-/* #ifdef RECP_MUL_MOD */
-int BN_mod_exp_recp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
- const BIGNUM *m, BN_CTX *ctx)
- {
- int i,j,bits,ret=0,wstart,wend,window,wvalue;
- int start=1,ts=0;
- BIGNUM *aa;
- BIGNUM val[TABLE_SIZE];
- BN_RECP_CTX recp;
-
- bits=BN_num_bits(p);
-
- if (bits == 0)
- {
- BN_one(r);
- return(1);
- }
-
- BN_CTX_start(ctx);
- if ((aa = BN_CTX_get(ctx)) == NULL) goto err;
-
- BN_RECP_CTX_init(&recp);
- if (BN_RECP_CTX_set(&recp,m,ctx) <= 0) goto err;
-
- BN_init(&(val[0]));
- ts=1;
-
- if (!BN_mod(&(val[0]),a,m,ctx)) goto err; /* 1 */
- if (!BN_mod_mul_reciprocal(aa,&(val[0]),&(val[0]),&recp,ctx))
- goto err; /* 2 */
-
- if (bits <= 17) /* This is probably 3 or 0x10001, so just do singles */
- window=1;
- else if (bits >= 256)
- window=5; /* max size of window */
- else if (bits >= 128)
- window=4;
- else
- window=3;
-
- j=1<<(window-1);
- for (i=1; i<j; i++)
- {
- BN_init(&val[i]);
- if (!BN_mod_mul_reciprocal(&(val[i]),&(val[i-1]),aa,&recp,ctx))
- goto err;
- }
- ts=i;
-
- start=1; /* This is used to avoid multiplication etc
- * when there is only the value '1' in the
- * buffer. */
- wvalue=0; /* The 'value' of the window */
- wstart=bits-1; /* The top bit of the window */
- wend=0; /* The bottom bit of the window */
-
- if (!BN_one(r)) goto err;
-
- for (;;)
- {
- if (BN_is_bit_set(p,wstart) == 0)
- {
- if (!start)
- if (!BN_mod_mul_reciprocal(r,r,r,&recp,ctx))
- goto err;
- if (wstart == 0) break;
- wstart--;
- continue;
- }
- /* We now have wstart on a 'set' bit, we now need to work out
- * how bit a window to do. To do this we need to scan
- * forward until the last set bit before the end of the
- * window */
- j=wstart;
- wvalue=1;
- wend=0;
- for (i=1; i<window; i++)
- {
- if (wstart-i < 0) break;
- if (BN_is_bit_set(p,wstart-i))
- {
- wvalue<<=(i-wend);
- wvalue|=1;
- wend=i;
- }
- }
-
- /* wend is the size of the current window */
- j=wend+1;
- /* add the 'bytes above' */
- if (!start)
- for (i=0; i<j; i++)
- {
- if (!BN_mod_mul_reciprocal(r,r,r,&recp,ctx))
- goto err;
- }
-
- /* wvalue will be an odd number < 2^window */
- if (!BN_mod_mul_reciprocal(r,r,&(val[wvalue>>1]),&recp,ctx))
- goto err;
-
- /* move the 'window' down further */
- wstart-=wend+1;
- wvalue=0;
- start=0;
- if (wstart < 0) break;
- }
- ret=1;
-err:
- BN_CTX_end(ctx);
- for (i=0; i<ts; i++)
- BN_clear_free(&(val[i]));
- BN_RECP_CTX_free(&recp);
- return(ret);
- }
-/* #endif */
-
-/* #ifdef MONT_MUL_MOD */
-int BN_mod_exp_mont(BIGNUM *rr, BIGNUM *a, const BIGNUM *p,
- const BIGNUM *m, BN_CTX *ctx, BN_MONT_CTX *in_mont)
- {
- int i,j,bits,ret=0,wstart,wend,window,wvalue;
- int start=1,ts=0;
- BIGNUM *d,*r;
- BIGNUM *aa;
- BIGNUM val[TABLE_SIZE];
- BN_MONT_CTX *mont=NULL;
-
- bn_check_top(a);
- bn_check_top(p);
- bn_check_top(m);
-
-#ifdef ATALLA
- if(!tried_atalla && BN_mod_exp_atalla(rr,a,p,m))
- return 1;
-/* If it fails, try the other methods */
+ return ret & BN_MASK2;
+}
+#endif
+
+/*
+ * BN_mod_exp_mont_consttime() stores the precomputed powers in a specific
+ * layout so that accessing any of these table values shows the same access
+ * pattern as far as cache lines are concerned. The following functions are
+ * used to transfer a BIGNUM from/to that table.
+ */
+
+static int MOD_EXP_CTIME_COPY_TO_PREBUF(const BIGNUM *b, int top,
+ unsigned char *buf, int idx,
+ int width)
+{
+ size_t i, j;
+
+ if (top > b->top)
+ top = b->top; /* this works because 'buf' is explicitly
+ * zeroed */
+ for (i = 0, j = idx; i < top * sizeof b->d[0]; i++, j += width) {
+ buf[j] = ((unsigned char *)b->d)[i];
+ }
+
+ return 1;
+}
+
+static int MOD_EXP_CTIME_COPY_FROM_PREBUF(BIGNUM *b, int top,
+ unsigned char *buf, int idx,
+ int width)
+{
+ size_t i, j;
+
+ if (bn_wexpand(b, top) == NULL)
+ return 0;
+
+ for (i = 0, j = idx; i < top * sizeof b->d[0]; i++, j += width) {
+ ((unsigned char *)b->d)[i] = buf[j];
+ }
+
+ b->top = top;
+ bn_correct_top(b);
+ return 1;
+}
+
+/*
+ * Given a pointer value, compute the next address that is a cache line
+ * multiple.
+ */
+#define MOD_EXP_CTIME_ALIGN(x_) \
+ ((unsigned char*)(x_) + (MOD_EXP_CTIME_MIN_CACHE_LINE_WIDTH - (((size_t)(x_)) & (MOD_EXP_CTIME_MIN_CACHE_LINE_MASK))))
+
+/*
+ * This variant of BN_mod_exp_mont() uses fixed windows and the special
+ * precomputation memory layout to limit data-dependency to a minimum to
+ * protect secret exponents (cf. the hyper-threading timing attacks pointed
+ * out by Colin Percival,
+ * http://www.daemong-consideredperthreading-considered-harmful/)
+ */
+int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
+ const BIGNUM *m, BN_CTX *ctx,
+ BN_MONT_CTX *in_mont)
+{
+ int i, bits, ret = 0, window, wvalue;
+ int top;
+ BN_MONT_CTX *mont = NULL;
+
+ int numPowers;
+ unsigned char *powerbufFree = NULL;
+ int powerbufLen = 0;
+ unsigned char *powerbuf = NULL;
+ BIGNUM tmp, am;
+#if defined(SPARC_T4_MONT)
+ unsigned int t4 = 0;
+#endif
+
+ bn_check_top(a);
+ bn_check_top(p);
+ bn_check_top(m);
+
+ if (!BN_is_odd(m)) {
+ BNerr(BN_F_BN_MOD_EXP_MONT_CONSTTIME, BN_R_CALLED_WITH_EVEN_MODULUS);
+ return (0);
+ }
+
+ top = m->top;
+
+ bits = BN_num_bits(p);
+ if (bits == 0) {
+ ret = BN_one(rr);
+ return ret;
+ }
+
+ BN_CTX_start(ctx);
+
+ /*
+ * Allocate a montgomery context if it was not supplied by the caller. If
+ * this is not done, things will break in the montgomery part.
+ */
+ if (in_mont != NULL)
+ mont = in_mont;
+ else {
+ if ((mont = BN_MONT_CTX_new()) == NULL)
+ goto err;
+ if (!BN_MONT_CTX_set(mont, m, ctx))
+ goto err;
+ }
+
+#ifdef RSAZ_ENABLED
+ /*
+ * If the size of the operands allow it, perform the optimized
+ * RSAZ exponentiation. For further information see
+ * crypto/bn/rsaz_exp.c and accompanying assembly modules.
+ */
+ if ((16 == a->top) && (16 == p->top) && (BN_num_bits(m) == 1024)
+ && rsaz_avx2_eligible()) {
+ if (NULL == bn_wexpand(rr, 16))
+ goto err;
+ RSAZ_1024_mod_exp_avx2(rr->d, a->d, p->d, m->d, mont->RR.d,
+ mont->n0[0]);
+ rr->top = 16;
+ rr->neg = 0;
+ bn_correct_top(rr);
+ ret = 1;
+ goto err;
+ } else if ((8 == a->top) && (8 == p->top) && (BN_num_bits(m) == 512)) {
+ if (NULL == bn_wexpand(rr, 8))
+ goto err;
+ RSAZ_512_mod_exp(rr->d, a->d, p->d, m->d, mont->n0[0], mont->RR.d);
+ rr->top = 8;
+ rr->neg = 0;
+ bn_correct_top(rr);
+ ret = 1;
+ goto err;
+ }
+#endif
+
+ /* Get the window size to use with size of p. */
+ window = BN_window_bits_for_ctime_exponent_size(bits);
+#if defined(SPARC_T4_MONT)
+ if (window >= 5 && (top & 15) == 0 && top <= 64 &&
+ (OPENSSL_sparcv9cap_P[1] & (CFR_MONTMUL | CFR_MONTSQR)) ==
+ (CFR_MONTMUL | CFR_MONTSQR) && (t4 = OPENSSL_sparcv9cap_P[0]))
+ window = 5;
+ else
+#endif
+#if defined(OPENSSL_BN_ASM_MONT5)
+ if (window >= 5) {
+ window = 5; /* ~5% improvement for RSA2048 sign, and even
+ * for RSA4096 */
+ if ((top & 7) == 0)
+ powerbufLen += 2 * top * sizeof(m->d[0]);
+ }
+#endif
+ (void)0;
+
+ /*
+ * Allocate a buffer large enough to hold all of the pre-computed powers
+ * of am, am itself and tmp.
+ */
+ numPowers = 1 << window;
+ powerbufLen += sizeof(m->d[0]) * (top * numPowers +
+ ((2 * top) >
+ numPowers ? (2 * top) : numPowers));
+#ifdef alloca
+ if (powerbufLen < 3072)
+ powerbufFree =
+ alloca(powerbufLen + MOD_EXP_CTIME_MIN_CACHE_LINE_WIDTH);
+ else
+#endif
+ if ((powerbufFree =
+ OPENSSL_malloc(powerbufLen + MOD_EXP_CTIME_MIN_CACHE_LINE_WIDTH))
+ == NULL)
+ goto err;
+
+ powerbuf = MOD_EXP_CTIME_ALIGN(powerbufFree);
+ memset(powerbuf, 0, powerbufLen);
+
+#ifdef alloca
+ if (powerbufLen < 3072)
+ powerbufFree = NULL;
+#endif
+
+ /* lay down tmp and am right after powers table */
+ tmp.d = (BN_ULONG *)(powerbuf + sizeof(m->d[0]) * top * numPowers);
+ am.d = tmp.d + top;
+ tmp.top = am.top = 0;
+ tmp.dmax = am.dmax = top;
+ tmp.neg = am.neg = 0;
+ tmp.flags = am.flags = BN_FLG_STATIC_DATA;
+
+ /* prepare a^0 in Montgomery domain */
+#if 1 /* by Shay Gueron's suggestion */
+ if (m->d[top - 1] & (((BN_ULONG)1) << (BN_BITS2 - 1))) {
+ /* 2^(top*BN_BITS2) - m */
+ tmp.d[0] = (0 - m->d[0]) & BN_MASK2;
+ for (i = 1; i < top; i++)
+ tmp.d[i] = (~m->d[i]) & BN_MASK2;
+ tmp.top = top;
+ } else
+#endif
+ if (!BN_to_montgomery(&tmp, BN_value_one(), mont, ctx))
+ goto err;
+
+ /* prepare a^1 in Montgomery domain */
+ if (a->neg || BN_ucmp(a, m) >= 0) {
+ if (!BN_mod(&am, a, m, ctx))
+ goto err;
+ if (!BN_to_montgomery(&am, &am, mont, ctx))
+ goto err;
+ } else if (!BN_to_montgomery(&am, a, mont, ctx))
+ goto err;
+
+#if defined(SPARC_T4_MONT)
+ if (t4) {
+ typedef int (*bn_pwr5_mont_f) (BN_ULONG *tp, const BN_ULONG *np,
+ const BN_ULONG *n0, const void *table,
+ int power, int bits);
+ int bn_pwr5_mont_t4_8(BN_ULONG *tp, const BN_ULONG *np,
+ const BN_ULONG *n0, const void *table,
+ int power, int bits);
+ int bn_pwr5_mont_t4_16(BN_ULONG *tp, const BN_ULONG *np,
+ const BN_ULONG *n0, const void *table,
+ int power, int bits);
+ int bn_pwr5_mont_t4_24(BN_ULONG *tp, const BN_ULONG *np,
+ const BN_ULONG *n0, const void *table,
+ int power, int bits);
+ int bn_pwr5_mont_t4_32(BN_ULONG *tp, const BN_ULONG *np,
+ const BN_ULONG *n0, const void *table,
+ int power, int bits);
+ static const bn_pwr5_mont_f pwr5_funcs[4] = {
+ bn_pwr5_mont_t4_8, bn_pwr5_mont_t4_16,
+ bn_pwr5_mont_t4_24, bn_pwr5_mont_t4_32
+ };
+ bn_pwr5_mont_f pwr5_worker = pwr5_funcs[top / 16 - 1];
+
+ typedef int (*bn_mul_mont_f) (BN_ULONG *rp, const BN_ULONG *ap,
+ const void *bp, const BN_ULONG *np,
+ const BN_ULONG *n0);
+ int bn_mul_mont_t4_8(BN_ULONG *rp, const BN_ULONG *ap, const void *bp,
+ const BN_ULONG *np, const BN_ULONG *n0);
+ int bn_mul_mont_t4_16(BN_ULONG *rp, const BN_ULONG *ap,
+ const void *bp, const BN_ULONG *np,
+ const BN_ULONG *n0);
+ int bn_mul_mont_t4_24(BN_ULONG *rp, const BN_ULONG *ap,
+ const void *bp, const BN_ULONG *np,
+ const BN_ULONG *n0);
+ int bn_mul_mont_t4_32(BN_ULONG *rp, const BN_ULONG *ap,
+ const void *bp, const BN_ULONG *np,
+ const BN_ULONG *n0);
+ static const bn_mul_mont_f mul_funcs[4] = {
+ bn_mul_mont_t4_8, bn_mul_mont_t4_16,
+ bn_mul_mont_t4_24, bn_mul_mont_t4_32
+ };
+ bn_mul_mont_f mul_worker = mul_funcs[top / 16 - 1];
+
+ void bn_mul_mont_vis3(BN_ULONG *rp, const BN_ULONG *ap,
+ const void *bp, const BN_ULONG *np,
+ const BN_ULONG *n0, int num);
+ void bn_mul_mont_t4(BN_ULONG *rp, const BN_ULONG *ap,
+ const void *bp, const BN_ULONG *np,
+ const BN_ULONG *n0, int num);
+ void bn_mul_mont_gather5_t4(BN_ULONG *rp, const BN_ULONG *ap,
+ const void *table, const BN_ULONG *np,
+ const BN_ULONG *n0, int num, int power);
+ void bn_flip_n_scatter5_t4(const BN_ULONG *inp, size_t num,
+ void *table, size_t power);
+ void bn_gather5_t4(BN_ULONG *out, size_t num,
+ void *table, size_t power);
+ void bn_flip_t4(BN_ULONG *dst, BN_ULONG *src, size_t num);
+
+ BN_ULONG *np = mont->N.d, *n0 = mont->n0;
+ int stride = 5 * (6 - (top / 16 - 1)); /* multiple of 5, but less
+ * than 32 */
+
+ /*
+ * BN_to_montgomery can contaminate words above .top [in
+ * BN_DEBUG[_DEBUG] build]...
+ */
+ for (i = am.top; i < top; i++)
+ am.d[i] = 0;
+ for (i = tmp.top; i < top; i++)
+ tmp.d[i] = 0;
+
+ bn_flip_n_scatter5_t4(tmp.d, top, powerbuf, 0);
+ bn_flip_n_scatter5_t4(am.d, top, powerbuf, 1);
+ if (!(*mul_worker) (tmp.d, am.d, am.d, np, n0) &&
+ !(*mul_worker) (tmp.d, am.d, am.d, np, n0))
+ bn_mul_mont_vis3(tmp.d, am.d, am.d, np, n0, top);
+ bn_flip_n_scatter5_t4(tmp.d, top, powerbuf, 2);
+
+ for (i = 3; i < 32; i++) {
+ /* Calculate a^i = a^(i-1) * a */
+ if (!(*mul_worker) (tmp.d, tmp.d, am.d, np, n0) &&
+ !(*mul_worker) (tmp.d, tmp.d, am.d, np, n0))
+ bn_mul_mont_vis3(tmp.d, tmp.d, am.d, np, n0, top);
+ bn_flip_n_scatter5_t4(tmp.d, top, powerbuf, i);
+ }
+
+ /* switch to 64-bit domain */
+ np = alloca(top * sizeof(BN_ULONG));
+ top /= 2;
+ bn_flip_t4(np, mont->N.d, top);
+
+ bits--;
+ for (wvalue = 0, i = bits % 5; i >= 0; i--, bits--)
+ wvalue = (wvalue << 1) + BN_is_bit_set(p, bits);
+ bn_gather5_t4(tmp.d, top, powerbuf, wvalue);
+
+ /*
+ * Scan the exponent one window at a time starting from the most
+ * significant bits.
+ */
+ while (bits >= 0) {
+ if (bits < stride)
+ stride = bits + 1;
+ bits -= stride;
+ wvalue = bn_get_bits(p, bits + 1);
+
+ if ((*pwr5_worker) (tmp.d, np, n0, powerbuf, wvalue, stride))
+ continue;
+ /* retry once and fall back */
+ if ((*pwr5_worker) (tmp.d, np, n0, powerbuf, wvalue, stride))
+ continue;
+
+ bits += stride - 5;
+ wvalue >>= stride - 5;
+ wvalue &= 31;
+ bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top);
+ bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top);
+ bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top);
+ bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top);
+ bn_mul_mont_t4(tmp.d, tmp.d, tmp.d, np, n0, top);
+ bn_mul_mont_gather5_t4(tmp.d, tmp.d, powerbuf, np, n0, top,
+ wvalue);
+ }
+
+ bn_flip_t4(tmp.d, tmp.d, top);
+ top *= 2;
+ /* back to 32-bit domain */
+ tmp.top = top;
+ bn_correct_top(&tmp);
+ OPENSSL_cleanse(np, top * sizeof(BN_ULONG));
+ } else