Don't use assertions to check application-provided arguments;

[oweals/openssl.git] / crypto / bn / bn_gf2m.c

diff --git a/crypto/bn/bn_gf2m.c b/crypto/bn/bn_gf2m.c
index 334a314283a2665185ea405bbbfcdab087126322..306f029f2789c907c6fcc949a760622f6b977aa8 100644 (file)
--- a/crypto/bn/bn_gf2m.c
+++ b/crypto/bn/bn_gf2m.c
@@ -228,7 +228,7 @@ static void bn_GF2m_mul_1x1(BN_ULONG *r1, BN_ULONG *r0, const BN_ULONG a, const
        BN_ULONG tab[16], top3b = a >> 61;
        register BN_ULONG a1, a2, a4, a8;
 
-       a1 = a & (0x1FFFFFFFFFFFFFFF); a2 = a1 << 1; a4 = a2 << 1; a8 = a4 << 1;
+       a1 = a & (0x1FFFFFFFFFFFFFFFULL); a2 = a1 << 1; a4 = a2 << 1; a8 = a4 << 1;
 
        tab[ 0] = 0;     tab[ 1] = a1;       tab[ 2] = a2;       tab[ 3] = a1^a2;
        tab[ 4] = a4;    tab[ 5] = a1^a4;    tab[ 6] = a2^a4;    tab[ 7] = a1^a2^a4;
@@ -288,6 +288,9 @@ int BN_GF2m_add(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
        int i;
        const BIGNUM *at, *bt;
 
+       bn_check_top(a);
+       bn_check_top(b);
+
        if (a->top < b->top) { at = b; bt = a; }
        else { at = a; bt = b; }
 
@@ -322,10 +325,15 @@ int BN_GF2m_mod_arr(BIGNUM *r, const BIGNUM *a, const unsigned int p[])
        int j, k;
        int n, dN, d0, d1;
        BN_ULONG zz, *z;
-       
+
+       bn_check_top(a);
+
        if (!p[0])
+               {
                /* reduction mod 1 => return 0 */
-               return BN_zero(r);
+               BN_zero(r);
+               return 1;
+               }
 
        /* Since the algorithm does reduction in the r value, if a != r, copy
         * the contents of a into r so we can do reduction in r. 
@@ -376,7 +384,11 @@ int BN_GF2m_mod_arr(BIGNUM *r, const BIGNUM *a, const unsigned int p[])
                if (zz == 0) break;
                d1 = BN_BITS2 - d0;
                
-               if (d0) z[dN] = (z[dN] << d1) >> d1; /* clear up the top d1 bits */
+               /* clear up the top d1 bits */
+               if (d0)
+                       z[dN] = (z[dN] << d1) >> d1;
+               else
+                       z[dN] = 0;
                z[0] ^= zz; /* reduction t^0 component */
 
                for (k = 1; p[k] != 0; k++)
@@ -397,7 +409,6 @@ int BN_GF2m_mod_arr(BIGNUM *r, const BIGNUM *a, const unsigned int p[])
                }
 
        bn_correct_top(r);
-       
        return 1;
        }
 
@@ -412,6 +423,8 @@ int BN_GF2m_mod(BIGNUM *r, const BIGNUM *a, const BIGNUM *p)
        int ret = 0;
        const int max = BN_num_bits(p);
        unsigned int *arr=NULL;
+       bn_check_top(a);
+       bn_check_top(p);
        if ((arr = (unsigned int *)OPENSSL_malloc(sizeof(unsigned int) * max)) == NULL) goto err;
        ret = BN_GF2m_poly2arr(p, arr, max);
        if (!ret || ret > max)
@@ -421,7 +434,7 @@ int BN_GF2m_mod(BIGNUM *r, const BIGNUM *a, const BIGNUM *p)
                }
        ret = BN_GF2m_mod_arr(r, a, arr);
        bn_check_top(r);
-  err:
+err:
        if (arr) OPENSSL_free(arr);
        return ret;
        }
@@ -435,12 +448,14 @@ int       BN_GF2m_mod_mul_arr(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const unsig
        int zlen, i, j, k, ret = 0;
        BIGNUM *s;
        BN_ULONG x1, x0, y1, y0, zz[4];
-       
+
+       bn_check_top(a);
+       bn_check_top(b);
+
        if (a == b)
                {
                return BN_GF2m_mod_sqr_arr(r, a, p, ctx);
                }
-       
 
        BN_CTX_start(ctx);
        if ((s = BN_CTX_get(ctx)) == NULL) goto err;
@@ -469,10 +484,9 @@ int        BN_GF2m_mod_mul_arr(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const unsig
                ret = 1;
        bn_check_top(r);
 
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
-       
        }
 
 /* Compute the product of two polynomials a and b, reduce modulo p, and store
@@ -487,6 +501,9 @@ int BN_GF2m_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *p
        int ret = 0;
        const int max = BN_num_bits(p);
        unsigned int *arr=NULL;
+       bn_check_top(a);
+       bn_check_top(b);
+       bn_check_top(p);
        if ((arr = (unsigned int *)OPENSSL_malloc(sizeof(unsigned int) * max)) == NULL) goto err;
        ret = BN_GF2m_poly2arr(p, arr, max);
        if (!ret || ret > max)
@@ -496,7 +513,7 @@ int BN_GF2m_mod_mul(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *p
                }
        ret = BN_GF2m_mod_mul_arr(r, a, b, arr, ctx);
        bn_check_top(r);
-  err:
+err:
        if (arr) OPENSSL_free(arr);
        return ret;
        }
@@ -507,7 +524,8 @@ int BN_GF2m_mod_sqr_arr(BIGNUM *r, const BIGNUM *a, const unsigned int p[], BN_C
        {
        int i, ret = 0;
        BIGNUM *s;
-       
+
+       bn_check_top(a);
        BN_CTX_start(ctx);
        if ((s = BN_CTX_get(ctx)) == NULL) return 0;
        if (!bn_wexpand(s, 2 * a->top)) goto err;
@@ -523,7 +541,7 @@ int BN_GF2m_mod_sqr_arr(BIGNUM *r, const BIGNUM *a, const unsigned int p[], BN_C
        if (!BN_GF2m_mod_arr(r, s, p)) goto err;
        bn_check_top(r);
        ret = 1;
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
        }
@@ -539,6 +557,9 @@ int BN_GF2m_mod_sqr(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
        int ret = 0;
        const int max = BN_num_bits(p);
        unsigned int *arr=NULL;
+
+       bn_check_top(a);
+       bn_check_top(p);
        if ((arr = (unsigned int *)OPENSSL_malloc(sizeof(unsigned int) * max)) == NULL) goto err;
        ret = BN_GF2m_poly2arr(p, arr, max);
        if (!ret || ret > max)
@@ -548,7 +569,7 @@ int BN_GF2m_mod_sqr(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
                }
        ret = BN_GF2m_mod_sqr_arr(r, a, arr, ctx);
        bn_check_top(r);
-  err:
+err:
        if (arr) OPENSSL_free(arr);
        return ret;
        }
@@ -564,6 +585,9 @@ int BN_GF2m_mod_inv(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
        BIGNUM *b, *c, *u, *v, *tmp;
        int ret = 0;
 
+       bn_check_top(a);
+       bn_check_top(p);
+
        BN_CTX_start(ctx);
        
        b = BN_CTX_get(ctx);
@@ -573,7 +597,6 @@ int BN_GF2m_mod_inv(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
        if (v == NULL) goto err;
 
        if (!BN_one(b)) goto err;
-       if (!BN_zero(c)) goto err;
        if (!BN_GF2m_mod(u, a, p)) goto err;
        if (!BN_copy(v, p)) goto err;
 
@@ -608,7 +631,7 @@ int BN_GF2m_mod_inv(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
        bn_check_top(r);
        ret = 1;
 
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
        }
@@ -624,6 +647,7 @@ int BN_GF2m_mod_inv_arr(BIGNUM *r, const BIGNUM *xx, const unsigned int p[], BN_
        BIGNUM *field;
        int ret = 0;
 
+       bn_check_top(xx);
        BN_CTX_start(ctx);
        if ((field = BN_CTX_get(ctx)) == NULL) goto err;
        if (!BN_GF2m_arr2poly(p, field)) goto err;
@@ -631,7 +655,7 @@ int BN_GF2m_mod_inv_arr(BIGNUM *r, const BIGNUM *xx, const unsigned int p[], BN_
        ret = BN_GF2m_mod_inv(r, xx, field, ctx);
        bn_check_top(r);
 
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
        }
@@ -645,7 +669,11 @@ int BN_GF2m_mod_div(BIGNUM *r, const BIGNUM *y, const BIGNUM *x, const BIGNUM *p
        {
        BIGNUM *xinv = NULL;
        int ret = 0;
-       
+
+       bn_check_top(y);
+       bn_check_top(x);
+       bn_check_top(p);
+
        BN_CTX_start(ctx);
        xinv = BN_CTX_get(ctx);
        if (xinv == NULL) goto err;
@@ -655,7 +683,7 @@ int BN_GF2m_mod_div(BIGNUM *r, const BIGNUM *y, const BIGNUM *x, const BIGNUM *p
        bn_check_top(r);
        ret = 1;
 
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
        }
@@ -671,6 +699,10 @@ int BN_GF2m_mod_div(BIGNUM *r, const BIGNUM *y, const BIGNUM *x, const BIGNUM *p
        BIGNUM *a, *b, *u, *v;
        int ret = 0;
 
+       bn_check_top(y);
+       bn_check_top(x);
+       bn_check_top(p);
+
        BN_CTX_start(ctx);
        
        a = BN_CTX_get(ctx);
@@ -683,7 +715,6 @@ int BN_GF2m_mod_div(BIGNUM *r, const BIGNUM *y, const BIGNUM *x, const BIGNUM *p
        if (!BN_GF2m_mod(u, y, p)) goto err;
        if (!BN_GF2m_mod(a, x, p)) goto err;
        if (!BN_copy(b, p)) goto err;
-       if (!BN_zero(v)) goto err;
        
        while (!BN_is_odd(a))
                {
@@ -724,7 +755,7 @@ int BN_GF2m_mod_div(BIGNUM *r, const BIGNUM *y, const BIGNUM *x, const BIGNUM *p
        bn_check_top(r);
        ret = 1;
 
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
        }
@@ -742,6 +773,9 @@ int BN_GF2m_mod_div_arr(BIGNUM *r, const BIGNUM *yy, const BIGNUM *xx, const uns
        BIGNUM *field;
        int ret = 0;
 
+       bn_check_top(yy);
+       bn_check_top(xx);
+
        BN_CTX_start(ctx);
        if ((field = BN_CTX_get(ctx)) == NULL) goto err;
        if (!BN_GF2m_arr2poly(p, field)) goto err;
@@ -749,7 +783,7 @@ int BN_GF2m_mod_div_arr(BIGNUM *r, const BIGNUM *yy, const BIGNUM *xx, const uns
        ret = BN_GF2m_mod_div(r, yy, xx, field, ctx);
        bn_check_top(r);
 
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
        }
@@ -763,13 +797,15 @@ int       BN_GF2m_mod_exp_arr(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const unsig
        {
        int ret = 0, i, n;
        BIGNUM *u;
-       
+
+       bn_check_top(a);
+       bn_check_top(b);
+
        if (BN_is_zero(b))
                return(BN_one(r));
 
        if (BN_abs_is_word(b, 1))
                return (BN_copy(r, a) != NULL);
-       
 
        BN_CTX_start(ctx);
        if ((u = BN_CTX_get(ctx)) == NULL) goto err;
@@ -787,10 +823,8 @@ int        BN_GF2m_mod_exp_arr(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const unsig
                }
        if (!BN_copy(r, u)) goto err;
        bn_check_top(r);
-
        ret = 1;
-
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
        }
@@ -807,6 +841,9 @@ int BN_GF2m_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *p
        int ret = 0;
        const int max = BN_num_bits(p);
        unsigned int *arr=NULL;
+       bn_check_top(a);
+       bn_check_top(b);
+       bn_check_top(p);
        if ((arr = (unsigned int *)OPENSSL_malloc(sizeof(unsigned int) * max)) == NULL) goto err;
        ret = BN_GF2m_poly2arr(p, arr, max);
        if (!ret || ret > max)
@@ -816,7 +853,7 @@ int BN_GF2m_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *b, const BIGNUM *p
                }
        ret = BN_GF2m_mod_exp_arr(r, a, b, arr, ctx);
        bn_check_top(r);
-  err:
+err:
        if (arr) OPENSSL_free(arr);
        return ret;
        }
@@ -830,19 +867,23 @@ int       BN_GF2m_mod_sqrt_arr(BIGNUM *r, const BIGNUM *a, const unsigned int p[], BN_
        int ret = 0;
        BIGNUM *u;
 
+       bn_check_top(a);
+
        if (!p[0])
+               {
                /* reduction mod 1 => return 0 */
-               return BN_zero(r);
-       
+               BN_zero(r);
+               return 1;
+               }
+
        BN_CTX_start(ctx);
        if ((u = BN_CTX_get(ctx)) == NULL) goto err;
        
-       if (!BN_zero(u)) goto err;
        if (!BN_set_bit(u, p[0] - 1)) goto err;
        ret = BN_GF2m_mod_exp_arr(r, a, u, p, ctx);
        bn_check_top(r);
 
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
        }
@@ -859,16 +900,18 @@ int BN_GF2m_mod_sqrt(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx)
        int ret = 0;
        const int max = BN_num_bits(p);
        unsigned int *arr=NULL;
+       bn_check_top(a);
+       bn_check_top(p);
        if ((arr = (unsigned int *)OPENSSL_malloc(sizeof(unsigned int) * max)) == NULL) goto err;
        ret = BN_GF2m_poly2arr(p, arr, max);
        if (!ret || ret > max)
                {
-               BNerr(BN_F_BN_GF2M_MOD_EXP,BN_R_INVALID_LENGTH);
+               BNerr(BN_F_BN_GF2M_MOD_SQRT,BN_R_INVALID_LENGTH);
                goto err;
                }
        ret = BN_GF2m_mod_sqrt_arr(r, a, arr, ctx);
        bn_check_top(r);
-  err:
+err:
        if (arr) OPENSSL_free(arr);
        return ret;
        }
@@ -882,9 +925,14 @@ int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const unsigned int p
        unsigned int j;
        BIGNUM *a, *z, *rho, *w, *w2, *tmp;
 
+       bn_check_top(a_);
+
        if (!p[0])
+               {
                /* reduction mod 1 => return 0 */
-               return BN_zero(r);
+               BN_zero(r);
+               return 1;
+               }
 
        BN_CTX_start(ctx);
        a = BN_CTX_get(ctx);
@@ -896,7 +944,8 @@ int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const unsigned int p
        
        if (BN_is_zero(a))
                {
-               ret = BN_zero(r);
+               BN_zero(r);
+               ret = 1;
                goto err;
                }
 
@@ -922,7 +971,7 @@ int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const unsigned int p
                        {
                        if (!BN_rand(rho, p[0], 0, 0)) goto err;
                        if (!BN_GF2m_mod_arr(rho, rho, p)) goto err;
-                       if (!BN_zero(z)) goto err;
+                       BN_zero(z);
                        if (!BN_copy(w, rho)) goto err;
                        for (j = 1; j <= p[0] - 1; j++)
                                {
@@ -943,14 +992,18 @@ int BN_GF2m_mod_solve_quad_arr(BIGNUM *r, const BIGNUM *a_, const unsigned int p
        
        if (!BN_GF2m_mod_sqr_arr(w, z, p, ctx)) goto err;
        if (!BN_GF2m_add(w, z, w)) goto err;
-       if (BN_GF2m_cmp(w, a)) goto err;
+       if (BN_GF2m_cmp(w, a))
+               {
+               BNerr(BN_F_BN_GF2M_MOD_SOLVE_QUAD_ARR, BN_R_NO_SOLUTION);
+               goto err;
+               }
 
        if (!BN_copy(r, z)) goto err;
        bn_check_top(r);
 
        ret = 1;
 
-  err:
+err:
        BN_CTX_end(ctx);
        return ret;
        }
@@ -966,6 +1019,8 @@ int BN_GF2m_mod_solve_quad(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *
        int ret = 0;
        const int max = BN_num_bits(p);
        unsigned int *arr=NULL;
+       bn_check_top(a);
+       bn_check_top(p);
        if ((arr = (unsigned int *)OPENSSL_malloc(sizeof(unsigned int) *
                                                max)) == NULL) goto err;
        ret = BN_GF2m_poly2arr(p, arr, max);
@@ -976,7 +1031,7 @@ int BN_GF2m_mod_solve_quad(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, BN_CTX *
                }
        ret = BN_GF2m_mod_solve_quad_arr(r, a, arr, ctx);
        bn_check_top(r);
-  err:
+err:
        if (arr) OPENSSL_free(arr);
        return ret;
        }
@@ -1025,14 +1080,16 @@ int BN_GF2m_arr2poly(const unsigned int p[], BIGNUM *a)
        {
        int i;
 
+       bn_check_top(a);
        BN_zero(a);
        for (i = 0; p[i] != 0; i++)
                {
-               BN_set_bit(a, p[i]);
+               if (BN_set_bit(a, p[i]) == 0)
+                       return 0;
                }
        BN_set_bit(a, 0);
        bn_check_top(a);
-       
+
        return 1;
        }