bn/bn_mont.c: move boundary condition check closer to caller.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: David Benjamin <davidben@google.com>
(Merged from https://github.com/openssl/openssl/pull/6662)

(cherry picked from commit 3c97e4121ecec20cfac433883cd4709580a05620)

diff --git a/crypto/bn/bn_mont.c b/crypto/bn/bn_mont.c
index dad3d07b3deb4eeb41d931ba209ce9b1d09103b0..f052d5612b72a6f725d0cd6cdbbae545e6f5ae4b 100644 (file)
--- a/crypto/bn/bn_mont.c
+++ b/crypto/bn/bn_mont.c
@@ -28,9 +28,9 @@ int BN_mod_mul_montgomery(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
 {
     BIGNUM *tmp;
     int ret = 0;
-#if defined(OPENSSL_BN_ASM_MONT) && defined(MONT_WORD)
     int num = mont->N.top;
 
+#if defined(OPENSSL_BN_ASM_MONT) && defined(MONT_WORD)
     if (num > 1 && a->top == num && b->top == num) {
         if (bn_wexpand(r, num) == NULL)
             return (0);
@@ -43,6 +43,9 @@ int BN_mod_mul_montgomery(BIGNUM *r, const BIGNUM *a, const BIGNUM *b,
     }
 #endif
 
+    if ((a->top + b->top) > 2 * num)
+        return 0;
+
     BN_CTX_start(ctx);
     tmp = BN_CTX_get(ctx);
     if (tmp == NULL)
@@ -95,8 +98,6 @@ static int BN_from_montgomery_word(BIGNUM *ret, BIGNUM *r, BN_MONT_CTX *mont)
 
     /* clear the top words of T */
     i = max - r->top;
-    if (i < 0)
-        return 0;
     if (i)
         memset(&rp[r->top], 0, sizeof(*rp) * i);