Fix an endless loop in BN_generate_prime_ex

Happens when trying to generate 4 or 5 bit safe primes.

[extended tests]

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9311)

diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c
index 03402c2cec000f322cf1a61605ca1804c97773a0..47e2f2357a8d88d58c6fc2dc7653674a6c4b6401 100644 (file)
--- a/crypto/bn/bn_prime.c
+++ b/crypto/bn/bn_prime.c
@@ -98,8 +98,12 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
         /* There are no prime numbers this small. */
         BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
         return 0;
-    } else if (bits == 2 && safe) {
-        /* The smallest safe prime (7) is three bits. */
+    } else if (add == NULL && safe && bits < 6 && bits != 3) {
+        /*
+         * The smallest safe prime (7) is three bits.
+         * But the following two safe primes with less than 6 bits (11, 23)
+         * are unreachable for BN_rand with BN_RAND_TOP_TWO.
+         */
         BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
         return 0;
     }

diff --git a/test/bntest.c b/test/bntest.c
index 8df6e0f34cc5231b7a34c3c19b2fcf4779d6a5bc..1e502103c0e6dfa22a6abcf2e926d381b5cc5fbc 100644 (file)
--- a/test/bntest.c
+++ b/test/bntest.c
@@ -2249,18 +2249,50 @@ static int test_expmodone(void)
     return ret;
 }
 
-static int test_smallprime(void)
+static int test_smallprime(int kBits)
 {
-    static const int kBits = 10;
     BIGNUM *r;
     int st = 0;
 
-    if (!TEST_ptr(r = BN_new())
-            || !TEST_true(BN_generate_prime_ex(r, (int)kBits, 0,
-                                               NULL, NULL, NULL))
-            || !TEST_int_eq(BN_num_bits(r), kBits))
+    if (!TEST_ptr(r = BN_new()))
+        goto err;
+
+    if (kBits <= 1) {
+        if (!TEST_false(BN_generate_prime_ex(r, kBits, 0,
+                                             NULL, NULL, NULL)))
+            goto err;
+    } else {
+        if (!TEST_true(BN_generate_prime_ex(r, kBits, 0,
+                                            NULL, NULL, NULL))
+                || !TEST_int_eq(BN_num_bits(r), kBits))
+            goto err;
+    }
+
+    st = 1;
+ err:
+    BN_free(r);
+    return st;
+}
+
+static int test_smallsafeprime(int kBits)
+{
+    BIGNUM *r;
+    int st = 0;
+
+    if (!TEST_ptr(r = BN_new()))
         goto err;
 
+    if (kBits <= 5 && kBits != 3) {
+        if (!TEST_false(BN_generate_prime_ex(r, kBits, 1,
+                                             NULL, NULL, NULL)))
+            goto err;
+    } else {
+        if (!TEST_true(BN_generate_prime_ex(r, kBits, 1,
+                                            NULL, NULL, NULL))
+                || !TEST_int_eq(BN_num_bits(r), kBits))
+            goto err;
+    }
+
     st = 1;
  err:
     BN_free(r);
@@ -2518,7 +2550,8 @@ int setup_tests(void)
         ADD_TEST(test_badmod);
         ADD_TEST(test_expmodzero);
         ADD_TEST(test_expmodone);
-        ADD_TEST(test_smallprime);
+        ADD_ALL_TESTS(test_smallprime, 16);
+        ADD_ALL_TESTS(test_smallsafeprime, 16);
         ADD_TEST(test_swap);
         ADD_TEST(test_ctx_consttime_flag);
 #ifndef OPENSSL_NO_EC2M