Fix an endless loop in BN_generate_prime_ex

Happens when trying to generate 4 or 5 bit safe primes.

[extended tests]

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9311)

(cherry picked from commit 291f616ced45c924d639d97fc9ca2cbeaad096cf)

diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c
index 4bbd7c88103145b66791eb66695ffb894da91e28..19b081f38eb7b2d744548c83daaa907df45a1f95 100644 (file)
--- a/crypto/bn/bn_prime.c
+++ b/crypto/bn/bn_prime.c
@@ -63,8 +63,12 @@ int BN_generate_prime_ex(BIGNUM *ret, int bits, int safe,
         /* There are no prime numbers this small. */
         BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
         return 0;
-    } else if (bits == 2 && safe) {
-        /* The smallest safe prime (7) is three bits. */
+    } else if (add == NULL && safe && bits < 6 && bits != 3) {
+        /*
+         * The smallest safe prime (7) is three bits.
+         * But the following two safe primes with less than 6 bits (11, 23)
+         * are unreachable for BN_rand with BN_RAND_TOP_TWO.
+         */
         BNerr(BN_F_BN_GENERATE_PRIME_EX, BN_R_BITS_TOO_SMALL);
         return 0;
     }

diff --git a/test/bntest.c b/test/bntest.c
index c68d7f6fb8796dabac1bf61216d6188a4c23236c..0bd9a06f728a5e04f06b830e44ff9c2b8f0e7065 100644 (file)
--- a/test/bntest.c
+++ b/test/bntest.c
@@ -2169,18 +2169,50 @@ static int test_expmodone(void)
     return ret;
 }
 
-static int test_smallprime(void)
+static int test_smallprime(int kBits)
 {
-    static const int kBits = 10;
     BIGNUM *r;
     int st = 0;
 
-    if (!TEST_ptr(r = BN_new())
-            || !TEST_true(BN_generate_prime_ex(r, (int)kBits, 0,
-                                               NULL, NULL, NULL))
-            || !TEST_int_eq(BN_num_bits(r), kBits))
+    if (!TEST_ptr(r = BN_new()))
+        goto err;
+
+    if (kBits <= 1) {
+        if (!TEST_false(BN_generate_prime_ex(r, kBits, 0,
+                                             NULL, NULL, NULL)))
+            goto err;
+    } else {
+        if (!TEST_true(BN_generate_prime_ex(r, kBits, 0,
+                                            NULL, NULL, NULL))
+                || !TEST_int_eq(BN_num_bits(r), kBits))
+            goto err;
+    }
+
+    st = 1;
+ err:
+    BN_free(r);
+    return st;
+}
+
+static int test_smallsafeprime(int kBits)
+{
+    BIGNUM *r;
+    int st = 0;
+
+    if (!TEST_ptr(r = BN_new()))
         goto err;
 
+    if (kBits <= 5 && kBits != 3) {
+        if (!TEST_false(BN_generate_prime_ex(r, kBits, 1,
+                                             NULL, NULL, NULL)))
+            goto err;
+    } else {
+        if (!TEST_true(BN_generate_prime_ex(r, kBits, 1,
+                                            NULL, NULL, NULL))
+                || !TEST_int_eq(BN_num_bits(r), kBits))
+            goto err;
+    }
+
     st = 1;
  err:
     BN_free(r);
@@ -2405,7 +2437,8 @@ int setup_tests(void)
         ADD_TEST(test_badmod);
         ADD_TEST(test_expmodzero);
         ADD_TEST(test_expmodone);
-        ADD_TEST(test_smallprime);
+        ADD_ALL_TESTS(test_smallprime, 16);
+        ADD_ALL_TESTS(test_smallsafeprime, 16);
         ADD_TEST(test_swap);
         ADD_TEST(test_ctx_consttime_flag);
 #ifndef OPENSSL_NO_EC2M