Fix BN_gcd errors for some curves

Those even order that do not play nicely with Montgomery arithmetic

Signed-off-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Emilia Käsper <emilia@openssl.org>

diff --git a/crypto/ec/ec_lib.c b/crypto/ec/ec_lib.c
index abb15a50769c73d849f5e0465fe94cd469443a1c..a34113c95342d367f4729e7d0ab65026262e09d8 100644 (file)
--- a/crypto/ec/ec_lib.c
+++ b/crypto/ec/ec_lib.c
@@ -327,13 +327,18 @@ int EC_GROUP_set_generator(EC_GROUP *group, const EC_POINT *generator,
     } else
         BN_zero(group->cofactor);
 
+ 
     /*
-     * We ignore the return value because some groups have an order with
+     * Some groups have an order with
      * factors of two, which makes the Montgomery setup fail.
      * |group->mont_data| will be NULL in this case.
      */
-    ec_precompute_mont_data(group);
+    if (BN_is_odd(group->order)) {
+        return ec_precompute_mont_data(group);
+    }
 
+    BN_MONT_CTX_free(group->mont_data);
+    group->mont_data = NULL;
     return 1;
 }