fixed public range check in ec_GF2m_simple_oct2point

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Bernd Edlinger <bernd.edlinger@hotmail.de>
(Merged from https://github.com/openssl/openssl/pull/8607)

(cherry picked from commit cad8347be23c5e0c0d9eea02d090d42daf2dd7a9)

diff --git a/crypto/ec/ec2_oct.c b/crypto/ec/ec2_oct.c
index ea88ce860a9a05c9ae02c7226f05bff81e79050a..2b411e26ba3c1f3070da8dd2a3507dd966d88724 100644 (file)
--- a/crypto/ec/ec2_oct.c
+++ b/crypto/ec/ec2_oct.c
@@ -251,7 +251,7 @@ int ec_GF2m_simple_oct2point(const EC_GROUP *group, EC_POINT *point,
                              BN_CTX *ctx)
 {
     point_conversion_form_t form;
-    int y_bit;
+    int y_bit, m;
     BN_CTX *new_ctx = NULL;
     BIGNUM *x, *y, *yxi;
     size_t field_len, enc_len;
@@ -284,7 +284,8 @@ int ec_GF2m_simple_oct2point(const EC_GROUP *group, EC_POINT *point,
         return EC_POINT_set_to_infinity(group, point);
     }
 
-    field_len = (EC_GROUP_get_degree(group) + 7) / 8;
+    m = EC_GROUP_get_degree(group);
+    field_len = (m + 7) / 8;
     enc_len =
         (form ==
          POINT_CONVERSION_COMPRESSED) ? 1 + field_len : 1 + 2 * field_len;
@@ -309,7 +310,7 @@ int ec_GF2m_simple_oct2point(const EC_GROUP *group, EC_POINT *point,
 
     if (!BN_bin2bn(buf + 1, field_len, x))
         goto err;
-    if (BN_ucmp(x, group->field) >= 0) {
+    if (BN_num_bits(x) > m) {
         ECerr(EC_F_EC_GF2M_SIMPLE_OCT2POINT, EC_R_INVALID_ENCODING);
         goto err;
     }
@@ -321,7 +322,7 @@ int ec_GF2m_simple_oct2point(const EC_GROUP *group, EC_POINT *point,
     } else {
         if (!BN_bin2bn(buf + 1 + field_len, field_len, y))
             goto err;
-        if (BN_ucmp(y, group->field) >= 0) {
+        if (BN_num_bits(y) > m) {
             ECerr(EC_F_EC_GF2M_SIMPLE_OCT2POINT, EC_R_INVALID_ENCODING);
             goto err;
         }