Use order not degree to calculate a buffer size in ecdsatest

Otherwise this can result in an incorrect calculation of the maximum
encoded integer length, meaning an insufficient buffer size is allocated.

Thanks to Billy Brumley for helping to track this down.

Fixes #8209

Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8237)

diff --git a/test/ecdsatest.c b/test/ecdsatest.c
index 004f39eb5ba490f80e4001cf53b372be43a32fc0..bc3adc004c240ba690fc1334686ed43054af86b8 100644 (file)
--- a/test/ecdsatest.c
+++ b/test/ecdsatest.c
@@ -223,7 +223,7 @@ static int test_builtin(void)
     const BIGNUM *sig_r, *sig_s;
     BIGNUM *modified_r = NULL, *modified_s = NULL;
     BIGNUM *unmodified_r = NULL, *unmodified_s = NULL;
-    unsigned int sig_len, degree, r_len, s_len, bn_len, buf_len;
+    unsigned int sig_len, order, r_len, s_len, bn_len, buf_len;
     int nid, ret = 0;
 
     /* fill digest values with some random data */
@@ -251,7 +251,7 @@ static int test_builtin(void)
                 || !TEST_true(EC_KEY_set_group(eckey, group)))
             goto builtin_err;
         EC_GROUP_free(group);
-        degree = EC_GROUP_get_degree(EC_KEY_get0_group(eckey));
+        order = EC_GROUP_order_bits(EC_KEY_get0_group(eckey));
 
         TEST_info("testing %s", OBJ_nid2sn(nid));
 
@@ -316,7 +316,7 @@ static int test_builtin(void)
         /* Store the two BIGNUMs in raw_buf. */
         r_len = BN_num_bytes(sig_r);
         s_len = BN_num_bytes(sig_s);
-        bn_len = (degree + 7) / 8;
+        bn_len = (order + 7) / 8;
         if (!TEST_false(r_len > bn_len)
                 || !TEST_false(s_len > bn_len))
             goto builtin_err;