Update control logic for BN_gcd

PR https://github.com/openssl/openssl/pull/10122 introduced changes to
the BN_gcd function and the control logic inside it accessed `g->d[0]`
irrespective of `g->top`.

When BN_add is called, in case the result is zero, `BN_zero` is called.
The latter behaves differently depending on the API compatibility level
flag: normally `g->d[0]` is cleared but in `no-deprecated` builds only
`g->top` is set to zero.

This commit uses bitwise logic to ensure that `g` is treated as zero if
`g->top` is zero, irrespective of `g->d[0]`.

Co-authored-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/10232)

diff --git a/crypto/bn/bn_gcd.c b/crypto/bn/bn_gcd.c
index fbefe4ab6abe6f143354487e2d99cc2dc635f2aa..bed9fca4d9337a84322bffc353d76e65c94cbc6c 100644 (file)
--- a/crypto/bn/bn_gcd.c
+++ b/crypto/bn/bn_gcd.c
@@ -593,7 +593,9 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
 
     for (i = 0; i < m; i++) {
         /* conditionally flip signs if delta is positive and g is odd */
-        cond = (-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1;
+        cond = (-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1
+            /* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */
+            & (~((g->top - 1) >> (sizeof(g->top) * 8 - 1)));
         delta = (-cond & -delta) | ((cond - 1) & delta);
         r->neg ^= cond;
         /* swap */
@@ -603,7 +605,10 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
         delta++;
         if (!BN_add(temp, g, r))
             goto err;
-        BN_consttime_swap(g->d[0] & 1, g, temp, top);
+        BN_consttime_swap(g->d[0] & 1 /* g is odd */
+                /* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */
+                & (~((g->top - 1) >> (sizeof(g->top) * 8 - 1))),
+                g, temp, top);
         if (!BN_rshift1(g, g))
             goto err;
     }