bn/bn_add.c: address performance regression.

Performance regression was reported for EC key generation between
1.0.2 and 1.1.x [in GH#2891]. It naturally depends on platform,
values between 6 and 9% were observed.

Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4743)

(cherry picked from commit a78324d95bd4568ce2c3b34bfa1d6f14cddf92ef)

diff --git a/crypto/bn/asm/x86_64-gcc.c b/crypto/bn/asm/x86_64-gcc.c
index 4f19abe43f15e9384834fcab74e459addabf9143..0ff3805a6110866bb47473c899973b140c828468 100644 (file)
--- a/crypto/bn/asm/x86_64-gcc.c
+++ b/crypto/bn/asm/x86_64-gcc.c
@@ -225,9 +225,10 @@ BN_ULONG bn_add_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
                   "       adcq    (%5,%2,8),%0    \n"
                   "       movq    %0,(%3,%2,8)    \n"
                   "       lea     1(%2),%2        \n"
-                  "       loop    1b              \n"
-                  "       sbbq    %0,%0           \n":"=&r" (ret), "+c"(n),
-                  "+r"(i)
+                  "       dec     %1              \n"
+                  "       jnz     1b              \n"
+                  "       sbbq    %0,%0           \n"
+                  :"=&r" (ret), "+c"(n), "+r"(i)
                   :"r"(rp), "r"(ap), "r"(bp)
                   :"cc", "memory");
 
@@ -251,9 +252,10 @@ BN_ULONG bn_sub_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
                   "       sbbq    (%5,%2,8),%0    \n"
                   "       movq    %0,(%3,%2,8)    \n"
                   "       lea     1(%2),%2        \n"
-                  "       loop    1b              \n"
-                  "       sbbq    %0,%0           \n":"=&r" (ret), "+c"(n),
-                  "+r"(i)
+                  "       dec     %1              \n"
+                  "       jnz     1b              \n"
+                  "       sbbq    %0,%0           \n"
+                  :"=&r" (ret), "+c"(n), "+r"(i)
                   :"r"(rp), "r"(ap), "r"(bp)
                   :"cc", "memory");
 

diff --git a/crypto/bn/bn_add.c b/crypto/bn/bn_add.c
index 6479650afba7c0d2e4273fb05ddf58172e6bc5e7..7cdefa77a17c57be6892ea650745b63f916e0dec 100644 (file)
--- a/crypto/bn/bn_add.c
+++ b/crypto/bn/bn_add.c
@@ -141,9 +141,13 @@ int BN_usub(BIGNUM *r, const BIGNUM *a, const BIGNUM *b)
         borrow &= (t1 == 0);
     }
 
+    while (max && *--rp == 0)
+        max--;
+
     r->top = max;
     r->neg = 0;
-    bn_correct_top(r);
+    bn_pollute(r);
+
     return 1;
 }