{arm64|x86_64}cpuid.pl: add special 16-byte case to OPENSSL_memcmp.

OPENSSL_memcmp is a must in GCM decrypt and general-purpose loop takes
quite a portion of execution time for short inputs, more than GHASH for
few-byte inputs according to profiler. Special 16-byte case takes it off
top five list in profiler output.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/6312)

diff --git a/crypto/arm64cpuid.pl b/crypto/arm64cpuid.pl
index daa2b17ada6669c39289a15b92e04cffd295cdfe..06c8add7a07561f1f00b2e5edb6826881771ba83 100755 (executable)
--- a/crypto/arm64cpuid.pl
+++ b/crypto/arm64cpuid.pl
@@ -115,6 +115,19 @@ OPENSSL_cleanse:
 CRYPTO_memcmp:
        eor     w3,w3,w3
        cbz     x2,.Lno_data    // len==0?
+       cmp     x2,#16
+       b.ne    .Loop_cmp
+       ldp     x8,x9,[x0]
+       ldp     x10,x11,[x1]
+       eor     x8,x8,x10
+       eor     x9,x9,x11
+       orr     x8,x8,x9
+       mov     x0,#1
+       cmp     x8,#0
+       csel    x0,xzr,x0,eq
+       ret
+
+.align 4
 .Loop_cmp:
        ldrb    w4,[x0],#1
        ldrb    w5,[x1],#1

diff --git a/crypto/x86_64cpuid.pl b/crypto/x86_64cpuid.pl
index 513d00560c59f4e263de34d56a45a9bc9b91b037..6423e803b759542dcc23999a8d1c239feccec4a7 100644 (file)
--- a/crypto/x86_64cpuid.pl
+++ b/crypto/x86_64cpuid.pl
@@ -271,6 +271,18 @@ CRYPTO_memcmp:
        xor     %r10,%r10
        cmp     \$0,$arg3
        je      .Lno_data
+       cmp     \$16,$arg3
+       jne     .Loop_cmp
+       mov     ($arg1),%r10
+       mov     8($arg1),%r11
+       mov     \$1,$arg3
+       xor     ($arg2),%r10
+       xor     8($arg2),%r11
+       or      %r11,%r10
+       cmovnz  $arg3,%rax
+       ret
+
+.align 16
 .Loop_cmp:
        mov     ($arg1),%r10b
        lea     1($arg1),$arg1