Engage GHASH for ARMv8.

diff --git a/Configure b/Configure
index 5d2bb63ef169cace2c95ba9f16297d834010f221..7d4ed2c32115020f3249685a03a16d8781cdf076 100755 (executable)
--- a/Configure
+++ b/Configure
@@ -136,8 +136,8 @@ my $alpha_asm="alphacpuid.o:bn_asm.o alpha-mont.o:::::sha1-alpha.o:::::::ghash-a
 my $mips64_asm=":bn-mips.o mips-mont.o::aes_cbc.o aes-mips.o:::sha1-mips.o sha256-mips.o sha512-mips.o::::::::";
 my $mips32_asm=$mips64_asm; $mips32_asm =~ s/\s*sha512\-mips\.o//;
 my $s390x_asm="s390xcap.o s390xcpuid.o:bn-s390x.o s390x-mont.o s390x-gf2m.o::aes-s390x.o aes-ctr.o aes-xts.o:::sha1-s390x.o sha256-s390x.o sha512-s390x.o::rc4-s390x.o:::::ghash-s390x.o:";
-my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o::aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o::void";
-my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o:::aes_core.o aes_cbc.o aesv8-armx.o:::sha1-armv8.o sha256-armv8.o sha512-armv8.o::::::::";
+my $armv4_asm="armcap.o armv4cpuid.o:bn_asm.o armv4-mont.o armv4-gf2m.o::aes_cbc.o aes-armv4.o bsaes-armv7.o aesv8-armx.o:::sha1-armv4-large.o sha256-armv4.o sha512-armv4.o:::::::ghash-armv4.o ghashv8-armx.o::void";
+my $aarch64_asm="armcap.o arm64cpuid.o mem_clr.o:::aes_core.o aes_cbc.o aesv8-armx.o:::sha1-armv8.o sha256-armv8.o sha512-armv8.o:::::::ghashv8-armx.o:";
 my $parisc11_asm="pariscid.o:bn_asm.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::32";
 my $parisc20_asm="pariscid.o:pa-risc2W.o parisc-mont.o::aes_core.o aes_cbc.o aes-parisc.o:::sha1-parisc.o sha256-parisc.o sha512-parisc.o::rc4-parisc.o:::::ghash-parisc.o::64";
 my $ppc64_asm="ppccpuid.o ppccap.o:bn-ppc.o ppc-mont.o ppc64-mont.o::aes_core.o aes_cbc.o aes-ppc.o vpaes-ppc.o aesp8-ppc.o:::sha1-ppc.o sha256-ppc.o sha512-ppc.o::::::::";
@@ -1617,7 +1617,7 @@ else      {
        $wp_obj="wp_block.o";
        }
 $cmll_obj=$cmll_enc    unless ($cmll_obj =~ /.o$/);
-if ($modes_obj =~ /ghash/)
+if ($modes_obj =~ /ghash\-/)
        {
        $cflags.=" -DGHASH_ASM";
        }

diff --git a/TABLE b/TABLE
index 519342a6d6dae93b501c6629e45189b1f60cd010..108ba87c026fbf566bb7762d7fb2204272a104a5 100644 (file)
--- a/TABLE
+++ b/TABLE
@@ -1110,7 +1110,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = 
 $cmll_obj     = 
-$modes_obj    = ghash-armv4.o
+$modes_obj    = ghash-armv4.o ghashv8-armx.o
 $engines_obj  = 
 $perlasm_scheme = void
 $dso_scheme   = dlfcn
@@ -4014,7 +4014,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = 
 $cmll_obj     = 
-$modes_obj    = 
+$modes_obj    = ghashv8-armx.o
 $engines_obj  = 
 $perlasm_scheme = linux64
 $dso_scheme   = dlfcn
@@ -4212,7 +4212,7 @@ $rmd160_obj   =
 $rc5_obj      = 
 $wp_obj       = 
 $cmll_obj     = 
-$modes_obj    = ghash-armv4.o
+$modes_obj    = ghash-armv4.o ghashv8-armx.o
 $engines_obj  = 
 $perlasm_scheme = void
 $dso_scheme   = dlfcn

diff --git a/crypto/modes/Makefile b/crypto/modes/Makefile
index b8cfa71fa3dff5887df1f564fc29c78cb38ac8c7..d03284e869d6f03a44c767679034d977d74432e2 100644 (file)
--- a/crypto/modes/Makefile
+++ b/crypto/modes/Makefile
@@ -58,14 +58,16 @@ ghash-alpha.s:      asm/ghash-alpha.pl
        (preproc=/tmp/$$$$.$@; trap "rm $$preproc" INT; \
        $(PERL) asm/ghash-alpha.pl > $$preproc && \
        $(CC) -E $$preproc > $@ && rm $$preproc)
-
 ghash-parisc.s:        asm/ghash-parisc.pl
        $(PERL) asm/ghash-parisc.pl $(PERLASM_SCHEME) $@
+ghashv8-armx.S:        asm/ghashv8-armx.pl
+       $(PERL) asm/ghashv8-armx.pl $(PERLASM_SCHEME) $@
 
 # GNU make "catch all"
 ghash-%.S:     asm/ghash-%.pl; $(PERL) $< $(PERLASM_SCHEME) $@
 
 ghash-armv4.o: ghash-armv4.S
+ghashv8-armx.o:        ghashv8-armx.S
 
 files:
        $(PERL) $(TOP)/util/files.pl Makefile >> $(TOP)/MINFO

diff --git a/crypto/modes/gcm128.c b/crypto/modes/gcm128.c
index 9afef1cf4e1c3b9136328080c971f2440fb4d83b..19cbcf5572cb9494062a5bed49d3e3ec0905a347 100644 (file)
--- a/crypto/modes/gcm128.c
+++ b/crypto/modes/gcm128.c
@@ -645,7 +645,7 @@ static void gcm_gmult_1bit(u64 Xi[2],const u64 H[2])
 
 #endif
 
-#if    TABLE_BITS==4 && defined(GHASH_ASM)
+#if    TABLE_BITS==4 && (defined(GHASH_ASM) || defined(OPENSSL_CPUID_OBJ))
 # if   !defined(I386_ONLY) && \
        (defined(__i386)        || defined(__i386__)    || \
         defined(__x86_64)      || defined(__x86_64__)  || \
@@ -676,14 +676,21 @@ void gcm_ghash_4bit_mmx(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len
 void gcm_gmult_4bit_x86(u64 Xi[2],const u128 Htable[16]);
 void gcm_ghash_4bit_x86(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
 #  endif
-# elif defined(__arm__) || defined(__arm)
+# elif defined(__arm__) || defined(__arm) || defined(__aarch64__)
 #  include "arm_arch.h"
 #  if __ARM_ARCH__>=7
 #   define GHASH_ASM_ARM
 #   define GCM_FUNCREF_4BIT
+#   define PMULL_CAPABLE       (OPENSSL_armcap_P & ARMV8_PMULL)
+#   if defined(__arm__) || defined(__arm)
+#    define NEON_CAPABLE       (OPENSSL_armcap_P & ARMV7_NEON)
+#   endif
 void gcm_init_neon(u128 Htable[16],const u64 Xi[2]);
 void gcm_gmult_neon(u64 Xi[2],const u128 Htable[16]);
 void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
+void gcm_init_v8(u128 Htable[16],const u64 Xi[2]);
+void gcm_gmult_v8(u64 Xi[2],const u128 Htable[16]);
+void gcm_ghash_v8(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
 #  endif
 # elif defined(__sparc__) || defined(__sparc)
 #  include "sparc_arch.h"
@@ -767,11 +774,21 @@ void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block)
        ctx->ghash = gcm_ghash_4bit;
 #  endif
 # elif defined(GHASH_ASM_ARM)
-       if (OPENSSL_armcap_P & ARMV7_NEON) {
+#  ifdef PMULL_CAPABLE
+       if (PMULL_CAPABLE) {
+               gcm_init_v8(ctx->Htable,ctx->H.u);
+               ctx->gmult = gcm_gmult_v8;
+               ctx->ghash = gcm_ghash_v8;
+       } else
+#  endif
+#  ifdef NEON_CAPABLE
+       if (NEON_CAPABLE) {
                gcm_init_neon(ctx->Htable,ctx->H.u);
                ctx->gmult = gcm_gmult_neon;
                ctx->ghash = gcm_ghash_neon;
-       } else {
+       } else
+#  endif
+       {
                gcm_init_4bit(ctx->Htable,ctx->H.u);
                ctx->gmult = gcm_gmult_4bit;
                ctx->ghash = gcm_ghash_4bit;