aesni-x86_64.pl: fix typo and optimize small block performance.

diff --git a/crypto/aes/asm/aesni-x86_64.pl b/crypto/aes/asm/aesni-x86_64.pl
index 27bb47c32634e7d7dfc909fe7abe682361921719..20164b6108082bb261da9134c343d7c38fa365b8 100644 (file)
--- a/crypto/aes/asm/aesni-x86_64.pl
+++ b/crypto/aes/asm/aesni-x86_64.pl
@@ -1014,7 +1014,7 @@ ___
 # does not update *ivec! (see crypto/modes/ctr128.c for details)
 #
 # Overhaul based on suggestions from Shay Gueron and Vlad Krasnov,
-# http://rt.openssl.org/Ticket/Display.html?id=3031&user=guest&pass=guest.
+# http://rt.openssl.org/Ticket/Display.html?id=3021&user=guest&pass=guest.
 # Keywords are full unroll and modulo-schedule counter calculations
 # with zero-round key xor.
 {
@@ -1058,9 +1058,9 @@ $code.=<<___;
        mov     12($key),$key0                  # 0-round key LSB
        movdqa  $inout0,0x00(%rsp)              # populate counter block
        bswap   $ctr
-       movdqa  $inout0,0x10(%rsp)
-       movdqa  $inout0,0x20(%rsp)
-       movdqa  $inout0,0x30(%rsp)
+       movdqa  $inout0,$inout1
+       movdqa  $inout0,$inout2
+       movdqa  $inout0,$inout3
        movdqa  $inout0,0x40(%rsp)
        movdqa  $inout0,0x50(%rsp)
        movdqa  $inout0,0x60(%rsp)
@@ -1074,15 +1074,18 @@ $code.=<<___;
         bswap  %r10d
        xor     $key0,%r9d
         xor    $key0,%r10d
-       mov     %r9d,0x10+12(%rsp)
+       pinsrd  \$3,%r9d,$inout1
        lea     3($ctr),%r9
-        mov    %r10d,0x20+12(%rsp)
+       movdqa  $inout1,0x10(%rsp)
+        pinsrd \$3,%r10d,$inout2
        bswap   %r9d
         lea    4($ctr),%r10
+        movdqa $inout2,0x20(%rsp)
        xor     $key0,%r9d
         bswap  %r10d
-       mov     %r9d,0x30+12(%rsp)
+       pinsrd  \$3,%r9d,$inout3
         xor    $key0,%r10d
+       movdqa  $inout3,0x30(%rsp)
        lea     5($ctr),%r9
         mov    %r10d,0x40+12(%rsp)
        bswap   %r9d
@@ -1099,9 +1102,6 @@ $code.=<<___;
 
        $movkey 0x10($key),$rndkey1
 
-       movdqa  0x10(%rsp),$inout1
-       movdqa  0x20(%rsp),$inout2
-       movdqa  0x30(%rsp),$inout3
        movdqa  0x40(%rsp),$inout4
        movdqa  0x50(%rsp),$inout5
 
@@ -1278,9 +1278,11 @@ $code.=<<___;
 .Lctr32_tail:
        lea     16($key),$key
        cmp     \$4,$len
-       jbe     .Lctr32_loop4
+       jb      .Lctr32_loop3
+       je      .Lctr32_loop4
 
        movdqa          0x60(%rsp),$inout6
+       pxor            $inout7,$inout7
 
        $movkey         16($key),$rndkey0
        aesenc          $rndkey1,$inout0
@@ -1290,17 +1292,16 @@ $code.=<<___;
        aesenc          $rndkey1,$inout2
        dec             $rounds
        aesenc          $rndkey1,$inout3
+        movups         ($inp),$in0
        aesenc          $rndkey1,$inout4
+        movups         0x10($inp),$in1
        aesenc          $rndkey1,$inout5
+        movups         0x20($inp),$in2
        aesenc          $rndkey1,$inout6
-       pxor            $inout7,$inout7
        $movkey         16($key),$rndkey1
 
        call            .Lenc_loop8_enter
 
-       movups  ($inp),$in0
-       movups  0x10($inp),$in1
-       movups  0x20($inp),$in2
        xorps   $in0,$inout0
        movups  0x30($inp),$in3
        xorps   $in1,$inout1
@@ -1337,9 +1338,36 @@ $code.=<<___;
        dec             $rounds
        jnz             .Lctr32_loop4
        aesenclast      $rndkey1,$inout0
+        movups         ($inp),$in0
        aesenclast      $rndkey1,$inout1
+        movups         0x10($inp),$in1
        aesenclast      $rndkey1,$inout2
+        movups         0x20($inp),$in2
        aesenclast      $rndkey1,$inout3
+        movups         0x30($inp),$in3
+
+       xorps   $in0,$inout0
+       movups  $inout0,($out)
+       xorps   $in1,$inout1
+       movups  $inout1,0x10($out)
+       xorps   $in2,$inout2
+       movups  $inout2,0x20($out)
+       xorps   $in3,$inout3
+       movups  $inout3,0x30($out)
+       jmp     .Lctr32_done
+
+.align 32
+.Lctr32_loop3:
+       aesenc          $rndkey1,$inout0
+       lea             16($key),$key
+       aesenc          $rndkey1,$inout1
+       aesenc          $rndkey1,$inout2
+       $movkey         ($key),$rndkey1
+       dec             $rounds
+       jnz             .Lctr32_loop3
+       aesenclast      $rndkey1,$inout0
+       aesenclast      $rndkey1,$inout1
+       aesenclast      $rndkey1,$inout2
 
        movups  ($inp),$in0
        xorps   $in0,$inout0
@@ -1355,12 +1383,6 @@ $code.=<<___;
        movups  0x20($inp),$in2
        xorps   $in2,$inout2
        movups  $inout2,0x20($out)
-       cmp     \$4,$len
-       jb      .Lctr32_done
-
-       movups  0x30($inp),$in3
-       xorps   $in3,$inout3
-       movups  $inout3,0x30($out)
        jmp     .Lctr32_done
 
 .align 16