crypto/aes/asm/aesni-mb-x86_64.pl

   1 #! /usr/bin/env perl
   2 # Copyright 2013-2016 The OpenSSL Project Authors. All Rights Reserved.
   3 #
   4 # Licensed under the OpenSSL license (the "License").  You may not use
   5 # this file except in compliance with the License.  You can obtain a copy
   6 # in the file LICENSE in the source distribution or at
   7 # https://www.openssl.org/source/license.html
   8
   9
  10 # ====================================================================
  11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
  12 # project. The module is, however, dual licensed under OpenSSL and
  13 # CRYPTOGAMS licenses depending on where you obtain it. For further
  14 # details see http://www.openssl.org/~appro/cryptogams/.
  15 # ====================================================================
  16
  17 # Multi-buffer AES-NI procedures process several independent buffers
  18 # in parallel by interleaving independent instructions.
  19 #
  20 # Cycles per byte for interleave factor 4:
  21 #
  22 #                       asymptotic      measured
  23 #                       ---------------------------
  24 # Westmere              5.00/4=1.25     5.13/4=1.28
  25 # Atom                  15.0/4=3.75     ?15.7/4=3.93
  26 # Sandy Bridge          5.06/4=1.27     5.18/4=1.29
  27 # Ivy Bridge            5.06/4=1.27     5.14/4=1.29
  28 # Haswell               4.44/4=1.11     4.44/4=1.11
  29 # Bulldozer             5.75/4=1.44     5.76/4=1.44
  30 #
  31 # Cycles per byte for interleave factor 8 (not implemented for
  32 # pre-AVX processors, where higher interleave factor incidentally
  33 # doesn't result in improvement):
  34 #
  35 #                       asymptotic      measured
  36 #                       ---------------------------
  37 # Sandy Bridge          5.06/8=0.64     7.10/8=0.89(*)
  38 # Ivy Bridge            5.06/8=0.64     7.14/8=0.89(*)
  39 # Haswell               5.00/8=0.63     5.00/8=0.63
  40 # Bulldozer             5.75/8=0.72     5.77/8=0.72
  41 #
  42 # (*)   Sandy/Ivy Bridge are known to handle high interleave factors
  43 #       suboptimally;
  44
  45 $flavour = shift;
  46 $output  = shift;
  47 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
  48
  49 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
  50
  51 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  52 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
  53 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
  54 die "can't locate x86_64-xlate.pl";
  55
  56 $avx=0;
  57
  58 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
  59                 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
  60         $avx = ($1>=2.19) + ($1>=2.22);
  61 }
  62
  63 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
  64            `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
  65         $avx = ($1>=2.09) + ($1>=2.10);
  66 }
  67
  68 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
  69            `ml64 2>&1` =~ /Version ([0-9]+)\./) {
  70         $avx = ($1>=10) + ($1>=11);
  71 }
  72
  73 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
  74         $avx = ($2>=3.0) + ($2>3.0);
  75 }
  76
  77 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
  78 *STDOUT=*OUT;
  79
  80 # void aesni_multi_cbc_encrypt (
  81 #     struct {  void *inp,*out; int blocks; double iv[2]; } inp[8];
  82 #     const AES_KEY *key,
  83 #     int num);         /* 1 or 2 */
  84 #
  85 $inp="%rdi";    # 1st arg
  86 $key="%rsi";    # 2nd arg
  87 $num="%edx";
  88
  89 @inptr=map("%r$_",(8..11));
  90 @outptr=map("%r$_",(12..15));
  91
  92 ($rndkey0,$rndkey1)=("%xmm0","%xmm1");
  93 @out=map("%xmm$_",(2..5));
  94 @inp=map("%xmm$_",(6..9));
  95 ($counters,$mask,$zero)=map("%xmm$_",(10..12));
  96
  97 ($rounds,$one,$sink,$offset)=("%eax","%ecx","%rbp","%rbx");
  98
  99 $code.=<<___;
 100 .text
 101
 102 .extern OPENSSL_ia32cap_P
 103
 104 .globl  aesni_multi_cbc_encrypt
 105 .type   aesni_multi_cbc_encrypt,\@function,3
 106 .align  32
 107 aesni_multi_cbc_encrypt:
 108 ___
 109 $code.=<<___ if ($avx);
 110         cmp     \$2,$num
 111         jb      .Lenc_non_avx
 112         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
 113         test    \$`1<<28`,%ecx                  # AVX bit
 114         jnz     _avx_cbc_enc_shortcut
 115         jmp     .Lenc_non_avx
 116 .align  16
 117 .Lenc_non_avx:
 118 ___
 119 $code.=<<___;
 120         mov     %rsp,%rax
 121         push    %rbx
 122         push    %rbp
 123         push    %r12
 124         push    %r13
 125         push    %r14
 126         push    %r15
 127 ___
 128 $code.=<<___ if ($win64);
 129         lea     -0xa8(%rsp),%rsp
 130         movaps  %xmm6,(%rsp)
 131         movaps  %xmm7,0x10(%rsp)
 132         movaps  %xmm8,0x20(%rsp)
 133         movaps  %xmm9,0x30(%rsp)
 134         movaps  %xmm10,0x40(%rsp)
 135         movaps  %xmm11,0x50(%rsp)
 136         movaps  %xmm12,0x60(%rsp)
 137         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler
 138         movaps  %xmm14,-0x58(%rax)
 139         movaps  %xmm15,-0x48(%rax)
 140 ___
 141 $code.=<<___;
 142         # stack layout
 143         #
 144         # +0    output sink
 145         # +16   input sink [original %rsp and $num]
 146         # +32   counters
 147
 148         sub     \$48,%rsp
 149         and     \$-64,%rsp
 150         mov     %rax,16(%rsp)                   # original %rsp
 151
 152 .Lenc4x_body:
 153         movdqu  ($key),$zero                    # 0-round key
 154         lea     0x78($key),$key                 # size optimization
 155         lea     40*2($inp),$inp
 156
 157 .Lenc4x_loop_grande:
 158         mov     $num,24(%rsp)                   # original $num
 159         xor     $num,$num
 160 ___
 161 for($i=0;$i<4;$i++) {
 162     $code.=<<___;
 163         mov     `40*$i+16-40*2`($inp),$one      # borrow $one for number of blocks
 164         mov     `40*$i+0-40*2`($inp),@inptr[$i]
 165         cmp     $num,$one
 166         mov     `40*$i+8-40*2`($inp),@outptr[$i]
 167         cmovg   $one,$num                       # find maximum
 168         test    $one,$one
 169         movdqu  `40*$i+24-40*2`($inp),@out[$i]  # load IV
 170         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 171         cmovle  %rsp,@inptr[$i]                 # cancel input
 172 ___
 173 }
 174 $code.=<<___;
 175         test    $num,$num
 176         jz      .Lenc4x_done
 177
 178         movups  0x10-0x78($key),$rndkey1
 179          pxor   $zero,@out[0]
 180         movups  0x20-0x78($key),$rndkey0
 181          pxor   $zero,@out[1]
 182         mov     0xf0-0x78($key),$rounds
 183          pxor   $zero,@out[2]
 184         movdqu  (@inptr[0]),@inp[0]             # load inputs
 185          pxor   $zero,@out[3]
 186         movdqu  (@inptr[1]),@inp[1]
 187          pxor   @inp[0],@out[0]
 188         movdqu  (@inptr[2]),@inp[2]
 189          pxor   @inp[1],@out[1]
 190         movdqu  (@inptr[3]),@inp[3]
 191          pxor   @inp[2],@out[2]
 192          pxor   @inp[3],@out[3]
 193         movdqa  32(%rsp),$counters              # load counters
 194         xor     $offset,$offset
 195         jmp     .Loop_enc4x
 196
 197 .align  32
 198 .Loop_enc4x:
 199         add     \$16,$offset
 200         lea     16(%rsp),$sink                  # sink pointer
 201         mov     \$1,$one                        # constant of 1
 202         sub     $offset,$sink
 203
 204         aesenc          $rndkey1,@out[0]
 205         prefetcht0      31(@inptr[0],$offset)   # prefetch input
 206         prefetcht0      31(@inptr[1],$offset)
 207         aesenc          $rndkey1,@out[1]
 208         prefetcht0      31(@inptr[2],$offset)
 209         prefetcht0      31(@inptr[2],$offset)
 210         aesenc          $rndkey1,@out[2]
 211         aesenc          $rndkey1,@out[3]
 212         movups          0x30-0x78($key),$rndkey1
 213 ___
 214 for($i=0;$i<4;$i++) {
 215 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
 216 $code.=<<___;
 217          cmp            `32+4*$i`(%rsp),$one
 218         aesenc          $rndkey,@out[0]
 219         aesenc          $rndkey,@out[1]
 220         aesenc          $rndkey,@out[2]
 221          cmovge         $sink,@inptr[$i]        # cancel input
 222          cmovg          $sink,@outptr[$i]       # sink output
 223         aesenc          $rndkey,@out[3]
 224         movups          `0x40+16*$i-0x78`($key),$rndkey
 225 ___
 226 }
 227 $code.=<<___;
 228          movdqa         $counters,$mask
 229         aesenc          $rndkey0,@out[0]
 230         prefetcht0      15(@outptr[0],$offset)  # prefetch output
 231         prefetcht0      15(@outptr[1],$offset)
 232         aesenc          $rndkey0,@out[1]
 233         prefetcht0      15(@outptr[2],$offset)
 234         prefetcht0      15(@outptr[3],$offset)
 235         aesenc          $rndkey0,@out[2]
 236         aesenc          $rndkey0,@out[3]
 237         movups          0x80-0x78($key),$rndkey0
 238          pxor           $zero,$zero
 239
 240         aesenc          $rndkey1,@out[0]
 241          pcmpgtd        $zero,$mask
 242          movdqu         -0x78($key),$zero       # reload 0-round key
 243         aesenc          $rndkey1,@out[1]
 244          paddd          $mask,$counters         # decrement counters
 245          movdqa         $counters,32(%rsp)      # update counters
 246         aesenc          $rndkey1,@out[2]
 247         aesenc          $rndkey1,@out[3]
 248         movups          0x90-0x78($key),$rndkey1
 249
 250         cmp     \$11,$rounds
 251
 252         aesenc          $rndkey0,@out[0]
 253         aesenc          $rndkey0,@out[1]
 254         aesenc          $rndkey0,@out[2]
 255         aesenc          $rndkey0,@out[3]
 256         movups          0xa0-0x78($key),$rndkey0
 257
 258         jb      .Lenc4x_tail
 259
 260         aesenc          $rndkey1,@out[0]
 261         aesenc          $rndkey1,@out[1]
 262         aesenc          $rndkey1,@out[2]
 263         aesenc          $rndkey1,@out[3]
 264         movups          0xb0-0x78($key),$rndkey1
 265
 266         aesenc          $rndkey0,@out[0]
 267         aesenc          $rndkey0,@out[1]
 268         aesenc          $rndkey0,@out[2]
 269         aesenc          $rndkey0,@out[3]
 270         movups          0xc0-0x78($key),$rndkey0
 271
 272         je      .Lenc4x_tail
 273
 274         aesenc          $rndkey1,@out[0]
 275         aesenc          $rndkey1,@out[1]
 276         aesenc          $rndkey1,@out[2]
 277         aesenc          $rndkey1,@out[3]
 278         movups          0xd0-0x78($key),$rndkey1
 279
 280         aesenc          $rndkey0,@out[0]
 281         aesenc          $rndkey0,@out[1]
 282         aesenc          $rndkey0,@out[2]
 283         aesenc          $rndkey0,@out[3]
 284         movups          0xe0-0x78($key),$rndkey0
 285         jmp     .Lenc4x_tail
 286
 287 .align  32
 288 .Lenc4x_tail:
 289         aesenc          $rndkey1,@out[0]
 290         aesenc          $rndkey1,@out[1]
 291         aesenc          $rndkey1,@out[2]
 292         aesenc          $rndkey1,@out[3]
 293          movdqu         (@inptr[0],$offset),@inp[0]
 294         movdqu          0x10-0x78($key),$rndkey1
 295
 296         aesenclast      $rndkey0,@out[0]
 297          movdqu         (@inptr[1],$offset),@inp[1]
 298          pxor           $zero,@inp[0]
 299         aesenclast      $rndkey0,@out[1]
 300          movdqu         (@inptr[2],$offset),@inp[2]
 301          pxor           $zero,@inp[1]
 302         aesenclast      $rndkey0,@out[2]
 303          movdqu         (@inptr[3],$offset),@inp[3]
 304          pxor           $zero,@inp[2]
 305         aesenclast      $rndkey0,@out[3]
 306         movdqu          0x20-0x78($key),$rndkey0
 307          pxor           $zero,@inp[3]
 308
 309         movups          @out[0],-16(@outptr[0],$offset)
 310          pxor           @inp[0],@out[0]
 311         movups          @out[1],-16(@outptr[1],$offset)
 312          pxor           @inp[1],@out[1]
 313         movups          @out[2],-16(@outptr[2],$offset)
 314          pxor           @inp[2],@out[2]
 315         movups          @out[3],-16(@outptr[3],$offset)
 316          pxor           @inp[3],@out[3]
 317
 318         dec     $num
 319         jnz     .Loop_enc4x
 320
 321         mov     16(%rsp),%rax                   # original %rsp
 322         mov     24(%rsp),$num
 323
 324         #pxor   @inp[0],@out[0]
 325         #pxor   @inp[1],@out[1]
 326         #movdqu @out[0],`40*0+24-40*2`($inp)    # output iv FIX ME!
 327         #pxor   @inp[2],@out[2]
 328         #movdqu @out[1],`40*1+24-40*2`($inp)
 329         #pxor   @inp[3],@out[3]
 330         #movdqu @out[2],`40*2+24-40*2`($inp)    # won't fix, let caller
 331         #movdqu @out[3],`40*3+24-40*2`($inp)    # figure this out...
 332
 333         lea     `40*4`($inp),$inp
 334         dec     $num
 335         jnz     .Lenc4x_loop_grande
 336
 337 .Lenc4x_done:
 338 ___
 339 $code.=<<___ if ($win64);
 340         movaps  -0xd8(%rax),%xmm6
 341         movaps  -0xc8(%rax),%xmm7
 342         movaps  -0xb8(%rax),%xmm8
 343         movaps  -0xa8(%rax),%xmm9
 344         movaps  -0x98(%rax),%xmm10
 345         movaps  -0x88(%rax),%xmm11
 346         movaps  -0x78(%rax),%xmm12
 347         #movaps -0x68(%rax),%xmm13
 348         #movaps -0x58(%rax),%xmm14
 349         #movaps -0x48(%rax),%xmm15
 350 ___
 351 $code.=<<___;
 352         mov     -48(%rax),%r15
 353         mov     -40(%rax),%r14
 354         mov     -32(%rax),%r13
 355         mov     -24(%rax),%r12
 356         mov     -16(%rax),%rbp
 357         mov     -8(%rax),%rbx
 358         lea     (%rax),%rsp
 359 .Lenc4x_epilogue:
 360         ret
 361 .size   aesni_multi_cbc_encrypt,.-aesni_multi_cbc_encrypt
 362
 363 .globl  aesni_multi_cbc_decrypt
 364 .type   aesni_multi_cbc_decrypt,\@function,3
 365 .align  32
 366 aesni_multi_cbc_decrypt:
 367 ___
 368 $code.=<<___ if ($avx);
 369         cmp     \$2,$num
 370         jb      .Ldec_non_avx
 371         mov     OPENSSL_ia32cap_P+4(%rip),%ecx
 372         test    \$`1<<28`,%ecx                  # AVX bit
 373         jnz     _avx_cbc_dec_shortcut
 374         jmp     .Ldec_non_avx
 375 .align  16
 376 .Ldec_non_avx:
 377 ___
 378 $code.=<<___;
 379         mov     %rsp,%rax
 380         push    %rbx
 381         push    %rbp
 382         push    %r12
 383         push    %r13
 384         push    %r14
 385         push    %r15
 386 ___
 387 $code.=<<___ if ($win64);
 388         lea     -0xa8(%rsp),%rsp
 389         movaps  %xmm6,(%rsp)
 390         movaps  %xmm7,0x10(%rsp)
 391         movaps  %xmm8,0x20(%rsp)
 392         movaps  %xmm9,0x30(%rsp)
 393         movaps  %xmm10,0x40(%rsp)
 394         movaps  %xmm11,0x50(%rsp)
 395         movaps  %xmm12,0x60(%rsp)
 396         movaps  %xmm13,-0x68(%rax)      # not used, saved to share se_handler
 397         movaps  %xmm14,-0x58(%rax)
 398         movaps  %xmm15,-0x48(%rax)
 399 ___
 400 $code.=<<___;
 401         # stack layout
 402         #
 403         # +0    output sink
 404         # +16   input sink [original %rsp and $num]
 405         # +32   counters
 406
 407         sub     \$48,%rsp
 408         and     \$-64,%rsp
 409         mov     %rax,16(%rsp)                   # original %rsp
 410
 411 .Ldec4x_body:
 412         movdqu  ($key),$zero                    # 0-round key
 413         lea     0x78($key),$key                 # size optimization
 414         lea     40*2($inp),$inp
 415
 416 .Ldec4x_loop_grande:
 417         mov     $num,24(%rsp)                   # original $num
 418         xor     $num,$num
 419 ___
 420 for($i=0;$i<4;$i++) {
 421     $code.=<<___;
 422         mov     `40*$i+16-40*2`($inp),$one      # borrow $one for number of blocks
 423         mov     `40*$i+0-40*2`($inp),@inptr[$i]
 424         cmp     $num,$one
 425         mov     `40*$i+8-40*2`($inp),@outptr[$i]
 426         cmovg   $one,$num                       # find maximum
 427         test    $one,$one
 428         movdqu  `40*$i+24-40*2`($inp),@inp[$i]  # load IV
 429         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 430         cmovle  %rsp,@inptr[$i]                 # cancel input
 431 ___
 432 }
 433 $code.=<<___;
 434         test    $num,$num
 435         jz      .Ldec4x_done
 436
 437         movups  0x10-0x78($key),$rndkey1
 438         movups  0x20-0x78($key),$rndkey0
 439         mov     0xf0-0x78($key),$rounds
 440         movdqu  (@inptr[0]),@out[0]             # load inputs
 441         movdqu  (@inptr[1]),@out[1]
 442          pxor   $zero,@out[0]
 443         movdqu  (@inptr[2]),@out[2]
 444          pxor   $zero,@out[1]
 445         movdqu  (@inptr[3]),@out[3]
 446          pxor   $zero,@out[2]
 447          pxor   $zero,@out[3]
 448         movdqa  32(%rsp),$counters              # load counters
 449         xor     $offset,$offset
 450         jmp     .Loop_dec4x
 451
 452 .align  32
 453 .Loop_dec4x:
 454         add     \$16,$offset
 455         lea     16(%rsp),$sink                  # sink pointer
 456         mov     \$1,$one                        # constant of 1
 457         sub     $offset,$sink
 458
 459         aesdec          $rndkey1,@out[0]
 460         prefetcht0      31(@inptr[0],$offset)   # prefetch input
 461         prefetcht0      31(@inptr[1],$offset)
 462         aesdec          $rndkey1,@out[1]
 463         prefetcht0      31(@inptr[2],$offset)
 464         prefetcht0      31(@inptr[3],$offset)
 465         aesdec          $rndkey1,@out[2]
 466         aesdec          $rndkey1,@out[3]
 467         movups          0x30-0x78($key),$rndkey1
 468 ___
 469 for($i=0;$i<4;$i++) {
 470 my $rndkey = ($i&1) ? $rndkey1 : $rndkey0;
 471 $code.=<<___;
 472          cmp            `32+4*$i`(%rsp),$one
 473         aesdec          $rndkey,@out[0]
 474         aesdec          $rndkey,@out[1]
 475         aesdec          $rndkey,@out[2]
 476          cmovge         $sink,@inptr[$i]        # cancel input
 477          cmovg          $sink,@outptr[$i]       # sink output
 478         aesdec          $rndkey,@out[3]
 479         movups          `0x40+16*$i-0x78`($key),$rndkey
 480 ___
 481 }
 482 $code.=<<___;
 483          movdqa         $counters,$mask
 484         aesdec          $rndkey0,@out[0]
 485         prefetcht0      15(@outptr[0],$offset)  # prefetch output
 486         prefetcht0      15(@outptr[1],$offset)
 487         aesdec          $rndkey0,@out[1]
 488         prefetcht0      15(@outptr[2],$offset)
 489         prefetcht0      15(@outptr[3],$offset)
 490         aesdec          $rndkey0,@out[2]
 491         aesdec          $rndkey0,@out[3]
 492         movups          0x80-0x78($key),$rndkey0
 493          pxor           $zero,$zero
 494
 495         aesdec          $rndkey1,@out[0]
 496          pcmpgtd        $zero,$mask
 497          movdqu         -0x78($key),$zero       # reload 0-round key
 498         aesdec          $rndkey1,@out[1]
 499          paddd          $mask,$counters         # decrement counters
 500          movdqa         $counters,32(%rsp)      # update counters
 501         aesdec          $rndkey1,@out[2]
 502         aesdec          $rndkey1,@out[3]
 503         movups          0x90-0x78($key),$rndkey1
 504
 505         cmp     \$11,$rounds
 506
 507         aesdec          $rndkey0,@out[0]
 508         aesdec          $rndkey0,@out[1]
 509         aesdec          $rndkey0,@out[2]
 510         aesdec          $rndkey0,@out[3]
 511         movups          0xa0-0x78($key),$rndkey0
 512
 513         jb      .Ldec4x_tail
 514
 515         aesdec          $rndkey1,@out[0]
 516         aesdec          $rndkey1,@out[1]
 517         aesdec          $rndkey1,@out[2]
 518         aesdec          $rndkey1,@out[3]
 519         movups          0xb0-0x78($key),$rndkey1
 520
 521         aesdec          $rndkey0,@out[0]
 522         aesdec          $rndkey0,@out[1]
 523         aesdec          $rndkey0,@out[2]
 524         aesdec          $rndkey0,@out[3]
 525         movups          0xc0-0x78($key),$rndkey0
 526
 527         je      .Ldec4x_tail
 528
 529         aesdec          $rndkey1,@out[0]
 530         aesdec          $rndkey1,@out[1]
 531         aesdec          $rndkey1,@out[2]
 532         aesdec          $rndkey1,@out[3]
 533         movups          0xd0-0x78($key),$rndkey1
 534
 535         aesdec          $rndkey0,@out[0]
 536         aesdec          $rndkey0,@out[1]
 537         aesdec          $rndkey0,@out[2]
 538         aesdec          $rndkey0,@out[3]
 539         movups          0xe0-0x78($key),$rndkey0
 540         jmp     .Ldec4x_tail
 541
 542 .align  32
 543 .Ldec4x_tail:
 544         aesdec          $rndkey1,@out[0]
 545         aesdec          $rndkey1,@out[1]
 546         aesdec          $rndkey1,@out[2]
 547          pxor           $rndkey0,@inp[0]
 548          pxor           $rndkey0,@inp[1]
 549         aesdec          $rndkey1,@out[3]
 550         movdqu          0x10-0x78($key),$rndkey1
 551          pxor           $rndkey0,@inp[2]
 552          pxor           $rndkey0,@inp[3]
 553         movdqu          0x20-0x78($key),$rndkey0
 554
 555         aesdeclast      @inp[0],@out[0]
 556         aesdeclast      @inp[1],@out[1]
 557          movdqu         -16(@inptr[0],$offset),@inp[0]  # load next IV
 558          movdqu         -16(@inptr[1],$offset),@inp[1]
 559         aesdeclast      @inp[2],@out[2]
 560         aesdeclast      @inp[3],@out[3]
 561          movdqu         -16(@inptr[2],$offset),@inp[2]
 562          movdqu         -16(@inptr[3],$offset),@inp[3]
 563
 564         movups          @out[0],-16(@outptr[0],$offset)
 565          movdqu         (@inptr[0],$offset),@out[0]
 566         movups          @out[1],-16(@outptr[1],$offset)
 567          movdqu         (@inptr[1],$offset),@out[1]
 568          pxor           $zero,@out[0]
 569         movups          @out[2],-16(@outptr[2],$offset)
 570          movdqu         (@inptr[2],$offset),@out[2]
 571          pxor           $zero,@out[1]
 572         movups          @out[3],-16(@outptr[3],$offset)
 573          movdqu         (@inptr[3],$offset),@out[3]
 574          pxor           $zero,@out[2]
 575          pxor           $zero,@out[3]
 576
 577         dec     $num
 578         jnz     .Loop_dec4x
 579
 580         mov     16(%rsp),%rax                   # original %rsp
 581         mov     24(%rsp),$num
 582
 583         lea     `40*4`($inp),$inp
 584         dec     $num
 585         jnz     .Ldec4x_loop_grande
 586
 587 .Ldec4x_done:
 588 ___
 589 $code.=<<___ if ($win64);
 590         movaps  -0xd8(%rax),%xmm6
 591         movaps  -0xc8(%rax),%xmm7
 592         movaps  -0xb8(%rax),%xmm8
 593         movaps  -0xa8(%rax),%xmm9
 594         movaps  -0x98(%rax),%xmm10
 595         movaps  -0x88(%rax),%xmm11
 596         movaps  -0x78(%rax),%xmm12
 597         #movaps -0x68(%rax),%xmm13
 598         #movaps -0x58(%rax),%xmm14
 599         #movaps -0x48(%rax),%xmm15
 600 ___
 601 $code.=<<___;
 602         mov     -48(%rax),%r15
 603         mov     -40(%rax),%r14
 604         mov     -32(%rax),%r13
 605         mov     -24(%rax),%r12
 606         mov     -16(%rax),%rbp
 607         mov     -8(%rax),%rbx
 608         lea     (%rax),%rsp
 609 .Ldec4x_epilogue:
 610         ret
 611 .size   aesni_multi_cbc_decrypt,.-aesni_multi_cbc_decrypt
 612 ___
 613
 614                                                 if ($avx) {{{
 615 my @ptr=map("%r$_",(8..15));
 616 my $offload=$sink;
 617
 618 my @out=map("%xmm$_",(2..9));
 619 my @inp=map("%xmm$_",(10..13));
 620 my ($counters,$zero)=("%xmm14","%xmm15");
 621
 622 $code.=<<___;
 623 .type   aesni_multi_cbc_encrypt_avx,\@function,3
 624 .align  32
 625 aesni_multi_cbc_encrypt_avx:
 626 _avx_cbc_enc_shortcut:
 627         mov     %rsp,%rax
 628         push    %rbx
 629         push    %rbp
 630         push    %r12
 631         push    %r13
 632         push    %r14
 633         push    %r15
 634 ___
 635 $code.=<<___ if ($win64);
 636         lea     -0xa8(%rsp),%rsp
 637         movaps  %xmm6,(%rsp)
 638         movaps  %xmm7,0x10(%rsp)
 639         movaps  %xmm8,0x20(%rsp)
 640         movaps  %xmm9,0x30(%rsp)
 641         movaps  %xmm10,0x40(%rsp)
 642         movaps  %xmm11,0x50(%rsp)
 643         movaps  %xmm12,-0x78(%rax)
 644         movaps  %xmm13,-0x68(%rax)
 645         movaps  %xmm14,-0x58(%rax)
 646         movaps  %xmm15,-0x48(%rax)
 647 ___
 648 $code.=<<___;
 649         # stack layout
 650         #
 651         # +0    output sink
 652         # +16   input sink [original %rsp and $num]
 653         # +32   counters
 654         # +64   distances between inputs and outputs
 655         # +128  off-load area for @inp[0..3]
 656
 657         sub     \$192,%rsp
 658         and     \$-128,%rsp
 659         mov     %rax,16(%rsp)                   # original %rsp
 660
 661 .Lenc8x_body:
 662         vzeroupper
 663         vmovdqu ($key),$zero                    # 0-round key
 664         lea     0x78($key),$key                 # size optimization
 665         lea     40*4($inp),$inp
 666         shr     \$1,$num
 667
 668 .Lenc8x_loop_grande:
 669         #mov    $num,24(%rsp)                   # original $num
 670         xor     $num,$num
 671 ___
 672 for($i=0;$i<8;$i++) {
 673   my $temp = $i ? $offload : $offset;
 674     $code.=<<___;
 675         mov     `40*$i+16-40*4`($inp),$one      # borrow $one for number of blocks
 676         mov     `40*$i+0-40*4`($inp),@ptr[$i]   # input pointer
 677         cmp     $num,$one
 678         mov     `40*$i+8-40*4`($inp),$temp      # output pointer
 679         cmovg   $one,$num                       # find maximum
 680         test    $one,$one
 681         vmovdqu `40*$i+24-40*4`($inp),@out[$i]  # load IV
 682         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 683         cmovle  %rsp,@ptr[$i]                   # cancel input
 684         sub     @ptr[$i],$temp                  # distance between input and output
 685         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
 686 ___
 687 }
 688 $code.=<<___;
 689         test    $num,$num
 690         jz      .Lenc8x_done
 691
 692         vmovups 0x10-0x78($key),$rndkey1
 693         vmovups 0x20-0x78($key),$rndkey0
 694         mov     0xf0-0x78($key),$rounds
 695
 696         vpxor   (@ptr[0]),$zero,@inp[0]         # load inputs and xor with 0-round
 697          lea    128(%rsp),$offload              # offload area
 698         vpxor   (@ptr[1]),$zero,@inp[1]
 699         vpxor   (@ptr[2]),$zero,@inp[2]
 700         vpxor   (@ptr[3]),$zero,@inp[3]
 701          vpxor  @inp[0],@out[0],@out[0]
 702         vpxor   (@ptr[4]),$zero,@inp[0]
 703          vpxor  @inp[1],@out[1],@out[1]
 704         vpxor   (@ptr[5]),$zero,@inp[1]
 705          vpxor  @inp[2],@out[2],@out[2]
 706         vpxor   (@ptr[6]),$zero,@inp[2]
 707          vpxor  @inp[3],@out[3],@out[3]
 708         vpxor   (@ptr[7]),$zero,@inp[3]
 709          vpxor  @inp[0],@out[4],@out[4]
 710         mov     \$1,$one                        # constant of 1
 711          vpxor  @inp[1],@out[5],@out[5]
 712          vpxor  @inp[2],@out[6],@out[6]
 713          vpxor  @inp[3],@out[7],@out[7]
 714         jmp     .Loop_enc8x
 715
 716 .align  32
 717 .Loop_enc8x:
 718 ___
 719 for($i=0;$i<8;$i++) {
 720 my $rndkey=($i&1)?$rndkey0:$rndkey1;
 721 $code.=<<___;
 722         vaesenc         $rndkey,@out[0],@out[0]
 723          cmp            32+4*$i(%rsp),$one
 724 ___
 725 $code.=<<___ if ($i);
 726          mov            64+8*$i(%rsp),$offset
 727 ___
 728 $code.=<<___;
 729         vaesenc         $rndkey,@out[1],@out[1]
 730         prefetcht0      31(@ptr[$i])                    # prefetch input
 731         vaesenc         $rndkey,@out[2],@out[2]
 732 ___
 733 $code.=<<___ if ($i>1);
 734         prefetcht0      15(@ptr[$i-2])                  # prefetch output
 735 ___
 736 $code.=<<___;
 737         vaesenc         $rndkey,@out[3],@out[3]
 738          lea            (@ptr[$i],$offset),$offset
 739          cmovge         %rsp,@ptr[$i]                   # cancel input
 740         vaesenc         $rndkey,@out[4],@out[4]
 741          cmovg          %rsp,$offset                    # sink output
 742         vaesenc         $rndkey,@out[5],@out[5]
 743          sub            @ptr[$i],$offset
 744         vaesenc         $rndkey,@out[6],@out[6]
 745          vpxor          16(@ptr[$i]),$zero,@inp[$i%4]   # load input and xor with 0-round
 746          mov            $offset,64+8*$i(%rsp)
 747         vaesenc         $rndkey,@out[7],@out[7]
 748         vmovups         `16*(3+$i)-0x78`($key),$rndkey
 749          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
 750 ___
 751 $code.=<<___ if ($i<4)
 752          vmovdqu        @inp[$i%4],`16*$i`($offload)    # off-load
 753 ___
 754 }
 755 $code.=<<___;
 756          vmovdqu        32(%rsp),$counters
 757         prefetcht0      15(@ptr[$i-2])                  # prefetch output
 758         prefetcht0      15(@ptr[$i-1])
 759         cmp     \$11,$rounds
 760         jb      .Lenc8x_tail
 761
 762         vaesenc         $rndkey1,@out[0],@out[0]
 763         vaesenc         $rndkey1,@out[1],@out[1]
 764         vaesenc         $rndkey1,@out[2],@out[2]
 765         vaesenc         $rndkey1,@out[3],@out[3]
 766         vaesenc         $rndkey1,@out[4],@out[4]
 767         vaesenc         $rndkey1,@out[5],@out[5]
 768         vaesenc         $rndkey1,@out[6],@out[6]
 769         vaesenc         $rndkey1,@out[7],@out[7]
 770         vmovups         0xb0-0x78($key),$rndkey1
 771
 772         vaesenc         $rndkey0,@out[0],@out[0]
 773         vaesenc         $rndkey0,@out[1],@out[1]
 774         vaesenc         $rndkey0,@out[2],@out[2]
 775         vaesenc         $rndkey0,@out[3],@out[3]
 776         vaesenc         $rndkey0,@out[4],@out[4]
 777         vaesenc         $rndkey0,@out[5],@out[5]
 778         vaesenc         $rndkey0,@out[6],@out[6]
 779         vaesenc         $rndkey0,@out[7],@out[7]
 780         vmovups         0xc0-0x78($key),$rndkey0
 781         je      .Lenc8x_tail
 782
 783         vaesenc         $rndkey1,@out[0],@out[0]
 784         vaesenc         $rndkey1,@out[1],@out[1]
 785         vaesenc         $rndkey1,@out[2],@out[2]
 786         vaesenc         $rndkey1,@out[3],@out[3]
 787         vaesenc         $rndkey1,@out[4],@out[4]
 788         vaesenc         $rndkey1,@out[5],@out[5]
 789         vaesenc         $rndkey1,@out[6],@out[6]
 790         vaesenc         $rndkey1,@out[7],@out[7]
 791         vmovups         0xd0-0x78($key),$rndkey1
 792
 793         vaesenc         $rndkey0,@out[0],@out[0]
 794         vaesenc         $rndkey0,@out[1],@out[1]
 795         vaesenc         $rndkey0,@out[2],@out[2]
 796         vaesenc         $rndkey0,@out[3],@out[3]
 797         vaesenc         $rndkey0,@out[4],@out[4]
 798         vaesenc         $rndkey0,@out[5],@out[5]
 799         vaesenc         $rndkey0,@out[6],@out[6]
 800         vaesenc         $rndkey0,@out[7],@out[7]
 801         vmovups         0xe0-0x78($key),$rndkey0
 802
 803 .Lenc8x_tail:
 804         vaesenc         $rndkey1,@out[0],@out[0]
 805          vpxor          $zero,$zero,$zero
 806         vaesenc         $rndkey1,@out[1],@out[1]
 807         vaesenc         $rndkey1,@out[2],@out[2]
 808          vpcmpgtd       $zero,$counters,$zero
 809         vaesenc         $rndkey1,@out[3],@out[3]
 810         vaesenc         $rndkey1,@out[4],@out[4]
 811          vpaddd         $counters,$zero,$zero           # decrement counters
 812          vmovdqu        48(%rsp),$counters
 813         vaesenc         $rndkey1,@out[5],@out[5]
 814          mov            64(%rsp),$offset                # pre-load 1st offset
 815         vaesenc         $rndkey1,@out[6],@out[6]
 816         vaesenc         $rndkey1,@out[7],@out[7]
 817         vmovups         0x10-0x78($key),$rndkey1
 818
 819         vaesenclast     $rndkey0,@out[0],@out[0]
 820          vmovdqa        $zero,32(%rsp)                  # update counters
 821          vpxor          $zero,$zero,$zero
 822         vaesenclast     $rndkey0,@out[1],@out[1]
 823         vaesenclast     $rndkey0,@out[2],@out[2]
 824          vpcmpgtd       $zero,$counters,$zero
 825         vaesenclast     $rndkey0,@out[3],@out[3]
 826         vaesenclast     $rndkey0,@out[4],@out[4]
 827          vpaddd         $zero,$counters,$counters       # decrement counters
 828          vmovdqu        -0x78($key),$zero               # 0-round
 829         vaesenclast     $rndkey0,@out[5],@out[5]
 830         vaesenclast     $rndkey0,@out[6],@out[6]
 831          vmovdqa        $counters,48(%rsp)              # update counters
 832         vaesenclast     $rndkey0,@out[7],@out[7]
 833         vmovups         0x20-0x78($key),$rndkey0
 834
 835         vmovups         @out[0],-16(@ptr[0])            # write output
 836          sub            $offset,@ptr[0]                 # switch to input
 837          vpxor          0x00($offload),@out[0],@out[0]
 838         vmovups         @out[1],-16(@ptr[1])
 839          sub            `64+1*8`(%rsp),@ptr[1]
 840          vpxor          0x10($offload),@out[1],@out[1]
 841         vmovups         @out[2],-16(@ptr[2])
 842          sub            `64+2*8`(%rsp),@ptr[2]
 843          vpxor          0x20($offload),@out[2],@out[2]
 844         vmovups         @out[3],-16(@ptr[3])
 845          sub            `64+3*8`(%rsp),@ptr[3]
 846          vpxor          0x30($offload),@out[3],@out[3]
 847         vmovups         @out[4],-16(@ptr[4])
 848          sub            `64+4*8`(%rsp),@ptr[4]
 849          vpxor          @inp[0],@out[4],@out[4]
 850         vmovups         @out[5],-16(@ptr[5])
 851          sub            `64+5*8`(%rsp),@ptr[5]
 852          vpxor          @inp[1],@out[5],@out[5]
 853         vmovups         @out[6],-16(@ptr[6])
 854          sub            `64+6*8`(%rsp),@ptr[6]
 855          vpxor          @inp[2],@out[6],@out[6]
 856         vmovups         @out[7],-16(@ptr[7])
 857          sub            `64+7*8`(%rsp),@ptr[7]
 858          vpxor          @inp[3],@out[7],@out[7]
 859
 860         dec     $num
 861         jnz     .Loop_enc8x
 862
 863         mov     16(%rsp),%rax                   # original %rsp
 864         #mov    24(%rsp),$num
 865         #lea    `40*8`($inp),$inp
 866         #dec    $num
 867         #jnz    .Lenc8x_loop_grande
 868
 869 .Lenc8x_done:
 870         vzeroupper
 871 ___
 872 $code.=<<___ if ($win64);
 873         movaps  -0xd8(%rax),%xmm6
 874         movaps  -0xc8(%rax),%xmm7
 875         movaps  -0xb8(%rax),%xmm8
 876         movaps  -0xa8(%rax),%xmm9
 877         movaps  -0x98(%rax),%xmm10
 878         movaps  -0x88(%rax),%xmm11
 879         movaps  -0x78(%rax),%xmm12
 880         movaps  -0x68(%rax),%xmm13
 881         movaps  -0x58(%rax),%xmm14
 882         movaps  -0x48(%rax),%xmm15
 883 ___
 884 $code.=<<___;
 885         mov     -48(%rax),%r15
 886         mov     -40(%rax),%r14
 887         mov     -32(%rax),%r13
 888         mov     -24(%rax),%r12
 889         mov     -16(%rax),%rbp
 890         mov     -8(%rax),%rbx
 891         lea     (%rax),%rsp
 892 .Lenc8x_epilogue:
 893         ret
 894 .size   aesni_multi_cbc_encrypt_avx,.-aesni_multi_cbc_encrypt_avx
 895
 896 .type   aesni_multi_cbc_decrypt_avx,\@function,3
 897 .align  32
 898 aesni_multi_cbc_decrypt_avx:
 899 _avx_cbc_dec_shortcut:
 900         mov     %rsp,%rax
 901         push    %rbx
 902         push    %rbp
 903         push    %r12
 904         push    %r13
 905         push    %r14
 906         push    %r15
 907 ___
 908 $code.=<<___ if ($win64);
 909         lea     -0xa8(%rsp),%rsp
 910         movaps  %xmm6,(%rsp)
 911         movaps  %xmm7,0x10(%rsp)
 912         movaps  %xmm8,0x20(%rsp)
 913         movaps  %xmm9,0x30(%rsp)
 914         movaps  %xmm10,0x40(%rsp)
 915         movaps  %xmm11,0x50(%rsp)
 916         movaps  %xmm12,-0x78(%rax)
 917         movaps  %xmm13,-0x68(%rax)
 918         movaps  %xmm14,-0x58(%rax)
 919         movaps  %xmm15,-0x48(%rax)
 920 ___
 921 $code.=<<___;
 922         # stack layout
 923         #
 924         # +0    output sink
 925         # +16   input sink [original %rsp and $num]
 926         # +32   counters
 927         # +64   distances between inputs and outputs
 928         # +128  off-load area for @inp[0..3]
 929         # +192  IV/input offload
 930
 931         sub     \$256,%rsp
 932         and     \$-256,%rsp
 933         sub     \$192,%rsp
 934         mov     %rax,16(%rsp)                   # original %rsp
 935
 936 .Ldec8x_body:
 937         vzeroupper
 938         vmovdqu ($key),$zero                    # 0-round key
 939         lea     0x78($key),$key                 # size optimization
 940         lea     40*4($inp),$inp
 941         shr     \$1,$num
 942
 943 .Ldec8x_loop_grande:
 944         #mov    $num,24(%rsp)                   # original $num
 945         xor     $num,$num
 946 ___
 947 for($i=0;$i<8;$i++) {
 948   my $temp = $i ? $offload : $offset;
 949     $code.=<<___;
 950         mov     `40*$i+16-40*4`($inp),$one      # borrow $one for number of blocks
 951         mov     `40*$i+0-40*4`($inp),@ptr[$i]   # input pointer
 952         cmp     $num,$one
 953         mov     `40*$i+8-40*4`($inp),$temp      # output pointer
 954         cmovg   $one,$num                       # find maximum
 955         test    $one,$one
 956         vmovdqu `40*$i+24-40*4`($inp),@out[$i]  # load IV
 957         mov     $one,`32+4*$i`(%rsp)            # initialize counters
 958         cmovle  %rsp,@ptr[$i]                   # cancel input
 959         sub     @ptr[$i],$temp                  # distance between input and output
 960         mov     $temp,`64+8*$i`(%rsp)           # initialize distances
 961         vmovdqu @out[$i],`192+16*$i`(%rsp)      # offload IV
 962 ___
 963 }
 964 $code.=<<___;
 965         test    $num,$num
 966         jz      .Ldec8x_done
 967
 968         vmovups 0x10-0x78($key),$rndkey1
 969         vmovups 0x20-0x78($key),$rndkey0
 970         mov     0xf0-0x78($key),$rounds
 971          lea    192+128(%rsp),$offload          # offload area
 972
 973         vmovdqu (@ptr[0]),@out[0]               # load inputs
 974         vmovdqu (@ptr[1]),@out[1]
 975         vmovdqu (@ptr[2]),@out[2]
 976         vmovdqu (@ptr[3]),@out[3]
 977         vmovdqu (@ptr[4]),@out[4]
 978         vmovdqu (@ptr[5]),@out[5]
 979         vmovdqu (@ptr[6]),@out[6]
 980         vmovdqu (@ptr[7]),@out[7]
 981         vmovdqu @out[0],0x00($offload)          # offload inputs
 982         vpxor   $zero,@out[0],@out[0]           # xor inputs with 0-round
 983         vmovdqu @out[1],0x10($offload)
 984         vpxor   $zero,@out[1],@out[1]
 985         vmovdqu @out[2],0x20($offload)
 986         vpxor   $zero,@out[2],@out[2]
 987         vmovdqu @out[3],0x30($offload)
 988         vpxor   $zero,@out[3],@out[3]
 989         vmovdqu @out[4],0x40($offload)
 990         vpxor   $zero,@out[4],@out[4]
 991         vmovdqu @out[5],0x50($offload)
 992         vpxor   $zero,@out[5],@out[5]
 993         vmovdqu @out[6],0x60($offload)
 994         vpxor   $zero,@out[6],@out[6]
 995         vmovdqu @out[7],0x70($offload)
 996         vpxor   $zero,@out[7],@out[7]
 997         xor     \$0x80,$offload
 998         mov     \$1,$one                        # constant of 1
 999         jmp     .Loop_dec8x
1000
1001 .align  32
1002 .Loop_dec8x:
1003 ___
1004 for($i=0;$i<8;$i++) {
1005 my $rndkey=($i&1)?$rndkey0:$rndkey1;
1006 $code.=<<___;
1007         vaesdec         $rndkey,@out[0],@out[0]
1008          cmp            32+4*$i(%rsp),$one
1009 ___
1010 $code.=<<___ if ($i);
1011          mov            64+8*$i(%rsp),$offset
1012 ___
1013 $code.=<<___;
1014         vaesdec         $rndkey,@out[1],@out[1]
1015         prefetcht0      31(@ptr[$i])                    # prefetch input
1016         vaesdec         $rndkey,@out[2],@out[2]
1017 ___
1018 $code.=<<___ if ($i>1);
1019         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1020 ___
1021 $code.=<<___;
1022         vaesdec         $rndkey,@out[3],@out[3]
1023          lea            (@ptr[$i],$offset),$offset
1024          cmovge         %rsp,@ptr[$i]                   # cancel input
1025         vaesdec         $rndkey,@out[4],@out[4]
1026          cmovg          %rsp,$offset                    # sink output
1027         vaesdec         $rndkey,@out[5],@out[5]
1028          sub            @ptr[$i],$offset
1029         vaesdec         $rndkey,@out[6],@out[6]
1030          vmovdqu        16(@ptr[$i]),@inp[$i%4]         # load input
1031          mov            $offset,64+8*$i(%rsp)
1032         vaesdec         $rndkey,@out[7],@out[7]
1033         vmovups         `16*(3+$i)-0x78`($key),$rndkey
1034          lea            16(@ptr[$i],$offset),@ptr[$i]   # switch to output
1035 ___
1036 $code.=<<___ if ($i<4);
1037          vmovdqu        @inp[$i%4],`128+16*$i`(%rsp)    # off-load
1038 ___
1039 }
1040 $code.=<<___;
1041          vmovdqu        32(%rsp),$counters
1042         prefetcht0      15(@ptr[$i-2])                  # prefetch output
1043         prefetcht0      15(@ptr[$i-1])
1044         cmp     \$11,$rounds
1045         jb      .Ldec8x_tail
1046
1047         vaesdec         $rndkey1,@out[0],@out[0]
1048         vaesdec         $rndkey1,@out[1],@out[1]
1049         vaesdec         $rndkey1,@out[2],@out[2]
1050         vaesdec         $rndkey1,@out[3],@out[3]
1051         vaesdec         $rndkey1,@out[4],@out[4]
1052         vaesdec         $rndkey1,@out[5],@out[5]
1053         vaesdec         $rndkey1,@out[6],@out[6]
1054         vaesdec         $rndkey1,@out[7],@out[7]
1055         vmovups         0xb0-0x78($key),$rndkey1
1056
1057         vaesdec         $rndkey0,@out[0],@out[0]
1058         vaesdec         $rndkey0,@out[1],@out[1]
1059         vaesdec         $rndkey0,@out[2],@out[2]
1060         vaesdec         $rndkey0,@out[3],@out[3]
1061         vaesdec         $rndkey0,@out[4],@out[4]
1062         vaesdec         $rndkey0,@out[5],@out[5]
1063         vaesdec         $rndkey0,@out[6],@out[6]
1064         vaesdec         $rndkey0,@out[7],@out[7]
1065         vmovups         0xc0-0x78($key),$rndkey0
1066         je      .Ldec8x_tail
1067
1068         vaesdec         $rndkey1,@out[0],@out[0]
1069         vaesdec         $rndkey1,@out[1],@out[1]
1070         vaesdec         $rndkey1,@out[2],@out[2]
1071         vaesdec         $rndkey1,@out[3],@out[3]
1072         vaesdec         $rndkey1,@out[4],@out[4]
1073         vaesdec         $rndkey1,@out[5],@out[5]
1074         vaesdec         $rndkey1,@out[6],@out[6]
1075         vaesdec         $rndkey1,@out[7],@out[7]
1076         vmovups         0xd0-0x78($key),$rndkey1
1077
1078         vaesdec         $rndkey0,@out[0],@out[0]
1079         vaesdec         $rndkey0,@out[1],@out[1]
1080         vaesdec         $rndkey0,@out[2],@out[2]
1081         vaesdec         $rndkey0,@out[3],@out[3]
1082         vaesdec         $rndkey0,@out[4],@out[4]
1083         vaesdec         $rndkey0,@out[5],@out[5]
1084         vaesdec         $rndkey0,@out[6],@out[6]
1085         vaesdec         $rndkey0,@out[7],@out[7]
1086         vmovups         0xe0-0x78($key),$rndkey0
1087
1088 .Ldec8x_tail:
1089         vaesdec         $rndkey1,@out[0],@out[0]
1090          vpxor          $zero,$zero,$zero
1091         vaesdec         $rndkey1,@out[1],@out[1]
1092         vaesdec         $rndkey1,@out[2],@out[2]
1093          vpcmpgtd       $zero,$counters,$zero
1094         vaesdec         $rndkey1,@out[3],@out[3]
1095         vaesdec         $rndkey1,@out[4],@out[4]
1096          vpaddd         $counters,$zero,$zero           # decrement counters
1097          vmovdqu        48(%rsp),$counters
1098         vaesdec         $rndkey1,@out[5],@out[5]
1099          mov            64(%rsp),$offset                # pre-load 1st offset
1100         vaesdec         $rndkey1,@out[6],@out[6]
1101         vaesdec         $rndkey1,@out[7],@out[7]
1102         vmovups         0x10-0x78($key),$rndkey1
1103
1104         vaesdeclast     $rndkey0,@out[0],@out[0]
1105          vmovdqa        $zero,32(%rsp)                  # update counters
1106          vpxor          $zero,$zero,$zero
1107         vaesdeclast     $rndkey0,@out[1],@out[1]
1108         vpxor           0x00($offload),@out[0],@out[0]  # xor with IV
1109         vaesdeclast     $rndkey0,@out[2],@out[2]
1110         vpxor           0x10($offload),@out[1],@out[1]
1111          vpcmpgtd       $zero,$counters,$zero
1112         vaesdeclast     $rndkey0,@out[3],@out[3]
1113         vpxor           0x20($offload),@out[2],@out[2]
1114         vaesdeclast     $rndkey0,@out[4],@out[4]
1115         vpxor           0x30($offload),@out[3],@out[3]
1116          vpaddd         $zero,$counters,$counters       # decrement counters
1117          vmovdqu        -0x78($key),$zero               # 0-round
1118         vaesdeclast     $rndkey0,@out[5],@out[5]
1119         vpxor           0x40($offload),@out[4],@out[4]
1120         vaesdeclast     $rndkey0,@out[6],@out[6]
1121         vpxor           0x50($offload),@out[5],@out[5]
1122          vmovdqa        $counters,48(%rsp)              # update counters
1123         vaesdeclast     $rndkey0,@out[7],@out[7]
1124         vpxor           0x60($offload),@out[6],@out[6]
1125         vmovups         0x20-0x78($key),$rndkey0
1126
1127         vmovups         @out[0],-16(@ptr[0])            # write output
1128          sub            $offset,@ptr[0]                 # switch to input
1129          vmovdqu        128+0(%rsp),@out[0]
1130         vpxor           0x70($offload),@out[7],@out[7]
1131         vmovups         @out[1],-16(@ptr[1])
1132          sub            `64+1*8`(%rsp),@ptr[1]
1133          vmovdqu        @out[0],0x00($offload)
1134          vpxor          $zero,@out[0],@out[0]
1135          vmovdqu        128+16(%rsp),@out[1]
1136         vmovups         @out[2],-16(@ptr[2])
1137          sub            `64+2*8`(%rsp),@ptr[2]
1138          vmovdqu        @out[1],0x10($offload)
1139          vpxor          $zero,@out[1],@out[1]
1140          vmovdqu        128+32(%rsp),@out[2]
1141         vmovups         @out[3],-16(@ptr[3])
1142          sub            `64+3*8`(%rsp),@ptr[3]
1143          vmovdqu        @out[2],0x20($offload)
1144          vpxor          $zero,@out[2],@out[2]
1145          vmovdqu        128+48(%rsp),@out[3]
1146         vmovups         @out[4],-16(@ptr[4])
1147          sub            `64+4*8`(%rsp),@ptr[4]
1148          vmovdqu        @out[3],0x30($offload)
1149          vpxor          $zero,@out[3],@out[3]
1150          vmovdqu        @inp[0],0x40($offload)
1151          vpxor          @inp[0],$zero,@out[4]
1152         vmovups         @out[5],-16(@ptr[5])
1153          sub            `64+5*8`(%rsp),@ptr[5]
1154          vmovdqu        @inp[1],0x50($offload)
1155          vpxor          @inp[1],$zero,@out[5]
1156         vmovups         @out[6],-16(@ptr[6])
1157          sub            `64+6*8`(%rsp),@ptr[6]
1158          vmovdqu        @inp[2],0x60($offload)
1159          vpxor          @inp[2],$zero,@out[6]
1160         vmovups         @out[7],-16(@ptr[7])
1161          sub            `64+7*8`(%rsp),@ptr[7]
1162          vmovdqu        @inp[3],0x70($offload)
1163          vpxor          @inp[3],$zero,@out[7]
1164
1165         xor     \$128,$offload
1166         dec     $num
1167         jnz     .Loop_dec8x
1168
1169         mov     16(%rsp),%rax                   # original %rsp
1170         #mov    24(%rsp),$num
1171         #lea    `40*8`($inp),$inp
1172         #dec    $num
1173         #jnz    .Ldec8x_loop_grande
1174
1175 .Ldec8x_done:
1176         vzeroupper
1177 ___
1178 $code.=<<___ if ($win64);
1179         movaps  -0xd8(%rax),%xmm6
1180         movaps  -0xc8(%rax),%xmm7
1181         movaps  -0xb8(%rax),%xmm8
1182         movaps  -0xa8(%rax),%xmm9
1183         movaps  -0x98(%rax),%xmm10
1184         movaps  -0x88(%rax),%xmm11
1185         movaps  -0x78(%rax),%xmm12
1186         movaps  -0x68(%rax),%xmm13
1187         movaps  -0x58(%rax),%xmm14
1188         movaps  -0x48(%rax),%xmm15
1189 ___
1190 $code.=<<___;
1191         mov     -48(%rax),%r15
1192         mov     -40(%rax),%r14
1193         mov     -32(%rax),%r13
1194         mov     -24(%rax),%r12
1195         mov     -16(%rax),%rbp
1196         mov     -8(%rax),%rbx
1197         lea     (%rax),%rsp
1198 .Ldec8x_epilogue:
1199         ret
1200 .size   aesni_multi_cbc_decrypt_avx,.-aesni_multi_cbc_decrypt_avx
1201 ___
1202                                                 }}}
1203
1204 if ($win64) {
1205 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
1206 #               CONTEXT *context,DISPATCHER_CONTEXT *disp)
1207 $rec="%rcx";
1208 $frame="%rdx";
1209 $context="%r8";
1210 $disp="%r9";
1211
1212 $code.=<<___;
1213 .extern __imp_RtlVirtualUnwind
1214 .type   se_handler,\@abi-omnipotent
1215 .align  16
1216 se_handler:
1217         push    %rsi
1218         push    %rdi
1219         push    %rbx
1220         push    %rbp
1221         push    %r12
1222         push    %r13
1223         push    %r14
1224         push    %r15
1225         pushfq
1226         sub     \$64,%rsp
1227
1228         mov     120($context),%rax      # pull context->Rax
1229         mov     248($context),%rbx      # pull context->Rip
1230
1231         mov     8($disp),%rsi           # disp->ImageBase
1232         mov     56($disp),%r11          # disp->HandlerData
1233
1234         mov     0(%r11),%r10d           # HandlerData[0]
1235         lea     (%rsi,%r10),%r10        # prologue label
1236         cmp     %r10,%rbx               # context->Rip<.Lprologue
1237         jb      .Lin_prologue
1238
1239         mov     152($context),%rax      # pull context->Rsp
1240
1241         mov     4(%r11),%r10d           # HandlerData[1]
1242         lea     (%rsi,%r10),%r10        # epilogue label
1243         cmp     %r10,%rbx               # context->Rip>=.Lepilogue
1244         jae     .Lin_prologue
1245
1246         mov     16(%rax),%rax           # pull saved stack pointer
1247
1248         mov     -8(%rax),%rbx
1249         mov     -16(%rax),%rbp
1250         mov     -24(%rax),%r12
1251         mov     -32(%rax),%r13
1252         mov     -40(%rax),%r14
1253         mov     -48(%rax),%r15
1254         mov     %rbx,144($context)      # restore context->Rbx
1255         mov     %rbp,160($context)      # restore context->Rbp
1256         mov     %r12,216($context)      # restore cotnext->R12
1257         mov     %r13,224($context)      # restore cotnext->R13
1258         mov     %r14,232($context)      # restore cotnext->R14
1259         mov     %r15,240($context)      # restore cotnext->R15
1260
1261         lea     -56-10*16(%rax),%rsi
1262         lea     512($context),%rdi      # &context.Xmm6
1263         mov     \$20,%ecx
1264         .long   0xa548f3fc              # cld; rep movsq
1265
1266 .Lin_prologue:
1267         mov     8(%rax),%rdi
1268         mov     16(%rax),%rsi
1269         mov     %rax,152($context)      # restore context->Rsp
1270         mov     %rsi,168($context)      # restore context->Rsi
1271         mov     %rdi,176($context)      # restore context->Rdi
1272
1273         mov     40($disp),%rdi          # disp->ContextRecord
1274         mov     $context,%rsi           # context
1275         mov     \$154,%ecx              # sizeof(CONTEXT)
1276         .long   0xa548f3fc              # cld; rep movsq
1277
1278         mov     $disp,%rsi
1279         xor     %rcx,%rcx               # arg1, UNW_FLAG_NHANDLER
1280         mov     8(%rsi),%rdx            # arg2, disp->ImageBase
1281         mov     0(%rsi),%r8             # arg3, disp->ControlPc
1282         mov     16(%rsi),%r9            # arg4, disp->FunctionEntry
1283         mov     40(%rsi),%r10           # disp->ContextRecord
1284         lea     56(%rsi),%r11           # &disp->HandlerData
1285         lea     24(%rsi),%r12           # &disp->EstablisherFrame
1286         mov     %r10,32(%rsp)           # arg5
1287         mov     %r11,40(%rsp)           # arg6
1288         mov     %r12,48(%rsp)           # arg7
1289         mov     %rcx,56(%rsp)           # arg8, (NULL)
1290         call    *__imp_RtlVirtualUnwind(%rip)
1291
1292         mov     \$1,%eax                # ExceptionContinueSearch
1293         add     \$64,%rsp
1294         popfq
1295         pop     %r15
1296         pop     %r14
1297         pop     %r13
1298         pop     %r12
1299         pop     %rbp
1300         pop     %rbx
1301         pop     %rdi
1302         pop     %rsi
1303         ret
1304 .size   se_handler,.-se_handler
1305
1306 .section        .pdata
1307 .align  4
1308         .rva    .LSEH_begin_aesni_multi_cbc_encrypt
1309         .rva    .LSEH_end_aesni_multi_cbc_encrypt
1310         .rva    .LSEH_info_aesni_multi_cbc_encrypt
1311         .rva    .LSEH_begin_aesni_multi_cbc_decrypt
1312         .rva    .LSEH_end_aesni_multi_cbc_decrypt
1313         .rva    .LSEH_info_aesni_multi_cbc_decrypt
1314 ___
1315 $code.=<<___ if ($avx);
1316         .rva    .LSEH_begin_aesni_multi_cbc_encrypt_avx
1317         .rva    .LSEH_end_aesni_multi_cbc_encrypt_avx
1318         .rva    .LSEH_info_aesni_multi_cbc_encrypt_avx
1319         .rva    .LSEH_begin_aesni_multi_cbc_decrypt_avx
1320         .rva    .LSEH_end_aesni_multi_cbc_decrypt_avx
1321         .rva    .LSEH_info_aesni_multi_cbc_decrypt_avx
1322 ___
1323 $code.=<<___;
1324 .section        .xdata
1325 .align  8
1326 .LSEH_info_aesni_multi_cbc_encrypt:
1327         .byte   9,0,0,0
1328         .rva    se_handler
1329         .rva    .Lenc4x_body,.Lenc4x_epilogue           # HandlerData[]
1330 .LSEH_info_aesni_multi_cbc_decrypt:
1331         .byte   9,0,0,0
1332         .rva    se_handler
1333         .rva    .Ldec4x_body,.Ldec4x_epilogue           # HandlerData[]
1334 ___
1335 $code.=<<___ if ($avx);
1336 .LSEH_info_aesni_multi_cbc_encrypt_avx:
1337         .byte   9,0,0,0
1338         .rva    se_handler
1339         .rva    .Lenc8x_body,.Lenc8x_epilogue           # HandlerData[]
1340 .LSEH_info_aesni_multi_cbc_decrypt_avx:
1341         .byte   9,0,0,0
1342         .rva    se_handler
1343         .rva    .Ldec8x_body,.Ldec8x_epilogue           # HandlerData[]
1344 ___
1345 }
1346 ####################################################################
1347
1348 sub rex {
1349   local *opcode=shift;
1350   my ($dst,$src)=@_;
1351   my $rex=0;
1352
1353     $rex|=0x04                  if($dst>=8);
1354     $rex|=0x01                  if($src>=8);
1355     push @opcode,$rex|0x40      if($rex);
1356 }
1357
1358 sub aesni {
1359   my $line=shift;
1360   my @opcode=(0x66);
1361
1362     if ($line=~/(aeskeygenassist)\s+\$([x0-9a-f]+),\s*%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1363         rex(\@opcode,$4,$3);
1364         push @opcode,0x0f,0x3a,0xdf;
1365         push @opcode,0xc0|($3&7)|(($4&7)<<3);   # ModR/M
1366         my $c=$2;
1367         push @opcode,$c=~/^0/?oct($c):$c;
1368         return ".byte\t".join(',',@opcode);
1369     }
1370     elsif ($line=~/(aes[a-z]+)\s+%xmm([0-9]+),\s*%xmm([0-9]+)/) {
1371         my %opcodelet = (
1372                 "aesimc" => 0xdb,
1373                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1374                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1375         );
1376         return undef if (!defined($opcodelet{$1}));
1377         rex(\@opcode,$3,$2);
1378         push @opcode,0x0f,0x38,$opcodelet{$1};
1379         push @opcode,0xc0|($2&7)|(($3&7)<<3);   # ModR/M
1380         return ".byte\t".join(',',@opcode);
1381     }
1382     elsif ($line=~/(aes[a-z]+)\s+([0x1-9a-fA-F]*)\(%rsp\),\s*%xmm([0-9]+)/) {
1383         my %opcodelet = (
1384                 "aesenc" => 0xdc,       "aesenclast" => 0xdd,
1385                 "aesdec" => 0xde,       "aesdeclast" => 0xdf
1386         );
1387         return undef if (!defined($opcodelet{$1}));
1388         my $off = $2;
1389         push @opcode,0x44 if ($3>=8);
1390         push @opcode,0x0f,0x38,$opcodelet{$1};
1391         push @opcode,0x44|(($3&7)<<3),0x24;     # ModR/M
1392         push @opcode,($off=~/^0/?oct($off):$off)&0xff;
1393         return ".byte\t".join(',',@opcode);
1394     }
1395     return $line;
1396 }
1397
1398 $code =~ s/\`([^\`]*)\`/eval($1)/gem;
1399 $code =~ s/\b(aes.*%xmm[0-9]+).*$/aesni($1)/gem;
1400
1401 print $code;
1402 close STDOUT;