crypto/aes/asm/aesni-x86.pl

   1 #!/usr/bin/env perl
   2
   3 # ====================================================================
   4 # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
   5 # project. The module is, however, dual licensed under OpenSSL and
   6 # CRYPTOGAMS licenses depending on where you obtain it. For further
   7 # details see http://www.openssl.org/~appro/cryptogams/.
   8 # ====================================================================
   9 #
  10 # This module implements support for Intel AES-NI extension. In
  11 # OpenSSL context it's used with Intel engine, but can also be used as
  12 # drop-in replacement for crypto/aes/asm/aes-586.pl [see below for
  13 # details].
  14
  15 $PREFIX="aesni";        # if $PREFIX is set to "AES", the script
  16                         # generates drop-in replacement for
  17                         # crypto/aes/asm/aes-586.pl:-)
  18 $inline=1;              # inline _aesni_[en|de]crypt
  19
  20 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
  21 push(@INC,"${dir}","${dir}../../perlasm");
  22 require "x86asm.pl";
  23
  24 &asm_init($ARGV[0],$0);
  25
  26 $movekey = eval($RREFIX eq "aseni" ? "*movaps" : "*movups");
  27
  28 $len="eax";
  29 $rounds="ecx";
  30 $key="edx";
  31 $inp="esi";
  32 $out="edi";
  33 $rounds_="ebx"; # backup copy for $rounds
  34 $key_="ebp";    # backup copy for $key
  35
  36 $inout0="xmm0";
  37 $inout1="xmm1";
  38 $inout2="xmm2";
  39 $rndkey0="xmm3";
  40 $rndkey1="xmm4";
  41 $ivec="xmm5";
  42 $in0="xmm6";
  43 $in1="xmm7";    $inout3="xmm7";
  44
  45 # Inline version of internal aesni_[en|de]crypt1
  46 sub aesni_inline_generate1
  47 { my $p=shift;
  48
  49     &$movekey           ($rndkey0,&QWP(0,$key));
  50     &$movekey           ($rndkey1,&QWP(16,$key));
  51     &lea                ($key,&DWP(32,$key));
  52     &pxor               ($inout0,$rndkey0);
  53     &set_label("${p}1_loop");
  54         eval"&aes${p}   ($inout0,$rndkey1)";
  55         &dec            ($rounds);
  56         &$movekey       ($rndkey1,&QWP(0,$key));
  57         &lea            ($key,&DWP(16,$key));
  58     &jnz                (&label("${p}1_loop"));
  59     eval"&aes${p}last   ($inout0,$rndkey1)";
  60 }
  61
  62 sub aesni_generate1     # fully unrolled loop
  63 { my $p=shift;
  64
  65     &function_begin_B("_aesni_${p}rypt1");
  66         &$movekey       ($rndkey0,&QWP(0,$key));
  67         &$movekey       ($rndkey1,&QWP(0x10,$key));
  68         &cmp            ($rounds,11);
  69         &pxor           ($inout0,$rndkey0);
  70         &$movekey       ($rndkey0,&QWP(0x20,$key));
  71         &lea            ($key,&DWP(0x30,$key));
  72         &jb             (&label("${p}128"));
  73         &lea            ($key,&DWP(0x20,$key));
  74         &je             (&label("${p}192"));
  75         &lea            ($key,&DWP(0x20,$key));
  76         eval"&aes${p}   ($inout0,$rndkey1)";
  77         &$movekey       ($rndkey1,&QWP(-0x40,$key));
  78         eval"&aes${p}   ($inout0,$rndkey0)";
  79         &$movekey       ($rndkey0,&QWP(-0x30,$key));
  80     &set_label("${p}192");
  81         eval"&aes${p}   ($inout0,$rndkey1)";
  82         &$movekey       ($rndkey1,&QWP(-0x20,$key));
  83         eval"&aes${p}   ($inout0,$rndkey0)";
  84         &$movekey       ($rndkey0,&QWP(-0x10,$key));
  85     &set_label("${p}128");
  86         eval"&aes${p}   ($inout0,$rndkey1)";
  87         &$movekey       ($rndkey1,&QWP(0,$key));
  88         eval"&aes${p}   ($inout0,$rndkey0)";
  89         &$movekey       ($rndkey0,&QWP(0x10,$key));
  90         eval"&aes${p}   ($inout0,$rndkey1)";
  91         &$movekey       ($rndkey1,&QWP(0x20,$key));
  92         eval"&aes${p}   ($inout0,$rndkey0)";
  93         &$movekey       ($rndkey0,&QWP(0x30,$key));
  94         eval"&aes${p}   ($inout0,$rndkey1)";
  95         &$movekey       ($rndkey1,&QWP(0x40,$key));
  96         eval"&aes${p}   ($inout0,$rndkey0)";
  97         &$movekey       ($rndkey0,&QWP(0x50,$key));
  98         eval"&aes${p}   ($inout0,$rndkey1)";
  99         &$movekey       ($rndkey1,&QWP(0x60,$key));
 100         eval"&aes${p}   ($inout0,$rndkey0)";
 101         &$movekey       ($rndkey0,&QWP(0x70,$key));
 102         eval"&aes${p}   ($inout0,$rndkey1)";
 103     eval"&aes${p}last   ($inout0,$rndkey0)";
 104     &ret();
 105     &function_end_B("_aesni_${p}rypt1");
 106 }
 107
 108 # void $PREFIX_encrypt (const void *inp,void *out,const AES_KEY *key);
 109 &aesni_generate1("enc") if (!$inline);
 110 &function_begin_B("${PREFIX}_encrypt");
 111         &mov    ("eax",&wparam(0));
 112         &mov    ($key,&wparam(2));
 113         &movups ($inout0,&QWP(0,"eax"));
 114         &mov    ($rounds,&DWP(240,$key));
 115         &mov    ("eax",&wparam(1));
 116         if ($inline)
 117         {   &aesni_inline_generate1("enc");     }
 118         else
 119         {   &call       ("_aesni_encrypt1");    }
 120         &movups (&QWP(0,"eax"),$inout0);
 121         &ret    ();
 122 &function_end_B("${PREFIX}_encrypt");
 123
 124 # void $PREFIX_decrypt (const void *inp,void *out,const AES_KEY *key);
 125 &aesni_generate1("dec") if(!$inline);
 126 &function_begin_B("${PREFIX}_decrypt");
 127         &mov    ("eax",&wparam(0));
 128         &mov    ($key,&wparam(2));
 129         &movups ($inout0,&QWP(0,"eax"));
 130         &mov    ($rounds,&DWP(240,$key));
 131         &mov    ("eax",&wparam(1));
 132         if ($inline)
 133         {   &aesni_inline_generate1("dec");     }
 134         else
 135         {   &call       ("_aesni_decrypt1");    }
 136         &movups (&QWP(0,"eax"),$inout0);
 137         &ret    ();
 138 &function_end_B("${PREFIX}_decrypt");
 139 \f
 140 # _aesni_[en|de]crypt[34] are private interfaces, N denotes interleave
 141 # factor. Why 3x subroutine is used in loops? Even though aes[enc|dec]
 142 # latency is 6, it turned out that it can be scheduled only every
 143 # *second* cycle. Thus 3x interleave is the one providing optimal
 144 # utilization, i.e. when subroutine's throughput is virtually same as
 145 # of non-interleaved subroutine [for number of input blocks up to 3].
 146 # This is why it makes no sense to implement 2x subroutine. As soon
 147 # as/if Intel improves throughput by making it possible to schedule
 148 # the instructions in question *every* cycles I would have to
 149 # implement 6x interleave and use it in loop...
 150 sub aesni_generate3
 151 { my $p=shift;
 152
 153     &function_begin_B("_aesni_${p}rypt3");
 154         &$movekey       ($rndkey0,&QWP(0,$key));
 155         &shr            ($rounds,1);
 156         &$movekey       ($rndkey1,&QWP(16,$key));
 157         &lea            ($key,&DWP(32,$key));
 158         &pxor           ($inout0,$rndkey0);
 159         &pxor           ($inout1,$rndkey0);
 160         &pxor           ($inout2,$rndkey0);
 161         &jmp            (&label("${p}3_loop"));
 162     &set_label("${p}3_loop",16);
 163         eval"&aes${p}   ($inout0,$rndkey1)";
 164         &$movekey       ($rndkey0,&QWP(0,$key));
 165         eval"&aes${p}   ($inout1,$rndkey1)";
 166         &dec            ($rounds);
 167         eval"&aes${p}   ($inout2,$rndkey1)";
 168         &$movekey       ($rndkey1,&QWP(16,$key));
 169         eval"&aes${p}   ($inout0,$rndkey0)";
 170         &lea            ($key,&DWP(32,$key));
 171         eval"&aes${p}   ($inout1,$rndkey0)";
 172         eval"&aes${p}   ($inout2,$rndkey0)";
 173         &jnz            (&label("${p}3_loop"));
 174     eval"&aes${p}       ($inout0,$rndkey1)";
 175     &$movekey           ($rndkey0,&QWP(0,$key));
 176     eval"&aes${p}       ($inout1,$rndkey1)";
 177     eval"&aes${p}       ($inout2,$rndkey1)";
 178     eval"&aes${p}last   ($inout0,$rndkey0)";
 179     eval"&aes${p}last   ($inout1,$rndkey0)";
 180     eval"&aes${p}last   ($inout2,$rndkey0)";
 181     &ret();
 182     &function_end_B("_aesni_${p}rypt3");
 183 }
 184
 185 # 4x interleave is implemented to improve small block performance,
 186 # most notably [and naturally] 4 block by ~30%. One can argue that one
 187 # should have implemented 5x as well, but improvement  would be <20%,
 188 # so it's not worth it...
 189 sub aesni_generate4
 190 { my $p=shift;
 191
 192     &function_begin_B("_aesni_${p}rypt4");
 193         &$movekey       ($rndkey0,&QWP(0,$key));
 194         &$movekey       ($rndkey1,&QWP(16,$key));
 195         &shr            ($rounds,1);
 196         &lea            ($key,&DWP(32,$key));
 197         &pxor           ($inout0,$rndkey0);
 198         &pxor           ($inout1,$rndkey0);
 199         &pxor           ($inout2,$rndkey0);
 200         &pxor           ($inout3,$rndkey0);
 201         &jmp            (&label("${p}3_loop"));
 202     &set_label("${p}3_loop",16);
 203         eval"&aes${p}   ($inout0,$rndkey1)";
 204         &$movekey       ($rndkey0,&QWP(0,$key));
 205         eval"&aes${p}   ($inout1,$rndkey1)";
 206         &dec            ($rounds);
 207         eval"&aes${p}   ($inout2,$rndkey1)";
 208         eval"&aes${p}   ($inout3,$rndkey1)";
 209         &$movekey       ($rndkey1,&QWP(16,$key));
 210         eval"&aes${p}   ($inout0,$rndkey0)";
 211         &lea            ($key,&DWP(32,$key));
 212         eval"&aes${p}   ($inout1,$rndkey0)";
 213         eval"&aes${p}   ($inout2,$rndkey0)";
 214         eval"&aes${p}   ($inout3,$rndkey0)";
 215         &jnz            (&label("${p}3_loop"));
 216     eval"&aes${p}       ($inout0,$rndkey1)";
 217     &$movekey           ($rndkey0,&QWP(0,$key));
 218     eval"&aes${p}       ($inout1,$rndkey1)";
 219     eval"&aes${p}       ($inout2,$rndkey1)";
 220     eval"&aes${p}       ($inout3,$rndkey1)";
 221     eval"&aes${p}last   ($inout0,$rndkey0)";
 222     eval"&aes${p}last   ($inout1,$rndkey0)";
 223     eval"&aes${p}last   ($inout2,$rndkey0)";
 224     eval"&aes${p}last   ($inout3,$rndkey0)";
 225     &ret();
 226     &function_end_B("_aesni_${p}rypt4");
 227 }
 228 &aesni_generate3("enc") if ($PREFIX eq "aesni");
 229 &aesni_generate3("dec");
 230 &aesni_generate4("enc") if ($PREFIX eq "aesni");
 231 &aesni_generate4("dec");
 232
 233 if ($PREFIX eq "aesni") {
 234 # void aesni_ecb_encrypt (const void *in, void *out,
 235 #                         size_t length, const AES_KEY *key,
 236 #                         int enc);
 237 &function_begin("aesni_ecb_encrypt");
 238         &mov    ($inp,&wparam(0));
 239         &mov    ($out,&wparam(1));
 240         &mov    ($len,&wparam(2));
 241         &mov    ($key,&wparam(3));
 242         &mov    ($rounds,&wparam(4));
 243         &cmp    ($len,16);
 244         &jb     (&label("ecb_ret"));
 245         &and    ($len,-16);
 246         &test   ($rounds,$rounds)
 247         &mov    ($rounds,&DWP(240,$key));
 248         &mov    ($key_,$key);           # backup $key
 249         &mov    ($rounds_,$rounds);     # backup $rounds
 250         &jz     (&label("ecb_decrypt"));
 251
 252         &sub    ($len,0x40);
 253         &jbe    (&label("ecb_enc_tail"));
 254         &jmp    (&label("ecb_enc_loop3"));
 255
 256 &set_label("ecb_enc_loop3",16);
 257         &movups ($inout0,&QWP(0,$inp));
 258         &movups ($inout1,&QWP(0x10,$inp));
 259         &movups ($inout2,&QWP(0x20,$inp));
 260         &call   ("_aesni_encrypt3");
 261         &sub    ($len,0x30);
 262         &lea    ($inp,&DWP(0x30,$inp));
 263         &lea    ($out,&DWP(0x30,$out));
 264         &movups (&QWP(-0x30,$out),$inout0);
 265         &mov    ($key,$key_);           # restore $key
 266         &movups (&QWP(-0x20,$out),$inout1);
 267         &mov    ($rounds,$rounds_);     # restore $rounds
 268         &movups (&QWP(-0x10,$out),$inout2);
 269         &ja     (&label("ecb_enc_loop3"));
 270
 271 &set_label("ecb_enc_tail");
 272         &add    ($len,0x40);
 273         &jz     (&label("ecb_ret"));
 274
 275         &cmp    ($len,0x10);
 276         &movups ($inout0,&QWP(0,$inp));
 277         &je     (&label("ecb_enc_one"));
 278         &cmp    ($len,0x20);
 279         &movups ($inout1,&QWP(0x10,$inp));
 280         &je     (&label("ecb_enc_two"));
 281         &cmp    ($len,0x30);
 282         &movups ($inout2,&QWP(0x20,$inp));
 283         &je     (&label("ecb_enc_three"));
 284         &movups ($inout3,&QWP(0x30,$inp));
 285         &call   ("_aesni_encrypt4");
 286         &movups (&QWP(0,$out),$inout0);
 287         &movups (&QWP(0x10,$out),$inout1);
 288         &movups (&QWP(0x20,$out),$inout2);
 289         &movups (&QWP(0x30,$out),$inout3);
 290         jmp     (&label("ecb_ret"));
 291
 292 &set_label("ecb_enc_one",16);
 293         if ($inline)
 294         {   &aesni_inline_generate1("enc");     }
 295         else
 296         {   &call       ("_aesni_encrypt1");    }
 297         &movups (&QWP(0,$out),$inout0);
 298         &jmp    (&label("ecb_ret"));
 299
 300 &set_label("ecb_enc_two",16);
 301         &call   ("_aesni_encrypt3");
 302         &movups (&QWP(0,$out),$inout0);
 303         &movups (&QWP(0x10,$out),$inout1);
 304         &jmp    (&label("ecb_ret"));
 305
 306 &set_label("ecb_enc_three",16);
 307         &call   ("_aesni_encrypt3");
 308         &movups (&QWP(0,$out),$inout0);
 309         &movups (&QWP(0x10,$out),$inout1);
 310         &movups (&QWP(0x20,$out),$inout2);
 311         &jmp    (&label("ecb_ret"));
 312
 313 &set_label("ecb_decrypt",16);
 314         &sub    ($len,0x40);
 315         &jbe    (&label("ecb_dec_tail"));
 316         &jmp    (&label("ecb_dec_loop3"));
 317
 318 &set_label("ecb_dec_loop3",16);
 319         &movups ($inout0,&QWP(0,$inp));
 320         &movups ($inout1,&QWP(0x10,$inp));
 321         &movups ($inout2,&QWP(0x20,$inp));
 322         &call   ("_aesni_decrypt3");
 323         &sub    ($len,0x30);
 324         &lea    ($inp,&DWP(0x30,$inp));
 325         &lea    ($out,&DWP(0x30,$out));
 326         &movups (&QWP(-0x30,$out),$inout0);
 327         &mov    ($key,$key_);           # restore $key
 328         &movups (&QWP(-0x20,$out),$inout1);
 329         &mov    ($rounds,$rounds_);     # restore $rounds
 330         &movups (&QWP(-0x10,$out),$inout2);
 331         &ja     (&label("ecb_dec_loop3"));
 332
 333 &set_label("ecb_dec_tail");
 334         &add    ($len,0x40);
 335         &jz     (&label("ecb_ret"));
 336
 337         &cmp    ($len,0x10);
 338         &movups ($inout0,&QWP(0,$inp));
 339         &je     (&label("ecb_dec_one"));
 340         &cmp    ($len,0x20);
 341         &movups ($inout1,&QWP(0x10,$inp));
 342         &je     (&label("ecb_dec_two"));
 343         &cmp    ($len,0x30);
 344         &movups ($inout2,&QWP(0x20,$inp));
 345         &je     (&label("ecb_dec_three"));
 346         &movups ($inout3,&QWP(0x30,$inp));
 347         &call   ("_aesni_decrypt4");
 348         &movups (&QWP(0,$out),$inout0);
 349         &movups (&QWP(0x10,$out),$inout1);
 350         &movups (&QWP(0x20,$out),$inout2);
 351         &movups (&QWP(0x30,$out),$inout3);
 352         &jmp    (&label("ecb_ret"));
 353
 354 &set_label("ecb_dec_one",16);
 355         if ($inline)
 356         {   &aesni_inline_generate1("dec");     }
 357         else
 358         {   &call       ("_aesni_decrypt1");    }
 359         &movups (&QWP(0,$out),$inout0);
 360         &jmp    (&label("ecb_ret"));
 361
 362 &set_label("ecb_dec_two",16);
 363         &call   ("_aesni_decrypt3");
 364         &movups (&QWP(0,$out),$inout0);
 365         &movups (&QWP(0x10,$out),$inout1);
 366         &jmp    (&label("ecb_ret"));
 367
 368 &set_label("ecb_dec_three",16);
 369         &call   ("_aesni_decrypt3");
 370         &movups (&QWP(0,$out),$inout0);
 371         &movups (&QWP(0x10,$out),$inout1);
 372         &movups (&QWP(0x20,$out),$inout2);
 373
 374 &set_label("ecb_ret");
 375 &function_end("aesni_ecb_encrypt");
 376 }
 377
 378 # void $PREFIX_cbc_encrypt (const void *inp, void *out,
 379 #                           size_t length, const AES_KEY *key,
 380 #                           unsigned char *ivp,const int enc);
 381 &function_begin("${PREFIX}_cbc_encrypt");
 382         &mov    ($inp,&wparam(0));
 383         &mov    ($out,&wparam(1));
 384         &mov    ($len,&wparam(2));
 385         &mov    ($key,&wparam(3));
 386         &test   ($len,$len);
 387         &mov    ($key_,&wparam(4));
 388         &jz     (&label("cbc_ret"));
 389
 390         &cmp    (&wparam(5),0);
 391         &movups ($ivec,&QWP(0,$key_));  # load IV
 392         &mov    ($rounds,&DWP(240,$key));
 393         &mov    ($key_,$key);           # backup $key
 394         &mov    ($rounds_,$rounds);     # backup $rounds
 395         &je     (&label("cbc_decrypt"));
 396
 397         &movaps ($inout0,$ivec);
 398         &cmp    ($len,16);
 399         &jb     (&label("cbc_enc_tail"));
 400         &sub    ($len,16);
 401         &jmp    (&label("cbc_enc_loop"));
 402
 403 &set_label("cbc_enc_loop",16);
 404         &movups ($ivec,&QWP(0,$inp));
 405         &lea    ($inp,&DWP(16,$inp));
 406         &pxor   ($inout0,$ivec);
 407         if ($inline)
 408         {   &aesni_inline_generate1("enc");     }
 409         else
 410         {   &call       ("_aesni_encrypt1");    }
 411         &sub    ($len,16);
 412         &lea    ($out,&DWP(16,$out));
 413         &mov    ($rounds,$rounds_);     # restore $rounds
 414         &mov    ($key,$key_);           # restore $key
 415         &movups (&QWP(-16,$out),$inout0);
 416         &jnc    (&label("cbc_enc_loop"));
 417         &add    ($len,16);
 418         &jnz    (&label("cbc_enc_tail"));
 419         &movaps ($ivec,$inout0);
 420         &jmp    (&label("cbc_ret"));
 421
 422 &set_label("cbc_enc_tail");
 423         &mov    ("ecx",$len);           # zaps $rounds
 424         &data_word(0xA4F3F689);         # rep movsb
 425         &mov    ("ecx",16);             # zero tail
 426         &sub    ("ecx",$len);
 427         &xor    ("eax","eax");          # zaps $len
 428         &data_word(0xAAF3F689);         # rep stosb
 429         &lea    ($out,&DWP(-16,$out));  # rewind $out by 1 block
 430         &mov    ($rounds,$rounds_);     # restore $rounds
 431         &mov    ($inp,$out);            # $inp and $out are the same
 432         &mov    ($key,$key_);           # restore $key
 433         &jmp    (&label("cbc_enc_loop"));
 434
 435 &set_label("cbc_decrypt",16);
 436         &sub    ($len,0x40);
 437         &jbe    (&label("cbc_dec_tail"));
 438         &jmp    (&label("cbc_dec_loop3"));
 439
 440 &set_label("cbc_dec_loop3",16);
 441         &movups ($inout0,&QWP(0,$inp));
 442         &movups ($inout1,&QWP(0x10,$inp));
 443         &movups ($inout2,&QWP(0x20,$inp));
 444         &movaps ($in0,$inout0);
 445         &movaps ($in1,$inout1);
 446         &call   ("_aesni_decrypt3");
 447         &sub    ($len,0x30);
 448         &lea    ($inp,&DWP(0x30,$inp));
 449         &lea    ($out,&DWP(0x30,$out));
 450         &pxor   ($inout0,$ivec);
 451         &pxor   ($inout1,$in0);
 452         &movups ($ivec,&QWP(-0x10,$inp));
 453         &pxor   ($inout2,$in1);
 454         &movups (&QWP(-0x30,$out),$inout0);
 455         &mov    ($rounds,$rounds_)      # restore $rounds
 456         &movups (&QWP(-0x20,$out),$inout1);
 457         &mov    ($key,$key_);           # restore $key
 458         &movups (&QWP(-0x10,$out),$inout2);
 459         &ja     (&label("cbc_dec_loop3"));
 460
 461 &set_label("cbc_dec_tail");
 462         &add    ($len,0x40);
 463         &jz     (&label("cbc_ret"));
 464
 465         &movups ($inout0,&QWP(0,$inp));
 466         &cmp    ($len,0x10);
 467         &movaps ($in0,$inout0);
 468         &jbe    (&label("cbc_dec_one"));
 469         &movups ($inout1,&QWP(0x10,$inp));
 470         &cmp    ($len,0x20);
 471         &movaps ($in1,$inout1);
 472         &jbe    (&label("cbc_dec_two"));
 473         &movups ($inout2,&QWP(0x20,$inp));
 474         &cmp    ($len,0x30);
 475         &jbe    (&label("cbc_dec_three"));
 476         &movups ($inout3,&QWP(0x30,$inp));
 477         &call   ("_aesni_decrypt4");
 478         &movups ($rndkey0,&QWP(0x10,$inp));
 479         &movups ($rndkey1,&QWP(0x20,$inp));
 480         &pxor   ($inout0,$ivec);
 481         &pxor   ($inout1,$in0);
 482         &movups ($ivec,&QWP(0x30,$inp));
 483         &movups (&QWP(0,$out),$inout0);
 484         &pxor   ($inout2,$rndkey0);
 485         &pxor   ($inout3,$rndkey1);
 486         &movups (&QWP(0x10,$out),$inout1);
 487         &movups (&QWP(0x20,$out),$inout2);
 488         &movaps ($inout0,$inout3);
 489         &lea    ($out,&DWP(0x30,$out));
 490         &jmp    (&label("cbc_dec_tail_collected"));
 491
 492 &set_label("cbc_dec_one");
 493         if ($inline)
 494         {   &aesni_inline_generate1("dec");     }
 495         else
 496         {   &call       ("_aesni_decrypt1");    }
 497         &pxor   ($inout0,$ivec);
 498         &movaps ($ivec,$in0);
 499         &jmp    (&label("cbc_dec_tail_collected"));
 500
 501 &set_label("cbc_dec_two");
 502         &call   ("_aesni_decrypt3");
 503         &pxor   ($inout0,$ivec);
 504         &pxor   ($inout1,$in0);
 505         &movups (&QWP(0,$out),$inout0);
 506         &movaps ($inout0,$inout1);
 507         &movaps ($ivec,$in1);
 508         &lea    ($out,&DWP(0x10,$out));
 509         &jmp    (&label("cbc_dec_tail_collected"));
 510
 511 &set_label("cbc_dec_three");
 512         &call   ("_aesni_decrypt3");
 513         &pxor   ($inout0,$ivec);
 514         &pxor   ($inout1,$in0);
 515         &pxor   ($inout2,$in1);
 516         &movups (&QWP(0,$out),$inout0);
 517         &movups (&QWP(0x10,$out),$inout1);
 518         &movaps ($inout0,$inout2);
 519         &movups ($ivec,&QWP(0x20,$inp));
 520         &lea    ($out,&DWP(0x20,$out));
 521
 522 &set_label("cbc_dec_tail_collected");
 523         &and    ($len,15);
 524         &jnz    (&label("cbc_dec_tail_partial"));
 525         &movups (&QWP(0,$out),$inout0);
 526         &jmp    (&label("cbc_ret"));
 527
 528 &set_label("cbc_dec_tail_partial");
 529         &mov    ($key_,"esp");
 530         &sub    ("esp",16);
 531         &and    ("esp",-16);
 532         &movaps (&QWP(0,"esp"),$inout0);
 533         &mov    ($inp,"esp");
 534         &mov    ("ecx",$len);
 535         &data_word(0xA4F3F689);         # rep movsb
 536         &mov    ("esp",$key_);
 537
 538 &set_label("cbc_ret");
 539         &mov    ($key_,&wparam(4));
 540         &movups (&QWP(0,$key_),$ivec);  # output IV
 541 &function_end("${PREFIX}_cbc_encrypt");
 542
 543 # Mechanical port from aesni-x86_64.pl.
 544 #
 545 # _aesni_set_encrypt_key is private interface,
 546 # input:
 547 #       "eax"   const unsigned char *userKey
 548 #       $rounds int bits
 549 #       $key    AES_KEY *key
 550 # output:
 551 #       "eax"   return code
 552 #       $round  rounds
 553
 554 &function_begin_B("_aesni_set_encrypt_key");
 555         &test   ("eax","eax");
 556         &jz     (&label("bad_pointer"));
 557         &test   ($key,$key);
 558         &jz     (&label("bad_pointer"));
 559
 560         &movups ("xmm0",&QWP(0,"eax")); # pull first 128 bits of *userKey
 561         &pxor   ("xmm4","xmm4");        # low dword of xmm4 is assumed 0
 562         &lea    ($key,&DWP(16,$key));
 563         &cmp    ($rounds,256);
 564         &je     (&label("14rounds"));
 565         &cmp    ($rounds,192);
 566         &je     (&label("12rounds"));
 567         &cmp    ($rounds,128);
 568         &jne    (&label("bad_keybits"));
 569
 570 &set_label("10rounds",16);
 571         &mov            ($rounds,9);
 572         &$movekey       (&QWP(-16,$key),"xmm0");        # round 0
 573         &aeskeygenassist("xmm1","xmm0",0x01);           # round 1
 574         &call           (&label("key_128_cold"));
 575         &aeskeygenassist("xmm1","xmm0",0x2);            # round 2
 576         &call           (&label("key_128"));
 577         &aeskeygenassist("xmm1","xmm0",0x04);           # round 3
 578         &call           (&label("key_128"));
 579         &aeskeygenassist("xmm1","xmm0",0x08);           # round 4
 580         &call           (&label("key_128"));
 581         &aeskeygenassist("xmm1","xmm0",0x10);           # round 5
 582         &call           (&label("key_128"));
 583         &aeskeygenassist("xmm1","xmm0",0x20);           # round 6
 584         &call           (&label("key_128"));
 585         &aeskeygenassist("xmm1","xmm0",0x40);           # round 7
 586         &call           (&label("key_128"));
 587         &aeskeygenassist("xmm1","xmm0",0x80);           # round 8
 588         &call           (&label("key_128"));
 589         &aeskeygenassist("xmm1","xmm0",0x1b);           # round 9
 590         &call           (&label("key_128"));
 591         &aeskeygenassist("xmm1","xmm0",0x36);           # round 10
 592         &call           (&label("key_128"));
 593         &$movekey       (&QWP(0,$key),"xmm0");
 594         &mov            (&DWP(80,$key),$rounds);
 595         &xor            ("eax","eax");
 596         &ret();
 597
 598 &set_label("key_128",16);
 599         &$movekey       (&QWP(0,$key),"xmm0");
 600         &lea            ($key,&DWP(16,$key));
 601 &set_label("key_128_cold");
 602         &shufps         ("xmm4","xmm0",0b00010000);
 603         &pxor           ("xmm0","xmm4");
 604         &shufps         ("xmm4","xmm0",0b10001100,);
 605         &pxor           ("xmm0","xmm4");
 606         &pshufd         ("xmm1","xmm1",0b11111111);     # critical path
 607         &pxor           ("xmm0","xmm1");
 608         &ret();
 609
 610 &set_label("12rounds",16);
 611         &movq           ("xmm2",&QWP(16,"eax"));        # remaining 1/3 of *userKey
 612         &mov            ($rounds,11);
 613         &$movekey       (&QWP(-16,$key),"xmm0")         # round 0
 614         &aeskeygenassist("xmm1","xmm2",0x01);           # round 1,2
 615         &call           (&label("key_192a_cold"));
 616         &aeskeygenassist("xmm1","xmm2",0x02);           # round 2,3
 617         &call           (&label("key_192b"));
 618         &aeskeygenassist("xmm1","xmm2",0x04);           # round 4,5
 619         &call           (&label("key_192a"));
 620         &aeskeygenassist("xmm1","xmm2",0x08);           # round 5,6
 621         &call           (&label("key_192b"));
 622         &aeskeygenassist("xmm1","xmm2",0x10);           # round 7,8
 623         &call           (&label("key_192a"));
 624         &aeskeygenassist("xmm1","xmm2",0x20);           # round 8,9
 625         &call           (&label("key_192b"));
 626         &aeskeygenassist("xmm1","xmm2",0x40);           # round 10,11
 627         &call           (&label("key_192a"));
 628         &aeskeygenassist("xmm1","xmm2",0x80);           # round 11,12
 629         &call           (&label("key_192b"));
 630         &$movekey       (&QWP(0,$key),"xmm0");
 631         &mov            (&DWP(48,$key),$rounds);
 632         &xor            ("eax","eax");
 633         &ret();
 634
 635 &set_label("key_192a",16);
 636         &$movekey       (&QWP(0,$key),"xmm0");
 637         &lea            ($key,&DWP(16,$key));
 638 &set_label("key_192a_cold",16);
 639         &movaps         ("xmm5","xmm2");
 640 &set_label("key_192b_warm");
 641         &shufps         ("xmm4","xmm0",0b00010000);
 642         &movaps         ("xmm3","xmm2");
 643         &pxor           ("xmm0","xmm4");
 644         &shufps         ("xmm4","xmm0",0b10001100);
 645         &pslldq         ("xmm3",4);
 646         &pxor           ("xmm0","xmm4");
 647         &pshufd         ("xmm1","xmm1",0b01010101);     # critical path
 648         &pxor           ("xmm2","xmm3");
 649         &pxor           ("xmm0","xmm1");
 650         &pshufd         ("xmm3","xmm0",0b11111111);
 651         &pxor           ("xmm2","xmm3");
 652         &ret();
 653
 654 &set_label("key_192b",16);
 655         &movaps         ("xmm3","xmm0");
 656         &shufps         ("xmm5","xmm0",0b01000100);
 657         &$movekey       (&QWP(0,$key),"xmm5");
 658         &shufps         ("xmm3","xmm2",0b01001110);
 659         &$movekey       (&QWP(16,$key),"xmm3");
 660         &lea            ($key,&DWP(32,$key));
 661         &jmp            (&label("key_192b_warm"));
 662
 663 &set_label("14rounds",16);
 664         &movups         ("xmm2",&QWP(16,"eax"));        # remaining half of *userKey
 665         &mov            ($rounds,13);
 666         &lea            ($key,&DWP(16,$key));
 667         &$movekey       (&QWP(-32,$key),"xmm0");        # round 0
 668         &$movekey       (&QWP(-16,$key),"xmm2");        # round 1
 669         &aeskeygenassist("xmm1","xmm2",0x01);           # round 2
 670         &call           (&label("key_256a_cold"));
 671         &aeskeygenassist("xmm1","xmm0",0x01);           # round 3
 672         &call           (&label("key_256b"));
 673         &aeskeygenassist("xmm1","xmm2",0x02);           # round 4
 674         &call           (&label("key_256a"));
 675         &aeskeygenassist("xmm1","xmm0",0x02);           # round 5
 676         &call           (&label("key_256b"));
 677         &aeskeygenassist("xmm1","xmm2",0x04);           # round 6
 678         &call           (&label("key_256a"));
 679         &aeskeygenassist("xmm1","xmm0",0x04);           # round 7
 680         &call           (&label("key_256b"));
 681         &aeskeygenassist("xmm1","xmm2",0x08);           # round 8
 682         &call           (&label("key_256a"));
 683         &aeskeygenassist("xmm1","xmm0",0x08);           # round 9
 684         &call           (&label("key_256b"));
 685         &aeskeygenassist("xmm1","xmm2",0x10);           # round 10
 686         &call           (&label("key_256a"));
 687         &aeskeygenassist("xmm1","xmm0",0x10);           # round 11
 688         &call           (&label("key_256b"));
 689         &aeskeygenassist("xmm1","xmm2",0x20);           # round 12
 690         &call           (&label("key_256a"));
 691         &aeskeygenassist("xmm1","xmm0",0x20);           # round 13
 692         &call           (&label("key_256b"));
 693         &aeskeygenassist("xmm1","xmm2",0x40);           # round 14
 694         &call           (&label("key_256a"));
 695         &$movekey       (&QWP(0,$key),"xmm0");
 696         &mov            (&DWP(16,$key),$rounds);
 697         &xor            ("eax","eax");
 698         &ret();
 699
 700 &set_label("key_256a",16);
 701         &$movekey       (&QWP(0,$key),"xmm2");
 702         &lea            ($key,&DWP(16,$key));
 703 &set_label("key_256a_cold");
 704         &shufps         ("xmm4","xmm0",0b00010000);
 705         &pxor           ("xmm0","xmm4");
 706         &shufps         ("xmm4","xmm0",0b10001100);
 707         &pxor           ("xmm0","xmm4");
 708         &pshufd         ("xmm1","xmm1",0b11111111);     # critical path
 709         &pxor           ("xmm0","xmm1");
 710         &ret();
 711
 712 &set_label("key_256b",16);
 713         &$movekey       (&QWP(0,$key),"xmm0");
 714         &lea            ($key,&DWP(16,$key));
 715
 716         &shufps         ("xmm4","xmm2",0b00010000);
 717         &pxor           ("xmm2","xmm4");
 718         &shufps         ("xmm4","xmm2",0b10001100);
 719         &pxor           ("xmm2","xmm4");
 720         &pshufd         ("xmm1","xmm1",0b10101010);     # critical path
 721         &pxor           ("xmm2","xmm1");
 722         &ret();
 723
 724 &set_label("bad_pointer",4);
 725         &mov    ("eax",-1);
 726         &ret    ();
 727 &set_label("bad_keybits",4);
 728         &mov    ("eax",-2);
 729         &ret    ();
 730 &function_end_B("_aesni_set_encrypt_key");
 731
 732 # int $PREFIX_set_encrypt_key (const unsigned char *userKey, int bits,
 733 #                              AES_KEY *key)
 734 &function_begin_B("${PREFIX}_set_encrypt_key");
 735         &mov    ("eax",&wparam(0));
 736         &mov    ($rounds,&wparam(1));
 737         &mov    ($key,&wparam(2));
 738         &call   ("_aesni_set_encrypt_key");
 739         &ret    ();
 740 &function_end_B("${PREFIX}_set_encrypt_key");
 741
 742 # int $PREFIX_set_decrypt_key (const unsigned char *userKey, int bits,
 743 #                              AES_KEY *key)
 744 &function_begin_B("${PREFIX}_set_decrypt_key");
 745         &mov    ("eax",&wparam(0));
 746         &mov    ($rounds,&wparam(1));
 747         &mov    ($key,&wparam(2));
 748         &call   ("_aesni_set_encrypt_key");
 749         &mov    ($key,&wparam(2));
 750         &shl    ($rounds,4)     # rounds-1 after _aesni_set_encrypt_key
 751         &test   ("eax","eax");
 752         &jnz    (&label("dec_key_ret"));
 753         &lea    ("eax",&DWP(16,$key,$rounds));  # end of key schedule
 754
 755         &$movekey       ("xmm0",&QWP(0,$key));  # just swap
 756         &$movekey       ("xmm1",&QWP(0,"eax"));
 757         &$movekey       (&QWP(0,"eax"),"xmm0");
 758         &$movekey       (&QWP(0,$key),"xmm1");
 759         &lea            ($key,&DWP(16,$key));
 760         &lea            ("eax",&DWP(-16,"eax"));
 761
 762 &set_label("dec_key_inverse");
 763         &$movekey       ("xmm0",&QWP(0,$key));  # swap and inverse
 764         &$movekey       ("xmm1",&QWP(0,"eax"));
 765         &aesimc         ("xmm0","xmm0");
 766         &aesimc         ("xmm1","xmm1");
 767         &lea            ($key,&DWP(16,$key));
 768         &lea            ("eax",&DWP(-16,"eax"));
 769         &cmp            ("eax",$key);
 770         &$movekey       (&QWP(16,"eax"),"xmm0");
 771         &$movekey       (&QWP(-16,$key),"xmm1");
 772         &ja             (&label("dec_key_inverse"));
 773
 774         &$movekey       ("xmm0",&QWP(0,$key));  # inverse middle
 775         &aesimc         ("xmm0","xmm0");
 776         &$movekey       (&QWP(0,$key),"xmm0");
 777
 778         &xor            ("eax","eax");          # return success
 779 &set_label("dec_key_ret");
 780         &ret    ();
 781 &function_end_B("${PREFIX}_set_decrypt_key");
 782 &asciz("AES for Intel AES-NI, CRYPTOGAMS by <appro\@openssl.org>");
 783
 784 &asm_finish();