2 # Copyright 2005-2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. Rights for redistribution and usage in source and binary
13 # forms are granted according to the OpenSSL license.
14 # ====================================================================
16 # sha256/512_block procedure for x86_64.
18 # 40% improvement over compiler-generated code on Opteron. On EM64T
19 # sha256 was observed to run >80% faster and sha512 - >40%. No magical
20 # tricks, just straight implementation... I really wonder why gcc
21 # [being armed with inline assembler] fails to generate as fast code.
22 # The only thing which is cool about this module is that it's very
23 # same instruction sequence used for both SHA-256 and SHA-512. In
24 # former case the instructions operate on 32-bit operands, while in
25 # latter - on 64-bit ones. All I had to do is to get one flavor right,
26 # the other one passed the test right away:-)
28 # sha256_block runs in ~1005 cycles on Opteron, which gives you
29 # asymptotic performance of 64*1000/1005=63.7MBps times CPU clock
30 # frequency in GHz. sha512_block runs in ~1275 cycles, which results
31 # in 128*1000/1275=100MBps per GHz. Is there room for improvement?
32 # Well, if you compare it to IA-64 implementation, which maintains
33 # X[16] in register bank[!], tends to 4 instructions per CPU clock
34 # cycle and runs in 1003 cycles, 1275 is very good result for 3-way
35 # issue Opteron pipeline and X[16] maintained in memory. So that *if*
36 # there is a way to improve it, *then* the only way would be to try to
37 # offload X[16] updates to SSE unit, but that would require "deeper"
38 # loop unroll, which in turn would naturally cause size blow-up, not
39 # to mention increased complexity! And once again, only *if* it's
40 # actually possible to noticeably improve overall ILP, instruction
41 # level parallelism, on a given CPU implementation in this case.
43 # Special note on Intel EM64T. While Opteron CPU exhibits perfect
44 # performance ratio of 1.5 between 64- and 32-bit flavors [see above],
45 # [currently available] EM64T CPUs apparently are far from it. On the
46 # contrary, 64-bit version, sha512_block, is ~30% *slower* than 32-bit
47 # sha256_block:-( This is presumably because 64-bit shifts/rotates
48 # apparently are not atomic instructions, but implemented in microcode.
52 # Optimization including one of Pavel Semjanov's ideas, alternative
53 # Maj, resulted in >=5% improvement on most CPUs, +20% SHA256 and
54 # unfortunately -2% SHA512 on P4 [which nobody should care about
59 # Add SIMD code paths, see below for improvement coefficients. SSSE3
60 # code path was not attempted for SHA512, because improvement is not
61 # estimated to be high enough, noticeably less than 9%, to justify
62 # the effort, not on pre-AVX processors. [Obviously with exclusion
63 # for VIA Nano, but it has SHA512 instruction that is faster and
64 # should be used instead.] For reference, corresponding estimated
65 # upper limit for improvement for SSSE3 SHA256 is 28%. The fact that
66 # higher coefficients are observed on VIA Nano and Bulldozer has more
67 # to do with specifics of their architecture [which is topic for
68 # separate discussion].
72 # Add AVX2 code path. Two consecutive input blocks are loaded to
73 # 256-bit %ymm registers, with data from first block to least
74 # significant 128-bit halves and data from second to most significant.
75 # The data is then processed with same SIMD instruction sequence as
76 # for AVX, but with %ymm as operands. Side effect is increased stack
77 # frame, 448 additional bytes in SHA256 and 1152 in SHA512, and 1.2KB
82 # Add support for Intel SHA Extensions.
84 ######################################################################
85 # Current performance in cycles per processed byte (less is better):
87 # SHA256 SSSE3 AVX/XOP(*) SHA512 AVX/XOP(*)
89 # AMD K8 14.9 - - 9.57 -
91 # Core 2 15.6 13.8(+13%) - 9.97 -
92 # Westmere 14.8 12.3(+19%) - 9.58 -
93 # Sandy Bridge 17.4 14.2(+23%) 11.6(+50%(**)) 11.2 8.10(+38%(**))
94 # Ivy Bridge 12.6 10.5(+20%) 10.3(+22%) 8.17 7.22(+13%)
95 # Haswell 12.2 9.28(+31%) 7.80(+56%) 7.66 5.40(+42%)
96 # Skylake 11.4 9.03(+26%) 7.70(+48%) 7.25 5.20(+40%)
97 # Bulldozer 21.1 13.6(+54%) 13.6(+54%(***)) 13.5 8.58(+57%)
98 # VIA Nano 23.0 16.5(+39%) - 14.7 -
99 # Atom 23.0 18.9(+22%) - 14.7 -
100 # Silvermont 27.4 20.6(+33%) - 17.5 -
101 # Goldmont 18.9 14.3(+32%) 4.16(+350%) 12.0 -
103 # (*) whichever best applicable, including SHAEXT;
104 # (**) switch from ror to shrd stands for fair share of improvement;
105 # (***) execution time is fully determined by remaining integer-only
106 # part, body_00_15; reducing the amount of SIMD instructions
107 # below certain limit makes no difference/sense; to conserve
108 # space SHA256 XOP code path is therefore omitted;
112 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
114 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
116 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
117 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
118 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
119 die "can't locate x86_64-xlate.pl";
121 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
122 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
123 $avx = ($1>=2.19) + ($1>=2.22);
126 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
127 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
128 $avx = ($1>=2.09) + ($1>=2.10);
131 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
132 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
133 $avx = ($1>=10) + ($1>=11);
136 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
137 $avx = ($2>=3.0) + ($2>3.0);
140 $shaext=1; ### set to zero if compiling for 1.0.1
141 $avx=1 if (!$shaext && $avx);
143 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
146 if ($output =~ /512/) {
147 $func="sha512_block_data_order";
150 @ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%rax","%rbx","%rcx","%rdx",
151 "%r8", "%r9", "%r10","%r11");
152 ($T1,$a0,$a1,$a2,$a3)=("%r12","%r13","%r14","%r15","%rdi");
159 $func="sha256_block_data_order";
162 @ROT=($A,$B,$C,$D,$E,$F,$G,$H)=("%eax","%ebx","%ecx","%edx",
163 "%r8d","%r9d","%r10d","%r11d");
164 ($T1,$a0,$a1,$a2,$a3)=("%r12d","%r13d","%r14d","%r15d","%edi");
172 $ctx="%rdi"; # 1st arg, zapped by $a3
173 $inp="%rsi"; # 2nd arg
176 $_ctx="16*$SZ+0*8(%rsp)";
177 $_inp="16*$SZ+1*8(%rsp)";
178 $_end="16*$SZ+2*8(%rsp)";
179 $_rsp="16*$SZ+3*8(%rsp)";
180 $framesz="16*$SZ+4*8";
184 { my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
186 $STRIDE += 16 if ($i%(16/$SZ)==(16/$SZ-1));
189 ror \$`$Sigma1[2]-$Sigma1[1]`,$a0
193 ror \$`$Sigma0[2]-$Sigma0[1]`,$a1
196 mov $T1,`$SZ*($i&0xf)`(%rsp)
200 ror \$`$Sigma1[1]-$Sigma1[0]`,$a0
202 xor $g,$a2 # Ch(e,f,g)=((f^g)&e)^g
204 ror \$`$Sigma0[1]-$Sigma0[0]`,$a1
206 add $a2,$T1 # T1+=Ch(e,f,g)
209 add ($Tbl),$T1 # T1+=K[round]
212 xor $b,$a2 # a^b, b^c in next round
213 ror \$$Sigma1[0],$a0 # Sigma1(e)
217 ror \$$Sigma0[0],$a1 # Sigma0(a)
218 add $a0,$T1 # T1+=Sigma1(e)
220 xor $a3,$h # h=Maj(a,b,c)=Ch(a^b,c,b)
224 lea $STRIDE($Tbl),$Tbl # round++
226 $code.=<<___ if ($i<15);
227 add $a1,$h # h+=Sigma0(a)
229 ($a2,$a3) = ($a3,$a2);
233 { my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_;
236 mov `$SZ*(($i+1)&0xf)`(%rsp),$a0
237 mov `$SZ*(($i+14)&0xf)`(%rsp),$a2
240 ror \$`$sigma0[1]-$sigma0[0]`,$a0
241 add $a1,$a # modulo-scheduled h+=Sigma0(a)
243 ror \$`$sigma1[1]-$sigma1[0]`,$a2
252 xor $a0,$T1 # sigma0(X[(i+1)&0xf])
253 xor $a1,$a2 # sigma1(X[(i+14)&0xf])
254 add `$SZ*(($i+9)&0xf)`(%rsp),$T1
256 add `$SZ*($i&0xf)`(%rsp),$T1
267 .extern OPENSSL_ia32cap_P
269 .type $func,\@function,3
273 $code.=<<___ if ($SZ==4 || $avx);
274 lea OPENSSL_ia32cap_P(%rip),%r11
279 $code.=<<___ if ($SZ==4 && $shaext);
280 test \$`1<<29`,%r11d # check for SHA
283 $code.=<<___ if ($avx && $SZ==8);
284 test \$`1<<11`,%r10d # check for XOP
287 $code.=<<___ if ($avx>1);
288 and \$`1<<8|1<<5|1<<3`,%r11d # check for BMI2+AVX2+BMI1
289 cmp \$`1<<8|1<<5|1<<3`,%r11d
292 $code.=<<___ if ($avx);
293 and \$`1<<30`,%r9d # mask "Intel CPU" bit
294 and \$`1<<28|1<<9`,%r10d # mask AVX and SSSE3 bits
296 cmp \$`1<<28|1<<9|1<<30`,%r10d
299 $code.=<<___ if ($SZ==4);
310 mov %rsp,%r11 # copy %rsp
311 shl \$4,%rdx # num*16
313 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
314 and \$-64,%rsp # align stack frame
315 mov $ctx,$_ctx # save ctx, 1st arg
316 mov $inp,$_inp # save inp, 2nd arh
317 mov %rdx,$_end # save end pointer, "3rd" arg
318 mov %r11,$_rsp # save copy of %rsp
334 lea $TABLE(%rip),$Tbl
337 for($i=0;$i<16;$i++) {
338 $code.=" mov $SZ*$i($inp),$T1\n";
339 $code.=" mov @ROT[4],$a0\n";
340 $code.=" mov @ROT[0],$a1\n";
341 $code.=" bswap $T1\n";
342 &ROUND_00_15($i,@ROT);
343 unshift(@ROT,pop(@ROT));
351 &ROUND_16_XX($i,@ROT);
352 unshift(@ROT,pop(@ROT));
356 cmpb \$0,`$SZ-1`($Tbl)
360 add $a1,$A # modulo-scheduled h+=Sigma0(a)
361 lea 16*$SZ($inp),$inp
400 .type $TABLE,\@object
402 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
403 .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
404 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
405 .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
406 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
407 .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
408 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
409 .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
410 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
411 .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
412 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
413 .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
414 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
415 .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
416 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
417 .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
418 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
419 .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
420 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
421 .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
422 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
423 .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
424 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
425 .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
426 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
427 .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
428 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
429 .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
430 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
431 .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
432 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
433 .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
435 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f
436 .long 0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f
437 .long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff
438 .long 0x03020100,0x0b0a0908,0xffffffff,0xffffffff
439 .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908
440 .long 0xffffffff,0xffffffff,0x03020100,0x0b0a0908
441 .asciz "SHA256 block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
446 .type $TABLE,\@object
448 .quad 0x428a2f98d728ae22,0x7137449123ef65cd
449 .quad 0x428a2f98d728ae22,0x7137449123ef65cd
450 .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
451 .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
452 .quad 0x3956c25bf348b538,0x59f111f1b605d019
453 .quad 0x3956c25bf348b538,0x59f111f1b605d019
454 .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
455 .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
456 .quad 0xd807aa98a3030242,0x12835b0145706fbe
457 .quad 0xd807aa98a3030242,0x12835b0145706fbe
458 .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
459 .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
460 .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
461 .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
462 .quad 0x9bdc06a725c71235,0xc19bf174cf692694
463 .quad 0x9bdc06a725c71235,0xc19bf174cf692694
464 .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
465 .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
466 .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
467 .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
468 .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
469 .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
470 .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
471 .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
472 .quad 0x983e5152ee66dfab,0xa831c66d2db43210
473 .quad 0x983e5152ee66dfab,0xa831c66d2db43210
474 .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
475 .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
476 .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
477 .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
478 .quad 0x06ca6351e003826f,0x142929670a0e6e70
479 .quad 0x06ca6351e003826f,0x142929670a0e6e70
480 .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
481 .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
482 .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
483 .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
484 .quad 0x650a73548baf63de,0x766a0abb3c77b2a8
485 .quad 0x650a73548baf63de,0x766a0abb3c77b2a8
486 .quad 0x81c2c92e47edaee6,0x92722c851482353b
487 .quad 0x81c2c92e47edaee6,0x92722c851482353b
488 .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
489 .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
490 .quad 0xc24b8b70d0f89791,0xc76c51a30654be30
491 .quad 0xc24b8b70d0f89791,0xc76c51a30654be30
492 .quad 0xd192e819d6ef5218,0xd69906245565a910
493 .quad 0xd192e819d6ef5218,0xd69906245565a910
494 .quad 0xf40e35855771202a,0x106aa07032bbd1b8
495 .quad 0xf40e35855771202a,0x106aa07032bbd1b8
496 .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
497 .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
498 .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
499 .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
500 .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
501 .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
502 .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
503 .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
504 .quad 0x748f82ee5defb2fc,0x78a5636f43172f60
505 .quad 0x748f82ee5defb2fc,0x78a5636f43172f60
506 .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
507 .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
508 .quad 0x90befffa23631e28,0xa4506cebde82bde9
509 .quad 0x90befffa23631e28,0xa4506cebde82bde9
510 .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
511 .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
512 .quad 0xca273eceea26619c,0xd186b8c721c0c207
513 .quad 0xca273eceea26619c,0xd186b8c721c0c207
514 .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
515 .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
516 .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
517 .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
518 .quad 0x113f9804bef90dae,0x1b710b35131c471b
519 .quad 0x113f9804bef90dae,0x1b710b35131c471b
520 .quad 0x28db77f523047d84,0x32caab7b40c72493
521 .quad 0x28db77f523047d84,0x32caab7b40c72493
522 .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
523 .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
524 .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
525 .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
526 .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
527 .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
529 .quad 0x0001020304050607,0x08090a0b0c0d0e0f
530 .quad 0x0001020304050607,0x08090a0b0c0d0e0f
531 .asciz "SHA512 block transform for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
535 ######################################################################
538 if ($SZ==4 && $shaext) {{{
539 ######################################################################
540 # Intel SHA Extensions implementation of SHA256 update function.
542 my ($ctx,$inp,$num,$Tbl)=("%rdi","%rsi","%rdx","%rcx");
544 my ($Wi,$ABEF,$CDGH,$TMP,$BSWAP,$ABEF_SAVE,$CDGH_SAVE)=map("%xmm$_",(0..2,7..10));
545 my @MSG=map("%xmm$_",(3..6));
548 .type sha256_block_data_order_shaext,\@function,3
550 sha256_block_data_order_shaext:
553 $code.=<<___ if ($win64);
554 lea `-8-5*16`(%rsp),%rsp
555 movaps %xmm6,-8-5*16(%rax)
556 movaps %xmm7,-8-4*16(%rax)
557 movaps %xmm8,-8-3*16(%rax)
558 movaps %xmm9,-8-2*16(%rax)
559 movaps %xmm10,-8-1*16(%rax)
563 lea K256+0x80(%rip),$Tbl
564 movdqu ($ctx),$ABEF # DCBA
565 movdqu 16($ctx),$CDGH # HGFE
566 movdqa 0x200-0x80($Tbl),$TMP # byte swap mask
568 pshufd \$0x1b,$ABEF,$Wi # ABCD
569 pshufd \$0xb1,$ABEF,$ABEF # CDAB
570 pshufd \$0x1b,$CDGH,$CDGH # EFGH
571 movdqa $TMP,$BSWAP # offload
572 palignr \$8,$CDGH,$ABEF # ABEF
573 punpcklqdq $Wi,$CDGH # CDGH
578 movdqu ($inp),@MSG[0]
579 movdqu 0x10($inp),@MSG[1]
580 movdqu 0x20($inp),@MSG[2]
582 movdqu 0x30($inp),@MSG[3]
584 movdqa 0*32-0x80($Tbl),$Wi
587 movdqa $CDGH,$CDGH_SAVE # offload
588 sha256rnds2 $ABEF,$CDGH # 0-3
589 pshufd \$0x0e,$Wi,$Wi
591 movdqa $ABEF,$ABEF_SAVE # offload
592 sha256rnds2 $CDGH,$ABEF
594 movdqa 1*32-0x80($Tbl),$Wi
597 sha256rnds2 $ABEF,$CDGH # 4-7
598 pshufd \$0x0e,$Wi,$Wi
600 sha256msg1 @MSG[1],@MSG[0]
601 sha256rnds2 $CDGH,$ABEF
603 movdqa 2*32-0x80($Tbl),$Wi
606 sha256rnds2 $ABEF,$CDGH # 8-11
607 pshufd \$0x0e,$Wi,$Wi
609 palignr \$4,@MSG[2],$TMP
612 sha256msg1 @MSG[2],@MSG[1]
613 sha256rnds2 $CDGH,$ABEF
615 movdqa 3*32-0x80($Tbl),$Wi
617 sha256msg2 @MSG[3],@MSG[0]
618 sha256rnds2 $ABEF,$CDGH # 12-15
619 pshufd \$0x0e,$Wi,$Wi
621 palignr \$4,@MSG[3],$TMP
624 sha256msg1 @MSG[3],@MSG[2]
625 sha256rnds2 $CDGH,$ABEF
627 for($i=4;$i<16-3;$i++) {
629 movdqa $i*32-0x80($Tbl),$Wi
631 sha256msg2 @MSG[0],@MSG[1]
632 sha256rnds2 $ABEF,$CDGH # 16-19...
633 pshufd \$0x0e,$Wi,$Wi
635 palignr \$4,@MSG[0],$TMP
638 sha256msg1 @MSG[0],@MSG[3]
639 sha256rnds2 $CDGH,$ABEF
641 push(@MSG,shift(@MSG));
644 movdqa 13*32-0x80($Tbl),$Wi
646 sha256msg2 @MSG[0],@MSG[1]
647 sha256rnds2 $ABEF,$CDGH # 52-55
648 pshufd \$0x0e,$Wi,$Wi
650 palignr \$4,@MSG[0],$TMP
651 sha256rnds2 $CDGH,$ABEF
654 movdqa 14*32-0x80($Tbl),$Wi
656 sha256rnds2 $ABEF,$CDGH # 56-59
657 pshufd \$0x0e,$Wi,$Wi
658 sha256msg2 @MSG[1],@MSG[2]
660 sha256rnds2 $CDGH,$ABEF
662 movdqa 15*32-0x80($Tbl),$Wi
665 sha256rnds2 $ABEF,$CDGH # 60-63
666 pshufd \$0x0e,$Wi,$Wi
669 sha256rnds2 $CDGH,$ABEF
671 paddd $CDGH_SAVE,$CDGH
672 paddd $ABEF_SAVE,$ABEF
675 pshufd \$0xb1,$CDGH,$CDGH # DCHG
676 pshufd \$0x1b,$ABEF,$TMP # FEBA
677 pshufd \$0xb1,$ABEF,$ABEF # BAFE
678 punpckhqdq $CDGH,$ABEF # DCBA
679 palignr \$8,$TMP,$CDGH # HGFE
682 movdqu $CDGH,16($ctx)
684 $code.=<<___ if ($win64);
685 movaps -8-5*16(%rax),%xmm6
686 movaps -8-4*16(%rax),%xmm7
687 movaps -8-3*16(%rax),%xmm8
688 movaps -8-2*16(%rax),%xmm9
689 movaps -8-1*16(%rax),%xmm10
695 .size sha256_block_data_order_shaext,.-sha256_block_data_order_shaext
701 my ($a,$b,$c,$d,$e,$f,$g,$h);
703 sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm
704 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
706 $arg = "\$$arg" if ($arg*1 eq $arg);
707 $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n";
712 '($a,$b,$c,$d,$e,$f,$g,$h)=@ROT;'.
714 '&ror ($a0,$Sigma1[2]-$Sigma1[1])',
718 '&ror ($a1,$Sigma0[2]-$Sigma0[1])',
720 '&xor ($a4,$g)', # f^g
722 '&ror ($a0,$Sigma1[1]-$Sigma1[0])',
724 '&and ($a4,$e)', # (f^g)&e
727 '&add ($h,$SZ*($i&15)."(%rsp)")', # h+=X[i]+K[i]
730 '&xor ($a4,$g)', # Ch(e,f,g)=((f^g)&e)^g
731 '&ror ($a1,$Sigma0[1]-$Sigma0[0])',
732 '&xor ($a2,$b)', # a^b, b^c in next round
734 '&add ($h,$a4)', # h+=Ch(e,f,g)
735 '&ror ($a0,$Sigma1[0])', # Sigma1(e)
736 '&and ($a3,$a2)', # (b^c)&(a^b)
739 '&add ($h,$a0)', # h+=Sigma1(e)
740 '&xor ($a3,$b)', # Maj(a,b,c)=Ch(a^b,c,b)
742 '&ror ($a1,$Sigma0[0])', # Sigma0(a)
743 '&add ($d,$h)', # d+=h
744 '&add ($h,$a3)', # h+=Maj(a,b,c)
747 '&add ($a1,$h);'. # h+=Sigma0(a)
748 '($a2,$a3) = ($a3,$a2); unshift(@ROT,pop(@ROT)); $i++;'
752 ######################################################################
755 if ($SZ==4) { # SHA256 only
756 my @X = map("%xmm$_",(0..3));
757 my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%xmm$_",(4..9));
760 .type ${func}_ssse3,\@function,3
770 mov %rsp,%r11 # copy %rsp
771 shl \$4,%rdx # num*16
772 sub \$`$framesz+$win64*16*4`,%rsp
773 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
774 and \$-64,%rsp # align stack frame
775 mov $ctx,$_ctx # save ctx, 1st arg
776 mov $inp,$_inp # save inp, 2nd arh
777 mov %rdx,$_end # save end pointer, "3rd" arg
778 mov %r11,$_rsp # save copy of %rsp
780 $code.=<<___ if ($win64);
781 movaps %xmm6,16*$SZ+32(%rsp)
782 movaps %xmm7,16*$SZ+48(%rsp)
783 movaps %xmm8,16*$SZ+64(%rsp)
784 movaps %xmm9,16*$SZ+80(%rsp)
800 #movdqa $TABLE+`$SZ*2*$rounds`+32(%rip),$t4
801 #movdqa $TABLE+`$SZ*2*$rounds`+64(%rip),$t5
805 movdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
806 movdqu 0x00($inp),@X[0]
807 movdqu 0x10($inp),@X[1]
808 movdqu 0x20($inp),@X[2]
810 movdqu 0x30($inp),@X[3]
811 lea $TABLE(%rip),$Tbl
813 movdqa 0x00($Tbl),$t0
814 movdqa 0x20($Tbl),$t1
817 movdqa 0x40($Tbl),$t2
819 movdqa 0x60($Tbl),$t3
823 movdqa $t0,0x00(%rsp)
825 movdqa $t1,0x10(%rsp)
827 movdqa $t2,0x20(%rsp)
829 movdqa $t3,0x30(%rsp)
835 sub \$`-16*2*$SZ`,$Tbl # size optimization
837 sub Xupdate_256_SSSE3 () {
839 '&movdqa ($t0,@X[1]);',
840 '&movdqa ($t3,@X[3])',
841 '&palignr ($t0,@X[0],$SZ)', # X[1..4]
842 '&palignr ($t3,@X[2],$SZ);', # X[9..12]
844 '&movdqa ($t2,$t0);',
845 '&psrld ($t0,$sigma0[2])',
846 '&paddd (@X[0],$t3);', # X[0..3] += X[9..12]
847 '&psrld ($t2,$sigma0[0])',
848 '&pshufd ($t3,@X[3],0b11111010)',# X[14..15]
849 '&pslld ($t1,8*$SZ-$sigma0[1]);'.
851 '&psrld ($t2,$sigma0[1]-$sigma0[0]);'.
853 '&pslld ($t1,$sigma0[1]-$sigma0[0]);'.
856 '&pxor ($t0,$t1);', # sigma0(X[1..4])
857 '&psrld ($t3,$sigma1[2])',
858 '&paddd (@X[0],$t0);', # X[0..3] += sigma0(X[1..4])
859 '&psrlq ($t2,$sigma1[0])',
861 '&psrlq ($t2,$sigma1[1]-$sigma1[0])',
863 '&pshufb ($t3,$t4)', # sigma1(X[14..15])
864 '&paddd (@X[0],$t3)', # X[0..1] += sigma1(X[14..15])
865 '&pshufd ($t3,@X[0],0b01010000)',# X[16..17]
866 '&movdqa ($t2,$t3);',
867 '&psrld ($t3,$sigma1[2])',
868 '&psrlq ($t2,$sigma1[0])',
870 '&psrlq ($t2,$sigma1[1]-$sigma1[0])',
872 '&movdqa ($t2,16*2*$j."($Tbl)")',
874 '&paddd (@X[0],$t3)' # X[2..3] += sigma1(X[16..17])
878 sub SSSE3_256_00_47 () {
882 my @insns = (&$body,&$body,&$body,&$body); # 104 instructions
885 foreach (Xupdate_256_SSSE3()) { # 36 instructions
891 } else { # squeeze extra 4% on Westmere and 19% on Atom
892 eval(shift(@insns)); #@
897 eval(shift(@insns)); #@
900 eval(shift(@insns)); #@
902 &palignr ($t0,@X[0],$SZ); # X[1..4]
905 &palignr ($t3,@X[2],$SZ); # X[9..12]
909 eval(shift(@insns)); #@
914 eval(shift(@insns)); #@
916 &psrld ($t0,$sigma0[2]);
920 &paddd (@X[0],$t3); # X[0..3] += X[9..12]
921 eval(shift(@insns)); #@
923 &psrld ($t2,$sigma0[0]);
926 &pshufd ($t3,@X[3],0b11111010); # X[4..15]
928 eval(shift(@insns)); #@
929 &pslld ($t1,8*$SZ-$sigma0[1]);
933 eval(shift(@insns)); #@
936 eval(shift(@insns)); #@
937 &psrld ($t2,$sigma0[1]-$sigma0[0]);
942 &pslld ($t1,$sigma0[1]-$sigma0[0]);
947 eval(shift(@insns)); #@
951 &pxor ($t0,$t1); # sigma0(X[1..4])
952 eval(shift(@insns)); #@
955 &psrld ($t3,$sigma1[2]);
958 &paddd (@X[0],$t0); # X[0..3] += sigma0(X[1..4])
959 eval(shift(@insns)); #@
961 &psrlq ($t2,$sigma1[0]);
966 eval(shift(@insns)); #@
969 eval(shift(@insns)); #@
970 &psrlq ($t2,$sigma1[1]-$sigma1[0]);
974 eval(shift(@insns)); #@
977 #&pshufb ($t3,$t4); # sigma1(X[14..15])
978 &pshufd ($t3,$t3,0b10000000);
984 eval(shift(@insns)); #@
987 eval(shift(@insns)); #@
988 &paddd (@X[0],$t3); # X[0..1] += sigma1(X[14..15])
992 &pshufd ($t3,@X[0],0b01010000); # X[16..17]
994 eval(shift(@insns)); #@
999 &psrld ($t3,$sigma1[2]);
1000 eval(shift(@insns));
1001 eval(shift(@insns)); #@
1002 &psrlq ($t2,$sigma1[0]);
1003 eval(shift(@insns));
1004 eval(shift(@insns));
1006 eval(shift(@insns)); #@
1007 eval(shift(@insns));
1008 eval(shift(@insns));
1009 eval(shift(@insns)); #@
1010 eval(shift(@insns));
1011 &psrlq ($t2,$sigma1[1]-$sigma1[0]);
1012 eval(shift(@insns));
1013 eval(shift(@insns));
1014 eval(shift(@insns));
1016 eval(shift(@insns));
1017 eval(shift(@insns));
1018 eval(shift(@insns)); #@
1020 &pshufd ($t3,$t3,0b00001000);
1021 eval(shift(@insns));
1022 eval(shift(@insns));
1023 &movdqa ($t2,16*2*$j."($Tbl)");
1024 eval(shift(@insns)); #@
1025 eval(shift(@insns));
1027 eval(shift(@insns));
1028 eval(shift(@insns));
1029 eval(shift(@insns));
1030 &paddd (@X[0],$t3); # X[2..3] += sigma1(X[16..17])
1031 eval(shift(@insns)); #@
1032 eval(shift(@insns));
1033 eval(shift(@insns));
1036 foreach (@insns) { eval; } # remaining instructions
1037 &movdqa (16*$j."(%rsp)",$t2);
1040 for ($i=0,$j=0; $j<4; $j++) {
1041 &SSSE3_256_00_47($j,\&body_00_15,@X);
1042 push(@X,shift(@X)); # rotate(@X)
1044 &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0);
1045 &jne (".Lssse3_00_47");
1047 for ($i=0; $i<16; ) {
1048 foreach(body_00_15()) { eval; }
1055 lea 16*$SZ($inp),$inp
1078 $code.=<<___ if ($win64);
1079 movaps 16*$SZ+32(%rsp),%xmm6
1080 movaps 16*$SZ+48(%rsp),%xmm7
1081 movaps 16*$SZ+64(%rsp),%xmm8
1082 movaps 16*$SZ+80(%rsp),%xmm9
1094 .size ${func}_ssse3,.-${func}_ssse3
1099 ######################################################################
1102 if ($SZ==8) { # SHA512 only
1104 .type ${func}_xop,\@function,3
1114 mov %rsp,%r11 # copy %rsp
1115 shl \$4,%rdx # num*16
1116 sub \$`$framesz+$win64*16*($SZ==4?4:6)`,%rsp
1117 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
1118 and \$-64,%rsp # align stack frame
1119 mov $ctx,$_ctx # save ctx, 1st arg
1120 mov $inp,$_inp # save inp, 2nd arh
1121 mov %rdx,$_end # save end pointer, "3rd" arg
1122 mov %r11,$_rsp # save copy of %rsp
1124 $code.=<<___ if ($win64);
1125 movaps %xmm6,16*$SZ+32(%rsp)
1126 movaps %xmm7,16*$SZ+48(%rsp)
1127 movaps %xmm8,16*$SZ+64(%rsp)
1128 movaps %xmm9,16*$SZ+80(%rsp)
1130 $code.=<<___ if ($win64 && $SZ>4);
1131 movaps %xmm10,16*$SZ+96(%rsp)
1132 movaps %xmm11,16*$SZ+112(%rsp)
1148 if ($SZ==4) { # SHA256
1149 my @X = map("%xmm$_",(0..3));
1150 my ($t0,$t1,$t2,$t3) = map("%xmm$_",(4..7));
1155 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1156 vmovdqu 0x00($inp),@X[0]
1157 vmovdqu 0x10($inp),@X[1]
1158 vmovdqu 0x20($inp),@X[2]
1159 vmovdqu 0x30($inp),@X[3]
1160 vpshufb $t3,@X[0],@X[0]
1161 lea $TABLE(%rip),$Tbl
1162 vpshufb $t3,@X[1],@X[1]
1163 vpshufb $t3,@X[2],@X[2]
1164 vpaddd 0x00($Tbl),@X[0],$t0
1165 vpshufb $t3,@X[3],@X[3]
1166 vpaddd 0x20($Tbl),@X[1],$t1
1167 vpaddd 0x40($Tbl),@X[2],$t2
1168 vpaddd 0x60($Tbl),@X[3],$t3
1169 vmovdqa $t0,0x00(%rsp)
1171 vmovdqa $t1,0x10(%rsp)
1173 vmovdqa $t2,0x20(%rsp)
1175 vmovdqa $t3,0x30(%rsp)
1181 sub \$`-16*2*$SZ`,$Tbl # size optimization
1183 sub XOP_256_00_47 () {
1187 my @insns = (&$body,&$body,&$body,&$body); # 104 instructions
1189 &vpalignr ($t0,@X[1],@X[0],$SZ); # X[1..4]
1190 eval(shift(@insns));
1191 eval(shift(@insns));
1192 &vpalignr ($t3,@X[3],@X[2],$SZ); # X[9..12]
1193 eval(shift(@insns));
1194 eval(shift(@insns));
1195 &vprotd ($t1,$t0,8*$SZ-$sigma0[1]);
1196 eval(shift(@insns));
1197 eval(shift(@insns));
1198 &vpsrld ($t0,$t0,$sigma0[2]);
1199 eval(shift(@insns));
1200 eval(shift(@insns));
1201 &vpaddd (@X[0],@X[0],$t3); # X[0..3] += X[9..12]
1202 eval(shift(@insns));
1203 eval(shift(@insns));
1204 eval(shift(@insns));
1205 eval(shift(@insns));
1206 &vprotd ($t2,$t1,$sigma0[1]-$sigma0[0]);
1207 eval(shift(@insns));
1208 eval(shift(@insns));
1209 &vpxor ($t0,$t0,$t1);
1210 eval(shift(@insns));
1211 eval(shift(@insns));
1212 eval(shift(@insns));
1213 eval(shift(@insns));
1214 &vprotd ($t3,@X[3],8*$SZ-$sigma1[1]);
1215 eval(shift(@insns));
1216 eval(shift(@insns));
1217 &vpxor ($t0,$t0,$t2); # sigma0(X[1..4])
1218 eval(shift(@insns));
1219 eval(shift(@insns));
1220 &vpsrld ($t2,@X[3],$sigma1[2]);
1221 eval(shift(@insns));
1222 eval(shift(@insns));
1223 &vpaddd (@X[0],@X[0],$t0); # X[0..3] += sigma0(X[1..4])
1224 eval(shift(@insns));
1225 eval(shift(@insns));
1226 &vprotd ($t1,$t3,$sigma1[1]-$sigma1[0]);
1227 eval(shift(@insns));
1228 eval(shift(@insns));
1229 &vpxor ($t3,$t3,$t2);
1230 eval(shift(@insns));
1231 eval(shift(@insns));
1232 eval(shift(@insns));
1233 eval(shift(@insns));
1234 &vpxor ($t3,$t3,$t1); # sigma1(X[14..15])
1235 eval(shift(@insns));
1236 eval(shift(@insns));
1237 eval(shift(@insns));
1238 eval(shift(@insns));
1239 &vpsrldq ($t3,$t3,8);
1240 eval(shift(@insns));
1241 eval(shift(@insns));
1242 eval(shift(@insns));
1243 eval(shift(@insns));
1244 &vpaddd (@X[0],@X[0],$t3); # X[0..1] += sigma1(X[14..15])
1245 eval(shift(@insns));
1246 eval(shift(@insns));
1247 eval(shift(@insns));
1248 eval(shift(@insns));
1249 &vprotd ($t3,@X[0],8*$SZ-$sigma1[1]);
1250 eval(shift(@insns));
1251 eval(shift(@insns));
1252 &vpsrld ($t2,@X[0],$sigma1[2]);
1253 eval(shift(@insns));
1254 eval(shift(@insns));
1255 &vprotd ($t1,$t3,$sigma1[1]-$sigma1[0]);
1256 eval(shift(@insns));
1257 eval(shift(@insns));
1258 &vpxor ($t3,$t3,$t2);
1259 eval(shift(@insns));
1260 eval(shift(@insns));
1261 eval(shift(@insns));
1262 eval(shift(@insns));
1263 &vpxor ($t3,$t3,$t1); # sigma1(X[16..17])
1264 eval(shift(@insns));
1265 eval(shift(@insns));
1266 eval(shift(@insns));
1267 eval(shift(@insns));
1268 &vpslldq ($t3,$t3,8); # 22 instructions
1269 eval(shift(@insns));
1270 eval(shift(@insns));
1271 eval(shift(@insns));
1272 eval(shift(@insns));
1273 &vpaddd (@X[0],@X[0],$t3); # X[2..3] += sigma1(X[16..17])
1274 eval(shift(@insns));
1275 eval(shift(@insns));
1276 eval(shift(@insns));
1277 eval(shift(@insns));
1278 &vpaddd ($t2,@X[0],16*2*$j."($Tbl)");
1279 foreach (@insns) { eval; } # remaining instructions
1280 &vmovdqa (16*$j."(%rsp)",$t2);
1283 for ($i=0,$j=0; $j<4; $j++) {
1284 &XOP_256_00_47($j,\&body_00_15,@X);
1285 push(@X,shift(@X)); # rotate(@X)
1287 &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0);
1288 &jne (".Lxop_00_47");
1290 for ($i=0; $i<16; ) {
1291 foreach(body_00_15()) { eval; }
1295 my @X = map("%xmm$_",(0..7));
1296 my ($t0,$t1,$t2,$t3) = map("%xmm$_",(8..11));
1301 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1302 vmovdqu 0x00($inp),@X[0]
1303 lea $TABLE+0x80(%rip),$Tbl # size optimization
1304 vmovdqu 0x10($inp),@X[1]
1305 vmovdqu 0x20($inp),@X[2]
1306 vpshufb $t3,@X[0],@X[0]
1307 vmovdqu 0x30($inp),@X[3]
1308 vpshufb $t3,@X[1],@X[1]
1309 vmovdqu 0x40($inp),@X[4]
1310 vpshufb $t3,@X[2],@X[2]
1311 vmovdqu 0x50($inp),@X[5]
1312 vpshufb $t3,@X[3],@X[3]
1313 vmovdqu 0x60($inp),@X[6]
1314 vpshufb $t3,@X[4],@X[4]
1315 vmovdqu 0x70($inp),@X[7]
1316 vpshufb $t3,@X[5],@X[5]
1317 vpaddq -0x80($Tbl),@X[0],$t0
1318 vpshufb $t3,@X[6],@X[6]
1319 vpaddq -0x60($Tbl),@X[1],$t1
1320 vpshufb $t3,@X[7],@X[7]
1321 vpaddq -0x40($Tbl),@X[2],$t2
1322 vpaddq -0x20($Tbl),@X[3],$t3
1323 vmovdqa $t0,0x00(%rsp)
1324 vpaddq 0x00($Tbl),@X[4],$t0
1325 vmovdqa $t1,0x10(%rsp)
1326 vpaddq 0x20($Tbl),@X[5],$t1
1327 vmovdqa $t2,0x20(%rsp)
1328 vpaddq 0x40($Tbl),@X[6],$t2
1329 vmovdqa $t3,0x30(%rsp)
1330 vpaddq 0x60($Tbl),@X[7],$t3
1331 vmovdqa $t0,0x40(%rsp)
1333 vmovdqa $t1,0x50(%rsp)
1335 vmovdqa $t2,0x60(%rsp)
1337 vmovdqa $t3,0x70(%rsp)
1343 add \$`16*2*$SZ`,$Tbl
1345 sub XOP_512_00_47 () {
1349 my @insns = (&$body,&$body); # 52 instructions
1351 &vpalignr ($t0,@X[1],@X[0],$SZ); # X[1..2]
1352 eval(shift(@insns));
1353 eval(shift(@insns));
1354 &vpalignr ($t3,@X[5],@X[4],$SZ); # X[9..10]
1355 eval(shift(@insns));
1356 eval(shift(@insns));
1357 &vprotq ($t1,$t0,8*$SZ-$sigma0[1]);
1358 eval(shift(@insns));
1359 eval(shift(@insns));
1360 &vpsrlq ($t0,$t0,$sigma0[2]);
1361 eval(shift(@insns));
1362 eval(shift(@insns));
1363 &vpaddq (@X[0],@X[0],$t3); # X[0..1] += X[9..10]
1364 eval(shift(@insns));
1365 eval(shift(@insns));
1366 eval(shift(@insns));
1367 eval(shift(@insns));
1368 &vprotq ($t2,$t1,$sigma0[1]-$sigma0[0]);
1369 eval(shift(@insns));
1370 eval(shift(@insns));
1371 &vpxor ($t0,$t0,$t1);
1372 eval(shift(@insns));
1373 eval(shift(@insns));
1374 eval(shift(@insns));
1375 eval(shift(@insns));
1376 &vprotq ($t3,@X[7],8*$SZ-$sigma1[1]);
1377 eval(shift(@insns));
1378 eval(shift(@insns));
1379 &vpxor ($t0,$t0,$t2); # sigma0(X[1..2])
1380 eval(shift(@insns));
1381 eval(shift(@insns));
1382 &vpsrlq ($t2,@X[7],$sigma1[2]);
1383 eval(shift(@insns));
1384 eval(shift(@insns));
1385 &vpaddq (@X[0],@X[0],$t0); # X[0..1] += sigma0(X[1..2])
1386 eval(shift(@insns));
1387 eval(shift(@insns));
1388 &vprotq ($t1,$t3,$sigma1[1]-$sigma1[0]);
1389 eval(shift(@insns));
1390 eval(shift(@insns));
1391 &vpxor ($t3,$t3,$t2);
1392 eval(shift(@insns));
1393 eval(shift(@insns));
1394 eval(shift(@insns));
1395 eval(shift(@insns));
1396 &vpxor ($t3,$t3,$t1); # sigma1(X[14..15])
1397 eval(shift(@insns));
1398 eval(shift(@insns));
1399 eval(shift(@insns));
1400 eval(shift(@insns));
1401 &vpaddq (@X[0],@X[0],$t3); # X[0..1] += sigma1(X[14..15])
1402 eval(shift(@insns));
1403 eval(shift(@insns));
1404 eval(shift(@insns));
1405 eval(shift(@insns));
1406 &vpaddq ($t2,@X[0],16*2*$j-0x80."($Tbl)");
1407 foreach (@insns) { eval; } # remaining instructions
1408 &vmovdqa (16*$j."(%rsp)",$t2);
1411 for ($i=0,$j=0; $j<8; $j++) {
1412 &XOP_512_00_47($j,\&body_00_15,@X);
1413 push(@X,shift(@X)); # rotate(@X)
1415 &cmpb ($SZ-1+16*2*$SZ-0x80."($Tbl)",0);
1416 &jne (".Lxop_00_47");
1418 for ($i=0; $i<16; ) {
1419 foreach(body_00_15()) { eval; }
1427 lea 16*$SZ($inp),$inp
1451 $code.=<<___ if ($win64);
1452 movaps 16*$SZ+32(%rsp),%xmm6
1453 movaps 16*$SZ+48(%rsp),%xmm7
1454 movaps 16*$SZ+64(%rsp),%xmm8
1455 movaps 16*$SZ+80(%rsp),%xmm9
1457 $code.=<<___ if ($win64 && $SZ>4);
1458 movaps 16*$SZ+96(%rsp),%xmm10
1459 movaps 16*$SZ+112(%rsp),%xmm11
1471 .size ${func}_xop,.-${func}_xop
1474 ######################################################################
1475 # AVX+shrd code path
1477 local *ror = sub { &shrd(@_[0],@_) };
1480 .type ${func}_avx,\@function,3
1490 mov %rsp,%r11 # copy %rsp
1491 shl \$4,%rdx # num*16
1492 sub \$`$framesz+$win64*16*($SZ==4?4:6)`,%rsp
1493 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
1494 and \$-64,%rsp # align stack frame
1495 mov $ctx,$_ctx # save ctx, 1st arg
1496 mov $inp,$_inp # save inp, 2nd arh
1497 mov %rdx,$_end # save end pointer, "3rd" arg
1498 mov %r11,$_rsp # save copy of %rsp
1500 $code.=<<___ if ($win64);
1501 movaps %xmm6,16*$SZ+32(%rsp)
1502 movaps %xmm7,16*$SZ+48(%rsp)
1503 movaps %xmm8,16*$SZ+64(%rsp)
1504 movaps %xmm9,16*$SZ+80(%rsp)
1506 $code.=<<___ if ($win64 && $SZ>4);
1507 movaps %xmm10,16*$SZ+96(%rsp)
1508 movaps %xmm11,16*$SZ+112(%rsp)
1523 if ($SZ==4) { # SHA256
1524 my @X = map("%xmm$_",(0..3));
1525 my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%xmm$_",(4..9));
1528 vmovdqa $TABLE+`$SZ*2*$rounds`+32(%rip),$t4
1529 vmovdqa $TABLE+`$SZ*2*$rounds`+64(%rip),$t5
1533 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1534 vmovdqu 0x00($inp),@X[0]
1535 vmovdqu 0x10($inp),@X[1]
1536 vmovdqu 0x20($inp),@X[2]
1537 vmovdqu 0x30($inp),@X[3]
1538 vpshufb $t3,@X[0],@X[0]
1539 lea $TABLE(%rip),$Tbl
1540 vpshufb $t3,@X[1],@X[1]
1541 vpshufb $t3,@X[2],@X[2]
1542 vpaddd 0x00($Tbl),@X[0],$t0
1543 vpshufb $t3,@X[3],@X[3]
1544 vpaddd 0x20($Tbl),@X[1],$t1
1545 vpaddd 0x40($Tbl),@X[2],$t2
1546 vpaddd 0x60($Tbl),@X[3],$t3
1547 vmovdqa $t0,0x00(%rsp)
1549 vmovdqa $t1,0x10(%rsp)
1551 vmovdqa $t2,0x20(%rsp)
1553 vmovdqa $t3,0x30(%rsp)
1559 sub \$`-16*2*$SZ`,$Tbl # size optimization
1561 sub Xupdate_256_AVX () {
1563 '&vpalignr ($t0,@X[1],@X[0],$SZ)', # X[1..4]
1564 '&vpalignr ($t3,@X[3],@X[2],$SZ)', # X[9..12]
1565 '&vpsrld ($t2,$t0,$sigma0[0]);',
1566 '&vpaddd (@X[0],@X[0],$t3)', # X[0..3] += X[9..12]
1567 '&vpsrld ($t3,$t0,$sigma0[2])',
1568 '&vpslld ($t1,$t0,8*$SZ-$sigma0[1]);',
1569 '&vpxor ($t0,$t3,$t2)',
1570 '&vpshufd ($t3,@X[3],0b11111010)',# X[14..15]
1571 '&vpsrld ($t2,$t2,$sigma0[1]-$sigma0[0]);',
1572 '&vpxor ($t0,$t0,$t1)',
1573 '&vpslld ($t1,$t1,$sigma0[1]-$sigma0[0]);',
1574 '&vpxor ($t0,$t0,$t2)',
1575 '&vpsrld ($t2,$t3,$sigma1[2]);',
1576 '&vpxor ($t0,$t0,$t1)', # sigma0(X[1..4])
1577 '&vpsrlq ($t3,$t3,$sigma1[0]);',
1578 '&vpaddd (@X[0],@X[0],$t0)', # X[0..3] += sigma0(X[1..4])
1579 '&vpxor ($t2,$t2,$t3);',
1580 '&vpsrlq ($t3,$t3,$sigma1[1]-$sigma1[0])',
1581 '&vpxor ($t2,$t2,$t3)',
1582 '&vpshufb ($t2,$t2,$t4)', # sigma1(X[14..15])
1583 '&vpaddd (@X[0],@X[0],$t2)', # X[0..1] += sigma1(X[14..15])
1584 '&vpshufd ($t3,@X[0],0b01010000)',# X[16..17]
1585 '&vpsrld ($t2,$t3,$sigma1[2])',
1586 '&vpsrlq ($t3,$t3,$sigma1[0])',
1587 '&vpxor ($t2,$t2,$t3);',
1588 '&vpsrlq ($t3,$t3,$sigma1[1]-$sigma1[0])',
1589 '&vpxor ($t2,$t2,$t3)',
1590 '&vpshufb ($t2,$t2,$t5)',
1591 '&vpaddd (@X[0],@X[0],$t2)' # X[2..3] += sigma1(X[16..17])
1595 sub AVX_256_00_47 () {
1599 my @insns = (&$body,&$body,&$body,&$body); # 104 instructions
1601 foreach (Xupdate_256_AVX()) { # 29 instructions
1603 eval(shift(@insns));
1604 eval(shift(@insns));
1605 eval(shift(@insns));
1607 &vpaddd ($t2,@X[0],16*2*$j."($Tbl)");
1608 foreach (@insns) { eval; } # remaining instructions
1609 &vmovdqa (16*$j."(%rsp)",$t2);
1612 for ($i=0,$j=0; $j<4; $j++) {
1613 &AVX_256_00_47($j,\&body_00_15,@X);
1614 push(@X,shift(@X)); # rotate(@X)
1616 &cmpb ($SZ-1+16*2*$SZ."($Tbl)",0);
1617 &jne (".Lavx_00_47");
1619 for ($i=0; $i<16; ) {
1620 foreach(body_00_15()) { eval; }
1624 my @X = map("%xmm$_",(0..7));
1625 my ($t0,$t1,$t2,$t3) = map("%xmm$_",(8..11));
1631 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1632 vmovdqu 0x00($inp),@X[0]
1633 lea $TABLE+0x80(%rip),$Tbl # size optimization
1634 vmovdqu 0x10($inp),@X[1]
1635 vmovdqu 0x20($inp),@X[2]
1636 vpshufb $t3,@X[0],@X[0]
1637 vmovdqu 0x30($inp),@X[3]
1638 vpshufb $t3,@X[1],@X[1]
1639 vmovdqu 0x40($inp),@X[4]
1640 vpshufb $t3,@X[2],@X[2]
1641 vmovdqu 0x50($inp),@X[5]
1642 vpshufb $t3,@X[3],@X[3]
1643 vmovdqu 0x60($inp),@X[6]
1644 vpshufb $t3,@X[4],@X[4]
1645 vmovdqu 0x70($inp),@X[7]
1646 vpshufb $t3,@X[5],@X[5]
1647 vpaddq -0x80($Tbl),@X[0],$t0
1648 vpshufb $t3,@X[6],@X[6]
1649 vpaddq -0x60($Tbl),@X[1],$t1
1650 vpshufb $t3,@X[7],@X[7]
1651 vpaddq -0x40($Tbl),@X[2],$t2
1652 vpaddq -0x20($Tbl),@X[3],$t3
1653 vmovdqa $t0,0x00(%rsp)
1654 vpaddq 0x00($Tbl),@X[4],$t0
1655 vmovdqa $t1,0x10(%rsp)
1656 vpaddq 0x20($Tbl),@X[5],$t1
1657 vmovdqa $t2,0x20(%rsp)
1658 vpaddq 0x40($Tbl),@X[6],$t2
1659 vmovdqa $t3,0x30(%rsp)
1660 vpaddq 0x60($Tbl),@X[7],$t3
1661 vmovdqa $t0,0x40(%rsp)
1663 vmovdqa $t1,0x50(%rsp)
1665 vmovdqa $t2,0x60(%rsp)
1667 vmovdqa $t3,0x70(%rsp)
1673 add \$`16*2*$SZ`,$Tbl
1675 sub Xupdate_512_AVX () {
1677 '&vpalignr ($t0,@X[1],@X[0],$SZ)', # X[1..2]
1678 '&vpalignr ($t3,@X[5],@X[4],$SZ)', # X[9..10]
1679 '&vpsrlq ($t2,$t0,$sigma0[0])',
1680 '&vpaddq (@X[0],@X[0],$t3);', # X[0..1] += X[9..10]
1681 '&vpsrlq ($t3,$t0,$sigma0[2])',
1682 '&vpsllq ($t1,$t0,8*$SZ-$sigma0[1]);',
1683 '&vpxor ($t0,$t3,$t2)',
1684 '&vpsrlq ($t2,$t2,$sigma0[1]-$sigma0[0]);',
1685 '&vpxor ($t0,$t0,$t1)',
1686 '&vpsllq ($t1,$t1,$sigma0[1]-$sigma0[0]);',
1687 '&vpxor ($t0,$t0,$t2)',
1688 '&vpsrlq ($t3,@X[7],$sigma1[2]);',
1689 '&vpxor ($t0,$t0,$t1)', # sigma0(X[1..2])
1690 '&vpsllq ($t2,@X[7],8*$SZ-$sigma1[1]);',
1691 '&vpaddq (@X[0],@X[0],$t0)', # X[0..1] += sigma0(X[1..2])
1692 '&vpsrlq ($t1,@X[7],$sigma1[0]);',
1693 '&vpxor ($t3,$t3,$t2)',
1694 '&vpsllq ($t2,$t2,$sigma1[1]-$sigma1[0]);',
1695 '&vpxor ($t3,$t3,$t1)',
1696 '&vpsrlq ($t1,$t1,$sigma1[1]-$sigma1[0]);',
1697 '&vpxor ($t3,$t3,$t2)',
1698 '&vpxor ($t3,$t3,$t1)', # sigma1(X[14..15])
1699 '&vpaddq (@X[0],@X[0],$t3)', # X[0..1] += sigma1(X[14..15])
1703 sub AVX_512_00_47 () {
1707 my @insns = (&$body,&$body); # 52 instructions
1709 foreach (Xupdate_512_AVX()) { # 23 instructions
1711 eval(shift(@insns));
1712 eval(shift(@insns));
1714 &vpaddq ($t2,@X[0],16*2*$j-0x80."($Tbl)");
1715 foreach (@insns) { eval; } # remaining instructions
1716 &vmovdqa (16*$j."(%rsp)",$t2);
1719 for ($i=0,$j=0; $j<8; $j++) {
1720 &AVX_512_00_47($j,\&body_00_15,@X);
1721 push(@X,shift(@X)); # rotate(@X)
1723 &cmpb ($SZ-1+16*2*$SZ-0x80."($Tbl)",0);
1724 &jne (".Lavx_00_47");
1726 for ($i=0; $i<16; ) {
1727 foreach(body_00_15()) { eval; }
1735 lea 16*$SZ($inp),$inp
1759 $code.=<<___ if ($win64);
1760 movaps 16*$SZ+32(%rsp),%xmm6
1761 movaps 16*$SZ+48(%rsp),%xmm7
1762 movaps 16*$SZ+64(%rsp),%xmm8
1763 movaps 16*$SZ+80(%rsp),%xmm9
1765 $code.=<<___ if ($win64 && $SZ>4);
1766 movaps 16*$SZ+96(%rsp),%xmm10
1767 movaps 16*$SZ+112(%rsp),%xmm11
1779 .size ${func}_avx,.-${func}_avx
1783 ######################################################################
1784 # AVX2+BMI code path
1786 my $a5=$SZ==4?"%esi":"%rsi"; # zap $inp
1790 sub bodyx_00_15 () {
1791 # at start $a1 should be zero, $a3 - $b^$c and $a4 copy of $f
1793 '($a,$b,$c,$d,$e,$f,$g,$h)=@ROT;'.
1795 '&add ($h,(32*($i/(16/$SZ))+$SZ*($i%(16/$SZ)))%$PUSH8.$base)', # h+=X[i]+K[i]
1796 '&and ($a4,$e)', # f&e
1797 '&rorx ($a0,$e,$Sigma1[2])',
1798 '&rorx ($a2,$e,$Sigma1[1])',
1800 '&lea ($a,"($a,$a1)")', # h+=Sigma0(a) from the past
1801 '&lea ($h,"($h,$a4)")',
1802 '&andn ($a4,$e,$g)', # ~e&g
1805 '&rorx ($a1,$e,$Sigma1[0])',
1806 '&lea ($h,"($h,$a4)")', # h+=Ch(e,f,g)=(e&f)+(~e&g)
1807 '&xor ($a0,$a1)', # Sigma1(e)
1810 '&rorx ($a4,$a,$Sigma0[2])',
1811 '&lea ($h,"($h,$a0)")', # h+=Sigma1(e)
1812 '&xor ($a2,$b)', # a^b, b^c in next round
1813 '&rorx ($a1,$a,$Sigma0[1])',
1815 '&rorx ($a0,$a,$Sigma0[0])',
1816 '&lea ($d,"($d,$h)")', # d+=h
1817 '&and ($a3,$a2)', # (b^c)&(a^b)
1820 '&xor ($a3,$b)', # Maj(a,b,c)=Ch(a^b,c,b)
1821 '&xor ($a1,$a0)', # Sigma0(a)
1822 '&lea ($h,"($h,$a3)");'. # h+=Maj(a,b,c)
1823 '&mov ($a4,$e)', # copy of f in future
1825 '($a2,$a3) = ($a3,$a2); unshift(@ROT,pop(@ROT)); $i++;'
1827 # and at the finish one has to $a+=$a1
1831 .type ${func}_avx2,\@function,3
1841 mov %rsp,%r11 # copy %rsp
1842 sub \$`2*$SZ*$rounds+4*8+$win64*16*($SZ==4?4:6)`,%rsp
1843 shl \$4,%rdx # num*16
1844 and \$-256*$SZ,%rsp # align stack frame
1845 lea ($inp,%rdx,$SZ),%rdx # inp+num*16*$SZ
1846 add \$`2*$SZ*($rounds-8)`,%rsp
1847 mov $ctx,$_ctx # save ctx, 1st arg
1848 mov $inp,$_inp # save inp, 2nd arh
1849 mov %rdx,$_end # save end pointer, "3rd" arg
1850 mov %r11,$_rsp # save copy of %rsp
1852 $code.=<<___ if ($win64);
1853 movaps %xmm6,16*$SZ+32(%rsp)
1854 movaps %xmm7,16*$SZ+48(%rsp)
1855 movaps %xmm8,16*$SZ+64(%rsp)
1856 movaps %xmm9,16*$SZ+80(%rsp)
1858 $code.=<<___ if ($win64 && $SZ>4);
1859 movaps %xmm10,16*$SZ+96(%rsp)
1860 movaps %xmm11,16*$SZ+112(%rsp)
1866 sub \$-16*$SZ,$inp # inp++, size optimization
1868 mov $inp,%r12 # borrow $T1
1870 cmp %rdx,$inp # $_end
1872 cmove %rsp,%r12 # next block or random data
1879 if ($SZ==4) { # SHA256
1880 my @X = map("%ymm$_",(0..3));
1881 my ($t0,$t1,$t2,$t3, $t4,$t5) = map("%ymm$_",(4..9));
1884 vmovdqa $TABLE+`$SZ*2*$rounds`+32(%rip),$t4
1885 vmovdqa $TABLE+`$SZ*2*$rounds`+64(%rip),$t5
1889 vmovdqa $TABLE+`$SZ*2*$rounds`(%rip),$t3
1890 vmovdqu -16*$SZ+0($inp),%xmm0
1891 vmovdqu -16*$SZ+16($inp),%xmm1
1892 vmovdqu -16*$SZ+32($inp),%xmm2
1893 vmovdqu -16*$SZ+48($inp),%xmm3
1894 #mov $inp,$_inp # offload $inp
1895 vinserti128 \$1,(%r12),@X[0],@X[0]
1896 vinserti128 \$1,16(%r12),@X[1],@X[1]
1897 vpshufb $t3,@X[0],@X[0]
1898 vinserti128 \$1,32(%r12),@X[2],@X[2]
1899 vpshufb $t3,@X[1],@X[1]
1900 vinserti128 \$1,48(%r12),@X[3],@X[3]
1902 lea $TABLE(%rip),$Tbl
1903 vpshufb $t3,@X[2],@X[2]
1904 vpaddd 0x00($Tbl),@X[0],$t0
1905 vpshufb $t3,@X[3],@X[3]
1906 vpaddd 0x20($Tbl),@X[1],$t1
1907 vpaddd 0x40($Tbl),@X[2],$t2
1908 vpaddd 0x60($Tbl),@X[3],$t3
1909 vmovdqa $t0,0x00(%rsp)
1911 vmovdqa $t1,0x20(%rsp)
1912 lea -$PUSH8(%rsp),%rsp
1914 vmovdqa $t2,0x00(%rsp)
1916 vmovdqa $t3,0x20(%rsp)
1918 sub \$-16*2*$SZ,$Tbl # size optimization
1925 sub AVX2_256_00_47 () {
1929 my @insns = (&$body,&$body,&$body,&$body); # 96 instructions
1930 my $base = "+2*$PUSH8(%rsp)";
1932 &lea ("%rsp","-$PUSH8(%rsp)") if (($j%2)==0);
1933 foreach (Xupdate_256_AVX()) { # 29 instructions
1935 eval(shift(@insns));
1936 eval(shift(@insns));
1937 eval(shift(@insns));
1939 &vpaddd ($t2,@X[0],16*2*$j."($Tbl)");
1940 foreach (@insns) { eval; } # remaining instructions
1941 &vmovdqa ((32*$j)%$PUSH8."(%rsp)",$t2);
1944 for ($i=0,$j=0; $j<4; $j++) {
1945 &AVX2_256_00_47($j,\&bodyx_00_15,@X);
1946 push(@X,shift(@X)); # rotate(@X)
1948 &lea ($Tbl,16*2*$SZ."($Tbl)");
1949 &cmpb (($SZ-1)."($Tbl)",0);
1950 &jne (".Lavx2_00_47");
1952 for ($i=0; $i<16; ) {
1953 my $base=$i<8?"+$PUSH8(%rsp)":"(%rsp)";
1954 foreach(bodyx_00_15()) { eval; }
1957 my @X = map("%ymm$_",(0..7));
1958 my ($t0,$t1,$t2,$t3) = map("%ymm$_",(8..11));
1964 vmovdqu -16*$SZ($inp),%xmm0
1965 vmovdqu -16*$SZ+16($inp),%xmm1
1966 vmovdqu -16*$SZ+32($inp),%xmm2
1967 lea $TABLE+0x80(%rip),$Tbl # size optimization
1968 vmovdqu -16*$SZ+48($inp),%xmm3
1969 vmovdqu -16*$SZ+64($inp),%xmm4
1970 vmovdqu -16*$SZ+80($inp),%xmm5
1971 vmovdqu -16*$SZ+96($inp),%xmm6
1972 vmovdqu -16*$SZ+112($inp),%xmm7
1973 #mov $inp,$_inp # offload $inp
1974 vmovdqa `$SZ*2*$rounds-0x80`($Tbl),$t2
1975 vinserti128 \$1,(%r12),@X[0],@X[0]
1976 vinserti128 \$1,16(%r12),@X[1],@X[1]
1977 vpshufb $t2,@X[0],@X[0]
1978 vinserti128 \$1,32(%r12),@X[2],@X[2]
1979 vpshufb $t2,@X[1],@X[1]
1980 vinserti128 \$1,48(%r12),@X[3],@X[3]
1981 vpshufb $t2,@X[2],@X[2]
1982 vinserti128 \$1,64(%r12),@X[4],@X[4]
1983 vpshufb $t2,@X[3],@X[3]
1984 vinserti128 \$1,80(%r12),@X[5],@X[5]
1985 vpshufb $t2,@X[4],@X[4]
1986 vinserti128 \$1,96(%r12),@X[6],@X[6]
1987 vpshufb $t2,@X[5],@X[5]
1988 vinserti128 \$1,112(%r12),@X[7],@X[7]
1990 vpaddq -0x80($Tbl),@X[0],$t0
1991 vpshufb $t2,@X[6],@X[6]
1992 vpaddq -0x60($Tbl),@X[1],$t1
1993 vpshufb $t2,@X[7],@X[7]
1994 vpaddq -0x40($Tbl),@X[2],$t2
1995 vpaddq -0x20($Tbl),@X[3],$t3
1996 vmovdqa $t0,0x00(%rsp)
1997 vpaddq 0x00($Tbl),@X[4],$t0
1998 vmovdqa $t1,0x20(%rsp)
1999 vpaddq 0x20($Tbl),@X[5],$t1
2000 vmovdqa $t2,0x40(%rsp)
2001 vpaddq 0x40($Tbl),@X[6],$t2
2002 vmovdqa $t3,0x60(%rsp)
2003 lea -$PUSH8(%rsp),%rsp
2004 vpaddq 0x60($Tbl),@X[7],$t3
2005 vmovdqa $t0,0x00(%rsp)
2007 vmovdqa $t1,0x20(%rsp)
2009 vmovdqa $t2,0x40(%rsp)
2011 vmovdqa $t3,0x60(%rsp)
2020 sub AVX2_512_00_47 () {
2024 my @insns = (&$body,&$body); # 48 instructions
2025 my $base = "+2*$PUSH8(%rsp)";
2027 &lea ("%rsp","-$PUSH8(%rsp)") if (($j%4)==0);
2028 foreach (Xupdate_512_AVX()) { # 23 instructions
2031 eval(shift(@insns));
2032 eval(shift(@insns));
2033 eval(shift(@insns));
2036 &vpaddq ($t2,@X[0],16*2*$j-0x80."($Tbl)");
2037 foreach (@insns) { eval; } # remaining instructions
2038 &vmovdqa ((32*$j)%$PUSH8."(%rsp)",$t2);
2041 for ($i=0,$j=0; $j<8; $j++) {
2042 &AVX2_512_00_47($j,\&bodyx_00_15,@X);
2043 push(@X,shift(@X)); # rotate(@X)
2045 &lea ($Tbl,16*2*$SZ."($Tbl)");
2046 &cmpb (($SZ-1-0x80)."($Tbl)",0);
2047 &jne (".Lavx2_00_47");
2049 for ($i=0; $i<16; ) {
2050 my $base=$i<8?"+$PUSH8(%rsp)":"(%rsp)";
2051 foreach(bodyx_00_15()) { eval; }
2055 mov `2*$SZ*$rounds`(%rsp),$ctx # $_ctx
2057 #mov `2*$SZ*$rounds+8`(%rsp),$inp # $_inp
2058 lea `2*$SZ*($rounds-8)`(%rsp),$Tbl
2078 cmp `$PUSH8+2*8`($Tbl),$inp # $_end
2089 for ($i=0; $i<8; ) {
2090 my $base="+16($Tbl)";
2091 foreach(bodyx_00_15()) { eval; }
2094 lea -$PUSH8($Tbl),$Tbl
2098 mov `2*$SZ*$rounds`(%rsp),$ctx # $_ctx
2100 #mov `2*$SZ*$rounds+8`(%rsp),$inp # $_inp
2101 lea `2*$SZ*($rounds-8)`(%rsp),%rsp
2109 lea `2*16*$SZ`($inp),$inp # inp+=2
2116 cmove %rsp,%r12 # next block or stale data
2133 $code.=<<___ if ($win64);
2134 movaps 16*$SZ+32(%rsp),%xmm6
2135 movaps 16*$SZ+48(%rsp),%xmm7
2136 movaps 16*$SZ+64(%rsp),%xmm8
2137 movaps 16*$SZ+80(%rsp),%xmm9
2139 $code.=<<___ if ($win64 && $SZ>4);
2140 movaps 16*$SZ+96(%rsp),%xmm10
2141 movaps 16*$SZ+112(%rsp),%xmm11
2153 .size ${func}_avx2,.-${func}_avx2
2158 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
2159 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
2167 .extern __imp_RtlVirtualUnwind
2168 .type se_handler,\@abi-omnipotent
2182 mov 120($context),%rax # pull context->Rax
2183 mov 248($context),%rbx # pull context->Rip
2185 mov 8($disp),%rsi # disp->ImageBase
2186 mov 56($disp),%r11 # disp->HanderlData
2188 mov 0(%r11),%r10d # HandlerData[0]
2189 lea (%rsi,%r10),%r10 # prologue label
2190 cmp %r10,%rbx # context->Rip<prologue label
2193 mov 152($context),%rax # pull context->Rsp
2195 mov 4(%r11),%r10d # HandlerData[1]
2196 lea (%rsi,%r10),%r10 # epilogue label
2197 cmp %r10,%rbx # context->Rip>=epilogue label
2200 $code.=<<___ if ($avx>1);
2201 lea .Lavx2_shortcut(%rip),%r10
2202 cmp %r10,%rbx # context->Rip<avx2_shortcut
2206 add \$`2*$SZ*($rounds-8)`,%rax
2210 mov %rax,%rsi # put aside Rsp
2211 mov 16*$SZ+3*8(%rax),%rax # pull $_rsp
2220 mov %rbx,144($context) # restore context->Rbx
2221 mov %rbp,160($context) # restore context->Rbp
2222 mov %r12,216($context) # restore context->R12
2223 mov %r13,224($context) # restore context->R13
2224 mov %r14,232($context) # restore context->R14
2225 mov %r15,240($context) # restore context->R15
2227 lea .Lepilogue(%rip),%r10
2229 jb .Lin_prologue # non-AVX code
2231 lea 16*$SZ+4*8(%rsi),%rsi # Xmm6- save area
2232 lea 512($context),%rdi # &context.Xmm6
2233 mov \$`$SZ==4?8:12`,%ecx
2234 .long 0xa548f3fc # cld; rep movsq
2239 mov %rax,152($context) # restore context->Rsp
2240 mov %rsi,168($context) # restore context->Rsi
2241 mov %rdi,176($context) # restore context->Rdi
2243 mov 40($disp),%rdi # disp->ContextRecord
2244 mov $context,%rsi # context
2245 mov \$154,%ecx # sizeof(CONTEXT)
2246 .long 0xa548f3fc # cld; rep movsq
2249 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
2250 mov 8(%rsi),%rdx # arg2, disp->ImageBase
2251 mov 0(%rsi),%r8 # arg3, disp->ControlPc
2252 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
2253 mov 40(%rsi),%r10 # disp->ContextRecord
2254 lea 56(%rsi),%r11 # &disp->HandlerData
2255 lea 24(%rsi),%r12 # &disp->EstablisherFrame
2256 mov %r10,32(%rsp) # arg5
2257 mov %r11,40(%rsp) # arg6
2258 mov %r12,48(%rsp) # arg7
2259 mov %rcx,56(%rsp) # arg8, (NULL)
2260 call *__imp_RtlVirtualUnwind(%rip)
2262 mov \$1,%eax # ExceptionContinueSearch
2274 .size se_handler,.-se_handler
2277 $code.=<<___ if ($SZ==4 && $shaext);
2278 .type shaext_handler,\@abi-omnipotent
2292 mov 120($context),%rax # pull context->Rax
2293 mov 248($context),%rbx # pull context->Rip
2295 lea .Lprologue_shaext(%rip),%r10
2296 cmp %r10,%rbx # context->Rip<.Lprologue
2299 lea .Lepilogue_shaext(%rip),%r10
2300 cmp %r10,%rbx # context->Rip>=.Lepilogue
2303 lea -8-5*16(%rax),%rsi
2304 lea 512($context),%rdi # &context.Xmm6
2306 .long 0xa548f3fc # cld; rep movsq
2309 .size shaext_handler,.-shaext_handler
2315 .rva .LSEH_begin_$func
2316 .rva .LSEH_end_$func
2317 .rva .LSEH_info_$func
2319 $code.=<<___ if ($SZ==4 && $shaext);
2320 .rva .LSEH_begin_${func}_shaext
2321 .rva .LSEH_end_${func}_shaext
2322 .rva .LSEH_info_${func}_shaext
2324 $code.=<<___ if ($SZ==4);
2325 .rva .LSEH_begin_${func}_ssse3
2326 .rva .LSEH_end_${func}_ssse3
2327 .rva .LSEH_info_${func}_ssse3
2329 $code.=<<___ if ($avx && $SZ==8);
2330 .rva .LSEH_begin_${func}_xop
2331 .rva .LSEH_end_${func}_xop
2332 .rva .LSEH_info_${func}_xop
2334 $code.=<<___ if ($avx);
2335 .rva .LSEH_begin_${func}_avx
2336 .rva .LSEH_end_${func}_avx
2337 .rva .LSEH_info_${func}_avx
2339 $code.=<<___ if ($avx>1);
2340 .rva .LSEH_begin_${func}_avx2
2341 .rva .LSEH_end_${func}_avx2
2342 .rva .LSEH_info_${func}_avx2
2350 .rva .Lprologue,.Lepilogue # HandlerData[]
2352 $code.=<<___ if ($SZ==4 && $shaext);
2353 .LSEH_info_${func}_shaext:
2357 $code.=<<___ if ($SZ==4);
2358 .LSEH_info_${func}_ssse3:
2361 .rva .Lprologue_ssse3,.Lepilogue_ssse3 # HandlerData[]
2363 $code.=<<___ if ($avx && $SZ==8);
2364 .LSEH_info_${func}_xop:
2367 .rva .Lprologue_xop,.Lepilogue_xop # HandlerData[]
2369 $code.=<<___ if ($avx);
2370 .LSEH_info_${func}_avx:
2373 .rva .Lprologue_avx,.Lepilogue_avx # HandlerData[]
2375 $code.=<<___ if ($avx>1);
2376 .LSEH_info_${func}_avx2:
2379 .rva .Lprologue_avx2,.Lepilogue_avx2 # HandlerData[]
2386 "sha256rnds2" => 0xcb,
2387 "sha256msg1" => 0xcc,
2388 "sha256msg2" => 0xcd );
2390 if (defined($opcodelet{$instr}) && @_[0] =~ /%xmm([0-7]),\s*%xmm([0-7])/) {
2391 my @opcode=(0x0f,0x38);
2392 push @opcode,$opcodelet{$instr};
2393 push @opcode,0xc0|($1&7)|(($2&7)<<3); # ModR/M
2394 return ".byte\t".join(',',@opcode);
2396 return $instr."\t".@_[0];
2400 foreach (split("\n",$code)) {
2401 s/\`([^\`]*)\`/eval $1/geo;
2403 s/\b(sha256[^\s]*)\s+(.*)/sha256op38($1,$2)/geo;