3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # SHA256 block transform for x86. September 2007.
12 # Performance improvement over compiler generated code varies from
13 # 10% to 40% [see below]. Not very impressive on some ยต-archs, but
14 # it's 5 times smaller and optimizies amount of writes.
18 # Optimization including two of Pavel Semjanov's ideas, alternative
19 # Maj and full unroll, resulted in ~20-25% improvement on most CPUs,
20 # ~7% on Pentium, ~40% on Atom. As fully unrolled loop body is almost
21 # 15x larger, 8KB vs. 560B, it's fired only for longer inputs. But not
22 # on P4, where it kills performance, nor Sandy Bridge, where folded
23 # loop is approximately as fast...
27 # Add AMD XOP-specific code path, >30% improvement on Bulldozer over
28 # May version, >60% over original. Add AVX+shrd code path, >25%
29 # improvement on Sandy Bridge over May version, 60% over original.
31 # Performance in clock cycles per processed byte (less is better):
33 # PIII P4 AMD K8 Core2 SB Atom Bldzr
34 # gcc 36 41 27 26 25 50 36
35 # icc 33 38 25 23 - - -
36 # x86 asm(*) 27/24 28 19/15.5 18/15.6 12.3 30/25 16.6
37 # x86_64 asm(**) 17.5 15.1 13.9 11.6 22 13.7
39 # (*) numbers after slash are for unrolled loop, where available,
40 # otherwise best applicable such as AVX/XOP;
41 # (**) x86_64 assembly performance is presented for reference
44 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
45 push(@INC,"${dir}","${dir}../../perlasm");
48 &asm_init($ARGV[0],"sha512-586.pl",$ARGV[$#ARGV] eq "386");
51 for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }
54 `$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
55 =~ /GNU assembler version ([2-9]\.[0-9]+)/ &&
56 $1>=2.19); # first version supporting AVX
58 $ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32n" &&
59 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/ &&
60 $1>=2.03); # first version supporting AVX
62 $ymm=1 if ($xmm && !$ymm && $ARGV[0] eq "win32" &&
63 `ml 2>&1` =~ /Version ([0-9]+)\./ &&
64 $1>=10); # first version supporting AVX
66 $unroll_after = 64*4; # If pre-evicted from L1P cache first spin of
67 # fully unrolled loop was measured to run about
68 # 3-4x slower. If slowdown coefficient is N and
69 # unrolled loop is m times faster, then you break
70 # even at (N-1)/(m-1) blocks. Then it needs to be
71 # adjusted for probability of code being evicted,
72 # code size/cache size=1/4. Typical m is 1.15...
89 &mov ($T,"ecx"); # "ecx" is preloaded
90 &mov ("esi",&DWP(4*(9+15+16-14),"esp"));
98 &xor ($T,"ecx"); # T = sigma0(X[-15])
100 &add ($T,&DWP(4*(9+15+16),"esp")); # T += X[-16]
102 &add ($T,&DWP(4*(9+15+16-9),"esp")); # T += X[-7]
103 #&xor ("edi","esi") # sigma1(X[-2])
104 # &add ($T,"edi"); # T += sigma1(X[-2])
105 # &mov (&DWP(4*(9+15),"esp"),$T); # save X[0]
113 &xor ("edi","esi") if ($in_16_63); # sigma1(X[-2])
116 &add ($T,"edi") if ($in_16_63); # T += sigma1(X[-2])
120 &mov ($T,&DWP(4*(9+15),"esp")) if (!$in_16_63);
121 &mov (&DWP(4*(9+15),"esp"),$T) if ($in_16_63); # save X[0]
124 &mov ($Eoff,$E); # modulo-scheduled
126 &add ($T,$Hoff); # T += h
127 &xor ("esi","edi"); # Ch(e,f,g)
128 &ror ($E,6); # Sigma1(e)
130 &add ($T,"esi"); # T += Ch(e,f,g)
133 &add ($T,$E); # T += Sigma1(e)
136 &mov ($Aoff,$A); # modulo-scheduled
137 &lea ("esp",&DWP(-4,"esp"));
139 &mov ("esi",&DWP(0,$K256));
141 &mov ($E,$Eoff); # e in next iteration, d in this one
142 &xor ($A,"edi"); # a ^= b
143 &ror ("ecx",2); # Sigma0(a)
145 &add ($T,"esi"); # T+= K[i]
146 &mov (&DWP(0,"esp"),$A); # (b^c) in next round
147 &add ($E,$T); # d += T
148 &and ($A,&DWP(4,"esp")); # a &= (b^c)
149 &add ($T,"ecx"); # T += Sigma0(a)
150 &xor ($A,"edi"); # h = Maj(a,b,c) = Ch(a^b,c,b)
151 &mov ("ecx",&DWP(4*(9+15+16-1),"esp")) if ($in_16_63); # preload T
153 &add ($A,$T); # h += T
156 &external_label("OPENSSL_ia32cap_P") if (!$i386);
158 &function_begin("sha256_block_data_order");
159 &mov ("esi",wparam(0)); # ctx
160 &mov ("edi",wparam(1)); # inp
161 &mov ("eax",wparam(2)); # num
162 &mov ("ebx","esp"); # saved sp
164 &call (&label("pic_point")); # make it PIC!
165 &set_label("pic_point");
167 &lea ($K256,&DWP(&label("K256")."-".&label("pic_point"),$K256));
174 &mov (&DWP(0,"esp"),"esi"); # ctx
175 &mov (&DWP(4,"esp"),"edi"); # inp
176 &mov (&DWP(8,"esp"),"eax"); # inp+num*128
177 &mov (&DWP(12,"esp"),"ebx"); # saved sp
179 &picmeup("edx","OPENSSL_ia32cap_P",$K256,&label("K256"));
180 &mov ("ecx",&DWP(0,"edx"));
181 &mov ("edx",&DWP(4,"edx"));
182 &test ("ecx",1<<20); # check for P4
183 &jnz (&label("loop"));
184 &test ("edx",1<<11); # check for XOP
185 &jnz (&label("XOP")) if ($ymm);
186 &and ("ecx",1<<30); # mask "Intel CPU" bit
187 &and ("edx",1<<28); # mask AVX bit
189 &cmp ("ecx",1<<28|1<<30);
190 &je (&label("AVX")) if ($ymm);
191 &je (&label("loop_shrd")) if (!$ymm);
194 &cmp ("eax",$unroll_after);
195 &jae (&label("unrolled"));
197 &jmp (&label("loop"));
202 &set_label("loop$suffix",16);
203 # copy input block to stack reversing byte and dword order
204 for($i=0;$i<4;$i++) {
205 &mov ("eax",&DWP($i*16+0,"edi"));
206 &mov ("ebx",&DWP($i*16+4,"edi"));
207 &mov ("ecx",&DWP($i*16+8,"edi"));
209 &mov ("edx",&DWP($i*16+12,"edi"));
219 &lea ("esp",&DWP(-4*9,"esp"));# place for A,B,C,D,E,F,G,H
220 &mov (&DWP(4*(9+16)+4,"esp"),"edi");
222 # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack
223 &mov ($A,&DWP(0,"esi"));
224 &mov ("ebx",&DWP(4,"esi"));
225 &mov ("ecx",&DWP(8,"esi"));
226 &mov ("edi",&DWP(12,"esi"));
232 &mov (&DWP(0,"esp"),"ebx"); # magic
233 &mov ($E,&DWP(16,"esi"));
234 &mov ("ebx",&DWP(20,"esi"));
235 &mov ("ecx",&DWP(24,"esi"));
236 &mov ("edi",&DWP(28,"esi"));
242 &set_label("00_15$suffix",16);
246 &cmp ("esi",0xc19bf174);
247 &jne (&label("00_15$suffix"));
249 &mov ("ecx",&DWP(4*(9+15+16-1),"esp")); # preloaded in BODY_00_15(1)
250 &jmp (&label("16_63$suffix"));
252 &set_label("16_63$suffix",16);
256 &cmp ("esi",0xc67178f2);
257 &jne (&label("16_63$suffix"));
259 &mov ("esi",&DWP(4*(9+16+64)+0,"esp"));#ctx
262 # &mov ("edi",$Coff);
264 &add ($A,&DWP(0,"esi"));
265 &add ("ebx",&DWP(4,"esi"));
266 &add ("edi",&DWP(8,"esi"));
267 &add ("ecx",&DWP(12,"esi"));
268 &mov (&DWP(0,"esi"),$A);
269 &mov (&DWP(4,"esi"),"ebx");
270 &mov (&DWP(8,"esi"),"edi");
271 &mov (&DWP(12,"esi"),"ecx");
276 &mov ("edi",&DWP(4*(9+16+64)+4,"esp"));#inp
277 &add ($E,&DWP(16,"esi"));
278 &add ("eax",&DWP(20,"esi"));
279 &add ("ebx",&DWP(24,"esi"));
280 &add ("ecx",&DWP(28,"esi"));
281 &mov (&DWP(16,"esi"),$E);
282 &mov (&DWP(20,"esi"),"eax");
283 &mov (&DWP(24,"esi"),"ebx");
284 &mov (&DWP(28,"esi"),"ecx");
286 &lea ("esp",&DWP(4*(9+16+64),"esp"));# destroy frame
287 &sub ($K256,4*64); # rewind K
289 &cmp ("edi",&DWP(8,"esp")); # are we done yet?
290 &jb (&label("loop$suffix"));
293 &mov ("esp",&DWP(12,"esp")); # restore sp
295 if (!$i386 && !$ymm) {
296 # ~20% improvement on Sandy Bridge
297 local *ror = sub { &shrd(@_[0],@_) };
298 &COMPACT_LOOP("_shrd");
299 &mov ("esp",&DWP(12,"esp")); # restore sp
303 &set_label("K256",64); # Yes! I keep it in the code segment!
304 @K256=( 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,
305 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,
306 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,
307 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,
308 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,
309 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,
310 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,
311 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,
312 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,
313 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,
314 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,
315 0xd192e819,0xd6990624,0xf40e3585,0x106aa070,
316 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,
317 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,
318 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,
319 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 );
321 &data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f);
323 if (!$i386 && $unroll_after) {
326 &set_label("unrolled",16);
327 &lea ("esp",&DWP(-96,"esp"));
328 # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack
329 &mov ($AH[0],&DWP(0,"esi"));
330 &mov ($AH[1],&DWP(4,"esi"));
331 &mov ("ecx",&DWP(8,"esi"));
332 &mov ("ebx",&DWP(12,"esi"));
333 #&mov (&DWP(0,"esp"),$AH[0]);
334 &mov (&DWP(4,"esp"),$AH[1]);
335 &xor ($AH[1],"ecx"); # magic
336 &mov (&DWP(8,"esp"),"ecx");
337 &mov (&DWP(12,"esp"),"ebx");
338 &mov ($E,&DWP(16,"esi"));
339 &mov ("ebx",&DWP(20,"esi"));
340 &mov ("ecx",&DWP(24,"esi"));
341 &mov ("esi",&DWP(28,"esi"));
342 #&mov (&DWP(16,"esp"),$E);
343 &mov (&DWP(20,"esp"),"ebx");
344 &mov (&DWP(24,"esp"),"ecx");
345 &mov (&DWP(28,"esp"),"esi");
346 &jmp (&label("grand_loop"));
348 &set_label("grand_loop",16);
349 # copy input block to stack reversing byte order
350 for($i=0;$i<5;$i++) {
351 &mov ("ebx",&DWP(12*$i+0,"edi"));
352 &mov ("ecx",&DWP(12*$i+4,"edi"));
354 &mov ("esi",&DWP(12*$i+8,"edi"));
356 &mov (&DWP(32+12*$i+0,"esp"),"ebx");
358 &mov (&DWP(32+12*$i+4,"esp"),"ecx");
359 &mov (&DWP(32+12*$i+8,"esp"),"esi");
361 &mov ("ebx",&DWP($i*12,"edi"));
364 &mov (&DWP(96+4,"esp"),"edi");
365 &mov (&DWP(32+12*$i,"esp"),"ebx");
367 my ($t1,$t2) = ("ecx","esi");
368 my ($a,$b,$c,$d,$e,$f,$g,$h)=(0..7); # offsets
369 sub off { &DWP(4*(((shift)-$i)&7),"esp"); }
371 for ($i=0;$i<64;$i++) {
374 &mov ($T,$t1); # $t1 is preloaded
375 # &mov ($t2,&DWP(32+4*(($i+14)&15),"esp"));
383 &xor ($T,$t1); # T = sigma0(X[-15])
385 &add ($T,&DWP(32+4*($i&15),"esp")); # T += X[-16]
387 &add ($T,&DWP(32+4*(($i+9)&15),"esp")); # T += X[-7]
388 #&xor ("edi",$t2) # sigma1(X[-2])
389 # &add ($T,"edi"); # T += sigma1(X[-2])
390 # &mov (&DWP(4*(9+15),"esp"),$T); # save X[0]
393 &xor ("edi",$t2) if ($i>=16); # sigma1(X[-2])
396 &add ($T,"edi") if ($i>=16); # T += sigma1(X[-2])
397 &mov ("edi",&off($g));
399 &mov ($T,&DWP(32+4*($i&15),"esp")) if ($i<16); # X[i]
400 &mov (&DWP(32+4*($i&15),"esp"),$T) if ($i>=16 && $i<62); # save X[0]
404 &mov (&off($e),$t1); # save $E, modulo-scheduled
406 &add ($T,&off($h)); # T += h
407 &xor ("edi",$t2); # Ch(e,f,g)
408 &ror ($E,6); # Sigma1(e)
410 &add ($T,"edi"); # T += Ch(e,f,g)
414 &mov ("edi",&off($b));
416 &mov (&off($a),$AH[0]); # save $A, modulo-scheduled
417 &xor ($AH[0],"edi"); # a ^= b, (b^c) in next round
419 &and ($AH[1],$AH[0]); # (b^c) &= (a^b)
420 &lea ($E,&DWP(@K256[$i],$T,$E)); # T += Sigma1(1)+K[i]
422 &xor ($AH[1],"edi"); # h = Maj(a,b,c) = Ch(a^b,c,b)
423 &mov ($t2,&DWP(32+4*(($i+2)&15),"esp")) if ($i>=15 && $i<63);
424 &ror ($t1,2); # Sigma0(a)
426 &add ($AH[1],$E); # h += T
427 &add ($E,&off($d)); # d += T
428 &add ($AH[1],$t1); # h += Sigma0(a)
429 &mov ($t1,&DWP(32+4*(($i+15)&15),"esp")) if ($i>=15 && $i<63);
431 @AH = reverse(@AH); # rotate(a,h)
432 ($t1,$t2) = ($t2,$t1); # rotate(t1,t2)
434 &mov ("esi",&DWP(96,"esp")); #ctx
435 #&mov ($AH[0],&DWP(0,"esp"));
436 &xor ($AH[1],"edi"); #&mov ($AH[1],&DWP(4,"esp"));
437 #&mov ("edi", &DWP(8,"esp"));
438 &mov ("ecx",&DWP(12,"esp"));
439 &add ($AH[0],&DWP(0,"esi"));
440 &add ($AH[1],&DWP(4,"esi"));
441 &add ("edi",&DWP(8,"esi"));
442 &add ("ecx",&DWP(12,"esi"));
443 &mov (&DWP(0,"esi"),$AH[0]);
444 &mov (&DWP(4,"esi"),$AH[1]);
445 &mov (&DWP(8,"esi"),"edi");
446 &mov (&DWP(12,"esi"),"ecx");
447 #&mov (&DWP(0,"esp"),$AH[0]);
448 &mov (&DWP(4,"esp"),$AH[1]);
449 &xor ($AH[1],"edi"); # magic
450 &mov (&DWP(8,"esp"),"edi");
451 &mov (&DWP(12,"esp"),"ecx");
452 #&mov ($E,&DWP(16,"esp"));
453 &mov ("edi",&DWP(20,"esp"));
454 &mov ("ebx",&DWP(24,"esp"));
455 &mov ("ecx",&DWP(28,"esp"));
456 &add ($E,&DWP(16,"esi"));
457 &add ("edi",&DWP(20,"esi"));
458 &add ("ebx",&DWP(24,"esi"));
459 &add ("ecx",&DWP(28,"esi"));
460 &mov (&DWP(16,"esi"),$E);
461 &mov (&DWP(20,"esi"),"edi");
462 &mov (&DWP(24,"esi"),"ebx");
463 &mov (&DWP(28,"esi"),"ecx");
464 #&mov (&DWP(16,"esp"),$E);
465 &mov (&DWP(20,"esp"),"edi");
466 &mov ("edi",&DWP(96+4,"esp")); # inp
467 &mov (&DWP(24,"esp"),"ebx");
468 &mov (&DWP(28,"esp"),"ecx");
470 &cmp ("edi",&DWP(96+8,"esp")); # are we done yet?
471 &jb (&label("grand_loop"));
473 &mov ("esp",&DWP(96+12,"esp")); # restore sp
477 my @X = map("xmm$_",(0..3));
478 my ($t0,$t1,$t2,$t3) = map("xmm$_",(4..7));
481 &set_label("XOP",16);
482 &lea ("esp",&DWP(-96,"esp"));
484 # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack
485 &mov ($AH[0],&DWP(0,"esi"));
486 &mov ($AH[1],&DWP(4,"esi"));
487 &mov ("ecx",&DWP(8,"esi"));
488 &mov ("edi",&DWP(12,"esi"));
489 #&mov (&DWP(0,"esp"),$AH[0]);
490 &mov (&DWP(4,"esp"),$AH[1]);
491 &xor ($AH[1],"ecx"); # magic
492 &mov (&DWP(8,"esp"),"ecx");
493 &mov (&DWP(12,"esp"),"edi");
494 &mov ($E,&DWP(16,"esi"));
495 &mov ("edi",&DWP(20,"esi"));
496 &mov ("ecx",&DWP(24,"esi"));
497 &mov ("esi",&DWP(28,"esi"));
498 #&mov (&DWP(16,"esp"),$E);
499 &mov (&DWP(20,"esp"),"edi");
500 &mov ("edi",&DWP(96+4,"esp")); # inp
501 &mov (&DWP(24,"esp"),"ecx");
502 &mov (&DWP(28,"esp"),"esi");
503 &vmovdqa ($t3,&QWP(256,$K256));
504 &jmp (&label("grand_xop"));
506 &set_label("grand_xop",16);
507 # load input, reverse byte order, add K256[0..15], save to stack
508 &vmovdqu (@X[0],&QWP(0,"edi"));
509 &vmovdqu (@X[1],&QWP(16,"edi"));
510 &vmovdqu (@X[2],&QWP(32,"edi"));
511 &vmovdqu (@X[3],&QWP(48,"edi"));
513 &vpshufb (@X[0],@X[0],$t3);
514 &mov (&DWP(96+4,"esp"),"edi");
515 &vpshufb (@X[1],@X[1],$t3);
516 &vpshufb (@X[2],@X[2],$t3);
517 &vpaddd ($t0,@X[0],&QWP(0,$K256));
518 &vpshufb (@X[3],@X[3],$t3);
519 &vpaddd ($t1,@X[1],&QWP(16,$K256));
520 &vpaddd ($t2,@X[2],&QWP(32,$K256));
521 &vpaddd ($t3,@X[3],&QWP(48,$K256));
522 &vmovdqa (&QWP(32+0,"esp"),$t0);
523 &vmovdqa (&QWP(32+16,"esp"),$t1);
524 &vmovdqa (&QWP(32+32,"esp"),$t2);
525 &vmovdqa (&QWP(32+48,"esp"),$t3);
526 &jmp (&label("xop_00_47"));
528 &set_label("xop_00_47",16);
535 my @insns = (&$body,&$body,&$body,&$body); # 120 instructions
537 &vpalignr ($t0,@X[1],@X[0],4); # X[1..4]
540 &vpalignr ($t3,@X[3],@X[2],4); # X[9..12]
543 &vprotd ($t1,$t0,14);
547 &vpaddd (@X[0],@X[0],$t3); # X[0..3] += X[9..12]
552 &vprotd ($t2,$t1,25-14);
553 &vpxor ($t0,$t0,$t1);
558 &vprotd ($t3,@X[3],13);
559 &vpxor ($t0,$t0,$t2); # sigma0(X[1..4])
562 &vpsrld ($t2,@X[3],10);
565 &vpaddd (@X[0],@X[0],$t0); # X[0..3] += sigma0(X[1..4])
568 &vprotd ($t1,$t3,15-13);
569 &vpxor ($t3,$t3,$t2);
574 &vpxor ($t3,$t3,$t1); # sigma1(X[14..15])
579 &vpsrldq ($t3,$t3,8);
584 &vpaddd (@X[0],@X[0],$t3); # X[0..1] += sigma1(X[14..15])
589 &vprotd ($t3,@X[0],13);
592 &vpsrld ($t2,@X[0],10);
595 &vprotd ($t1,$t3,15-13);
598 &vpxor ($t3,$t3,$t2);
603 &vpxor ($t3,$t3,$t1); # sigma1(X[16..17])
608 &vpslldq ($t3,$t3,8); # 22 instructions
613 &vpaddd (@X[0],@X[0],$t3); # X[2..3] += sigma1(X[16..17])
618 &vpaddd ($t2,@X[0],&QWP(16*$j,$K256));
620 foreach (@insns) { eval; } # remaining instructions
622 &vmovdqa (&QWP(32+16*$j,"esp"),$t2);
628 '&mov ("esi",&off($f));',
630 '&mov ("edi",&off($g));',
632 '&xor ("esi","edi");',
634 '&and ("esi","ecx");',
635 '&mov (&off($e),"ecx");', # save $E, modulo-scheduled
637 '&xor ("edi","esi");', # Ch(e,f,g)
638 '&ror ($E,6);', # T = Sigma1(e)
639 '&mov ("ecx",$AH[0]);',
640 '&mov ("esi",$AH[0]);',
641 '&add ($E,&off($h));', # T += h
643 '&ror ("ecx",22-13);',
644 '&add ($E,"edi");', # T += Ch(e,f,g)
645 '&mov ("edi",&off($b));',
646 '&xor ("ecx",$AH[0]);',
647 '&mov (&off($a),$AH[0]);', # save $A, modulo-scheduled
648 '&xor ($AH[0],"edi");', # a ^= b, (b^c) in next round
649 '&ror ("ecx",13-2);',
650 '&and ($AH[1],$AH[0]);', # (b^c) &= (a^b)
651 '&add ($E,&DWP(32+4*($i&15),"esp"));', # T += K[i]+X[i]
652 '&xor ("ecx","esi");',
653 '&xor ($AH[1],"edi");', # h = Maj(a,b,c) = Ch(a^b,c,b)
654 '&ror ("ecx",2);', # Sigma0(a)
656 '&add ($AH[1],$E);', # h += T
657 '&add ($E,&off($d));', # d += T
658 '&add ($AH[1],"ecx");'. # h += Sigma0(a)
660 '@AH = reverse(@AH); $i++;' # rotate(a,h)
664 for ($i=0,$j=0; $j<4; $j++) {
665 &XOP_00_47($j,\&body_00_15,@X);
666 push(@X,shift(@X)); # rotate(@X)
668 &cmp (&DWP(16*$j,$K256),0x00010203);
669 &jne (&label("xop_00_47"));
671 for ($i=0; $i<16; ) {
672 foreach(body_00_15()) { eval; }
675 &mov ("esi",&DWP(96,"esp")); #ctx
676 #&mov ($AH[0],&DWP(0,"esp"));
677 &xor ($AH[1],"edi"); #&mov ($AH[1],&DWP(4,"esp"));
678 #&mov ("edi", &DWP(8,"esp"));
679 &mov ("ecx",&DWP(12,"esp"));
680 &add ($AH[0],&DWP(0,"esi"));
681 &add ($AH[1],&DWP(4,"esi"));
682 &add ("edi",&DWP(8,"esi"));
683 &add ("ecx",&DWP(12,"esi"));
684 &mov (&DWP(0,"esi"),$AH[0]);
685 &mov (&DWP(4,"esi"),$AH[1]);
686 &mov (&DWP(8,"esi"),"edi");
687 &mov (&DWP(12,"esi"),"ecx");
688 #&mov (&DWP(0,"esp"),$AH[0]);
689 &mov (&DWP(4,"esp"),$AH[1]);
690 &xor ($AH[1],"edi"); # magic
691 &mov (&DWP(8,"esp"),"edi");
692 &mov (&DWP(12,"esp"),"ecx");
693 #&mov ($E,&DWP(16,"esp"));
694 &mov ("edi",&DWP(20,"esp"));
695 &mov ("ecx",&DWP(24,"esp"));
696 &add ($E,&DWP(16,"esi"));
697 &add ("edi",&DWP(20,"esi"));
698 &add ("ecx",&DWP(24,"esi"));
699 &mov (&DWP(16,"esi"),$E);
700 &mov (&DWP(20,"esi"),"edi");
701 &mov (&DWP(20,"esp"),"edi");
702 &mov ("edi",&DWP(28,"esp"));
703 &mov (&DWP(24,"esi"),"ecx");
704 #&mov (&DWP(16,"esp"),$E);
705 &add ("edi",&DWP(28,"esi"));
706 &mov (&DWP(24,"esp"),"ecx");
707 &mov (&DWP(28,"esi"),"edi");
708 &mov (&DWP(28,"esp"),"edi");
709 &mov ("edi",&DWP(96+4,"esp")); # inp
711 &vmovdqa ($t3,&QWP(64,$K256));
712 &sub ($K256,3*64); # rewind K
713 &cmp ("edi",&DWP(96+8,"esp")); # are we done yet?
714 &jb (&label("grand_xop"));
716 &mov ("esp",&DWP(96+12,"esp")); # restore sp
720 &set_label("AVX",16);
721 &lea ("esp",&DWP(-96,"esp"));
723 # copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack
724 &mov ($AH[0],&DWP(0,"esi"));
725 &mov ($AH[1],&DWP(4,"esi"));
726 &mov ("ecx",&DWP(8,"esi"));
727 &mov ("edi",&DWP(12,"esi"));
728 #&mov (&DWP(0,"esp"),$AH[0]);
729 &mov (&DWP(4,"esp"),$AH[1]);
730 &xor ($AH[1],"ecx"); # magic
731 &mov (&DWP(8,"esp"),"ecx");
732 &mov (&DWP(12,"esp"),"edi");
733 &mov ($E,&DWP(16,"esi"));
734 &mov ("edi",&DWP(20,"esi"));
735 &mov ("ecx",&DWP(24,"esi"));
736 &mov ("esi",&DWP(28,"esi"));
737 #&mov (&DWP(16,"esp"),$E);
738 &mov (&DWP(20,"esp"),"edi");
739 &mov ("edi",&DWP(96+4,"esp")); # inp
740 &mov (&DWP(24,"esp"),"ecx");
741 &mov (&DWP(28,"esp"),"esi");
742 &vmovdqa ($t3,&QWP(256,$K256));
743 &jmp (&label("grand_avx"));
745 &set_label("grand_avx",16);
746 # load input, reverse byte order, add K256[0..15], save to stack
747 &vmovdqu (@X[0],&QWP(0,"edi"));
748 &vmovdqu (@X[1],&QWP(16,"edi"));
749 &vmovdqu (@X[2],&QWP(32,"edi"));
750 &vmovdqu (@X[3],&QWP(48,"edi"));
752 &vpshufb (@X[0],@X[0],$t3);
753 &mov (&DWP(96+4,"esp"),"edi");
754 &vpshufb (@X[1],@X[1],$t3);
755 &vpshufb (@X[2],@X[2],$t3);
756 &vpaddd ($t0,@X[0],&QWP(0,$K256));
757 &vpshufb (@X[3],@X[3],$t3);
758 &vpaddd ($t1,@X[1],&QWP(16,$K256));
759 &vpaddd ($t2,@X[2],&QWP(32,$K256));
760 &vpaddd ($t3,@X[3],&QWP(48,$K256));
761 &vmovdqa (&QWP(32+0,"esp"),$t0);
762 &vmovdqa (&QWP(32+16,"esp"),$t1);
763 &vmovdqa (&QWP(32+32,"esp"),$t2);
764 &vmovdqa (&QWP(32+48,"esp"),$t3);
765 &jmp (&label("avx_00_47"));
767 &set_label("avx_00_47",16);
772 '&vpalignr ($t0,@X[1],@X[0],4);', # X[1..4]
773 '&vpalignr ($t3,@X[3],@X[2],4);', # X[9..12]
774 '&vpsrld ($t2,$t0,7);',
775 '&vpaddd (@X[0],@X[0],$t3);', # X[0..3] += X[9..16]
776 '&vpsrld ($t3,$t0,3);',
777 '&vpslld ($t1,$t0,14);',
778 '&vpxor ($t0,$t3,$t2);',
779 '&vpshufd ($t3,@X[3],0b11111010)',# X[14..15]
780 '&vpsrld ($t2,$t2,18-7);',
781 '&vpxor ($t0,$t0,$t1);',
782 '&vpslld ($t1,$t1,25-14);',
783 '&vpxor ($t0,$t0,$t2);',
784 '&vpsrld ($t2,$t3,10);',
785 '&vpxor ($t0,$t0,$t1);', # sigma0(X[1..4])
786 '&vpsrlq ($t1,$t3,17);',
787 '&vpaddd (@X[0],@X[0],$t0);', # X[0..3] += sigma0(X[1..4])
788 '&vpxor ($t2,$t2,$t1);',
789 '&vpsrlq ($t3,$t3,19);',
790 '&vpxor ($t2,$t2,$t3);', # sigma1(X[14..15]
791 '&vpshufd ($t3,$t2,0b10000100);',
792 '&vpsrldq ($t3,$t3,8);',
793 '&vpaddd (@X[0],@X[0],$t3);', # X[0..1] += sigma1(X[14..15])
794 '&vpshufd ($t3,@X[0],0b01010000)',# X[16..17]
795 '&vpsrld ($t2,$t3,10);',
796 '&vpsrlq ($t1,$t3,17);',
797 '&vpxor ($t2,$t2,$t1);',
798 '&vpsrlq ($t3,$t3,19);',
799 '&vpxor ($t2,$t2,$t3);', # sigma1(X[16..17]
800 '&vpshufd ($t3,$t2,0b11101000);',
801 '&vpslldq ($t3,$t3,8);',
802 '&vpaddd (@X[0],@X[0],$t3);' # X[2..3] += sigma1(X[16..17])
806 local *ror = sub { &shrd(@_[0],@_) };
811 my @insns = (&$body,&$body,&$body,&$body); # 120 instructions
813 foreach (Xupdate_AVX()) { # 31 instructions
819 &vpaddd ($t2,@X[0],&QWP(16*$j,$K256));
820 foreach (@insns) { eval; } # remaining instructions
821 &vmovdqa (&QWP(32+16*$j,"esp"),$t2);
824 for ($i=0,$j=0; $j<4; $j++) {
825 &AVX_00_47($j,\&body_00_15,@X);
826 push(@X,shift(@X)); # rotate(@X)
828 &cmp (&DWP(16*$j,$K256),0x00010203);
829 &jne (&label("avx_00_47"));
831 for ($i=0; $i<16; ) {
832 foreach(body_00_15()) { eval; }
835 &mov ("esi",&DWP(96,"esp")); #ctx
836 #&mov ($AH[0],&DWP(0,"esp"));
837 &xor ($AH[1],"edi"); #&mov ($AH[1],&DWP(4,"esp"));
838 #&mov ("edi", &DWP(8,"esp"));
839 &mov ("ecx",&DWP(12,"esp"));
840 &add ($AH[0],&DWP(0,"esi"));
841 &add ($AH[1],&DWP(4,"esi"));
842 &add ("edi",&DWP(8,"esi"));
843 &add ("ecx",&DWP(12,"esi"));
844 &mov (&DWP(0,"esi"),$AH[0]);
845 &mov (&DWP(4,"esi"),$AH[1]);
846 &mov (&DWP(8,"esi"),"edi");
847 &mov (&DWP(12,"esi"),"ecx");
848 #&mov (&DWP(0,"esp"),$AH[0]);
849 &mov (&DWP(4,"esp"),$AH[1]);
850 &xor ($AH[1],"edi"); # magic
851 &mov (&DWP(8,"esp"),"edi");
852 &mov (&DWP(12,"esp"),"ecx");
853 #&mov ($E,&DWP(16,"esp"));
854 &mov ("edi",&DWP(20,"esp"));
855 &mov ("ecx",&DWP(24,"esp"));
856 &add ($E,&DWP(16,"esi"));
857 &add ("edi",&DWP(20,"esi"));
858 &add ("ecx",&DWP(24,"esi"));
859 &mov (&DWP(16,"esi"),$E);
860 &mov (&DWP(20,"esi"),"edi");
861 &mov (&DWP(20,"esp"),"edi");
862 &mov ("edi",&DWP(28,"esp"));
863 &mov (&DWP(24,"esi"),"ecx");
864 #&mov (&DWP(16,"esp"),$E);
865 &add ("edi",&DWP(28,"esi"));
866 &mov (&DWP(24,"esp"),"ecx");
867 &mov (&DWP(28,"esi"),"edi");
868 &mov (&DWP(28,"esp"),"edi");
869 &mov ("edi",&DWP(96+4,"esp")); # inp
871 &vmovdqa ($t3,&QWP(64,$K256));
872 &sub ($K256,3*64); # rewind K
873 &cmp ("edi",&DWP(96+8,"esp")); # are we done yet?
874 &jb (&label("grand_avx"));
876 &mov ("esp",&DWP(96+12,"esp")); # restore sp
881 &function_end_B("sha256_block_data_order");
882 &asciz("SHA256 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>");