3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # This module implements Poly1305 hash for x86.
14 # Numbers are cycles per processed byte with poly1305_blocks alone,
15 # measured with rdtsc at fixed clock frequency.
17 # IALU/gcc-3.4(*) SSE2(**) AVX2
21 # Core 2 4.85/+90% 1.80
22 # Westmere 4.58/+100% 1.43
23 # Sandy Bridge 3.90/+100% 1.36
24 # Haswell 3.88/+70% 1.18 0.72
25 # Silvermont 11.0/+40% 4.80
26 # VIA Nano 6.71/+90% 2.47
27 # Sledgehammer 3.51/+180% 4.27
28 # Bulldozer 4.53/+140% 1.31
30 # (*) gcc 4.8 for some reason generated worse code;
31 # (**) besides SSE2 there are floating-point and AVX options; FP
32 # is deemed unnecessary, because pre-SSE2 processor are too
33 # old to care about, while it's not the fastest option on
34 # SSE2-capable ones; AVX is omitted, because it doesn't give
35 # a lot of improvement, 5-10% depending on processor;
37 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
38 push(@INC,"${dir}","${dir}../../perlasm");
41 &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
44 for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
47 &static_label("const_sse2");
48 &static_label("enter_blocks");
49 &static_label("enter_emit");
50 &external_label("OPENSSL_ia32cap_P");
52 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
53 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
54 $avx = ($1>=2.19) + ($1>=2.22);
57 if (!$avx && $ARGV[0] eq "win32n" &&
58 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
59 $avx = ($1>=2.09) + ($1>=2.10);
62 if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {
63 $avx = ($2>=3.0) + ($2>3.0);
67 ########################################################################
68 # Layout of opaque area is following.
70 # unsigned __int32 h[5]; # current hash value base 2^32
71 # unsigned __int32 pad; # is_base2_26 in vector context
72 # unsigned __int32 r[4]; # key value base 2^32
75 &function_begin("poly1305_init");
76 &mov ("edi",&wparam(0)); # context
77 &mov ("esi",&wparam(1)); # key
78 &mov ("ebp",&wparam(2)); # function table
81 &mov (&DWP(4*0,"edi"),"eax"); # zero hash value
82 &mov (&DWP(4*1,"edi"),"eax");
83 &mov (&DWP(4*2,"edi"),"eax");
84 &mov (&DWP(4*3,"edi"),"eax");
85 &mov (&DWP(4*4,"edi"),"eax");
86 &mov (&DWP(4*5,"edi"),"eax"); # is_base2_26
89 &je (&label("nokey"));
92 &call (&label("pic_point"));
93 &set_label("pic_point");
96 &lea ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
97 &lea ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
99 &picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
100 &mov ("ecx",&DWP(0,"edi"));
101 &and ("ecx",1<<26|1<<24);
102 &cmp ("ecx",1<<26|1<<24); # SSE2 and XMM?
103 &jne (&label("no_sse2"));
105 &lea ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
106 &lea ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
109 &mov ("ecx",&DWP(8,"edi"));
110 &test ("ecx",1<<5); # AVX2?
111 &jz (&label("no_sse2"));
113 &lea ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
115 &set_label("no_sse2");
116 &mov ("edi",&wparam(0)); # reload context
117 &mov (&DWP(0,"ebp"),"eax"); # fill function table
118 &mov (&DWP(4,"ebp"),"edx");
121 &mov ("eax",&DWP(4*0,"esi")); # load input key
122 &mov ("ebx",&DWP(4*1,"esi"));
123 &mov ("ecx",&DWP(4*2,"esi"));
124 &mov ("edx",&DWP(4*3,"esi"));
125 &and ("eax",0x0fffffff);
126 &and ("ebx",0x0ffffffc);
127 &and ("ecx",0x0ffffffc);
128 &and ("edx",0x0ffffffc);
129 &mov (&DWP(4*6,"edi"),"eax");
130 &mov (&DWP(4*7,"edi"),"ebx");
131 &mov (&DWP(4*8,"edi"),"ecx");
132 &mov (&DWP(4*9,"edi"),"edx");
136 &function_end("poly1305_init");
138 ($h0,$h1,$h2,$h3,$h4,
141 $s1,$s2,$s3)=map(4*$_,(0..15));
143 &function_begin("poly1305_blocks");
144 &mov ("edi",&wparam(0)); # ctx
145 &mov ("esi",&wparam(1)); # inp
146 &mov ("ecx",&wparam(2)); # len
147 &set_label("enter_blocks");
149 &jz (&label("nodata"));
152 &mov ("eax",&DWP(4*6,"edi")); # r0
153 &mov ("ebx",&DWP(4*7,"edi")); # r1
154 &lea ("ebp",&DWP(0,"esi","ecx")); # end of input
155 &mov ("ecx",&DWP(4*8,"edi")); # r2
156 &mov ("edx",&DWP(4*9,"edi")); # r3
158 &mov (&wparam(2),"ebp");
161 &mov (&DWP($r0,"esp"),"eax"); # r0
164 &mov (&DWP($r1,"esp"),"ebx"); # r1
165 &add ("eax","ebx"); # s1
168 &mov (&DWP($r2,"esp"),"ecx"); # r2
169 &add ("ebx","ecx"); # s2
172 &mov (&DWP($r3,"esp"),"edx"); # r3
173 &add ("ecx","edx"); # s3
174 &mov (&DWP($s1,"esp"),"eax"); # s1
175 &mov (&DWP($s2,"esp"),"ebx"); # s2
176 &mov (&DWP($s3,"esp"),"ecx"); # s3
178 &mov ("eax",&DWP(4*0,"edi")); # load hash value
179 &mov ("ebx",&DWP(4*1,"edi"));
180 &mov ("ecx",&DWP(4*2,"edi"));
181 &mov ("esi",&DWP(4*3,"edi"));
182 &mov ("edi",&DWP(4*4,"edi"));
183 &jmp (&label("loop"));
185 &set_label("loop",32);
186 &add ("eax",&DWP(4*0,"ebp")); # accumulate input
187 &adc ("ebx",&DWP(4*1,"ebp"));
188 &adc ("ecx",&DWP(4*2,"ebp"));
189 &adc ("esi",&DWP(4*3,"ebp"));
190 &lea ("ebp",&DWP(4*4,"ebp"));
191 &adc ("edi",&wparam(3)); # padbit
193 &mov (&DWP($h0,"esp"),"eax"); # put aside hash[+inp]
194 &mov (&DWP($h3,"esp"),"esi");
196 &mul (&DWP($r0,"esp")); # h0*r0
197 &mov (&DWP($h4,"esp"),"edi");
199 &mov ("eax","ebx"); # h1
201 &mul (&DWP($s3,"esp")); # h1*s3
203 &mov ("eax","ecx"); # h2
205 &mul (&DWP($s2,"esp")); # h2*s2
207 &mov ("eax",&DWP($h3,"esp"));
209 &mul (&DWP($s1,"esp")); # h3*s1
211 &mov ("eax",&DWP($h0,"esp"));
214 &mul (&DWP($r1,"esp")); # h0*r1
215 &mov (&DWP($d0,"esp"),"edi");
218 &mov ("eax","ebx"); # h1
220 &mul (&DWP($r0,"esp")); # h1*r0
222 &mov ("eax","ecx"); # h2
224 &mul (&DWP($s3,"esp")); # h2*s3
226 &mov ("eax",&DWP($h3,"esp"));
228 &mul (&DWP($s2,"esp")); # h3*s2
230 &mov ("eax",&DWP($h4,"esp"));
232 &imul ("eax",&DWP($s1,"esp")); # h4*s1
234 &mov ("eax",&DWP($h0,"esp"));
237 &mul (&DWP($r2,"esp")); # h0*r2
238 &mov (&DWP($d1,"esp"),"esi");
241 &mov ("eax","ebx"); # h1
243 &mul (&DWP($r1,"esp")); # h1*r1
245 &mov ("eax","ecx"); # h2
247 &mul (&DWP($r0,"esp")); # h2*r0
249 &mov ("eax",&DWP($h3,"esp"));
251 &mul (&DWP($s3,"esp")); # h3*s3
253 &mov ("eax",&DWP($h4,"esp"));
255 &imul ("eax",&DWP($s2,"esp")); # h4*s2
257 &mov ("eax",&DWP($h0,"esp"));
260 &mul (&DWP($r3,"esp")); # h0*r3
261 &mov (&DWP($d2,"esp"),"edi");
264 &mov ("eax","ebx"); # h1
266 &mul (&DWP($r2,"esp")); # h1*r2
268 &mov ("eax","ecx"); # h2
270 &mul (&DWP($r1,"esp")); # h2*r1
272 &mov ("eax",&DWP($h3,"esp"));
274 &mul (&DWP($r0,"esp")); # h3*r0
276 &mov ("ecx",&DWP($h4,"esp"));
280 &imul ("ecx",&DWP($s3,"esp")); # h4*s3
282 &mov ("eax",&DWP($d0,"esp"));
285 &imul ("edx",&DWP($r0,"esp")); # h4*r0
288 &mov ("ebx",&DWP($d1,"esp"));
289 &mov ("ecx",&DWP($d2,"esp"));
291 &mov ("edi","edx"); # last reduction step
294 &lea ("edx",&DWP(0,"edx","edx",4)); # *5
300 &cmp ("ebp",&wparam(2)); # done yet?
301 &jne (&label("loop"));
303 &mov ("edx",&wparam(0)); # ctx
305 &mov (&DWP(4*0,"edx"),"eax"); # store hash value
306 &mov (&DWP(4*1,"edx"),"ebx");
307 &mov (&DWP(4*2,"edx"),"ecx");
308 &mov (&DWP(4*3,"edx"),"esi");
309 &mov (&DWP(4*4,"edx"),"edi");
310 &set_label("nodata");
311 &function_end("poly1305_blocks");
313 &function_begin("poly1305_emit");
314 &mov ("ebp",&wparam(0)); # context
315 &set_label("enter_emit");
316 &mov ("edi",&wparam(1)); # output
317 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
318 &mov ("ebx",&DWP(4*1,"ebp"));
319 &mov ("ecx",&DWP(4*2,"ebp"));
320 &mov ("edx",&DWP(4*3,"ebp"));
321 &mov ("esi",&DWP(4*4,"ebp"));
323 &add ("eax",5); # compare to modulus
328 &shr ("esi",2); # did it carry/borrow?
329 &neg ("esi"); # do we choose hash-modulus?
335 &mov (&DWP(4*0,"edi"),"eax");
336 &mov (&DWP(4*1,"edi"),"ebx");
337 &mov (&DWP(4*2,"edi"),"ecx");
338 &mov (&DWP(4*3,"edi"),"edx");
340 ¬ ("esi"); # or original hash value?
341 &mov ("eax",&DWP(4*0,"ebp"));
342 &mov ("ebx",&DWP(4*1,"ebp"));
343 &mov ("ecx",&DWP(4*2,"ebp"));
344 &mov ("edx",&DWP(4*3,"ebp"));
345 &mov ("ebp",&wparam(2));
350 &or ("eax",&DWP(4*0,"edi"));
351 &or ("ebx",&DWP(4*1,"edi"));
352 &or ("ecx",&DWP(4*2,"edi"));
353 &or ("edx",&DWP(4*3,"edi"));
355 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
356 &adc ("ebx",&DWP(4*1,"ebp"));
357 &adc ("ecx",&DWP(4*2,"ebp"));
358 &adc ("edx",&DWP(4*3,"ebp"));
360 &mov (&DWP(4*0,"edi"),"eax");
361 &mov (&DWP(4*1,"edi"),"ebx");
362 &mov (&DWP(4*2,"edi"),"ecx");
363 &mov (&DWP(4*3,"edi"),"edx");
364 &function_end("poly1305_emit");
367 ########################################################################
368 # Layout of opaque area is following.
370 # unsigned __int32 h[5]; # current hash value base 2^26
371 # unsigned __int32 is_base2_26;
372 # unsigned __int32 r[4]; # key value base 2^32
373 # unsigned __int32 pad[2];
374 # struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
376 # where r^n are base 2^26 digits of degrees of multiplier key. There are
377 # 5 digits, but last four are interleaved with multiples of 5, totalling
378 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
380 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
381 my $MASK=$T2; # borrow and keep in mind
384 &function_begin_B("_poly1305_init_sse2");
385 &movdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
386 &lea ("edi",&DWP(16*3,"edi")); # size optimization
388 &sub ("esp",16*(9+5));
391 #&pand ($D4,&QWP(96,"ebx")); # magic mask
392 &movq ($MASK,&QWP(64,"ebx"));
398 &pand ($D0,$MASK); # -> base 2^26
409 &lea ("edx",&DWP(16*9,"esp")); # size optimization
411 &set_label("square");
412 &movdqa (&QWP(16*0,"esp"),$D0);
413 &movdqa (&QWP(16*1,"esp"),$D1);
414 &movdqa (&QWP(16*2,"esp"),$D2);
415 &movdqa (&QWP(16*3,"esp"),$D3);
416 &movdqa (&QWP(16*4,"esp"),$D4);
422 &paddd ($T1,$D1); # *5
423 &paddd ($T0,$D2); # *5
424 &movdqa (&QWP(16*5,"esp"),$T1);
425 &movdqa (&QWP(16*6,"esp"),$T0);
430 &paddd ($T1,$D3); # *5
431 &paddd ($T0,$D4); # *5
432 &movdqa (&QWP(16*7,"esp"),$T1);
433 &movdqa (&QWP(16*8,"esp"),$T0);
435 &pshufd ($T1,$D0,0b01000100);
437 &pshufd ($D1,$D1,0b01000100);
438 &pshufd ($D2,$D2,0b01000100);
439 &pshufd ($D3,$D3,0b01000100);
440 &pshufd ($D4,$D4,0b01000100);
441 &movdqa (&QWP(16*0,"edx"),$T1);
442 &movdqa (&QWP(16*1,"edx"),$D1);
443 &movdqa (&QWP(16*2,"edx"),$D2);
444 &movdqa (&QWP(16*3,"edx"),$D3);
445 &movdqa (&QWP(16*4,"edx"),$D4);
447 ################################################################
448 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
449 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
450 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
451 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
452 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
454 &pmuludq ($D4,$D0); # h4*r0
455 &pmuludq ($D3,$D0); # h3*r0
456 &pmuludq ($D2,$D0); # h2*r0
457 &pmuludq ($D1,$D0); # h1*r0
458 &pmuludq ($D0,$T1); # h0*r0
462 my $base = shift; $base = "esp" if (!defined($base));
464 ################################################################
465 # As for choice to "rotate" $T0-$T2 in order to move paddq
466 # past next multiplication. While it makes code harder to read
467 # and doesn't have significant effect on most processors, it
468 # makes a lot of difference on Atom, up to 30% improvement.
471 &pmuludq ($T0,&QWP(16*3,$base)); # r1*h3
473 &pmuludq ($T1,&QWP(16*2,$base)); # r1*h2
476 &pmuludq ($T2,&QWP(16*1,$base)); # r1*h1
479 &pmuludq ($T0,&QWP(16*0,$base)); # r1*h0
481 &pmuludq ($T1,&QWP(16*4,$base)); # s1*h4
482 &$load ($T2,2); # r2^n
486 &pmuludq ($T2,&QWP(16*2,$base)); # r2*h2
489 &pmuludq ($T0,&QWP(16*1,$base)); # r2*h1
491 &$load ($T2,6); # s2^n
492 &pmuludq ($T1,&QWP(16*0,$base)); # r2*h0
495 &pmuludq ($T2,&QWP(16*4,$base)); # s2*h4
497 &pmuludq ($T0,&QWP(16*3,$base)); # s2*h3
498 &$load ($T1,3); # r3^n
502 &pmuludq ($T1,&QWP(16*1,$base)); # r3*h1
504 &$load ($T0,7); # s3^n
505 &pmuludq ($T2,&QWP(16*0,$base)); # r3*h0
508 &pmuludq ($T0,&QWP(16*4,$base)); # s3*h4
511 &pmuludq ($T1,&QWP(16*3,$base)); # s3*h3
513 &pmuludq ($T2,&QWP(16*2,$base)); # s3*h2
514 &$load ($T0,4); # r4^n
517 &$load ($T1,8); # s4^n
518 &pmuludq ($T0,&QWP(16*0,$base)); # r4*h0
521 &pmuludq ($T1,&QWP(16*4,$base)); # s4*h4
524 &pmuludq ($T2,&QWP(16*1,$base)); # s4*h1
527 &pmuludq ($T0,&QWP(16*2,$base)); # s4*h2
529 &pmuludq ($T1,&QWP(16*3,$base)); # s4*h3
530 &movdqa ($MASK,&QWP(64,"ebx"));
534 &pmuladd (sub { my ($reg,$i)=@_;
535 &movdqa ($reg,&QWP(16*$i,"esp"));
539 ################################################################
540 # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
546 &paddq ($T0,$D4); # h3 -> h4
551 &paddq ($T1,$D1); # h0 -> h1
556 &paddd ($D0,$T0); # favour paddd when
561 &paddq ($T1,$D2); # h1 -> h2
565 &paddd ($T0,$D0); # h4 -> h0
567 &paddd ($T1,$D3); # h2 -> h3
573 &paddd ($D1,$T0); # h0 -> h1
575 &paddd ($D4,$T1); # h3 -> h4
580 &jz (&label("square_break"));
582 &punpcklqdq ($D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
583 &punpcklqdq ($D1,&QWP(16*1,"esp"));
584 &punpcklqdq ($D2,&QWP(16*2,"esp"));
585 &punpcklqdq ($D3,&QWP(16*3,"esp"));
586 &punpcklqdq ($D4,&QWP(16*4,"esp"));
587 &jmp (&label("square"));
589 &set_label("square_break");
590 &psllq ($D0,32); # -> r^3:0:r^4:0
595 &por ($D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
596 &por ($D1,&QWP(16*1,"esp"));
597 &por ($D2,&QWP(16*2,"esp"));
598 &por ($D3,&QWP(16*3,"esp"));
599 &por ($D4,&QWP(16*4,"esp"));
601 &pshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
602 &pshufd ($D1,$D1,0b10001101);
603 &pshufd ($D2,$D2,0b10001101);
604 &pshufd ($D3,$D3,0b10001101);
605 &pshufd ($D4,$D4,0b10001101);
607 &movdqu (&QWP(16*0,"edi"),$D0); # save the table
608 &movdqu (&QWP(16*1,"edi"),$D1);
609 &movdqu (&QWP(16*2,"edi"),$D2);
610 &movdqu (&QWP(16*3,"edi"),$D3);
611 &movdqu (&QWP(16*4,"edi"),$D4);
617 &paddd ($T1,$D1); # *5
618 &paddd ($T0,$D2); # *5
619 &movdqu (&QWP(16*5,"edi"),$T1);
620 &movdqu (&QWP(16*6,"edi"),$T0);
625 &paddd ($T1,$D3); # *5
626 &paddd ($T0,$D4); # *5
627 &movdqu (&QWP(16*7,"edi"),$T1);
628 &movdqu (&QWP(16*8,"edi"),$T0);
631 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
633 &function_end_B("_poly1305_init_sse2");
636 &function_begin("_poly1305_blocks_sse2");
637 &mov ("edi",&wparam(0)); # ctx
638 &mov ("esi",&wparam(1)); # inp
639 &mov ("ecx",&wparam(2)); # len
641 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
643 &jz (&label("nodata"));
645 &jae (&label("enter_sse2"));
646 &test ("eax","eax"); # is_base2_26?
647 &jz (&label("enter_blocks"));
649 &set_label("enter_sse2",16);
650 &call (&label("pic_point"));
651 &set_label("pic_point");
653 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
655 &test ("eax","eax"); # is_base2_26?
656 &jnz (&label("base2_26"));
658 &call ("_poly1305_init_sse2");
660 ################################################# base 2^32 -> base 2^26
661 &mov ("eax",&DWP(0,"edi"));
662 &mov ("ecx",&DWP(3,"edi"));
663 &mov ("edx",&DWP(6,"edi"));
664 &mov ("esi",&DWP(9,"edi"));
665 &mov ("ebp",&DWP(13,"edi"));
666 &mov (&DWP(4*5,"edi"),1); # is_base2_26
669 &and ("eax",0x3ffffff);
671 &and ("ecx",0x3ffffff);
673 &and ("edx",0x3ffffff);
681 &mov ("esi",&wparam(1)); # [reload] inp
682 &mov ("ecx",&wparam(2)); # [reload] len
683 &jmp (&label("base2_32"));
685 &set_label("base2_26",16);
686 &movd ($D0,&DWP(4*0,"edi")); # load hash value
687 &movd ($D1,&DWP(4*1,"edi"));
688 &movd ($D2,&DWP(4*2,"edi"));
689 &movd ($D3,&DWP(4*3,"edi"));
690 &movd ($D4,&DWP(4*4,"edi"));
691 &movdqa ($MASK,&QWP(64,"ebx"));
693 &set_label("base2_32");
694 &mov ("eax",&wparam(3)); # padbit
697 &sub ("esp",16*(5+5+5+9+9));
700 &lea ("edi",&DWP(16*3,"edi")); # size optimization
701 &shl ("eax",24); # padbit
704 &jz (&label("even"));
706 ################################################################
707 # process single block, with SSE2, because it's still faster
708 # even though half of result is discarded
710 &movdqu ($T1,&QWP(0,"esi")); # input
711 &lea ("esi",&DWP(16,"esi"));
713 &movdqa ($T0,$T1); # -> base 2^26 ...
715 &paddd ($D0,$T1); # ... and accumuate
734 &movd ($T0,"eax"); # padbit
736 &movd ($T1,&DWP(16*0+12,"edi")); # r0
739 &movdqa (&QWP(16*0,"esp"),$D0);
740 &movdqa (&QWP(16*1,"esp"),$D1);
741 &movdqa (&QWP(16*2,"esp"),$D2);
742 &movdqa (&QWP(16*3,"esp"),$D3);
743 &movdqa (&QWP(16*4,"esp"),$D4);
745 ################################################################
746 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
747 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
748 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
749 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
750 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
752 &pmuludq ($D0,$T1); # h4*r0
753 &pmuludq ($D1,$T1); # h3*r0
754 &pmuludq ($D2,$T1); # h2*r0
755 &movd ($T0,&DWP(16*1+12,"edi")); # r1
756 &pmuludq ($D3,$T1); # h1*r0
757 &pmuludq ($D4,$T1); # h0*r0
759 &pmuladd (sub { my ($reg,$i)=@_;
760 &movd ($reg,&DWP(16*$i+12,"edi"));
766 &jz (&label("done"));
769 &lea ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
770 &lea ("eax",&DWP(-16*2,"esi"));
773 ################################################################
774 # expand and copy pre-calculated table to stack
776 &movdqu ($T0,&QWP(16*0,"edi")); # r^1:r^2:r^3:r^4
777 &pshufd ($T1,$T0,0b01000100); # duplicate r^3:r^4
778 &cmovb ("esi","eax");
779 &pshufd ($T0,$T0,0b11101110); # duplicate r^1:r^2
780 &movdqa (&QWP(16*0,"edx"),$T1);
781 &lea ("eax",&DWP(16*10,"esp"));
782 &movdqu ($T1,&QWP(16*1,"edi"));
783 &movdqa (&QWP(16*(0-9),"edx"),$T0);
784 &pshufd ($T0,$T1,0b01000100);
785 &pshufd ($T1,$T1,0b11101110);
786 &movdqa (&QWP(16*1,"edx"),$T0);
787 &movdqu ($T0,&QWP(16*2,"edi"));
788 &movdqa (&QWP(16*(1-9),"edx"),$T1);
789 &pshufd ($T1,$T0,0b01000100);
790 &pshufd ($T0,$T0,0b11101110);
791 &movdqa (&QWP(16*2,"edx"),$T1);
792 &movdqu ($T1,&QWP(16*3,"edi"));
793 &movdqa (&QWP(16*(2-9),"edx"),$T0);
794 &pshufd ($T0,$T1,0b01000100);
795 &pshufd ($T1,$T1,0b11101110);
796 &movdqa (&QWP(16*3,"edx"),$T0);
797 &movdqu ($T0,&QWP(16*4,"edi"));
798 &movdqa (&QWP(16*(3-9),"edx"),$T1);
799 &pshufd ($T1,$T0,0b01000100);
800 &pshufd ($T0,$T0,0b11101110);
801 &movdqa (&QWP(16*4,"edx"),$T1);
802 &movdqu ($T1,&QWP(16*5,"edi"));
803 &movdqa (&QWP(16*(4-9),"edx"),$T0);
804 &pshufd ($T0,$T1,0b01000100);
805 &pshufd ($T1,$T1,0b11101110);
806 &movdqa (&QWP(16*5,"edx"),$T0);
807 &movdqu ($T0,&QWP(16*6,"edi"));
808 &movdqa (&QWP(16*(5-9),"edx"),$T1);
809 &pshufd ($T1,$T0,0b01000100);
810 &pshufd ($T0,$T0,0b11101110);
811 &movdqa (&QWP(16*6,"edx"),$T1);
812 &movdqu ($T1,&QWP(16*7,"edi"));
813 &movdqa (&QWP(16*(6-9),"edx"),$T0);
814 &pshufd ($T0,$T1,0b01000100);
815 &pshufd ($T1,$T1,0b11101110);
816 &movdqa (&QWP(16*7,"edx"),$T0);
817 &movdqu ($T0,&QWP(16*8,"edi"));
818 &movdqa (&QWP(16*(7-9),"edx"),$T1);
819 &pshufd ($T1,$T0,0b01000100);
820 &pshufd ($T0,$T0,0b11101110);
821 &movdqa (&QWP(16*8,"edx"),$T1);
822 &movdqa (&QWP(16*(8-9),"edx"),$T0);
825 my ($inpbase,$offbase)=@_;
827 &movdqu ($T0,&QWP($inpbase+0,"esi")); # load input
828 &movdqu ($T1,&QWP($inpbase+16,"esi"));
829 &lea ("esi",&DWP(16*2,"esi"));
831 &movdqa (&QWP($offbase+16*2,"esp"),$D2);
832 &movdqa (&QWP($offbase+16*3,"esp"),$D3);
833 &movdqa (&QWP($offbase+16*4,"esp"),$D4);
835 &movdqa ($D2,$T0); # splat input
840 &punpcklqdq ($D2,$D3); # 2:3
841 &punpckhqdq ($D4,$T1); # 4
842 &punpcklqdq ($T0,$T1); # 0:1
850 &pand ($T0,$MASK); # 0
851 &pand ($T1,$MASK); # 1
852 &pand ($D2,$MASK); # 2
853 &pand ($D3,$MASK); # 3
854 &por ($D4,&QWP(0,"ebx")); # padbit, yes, always
856 &movdqa (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
857 &movdqa (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
859 &load_input (16*2,16*5);
861 &jbe (&label("skip_loop"));
862 &jmp (&label("loop"));
864 &set_label("loop",32);
865 ################################################################
866 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
867 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
868 # \___________________/
869 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
870 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
871 # \___________________/ \____________________/
872 ################################################################
874 &movdqa ($T2,&QWP(16*(0-9),"edx")); # r0^2
875 &movdqa (&QWP(16*1,"eax"),$T1);
876 &movdqa (&QWP(16*2,"eax"),$D2);
877 &movdqa (&QWP(16*3,"eax"),$D3);
878 &movdqa (&QWP(16*4,"eax"),$D4);
880 ################################################################
881 # d4 = h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
882 # d3 = h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
883 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
884 # d1 = h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
885 # d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
888 &pmuludq ($T0,$T2); # h0*r0
890 &pmuludq ($T1,$T2); # h1*r0
891 &pmuludq ($D2,$T2); # h2*r0
892 &pmuludq ($D3,$T2); # h3*r0
893 &pmuludq ($D4,$T2); # h4*r0
898 &pmuludq ($D0,&$addr(8)); # h1*s4
900 &pmuludq ($D1,&$addr(1)); # h0*r1
903 &pmuludq ($T2,&$addr(2)); # h0*r2
906 &pmuludq ($T0,&$addr(3)); # h0*r3
908 &movdqa ($T2,&QWP(16*1,"eax")); # pull h1
909 &pmuludq ($T1,&$addr(4)); # h0*r4
913 &pmuludq ($T2,&$addr(1)); # h1*r1
916 &pmuludq ($T0,&$addr(2)); # h1*r2
918 &movdqa ($T2,&QWP(16*2,"eax")); # pull h2
919 &pmuludq ($T1,&$addr(3)); # h1*r3
922 &pmuludq ($T2,&$addr(7)); # h2*s3
925 &pmuludq ($T0,&$addr(8)); # h2*s4
929 &pmuludq ($T1,&$addr(1)); # h2*r1
931 &movdqa ($T0,&QWP(16*3,"eax")); # pull h3
932 &pmuludq ($T2,&$addr(2)); # h2*r2
935 &pmuludq ($T0,&$addr(6)); # h3*s2
938 &pmuludq ($T1,&$addr(7)); # h3*s3
941 &pmuludq ($T2,&$addr(8)); # h3*s4
944 &movdqa ($T1,&QWP(16*4,"eax")); # pull h4
945 &pmuludq ($T0,&$addr(1)); # h3*r1
948 &pmuludq ($T1,&$addr(8)); # h4*s4
951 &pmuludq ($T2,&$addr(5)); # h4*s1
954 &pmuludq ($T0,&$addr(6)); # h4*s2
956 &movdqa ($MASK,&QWP(64,"ebx"));
957 &pmuludq ($T1,&$addr(7)); # h4*s3
961 &pmuladd_alt (sub { my $i=shift; &QWP(16*($i-9),"edx"); });
963 &load_input (-16*2,0);
964 &lea ("eax",&DWP(-16*2,"esi"));
967 &paddd ($T0,&QWP(16*(5+0),"esp")); # add hash value
968 &paddd ($T1,&QWP(16*(5+1),"esp"));
969 &paddd ($D2,&QWP(16*(5+2),"esp"));
970 &paddd ($D3,&QWP(16*(5+3),"esp"));
971 &paddd ($D4,&QWP(16*(5+4),"esp"));
973 &cmovb ("esi","eax");
974 &lea ("eax",&DWP(16*10,"esp"));
976 &movdqa ($T2,&QWP(16*0,"edx")); # r0^4
977 &movdqa (&QWP(16*1,"esp"),$D1);
978 &movdqa (&QWP(16*1,"eax"),$T1);
979 &movdqa (&QWP(16*2,"eax"),$D2);
980 &movdqa (&QWP(16*3,"eax"),$D3);
981 &movdqa (&QWP(16*4,"eax"),$D4);
983 ################################################################
984 # d4 += h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
985 # d3 += h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
986 # d2 += h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
987 # d1 += h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
988 # d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
991 &pmuludq ($T0,$T2); # h0*r0
994 &pmuludq ($T1,$T2); # h1*r0
995 &pmuludq ($D2,$T2); # h2*r0
996 &pmuludq ($D3,$T2); # h3*r0
997 &pmuludq ($D4,$T2); # h4*r0
999 &paddq ($T1,&QWP(16*1,"esp"));
1000 &paddq ($D2,&QWP(16*2,"esp"));
1001 &paddq ($D3,&QWP(16*3,"esp"));
1002 &paddq ($D4,&QWP(16*4,"esp"));
1004 &pmuladd_alt (sub { my $i=shift; &QWP(16*$i,"edx"); });
1008 &load_input (16*2,16*5);
1010 &ja (&label("loop"));
1012 &set_label("skip_loop");
1013 ################################################################
1014 # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1016 &pshufd ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
1018 &jnz (&label("long_tail"));
1020 &paddd ($T0,$D0); # add hash value
1022 &paddd ($D2,&QWP(16*7,"esp"));
1023 &paddd ($D3,&QWP(16*8,"esp"));
1024 &paddd ($D4,&QWP(16*9,"esp"));
1026 &set_label("long_tail");
1028 &movdqa (&QWP(16*0,"eax"),$T0);
1029 &movdqa (&QWP(16*1,"eax"),$T1);
1030 &movdqa (&QWP(16*2,"eax"),$D2);
1031 &movdqa (&QWP(16*3,"eax"),$D3);
1032 &movdqa (&QWP(16*4,"eax"),$D4);
1034 ################################################################
1035 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1036 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1037 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1038 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1039 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1041 &pmuludq ($T0,$T2); # h0*r0
1042 &pmuludq ($T1,$T2); # h1*r0
1043 &pmuludq ($D2,$T2); # h2*r0
1045 &pshufd ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
1046 &pmuludq ($D3,$T2); # h3*r0
1048 &pmuludq ($D4,$T2); # h4*r0
1050 &pmuladd (sub { my ($reg,$i)=@_;
1051 &pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
1054 &jz (&label("short_tail"));
1056 &load_input (-16*2,0);
1058 &pshufd ($T2,&QWP(16*0,"edx"),0x10); # r0^n
1059 &paddd ($T0,&QWP(16*5,"esp")); # add hash value
1060 &paddd ($T1,&QWP(16*6,"esp"));
1061 &paddd ($D2,&QWP(16*7,"esp"));
1062 &paddd ($D3,&QWP(16*8,"esp"));
1063 &paddd ($D4,&QWP(16*9,"esp"));
1065 ################################################################
1066 # multiply inp[0:1] by r^4:r^3 and accumulate
1068 &movdqa (&QWP(16*0,"esp"),$T0);
1069 &pmuludq ($T0,$T2); # h0*r0
1070 &movdqa (&QWP(16*1,"esp"),$T1);
1071 &pmuludq ($T1,$T2); # h1*r0
1074 &pmuludq ($D2,$T2); # h2*r0
1077 &pmuludq ($D3,$T2); # h3*r0
1078 &paddq ($D2,&QWP(16*2,"esp"));
1079 &movdqa (&QWP(16*2,"esp"),$T0);
1080 &pshufd ($T0,&QWP(16*1,"edx"),0x10); # r1^n
1081 &paddq ($D3,&QWP(16*3,"esp"));
1082 &movdqa (&QWP(16*3,"esp"),$T1);
1084 &pmuludq ($D4,$T2); # h4*r0
1085 &paddq ($D4,&QWP(16*4,"esp"));
1086 &movdqa (&QWP(16*4,"esp"),$T1);
1088 &pmuladd (sub { my ($reg,$i)=@_;
1089 &pshufd ($reg,&QWP(16*$i,"edx"),0x10);
1092 &set_label("short_tail");
1096 ################################################################
1097 # horizontal addition
1099 &pshufd ($T1,$D0,0b01001110);
1100 &pshufd ($T0,$D1,0b01001110);
1102 &pshufd ($T1,$D2,0b01001110);
1104 &pshufd ($T0,$D3,0b01001110);
1106 &pshufd ($T1,$D4,0b01001110);
1111 &movd (&DWP(-16*3+4*0,"edi"),$D0); # store hash value
1112 &movd (&DWP(-16*3+4*1,"edi"),$D1);
1113 &movd (&DWP(-16*3+4*2,"edi"),$D2);
1114 &movd (&DWP(-16*3+4*3,"edi"),$D3);
1115 &movd (&DWP(-16*3+4*4,"edi"),$D4);
1116 &set_label("nodata");
1118 &function_end("_poly1305_blocks_sse2");
1121 &function_begin("_poly1305_emit_sse2");
1122 &mov ("ebp",&wparam(0)); # context
1124 &cmp (&DWP(4*5,"ebp"),0); # is_base2_26?
1125 &je (&label("enter_emit"));
1127 &mov ("eax",&DWP(4*0,"ebp")); # load hash value
1128 &mov ("edi",&DWP(4*1,"ebp"));
1129 &mov ("ecx",&DWP(4*2,"ebp"));
1130 &mov ("edx",&DWP(4*3,"ebp"));
1131 &mov ("esi",&DWP(4*4,"ebp"));
1133 &mov ("ebx","edi"); # base 2^26 -> base 2^32
1155 &adc ("esi",0); # can be partially reduced
1157 &mov ("edi","esi"); # final reduction
1160 &lea ("ebp",&DWP(0,"edi","edi",4)); # *5
1161 &mov ("edi",&wparam(1)); # output
1163 &mov ("ebp",&wparam(2)); # key
1168 &movd ($D0,"eax"); # offload original hash value
1169 &add ("eax",5); # compare to modulus
1177 &shr ("esi",2); # did it carry/borrow?
1179 &neg ("esi"); # do we choose (hash-modulus) ...
1184 &mov (&DWP(4*0,"edi"),"eax");
1186 &mov (&DWP(4*1,"edi"),"ebx");
1188 &mov (&DWP(4*2,"edi"),"ecx");
1190 &mov (&DWP(4*3,"edi"),"edx");
1193 ¬ ("esi"); # ... or original hash value?
1196 &or ("eax",&DWP(4*0,"edi"));
1198 &or ("ebx",&DWP(4*1,"edi"));
1200 &or ("ecx",&DWP(4*2,"edi"));
1201 &or ("edx",&DWP(4*3,"edi"));
1203 &add ("eax",&DWP(4*0,"ebp")); # accumulate key
1204 &adc ("ebx",&DWP(4*1,"ebp"));
1205 &mov (&DWP(4*0,"edi"),"eax");
1206 &adc ("ecx",&DWP(4*2,"ebp"));
1207 &mov (&DWP(4*1,"edi"),"ebx");
1208 &adc ("edx",&DWP(4*3,"ebp"));
1209 &mov (&DWP(4*2,"edi"),"ecx");
1210 &mov (&DWP(4*3,"edi"),"edx");
1211 &function_end("_poly1305_emit_sse2");
1214 ########################################################################
1215 # Note that poly1305_init_avx2 operates on %xmm, I could have used
1216 # poly1305_init_sse2...
1219 &function_begin_B("_poly1305_init_avx2");
1220 &vmovdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
1221 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1223 &sub ("esp",16*(9+5));
1226 #&vpand ($D4,$D4,&QWP(96,"ebx")); # magic mask
1227 &vmovdqa ($MASK,&QWP(64,"ebx"));
1229 &vpand ($D0,$D4,$MASK); # -> base 2^26
1230 &vpsrlq ($D1,$D4,26);
1231 &vpsrldq ($D3,$D4,6);
1232 &vpand ($D1,$D1,$MASK);
1234 &vpsrlq ($D3,$D3,30);
1235 &vpand ($D2,$D2,$MASK);
1236 &vpand ($D3,$D3,$MASK);
1237 &vpsrldq ($D4,$D4,13);
1239 &lea ("edx",&DWP(16*9,"esp")); # size optimization
1241 &set_label("square");
1242 &vmovdqa (&QWP(16*0,"esp"),$D0);
1243 &vmovdqa (&QWP(16*1,"esp"),$D1);
1244 &vmovdqa (&QWP(16*2,"esp"),$D2);
1245 &vmovdqa (&QWP(16*3,"esp"),$D3);
1246 &vmovdqa (&QWP(16*4,"esp"),$D4);
1248 &vpslld ($T1,$D1,2);
1249 &vpslld ($T0,$D2,2);
1250 &vpaddd ($T1,$T1,$D1); # *5
1251 &vpaddd ($T0,$T0,$D2); # *5
1252 &vmovdqa (&QWP(16*5,"esp"),$T1);
1253 &vmovdqa (&QWP(16*6,"esp"),$T0);
1254 &vpslld ($T1,$D3,2);
1255 &vpslld ($T0,$D4,2);
1256 &vpaddd ($T1,$T1,$D3); # *5
1257 &vpaddd ($T0,$T0,$D4); # *5
1258 &vmovdqa (&QWP(16*7,"esp"),$T1);
1259 &vmovdqa (&QWP(16*8,"esp"),$T0);
1261 &vpshufd ($T0,$D0,0b01000100);
1263 &vpshufd ($D1,$D1,0b01000100);
1264 &vpshufd ($D2,$D2,0b01000100);
1265 &vpshufd ($D3,$D3,0b01000100);
1266 &vpshufd ($D4,$D4,0b01000100);
1267 &vmovdqa (&QWP(16*0,"edx"),$T0);
1268 &vmovdqa (&QWP(16*1,"edx"),$D1);
1269 &vmovdqa (&QWP(16*2,"edx"),$D2);
1270 &vmovdqa (&QWP(16*3,"edx"),$D3);
1271 &vmovdqa (&QWP(16*4,"edx"),$D4);
1273 ################################################################
1274 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1275 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1276 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1277 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1278 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1280 &vpmuludq ($D4,$D4,$D0); # h4*r0
1281 &vpmuludq ($D3,$D3,$D0); # h3*r0
1282 &vpmuludq ($D2,$D2,$D0); # h2*r0
1283 &vpmuludq ($D1,$D1,$D0); # h1*r0
1284 &vpmuludq ($D0,$T0,$D0); # h0*r0
1286 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # r1*h3
1287 &vpaddq ($D4,$D4,$T0);
1288 &vpmuludq ($T2,$T1,&QWP(16*2,"edx")); # r1*h2
1289 &vpaddq ($D3,$D3,$T2);
1290 &vpmuludq ($T0,$T1,&QWP(16*1,"edx")); # r1*h1
1291 &vpaddq ($D2,$D2,$T0);
1292 &vmovdqa ($T2,&QWP(16*5,"esp")); # s1
1293 &vpmuludq ($T1,$T1,&QWP(16*0,"edx")); # r1*h0
1294 &vpaddq ($D1,$D1,$T1);
1295 &vmovdqa ($T0,&QWP(16*2,"esp")); # r2
1296 &vpmuludq ($T2,$T2,&QWP(16*4,"edx")); # s1*h4
1297 &vpaddq ($D0,$D0,$T2);
1299 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # r2*h2
1300 &vpaddq ($D4,$D4,$T1);
1301 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r2*h1
1302 &vpaddq ($D3,$D3,$T2);
1303 &vmovdqa ($T1,&QWP(16*6,"esp")); # s2
1304 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r2*h0
1305 &vpaddq ($D2,$D2,$T0);
1306 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s2*h4
1307 &vpaddq ($D1,$D1,$T2);
1308 &vmovdqa ($T0,&QWP(16*3,"esp")); # r3
1309 &vpmuludq ($T1,$T1,&QWP(16*3,"edx")); # s2*h3
1310 &vpaddq ($D0,$D0,$T1);
1312 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r3*h1
1313 &vpaddq ($D4,$D4,$T2);
1314 &vmovdqa ($T1,&QWP(16*7,"esp")); # s3
1315 &vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r3*h0
1316 &vpaddq ($D3,$D3,$T0);
1317 &vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s3*h4
1318 &vpaddq ($D2,$D2,$T2);
1319 &vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # s3*h3
1320 &vpaddq ($D1,$D1,$T0);
1321 &vmovdqa ($T2,&QWP(16*4,"esp")); # r4
1322 &vpmuludq ($T1,$T1,&QWP(16*2,"edx")); # s3*h2
1323 &vpaddq ($D0,$D0,$T1);
1325 &vmovdqa ($T0,&QWP(16*8,"esp")); # s4
1326 &vpmuludq ($T2,$T2,&QWP(16*0,"edx")); # r4*h0
1327 &vpaddq ($D4,$D4,$T2);
1328 &vpmuludq ($T1,$T0,&QWP(16*4,"edx")); # s4*h4
1329 &vpaddq ($D3,$D3,$T1);
1330 &vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # s4*h1
1331 &vpaddq ($D0,$D0,$T2);
1332 &vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # s4*h2
1333 &vpaddq ($D1,$D1,$T1);
1334 &vmovdqa ($MASK,&QWP(64,"ebx"));
1335 &vpmuludq ($T0,$T0,&QWP(16*3,"edx")); # s4*h3
1336 &vpaddq ($D2,$D2,$T0);
1338 ################################################################
1340 &vpsrlq ($T0,$D3,26);
1341 &vpand ($D3,$D3,$MASK);
1342 &vpsrlq ($T1,$D0,26);
1343 &vpand ($D0,$D0,$MASK);
1344 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1345 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1346 &vpsrlq ($T0,$D4,26);
1347 &vpand ($D4,$D4,$MASK);
1348 &vpsrlq ($T1,$D1,26);
1349 &vpand ($D1,$D1,$MASK);
1350 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1351 &vpaddd ($D0,$D0,$T0);
1352 &vpsllq ($T0,$T0,2);
1353 &vpsrlq ($T1,$D2,26);
1354 &vpand ($D2,$D2,$MASK);
1355 &vpaddd ($D0,$D0,$T0); # h4 -> h0
1356 &vpaddd ($D3,$D3,$T1); # h2 -> h3
1357 &vpsrlq ($T1,$D3,26);
1358 &vpsrlq ($T0,$D0,26);
1359 &vpand ($D0,$D0,$MASK);
1360 &vpand ($D3,$D3,$MASK);
1361 &vpaddd ($D1,$D1,$T0); # h0 -> h1
1362 &vpaddd ($D4,$D4,$T1); # h3 -> h4
1365 &jz (&label("square_break"));
1367 &vpunpcklqdq ($D0,$D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
1368 &vpunpcklqdq ($D1,$D1,&QWP(16*1,"esp"));
1369 &vpunpcklqdq ($D2,$D2,&QWP(16*2,"esp"));
1370 &vpunpcklqdq ($D3,$D3,&QWP(16*3,"esp"));
1371 &vpunpcklqdq ($D4,$D4,&QWP(16*4,"esp"));
1372 &jmp (&label("square"));
1374 &set_label("square_break");
1375 &vpsllq ($D0,$D0,32); # -> r^3:0:r^4:0
1376 &vpsllq ($D1,$D1,32);
1377 &vpsllq ($D2,$D2,32);
1378 &vpsllq ($D3,$D3,32);
1379 &vpsllq ($D4,$D4,32);
1380 &vpor ($D0,$D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
1381 &vpor ($D1,$D1,&QWP(16*1,"esp"));
1382 &vpor ($D2,$D2,&QWP(16*2,"esp"));
1383 &vpor ($D3,$D3,&QWP(16*3,"esp"));
1384 &vpor ($D4,$D4,&QWP(16*4,"esp"));
1386 &vpshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
1387 &vpshufd ($D1,$D1,0b10001101);
1388 &vpshufd ($D2,$D2,0b10001101);
1389 &vpshufd ($D3,$D3,0b10001101);
1390 &vpshufd ($D4,$D4,0b10001101);
1392 &vmovdqu (&QWP(16*0,"edi"),$D0); # save the table
1393 &vmovdqu (&QWP(16*1,"edi"),$D1);
1394 &vmovdqu (&QWP(16*2,"edi"),$D2);
1395 &vmovdqu (&QWP(16*3,"edi"),$D3);
1396 &vmovdqu (&QWP(16*4,"edi"),$D4);
1398 &vpslld ($T1,$D1,2);
1399 &vpslld ($T0,$D2,2);
1400 &vpaddd ($T1,$T1,$D1); # *5
1401 &vpaddd ($T0,$T0,$D2); # *5
1402 &vmovdqu (&QWP(16*5,"edi"),$T1);
1403 &vmovdqu (&QWP(16*6,"edi"),$T0);
1404 &vpslld ($T1,$D3,2);
1405 &vpslld ($T0,$D4,2);
1406 &vpaddd ($T1,$T1,$D3); # *5
1407 &vpaddd ($T0,$T0,$D4); # *5
1408 &vmovdqu (&QWP(16*7,"edi"),$T1);
1409 &vmovdqu (&QWP(16*8,"edi"),$T0);
1412 &lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
1414 &function_end_B("_poly1305_init_avx2");
1416 ########################################################################
1417 # now it's time to switch to %ymm
1419 my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
1422 sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
1425 &function_begin("_poly1305_blocks_avx2");
1426 &mov ("edi",&wparam(0)); # ctx
1427 &mov ("esi",&wparam(1)); # inp
1428 &mov ("ecx",&wparam(2)); # len
1430 &mov ("eax",&DWP(4*5,"edi")); # is_base2_26
1432 &jz (&label("nodata"));
1434 &jae (&label("enter_avx2"));
1435 &test ("eax","eax"); # is_base2_26?
1436 &jz (&label("enter_blocks"));
1438 &set_label("enter_avx2",16);
1441 &call (&label("pic_point"));
1442 &set_label("pic_point");
1444 &lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
1446 &test ("eax","eax"); # is_base2_26?
1447 &jnz (&label("base2_26"));
1449 &call ("_poly1305_init_avx2");
1451 ################################################# base 2^32 -> base 2^26
1452 &mov ("eax",&DWP(0,"edi"));
1453 &mov ("ecx",&DWP(3,"edi"));
1454 &mov ("edx",&DWP(6,"edi"));
1455 &mov ("esi",&DWP(9,"edi"));
1456 &mov ("ebp",&DWP(13,"edi"));
1459 &and ("eax",0x3ffffff);
1461 &and ("ecx",0x3ffffff);
1463 &and ("edx",0x3ffffff);
1465 &mov (&DWP(4*0,"edi"),"eax");
1466 &mov (&DWP(4*1,"edi"),"ecx");
1467 &mov (&DWP(4*2,"edi"),"edx");
1468 &mov (&DWP(4*3,"edi"),"esi");
1469 &mov (&DWP(4*4,"edi"),"ebp");
1470 &mov (&DWP(4*5,"edi"),1); # is_base2_26
1472 &mov ("esi",&wparam(1)); # [reload] inp
1473 &mov ("ecx",&wparam(2)); # [reload] len
1475 &set_label("base2_26");
1476 &mov ("eax",&wparam(3)); # padbit
1479 &sub ("esp",32*(5+9));
1480 &and ("esp",-512); # ensure that frame
1481 # doesn't cross page
1482 # boundary, which is
1484 # misaligned 32-byte
1487 ################################################################
1488 # expand and copy pre-calculated table to stack
1490 &vmovdqu (&X($D0),&QWP(16*(3+0),"edi"));
1491 &lea ("edx",&DWP(32*5+128,"esp")); # +128 size optimization
1492 &vmovdqu (&X($D1),&QWP(16*(3+1),"edi"));
1493 &vmovdqu (&X($D2),&QWP(16*(3+2),"edi"));
1494 &vmovdqu (&X($D3),&QWP(16*(3+3),"edi"));
1495 &vmovdqu (&X($D4),&QWP(16*(3+4),"edi"));
1496 &lea ("edi",&DWP(16*3,"edi")); # size optimization
1497 &vpermq ($D0,$D0,0b01000000); # 00001234 -> 12343434
1498 &vpermq ($D1,$D1,0b01000000);
1499 &vpermq ($D2,$D2,0b01000000);
1500 &vpermq ($D3,$D3,0b01000000);
1501 &vpermq ($D4,$D4,0b01000000);
1502 &vpshufd ($D0,$D0,0b11001000); # 12343434 -> 14243444
1503 &vpshufd ($D1,$D1,0b11001000);
1504 &vpshufd ($D2,$D2,0b11001000);
1505 &vpshufd ($D3,$D3,0b11001000);
1506 &vpshufd ($D4,$D4,0b11001000);
1507 &vmovdqa (&QWP(32*0-128,"edx"),$D0);
1508 &vmovdqu (&X($D0),&QWP(16*5,"edi"));
1509 &vmovdqa (&QWP(32*1-128,"edx"),$D1);
1510 &vmovdqu (&X($D1),&QWP(16*6,"edi"));
1511 &vmovdqa (&QWP(32*2-128,"edx"),$D2);
1512 &vmovdqu (&X($D2),&QWP(16*7,"edi"));
1513 &vmovdqa (&QWP(32*3-128,"edx"),$D3);
1514 &vmovdqu (&X($D3),&QWP(16*8,"edi"));
1515 &vmovdqa (&QWP(32*4-128,"edx"),$D4);
1516 &vpermq ($D0,$D0,0b01000000);
1517 &vpermq ($D1,$D1,0b01000000);
1518 &vpermq ($D2,$D2,0b01000000);
1519 &vpermq ($D3,$D3,0b01000000);
1520 &vpshufd ($D0,$D0,0b11001000);
1521 &vpshufd ($D1,$D1,0b11001000);
1522 &vpshufd ($D2,$D2,0b11001000);
1523 &vpshufd ($D3,$D3,0b11001000);
1524 &vmovdqa (&QWP(32*5-128,"edx"),$D0);
1525 &vmovd (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
1526 &vmovdqa (&QWP(32*6-128,"edx"),$D1);
1527 &vmovd (&X($D1),&DWP(-16*3+4*1,"edi"));
1528 &vmovdqa (&QWP(32*7-128,"edx"),$D2);
1529 &vmovd (&X($D2),&DWP(-16*3+4*2,"edi"));
1530 &vmovdqa (&QWP(32*8-128,"edx"),$D3);
1531 &vmovd (&X($D3),&DWP(-16*3+4*3,"edi"));
1532 &vmovd (&X($D4),&DWP(-16*3+4*4,"edi"));
1533 &vmovdqa ($MASK,&QWP(64,"ebx"));
1534 &neg ("eax"); # padbit
1537 &jz (&label("even"));
1543 &vmovdqu (&X($T0),&QWP(16*0,"esi"));
1545 &jb (&label("one"));
1547 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1548 &je (&label("two"));
1550 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1551 &lea ("esi",&DWP(16*3,"esi"));
1552 &lea ("ebx",&DWP(8,"ebx")); # three padbits
1553 &lea ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
1554 &jmp (&label("tail"));
1557 &lea ("esi",&DWP(16*2,"esi"));
1558 &lea ("ebx",&DWP(16,"ebx")); # two padbits
1559 &lea ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
1560 &jmp (&label("tail"));
1563 &lea ("esi",&DWP(16*1,"esi"));
1564 &vpxor ($T1,$T1,$T1);
1565 &lea ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
1566 &lea ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
1567 &jmp (&label("tail"));
1569 # (*) spots marked with '--' are data from next table entry, but they
1570 # are multiplied by 0 and therefore rendered insignificant
1572 &set_label("even",32);
1573 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1574 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1575 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1576 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1577 &lea ("esi",&DWP(16*4,"esi"));
1579 &jz (&label("tail"));
1582 ################################################################
1583 # ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
1584 # ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
1585 # ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
1586 # ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
1587 # \________/ \_______/
1588 ################################################################
1591 &vmovdqa (&QWP(32*2,"esp"),$D2);
1592 &vpsrldq ($D2,$T0,6); # splat input
1593 &vmovdqa (&QWP(32*0,"esp"),$D0);
1594 &vpsrldq ($D0,$T1,6);
1595 &vmovdqa (&QWP(32*1,"esp"),$D1);
1596 &vpunpckhqdq ($D1,$T0,$T1); # 4
1597 &vpunpcklqdq ($T0,$T0,$T1); # 0:1
1598 &vpunpcklqdq ($D2,$D2,$D0); # 2:3
1600 &vpsrlq ($D0,$D2,30);
1601 &vpsrlq ($D2,$D2,4);
1602 &vpsrlq ($T1,$T0,26);
1603 &vpsrlq ($D1,$D1,40); # 4
1604 &vpand ($D2,$D2,$MASK); # 2
1605 &vpand ($T0,$T0,$MASK); # 0
1606 &vpand ($T1,$T1,$MASK); # 1
1607 &vpand ($D0,$D0,$MASK); # 3 (*)
1608 &vpor ($D1,$D1,&QWP(0,"ebx")); # padbit, yes, always
1610 # (*) note that output is counterintuitive, inp[3:4] is
1611 # returned in $D1-2, while $D3-4 are preserved;
1618 &vpaddq ($D2,$D2,&QWP(32*2,"esp")); # add hash value
1619 &vpaddq ($T0,$T0,&QWP(32*0,"esp"));
1620 &vpaddq ($T1,$T1,&QWP(32*1,"esp"));
1621 &vpaddq ($D0,$D0,$D3);
1622 &vpaddq ($D1,$D1,$D4);
1624 ################################################################
1625 # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4
1626 # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0
1627 # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
1628 # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2
1629 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
1631 &vpmuludq ($D3,$D2,&$addr(1)); # d3 = h2*r1
1632 &vmovdqa (QWP(32*1,"esp"),$T1);
1633 &vpmuludq ($D4,$D2,&$addr(2)); # d4 = h2*r2
1634 &vmovdqa (QWP(32*3,"esp"),$D0);
1635 &vpmuludq ($D0,$D2,&$addr(7)); # d0 = h2*s3
1636 &vmovdqa (QWP(32*4,"esp"),$D1);
1637 &vpmuludq ($D1,$D2,&$addr(8)); # d1 = h2*s4
1638 &vpmuludq ($D2,$D2,&$addr(0)); # d2 = h2*r0
1640 &vpmuludq ($T2,$T0,&$addr(3)); # h0*r3
1641 &vpaddq ($D3,$D3,$T2); # d3 += h0*r3
1642 &vpmuludq ($T1,$T0,&$addr(4)); # h0*r4
1643 &vpaddq ($D4,$D4,$T1); # d4 + h0*r4
1644 &vpmuludq ($T2,$T0,&$addr(0)); # h0*r0
1645 &vpaddq ($D0,$D0,$T2); # d0 + h0*r0
1646 &vmovdqa ($T2,&QWP(32*1,"esp")); # h1
1647 &vpmuludq ($T1,$T0,&$addr(1)); # h0*r1
1648 &vpaddq ($D1,$D1,$T1); # d1 += h0*r1
1649 &vpmuludq ($T0,$T0,&$addr(2)); # h0*r2
1650 &vpaddq ($D2,$D2,$T0); # d2 += h0*r2
1652 &vpmuludq ($T1,$T2,&$addr(2)); # h1*r2
1653 &vpaddq ($D3,$D3,$T1); # d3 += h1*r2
1654 &vpmuludq ($T0,$T2,&$addr(3)); # h1*r3
1655 &vpaddq ($D4,$D4,$T0); # d4 += h1*r3
1656 &vpmuludq ($T1,$T2,&$addr(8)); # h1*s4
1657 &vpaddq ($D0,$D0,$T1); # d0 += h1*s4
1658 &vmovdqa ($T1,&QWP(32*3,"esp")); # h3
1659 &vpmuludq ($T0,$T2,&$addr(0)); # h1*r0
1660 &vpaddq ($D1,$D1,$T0); # d1 += h1*r0
1661 &vpmuludq ($T2,$T2,&$addr(1)); # h1*r1
1662 &vpaddq ($D2,$D2,$T2); # d2 += h1*r1
1664 &vpmuludq ($T0,$T1,&$addr(0)); # h3*r0
1665 &vpaddq ($D3,$D3,$T0); # d3 += h3*r0
1666 &vpmuludq ($T2,$T1,&$addr(1)); # h3*r1
1667 &vpaddq ($D4,$D4,$T2); # d4 += h3*r1
1668 &vpmuludq ($T0,$T1,&$addr(6)); # h3*s2
1669 &vpaddq ($D0,$D0,$T0); # d0 += h3*s2
1670 &vmovdqa ($T0,&QWP(32*4,"esp")); # h4
1671 &vpmuludq ($T2,$T1,&$addr(7)); # h3*s3
1672 &vpaddq ($D1,$D1,$T2); # d1+= h3*s3
1673 &vpmuludq ($T1,$T1,&$addr(8)); # h3*s4
1674 &vpaddq ($D2,$D2,$T1); # d2 += h3*s4
1676 &vpmuludq ($T2,$T0,&$addr(8)); # h4*s4
1677 &vpaddq ($D3,$D3,$T2); # d3 += h4*s4
1678 &vpmuludq ($T1,$T0,&$addr(5)); # h4*s1
1679 &vpaddq ($D0,$D0,$T1); # d0 += h4*s1
1680 &vpmuludq ($T2,$T0,&$addr(0)); # h4*r0
1681 &vpaddq ($D4,$D4,$T2); # d4 += h4*r0
1682 &vmovdqa ($MASK,&QWP(64,"ebx"));
1683 &vpmuludq ($T1,$T0,&$addr(6)); # h4*s2
1684 &vpaddq ($D1,$D1,$T1); # d1 += h4*s2
1685 &vpmuludq ($T0,$T0,&$addr(7)); # h4*s3
1686 &vpaddq ($D2,$D2,$T0); # d2 += h4*s3
1688 &vpmuladd (sub { my $i=shift; &QWP(32*$i-128,"edx"); });
1690 sub vlazy_reduction {
1691 ################################################################
1694 &vpsrlq ($T0,$D3,26);
1695 &vpand ($D3,$D3,$MASK);
1696 &vpsrlq ($T1,$D0,26);
1697 &vpand ($D0,$D0,$MASK);
1698 &vpaddq ($D4,$D4,$T0); # h3 -> h4
1699 &vpaddq ($D1,$D1,$T1); # h0 -> h1
1700 &vpsrlq ($T0,$D4,26);
1701 &vpand ($D4,$D4,$MASK);
1702 &vpsrlq ($T1,$D1,26);
1703 &vpand ($D1,$D1,$MASK);
1704 &vpaddq ($D2,$D2,$T1); # h1 -> h2
1705 &vpaddd ($D0,$D0,$T0);
1706 &vpsllq ($T0,$T0,2);
1707 &vpsrlq ($T1,$D2,26);
1708 &vpand ($D2,$D2,$MASK);
1709 &vpaddd ($D0,$D0,$T0); # h4 -> h0
1710 &vpaddd ($D3,$D3,$T1); # h2 -> h3
1711 &vpsrlq ($T1,$D3,26);
1712 &vpsrlq ($T0,$D0,26);
1713 &vpand ($D0,$D0,$MASK);
1714 &vpand ($D3,$D3,$MASK);
1715 &vpaddd ($D1,$D1,$T0); # h0 -> h1
1716 &vpaddd ($D4,$D4,$T1); # h3 -> h4
1720 &vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
1721 &vmovdqu (&X($T1),&QWP(16*1,"esi"));
1722 &vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
1723 &vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
1724 &lea ("esi",&DWP(16*4,"esi"));
1726 &jnz (&label("loop"));
1730 &and ("ebx",-64); # restore pointer
1732 &vpmuladd (sub { my $i=shift; &QWP(4+32*$i-128,"edx"); });
1736 ################################################################
1737 # horizontal addition
1739 &vpsrldq ($T0,$D0,8);
1740 &vpsrldq ($T1,$D1,8);
1741 &vpaddq ($D0,$D0,$T0);
1742 &vpsrldq ($T0,$D2,8);
1743 &vpaddq ($D1,$D1,$T1);
1744 &vpsrldq ($T1,$D3,8);
1745 &vpaddq ($D2,$D2,$T0);
1746 &vpsrldq ($T0,$D4,8);
1747 &vpaddq ($D3,$D3,$T1);
1748 &vpermq ($T1,$D0,2); # keep folding
1749 &vpaddq ($D4,$D4,$T0);
1750 &vpermq ($T0,$D1,2);
1751 &vpaddq ($D0,$D0,$T1);
1752 &vpermq ($T1,$D2,2);
1753 &vpaddq ($D1,$D1,$T0);
1754 &vpermq ($T0,$D3,2);
1755 &vpaddq ($D2,$D2,$T1);
1756 &vpermq ($T1,$D4,2);
1757 &vpaddq ($D3,$D3,$T0);
1758 &vpaddq ($D4,$D4,$T1);
1761 &je (&label("done"));
1763 ################################################################
1764 # clear all but single word
1766 &vpshufd (&X($D0),&X($D0),0b11111100);
1767 &lea ("edx",&DWP(32*5+128,"esp")); # restore pointer
1768 &vpshufd (&X($D1),&X($D1),0b11111100);
1769 &vpshufd (&X($D2),&X($D2),0b11111100);
1770 &vpshufd (&X($D3),&X($D3),0b11111100);
1771 &vpshufd (&X($D4),&X($D4),0b11111100);
1772 &jmp (&label("even"));
1774 &set_label("done",16);
1775 &vmovd (&DWP(-16*3+4*0,"edi"),"xmm0"); # store hash value
1776 &vmovd (&DWP(-16*3+4*1,"edi"),"xmm1");
1777 &vmovd (&DWP(-16*3+4*2,"edi"),"xmm2");
1778 &vmovd (&DWP(-16*3+4*3,"edi"),"xmm3");
1779 &vmovd (&DWP(-16*3+4*4,"edi"),"xmm4");
1781 &set_label("nodata");
1783 &function_end("_poly1305_blocks_avx2");
1785 &set_label("const_sse2",64);
1786 &data_word(1<<24,0, 1<<24,0, 1<<24,0, 1<<24,0);
1787 &data_word(0,0, 0,0, 0,0, 0,0);
1788 &data_word(0x03ffffff,0,0x03ffffff,0, 0x03ffffff,0, 0x03ffffff,0);
1789 &data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
1791 &asciz ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");