2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
17 # This module implements Poly1305 hash for x86_64.
25 # Add AVX512F+VL+BW code path.
27 # Numbers are cycles per processed byte with poly1305_blocks alone,
28 # measured with rdtsc at fixed clock frequency.
30 # IALU/gcc-4.8(*) AVX(**) AVX2
33 # Westmere 1.88/+120% -
34 # Sandy Bridge 1.39/+140% 1.10
35 # Haswell 1.14/+175% 1.11 0.65
36 # Skylake 1.13/+120% 0.96 0.51
37 # Silvermont 2.83/+95% -
38 # Goldmont 1.70/+180% -
39 # VIA Nano 1.82/+150% -
40 # Sledgehammer 1.38/+160% -
41 # Bulldozer 2.30/+130% 0.97
42 # Ryzen 1.15/+200% 1.08 1.18
44 # (*) improvement coefficients relative to clang are more modest and
45 # are ~50% on most processors, in both cases we are comparing to
47 # (**) SSE2 implementation was attempted, but among non-AVX processors
48 # it was faster than integer-only code only on older Intel P4 and
49 # Core processors, 50-30%, less newer processor is, but slower on
50 # contemporary ones, for example almost 2x slower on Atom, and as
51 # former are naturally disappearing, SSE2 is deemed unnecessary;
55 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
57 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
59 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
60 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
61 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
62 die "can't locate x86_64-xlate.pl";
64 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
65 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
66 $avx = ($1>=2.19) + ($1>=2.22) + ($1>=2.25) + ($1>=2.26);
69 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
70 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) {
71 $avx = ($1>=2.09) + ($1>=2.10) + 2 * ($1>=2.12);
72 $avx += 2 if ($1==2.11 && $2>=8);
75 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
76 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
77 $avx = ($1>=10) + ($1>=12);
80 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
81 $avx = ($2>=3.0) + ($2>3.0);
84 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
87 my ($ctx,$inp,$len,$padbit)=("%rdi","%rsi","%rdx","%rcx");
88 my ($mac,$nonce)=($inp,$len); # *_emit arguments
89 my ($d1,$d2,$d3, $r0,$r1,$s1)=map("%r$_",(8..13));
90 my ($h0,$h1,$h2)=("%r14","%rbx","%rbp");
92 sub poly1305_iteration {
93 # input: copy of $r1 in %rax, $h0-$h2, $r0-$r1
94 # output: $h0-$h2 *= $r0-$r1
102 mov %rax,$h0 # future $h0
112 mov $h2,$h1 # borrow $h1
116 imulq $s1,$h1 # h2*s1
121 imulq $r0,$h2 # h2*r0
123 mov \$-4,%rax # mask value
126 and $d3,%rax # last reduction step
137 ########################################################################
138 # Layout of opaque area is following.
140 # unsigned __int64 h[3]; # current hash value base 2^64
141 # unsigned __int64 r[2]; # key value base 2^64
146 .extern OPENSSL_ia32cap_P
149 .hidden poly1305_init
150 .globl poly1305_blocks
151 .hidden poly1305_blocks
153 .hidden poly1305_emit
155 .type poly1305_init,\@function,3
159 mov %rax,0($ctx) # initialize hash value
166 lea poly1305_blocks(%rip),%r10
167 lea poly1305_emit(%rip),%r11
169 $code.=<<___ if ($avx);
170 mov OPENSSL_ia32cap_P+4(%rip),%r9
171 lea poly1305_blocks_avx(%rip),%rax
172 lea poly1305_emit_avx(%rip),%rcx
173 bt \$`60-32`,%r9 # AVX?
177 $code.=<<___ if ($avx>1);
178 lea poly1305_blocks_avx2(%rip),%rax
179 bt \$`5+32`,%r9 # AVX2?
182 $code.=<<___ if ($avx>3);
183 mov \$`(1<<31|1<<21|1<<16)`,%rax
190 mov \$0x0ffffffc0fffffff,%rax
191 mov \$0x0ffffffc0ffffffc,%rcx
197 $code.=<<___ if ($flavour !~ /elf32/);
201 $code.=<<___ if ($flavour =~ /elf32/);
209 .size poly1305_init,.-poly1305_init
211 .type poly1305_blocks,\@function,4
217 jz .Lno_data # too short
233 mov $len,%r15 # reassign $len
235 mov 24($ctx),$r0 # load r
238 mov 0($ctx),$h0 # load hash value
245 add $r1,$s1 # s1 = r1 + (r1 >> 2)
250 add 0($inp),$h0 # accumulate input
255 &poly1305_iteration();
261 mov $h0,0($ctx) # store hash value
278 .cfi_adjust_cfa_offset -48
283 .size poly1305_blocks,.-poly1305_blocks
285 .type poly1305_emit,\@function,3
289 mov 0($ctx),%r8 # load hash value
294 add \$5,%r8 # compare to modulus
298 shr \$2,%r10 # did 130-bit value overfow?
302 add 0($nonce),%rax # accumulate nonce
304 mov %rax,0($mac) # write result
308 .size poly1305_emit,.-poly1305_emit
312 ########################################################################
313 # Layout of opaque area is following.
315 # unsigned __int32 h[5]; # current hash value base 2^26
316 # unsigned __int32 is_base2_26;
317 # unsigned __int64 r[2]; # key value base 2^64
318 # unsigned __int64 pad;
319 # struct { unsigned __int32 r^2, r^1, r^4, r^3; } r[9];
321 # where r^n are base 2^26 digits of degrees of multiplier key. There are
322 # 5 digits, but last four are interleaved with multiples of 5, totalling
323 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
325 my ($H0,$H1,$H2,$H3,$H4, $T0,$T1,$T2,$T3,$T4, $D0,$D1,$D2,$D3,$D4, $MASK) =
326 map("%xmm$_",(0..15));
329 .type __poly1305_block,\@abi-omnipotent
333 &poly1305_iteration();
336 .size __poly1305_block,.-__poly1305_block
338 .type __poly1305_init_avx,\@abi-omnipotent
345 lea 48+64($ctx),$ctx # size optimization
348 call __poly1305_block # r^2
350 mov \$0x3ffffff,%eax # save interleaved r^2 and r base 2^26
356 mov %eax,`16*0+0-64`($ctx)
358 mov %edx,`16*0+4-64`($ctx)
365 mov %eax,`16*1+0-64`($ctx)
366 lea (%rax,%rax,4),%eax # *5
367 mov %edx,`16*1+4-64`($ctx)
368 lea (%rdx,%rdx,4),%edx # *5
369 mov %eax,`16*2+0-64`($ctx)
371 mov %edx,`16*2+4-64`($ctx)
382 mov %eax,`16*3+0-64`($ctx)
383 lea (%rax,%rax,4),%eax # *5
384 mov %edx,`16*3+4-64`($ctx)
385 lea (%rdx,%rdx,4),%edx # *5
386 mov %eax,`16*4+0-64`($ctx)
388 mov %edx,`16*4+4-64`($ctx)
397 mov %eax,`16*5+0-64`($ctx)
398 lea (%rax,%rax,4),%eax # *5
399 mov %edx,`16*5+4-64`($ctx)
400 lea (%rdx,%rdx,4),%edx # *5
401 mov %eax,`16*6+0-64`($ctx)
403 mov %edx,`16*6+4-64`($ctx)
409 mov $d1#d,`16*7+0-64`($ctx)
410 lea ($d1,$d1,4),$d1 # *5
411 mov $d2#d,`16*7+4-64`($ctx)
412 lea ($d2,$d2,4),$d2 # *5
413 mov $d1#d,`16*8+0-64`($ctx)
414 mov $d2#d,`16*8+4-64`($ctx)
417 call __poly1305_block # r^3
419 mov \$0x3ffffff,%eax # save r^3 base 2^26
423 mov %eax,`16*0+12-64`($ctx)
427 mov %edx,`16*1+12-64`($ctx)
428 lea (%rdx,%rdx,4),%edx # *5
430 mov %edx,`16*2+12-64`($ctx)
436 mov %eax,`16*3+12-64`($ctx)
437 lea (%rax,%rax,4),%eax # *5
439 mov %eax,`16*4+12-64`($ctx)
444 mov %edx,`16*5+12-64`($ctx)
445 lea (%rdx,%rdx,4),%edx # *5
447 mov %edx,`16*6+12-64`($ctx)
452 mov $d1#d,`16*7+12-64`($ctx)
453 lea ($d1,$d1,4),$d1 # *5
454 mov $d1#d,`16*8+12-64`($ctx)
457 call __poly1305_block # r^4
459 mov \$0x3ffffff,%eax # save r^4 base 2^26
463 mov %eax,`16*0+8-64`($ctx)
467 mov %edx,`16*1+8-64`($ctx)
468 lea (%rdx,%rdx,4),%edx # *5
470 mov %edx,`16*2+8-64`($ctx)
476 mov %eax,`16*3+8-64`($ctx)
477 lea (%rax,%rax,4),%eax # *5
479 mov %eax,`16*4+8-64`($ctx)
484 mov %edx,`16*5+8-64`($ctx)
485 lea (%rdx,%rdx,4),%edx # *5
487 mov %edx,`16*6+8-64`($ctx)
492 mov $d1#d,`16*7+8-64`($ctx)
493 lea ($d1,$d1,4),$d1 # *5
494 mov $d1#d,`16*8+8-64`($ctx)
496 lea -48-64($ctx),$ctx # size [de-]optimization
498 .size __poly1305_init_avx,.-__poly1305_init_avx
500 .type poly1305_blocks_avx,\@function,4
504 mov 20($ctx),%r8d # is_base2_26
536 mov $len,%r15 # reassign $len
538 mov 0($ctx),$d1 # load hash value
542 mov 24($ctx),$r0 # load r
545 ################################# base 2^26 -> base 2^64
547 and \$`-1*(1<<31)`,$d1
548 mov $d2,$r1 # borrow $r1
550 and \$`-1*(1<<31)`,$d2
564 adc \$0,$h2 # can be partially reduced...
566 mov \$-4,$d2 # ... so reduce
579 add $r1,$s1 # s1 = r1 + (r1 >> 2)
581 add 0($inp),$h0 # accumulate input
586 call __poly1305_block
588 test $padbit,$padbit # if $padbit is zero,
589 jz .Lstore_base2_64_avx # store hash in base 2^64 format
591 ################################# base 2^64 -> base 2^26
598 and \$0x3ffffff,%rax # h[0]
600 and \$0x3ffffff,%rdx # h[1]
604 and \$0x3ffffff,$h0 # h[2]
606 and \$0x3ffffff,$h1 # h[3]
610 jz .Lstore_base2_26_avx
620 .Lstore_base2_64_avx:
623 mov $h2,16($ctx) # note that is_base2_26 is zeroed
627 .Lstore_base2_26_avx:
628 mov %rax#d,0($ctx) # store hash value base 2^26
648 .cfi_adjust_cfa_offset -48
650 .Lblocks_avx_epilogue:
671 mov $len,%r15 # reassign $len
673 mov 24($ctx),$r0 # load r
676 mov 0($ctx),$h0 # load hash value
683 add $r1,$s1 # s1 = r1 + (r1 >> 2)
688 add 0($inp),$h0 # accumulate input
694 call __poly1305_block
697 ################################# base 2^64 -> base 2^26
704 and \$0x3ffffff,%rax # h[0]
706 and \$0x3ffffff,%rdx # h[1]
710 and \$0x3ffffff,$h0 # h[2]
712 and \$0x3ffffff,$h1 # h[3]
720 movl \$1,20($ctx) # set is_base2_26
722 call __poly1305_init_avx
741 .cfi_adjust_cfa_offset -48
742 .Lbase2_64_avx_epilogue:
749 vmovd 4*0($ctx),$H0 # load hash value
757 $code.=<<___ if (!$win64);
759 .cfi_def_cfa %r11,0x60
762 $code.=<<___ if ($win64);
765 vmovdqa %xmm6,0x50(%r11)
766 vmovdqa %xmm7,0x60(%r11)
767 vmovdqa %xmm8,0x70(%r11)
768 vmovdqa %xmm9,0x80(%r11)
769 vmovdqa %xmm10,0x90(%r11)
770 vmovdqa %xmm11,0xa0(%r11)
771 vmovdqa %xmm12,0xb0(%r11)
772 vmovdqa %xmm13,0xc0(%r11)
773 vmovdqa %xmm14,0xd0(%r11)
774 vmovdqa %xmm15,0xe0(%r11)
782 vmovdqu `16*3`($ctx),$D4 # preload r0^2
783 lea `16*3+64`($ctx),$ctx # size optimization
784 lea .Lconst(%rip),%rcx
786 ################################################################
788 vmovdqu 16*2($inp),$T0
789 vmovdqu 16*3($inp),$T1
790 vmovdqa 64(%rcx),$MASK # .Lmask26
792 vpsrldq \$6,$T0,$T2 # splat input
794 vpunpckhqdq $T1,$T0,$T4 # 4
795 vpunpcklqdq $T1,$T0,$T0 # 0:1
796 vpunpcklqdq $T3,$T2,$T3 # 2:3
798 vpsrlq \$40,$T4,$T4 # 4
800 vpand $MASK,$T0,$T0 # 0
802 vpand $MASK,$T1,$T1 # 1
804 vpand $MASK,$T2,$T2 # 2
805 vpand $MASK,$T3,$T3 # 3
806 vpor 32(%rcx),$T4,$T4 # padbit, yes, always
810 # expand and copy pre-calculated table to stack
811 vmovdqu `16*1-64`($ctx),$D1
812 vmovdqu `16*2-64`($ctx),$D2
813 vpshufd \$0xEE,$D4,$D3 # 34xx -> 3434
814 vpshufd \$0x44,$D4,$D0 # xx12 -> 1212
815 vmovdqa $D3,-0x90(%r11)
816 vmovdqa $D0,0x00(%rsp)
817 vpshufd \$0xEE,$D1,$D4
818 vmovdqu `16*3-64`($ctx),$D0
819 vpshufd \$0x44,$D1,$D1
820 vmovdqa $D4,-0x80(%r11)
821 vmovdqa $D1,0x10(%rsp)
822 vpshufd \$0xEE,$D2,$D3
823 vmovdqu `16*4-64`($ctx),$D1
824 vpshufd \$0x44,$D2,$D2
825 vmovdqa $D3,-0x70(%r11)
826 vmovdqa $D2,0x20(%rsp)
827 vpshufd \$0xEE,$D0,$D4
828 vmovdqu `16*5-64`($ctx),$D2
829 vpshufd \$0x44,$D0,$D0
830 vmovdqa $D4,-0x60(%r11)
831 vmovdqa $D0,0x30(%rsp)
832 vpshufd \$0xEE,$D1,$D3
833 vmovdqu `16*6-64`($ctx),$D0
834 vpshufd \$0x44,$D1,$D1
835 vmovdqa $D3,-0x50(%r11)
836 vmovdqa $D1,0x40(%rsp)
837 vpshufd \$0xEE,$D2,$D4
838 vmovdqu `16*7-64`($ctx),$D1
839 vpshufd \$0x44,$D2,$D2
840 vmovdqa $D4,-0x40(%r11)
841 vmovdqa $D2,0x50(%rsp)
842 vpshufd \$0xEE,$D0,$D3
843 vmovdqu `16*8-64`($ctx),$D2
844 vpshufd \$0x44,$D0,$D0
845 vmovdqa $D3,-0x30(%r11)
846 vmovdqa $D0,0x60(%rsp)
847 vpshufd \$0xEE,$D1,$D4
848 vpshufd \$0x44,$D1,$D1
849 vmovdqa $D4,-0x20(%r11)
850 vmovdqa $D1,0x70(%rsp)
851 vpshufd \$0xEE,$D2,$D3
852 vmovdqa 0x00(%rsp),$D4 # preload r0^2
853 vpshufd \$0x44,$D2,$D2
854 vmovdqa $D3,-0x10(%r11)
855 vmovdqa $D2,0x80(%rsp)
861 ################################################################
862 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
863 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
864 # \___________________/
865 # ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
866 # ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
867 # \___________________/ \____________________/
869 # Note that we start with inp[2:3]*r^2. This is because it
870 # doesn't depend on reduction in previous iteration.
871 ################################################################
872 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
873 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
874 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
875 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
876 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
878 # though note that $Tx and $Hx are "reversed" in this section,
879 # and $D4 is preloaded with r0^2...
881 vpmuludq $T0,$D4,$D0 # d0 = h0*r0
882 vpmuludq $T1,$D4,$D1 # d1 = h1*r0
883 vmovdqa $H2,0x20(%r11) # offload hash
884 vpmuludq $T2,$D4,$D2 # d3 = h2*r0
885 vmovdqa 0x10(%rsp),$H2 # r1^2
886 vpmuludq $T3,$D4,$D3 # d3 = h3*r0
887 vpmuludq $T4,$D4,$D4 # d4 = h4*r0
889 vmovdqa $H0,0x00(%r11) #
890 vpmuludq 0x20(%rsp),$T4,$H0 # h4*s1
891 vmovdqa $H1,0x10(%r11) #
892 vpmuludq $T3,$H2,$H1 # h3*r1
893 vpaddq $H0,$D0,$D0 # d0 += h4*s1
894 vpaddq $H1,$D4,$D4 # d4 += h3*r1
895 vmovdqa $H3,0x30(%r11) #
896 vpmuludq $T2,$H2,$H0 # h2*r1
897 vpmuludq $T1,$H2,$H1 # h1*r1
898 vpaddq $H0,$D3,$D3 # d3 += h2*r1
899 vmovdqa 0x30(%rsp),$H3 # r2^2
900 vpaddq $H1,$D2,$D2 # d2 += h1*r1
901 vmovdqa $H4,0x40(%r11) #
902 vpmuludq $T0,$H2,$H2 # h0*r1
903 vpmuludq $T2,$H3,$H0 # h2*r2
904 vpaddq $H2,$D1,$D1 # d1 += h0*r1
906 vmovdqa 0x40(%rsp),$H4 # s2^2
907 vpaddq $H0,$D4,$D4 # d4 += h2*r2
908 vpmuludq $T1,$H3,$H1 # h1*r2
909 vpmuludq $T0,$H3,$H3 # h0*r2
910 vpaddq $H1,$D3,$D3 # d3 += h1*r2
911 vmovdqa 0x50(%rsp),$H2 # r3^2
912 vpaddq $H3,$D2,$D2 # d2 += h0*r2
913 vpmuludq $T4,$H4,$H0 # h4*s2
914 vpmuludq $T3,$H4,$H4 # h3*s2
915 vpaddq $H0,$D1,$D1 # d1 += h4*s2
916 vmovdqa 0x60(%rsp),$H3 # s3^2
917 vpaddq $H4,$D0,$D0 # d0 += h3*s2
919 vmovdqa 0x80(%rsp),$H4 # s4^2
920 vpmuludq $T1,$H2,$H1 # h1*r3
921 vpmuludq $T0,$H2,$H2 # h0*r3
922 vpaddq $H1,$D4,$D4 # d4 += h1*r3
923 vpaddq $H2,$D3,$D3 # d3 += h0*r3
924 vpmuludq $T4,$H3,$H0 # h4*s3
925 vpmuludq $T3,$H3,$H1 # h3*s3
926 vpaddq $H0,$D2,$D2 # d2 += h4*s3
927 vmovdqu 16*0($inp),$H0 # load input
928 vpaddq $H1,$D1,$D1 # d1 += h3*s3
929 vpmuludq $T2,$H3,$H3 # h2*s3
930 vpmuludq $T2,$H4,$T2 # h2*s4
931 vpaddq $H3,$D0,$D0 # d0 += h2*s3
933 vmovdqu 16*1($inp),$H1 #
934 vpaddq $T2,$D1,$D1 # d1 += h2*s4
935 vpmuludq $T3,$H4,$T3 # h3*s4
936 vpmuludq $T4,$H4,$T4 # h4*s4
937 vpsrldq \$6,$H0,$H2 # splat input
938 vpaddq $T3,$D2,$D2 # d2 += h3*s4
939 vpaddq $T4,$D3,$D3 # d3 += h4*s4
940 vpsrldq \$6,$H1,$H3 #
941 vpmuludq 0x70(%rsp),$T0,$T4 # h0*r4
942 vpmuludq $T1,$H4,$T0 # h1*s4
943 vpunpckhqdq $H1,$H0,$H4 # 4
944 vpaddq $T4,$D4,$D4 # d4 += h0*r4
945 vmovdqa -0x90(%r11),$T4 # r0^4
946 vpaddq $T0,$D0,$D0 # d0 += h1*s4
948 vpunpcklqdq $H1,$H0,$H0 # 0:1
949 vpunpcklqdq $H3,$H2,$H3 # 2:3
951 #vpsrlq \$40,$H4,$H4 # 4
952 vpsrldq \$`40/8`,$H4,$H4 # 4
954 vpand $MASK,$H0,$H0 # 0
956 vpand $MASK,$H1,$H1 # 1
957 vpand 0(%rcx),$H4,$H4 # .Lmask24
959 vpand $MASK,$H2,$H2 # 2
960 vpand $MASK,$H3,$H3 # 3
961 vpor 32(%rcx),$H4,$H4 # padbit, yes, always
963 vpaddq 0x00(%r11),$H0,$H0 # add hash value
964 vpaddq 0x10(%r11),$H1,$H1
965 vpaddq 0x20(%r11),$H2,$H2
966 vpaddq 0x30(%r11),$H3,$H3
967 vpaddq 0x40(%r11),$H4,$H4
974 ################################################################
975 # Now we accumulate (inp[0:1]+hash)*r^4
976 ################################################################
977 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
978 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
979 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
980 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
981 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
983 vpmuludq $H0,$T4,$T0 # h0*r0
984 vpmuludq $H1,$T4,$T1 # h1*r0
987 vmovdqa -0x80(%r11),$T2 # r1^4
988 vpmuludq $H2,$T4,$T0 # h2*r0
989 vpmuludq $H3,$T4,$T1 # h3*r0
992 vpmuludq $H4,$T4,$T4 # h4*r0
993 vpmuludq -0x70(%r11),$H4,$T0 # h4*s1
996 vpaddq $T0,$D0,$D0 # d0 += h4*s1
997 vpmuludq $H2,$T2,$T1 # h2*r1
998 vpmuludq $H3,$T2,$T0 # h3*r1
999 vpaddq $T1,$D3,$D3 # d3 += h2*r1
1000 vmovdqa -0x60(%r11),$T3 # r2^4
1001 vpaddq $T0,$D4,$D4 # d4 += h3*r1
1002 vpmuludq $H1,$T2,$T1 # h1*r1
1003 vpmuludq $H0,$T2,$T2 # h0*r1
1004 vpaddq $T1,$D2,$D2 # d2 += h1*r1
1005 vpaddq $T2,$D1,$D1 # d1 += h0*r1
1007 vmovdqa -0x50(%r11),$T4 # s2^4
1008 vpmuludq $H2,$T3,$T0 # h2*r2
1009 vpmuludq $H1,$T3,$T1 # h1*r2
1010 vpaddq $T0,$D4,$D4 # d4 += h2*r2
1011 vpaddq $T1,$D3,$D3 # d3 += h1*r2
1012 vmovdqa -0x40(%r11),$T2 # r3^4
1013 vpmuludq $H0,$T3,$T3 # h0*r2
1014 vpmuludq $H4,$T4,$T0 # h4*s2
1015 vpaddq $T3,$D2,$D2 # d2 += h0*r2
1016 vpaddq $T0,$D1,$D1 # d1 += h4*s2
1017 vmovdqa -0x30(%r11),$T3 # s3^4
1018 vpmuludq $H3,$T4,$T4 # h3*s2
1019 vpmuludq $H1,$T2,$T1 # h1*r3
1020 vpaddq $T4,$D0,$D0 # d0 += h3*s2
1022 vmovdqa -0x10(%r11),$T4 # s4^4
1023 vpaddq $T1,$D4,$D4 # d4 += h1*r3
1024 vpmuludq $H0,$T2,$T2 # h0*r3
1025 vpmuludq $H4,$T3,$T0 # h4*s3
1026 vpaddq $T2,$D3,$D3 # d3 += h0*r3
1027 vpaddq $T0,$D2,$D2 # d2 += h4*s3
1028 vmovdqu 16*2($inp),$T0 # load input
1029 vpmuludq $H3,$T3,$T2 # h3*s3
1030 vpmuludq $H2,$T3,$T3 # h2*s3
1031 vpaddq $T2,$D1,$D1 # d1 += h3*s3
1032 vmovdqu 16*3($inp),$T1 #
1033 vpaddq $T3,$D0,$D0 # d0 += h2*s3
1035 vpmuludq $H2,$T4,$H2 # h2*s4
1036 vpmuludq $H3,$T4,$H3 # h3*s4
1037 vpsrldq \$6,$T0,$T2 # splat input
1038 vpaddq $H2,$D1,$D1 # d1 += h2*s4
1039 vpmuludq $H4,$T4,$H4 # h4*s4
1040 vpsrldq \$6,$T1,$T3 #
1041 vpaddq $H3,$D2,$H2 # h2 = d2 + h3*s4
1042 vpaddq $H4,$D3,$H3 # h3 = d3 + h4*s4
1043 vpmuludq -0x20(%r11),$H0,$H4 # h0*r4
1044 vpmuludq $H1,$T4,$H0
1045 vpunpckhqdq $T1,$T0,$T4 # 4
1046 vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4
1047 vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4
1049 vpunpcklqdq $T1,$T0,$T0 # 0:1
1050 vpunpcklqdq $T3,$T2,$T3 # 2:3
1052 #vpsrlq \$40,$T4,$T4 # 4
1053 vpsrldq \$`40/8`,$T4,$T4 # 4
1055 vmovdqa 0x00(%rsp),$D4 # preload r0^2
1056 vpand $MASK,$T0,$T0 # 0
1058 vpand $MASK,$T1,$T1 # 1
1059 vpand 0(%rcx),$T4,$T4 # .Lmask24
1061 vpand $MASK,$T2,$T2 # 2
1062 vpand $MASK,$T3,$T3 # 3
1063 vpor 32(%rcx),$T4,$T4 # padbit, yes, always
1065 ################################################################
1066 # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
1071 vpaddq $D3,$H4,$H4 # h3 -> h4
1075 vpaddq $D0,$D1,$H1 # h0 -> h1
1082 vpaddq $D1,$H2,$H2 # h1 -> h2
1086 vpaddq $D0,$H0,$H0 # h4 -> h0
1090 vpaddq $D2,$H3,$H3 # h2 -> h3
1094 vpaddq $D0,$H1,$H1 # h0 -> h1
1098 vpaddq $D3,$H4,$H4 # h3 -> h4
1103 ################################################################
1104 # multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
1106 vpshufd \$0x10,$D4,$D4 # r0^n, xx12 -> x1x2
1117 vmovdqa $H2,0x20(%r11)
1118 vmovdqa $H0,0x00(%r11)
1119 vmovdqa $H1,0x10(%r11)
1120 vmovdqa $H3,0x30(%r11)
1121 vmovdqa $H4,0x40(%r11)
1123 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1124 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1125 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1126 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1127 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1129 vpmuludq $T2,$D4,$D2 # d2 = h2*r0
1130 vpmuludq $T0,$D4,$D0 # d0 = h0*r0
1131 vpshufd \$0x10,`16*1-64`($ctx),$H2 # r1^n
1132 vpmuludq $T1,$D4,$D1 # d1 = h1*r0
1133 vpmuludq $T3,$D4,$D3 # d3 = h3*r0
1134 vpmuludq $T4,$D4,$D4 # d4 = h4*r0
1136 vpmuludq $T3,$H2,$H0 # h3*r1
1137 vpaddq $H0,$D4,$D4 # d4 += h3*r1
1138 vpshufd \$0x10,`16*2-64`($ctx),$H3 # s1^n
1139 vpmuludq $T2,$H2,$H1 # h2*r1
1140 vpaddq $H1,$D3,$D3 # d3 += h2*r1
1141 vpshufd \$0x10,`16*3-64`($ctx),$H4 # r2^n
1142 vpmuludq $T1,$H2,$H0 # h1*r1
1143 vpaddq $H0,$D2,$D2 # d2 += h1*r1
1144 vpmuludq $T0,$H2,$H2 # h0*r1
1145 vpaddq $H2,$D1,$D1 # d1 += h0*r1
1146 vpmuludq $T4,$H3,$H3 # h4*s1
1147 vpaddq $H3,$D0,$D0 # d0 += h4*s1
1149 vpshufd \$0x10,`16*4-64`($ctx),$H2 # s2^n
1150 vpmuludq $T2,$H4,$H1 # h2*r2
1151 vpaddq $H1,$D4,$D4 # d4 += h2*r2
1152 vpmuludq $T1,$H4,$H0 # h1*r2
1153 vpaddq $H0,$D3,$D3 # d3 += h1*r2
1154 vpshufd \$0x10,`16*5-64`($ctx),$H3 # r3^n
1155 vpmuludq $T0,$H4,$H4 # h0*r2
1156 vpaddq $H4,$D2,$D2 # d2 += h0*r2
1157 vpmuludq $T4,$H2,$H1 # h4*s2
1158 vpaddq $H1,$D1,$D1 # d1 += h4*s2
1159 vpshufd \$0x10,`16*6-64`($ctx),$H4 # s3^n
1160 vpmuludq $T3,$H2,$H2 # h3*s2
1161 vpaddq $H2,$D0,$D0 # d0 += h3*s2
1163 vpmuludq $T1,$H3,$H0 # h1*r3
1164 vpaddq $H0,$D4,$D4 # d4 += h1*r3
1165 vpmuludq $T0,$H3,$H3 # h0*r3
1166 vpaddq $H3,$D3,$D3 # d3 += h0*r3
1167 vpshufd \$0x10,`16*7-64`($ctx),$H2 # r4^n
1168 vpmuludq $T4,$H4,$H1 # h4*s3
1169 vpaddq $H1,$D2,$D2 # d2 += h4*s3
1170 vpshufd \$0x10,`16*8-64`($ctx),$H3 # s4^n
1171 vpmuludq $T3,$H4,$H0 # h3*s3
1172 vpaddq $H0,$D1,$D1 # d1 += h3*s3
1173 vpmuludq $T2,$H4,$H4 # h2*s3
1174 vpaddq $H4,$D0,$D0 # d0 += h2*s3
1176 vpmuludq $T0,$H2,$H2 # h0*r4
1177 vpaddq $H2,$D4,$D4 # h4 = d4 + h0*r4
1178 vpmuludq $T4,$H3,$H1 # h4*s4
1179 vpaddq $H1,$D3,$D3 # h3 = d3 + h4*s4
1180 vpmuludq $T3,$H3,$H0 # h3*s4
1181 vpaddq $H0,$D2,$D2 # h2 = d2 + h3*s4
1182 vpmuludq $T2,$H3,$H1 # h2*s4
1183 vpaddq $H1,$D1,$D1 # h1 = d1 + h2*s4
1184 vpmuludq $T1,$H3,$H3 # h1*s4
1185 vpaddq $H3,$D0,$D0 # h0 = d0 + h1*s4
1189 vmovdqu 16*0($inp),$H0 # load input
1190 vmovdqu 16*1($inp),$H1
1192 vpsrldq \$6,$H0,$H2 # splat input
1194 vpunpckhqdq $H1,$H0,$H4 # 4
1195 vpunpcklqdq $H1,$H0,$H0 # 0:1
1196 vpunpcklqdq $H3,$H2,$H3 # 2:3
1198 vpsrlq \$40,$H4,$H4 # 4
1200 vpand $MASK,$H0,$H0 # 0
1202 vpand $MASK,$H1,$H1 # 1
1204 vpand $MASK,$H2,$H2 # 2
1205 vpand $MASK,$H3,$H3 # 3
1206 vpor 32(%rcx),$H4,$H4 # padbit, yes, always
1208 vpshufd \$0x32,`16*0-64`($ctx),$T4 # r0^n, 34xx -> x3x4
1209 vpaddq 0x00(%r11),$H0,$H0
1210 vpaddq 0x10(%r11),$H1,$H1
1211 vpaddq 0x20(%r11),$H2,$H2
1212 vpaddq 0x30(%r11),$H3,$H3
1213 vpaddq 0x40(%r11),$H4,$H4
1215 ################################################################
1216 # multiply (inp[0:1]+hash) by r^4:r^3 and accumulate
1218 vpmuludq $H0,$T4,$T0 # h0*r0
1219 vpaddq $T0,$D0,$D0 # d0 += h0*r0
1220 vpmuludq $H1,$T4,$T1 # h1*r0
1221 vpaddq $T1,$D1,$D1 # d1 += h1*r0
1222 vpmuludq $H2,$T4,$T0 # h2*r0
1223 vpaddq $T0,$D2,$D2 # d2 += h2*r0
1224 vpshufd \$0x32,`16*1-64`($ctx),$T2 # r1^n
1225 vpmuludq $H3,$T4,$T1 # h3*r0
1226 vpaddq $T1,$D3,$D3 # d3 += h3*r0
1227 vpmuludq $H4,$T4,$T4 # h4*r0
1228 vpaddq $T4,$D4,$D4 # d4 += h4*r0
1230 vpmuludq $H3,$T2,$T0 # h3*r1
1231 vpaddq $T0,$D4,$D4 # d4 += h3*r1
1232 vpshufd \$0x32,`16*2-64`($ctx),$T3 # s1
1233 vpmuludq $H2,$T2,$T1 # h2*r1
1234 vpaddq $T1,$D3,$D3 # d3 += h2*r1
1235 vpshufd \$0x32,`16*3-64`($ctx),$T4 # r2
1236 vpmuludq $H1,$T2,$T0 # h1*r1
1237 vpaddq $T0,$D2,$D2 # d2 += h1*r1
1238 vpmuludq $H0,$T2,$T2 # h0*r1
1239 vpaddq $T2,$D1,$D1 # d1 += h0*r1
1240 vpmuludq $H4,$T3,$T3 # h4*s1
1241 vpaddq $T3,$D0,$D0 # d0 += h4*s1
1243 vpshufd \$0x32,`16*4-64`($ctx),$T2 # s2
1244 vpmuludq $H2,$T4,$T1 # h2*r2
1245 vpaddq $T1,$D4,$D4 # d4 += h2*r2
1246 vpmuludq $H1,$T4,$T0 # h1*r2
1247 vpaddq $T0,$D3,$D3 # d3 += h1*r2
1248 vpshufd \$0x32,`16*5-64`($ctx),$T3 # r3
1249 vpmuludq $H0,$T4,$T4 # h0*r2
1250 vpaddq $T4,$D2,$D2 # d2 += h0*r2
1251 vpmuludq $H4,$T2,$T1 # h4*s2
1252 vpaddq $T1,$D1,$D1 # d1 += h4*s2
1253 vpshufd \$0x32,`16*6-64`($ctx),$T4 # s3
1254 vpmuludq $H3,$T2,$T2 # h3*s2
1255 vpaddq $T2,$D0,$D0 # d0 += h3*s2
1257 vpmuludq $H1,$T3,$T0 # h1*r3
1258 vpaddq $T0,$D4,$D4 # d4 += h1*r3
1259 vpmuludq $H0,$T3,$T3 # h0*r3
1260 vpaddq $T3,$D3,$D3 # d3 += h0*r3
1261 vpshufd \$0x32,`16*7-64`($ctx),$T2 # r4
1262 vpmuludq $H4,$T4,$T1 # h4*s3
1263 vpaddq $T1,$D2,$D2 # d2 += h4*s3
1264 vpshufd \$0x32,`16*8-64`($ctx),$T3 # s4
1265 vpmuludq $H3,$T4,$T0 # h3*s3
1266 vpaddq $T0,$D1,$D1 # d1 += h3*s3
1267 vpmuludq $H2,$T4,$T4 # h2*s3
1268 vpaddq $T4,$D0,$D0 # d0 += h2*s3
1270 vpmuludq $H0,$T2,$T2 # h0*r4
1271 vpaddq $T2,$D4,$D4 # d4 += h0*r4
1272 vpmuludq $H4,$T3,$T1 # h4*s4
1273 vpaddq $T1,$D3,$D3 # d3 += h4*s4
1274 vpmuludq $H3,$T3,$T0 # h3*s4
1275 vpaddq $T0,$D2,$D2 # d2 += h3*s4
1276 vpmuludq $H2,$T3,$T1 # h2*s4
1277 vpaddq $T1,$D1,$D1 # d1 += h2*s4
1278 vpmuludq $H1,$T3,$T3 # h1*s4
1279 vpaddq $T3,$D0,$D0 # d0 += h1*s4
1282 ################################################################
1283 # horizontal addition
1296 ################################################################
1301 vpaddq $H3,$D4,$D4 # h3 -> h4
1305 vpaddq $H0,$D1,$D1 # h0 -> h1
1312 vpaddq $H1,$D2,$D2 # h1 -> h2
1316 vpaddq $H4,$D0,$D0 # h4 -> h0
1320 vpaddq $H2,$D3,$D3 # h2 -> h3
1324 vpaddq $H0,$D1,$D1 # h0 -> h1
1328 vpaddq $H3,$D4,$D4 # h3 -> h4
1330 vmovd $D0,`4*0-48-64`($ctx) # save partially reduced
1331 vmovd $D1,`4*1-48-64`($ctx)
1332 vmovd $D2,`4*2-48-64`($ctx)
1333 vmovd $D3,`4*3-48-64`($ctx)
1334 vmovd $D4,`4*4-48-64`($ctx)
1336 $code.=<<___ if ($win64);
1337 vmovdqa 0x50(%r11),%xmm6
1338 vmovdqa 0x60(%r11),%xmm7
1339 vmovdqa 0x70(%r11),%xmm8
1340 vmovdqa 0x80(%r11),%xmm9
1341 vmovdqa 0x90(%r11),%xmm10
1342 vmovdqa 0xa0(%r11),%xmm11
1343 vmovdqa 0xb0(%r11),%xmm12
1344 vmovdqa 0xc0(%r11),%xmm13
1345 vmovdqa 0xd0(%r11),%xmm14
1346 vmovdqa 0xe0(%r11),%xmm15
1350 $code.=<<___ if (!$win64);
1358 .size poly1305_blocks_avx,.-poly1305_blocks_avx
1360 .type poly1305_emit_avx,\@function,3
1363 cmpl \$0,20($ctx) # is_base2_26?
1366 mov 0($ctx),%eax # load hash value base 2^26
1372 shl \$26,%rcx # base 2^26 -> base 2^64
1388 mov %r10,%rax # could be partially reduced, so reduce
1399 add \$5,%r8 # compare to modulus
1403 shr \$2,%r10 # did 130-bit value overfow?
1407 add 0($nonce),%rax # accumulate nonce
1409 mov %rax,0($mac) # write result
1413 .size poly1305_emit_avx,.-poly1305_emit_avx
1417 my ($H0,$H1,$H2,$H3,$H4, $MASK, $T4,$T0,$T1,$T2,$T3, $D0,$D1,$D2,$D3,$D4) =
1418 map("%ymm$_",(0..15));
1422 .type poly1305_blocks_avx2,\@function,4
1424 poly1305_blocks_avx2:
1426 mov 20($ctx),%r8d # is_base2_26
1458 mov $len,%r15 # reassign $len
1460 mov 0($ctx),$d1 # load hash value
1464 mov 24($ctx),$r0 # load r
1467 ################################# base 2^26 -> base 2^64
1469 and \$`-1*(1<<31)`,$d1
1470 mov $d2,$r1 # borrow $r1
1472 and \$`-1*(1<<31)`,$d2
1486 adc \$0,$h2 # can be partially reduced...
1488 mov \$-4,$d2 # ... so reduce
1501 add $r1,$s1 # s1 = r1 + (r1 >> 2)
1503 .Lbase2_26_pre_avx2:
1504 add 0($inp),$h0 # accumulate input
1510 call __poly1305_block
1514 jnz .Lbase2_26_pre_avx2
1516 test $padbit,$padbit # if $padbit is zero,
1517 jz .Lstore_base2_64_avx2 # store hash in base 2^64 format
1519 ################################# base 2^64 -> base 2^26
1526 and \$0x3ffffff,%rax # h[0]
1528 and \$0x3ffffff,%rdx # h[1]
1532 and \$0x3ffffff,$h0 # h[2]
1534 and \$0x3ffffff,$h1 # h[3]
1538 jz .Lstore_base2_26_avx2
1548 .Lstore_base2_64_avx2:
1551 mov $h2,16($ctx) # note that is_base2_26 is zeroed
1555 .Lstore_base2_26_avx2:
1556 mov %rax#d,0($ctx) # store hash value base 2^26
1576 .cfi_adjust_cfa_offset -48
1578 .Lblocks_avx2_epilogue:
1597 .Lbase2_64_avx2_body:
1599 mov $len,%r15 # reassign $len
1601 mov 24($ctx),$r0 # load r
1604 mov 0($ctx),$h0 # load hash value
1611 add $r1,$s1 # s1 = r1 + (r1 >> 2)
1616 .Lbase2_64_pre_avx2:
1617 add 0($inp),$h0 # accumulate input
1623 call __poly1305_block
1627 jnz .Lbase2_64_pre_avx2
1630 ################################# base 2^64 -> base 2^26
1637 and \$0x3ffffff,%rax # h[0]
1639 and \$0x3ffffff,%rdx # h[1]
1643 and \$0x3ffffff,$h0 # h[2]
1645 and \$0x3ffffff,$h1 # h[3]
1653 movl \$1,20($ctx) # set is_base2_26
1655 call __poly1305_init_avx
1658 mov %r15,$len # restore $len
1659 mov OPENSSL_ia32cap_P+8(%rip),%r10d
1660 mov \$`(1<<31|1<<30|1<<16)`,%r11d
1676 .cfi_adjust_cfa_offset -48
1677 .Lbase2_64_avx2_epilogue:
1684 mov OPENSSL_ia32cap_P+8(%rip),%r10d
1685 mov \$`(1<<31|1<<30|1<<16)`,%r11d
1686 vmovd 4*0($ctx),%x#$H0 # load hash value base 2^26
1687 vmovd 4*1($ctx),%x#$H1
1688 vmovd 4*2($ctx),%x#$H2
1689 vmovd 4*3($ctx),%x#$H3
1690 vmovd 4*4($ctx),%x#$H4
1694 $code.=<<___ if ($avx>2);
1698 cmp %r11d,%r10d # check for AVX512F+BW+VL
1702 $code.=<<___ if (!$win64);
1704 .cfi_def_cfa %r11,16
1707 $code.=<<___ if ($win64);
1708 lea -0xf8(%rsp),%r11
1710 vmovdqa %xmm6,0x50(%r11)
1711 vmovdqa %xmm7,0x60(%r11)
1712 vmovdqa %xmm8,0x70(%r11)
1713 vmovdqa %xmm9,0x80(%r11)
1714 vmovdqa %xmm10,0x90(%r11)
1715 vmovdqa %xmm11,0xa0(%r11)
1716 vmovdqa %xmm12,0xb0(%r11)
1717 vmovdqa %xmm13,0xc0(%r11)
1718 vmovdqa %xmm14,0xd0(%r11)
1719 vmovdqa %xmm15,0xe0(%r11)
1723 lea .Lconst(%rip),%rcx
1724 lea 48+64($ctx),$ctx # size optimization
1725 vmovdqa 96(%rcx),$T0 # .Lpermd_avx2
1727 # expand and copy pre-calculated table to stack
1728 vmovdqu `16*0-64`($ctx),%x#$T2
1730 vmovdqu `16*1-64`($ctx),%x#$T3
1731 vmovdqu `16*2-64`($ctx),%x#$T4
1732 vmovdqu `16*3-64`($ctx),%x#$D0
1733 vmovdqu `16*4-64`($ctx),%x#$D1
1734 vmovdqu `16*5-64`($ctx),%x#$D2
1735 lea 0x90(%rsp),%rax # size optimization
1736 vmovdqu `16*6-64`($ctx),%x#$D3
1737 vpermd $T2,$T0,$T2 # 00003412 -> 14243444
1738 vmovdqu `16*7-64`($ctx),%x#$D4
1740 vmovdqu `16*8-64`($ctx),%x#$MASK
1742 vmovdqa $T2,0x00(%rsp)
1744 vmovdqa $T3,0x20-0x90(%rax)
1746 vmovdqa $T4,0x40-0x90(%rax)
1748 vmovdqa $D0,0x60-0x90(%rax)
1750 vmovdqa $D1,0x80-0x90(%rax)
1752 vmovdqa $D2,0xa0-0x90(%rax)
1753 vpermd $MASK,$T0,$MASK
1754 vmovdqa $D3,0xc0-0x90(%rax)
1755 vmovdqa $D4,0xe0-0x90(%rax)
1756 vmovdqa $MASK,0x100-0x90(%rax)
1757 vmovdqa 64(%rcx),$MASK # .Lmask26
1759 ################################################################
1761 vmovdqu 16*0($inp),%x#$T0
1762 vmovdqu 16*1($inp),%x#$T1
1763 vinserti128 \$1,16*2($inp),$T0,$T0
1764 vinserti128 \$1,16*3($inp),$T1,$T1
1767 vpsrldq \$6,$T0,$T2 # splat input
1769 vpunpckhqdq $T1,$T0,$T4 # 4
1770 vpunpcklqdq $T3,$T2,$T2 # 2:3
1771 vpunpcklqdq $T1,$T0,$T0 # 0:1
1776 vpsrlq \$40,$T4,$T4 # 4
1777 vpand $MASK,$T2,$T2 # 2
1778 vpand $MASK,$T0,$T0 # 0
1779 vpand $MASK,$T1,$T1 # 1
1780 vpand $MASK,$T3,$T3 # 3
1781 vpor 32(%rcx),$T4,$T4 # padbit, yes, always
1783 vpaddq $H2,$T2,$H2 # accumulate input
1790 ################################################################
1791 # ((inp[0]*r^4+inp[4])*r^4+inp[ 8])*r^4
1792 # ((inp[1]*r^4+inp[5])*r^4+inp[ 9])*r^3
1793 # ((inp[2]*r^4+inp[6])*r^4+inp[10])*r^2
1794 # ((inp[3]*r^4+inp[7])*r^4+inp[11])*r^1
1795 # \________/\__________/
1796 ################################################################
1797 #vpaddq $H2,$T2,$H2 # accumulate input
1799 vmovdqa `32*0`(%rsp),$T0 # r0^4
1801 vmovdqa `32*1`(%rsp),$T1 # r1^4
1803 vmovdqa `32*3`(%rsp),$T2 # r2^4
1805 vmovdqa `32*6-0x90`(%rax),$T3 # s3^4
1806 vmovdqa `32*8-0x90`(%rax),$S4 # s4^4
1808 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1809 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1810 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1811 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1812 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1814 # however, as h2 is "chronologically" first one available pull
1815 # corresponding operations up, so it's
1817 # d4 = h2*r2 + h4*r0 + h3*r1 + h1*r3 + h0*r4
1818 # d3 = h2*r1 + h3*r0 + h1*r2 + h0*r3 + h4*5*r4
1819 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1820 # d1 = h2*5*r4 + h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3
1821 # d0 = h2*5*r3 + h0*r0 + h4*5*r1 + h3*5*r2 + h1*5*r4
1823 vpmuludq $H2,$T0,$D2 # d2 = h2*r0
1824 vpmuludq $H2,$T1,$D3 # d3 = h2*r1
1825 vpmuludq $H2,$T2,$D4 # d4 = h2*r2
1826 vpmuludq $H2,$T3,$D0 # d0 = h2*s3
1827 vpmuludq $H2,$S4,$D1 # d1 = h2*s4
1829 vpmuludq $H0,$T1,$T4 # h0*r1
1830 vpmuludq $H1,$T1,$H2 # h1*r1, borrow $H2 as temp
1831 vpaddq $T4,$D1,$D1 # d1 += h0*r1
1832 vpaddq $H2,$D2,$D2 # d2 += h1*r1
1833 vpmuludq $H3,$T1,$T4 # h3*r1
1834 vpmuludq `32*2`(%rsp),$H4,$H2 # h4*s1
1835 vpaddq $T4,$D4,$D4 # d4 += h3*r1
1836 vpaddq $H2,$D0,$D0 # d0 += h4*s1
1837 vmovdqa `32*4-0x90`(%rax),$T1 # s2
1839 vpmuludq $H0,$T0,$T4 # h0*r0
1840 vpmuludq $H1,$T0,$H2 # h1*r0
1841 vpaddq $T4,$D0,$D0 # d0 += h0*r0
1842 vpaddq $H2,$D1,$D1 # d1 += h1*r0
1843 vpmuludq $H3,$T0,$T4 # h3*r0
1844 vpmuludq $H4,$T0,$H2 # h4*r0
1845 vmovdqu 16*0($inp),%x#$T0 # load input
1846 vpaddq $T4,$D3,$D3 # d3 += h3*r0
1847 vpaddq $H2,$D4,$D4 # d4 += h4*r0
1848 vinserti128 \$1,16*2($inp),$T0,$T0
1850 vpmuludq $H3,$T1,$T4 # h3*s2
1851 vpmuludq $H4,$T1,$H2 # h4*s2
1852 vmovdqu 16*1($inp),%x#$T1
1853 vpaddq $T4,$D0,$D0 # d0 += h3*s2
1854 vpaddq $H2,$D1,$D1 # d1 += h4*s2
1855 vmovdqa `32*5-0x90`(%rax),$H2 # r3
1856 vpmuludq $H1,$T2,$T4 # h1*r2
1857 vpmuludq $H0,$T2,$T2 # h0*r2
1858 vpaddq $T4,$D3,$D3 # d3 += h1*r2
1859 vpaddq $T2,$D2,$D2 # d2 += h0*r2
1860 vinserti128 \$1,16*3($inp),$T1,$T1
1863 vpmuludq $H1,$H2,$T4 # h1*r3
1864 vpmuludq $H0,$H2,$H2 # h0*r3
1865 vpsrldq \$6,$T0,$T2 # splat input
1866 vpaddq $T4,$D4,$D4 # d4 += h1*r3
1867 vpaddq $H2,$D3,$D3 # d3 += h0*r3
1868 vpmuludq $H3,$T3,$T4 # h3*s3
1869 vpmuludq $H4,$T3,$H2 # h4*s3
1871 vpaddq $T4,$D1,$D1 # d1 += h3*s3
1872 vpaddq $H2,$D2,$D2 # d2 += h4*s3
1873 vpunpckhqdq $T1,$T0,$T4 # 4
1875 vpmuludq $H3,$S4,$H3 # h3*s4
1876 vpmuludq $H4,$S4,$H4 # h4*s4
1877 vpunpcklqdq $T1,$T0,$T0 # 0:1
1878 vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4
1879 vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4
1880 vpunpcklqdq $T3,$T2,$T3 # 2:3
1881 vpmuludq `32*7-0x90`(%rax),$H0,$H4 # h0*r4
1882 vpmuludq $H1,$S4,$H0 # h1*s4
1883 vmovdqa 64(%rcx),$MASK # .Lmask26
1884 vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4
1885 vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4
1887 ################################################################
1888 # lazy reduction (interleaved with tail of input splat)
1892 vpaddq $D3,$H4,$H4 # h3 -> h4
1896 vpaddq $D0,$D1,$H1 # h0 -> h1
1905 vpaddq $D1,$H2,$H2 # h1 -> h2
1909 vpaddq $D4,$H0,$H0 # h4 -> h0
1911 vpand $MASK,$T2,$T2 # 2
1916 vpaddq $D2,$H3,$H3 # h2 -> h3
1918 vpaddq $T2,$H2,$H2 # modulo-scheduled
1923 vpaddq $D0,$H1,$H1 # h0 -> h1
1925 vpsrlq \$40,$T4,$T4 # 4
1929 vpaddq $D3,$H4,$H4 # h3 -> h4
1931 vpand $MASK,$T0,$T0 # 0
1932 vpand $MASK,$T1,$T1 # 1
1933 vpand $MASK,$T3,$T3 # 3
1934 vpor 32(%rcx),$T4,$T4 # padbit, yes, always
1941 ################################################################
1942 # while above multiplications were by r^4 in all lanes, in last
1943 # iteration we multiply least significant lane by r^4 and most
1944 # significant one by r, so copy of above except that references
1945 # to the precomputed table are displaced by 4...
1947 #vpaddq $H2,$T2,$H2 # accumulate input
1949 vmovdqu `32*0+4`(%rsp),$T0 # r0^4
1951 vmovdqu `32*1+4`(%rsp),$T1 # r1^4
1953 vmovdqu `32*3+4`(%rsp),$T2 # r2^4
1955 vmovdqu `32*6+4-0x90`(%rax),$T3 # s3^4
1956 vmovdqu `32*8+4-0x90`(%rax),$S4 # s4^4
1958 vpmuludq $H2,$T0,$D2 # d2 = h2*r0
1959 vpmuludq $H2,$T1,$D3 # d3 = h2*r1
1960 vpmuludq $H2,$T2,$D4 # d4 = h2*r2
1961 vpmuludq $H2,$T3,$D0 # d0 = h2*s3
1962 vpmuludq $H2,$S4,$D1 # d1 = h2*s4
1964 vpmuludq $H0,$T1,$T4 # h0*r1
1965 vpmuludq $H1,$T1,$H2 # h1*r1
1966 vpaddq $T4,$D1,$D1 # d1 += h0*r1
1967 vpaddq $H2,$D2,$D2 # d2 += h1*r1
1968 vpmuludq $H3,$T1,$T4 # h3*r1
1969 vpmuludq `32*2+4`(%rsp),$H4,$H2 # h4*s1
1970 vpaddq $T4,$D4,$D4 # d4 += h3*r1
1971 vpaddq $H2,$D0,$D0 # d0 += h4*s1
1973 vpmuludq $H0,$T0,$T4 # h0*r0
1974 vpmuludq $H1,$T0,$H2 # h1*r0
1975 vpaddq $T4,$D0,$D0 # d0 += h0*r0
1976 vmovdqu `32*4+4-0x90`(%rax),$T1 # s2
1977 vpaddq $H2,$D1,$D1 # d1 += h1*r0
1978 vpmuludq $H3,$T0,$T4 # h3*r0
1979 vpmuludq $H4,$T0,$H2 # h4*r0
1980 vpaddq $T4,$D3,$D3 # d3 += h3*r0
1981 vpaddq $H2,$D4,$D4 # d4 += h4*r0
1983 vpmuludq $H3,$T1,$T4 # h3*s2
1984 vpmuludq $H4,$T1,$H2 # h4*s2
1985 vpaddq $T4,$D0,$D0 # d0 += h3*s2
1986 vpaddq $H2,$D1,$D1 # d1 += h4*s2
1987 vmovdqu `32*5+4-0x90`(%rax),$H2 # r3
1988 vpmuludq $H1,$T2,$T4 # h1*r2
1989 vpmuludq $H0,$T2,$T2 # h0*r2
1990 vpaddq $T4,$D3,$D3 # d3 += h1*r2
1991 vpaddq $T2,$D2,$D2 # d2 += h0*r2
1993 vpmuludq $H1,$H2,$T4 # h1*r3
1994 vpmuludq $H0,$H2,$H2 # h0*r3
1995 vpaddq $T4,$D4,$D4 # d4 += h1*r3
1996 vpaddq $H2,$D3,$D3 # d3 += h0*r3
1997 vpmuludq $H3,$T3,$T4 # h3*s3
1998 vpmuludq $H4,$T3,$H2 # h4*s3
1999 vpaddq $T4,$D1,$D1 # d1 += h3*s3
2000 vpaddq $H2,$D2,$D2 # d2 += h4*s3
2002 vpmuludq $H3,$S4,$H3 # h3*s4
2003 vpmuludq $H4,$S4,$H4 # h4*s4
2004 vpaddq $H3,$D2,$H2 # h2 = d2 + h3*r4
2005 vpaddq $H4,$D3,$H3 # h3 = d3 + h4*r4
2006 vpmuludq `32*7+4-0x90`(%rax),$H0,$H4 # h0*r4
2007 vpmuludq $H1,$S4,$H0 # h1*s4
2008 vmovdqa 64(%rcx),$MASK # .Lmask26
2009 vpaddq $H4,$D4,$H4 # h4 = d4 + h0*r4
2010 vpaddq $H0,$D0,$H0 # h0 = d0 + h1*s4
2012 ################################################################
2013 # horizontal addition
2026 vpermq \$0x2,$H3,$T3
2027 vpermq \$0x2,$H4,$T4
2028 vpermq \$0x2,$H0,$T0
2029 vpermq \$0x2,$D1,$T1
2030 vpermq \$0x2,$H2,$T2
2037 ################################################################
2042 vpaddq $D3,$H4,$H4 # h3 -> h4
2046 vpaddq $D0,$D1,$H1 # h0 -> h1
2053 vpaddq $D1,$H2,$H2 # h1 -> h2
2057 vpaddq $D4,$H0,$H0 # h4 -> h0
2061 vpaddq $D2,$H3,$H3 # h2 -> h3
2065 vpaddq $D0,$H1,$H1 # h0 -> h1
2069 vpaddq $D3,$H4,$H4 # h3 -> h4
2071 vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced
2072 vmovd %x#$H1,`4*1-48-64`($ctx)
2073 vmovd %x#$H2,`4*2-48-64`($ctx)
2074 vmovd %x#$H3,`4*3-48-64`($ctx)
2075 vmovd %x#$H4,`4*4-48-64`($ctx)
2077 $code.=<<___ if ($win64);
2078 vmovdqa 0x50(%r11),%xmm6
2079 vmovdqa 0x60(%r11),%xmm7
2080 vmovdqa 0x70(%r11),%xmm8
2081 vmovdqa 0x80(%r11),%xmm9
2082 vmovdqa 0x90(%r11),%xmm10
2083 vmovdqa 0xa0(%r11),%xmm11
2084 vmovdqa 0xb0(%r11),%xmm12
2085 vmovdqa 0xc0(%r11),%xmm13
2086 vmovdqa 0xd0(%r11),%xmm14
2087 vmovdqa 0xe0(%r11),%xmm15
2091 $code.=<<___ if (!$win64);
2099 .size poly1305_blocks_avx2,.-poly1305_blocks_avx2
2101 #######################################################################
2103 # On entry we have input length divisible by 64. But since inner loop
2104 # processes 128 bytes per iteration, cases when length is not divisible
2105 # by 128 are handled by passing tail 64 bytes to .Ltail_avx2. For this
2106 # reason stack layout is kept identical to poly1305_blocks_avx2. If not
2107 # for this tail, we wouldn't have to even allocate stack frame...
2109 my ($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4) = map("%ymm$_",(16..24));
2110 my ($M0,$M1,$M2,$M3,$M4) = map("%ymm$_",(25..29));
2111 my $PADBIT="%zmm30";
2112 my $GATHER="%ymm31";
2115 .type poly1305_blocks_avx512,\@function,4
2117 poly1305_blocks_avx512:
2122 $code.=<<___ if (!$win64);
2124 .cfi_def_cfa %r11,16
2127 $code.=<<___ if ($win64);
2128 lea -0xf8(%rsp),%r11
2130 vmovdqa %xmm6,0x50(%r11)
2131 vmovdqa %xmm7,0x60(%r11)
2132 vmovdqa %xmm8,0x70(%r11)
2133 vmovdqa32 %xmm9,0x80(%r11)
2134 vmovdqa32 %xmm10,0x90(%r11)
2135 vmovdqa32 %xmm11,0xa0(%r11)
2136 vmovdqa32 %xmm12,0xb0(%r11)
2137 vmovdqa32 %xmm13,0xc0(%r11)
2138 vmovdqa32 %xmm14,0xd0(%r11)
2139 vmovdqa32 %xmm15,0xe0(%r11)
2143 lea .Lconst(%rip),%rcx
2144 lea 48+64($ctx),$ctx # size optimization
2145 vmovdqa 96(%rcx),$T2 # .Lpermd_avx2
2147 # expand pre-calculated table
2148 vmovdqu32 `16*0-64`($ctx),%x#$R0
2150 vmovdqu32 `16*1-64`($ctx),%x#$R1
2151 vmovdqu32 `16*2-64`($ctx),%x#$S1
2152 vmovdqu32 `16*3-64`($ctx),%x#$R2
2153 vmovdqu32 `16*4-64`($ctx),%x#$S2
2154 vmovdqu32 `16*5-64`($ctx),%x#$R3
2155 vmovdqu32 `16*6-64`($ctx),%x#$S3
2156 vmovdqu32 `16*7-64`($ctx),%x#$R4
2157 vmovdqu32 `16*8-64`($ctx),%x#$S4
2158 vpermd $R0,$T2,$R0 # 00003412 -> 14243444
2159 vmovdqa64 64(%rcx),$MASK # .Lmask26
2163 vmovdqa32 $R0,0x00(%rsp) # save in case $len%128 != 0
2164 vpsrlq \$32,$R0,$T0 # 14243444 -> 01020304
2166 vmovdqa32 $R1,0x20(%rsp)
2169 vmovdqa32 $S1,0x40(%rsp)
2172 vmovdqa32 $R2,0x60(%rsp)
2174 vmovdqa32 $S2,0x80(%rsp)
2175 vmovdqa32 $R3,0xa0(%rsp)
2176 vmovdqa32 $S3,0xc0(%rsp)
2177 vmovdqa32 $R4,0xe0(%rsp)
2178 vmovdqa32 $S4,0x100(%rsp)
2180 ################################################################
2181 # calculate 5th through 8th powers of the key
2183 # d0 = r0'*r0 + r1'*5*r4 + r2'*5*r3 + r3'*5*r2 + r4'*5*r1
2184 # d1 = r0'*r1 + r1'*r0 + r2'*5*r4 + r3'*5*r3 + r4'*5*r2
2185 # d2 = r0'*r2 + r1'*r1 + r2'*r0 + r3'*5*r4 + r4'*5*r3
2186 # d3 = r0'*r3 + r1'*r2 + r2'*r1 + r3'*r0 + r4'*5*r4
2187 # d4 = r0'*r4 + r1'*r3 + r2'*r2 + r3'*r1 + r4'*r0
2189 vpmuludq $T0,$R0,$D0 # d0 = r0'*r0
2190 vpmuludq $T0,$R1,$D1 # d1 = r0'*r1
2191 vpmuludq $T0,$R2,$D2 # d2 = r0'*r2
2192 vpmuludq $T0,$R3,$D3 # d3 = r0'*r3
2193 vpmuludq $T0,$R4,$D4 # d4 = r0'*r4
2196 vpmuludq $T1,$S4,$M0
2197 vpmuludq $T1,$R0,$M1
2198 vpmuludq $T1,$R1,$M2
2199 vpmuludq $T1,$R2,$M3
2200 vpmuludq $T1,$R3,$M4
2202 vpaddq $M0,$D0,$D0 # d0 += r1'*5*r4
2203 vpaddq $M1,$D1,$D1 # d1 += r1'*r0
2204 vpaddq $M2,$D2,$D2 # d2 += r1'*r1
2205 vpaddq $M3,$D3,$D3 # d3 += r1'*r2
2206 vpaddq $M4,$D4,$D4 # d4 += r1'*r3
2208 vpmuludq $T2,$S3,$M0
2209 vpmuludq $T2,$S4,$M1
2210 vpmuludq $T2,$R1,$M3
2211 vpmuludq $T2,$R2,$M4
2212 vpmuludq $T2,$R0,$M2
2214 vpaddq $M0,$D0,$D0 # d0 += r2'*5*r3
2215 vpaddq $M1,$D1,$D1 # d1 += r2'*5*r4
2216 vpaddq $M3,$D3,$D3 # d3 += r2'*r1
2217 vpaddq $M4,$D4,$D4 # d4 += r2'*r2
2218 vpaddq $M2,$D2,$D2 # d2 += r2'*r0
2220 vpmuludq $T3,$S2,$M0
2221 vpmuludq $T3,$R0,$M3
2222 vpmuludq $T3,$R1,$M4
2223 vpmuludq $T3,$S3,$M1
2224 vpmuludq $T3,$S4,$M2
2225 vpaddq $M0,$D0,$D0 # d0 += r3'*5*r2
2226 vpaddq $M3,$D3,$D3 # d3 += r3'*r0
2227 vpaddq $M4,$D4,$D4 # d4 += r3'*r1
2228 vpaddq $M1,$D1,$D1 # d1 += r3'*5*r3
2229 vpaddq $M2,$D2,$D2 # d2 += r3'*5*r4
2231 vpmuludq $T4,$S4,$M3
2232 vpmuludq $T4,$R0,$M4
2233 vpmuludq $T4,$S1,$M0
2234 vpmuludq $T4,$S2,$M1
2235 vpmuludq $T4,$S3,$M2
2236 vpaddq $M3,$D3,$D3 # d3 += r2'*5*r4
2237 vpaddq $M4,$D4,$D4 # d4 += r2'*r0
2238 vpaddq $M0,$D0,$D0 # d0 += r2'*5*r1
2239 vpaddq $M1,$D1,$D1 # d1 += r2'*5*r2
2240 vpaddq $M2,$D2,$D2 # d2 += r2'*5*r3
2242 ################################################################
2244 vmovdqu64 16*0($inp),%z#$T3
2245 vmovdqu64 16*4($inp),%z#$T4
2248 ################################################################
2252 vpandq $MASK,$D3,$D3
2253 vpaddq $M3,$D4,$D4 # d3 -> d4
2256 vpandq $MASK,$D0,$D0
2257 vpaddq $M0,$D1,$D1 # d0 -> d1
2260 vpandq $MASK,$D4,$D4
2263 vpandq $MASK,$D1,$D1
2264 vpaddq $M1,$D2,$D2 # d1 -> d2
2268 vpaddq $M4,$D0,$D0 # d4 -> d0
2271 vpandq $MASK,$D2,$D2
2272 vpaddq $M2,$D3,$D3 # d2 -> d3
2275 vpandq $MASK,$D0,$D0
2276 vpaddq $M0,$D1,$D1 # d0 -> d1
2279 vpandq $MASK,$D3,$D3
2280 vpaddq $M3,$D4,$D4 # d3 -> d4
2283 map(s/%y/%z/,($T4,$T0,$T1,$T2,$T3)); # switch to %zmm domain
2284 map(s/%y/%z/,($M4,$M0,$M1,$M2,$M3));
2285 map(s/%y/%z/,($D0,$D1,$D2,$D3,$D4));
2286 map(s/%y/%z/,($R0,$R1,$R2,$R3,$R4, $S1,$S2,$S3,$S4));
2287 map(s/%y/%z/,($H0,$H1,$H2,$H3,$H4));
2288 map(s/%y/%z/,($MASK));
2290 ################################################################
2291 # at this point we have 14243444 in $R0-$S4 and 05060708 in
2294 vpunpcklqdq $T4,$T3,$T0 # transpose input
2295 vpunpckhqdq $T4,$T3,$T4
2297 # ... since input 64-bit lanes are ordered as 73625140, we could
2298 # "vperm" it to 76543210 (here and in each loop iteration), *or*
2299 # we could just flow along, hence the goal for $R0-$S4 is
2300 # 1858286838784888 ...
2302 vmovdqa32 128(%rcx),$M0 # .Lpermd_avx512:
2306 vpermd $R0,$M0,$R0 # 14243444 -> 1---2---3---4---
2312 vpermd $D0,$M0,${R0}{%k1} # 05060708 -> 1858286838784888
2313 vpermd $D1,$M0,${R1}{%k1}
2314 vpermd $D2,$M0,${R2}{%k1}
2315 vpermd $D3,$M0,${R3}{%k1}
2316 vpermd $D4,$M0,${R4}{%k1}
2318 vpslld \$2,$R1,$S1 # *5
2327 vpbroadcastq %x#$MASK,$MASK
2328 vpbroadcastq 32(%rcx),$PADBIT # .L129
2330 vpsrlq \$52,$T0,$T2 # splat input
2335 vpsrlq \$40,$T4,$T4 # 4
2336 vpandq $MASK,$T2,$T2 # 2
2337 vpandq $MASK,$T0,$T0 # 0
2338 #vpandq $MASK,$T1,$T1 # 1
2339 #vpandq $MASK,$T3,$T3 # 3
2340 #vporq $PADBIT,$T4,$T4 # padbit, yes, always
2342 vpaddq $H2,$T2,$H2 # accumulate input
2349 ################################################################
2350 # ((inp[0]*r^8+inp[ 8])*r^8+inp[16])*r^8
2351 # ((inp[1]*r^8+inp[ 9])*r^8+inp[17])*r^7
2352 # ((inp[2]*r^8+inp[10])*r^8+inp[18])*r^6
2353 # ((inp[3]*r^8+inp[11])*r^8+inp[19])*r^5
2354 # ((inp[4]*r^8+inp[12])*r^8+inp[20])*r^4
2355 # ((inp[5]*r^8+inp[13])*r^8+inp[21])*r^3
2356 # ((inp[6]*r^8+inp[14])*r^8+inp[22])*r^2
2357 # ((inp[7]*r^8+inp[15])*r^8+inp[23])*r^1
2358 # \________/\___________/
2359 ################################################################
2360 #vpaddq $H2,$T2,$H2 # accumulate input
2362 # d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
2363 # d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
2364 # d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
2365 # d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
2366 # d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
2368 # however, as h2 is "chronologically" first one available pull
2369 # corresponding operations up, so it's
2371 # d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4
2372 # d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0
2373 # d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
2374 # d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2
2375 # d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
2377 vpmuludq $H2,$R1,$D3 # d3 = h2*r1
2379 vpmuludq $H2,$R2,$D4 # d4 = h2*r2
2380 vpandq $MASK,$T1,$T1 # 1
2381 vpmuludq $H2,$S3,$D0 # d0 = h2*s3
2382 vpandq $MASK,$T3,$T3 # 3
2383 vpmuludq $H2,$S4,$D1 # d1 = h2*s4
2384 vporq $PADBIT,$T4,$T4 # padbit, yes, always
2385 vpmuludq $H2,$R0,$D2 # d2 = h2*r0
2386 vpaddq $H1,$T1,$H1 # accumulate input
2390 vmovdqu64 16*0($inp),$T3 # load input
2391 vmovdqu64 16*4($inp),$T4
2393 vpmuludq $H0,$R3,$M3
2394 vpmuludq $H0,$R4,$M4
2395 vpmuludq $H0,$R0,$M0
2396 vpmuludq $H0,$R1,$M1
2397 vpaddq $M3,$D3,$D3 # d3 += h0*r3
2398 vpaddq $M4,$D4,$D4 # d4 += h0*r4
2399 vpaddq $M0,$D0,$D0 # d0 += h0*r0
2400 vpaddq $M1,$D1,$D1 # d1 += h0*r1
2402 vpmuludq $H1,$R2,$M3
2403 vpmuludq $H1,$R3,$M4
2404 vpmuludq $H1,$S4,$M0
2405 vpmuludq $H0,$R2,$M2
2406 vpaddq $M3,$D3,$D3 # d3 += h1*r2
2407 vpaddq $M4,$D4,$D4 # d4 += h1*r3
2408 vpaddq $M0,$D0,$D0 # d0 += h1*s4
2409 vpaddq $M2,$D2,$D2 # d2 += h0*r2
2411 vpunpcklqdq $T4,$T3,$T0 # transpose input
2412 vpunpckhqdq $T4,$T3,$T4
2414 vpmuludq $H3,$R0,$M3
2415 vpmuludq $H3,$R1,$M4
2416 vpmuludq $H1,$R0,$M1
2417 vpmuludq $H1,$R1,$M2
2418 vpaddq $M3,$D3,$D3 # d3 += h3*r0
2419 vpaddq $M4,$D4,$D4 # d4 += h3*r1
2420 vpaddq $M1,$D1,$D1 # d1 += h1*r0
2421 vpaddq $M2,$D2,$D2 # d2 += h1*r1
2423 vpmuludq $H4,$S4,$M3
2424 vpmuludq $H4,$R0,$M4
2425 vpmuludq $H3,$S2,$M0
2426 vpmuludq $H3,$S3,$M1
2427 vpaddq $M3,$D3,$D3 # d3 += h4*s4
2428 vpmuludq $H3,$S4,$M2
2429 vpaddq $M4,$D4,$D4 # d4 += h4*r0
2430 vpaddq $M0,$D0,$D0 # d0 += h3*s2
2431 vpaddq $M1,$D1,$D1 # d1 += h3*s3
2432 vpaddq $M2,$D2,$D2 # d2 += h3*s4
2434 vpmuludq $H4,$S1,$M0
2435 vpmuludq $H4,$S2,$M1
2436 vpmuludq $H4,$S3,$M2
2437 vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1
2438 vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2
2439 vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3
2441 ################################################################
2442 # lazy reduction (interleaved with input splat)
2444 vpsrlq \$52,$T0,$T2 # splat input
2448 vpandq $MASK,$D3,$D3
2449 vpaddq $H3,$D4,$H4 # h3 -> h4
2454 vpandq $MASK,$H0,$H0
2455 vpaddq $D0,$H1,$H1 # h0 -> h1
2457 vpandq $MASK,$T2,$T2 # 2
2460 vpandq $MASK,$H4,$H4
2463 vpandq $MASK,$H1,$H1
2464 vpaddq $D1,$H2,$H2 # h1 -> h2
2468 vpaddq $D4,$H0,$H0 # h4 -> h0
2470 vpaddq $T2,$H2,$H2 # modulo-scheduled
2474 vpandq $MASK,$H2,$H2
2475 vpaddq $D2,$D3,$H3 # h2 -> h3
2480 vpandq $MASK,$H0,$H0
2481 vpaddq $D0,$H1,$H1 # h0 -> h1
2483 vpsrlq \$40,$T4,$T4 # 4
2486 vpandq $MASK,$H3,$H3
2487 vpaddq $D3,$H4,$H4 # h3 -> h4
2489 vpandq $MASK,$T0,$T0 # 0
2490 #vpandq $MASK,$T1,$T1 # 1
2491 #vpandq $MASK,$T3,$T3 # 3
2492 #vporq $PADBIT,$T4,$T4 # padbit, yes, always
2498 ################################################################
2499 # while above multiplications were by r^8 in all lanes, in last
2500 # iteration we multiply least significant lane by r^8 and most
2501 # significant one by r, that's why table gets shifted...
2503 vpsrlq \$32,$R0,$R0 # 0105020603070408
2513 ################################################################
2514 # load either next or last 64 byte of input
2515 lea ($inp,$len),$inp
2517 #vpaddq $H2,$T2,$H2 # accumulate input
2520 vpmuludq $H2,$R1,$D3 # d3 = h2*r1
2521 vpmuludq $H2,$R2,$D4 # d4 = h2*r2
2522 vpmuludq $H2,$S3,$D0 # d0 = h2*s3
2523 vpandq $MASK,$T1,$T1 # 1
2524 vpmuludq $H2,$S4,$D1 # d1 = h2*s4
2525 vpandq $MASK,$T3,$T3 # 3
2526 vpmuludq $H2,$R0,$D2 # d2 = h2*r0
2527 vporq $PADBIT,$T4,$T4 # padbit, yes, always
2528 vpaddq $H1,$T1,$H1 # accumulate input
2532 vmovdqu64 16*0($inp),%x#$T0
2533 vpmuludq $H0,$R3,$M3
2534 vpmuludq $H0,$R4,$M4
2535 vpmuludq $H0,$R0,$M0
2536 vpmuludq $H0,$R1,$M1
2537 vpaddq $M3,$D3,$D3 # d3 += h0*r3
2538 vpaddq $M4,$D4,$D4 # d4 += h0*r4
2539 vpaddq $M0,$D0,$D0 # d0 += h0*r0
2540 vpaddq $M1,$D1,$D1 # d1 += h0*r1
2542 vmovdqu64 16*1($inp),%x#$T1
2543 vpmuludq $H1,$R2,$M3
2544 vpmuludq $H1,$R3,$M4
2545 vpmuludq $H1,$S4,$M0
2546 vpmuludq $H0,$R2,$M2
2547 vpaddq $M3,$D3,$D3 # d3 += h1*r2
2548 vpaddq $M4,$D4,$D4 # d4 += h1*r3
2549 vpaddq $M0,$D0,$D0 # d0 += h1*s4
2550 vpaddq $M2,$D2,$D2 # d2 += h0*r2
2552 vinserti64x2 \$1,16*2($inp),$T0,$T0
2553 vpmuludq $H3,$R0,$M3
2554 vpmuludq $H3,$R1,$M4
2555 vpmuludq $H1,$R0,$M1
2556 vpmuludq $H1,$R1,$M2
2557 vpaddq $M3,$D3,$D3 # d3 += h3*r0
2558 vpaddq $M4,$D4,$D4 # d4 += h3*r1
2559 vpaddq $M1,$D1,$D1 # d1 += h1*r0
2560 vpaddq $M2,$D2,$D2 # d2 += h1*r1
2562 vinserti64x2 \$1,16*3($inp),$T1,$T1
2563 vpmuludq $H4,$S4,$M3
2564 vpmuludq $H4,$R0,$M4
2565 vpmuludq $H3,$S2,$M0
2566 vpmuludq $H3,$S3,$M1
2567 vpmuludq $H3,$S4,$M2
2568 vpaddq $M3,$D3,$H3 # h3 = d3 + h4*s4
2569 vpaddq $M4,$D4,$D4 # d4 += h4*r0
2570 vpaddq $M0,$D0,$D0 # d0 += h3*s2
2571 vpaddq $M1,$D1,$D1 # d1 += h3*s3
2572 vpaddq $M2,$D2,$D2 # d2 += h3*s4
2574 vpmuludq $H4,$S1,$M0
2575 vpmuludq $H4,$S2,$M1
2576 vpmuludq $H4,$S3,$M2
2577 vpaddq $M0,$D0,$H0 # h0 = d0 + h4*s1
2578 vpaddq $M1,$D1,$H1 # h1 = d2 + h4*s2
2579 vpaddq $M2,$D2,$H2 # h2 = d3 + h4*s3
2581 ################################################################
2582 # horizontal addition
2597 vpermq \$0x2,$H3,$D3
2598 vpermq \$0x2,$H4,$D4
2599 vpermq \$0x2,$H0,$D0
2600 vpermq \$0x2,$H1,$D1
2601 vpermq \$0x2,$H2,$D2
2608 vextracti64x4 \$0x1,$H3,%y#$D3
2609 vextracti64x4 \$0x1,$H4,%y#$D4
2610 vextracti64x4 \$0x1,$H0,%y#$D0
2611 vextracti64x4 \$0x1,$H1,%y#$D1
2612 vextracti64x4 \$0x1,$H2,%y#$D2
2613 vpaddq $D3,$H3,${H3}{%k3}{z} # keep single qword in case
2614 vpaddq $D4,$H4,${H4}{%k3}{z} # it's passed to .Ltail_avx2
2615 vpaddq $D0,$H0,${H0}{%k3}{z}
2616 vpaddq $D1,$H1,${H1}{%k3}{z}
2617 vpaddq $D2,$H2,${H2}{%k3}{z}
2619 map(s/%z/%y/,($T0,$T1,$T2,$T3,$T4, $PADBIT));
2620 map(s/%z/%y/,($H0,$H1,$H2,$H3,$H4, $D0,$D1,$D2,$D3,$D4, $MASK));
2622 ################################################################
2623 # lazy reduction (interleaved with input splat)
2626 vpandq $MASK,$H3,$H3
2627 vpsrldq \$6,$T0,$T2 # splat input
2629 vpunpckhqdq $T1,$T0,$T4 # 4
2630 vpaddq $D3,$H4,$H4 # h3 -> h4
2633 vpandq $MASK,$H0,$H0
2634 vpunpcklqdq $T3,$T2,$T2 # 2:3
2635 vpunpcklqdq $T1,$T0,$T0 # 0:1
2636 vpaddq $D0,$H1,$H1 # h0 -> h1
2639 vpandq $MASK,$H4,$H4
2642 vpandq $MASK,$H1,$H1
2645 vpaddq $D1,$H2,$H2 # h1 -> h2
2650 vpsrlq \$40,$T4,$T4 # 4
2651 vpaddq $D4,$H0,$H0 # h4 -> h0
2654 vpandq $MASK,$H2,$H2
2655 vpandq $MASK,$T2,$T2 # 2
2656 vpandq $MASK,$T0,$T0 # 0
2657 vpaddq $D2,$H3,$H3 # h2 -> h3
2660 vpandq $MASK,$H0,$H0
2661 vpaddq $H2,$T2,$H2 # accumulate input for .Ltail_avx2
2662 vpandq $MASK,$T1,$T1 # 1
2663 vpaddq $D0,$H1,$H1 # h0 -> h1
2666 vpandq $MASK,$H3,$H3
2667 vpandq $MASK,$T3,$T3 # 3
2668 vporq $PADBIT,$T4,$T4 # padbit, yes, always
2669 vpaddq $D3,$H4,$H4 # h3 -> h4
2671 lea 0x90(%rsp),%rax # size optimization for .Ltail_avx2
2675 vpsubq $T2,$H2,$H2 # undo input accumulation
2676 vmovd %x#$H0,`4*0-48-64`($ctx)# save partially reduced
2677 vmovd %x#$H1,`4*1-48-64`($ctx)
2678 vmovd %x#$H2,`4*2-48-64`($ctx)
2679 vmovd %x#$H3,`4*3-48-64`($ctx)
2680 vmovd %x#$H4,`4*4-48-64`($ctx)
2683 $code.=<<___ if ($win64);
2684 movdqa 0x50(%r11),%xmm6
2685 movdqa 0x60(%r11),%xmm7
2686 movdqa 0x70(%r11),%xmm8
2687 movdqa 0x80(%r11),%xmm9
2688 movdqa 0x90(%r11),%xmm10
2689 movdqa 0xa0(%r11),%xmm11
2690 movdqa 0xb0(%r11),%xmm12
2691 movdqa 0xc0(%r11),%xmm13
2692 movdqa 0xd0(%r11),%xmm14
2693 movdqa 0xe0(%r11),%xmm15
2695 .Ldo_avx512_epilogue:
2697 $code.=<<___ if (!$win64);
2704 .size poly1305_blocks_avx512,.-poly1305_blocks_avx512
2707 ########################################################################
2708 # VPMADD52 version using 2^44 radix.
2710 # One can argue that base 2^52 would be more natural. Well, even though
2711 # some operations would be more natural, one has to recognize couple of
2712 # things. Base 2^52 doesn't provide advantage over base 2^44 if you look
2713 # at amount of multiply-n-accumulate operations. Secondly, it makes it
2714 # impossible to pre-compute multiples of 5 [referred to as s[]/sN in
2715 # reference implementations], which means that more such operations
2716 # would have to be performed in inner loop, which in turn makes critical
2717 # path longer. In other words, even though base 2^44 reduction might
2718 # look less elegant, overall critical path is actually shorter...
2720 ########################################################################
2721 # Layout of opaque area is following.
2723 # unsigned __int64 h[3]; # current hash value base 2^44
2724 # unsigned __int64 s[2]; # key value*20 base 2^44
2725 # unsigned __int64 r[3]; # key value base 2^44
2726 # struct { unsigned __int64 r^1, r^3, r^2, r^4; } R[4];
2727 # # r^n positions reflect
2728 # # placement in register, not
2729 # # memory, R[3] is R[1]*20
2732 .type poly1305_init_base2_44,\@function,3
2734 poly1305_init_base2_44:
2736 mov %rax,0($ctx) # initialize hash value
2741 lea poly1305_blocks_vpmadd52(%rip),%r10
2742 lea poly1305_emit_base2_44(%rip),%r11
2744 mov \$0x0ffffffc0fffffff,%rax
2745 mov \$0x0ffffffc0ffffffc,%rcx
2747 mov \$0x00000fffffffffff,%r8
2749 mov \$0x00000fffffffffff,%r9
2752 mov %r8,40($ctx) # r0
2755 mov %rax,48($ctx) # r1
2756 lea (%rax,%rax,4),%rax # *5
2757 mov %rcx,56($ctx) # r2
2758 shl \$2,%rax # magic <<2
2759 lea (%rcx,%rcx,4),%rcx # *5
2760 shl \$2,%rcx # magic <<2
2761 mov %rax,24($ctx) # s1
2762 mov %rcx,32($ctx) # s2
2763 movq \$-1,64($ctx) # write impossible value
2765 $code.=<<___ if ($flavour !~ /elf32/);
2769 $code.=<<___ if ($flavour =~ /elf32/);
2776 .size poly1305_init_base2_44,.-poly1305_init_base2_44
2779 my ($H0,$H1,$H2,$r2r1r0,$r1r0s2,$r0s2s1,$Dlo,$Dhi) = map("%ymm$_",(0..5,16,17));
2780 my ($T0,$inp_permd,$inp_shift,$PAD) = map("%ymm$_",(18..21));
2781 my ($reduc_mask,$reduc_rght,$reduc_left) = map("%ymm$_",(22..25));
2784 .type poly1305_blocks_vpmadd52,\@function,4
2786 poly1305_blocks_vpmadd52:
2788 jz .Lno_data_vpmadd52 # too short
2791 mov 64($ctx),%r8 # peek on power of the key
2793 # if powers of the key are not calculated yet, process up to 3
2794 # blocks with this single-block subroutine, otherwise ensure that
2795 # length is divisible by 2 blocks and pass the rest down to next
2800 cmp \$4,$len # is input long
2802 test %r8,%r8 # is power value impossible?
2805 and $len,%rax # is input of favourable length?
2806 jz .Lblocks_vpmadd52_4x
2812 lea .L2_44_inp_permd(%rip),%r10
2815 vmovq $padbit,%x#$PAD
2816 vmovdqa64 0(%r10),$inp_permd # .L2_44_inp_permd
2817 vmovdqa64 32(%r10),$inp_shift # .L2_44_inp_shift
2818 vpermq \$0xcf,$PAD,$PAD
2819 vmovdqa64 64(%r10),$reduc_mask # .L2_44_mask
2821 vmovdqu64 0($ctx),${Dlo}{%k7}{z} # load hash value
2822 vmovdqu64 40($ctx),${r2r1r0}{%k7}{z} # load keys
2823 vmovdqu64 32($ctx),${r1r0s2}{%k7}{z}
2824 vmovdqu64 24($ctx),${r0s2s1}{%k7}{z}
2826 vmovdqa64 96(%r10),$reduc_rght # .L2_44_shift_rgt
2827 vmovdqa64 128(%r10),$reduc_left # .L2_44_shift_lft
2833 vmovdqu32 0($inp),%x#$T0 # load input as ----3210
2836 vpermd $T0,$inp_permd,$T0 # ----3210 -> --322110
2837 vpsrlvq $inp_shift,$T0,$T0
2838 vpandq $reduc_mask,$T0,$T0
2841 vpaddq $T0,$Dlo,$Dlo # accumulate input
2843 vpermq \$0,$Dlo,${H0}{%k7}{z} # smash hash value
2844 vpermq \$0b01010101,$Dlo,${H1}{%k7}{z}
2845 vpermq \$0b10101010,$Dlo,${H2}{%k7}{z}
2847 vpxord $Dlo,$Dlo,$Dlo
2848 vpxord $Dhi,$Dhi,$Dhi
2850 vpmadd52luq $r2r1r0,$H0,$Dlo
2851 vpmadd52huq $r2r1r0,$H0,$Dhi
2853 vpmadd52luq $r1r0s2,$H1,$Dlo
2854 vpmadd52huq $r1r0s2,$H1,$Dhi
2856 vpmadd52luq $r0s2s1,$H2,$Dlo
2857 vpmadd52huq $r0s2s1,$H2,$Dhi
2859 vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost qword
2860 vpsllvq $reduc_left,$Dhi,$Dhi # 0 in topmost qword
2861 vpandq $reduc_mask,$Dlo,$Dlo
2863 vpaddq $T0,$Dhi,$Dhi
2865 vpermq \$0b10010011,$Dhi,$Dhi # 0 in lowest qword
2867 vpaddq $Dhi,$Dlo,$Dlo # note topmost qword :-)
2869 vpsrlvq $reduc_rght,$Dlo,$T0 # 0 in topmost word
2870 vpandq $reduc_mask,$Dlo,$Dlo
2872 vpermq \$0b10010011,$T0,$T0
2874 vpaddq $T0,$Dlo,$Dlo
2876 vpermq \$0b10010011,$Dlo,${T0}{%k1}{z}
2878 vpaddq $T0,$Dlo,$Dlo
2881 vpaddq $T0,$Dlo,$Dlo
2886 vmovdqu64 $Dlo,0($ctx){%k7} # store hash value
2889 jnz .Lblocks_vpmadd52_4x
2893 .size poly1305_blocks_vpmadd52,.-poly1305_blocks_vpmadd52
2897 ########################################################################
2898 # As implied by its name 4x subroutine processes 4 blocks in parallel
2899 # (but handles even 4*n+2 blocks lengths). It takes up to 4th key power
2900 # and is handled in 256-bit %ymm registers.
2902 my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17));
2903 my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23));
2904 my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31));
2907 .type poly1305_blocks_vpmadd52_4x,\@function,4
2909 poly1305_blocks_vpmadd52_4x:
2911 jz .Lno_data_vpmadd52_4x # too short
2914 mov 64($ctx),%r8 # peek on power of the key
2916 .Lblocks_vpmadd52_4x:
2917 vpbroadcastq $padbit,$PAD
2919 vmovdqa64 .Lx_mask44(%rip),$mask44
2921 vmovdqa64 .Lx_mask42(%rip),$mask42
2922 kmovw %eax,%k1 # used in 2x path
2924 test %r8,%r8 # is power value impossible?
2925 js .Linit_vpmadd52 # if it is, then init R[4]
2927 vmovq 0($ctx),%x#$H0 # load current hash value
2928 vmovq 8($ctx),%x#$H1
2929 vmovq 16($ctx),%x#$H2
2931 test \$3,$len # is length 4*n+2?
2932 jnz .Lblocks_vpmadd52_2x_do
2934 .Lblocks_vpmadd52_4x_do:
2935 vpbroadcastq 64($ctx),$R0 # load 4th power of the key
2936 vpbroadcastq 96($ctx),$R1
2937 vpbroadcastq 128($ctx),$R2
2938 vpbroadcastq 160($ctx),$S1
2940 .Lblocks_vpmadd52_4x_key_loaded:
2941 vpsllq \$2,$R2,$S2 # S2 = R2*5*4
2945 test \$7,$len # is len 8*n?
2946 jz .Lblocks_vpmadd52_8x
2948 vmovdqu64 16*0($inp),$T2 # load data
2949 vmovdqu64 16*2($inp),$T3
2952 vpunpcklqdq $T3,$T2,$T1 # transpose data
2953 vpunpckhqdq $T3,$T2,$T3
2955 # at this point 64-bit lanes are ordered as 3-1-2-0
2957 vpsrlq \$24,$T3,$T2 # splat the data
2959 vpaddq $T2,$H2,$H2 # accumulate input
2960 vpandq $mask44,$T1,$T0
2964 vpandq $mask44,$T1,$T1
2967 jz .Ltail_vpmadd52_4x
2968 jmp .Loop_vpmadd52_4x
2973 vmovq 24($ctx),%x#$S1 # load key
2974 vmovq 56($ctx),%x#$H2
2975 vmovq 32($ctx),%x#$S2
2976 vmovq 40($ctx),%x#$R0
2977 vmovq 48($ctx),%x#$R1
2985 .Lmul_init_vpmadd52:
2986 vpxorq $D0lo,$D0lo,$D0lo
2987 vpmadd52luq $H2,$S1,$D0lo
2988 vpxorq $D0hi,$D0hi,$D0hi
2989 vpmadd52huq $H2,$S1,$D0hi
2990 vpxorq $D1lo,$D1lo,$D1lo
2991 vpmadd52luq $H2,$S2,$D1lo
2992 vpxorq $D1hi,$D1hi,$D1hi
2993 vpmadd52huq $H2,$S2,$D1hi
2994 vpxorq $D2lo,$D2lo,$D2lo
2995 vpmadd52luq $H2,$R0,$D2lo
2996 vpxorq $D2hi,$D2hi,$D2hi
2997 vpmadd52huq $H2,$R0,$D2hi
2999 vpmadd52luq $H0,$R0,$D0lo
3000 vpmadd52huq $H0,$R0,$D0hi
3001 vpmadd52luq $H0,$R1,$D1lo
3002 vpmadd52huq $H0,$R1,$D1hi
3003 vpmadd52luq $H0,$R2,$D2lo
3004 vpmadd52huq $H0,$R2,$D2hi
3006 vpmadd52luq $H1,$S2,$D0lo
3007 vpmadd52huq $H1,$S2,$D0hi
3008 vpmadd52luq $H1,$R0,$D1lo
3009 vpmadd52huq $H1,$R0,$D1hi
3010 vpmadd52luq $H1,$R1,$D2lo
3011 vpmadd52huq $H1,$R1,$D2hi
3013 ################################################################
3015 vpsrlq \$44,$D0lo,$tmp
3016 vpsllq \$8,$D0hi,$D0hi
3017 vpandq $mask44,$D0lo,$H0
3018 vpaddq $tmp,$D0hi,$D0hi
3020 vpaddq $D0hi,$D1lo,$D1lo
3022 vpsrlq \$44,$D1lo,$tmp
3023 vpsllq \$8,$D1hi,$D1hi
3024 vpandq $mask44,$D1lo,$H1
3025 vpaddq $tmp,$D1hi,$D1hi
3027 vpaddq $D1hi,$D2lo,$D2lo
3029 vpsrlq \$42,$D2lo,$tmp
3030 vpsllq \$10,$D2hi,$D2hi
3031 vpandq $mask42,$D2lo,$H2
3032 vpaddq $tmp,$D2hi,$D2hi
3034 vpaddq $D2hi,$H0,$H0
3035 vpsllq \$2,$D2hi,$D2hi
3037 vpaddq $D2hi,$H0,$H0
3039 vpsrlq \$44,$H0,$tmp # additional step
3040 vpandq $mask44,$H0,$H0
3045 jz .Ldone_init_vpmadd52
3047 vpunpcklqdq $R1,$H1,$R1 # 1,2
3048 vpbroadcastq %x#$H1,%x#$H1 # 2,2
3049 vpunpcklqdq $R2,$H2,$R2
3050 vpbroadcastq %x#$H2,%x#$H2
3051 vpunpcklqdq $R0,$H0,$R0
3052 vpbroadcastq %x#$H0,%x#$H0
3054 vpsllq \$2,$R1,$S1 # S1 = R1*5*4
3055 vpsllq \$2,$R2,$S2 # S2 = R2*5*4
3061 jmp .Lmul_init_vpmadd52
3065 .Ldone_init_vpmadd52:
3066 vinserti128 \$1,%x#$R1,$H1,$R1 # 1,2,3,4
3067 vinserti128 \$1,%x#$R2,$H2,$R2
3068 vinserti128 \$1,%x#$R0,$H0,$R0
3070 vpermq \$0b11011000,$R1,$R1 # 1,3,2,4
3071 vpermq \$0b11011000,$R2,$R2
3072 vpermq \$0b11011000,$R0,$R0
3074 vpsllq \$2,$R1,$S1 # S1 = R1*5*4
3078 vmovq 0($ctx),%x#$H0 # load current hash value
3079 vmovq 8($ctx),%x#$H1
3080 vmovq 16($ctx),%x#$H2
3082 test \$3,$len # is length 4*n+2?
3083 jnz .Ldone_init_vpmadd52_2x
3085 vmovdqu64 $R0,64($ctx) # save key powers
3086 vpbroadcastq %x#$R0,$R0 # broadcast 4th power
3087 vmovdqu64 $R1,96($ctx)
3088 vpbroadcastq %x#$R1,$R1
3089 vmovdqu64 $R2,128($ctx)
3090 vpbroadcastq %x#$R2,$R2
3091 vmovdqu64 $S1,160($ctx)
3092 vpbroadcastq %x#$S1,$S1
3094 jmp .Lblocks_vpmadd52_4x_key_loaded
3098 .Ldone_init_vpmadd52_2x:
3099 vmovdqu64 $R0,64($ctx) # save key powers
3100 vpsrldq \$8,$R0,$R0 # 0-1-0-2
3101 vmovdqu64 $R1,96($ctx)
3103 vmovdqu64 $R2,128($ctx)
3105 vmovdqu64 $S1,160($ctx)
3107 jmp .Lblocks_vpmadd52_2x_key_loaded
3111 .Lblocks_vpmadd52_2x_do:
3112 vmovdqu64 128+8($ctx),${R2}{%k1}{z}# load 2nd and 1st key powers
3113 vmovdqu64 160+8($ctx),${S1}{%k1}{z}
3114 vmovdqu64 64+8($ctx),${R0}{%k1}{z}
3115 vmovdqu64 96+8($ctx),${R1}{%k1}{z}
3117 .Lblocks_vpmadd52_2x_key_loaded:
3118 vmovdqu64 16*0($inp),$T2 # load data
3122 vpunpcklqdq $T3,$T2,$T1 # transpose data
3123 vpunpckhqdq $T3,$T2,$T3
3125 # at this point 64-bit lanes are ordered as x-1-x-0
3127 vpsrlq \$24,$T3,$T2 # splat the data
3129 vpaddq $T2,$H2,$H2 # accumulate input
3130 vpandq $mask44,$T1,$T0
3134 vpandq $mask44,$T1,$T1
3136 jmp .Ltail_vpmadd52_2x
3141 #vpaddq $T2,$H2,$H2 # accumulate input
3145 vpxorq $D0lo,$D0lo,$D0lo
3146 vpmadd52luq $H2,$S1,$D0lo
3147 vpxorq $D0hi,$D0hi,$D0hi
3148 vpmadd52huq $H2,$S1,$D0hi
3149 vpxorq $D1lo,$D1lo,$D1lo
3150 vpmadd52luq $H2,$S2,$D1lo
3151 vpxorq $D1hi,$D1hi,$D1hi
3152 vpmadd52huq $H2,$S2,$D1hi
3153 vpxorq $D2lo,$D2lo,$D2lo
3154 vpmadd52luq $H2,$R0,$D2lo
3155 vpxorq $D2hi,$D2hi,$D2hi
3156 vpmadd52huq $H2,$R0,$D2hi
3158 vmovdqu64 16*0($inp),$T2 # load data
3159 vmovdqu64 16*2($inp),$T3
3161 vpmadd52luq $H0,$R0,$D0lo
3162 vpmadd52huq $H0,$R0,$D0hi
3163 vpmadd52luq $H0,$R1,$D1lo
3164 vpmadd52huq $H0,$R1,$D1hi
3165 vpmadd52luq $H0,$R2,$D2lo
3166 vpmadd52huq $H0,$R2,$D2hi
3168 vpunpcklqdq $T3,$T2,$T1 # transpose data
3169 vpunpckhqdq $T3,$T2,$T3
3170 vpmadd52luq $H1,$S2,$D0lo
3171 vpmadd52huq $H1,$S2,$D0hi
3172 vpmadd52luq $H1,$R0,$D1lo
3173 vpmadd52huq $H1,$R0,$D1hi
3174 vpmadd52luq $H1,$R1,$D2lo
3175 vpmadd52huq $H1,$R1,$D2hi
3177 ################################################################
3178 # partial reduction (interleaved with data splat)
3179 vpsrlq \$44,$D0lo,$tmp
3180 vpsllq \$8,$D0hi,$D0hi
3181 vpandq $mask44,$D0lo,$H0
3182 vpaddq $tmp,$D0hi,$D0hi
3186 vpaddq $D0hi,$D1lo,$D1lo
3188 vpsrlq \$44,$D1lo,$tmp
3189 vpsllq \$8,$D1hi,$D1hi
3190 vpandq $mask44,$D1lo,$H1
3191 vpaddq $tmp,$D1hi,$D1hi
3193 vpandq $mask44,$T1,$T0
3196 vpaddq $D1hi,$D2lo,$D2lo
3198 vpsrlq \$42,$D2lo,$tmp
3199 vpsllq \$10,$D2hi,$D2hi
3200 vpandq $mask42,$D2lo,$H2
3201 vpaddq $tmp,$D2hi,$D2hi
3203 vpaddq $T2,$H2,$H2 # accumulate input
3204 vpaddq $D2hi,$H0,$H0
3205 vpsllq \$2,$D2hi,$D2hi
3207 vpaddq $D2hi,$H0,$H0
3209 vpandq $mask44,$T1,$T1
3211 vpsrlq \$44,$H0,$tmp # additional step
3212 vpandq $mask44,$H0,$H0
3216 sub \$4,$len # len-=64
3217 jnz .Loop_vpmadd52_4x
3220 vmovdqu64 128($ctx),$R2 # load all key powers
3221 vmovdqu64 160($ctx),$S1
3222 vmovdqu64 64($ctx),$R0
3223 vmovdqu64 96($ctx),$R1
3226 vpsllq \$2,$R2,$S2 # S2 = R2*5*4
3230 #vpaddq $T2,$H2,$H2 # accumulate input
3234 vpxorq $D0lo,$D0lo,$D0lo
3235 vpmadd52luq $H2,$S1,$D0lo
3236 vpxorq $D0hi,$D0hi,$D0hi
3237 vpmadd52huq $H2,$S1,$D0hi
3238 vpxorq $D1lo,$D1lo,$D1lo
3239 vpmadd52luq $H2,$S2,$D1lo
3240 vpxorq $D1hi,$D1hi,$D1hi
3241 vpmadd52huq $H2,$S2,$D1hi
3242 vpxorq $D2lo,$D2lo,$D2lo
3243 vpmadd52luq $H2,$R0,$D2lo
3244 vpxorq $D2hi,$D2hi,$D2hi
3245 vpmadd52huq $H2,$R0,$D2hi
3247 vpmadd52luq $H0,$R0,$D0lo
3248 vpmadd52huq $H0,$R0,$D0hi
3249 vpmadd52luq $H0,$R1,$D1lo
3250 vpmadd52huq $H0,$R1,$D1hi
3251 vpmadd52luq $H0,$R2,$D2lo
3252 vpmadd52huq $H0,$R2,$D2hi
3254 vpmadd52luq $H1,$S2,$D0lo
3255 vpmadd52huq $H1,$S2,$D0hi
3256 vpmadd52luq $H1,$R0,$D1lo
3257 vpmadd52huq $H1,$R0,$D1hi
3258 vpmadd52luq $H1,$R1,$D2lo
3259 vpmadd52huq $H1,$R1,$D2hi
3261 ################################################################
3262 # horizontal addition
3266 vpsrldq \$8,$D0lo,$T0
3267 vpsrldq \$8,$D0hi,$H0
3268 vpsrldq \$8,$D1lo,$T1
3269 vpsrldq \$8,$D1hi,$H1
3270 vpaddq $T0,$D0lo,$D0lo
3271 vpaddq $H0,$D0hi,$D0hi
3272 vpsrldq \$8,$D2lo,$T2
3273 vpsrldq \$8,$D2hi,$H2
3274 vpaddq $T1,$D1lo,$D1lo
3275 vpaddq $H1,$D1hi,$D1hi
3276 vpermq \$0x2,$D0lo,$T0
3277 vpermq \$0x2,$D0hi,$H0
3278 vpaddq $T2,$D2lo,$D2lo
3279 vpaddq $H2,$D2hi,$D2hi
3281 vpermq \$0x2,$D1lo,$T1
3282 vpermq \$0x2,$D1hi,$H1
3283 vpaddq $T0,$D0lo,${D0lo}{%k1}{z}
3284 vpaddq $H0,$D0hi,${D0hi}{%k1}{z}
3285 vpermq \$0x2,$D2lo,$T2
3286 vpermq \$0x2,$D2hi,$H2
3287 vpaddq $T1,$D1lo,${D1lo}{%k1}{z}
3288 vpaddq $H1,$D1hi,${D1hi}{%k1}{z}
3289 vpaddq $T2,$D2lo,${D2lo}{%k1}{z}
3290 vpaddq $H2,$D2hi,${D2hi}{%k1}{z}
3292 ################################################################
3294 vpsrlq \$44,$D0lo,$tmp
3295 vpsllq \$8,$D0hi,$D0hi
3296 vpandq $mask44,$D0lo,$H0
3297 vpaddq $tmp,$D0hi,$D0hi
3299 vpaddq $D0hi,$D1lo,$D1lo
3301 vpsrlq \$44,$D1lo,$tmp
3302 vpsllq \$8,$D1hi,$D1hi
3303 vpandq $mask44,$D1lo,$H1
3304 vpaddq $tmp,$D1hi,$D1hi
3306 vpaddq $D1hi,$D2lo,$D2lo
3308 vpsrlq \$42,$D2lo,$tmp
3309 vpsllq \$10,$D2hi,$D2hi
3310 vpandq $mask42,$D2lo,$H2
3311 vpaddq $tmp,$D2hi,$D2hi
3313 vpaddq $D2hi,$H0,$H0
3314 vpsllq \$2,$D2hi,$D2hi
3316 vpaddq $D2hi,$H0,$H0
3318 vpsrlq \$44,$H0,$tmp # additional step
3319 vpandq $mask44,$H0,$H0
3322 # at this point $len is
3323 # either 4*n+2 or 0...
3324 sub \$2,$len # len-=32
3325 ja .Lblocks_vpmadd52_4x_do
3327 vmovq %x#$H0,0($ctx)
3328 vmovq %x#$H1,8($ctx)
3329 vmovq %x#$H2,16($ctx)
3332 .Lno_data_vpmadd52_4x:
3334 .size poly1305_blocks_vpmadd52_4x,.-poly1305_blocks_vpmadd52_4x
3338 ########################################################################
3339 # As implied by its name 8x subroutine processes 8 blocks in parallel...
3340 # This is intermediate version, as it's used only in cases when input
3341 # length is either 8*n, 8*n+1 or 8*n+2...
3343 my ($H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2) = map("%ymm$_",(0..5,16,17));
3344 my ($D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi) = map("%ymm$_",(18..23));
3345 my ($T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD) = map("%ymm$_",(24..31));
3346 my ($RR0,$RR1,$RR2,$SS1,$SS2) = map("%ymm$_",(6..10));
3349 .type poly1305_blocks_vpmadd52_8x,\@function,4
3351 poly1305_blocks_vpmadd52_8x:
3353 jz .Lno_data_vpmadd52_8x # too short
3356 mov 64($ctx),%r8 # peek on power of the key
3358 vmovdqa64 .Lx_mask44(%rip),$mask44
3359 vmovdqa64 .Lx_mask42(%rip),$mask42
3361 test %r8,%r8 # is power value impossible?
3362 js .Linit_vpmadd52 # if it is, then init R[4]
3364 vmovq 0($ctx),%x#$H0 # load current hash value
3365 vmovq 8($ctx),%x#$H1
3366 vmovq 16($ctx),%x#$H2
3368 .Lblocks_vpmadd52_8x:
3369 ################################################################
3370 # fist we calculate more key powers
3372 vmovdqu64 128($ctx),$R2 # load 1-3-2-4 powers
3373 vmovdqu64 160($ctx),$S1
3374 vmovdqu64 64($ctx),$R0
3375 vmovdqu64 96($ctx),$R1
3377 vpsllq \$2,$R2,$S2 # S2 = R2*5*4
3381 vpbroadcastq %x#$R2,$RR2 # broadcast 4th power
3382 vpbroadcastq %x#$R0,$RR0
3383 vpbroadcastq %x#$R1,$RR1
3385 vpxorq $D0lo,$D0lo,$D0lo
3386 vpmadd52luq $RR2,$S1,$D0lo
3387 vpxorq $D0hi,$D0hi,$D0hi
3388 vpmadd52huq $RR2,$S1,$D0hi
3389 vpxorq $D1lo,$D1lo,$D1lo
3390 vpmadd52luq $RR2,$S2,$D1lo
3391 vpxorq $D1hi,$D1hi,$D1hi
3392 vpmadd52huq $RR2,$S2,$D1hi
3393 vpxorq $D2lo,$D2lo,$D2lo
3394 vpmadd52luq $RR2,$R0,$D2lo
3395 vpxorq $D2hi,$D2hi,$D2hi
3396 vpmadd52huq $RR2,$R0,$D2hi
3398 vpmadd52luq $RR0,$R0,$D0lo
3399 vpmadd52huq $RR0,$R0,$D0hi
3400 vpmadd52luq $RR0,$R1,$D1lo
3401 vpmadd52huq $RR0,$R1,$D1hi
3402 vpmadd52luq $RR0,$R2,$D2lo
3403 vpmadd52huq $RR0,$R2,$D2hi
3405 vpmadd52luq $RR1,$S2,$D0lo
3406 vpmadd52huq $RR1,$S2,$D0hi
3407 vpmadd52luq $RR1,$R0,$D1lo
3408 vpmadd52huq $RR1,$R0,$D1hi
3409 vpmadd52luq $RR1,$R1,$D2lo
3410 vpmadd52huq $RR1,$R1,$D2hi
3412 ################################################################
3414 vpsrlq \$44,$D0lo,$tmp
3415 vpsllq \$8,$D0hi,$D0hi
3416 vpandq $mask44,$D0lo,$RR0
3417 vpaddq $tmp,$D0hi,$D0hi
3419 vpaddq $D0hi,$D1lo,$D1lo
3421 vpsrlq \$44,$D1lo,$tmp
3422 vpsllq \$8,$D1hi,$D1hi
3423 vpandq $mask44,$D1lo,$RR1
3424 vpaddq $tmp,$D1hi,$D1hi
3426 vpaddq $D1hi,$D2lo,$D2lo
3428 vpsrlq \$42,$D2lo,$tmp
3429 vpsllq \$10,$D2hi,$D2hi
3430 vpandq $mask42,$D2lo,$RR2
3431 vpaddq $tmp,$D2hi,$D2hi
3433 vpaddq $D2hi,$RR0,$RR0
3434 vpsllq \$2,$D2hi,$D2hi
3436 vpaddq $D2hi,$RR0,$RR0
3438 vpsrlq \$44,$RR0,$tmp # additional step
3439 vpandq $mask44,$RR0,$RR0
3441 vpaddq $tmp,$RR1,$RR1
3443 ################################################################
3444 # At this point Rx holds 1324 powers, RRx - 5768, and the goal
3445 # is 15263748, which reflects how data is loaded...
3447 vpunpcklqdq $R2,$RR2,$T2 # 3748
3448 vpunpckhqdq $R2,$RR2,$R2 # 1526
3449 vpunpcklqdq $R0,$RR0,$T0
3450 vpunpckhqdq $R0,$RR0,$R0
3451 vpunpcklqdq $R1,$RR1,$T1
3452 vpunpckhqdq $R1,$RR1,$R1
3454 ######## switch to %zmm
3455 map(s/%y/%z/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2);
3456 map(s/%y/%z/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi);
3457 map(s/%y/%z/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD);
3458 map(s/%y/%z/, $RR0,$RR1,$RR2,$SS1,$SS2);
3461 vshufi64x2 \$0x44,$R2,$T2,$RR2 # 15263748
3462 vshufi64x2 \$0x44,$R0,$T0,$RR0
3463 vshufi64x2 \$0x44,$R1,$T1,$RR1
3465 vmovdqu64 16*0($inp),$T2 # load data
3466 vmovdqu64 16*4($inp),$T3
3469 vpsllq \$2,$RR2,$SS2 # S2 = R2*5*4
3470 vpsllq \$2,$RR1,$SS1 # S1 = R1*5*4
3471 vpaddq $RR2,$SS2,$SS2
3472 vpaddq $RR1,$SS1,$SS1
3473 vpsllq \$2,$SS2,$SS2
3474 vpsllq \$2,$SS1,$SS1
3476 vpbroadcastq $padbit,$PAD
3477 vpbroadcastq %x#$mask44,$mask44
3478 vpbroadcastq %x#$mask42,$mask42
3480 vpbroadcastq %x#$SS1,$S1 # broadcast 8th power
3481 vpbroadcastq %x#$SS2,$S2
3482 vpbroadcastq %x#$RR0,$R0
3483 vpbroadcastq %x#$RR1,$R1
3484 vpbroadcastq %x#$RR2,$R2
3486 vpunpcklqdq $T3,$T2,$T1 # transpose data
3487 vpunpckhqdq $T3,$T2,$T3
3489 # at this point 64-bit lanes are ordered as 73625140
3491 vpsrlq \$24,$T3,$T2 # splat the data
3493 vpaddq $T2,$H2,$H2 # accumulate input
3494 vpandq $mask44,$T1,$T0
3498 vpandq $mask44,$T1,$T1
3501 jz .Ltail_vpmadd52_8x
3502 jmp .Loop_vpmadd52_8x
3506 #vpaddq $T2,$H2,$H2 # accumulate input
3510 vpxorq $D0lo,$D0lo,$D0lo
3511 vpmadd52luq $H2,$S1,$D0lo
3512 vpxorq $D0hi,$D0hi,$D0hi
3513 vpmadd52huq $H2,$S1,$D0hi
3514 vpxorq $D1lo,$D1lo,$D1lo
3515 vpmadd52luq $H2,$S2,$D1lo
3516 vpxorq $D1hi,$D1hi,$D1hi
3517 vpmadd52huq $H2,$S2,$D1hi
3518 vpxorq $D2lo,$D2lo,$D2lo
3519 vpmadd52luq $H2,$R0,$D2lo
3520 vpxorq $D2hi,$D2hi,$D2hi
3521 vpmadd52huq $H2,$R0,$D2hi
3523 vmovdqu64 16*0($inp),$T2 # load data
3524 vmovdqu64 16*4($inp),$T3
3526 vpmadd52luq $H0,$R0,$D0lo
3527 vpmadd52huq $H0,$R0,$D0hi
3528 vpmadd52luq $H0,$R1,$D1lo
3529 vpmadd52huq $H0,$R1,$D1hi
3530 vpmadd52luq $H0,$R2,$D2lo
3531 vpmadd52huq $H0,$R2,$D2hi
3533 vpunpcklqdq $T3,$T2,$T1 # transpose data
3534 vpunpckhqdq $T3,$T2,$T3
3535 vpmadd52luq $H1,$S2,$D0lo
3536 vpmadd52huq $H1,$S2,$D0hi
3537 vpmadd52luq $H1,$R0,$D1lo
3538 vpmadd52huq $H1,$R0,$D1hi
3539 vpmadd52luq $H1,$R1,$D2lo
3540 vpmadd52huq $H1,$R1,$D2hi
3542 ################################################################
3543 # partial reduction (interleaved with data splat)
3544 vpsrlq \$44,$D0lo,$tmp
3545 vpsllq \$8,$D0hi,$D0hi
3546 vpandq $mask44,$D0lo,$H0
3547 vpaddq $tmp,$D0hi,$D0hi
3551 vpaddq $D0hi,$D1lo,$D1lo
3553 vpsrlq \$44,$D1lo,$tmp
3554 vpsllq \$8,$D1hi,$D1hi
3555 vpandq $mask44,$D1lo,$H1
3556 vpaddq $tmp,$D1hi,$D1hi
3558 vpandq $mask44,$T1,$T0
3561 vpaddq $D1hi,$D2lo,$D2lo
3563 vpsrlq \$42,$D2lo,$tmp
3564 vpsllq \$10,$D2hi,$D2hi
3565 vpandq $mask42,$D2lo,$H2
3566 vpaddq $tmp,$D2hi,$D2hi
3568 vpaddq $T2,$H2,$H2 # accumulate input
3569 vpaddq $D2hi,$H0,$H0
3570 vpsllq \$2,$D2hi,$D2hi
3572 vpaddq $D2hi,$H0,$H0
3574 vpandq $mask44,$T1,$T1
3576 vpsrlq \$44,$H0,$tmp # additional step
3577 vpandq $mask44,$H0,$H0
3581 sub \$8,$len # len-=128
3582 jnz .Loop_vpmadd52_8x
3585 #vpaddq $T2,$H2,$H2 # accumulate input
3589 vpxorq $D0lo,$D0lo,$D0lo
3590 vpmadd52luq $H2,$SS1,$D0lo
3591 vpxorq $D0hi,$D0hi,$D0hi
3592 vpmadd52huq $H2,$SS1,$D0hi
3593 vpxorq $D1lo,$D1lo,$D1lo
3594 vpmadd52luq $H2,$SS2,$D1lo
3595 vpxorq $D1hi,$D1hi,$D1hi
3596 vpmadd52huq $H2,$SS2,$D1hi
3597 vpxorq $D2lo,$D2lo,$D2lo
3598 vpmadd52luq $H2,$RR0,$D2lo
3599 vpxorq $D2hi,$D2hi,$D2hi
3600 vpmadd52huq $H2,$RR0,$D2hi
3602 vpmadd52luq $H0,$RR0,$D0lo
3603 vpmadd52huq $H0,$RR0,$D0hi
3604 vpmadd52luq $H0,$RR1,$D1lo
3605 vpmadd52huq $H0,$RR1,$D1hi
3606 vpmadd52luq $H0,$RR2,$D2lo
3607 vpmadd52huq $H0,$RR2,$D2hi
3609 vpmadd52luq $H1,$SS2,$D0lo
3610 vpmadd52huq $H1,$SS2,$D0hi
3611 vpmadd52luq $H1,$RR0,$D1lo
3612 vpmadd52huq $H1,$RR0,$D1hi
3613 vpmadd52luq $H1,$RR1,$D2lo
3614 vpmadd52huq $H1,$RR1,$D2hi
3616 ################################################################
3617 # horizontal addition
3621 vpsrldq \$8,$D0lo,$T0
3622 vpsrldq \$8,$D0hi,$H0
3623 vpsrldq \$8,$D1lo,$T1
3624 vpsrldq \$8,$D1hi,$H1
3625 vpaddq $T0,$D0lo,$D0lo
3626 vpaddq $H0,$D0hi,$D0hi
3627 vpsrldq \$8,$D2lo,$T2
3628 vpsrldq \$8,$D2hi,$H2
3629 vpaddq $T1,$D1lo,$D1lo
3630 vpaddq $H1,$D1hi,$D1hi
3631 vpermq \$0x2,$D0lo,$T0
3632 vpermq \$0x2,$D0hi,$H0
3633 vpaddq $T2,$D2lo,$D2lo
3634 vpaddq $H2,$D2hi,$D2hi
3636 vpermq \$0x2,$D1lo,$T1
3637 vpermq \$0x2,$D1hi,$H1
3638 vpaddq $T0,$D0lo,$D0lo
3639 vpaddq $H0,$D0hi,$D0hi
3640 vpermq \$0x2,$D2lo,$T2
3641 vpermq \$0x2,$D2hi,$H2
3642 vpaddq $T1,$D1lo,$D1lo
3643 vpaddq $H1,$D1hi,$D1hi
3644 vextracti64x4 \$1,$D0lo,%y#$T0
3645 vextracti64x4 \$1,$D0hi,%y#$H0
3646 vpaddq $T2,$D2lo,$D2lo
3647 vpaddq $H2,$D2hi,$D2hi
3649 vextracti64x4 \$1,$D1lo,%y#$T1
3650 vextracti64x4 \$1,$D1hi,%y#$H1
3651 vextracti64x4 \$1,$D2lo,%y#$T2
3652 vextracti64x4 \$1,$D2hi,%y#$H2
3654 ######## switch back to %ymm
3655 map(s/%z/%y/, $H0,$H1,$H2,$R0,$R1,$R2,$S1,$S2);
3656 map(s/%z/%y/, $D0lo,$D0hi,$D1lo,$D1hi,$D2lo,$D2hi);
3657 map(s/%z/%y/, $T0,$T1,$T2,$T3,$mask44,$mask42,$tmp,$PAD);
3660 vpaddq $T0,$D0lo,${D0lo}{%k1}{z}
3661 vpaddq $H0,$D0hi,${D0hi}{%k1}{z}
3662 vpaddq $T1,$D1lo,${D1lo}{%k1}{z}
3663 vpaddq $H1,$D1hi,${D1hi}{%k1}{z}
3664 vpaddq $T2,$D2lo,${D2lo}{%k1}{z}
3665 vpaddq $H2,$D2hi,${D2hi}{%k1}{z}
3667 ################################################################
3669 vpsrlq \$44,$D0lo,$tmp
3670 vpsllq \$8,$D0hi,$D0hi
3671 vpandq $mask44,$D0lo,$H0
3672 vpaddq $tmp,$D0hi,$D0hi
3674 vpaddq $D0hi,$D1lo,$D1lo
3676 vpsrlq \$44,$D1lo,$tmp
3677 vpsllq \$8,$D1hi,$D1hi
3678 vpandq $mask44,$D1lo,$H1
3679 vpaddq $tmp,$D1hi,$D1hi
3681 vpaddq $D1hi,$D2lo,$D2lo
3683 vpsrlq \$42,$D2lo,$tmp
3684 vpsllq \$10,$D2hi,$D2hi
3685 vpandq $mask42,$D2lo,$H2
3686 vpaddq $tmp,$D2hi,$D2hi
3688 vpaddq $D2hi,$H0,$H0
3689 vpsllq \$2,$D2hi,$D2hi
3691 vpaddq $D2hi,$H0,$H0
3693 vpsrlq \$44,$H0,$tmp # additional step
3694 vpandq $mask44,$H0,$H0
3698 ################################################################
3700 vmovq %x#$H0,0($ctx)
3701 vmovq %x#$H1,8($ctx)
3702 vmovq %x#$H2,16($ctx)
3705 .Lno_data_vpmadd52_8x:
3707 .size poly1305_blocks_vpmadd52_8x,.-poly1305_blocks_vpmadd52_8x
3711 .type poly1305_emit_base2_44,\@function,3
3713 poly1305_emit_base2_44:
3714 mov 0($ctx),%r8 # load hash value
3730 add \$5,%r8 # compare to modulus
3734 shr \$2,%r10 # did 130-bit value overfow?
3738 add 0($nonce),%rax # accumulate nonce
3740 mov %rax,0($mac) # write result
3744 .size poly1305_emit_base2_44,.-poly1305_emit_base2_44
3751 .long 0x0ffffff,0,0x0ffffff,0,0x0ffffff,0,0x0ffffff,0
3753 .long `1<<24`,0,`1<<24`,0,`1<<24`,0,`1<<24`,0
3755 .long 0x3ffffff,0,0x3ffffff,0,0x3ffffff,0,0x3ffffff,0
3757 .long 2,2,2,3,2,0,2,1
3759 .long 0,0,0,1, 0,2,0,3, 0,4,0,5, 0,6,0,7
3762 .long 0,1,1,2,2,3,7,7
3766 .quad 0xfffffffffff,0xfffffffffff,0x3ffffffffff,0xffffffffffffffff
3774 .quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
3775 .quad 0xfffffffffff,0xfffffffffff,0xfffffffffff,0xfffffffffff
3777 .quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
3778 .quad 0x3ffffffffff,0x3ffffffffff,0x3ffffffffff,0x3ffffffffff
3783 .asciz "Poly1305 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
3787 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
3788 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
3796 .extern __imp_RtlVirtualUnwind
3797 .type se_handler,\@abi-omnipotent
3811 mov 120($context),%rax # pull context->Rax
3812 mov 248($context),%rbx # pull context->Rip
3814 mov 8($disp),%rsi # disp->ImageBase
3815 mov 56($disp),%r11 # disp->HandlerData
3817 mov 0(%r11),%r10d # HandlerData[0]
3818 lea (%rsi,%r10),%r10 # prologue label
3819 cmp %r10,%rbx # context->Rip<.Lprologue
3820 jb .Lcommon_seh_tail
3822 mov 152($context),%rax # pull context->Rsp
3824 mov 4(%r11),%r10d # HandlerData[1]
3825 lea (%rsi,%r10),%r10 # epilogue label
3826 cmp %r10,%rbx # context->Rip>=.Lepilogue
3827 jae .Lcommon_seh_tail
3837 mov %rbx,144($context) # restore context->Rbx
3838 mov %rbp,160($context) # restore context->Rbp
3839 mov %r12,216($context) # restore context->R12
3840 mov %r13,224($context) # restore context->R13
3841 mov %r14,232($context) # restore context->R14
3842 mov %r15,240($context) # restore context->R14
3844 jmp .Lcommon_seh_tail
3845 .size se_handler,.-se_handler
3847 .type avx_handler,\@abi-omnipotent
3861 mov 120($context),%rax # pull context->Rax
3862 mov 248($context),%rbx # pull context->Rip
3864 mov 8($disp),%rsi # disp->ImageBase
3865 mov 56($disp),%r11 # disp->HandlerData
3867 mov 0(%r11),%r10d # HandlerData[0]
3868 lea (%rsi,%r10),%r10 # prologue label
3869 cmp %r10,%rbx # context->Rip<prologue label
3870 jb .Lcommon_seh_tail
3872 mov 152($context),%rax # pull context->Rsp
3874 mov 4(%r11),%r10d # HandlerData[1]
3875 lea (%rsi,%r10),%r10 # epilogue label
3876 cmp %r10,%rbx # context->Rip>=epilogue label
3877 jae .Lcommon_seh_tail
3879 mov 208($context),%rax # pull context->R11
3883 lea 512($context),%rdi # &context.Xmm6
3885 .long 0xa548f3fc # cld; rep movsq
3890 mov %rax,152($context) # restore context->Rsp
3891 mov %rsi,168($context) # restore context->Rsi
3892 mov %rdi,176($context) # restore context->Rdi
3894 mov 40($disp),%rdi # disp->ContextRecord
3895 mov $context,%rsi # context
3896 mov \$154,%ecx # sizeof(CONTEXT)
3897 .long 0xa548f3fc # cld; rep movsq
3900 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
3901 mov 8(%rsi),%rdx # arg2, disp->ImageBase
3902 mov 0(%rsi),%r8 # arg3, disp->ControlPc
3903 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
3904 mov 40(%rsi),%r10 # disp->ContextRecord
3905 lea 56(%rsi),%r11 # &disp->HandlerData
3906 lea 24(%rsi),%r12 # &disp->EstablisherFrame
3907 mov %r10,32(%rsp) # arg5
3908 mov %r11,40(%rsp) # arg6
3909 mov %r12,48(%rsp) # arg7
3910 mov %rcx,56(%rsp) # arg8, (NULL)
3911 call *__imp_RtlVirtualUnwind(%rip)
3913 mov \$1,%eax # ExceptionContinueSearch
3925 .size avx_handler,.-avx_handler
3929 .rva .LSEH_begin_poly1305_init
3930 .rva .LSEH_end_poly1305_init
3931 .rva .LSEH_info_poly1305_init
3933 .rva .LSEH_begin_poly1305_blocks
3934 .rva .LSEH_end_poly1305_blocks
3935 .rva .LSEH_info_poly1305_blocks
3937 .rva .LSEH_begin_poly1305_emit
3938 .rva .LSEH_end_poly1305_emit
3939 .rva .LSEH_info_poly1305_emit
3941 $code.=<<___ if ($avx);
3942 .rva .LSEH_begin_poly1305_blocks_avx
3944 .rva .LSEH_info_poly1305_blocks_avx_1
3948 .rva .LSEH_info_poly1305_blocks_avx_2
3951 .rva .LSEH_end_poly1305_blocks_avx
3952 .rva .LSEH_info_poly1305_blocks_avx_3
3954 .rva .LSEH_begin_poly1305_emit_avx
3955 .rva .LSEH_end_poly1305_emit_avx
3956 .rva .LSEH_info_poly1305_emit_avx
3958 $code.=<<___ if ($avx>1);
3959 .rva .LSEH_begin_poly1305_blocks_avx2
3960 .rva .Lbase2_64_avx2
3961 .rva .LSEH_info_poly1305_blocks_avx2_1
3963 .rva .Lbase2_64_avx2
3965 .rva .LSEH_info_poly1305_blocks_avx2_2
3968 .rva .LSEH_end_poly1305_blocks_avx2
3969 .rva .LSEH_info_poly1305_blocks_avx2_3
3971 $code.=<<___ if ($avx>2);
3972 .rva .LSEH_begin_poly1305_blocks_avx512
3973 .rva .LSEH_end_poly1305_blocks_avx512
3974 .rva .LSEH_info_poly1305_blocks_avx512
3979 .LSEH_info_poly1305_init:
3982 .rva .LSEH_begin_poly1305_init,.LSEH_begin_poly1305_init
3984 .LSEH_info_poly1305_blocks:
3987 .rva .Lblocks_body,.Lblocks_epilogue
3989 .LSEH_info_poly1305_emit:
3992 .rva .LSEH_begin_poly1305_emit,.LSEH_begin_poly1305_emit
3994 $code.=<<___ if ($avx);
3995 .LSEH_info_poly1305_blocks_avx_1:
3998 .rva .Lblocks_avx_body,.Lblocks_avx_epilogue # HandlerData[]
4000 .LSEH_info_poly1305_blocks_avx_2:
4003 .rva .Lbase2_64_avx_body,.Lbase2_64_avx_epilogue # HandlerData[]
4005 .LSEH_info_poly1305_blocks_avx_3:
4008 .rva .Ldo_avx_body,.Ldo_avx_epilogue # HandlerData[]
4010 .LSEH_info_poly1305_emit_avx:
4013 .rva .LSEH_begin_poly1305_emit_avx,.LSEH_begin_poly1305_emit_avx
4015 $code.=<<___ if ($avx>1);
4016 .LSEH_info_poly1305_blocks_avx2_1:
4019 .rva .Lblocks_avx2_body,.Lblocks_avx2_epilogue # HandlerData[]
4021 .LSEH_info_poly1305_blocks_avx2_2:
4024 .rva .Lbase2_64_avx2_body,.Lbase2_64_avx2_epilogue # HandlerData[]
4026 .LSEH_info_poly1305_blocks_avx2_3:
4029 .rva .Ldo_avx2_body,.Ldo_avx2_epilogue # HandlerData[]
4031 $code.=<<___ if ($avx>2);
4032 .LSEH_info_poly1305_blocks_avx512:
4035 .rva .Ldo_avx512_body,.Ldo_avx512_epilogue # HandlerData[]
4039 foreach (split('\n',$code)) {
4040 s/\`([^\`]*)\`/eval($1)/ge;
4041 s/%r([a-z]+)#d/%e$1/g;
4042 s/%r([0-9]+)#d/%r$1d/g;
4043 s/%x#%[yz]/%x/g or s/%y#%z/%y/g or s/%z#%[yz]/%z/g;