2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
19 # ChaCha20 for x86_64.
21 # Performance in cycles per byte out of large buffer.
23 # IALU/gcc 4.8(i) 1xSSSE3/SSE2 4xSSSE3 8xAVX2
25 # P4 9.48/+99% -/22.7(ii) -
26 # Core2 7.83/+55% 7.90/8.08 4.35
27 # Westmere 7.19/+50% 5.60/6.70 3.00
28 # Sandy Bridge 8.31/+42% 5.45/6.76 2.72
29 # Ivy Bridge 6.71/+46% 5.40/6.49 2.41
30 # Haswell 5.92/+43% 5.20/6.45 2.42 1.23
31 # Silvermont 12.0/+33% 7.75/7.40 7.03(iii)
32 # Goldmont 10.6/+17% 5.10/- 3.28
33 # Sledgehammer 7.28/+52% -/14.2(ii) -
34 # Bulldozer 9.66/+28% 9.85/11.1 3.06(iv)
35 # VIA Nano 10.5/+46% 6.72/8.60 6.05
37 # (i) compared to older gcc 3.x one can observe >2x improvement on
39 # (ii) as it can be seen, SSE2 performance is too low on legacy
40 # processors; NxSSE2 results are naturally better, but not
41 # impressively better than IALU ones, which is why you won't
42 # find SSE2 code below;
43 # (iii) this is not optimal result for Atom because of MSROM
44 # limitations, SSE2 can do better, but gain is considered too
45 # low to justify the [maintenance] effort;
46 # (iv) Bulldozer actually executes 4xXOP code path that delivers 2.20;
50 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
52 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
54 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
55 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
56 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
57 die "can't locate x86_64-xlate.pl";
59 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
60 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
61 $avx = ($1>=2.19) + ($1>=2.22);
64 if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
65 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
66 $avx = ($1>=2.09) + ($1>=2.10);
69 if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
70 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
71 $avx = ($1>=10) + ($1>=11);
74 if (!$avx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9]\.[0-9]+)/) {
75 $avx = ($2>=3.0) + ($2>3.0);
78 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
81 # input parameter block
82 ($out,$inp,$len,$key,$counter)=("%rdi","%rsi","%rdx","%rcx","%r8");
87 .extern OPENSSL_ia32cap_P
101 .long 8,8,8,8,8,8,8,8
103 .byte 0x2,0x3,0x0,0x1, 0x6,0x7,0x4,0x5, 0xa,0xb,0x8,0x9, 0xe,0xf,0xc,0xd
105 .byte 0x3,0x0,0x1,0x2, 0x7,0x4,0x5,0x6, 0xb,0x8,0x9,0xa, 0xf,0xc,0xd,0xe
107 .asciz "expand 32-byte k"
108 .asciz "ChaCha20 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
111 sub AUTOLOAD() # thunk [simplified] 32-bit style perlasm
112 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
114 $arg = "\$$arg" if ($arg*1 eq $arg);
115 $code .= "\t$opcode\t".join(',',$arg,reverse @_)."\n";
118 @x=("%eax","%ebx","%ecx","%edx",map("%r${_}d",(8..11)),
119 "%nox","%nox","%nox","%nox",map("%r${_}d",(12..15)));
122 sub ROUND { # critical path is 24 cycles per round
123 my ($a0,$b0,$c0,$d0)=@_;
124 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
125 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
126 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
127 my ($xc,$xc_)=map("\"$_\"",@t);
128 my @x=map("\"$_\"",@x);
130 # Consider order in which variables are addressed by their
135 # 0 4 8 12 < even round
139 # 0 5 10 15 < odd round
144 # 'a', 'b' and 'd's are permanently allocated in registers,
145 # @x[0..7,12..15], while 'c's are maintained in memory. If
146 # you observe 'c' column, you'll notice that pair of 'c's is
147 # invariant between rounds. This means that we have to reload
148 # them once per round, in the middle. This is why you'll see
149 # bunch of 'c' stores and loads in the middle, but none in
150 # the beginning or end.
152 # Normally instructions would be interleaved to favour in-order
153 # execution. Generally out-of-order cores manage it gracefully,
154 # but not this time for some reason. As in-order execution
155 # cores are dying breed, old Atom is the only one around,
156 # instructions are left uninterleaved. Besides, Atom is better
157 # off executing 1xSSSE3 code anyway...
160 "&add (@x[$a0],@x[$b0])", # Q1
161 "&xor (@x[$d0],@x[$a0])",
163 "&add (@x[$a1],@x[$b1])", # Q2
164 "&xor (@x[$d1],@x[$a1])",
167 "&add ($xc,@x[$d0])",
168 "&xor (@x[$b0],$xc)",
170 "&add ($xc_,@x[$d1])",
171 "&xor (@x[$b1],$xc_)",
174 "&add (@x[$a0],@x[$b0])",
175 "&xor (@x[$d0],@x[$a0])",
177 "&add (@x[$a1],@x[$b1])",
178 "&xor (@x[$d1],@x[$a1])",
181 "&add ($xc,@x[$d0])",
182 "&xor (@x[$b0],$xc)",
184 "&add ($xc_,@x[$d1])",
185 "&xor (@x[$b1],$xc_)",
188 "&mov (\"4*$c0(%rsp)\",$xc)", # reload pair of 'c's
189 "&mov (\"4*$c1(%rsp)\",$xc_)",
190 "&mov ($xc,\"4*$c2(%rsp)\")",
191 "&mov ($xc_,\"4*$c3(%rsp)\")",
193 "&add (@x[$a2],@x[$b2])", # Q3
194 "&xor (@x[$d2],@x[$a2])",
196 "&add (@x[$a3],@x[$b3])", # Q4
197 "&xor (@x[$d3],@x[$a3])",
200 "&add ($xc,@x[$d2])",
201 "&xor (@x[$b2],$xc)",
203 "&add ($xc_,@x[$d3])",
204 "&xor (@x[$b3],$xc_)",
207 "&add (@x[$a2],@x[$b2])",
208 "&xor (@x[$d2],@x[$a2])",
210 "&add (@x[$a3],@x[$b3])",
211 "&xor (@x[$d3],@x[$a3])",
214 "&add ($xc,@x[$d2])",
215 "&xor (@x[$b2],$xc)",
217 "&add ($xc_,@x[$d3])",
218 "&xor (@x[$b3],$xc_)",
223 ########################################################################
224 # Generic code path that handles all lengths on pre-SSSE3 processors.
226 .globl ChaCha20_ctr32
227 .type ChaCha20_ctr32,\@function,5
232 mov OPENSSL_ia32cap_P+4(%rip),%r10
233 test \$`1<<(41-32)`,%r10d
244 #movdqa .Lsigma(%rip),%xmm0
246 movdqu 16($key),%xmm2
247 movdqu ($counter),%xmm3
248 movdqa .Lone(%rip),%xmm4
250 #movdqa %xmm0,4*0(%rsp) # key[0]
251 movdqa %xmm1,4*4(%rsp) # key[1]
252 movdqa %xmm2,4*8(%rsp) # key[2]
253 movdqa %xmm3,4*12(%rsp) # key[3]
254 mov $len,%rbp # reassign $len
259 mov \$0x61707865,@x[0] # 'expa'
260 mov \$0x3320646e,@x[1] # 'nd 3'
261 mov \$0x79622d32,@x[2] # '2-by'
262 mov \$0x6b206574,@x[3] # 'te k'
268 mov 4*13(%rsp),@x[13]
269 mov 4*14(%rsp),@x[14]
270 mov 4*15(%rsp),@x[15]
272 mov %rbp,64+0(%rsp) # save len
274 mov $inp,64+8(%rsp) # save inp
275 movq %xmm2,%rsi # "@x[8]"
276 mov $out,64+16(%rsp) # save out
278 shr \$32,%rdi # "@x[9]"
284 foreach (&ROUND (0, 4, 8,12)) { eval; }
285 foreach (&ROUND (0, 5,10,15)) { eval; }
290 mov @t[1],4*9(%rsp) # modulo-scheduled
292 mov 64(%rsp),%rbp # load len
294 mov 64+8(%rsp),$inp # load inp
295 paddd %xmm4,%xmm3 # increment counter
296 mov 64+16(%rsp),$out # load out
298 add \$0x61707865,@x[0] # 'expa'
299 add \$0x3320646e,@x[1] # 'nd 3'
300 add \$0x79622d32,@x[2] # '2-by'
301 add \$0x6b206574,@x[3] # 'te k'
306 add 4*12(%rsp),@x[12]
307 add 4*13(%rsp),@x[13]
308 add 4*14(%rsp),@x[14]
309 add 4*15(%rsp),@x[15]
310 paddd 4*8(%rsp),%xmm1
315 xor 4*0($inp),@x[0] # xor with input
323 movdqu 4*8($inp),%xmm0
324 xor 4*12($inp),@x[12]
325 xor 4*13($inp),@x[13]
326 xor 4*14($inp),@x[14]
327 xor 4*15($inp),@x[15]
328 lea 4*16($inp),$inp # inp+=64
331 movdqa %xmm2,4*8(%rsp)
332 movd %xmm3,4*12(%rsp)
334 mov @x[0],4*0($out) # write output
342 movdqu %xmm0,4*8($out)
343 mov @x[12],4*12($out)
344 mov @x[13],4*13($out)
345 mov @x[14],4*14($out)
346 mov @x[15],4*15($out)
347 lea 4*16($out),$out # out+=64
365 movdqa %xmm1,4*8(%rsp)
366 mov @x[12],4*12(%rsp)
367 mov @x[13],4*13(%rsp)
368 mov @x[14],4*14(%rsp)
369 mov @x[15],4*15(%rsp)
372 movzb ($inp,%rbx),%eax
373 movzb (%rsp,%rbx),%edx
376 mov %al,-1($out,%rbx)
390 .size ChaCha20_ctr32,.-ChaCha20_ctr32
393 ########################################################################
394 # SSSE3 code path that handles shorter lengths
396 my ($a,$b,$c,$d,$t,$t1,$rot16,$rot24)=map("%xmm$_",(0..7));
398 sub SSSE3ROUND { # critical path is 20 "SIMD ticks" per round
422 my $xframe = $win64 ? 32+32+8 : 24;
425 .type ChaCha20_ssse3,\@function,5
430 $code.=<<___ if ($avx);
431 test \$`1<<(43-32)`,%r10d
432 jnz .LChaCha20_4xop # XOP is fastest even if we use 1/4
435 cmp \$128,$len # we might throw away some data,
436 ja .LChaCha20_4x # but overall it won't be slower
446 sub \$64+$xframe,%rsp
448 $code.=<<___ if ($win64);
449 movaps %xmm6,64+32(%rsp)
450 movaps %xmm7,64+48(%rsp)
453 movdqa .Lsigma(%rip),$a
457 movdqa .Lrot16(%rip),$rot16
458 movdqa .Lrot24(%rip),$rot24
469 movdqa .Lone(%rip),$d
482 &pshufd ($c,$c,0b01001110);
483 &pshufd ($b,$b,0b00111001);
484 &pshufd ($d,$d,0b10010011);
488 &pshufd ($c,$c,0b01001110);
489 &pshufd ($b,$b,0b10010011);
490 &pshufd ($d,$d,0b00111001);
493 &jnz (".Loop_ssse3");
505 movdqu 0x10($inp),$t1
506 pxor $t,$a # xor with input
509 movdqu 0x30($inp),$t1
510 lea 0x40($inp),$inp # inp+=64
514 movdqu $a,0x00($out) # write output
518 lea 0x40($out),$out # out+=64
521 jnz .Loop_outer_ssse3
534 movzb ($inp,%rbx),%eax
535 movzb (%rsp,%rbx),%ecx
538 mov %al,-1($out,%rbx)
544 $code.=<<___ if ($win64);
545 movaps 64+32(%rsp),%xmm6
546 movaps 64+48(%rsp),%xmm7
549 add \$64+$xframe,%rsp
557 .size ChaCha20_ssse3,.-ChaCha20_ssse3
561 ########################################################################
562 # SSSE3 code path that handles longer messages.
564 # assign variables to favor Atom front-end
565 my ($xd0,$xd1,$xd2,$xd3, $xt0,$xt1,$xt2,$xt3,
566 $xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3)=map("%xmm$_",(0..15));
567 my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
568 "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3);
570 sub SSSE3_lane_ROUND {
571 my ($a0,$b0,$c0,$d0)=@_;
572 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
573 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
574 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
575 my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3);
576 my @x=map("\"$_\"",@xx);
578 # Consider order in which variables are addressed by their
583 # 0 4 8 12 < even round
587 # 0 5 10 15 < odd round
592 # 'a', 'b' and 'd's are permanently allocated in registers,
593 # @x[0..7,12..15], while 'c's are maintained in memory. If
594 # you observe 'c' column, you'll notice that pair of 'c's is
595 # invariant between rounds. This means that we have to reload
596 # them once per round, in the middle. This is why you'll see
597 # bunch of 'c' stores and loads in the middle, but none in
598 # the beginning or end.
601 "&paddd (@x[$a0],@x[$b0])", # Q1
602 "&paddd (@x[$a1],@x[$b1])", # Q2
603 "&pxor (@x[$d0],@x[$a0])",
604 "&pxor (@x[$d1],@x[$a1])",
605 "&pshufb (@x[$d0],$t1)",
606 "&pshufb (@x[$d1],$t1)",
608 "&paddd ($xc,@x[$d0])",
609 "&paddd ($xc_,@x[$d1])",
610 "&pxor (@x[$b0],$xc)",
611 "&pxor (@x[$b1],$xc_)",
612 "&movdqa ($t0,@x[$b0])",
613 "&pslld (@x[$b0],12)",
615 "&movdqa ($t1,@x[$b1])",
616 "&pslld (@x[$b1],12)",
617 "&por (@x[$b0],$t0)",
619 "&movdqa ($t0,'(%r11)')", # .Lrot24(%rip)
620 "&por (@x[$b1],$t1)",
622 "&paddd (@x[$a0],@x[$b0])",
623 "&paddd (@x[$a1],@x[$b1])",
624 "&pxor (@x[$d0],@x[$a0])",
625 "&pxor (@x[$d1],@x[$a1])",
626 "&pshufb (@x[$d0],$t0)",
627 "&pshufb (@x[$d1],$t0)",
629 "&paddd ($xc,@x[$d0])",
630 "&paddd ($xc_,@x[$d1])",
631 "&pxor (@x[$b0],$xc)",
632 "&pxor (@x[$b1],$xc_)",
633 "&movdqa ($t1,@x[$b0])",
634 "&pslld (@x[$b0],7)",
636 "&movdqa ($t0,@x[$b1])",
637 "&pslld (@x[$b1],7)",
638 "&por (@x[$b0],$t1)",
640 "&movdqa ($t1,'(%r10)')", # .Lrot16(%rip)
641 "&por (@x[$b1],$t0)",
643 "&movdqa (\"`16*($c0-8)`(%rsp)\",$xc)", # reload pair of 'c's
644 "&movdqa (\"`16*($c1-8)`(%rsp)\",$xc_)",
645 "&movdqa ($xc,\"`16*($c2-8)`(%rsp)\")",
646 "&movdqa ($xc_,\"`16*($c3-8)`(%rsp)\")",
648 "&paddd (@x[$a2],@x[$b2])", # Q3
649 "&paddd (@x[$a3],@x[$b3])", # Q4
650 "&pxor (@x[$d2],@x[$a2])",
651 "&pxor (@x[$d3],@x[$a3])",
652 "&pshufb (@x[$d2],$t1)",
653 "&pshufb (@x[$d3],$t1)",
655 "&paddd ($xc,@x[$d2])",
656 "&paddd ($xc_,@x[$d3])",
657 "&pxor (@x[$b2],$xc)",
658 "&pxor (@x[$b3],$xc_)",
659 "&movdqa ($t0,@x[$b2])",
660 "&pslld (@x[$b2],12)",
662 "&movdqa ($t1,@x[$b3])",
663 "&pslld (@x[$b3],12)",
664 "&por (@x[$b2],$t0)",
666 "&movdqa ($t0,'(%r11)')", # .Lrot24(%rip)
667 "&por (@x[$b3],$t1)",
669 "&paddd (@x[$a2],@x[$b2])",
670 "&paddd (@x[$a3],@x[$b3])",
671 "&pxor (@x[$d2],@x[$a2])",
672 "&pxor (@x[$d3],@x[$a3])",
673 "&pshufb (@x[$d2],$t0)",
674 "&pshufb (@x[$d3],$t0)",
676 "&paddd ($xc,@x[$d2])",
677 "&paddd ($xc_,@x[$d3])",
678 "&pxor (@x[$b2],$xc)",
679 "&pxor (@x[$b3],$xc_)",
680 "&movdqa ($t1,@x[$b2])",
681 "&pslld (@x[$b2],7)",
683 "&movdqa ($t0,@x[$b3])",
684 "&pslld (@x[$b3],7)",
685 "&por (@x[$b2],$t1)",
687 "&movdqa ($t1,'(%r10)')", # .Lrot16(%rip)
692 my $xframe = $win64 ? 0xa0 : 0;
695 .type ChaCha20_4x,\@function,5
701 $code.=<<___ if ($avx>1);
702 shr \$32,%r10 # OPENSSL_ia32cap_P+8
703 test \$`1<<5`,%r10 # test AVX2
710 and \$`1<<26|1<<22`,%r11 # isolate XSAVE+MOVBE
711 cmp \$`1<<22`,%r11 # check for MOVBE without XSAVE
712 je .Ldo_sse3_after_all # to detect Atom
716 sub \$0x148+$xframe,%rsp
718 ################ stack layout
719 # +0x00 SIMD equivalent of @x[8-12]
721 # +0x40 constant copy of key[0-2] smashed by lanes
723 # +0x100 SIMD counters (with nonce smashed by lanes)
726 $code.=<<___ if ($win64);
727 movaps %xmm6,-0x30(%r11)
728 movaps %xmm7,-0x20(%r11)
729 movaps %xmm8,-0x10(%r11)
730 movaps %xmm9,0x00(%r11)
731 movaps %xmm10,0x10(%r11)
732 movaps %xmm11,0x20(%r11)
733 movaps %xmm12,0x30(%r11)
734 movaps %xmm13,0x40(%r11)
735 movaps %xmm14,0x50(%r11)
736 movaps %xmm15,0x60(%r11)
739 movdqa .Lsigma(%rip),$xa3 # key[0]
740 movdqu ($key),$xb3 # key[1]
741 movdqu 16($key),$xt3 # key[2]
742 movdqu ($counter),$xd3 # key[3]
743 lea 0x100(%rsp),%rcx # size optimization
744 lea .Lrot16(%rip),%r10
745 lea .Lrot24(%rip),%r11
747 pshufd \$0x00,$xa3,$xa0 # smash key by lanes...
748 pshufd \$0x55,$xa3,$xa1
749 movdqa $xa0,0x40(%rsp) # ... and offload
750 pshufd \$0xaa,$xa3,$xa2
751 movdqa $xa1,0x50(%rsp)
752 pshufd \$0xff,$xa3,$xa3
753 movdqa $xa2,0x60(%rsp)
754 movdqa $xa3,0x70(%rsp)
756 pshufd \$0x00,$xb3,$xb0
757 pshufd \$0x55,$xb3,$xb1
758 movdqa $xb0,0x80-0x100(%rcx)
759 pshufd \$0xaa,$xb3,$xb2
760 movdqa $xb1,0x90-0x100(%rcx)
761 pshufd \$0xff,$xb3,$xb3
762 movdqa $xb2,0xa0-0x100(%rcx)
763 movdqa $xb3,0xb0-0x100(%rcx)
765 pshufd \$0x00,$xt3,$xt0 # "$xc0"
766 pshufd \$0x55,$xt3,$xt1 # "$xc1"
767 movdqa $xt0,0xc0-0x100(%rcx)
768 pshufd \$0xaa,$xt3,$xt2 # "$xc2"
769 movdqa $xt1,0xd0-0x100(%rcx)
770 pshufd \$0xff,$xt3,$xt3 # "$xc3"
771 movdqa $xt2,0xe0-0x100(%rcx)
772 movdqa $xt3,0xf0-0x100(%rcx)
774 pshufd \$0x00,$xd3,$xd0
775 pshufd \$0x55,$xd3,$xd1
776 paddd .Linc(%rip),$xd0 # don't save counters yet
777 pshufd \$0xaa,$xd3,$xd2
778 movdqa $xd1,0x110-0x100(%rcx)
779 pshufd \$0xff,$xd3,$xd3
780 movdqa $xd2,0x120-0x100(%rcx)
781 movdqa $xd3,0x130-0x100(%rcx)
787 movdqa 0x40(%rsp),$xa0 # re-load smashed key
788 movdqa 0x50(%rsp),$xa1
789 movdqa 0x60(%rsp),$xa2
790 movdqa 0x70(%rsp),$xa3
791 movdqa 0x80-0x100(%rcx),$xb0
792 movdqa 0x90-0x100(%rcx),$xb1
793 movdqa 0xa0-0x100(%rcx),$xb2
794 movdqa 0xb0-0x100(%rcx),$xb3
795 movdqa 0xc0-0x100(%rcx),$xt0 # "$xc0"
796 movdqa 0xd0-0x100(%rcx),$xt1 # "$xc1"
797 movdqa 0xe0-0x100(%rcx),$xt2 # "$xc2"
798 movdqa 0xf0-0x100(%rcx),$xt3 # "$xc3"
799 movdqa 0x100-0x100(%rcx),$xd0
800 movdqa 0x110-0x100(%rcx),$xd1
801 movdqa 0x120-0x100(%rcx),$xd2
802 movdqa 0x130-0x100(%rcx),$xd3
803 paddd .Lfour(%rip),$xd0 # next SIMD counters
806 movdqa $xt2,0x20(%rsp) # SIMD equivalent of "@x[10]"
807 movdqa $xt3,0x30(%rsp) # SIMD equivalent of "@x[11]"
808 movdqa (%r10),$xt3 # .Lrot16(%rip)
810 movdqa $xd0,0x100-0x100(%rcx) # save SIMD counters
816 foreach (&SSSE3_lane_ROUND(0, 4, 8,12)) { eval; }
817 foreach (&SSSE3_lane_ROUND(0, 5,10,15)) { eval; }
822 paddd 0x40(%rsp),$xa0 # accumulate key material
823 paddd 0x50(%rsp),$xa1
824 paddd 0x60(%rsp),$xa2
825 paddd 0x70(%rsp),$xa3
827 movdqa $xa0,$xt2 # "de-interlace" data
834 punpcklqdq $xa2,$xa0 # "a0"
836 punpcklqdq $xt3,$xt2 # "a2"
837 punpckhqdq $xa2,$xa1 # "a1"
838 punpckhqdq $xt3,$xa3 # "a3"
840 ($xa2,$xt2)=($xt2,$xa2);
842 paddd 0x80-0x100(%rcx),$xb0
843 paddd 0x90-0x100(%rcx),$xb1
844 paddd 0xa0-0x100(%rcx),$xb2
845 paddd 0xb0-0x100(%rcx),$xb3
847 movdqa $xa0,0x00(%rsp) # offload $xaN
848 movdqa $xa1,0x10(%rsp)
849 movdqa 0x20(%rsp),$xa0 # "xc2"
850 movdqa 0x30(%rsp),$xa1 # "xc3"
859 punpcklqdq $xb2,$xb0 # "b0"
861 punpcklqdq $xt3,$xt2 # "b2"
862 punpckhqdq $xb2,$xb1 # "b1"
863 punpckhqdq $xt3,$xb3 # "b3"
865 ($xb2,$xt2)=($xt2,$xb2);
866 my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1);
868 paddd 0xc0-0x100(%rcx),$xc0
869 paddd 0xd0-0x100(%rcx),$xc1
870 paddd 0xe0-0x100(%rcx),$xc2
871 paddd 0xf0-0x100(%rcx),$xc3
873 movdqa $xa2,0x20(%rsp) # keep offloading $xaN
874 movdqa $xa3,0x30(%rsp)
883 punpcklqdq $xc2,$xc0 # "c0"
885 punpcklqdq $xt3,$xt2 # "c2"
886 punpckhqdq $xc2,$xc1 # "c1"
887 punpckhqdq $xt3,$xc3 # "c3"
889 ($xc2,$xt2)=($xt2,$xc2);
890 ($xt0,$xt1)=($xa2,$xa3); # use $xaN as temporary
892 paddd 0x100-0x100(%rcx),$xd0
893 paddd 0x110-0x100(%rcx),$xd1
894 paddd 0x120-0x100(%rcx),$xd2
895 paddd 0x130-0x100(%rcx),$xd3
904 punpcklqdq $xd2,$xd0 # "d0"
906 punpcklqdq $xt3,$xt2 # "d2"
907 punpckhqdq $xd2,$xd1 # "d1"
908 punpckhqdq $xt3,$xd3 # "d3"
910 ($xd2,$xt2)=($xt2,$xd2);
915 movdqu 0x00($inp),$xt0 # xor with input
916 movdqu 0x10($inp),$xt1
917 movdqu 0x20($inp),$xt2
918 movdqu 0x30($inp),$xt3
919 pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember?
924 movdqu $xt0,0x00($out)
925 movdqu 0x40($inp),$xt0
926 movdqu $xt1,0x10($out)
927 movdqu 0x50($inp),$xt1
928 movdqu $xt2,0x20($out)
929 movdqu 0x60($inp),$xt2
930 movdqu $xt3,0x30($out)
931 movdqu 0x70($inp),$xt3
932 lea 0x80($inp),$inp # size optimization
938 movdqu $xt0,0x40($out)
939 movdqu 0x00($inp),$xt0
940 movdqu $xt1,0x50($out)
941 movdqu 0x10($inp),$xt1
942 movdqu $xt2,0x60($out)
943 movdqu 0x20($inp),$xt2
944 movdqu $xt3,0x70($out)
945 lea 0x80($out),$out # size optimization
946 movdqu 0x30($inp),$xt3
952 movdqu $xt0,0x00($out)
953 movdqu 0x40($inp),$xt0
954 movdqu $xt1,0x10($out)
955 movdqu 0x50($inp),$xt1
956 movdqu $xt2,0x20($out)
957 movdqu 0x60($inp),$xt2
958 movdqu $xt3,0x30($out)
959 movdqu 0x70($inp),$xt3
960 lea 0x80($inp),$inp # inp+=64*4
965 movdqu $xt0,0x40($out)
966 movdqu $xt1,0x50($out)
967 movdqu $xt2,0x60($out)
968 movdqu $xt3,0x70($out)
969 lea 0x80($out),$out # out+=64*4
984 #movdqa 0x00(%rsp),$xt0 # $xaN is offloaded, remember?
986 #movdqa $xt0,0x00(%rsp)
987 movdqa $xb0,0x10(%rsp)
988 movdqa $xc0,0x20(%rsp)
989 movdqa $xd0,0x30(%rsp)
994 movdqu 0x00($inp),$xt0 # xor with input
995 movdqu 0x10($inp),$xt1
996 movdqu 0x20($inp),$xt2
997 movdqu 0x30($inp),$xt3
998 pxor 0x00(%rsp),$xt0 # $xaxN is offloaded, remember?
1002 movdqu $xt0,0x00($out)
1003 movdqu $xt1,0x10($out)
1004 movdqu $xt2,0x20($out)
1005 movdqu $xt3,0x30($out)
1008 movdqa 0x10(%rsp),$xt0 # $xaN is offloaded, remember?
1009 lea 0x40($inp),$inp # inp+=64*1
1011 movdqa $xt0,0x00(%rsp)
1012 movdqa $xb1,0x10(%rsp)
1013 lea 0x40($out),$out # out+=64*1
1014 movdqa $xc1,0x20(%rsp)
1015 sub \$64,$len # len-=64*1
1016 movdqa $xd1,0x30(%rsp)
1021 movdqu 0x00($inp),$xt0 # xor with input
1022 movdqu 0x10($inp),$xt1
1023 movdqu 0x20($inp),$xt2
1024 movdqu 0x30($inp),$xt3
1025 pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember?
1030 movdqu $xt0,0x00($out)
1031 movdqu 0x40($inp),$xt0
1032 movdqu $xt1,0x10($out)
1033 movdqu 0x50($inp),$xt1
1034 movdqu $xt2,0x20($out)
1035 movdqu 0x60($inp),$xt2
1036 movdqu $xt3,0x30($out)
1037 movdqu 0x70($inp),$xt3
1038 pxor 0x10(%rsp),$xt0
1042 movdqu $xt0,0x40($out)
1043 movdqu $xt1,0x50($out)
1044 movdqu $xt2,0x60($out)
1045 movdqu $xt3,0x70($out)
1048 movdqa 0x20(%rsp),$xt0 # $xaN is offloaded, remember?
1049 lea 0x80($inp),$inp # inp+=64*2
1051 movdqa $xt0,0x00(%rsp)
1052 movdqa $xb2,0x10(%rsp)
1053 lea 0x80($out),$out # out+=64*2
1054 movdqa $xc2,0x20(%rsp)
1055 sub \$128,$len # len-=64*2
1056 movdqa $xd2,0x30(%rsp)
1061 movdqu 0x00($inp),$xt0 # xor with input
1062 movdqu 0x10($inp),$xt1
1063 movdqu 0x20($inp),$xt2
1064 movdqu 0x30($inp),$xt3
1065 pxor 0x00(%rsp),$xt0 # $xaN is offloaded, remember?
1070 movdqu $xt0,0x00($out)
1071 movdqu 0x40($inp),$xt0
1072 movdqu $xt1,0x10($out)
1073 movdqu 0x50($inp),$xt1
1074 movdqu $xt2,0x20($out)
1075 movdqu 0x60($inp),$xt2
1076 movdqu $xt3,0x30($out)
1077 movdqu 0x70($inp),$xt3
1078 lea 0x80($inp),$inp # size optimization
1079 pxor 0x10(%rsp),$xt0
1084 movdqu $xt0,0x40($out)
1085 movdqu 0x00($inp),$xt0
1086 movdqu $xt1,0x50($out)
1087 movdqu 0x10($inp),$xt1
1088 movdqu $xt2,0x60($out)
1089 movdqu 0x20($inp),$xt2
1090 movdqu $xt3,0x70($out)
1091 lea 0x80($out),$out # size optimization
1092 movdqu 0x30($inp),$xt3
1093 pxor 0x20(%rsp),$xt0
1097 movdqu $xt0,0x00($out)
1098 movdqu $xt1,0x10($out)
1099 movdqu $xt2,0x20($out)
1100 movdqu $xt3,0x30($out)
1103 movdqa 0x30(%rsp),$xt0 # $xaN is offloaded, remember?
1104 lea 0x40($inp),$inp # inp+=64*3
1106 movdqa $xt0,0x00(%rsp)
1107 movdqa $xb3,0x10(%rsp)
1108 lea 0x40($out),$out # out+=64*3
1109 movdqa $xc3,0x20(%rsp)
1110 sub \$192,$len # len-=64*3
1111 movdqa $xd3,0x30(%rsp)
1114 movzb ($inp,%r10),%eax
1115 movzb (%rsp,%r10),%ecx
1118 mov %al,-1($out,%r10)
1124 $code.=<<___ if ($win64);
1125 lea 0x140+0x30(%rsp),%r11
1126 movaps -0x30(%r11),%xmm6
1127 movaps -0x20(%r11),%xmm7
1128 movaps -0x10(%r11),%xmm8
1129 movaps 0x00(%r11),%xmm9
1130 movaps 0x10(%r11),%xmm10
1131 movaps 0x20(%r11),%xmm11
1132 movaps 0x30(%r11),%xmm12
1133 movaps 0x40(%r11),%xmm13
1134 movaps 0x50(%r11),%xmm14
1135 movaps 0x60(%r11),%xmm15
1138 add \$0x148+$xframe,%rsp
1140 .size ChaCha20_4x,.-ChaCha20_4x
1144 ########################################################################
1145 # XOP code path that handles all lengths.
1147 # There is some "anomaly" observed depending on instructions' size or
1148 # alignment. If you look closely at below code you'll notice that
1149 # sometimes argument order varies. The order affects instruction
1150 # encoding by making it larger, and such fiddling gives 5% performance
1151 # improvement. This is on FX-4100...
1153 my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3,
1154 $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%xmm$_",(0..15));
1155 my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
1156 $xt0,$xt1,$xt2,$xt3, $xd0,$xd1,$xd2,$xd3);
1158 sub XOP_lane_ROUND {
1159 my ($a0,$b0,$c0,$d0)=@_;
1160 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
1161 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
1162 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
1163 my @x=map("\"$_\"",@xx);
1166 "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1
1167 "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2
1168 "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3
1169 "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4
1170 "&vpxor (@x[$d0],@x[$a0],@x[$d0])",
1171 "&vpxor (@x[$d1],@x[$a1],@x[$d1])",
1172 "&vpxor (@x[$d2],@x[$a2],@x[$d2])",
1173 "&vpxor (@x[$d3],@x[$a3],@x[$d3])",
1174 "&vprotd (@x[$d0],@x[$d0],16)",
1175 "&vprotd (@x[$d1],@x[$d1],16)",
1176 "&vprotd (@x[$d2],@x[$d2],16)",
1177 "&vprotd (@x[$d3],@x[$d3],16)",
1179 "&vpaddd (@x[$c0],@x[$c0],@x[$d0])",
1180 "&vpaddd (@x[$c1],@x[$c1],@x[$d1])",
1181 "&vpaddd (@x[$c2],@x[$c2],@x[$d2])",
1182 "&vpaddd (@x[$c3],@x[$c3],@x[$d3])",
1183 "&vpxor (@x[$b0],@x[$c0],@x[$b0])",
1184 "&vpxor (@x[$b1],@x[$c1],@x[$b1])",
1185 "&vpxor (@x[$b2],@x[$b2],@x[$c2])", # flip
1186 "&vpxor (@x[$b3],@x[$b3],@x[$c3])", # flip
1187 "&vprotd (@x[$b0],@x[$b0],12)",
1188 "&vprotd (@x[$b1],@x[$b1],12)",
1189 "&vprotd (@x[$b2],@x[$b2],12)",
1190 "&vprotd (@x[$b3],@x[$b3],12)",
1192 "&vpaddd (@x[$a0],@x[$b0],@x[$a0])", # flip
1193 "&vpaddd (@x[$a1],@x[$b1],@x[$a1])", # flip
1194 "&vpaddd (@x[$a2],@x[$a2],@x[$b2])",
1195 "&vpaddd (@x[$a3],@x[$a3],@x[$b3])",
1196 "&vpxor (@x[$d0],@x[$a0],@x[$d0])",
1197 "&vpxor (@x[$d1],@x[$a1],@x[$d1])",
1198 "&vpxor (@x[$d2],@x[$a2],@x[$d2])",
1199 "&vpxor (@x[$d3],@x[$a3],@x[$d3])",
1200 "&vprotd (@x[$d0],@x[$d0],8)",
1201 "&vprotd (@x[$d1],@x[$d1],8)",
1202 "&vprotd (@x[$d2],@x[$d2],8)",
1203 "&vprotd (@x[$d3],@x[$d3],8)",
1205 "&vpaddd (@x[$c0],@x[$c0],@x[$d0])",
1206 "&vpaddd (@x[$c1],@x[$c1],@x[$d1])",
1207 "&vpaddd (@x[$c2],@x[$c2],@x[$d2])",
1208 "&vpaddd (@x[$c3],@x[$c3],@x[$d3])",
1209 "&vpxor (@x[$b0],@x[$c0],@x[$b0])",
1210 "&vpxor (@x[$b1],@x[$c1],@x[$b1])",
1211 "&vpxor (@x[$b2],@x[$b2],@x[$c2])", # flip
1212 "&vpxor (@x[$b3],@x[$b3],@x[$c3])", # flip
1213 "&vprotd (@x[$b0],@x[$b0],7)",
1214 "&vprotd (@x[$b1],@x[$b1],7)",
1215 "&vprotd (@x[$b2],@x[$b2],7)",
1216 "&vprotd (@x[$b3],@x[$b3],7)"
1220 my $xframe = $win64 ? 0xa0 : 0;
1223 .type ChaCha20_4xop,\@function,5
1227 lea -0x78(%rsp),%r11
1228 sub \$0x148+$xframe,%rsp
1230 ################ stack layout
1231 # +0x00 SIMD equivalent of @x[8-12]
1233 # +0x40 constant copy of key[0-2] smashed by lanes
1235 # +0x100 SIMD counters (with nonce smashed by lanes)
1238 $code.=<<___ if ($win64);
1239 movaps %xmm6,-0x30(%r11)
1240 movaps %xmm7,-0x20(%r11)
1241 movaps %xmm8,-0x10(%r11)
1242 movaps %xmm9,0x00(%r11)
1243 movaps %xmm10,0x10(%r11)
1244 movaps %xmm11,0x20(%r11)
1245 movaps %xmm12,0x30(%r11)
1246 movaps %xmm13,0x40(%r11)
1247 movaps %xmm14,0x50(%r11)
1248 movaps %xmm15,0x60(%r11)
1253 vmovdqa .Lsigma(%rip),$xa3 # key[0]
1254 vmovdqu ($key),$xb3 # key[1]
1255 vmovdqu 16($key),$xt3 # key[2]
1256 vmovdqu ($counter),$xd3 # key[3]
1257 lea 0x100(%rsp),%rcx # size optimization
1259 vpshufd \$0x00,$xa3,$xa0 # smash key by lanes...
1260 vpshufd \$0x55,$xa3,$xa1
1261 vmovdqa $xa0,0x40(%rsp) # ... and offload
1262 vpshufd \$0xaa,$xa3,$xa2
1263 vmovdqa $xa1,0x50(%rsp)
1264 vpshufd \$0xff,$xa3,$xa3
1265 vmovdqa $xa2,0x60(%rsp)
1266 vmovdqa $xa3,0x70(%rsp)
1268 vpshufd \$0x00,$xb3,$xb0
1269 vpshufd \$0x55,$xb3,$xb1
1270 vmovdqa $xb0,0x80-0x100(%rcx)
1271 vpshufd \$0xaa,$xb3,$xb2
1272 vmovdqa $xb1,0x90-0x100(%rcx)
1273 vpshufd \$0xff,$xb3,$xb3
1274 vmovdqa $xb2,0xa0-0x100(%rcx)
1275 vmovdqa $xb3,0xb0-0x100(%rcx)
1277 vpshufd \$0x00,$xt3,$xt0 # "$xc0"
1278 vpshufd \$0x55,$xt3,$xt1 # "$xc1"
1279 vmovdqa $xt0,0xc0-0x100(%rcx)
1280 vpshufd \$0xaa,$xt3,$xt2 # "$xc2"
1281 vmovdqa $xt1,0xd0-0x100(%rcx)
1282 vpshufd \$0xff,$xt3,$xt3 # "$xc3"
1283 vmovdqa $xt2,0xe0-0x100(%rcx)
1284 vmovdqa $xt3,0xf0-0x100(%rcx)
1286 vpshufd \$0x00,$xd3,$xd0
1287 vpshufd \$0x55,$xd3,$xd1
1288 vpaddd .Linc(%rip),$xd0,$xd0 # don't save counters yet
1289 vpshufd \$0xaa,$xd3,$xd2
1290 vmovdqa $xd1,0x110-0x100(%rcx)
1291 vpshufd \$0xff,$xd3,$xd3
1292 vmovdqa $xd2,0x120-0x100(%rcx)
1293 vmovdqa $xd3,0x130-0x100(%rcx)
1299 vmovdqa 0x40(%rsp),$xa0 # re-load smashed key
1300 vmovdqa 0x50(%rsp),$xa1
1301 vmovdqa 0x60(%rsp),$xa2
1302 vmovdqa 0x70(%rsp),$xa3
1303 vmovdqa 0x80-0x100(%rcx),$xb0
1304 vmovdqa 0x90-0x100(%rcx),$xb1
1305 vmovdqa 0xa0-0x100(%rcx),$xb2
1306 vmovdqa 0xb0-0x100(%rcx),$xb3
1307 vmovdqa 0xc0-0x100(%rcx),$xt0 # "$xc0"
1308 vmovdqa 0xd0-0x100(%rcx),$xt1 # "$xc1"
1309 vmovdqa 0xe0-0x100(%rcx),$xt2 # "$xc2"
1310 vmovdqa 0xf0-0x100(%rcx),$xt3 # "$xc3"
1311 vmovdqa 0x100-0x100(%rcx),$xd0
1312 vmovdqa 0x110-0x100(%rcx),$xd1
1313 vmovdqa 0x120-0x100(%rcx),$xd2
1314 vmovdqa 0x130-0x100(%rcx),$xd3
1315 vpaddd .Lfour(%rip),$xd0,$xd0 # next SIMD counters
1319 vmovdqa $xd0,0x100-0x100(%rcx) # save SIMD counters
1325 foreach (&XOP_lane_ROUND(0, 4, 8,12)) { eval; }
1326 foreach (&XOP_lane_ROUND(0, 5,10,15)) { eval; }
1331 vpaddd 0x40(%rsp),$xa0,$xa0 # accumulate key material
1332 vpaddd 0x50(%rsp),$xa1,$xa1
1333 vpaddd 0x60(%rsp),$xa2,$xa2
1334 vpaddd 0x70(%rsp),$xa3,$xa3
1336 vmovdqa $xt2,0x20(%rsp) # offload $xc2,3
1337 vmovdqa $xt3,0x30(%rsp)
1339 vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data
1340 vpunpckldq $xa3,$xa2,$xt3
1341 vpunpckhdq $xa1,$xa0,$xa0
1342 vpunpckhdq $xa3,$xa2,$xa2
1343 vpunpcklqdq $xt3,$xt2,$xa1 # "a0"
1344 vpunpckhqdq $xt3,$xt2,$xt2 # "a1"
1345 vpunpcklqdq $xa2,$xa0,$xa3 # "a2"
1346 vpunpckhqdq $xa2,$xa0,$xa0 # "a3"
1348 ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2);
1350 vpaddd 0x80-0x100(%rcx),$xb0,$xb0
1351 vpaddd 0x90-0x100(%rcx),$xb1,$xb1
1352 vpaddd 0xa0-0x100(%rcx),$xb2,$xb2
1353 vpaddd 0xb0-0x100(%rcx),$xb3,$xb3
1355 vmovdqa $xa0,0x00(%rsp) # offload $xa0,1
1356 vmovdqa $xa1,0x10(%rsp)
1357 vmovdqa 0x20(%rsp),$xa0 # "xc2"
1358 vmovdqa 0x30(%rsp),$xa1 # "xc3"
1360 vpunpckldq $xb1,$xb0,$xt2
1361 vpunpckldq $xb3,$xb2,$xt3
1362 vpunpckhdq $xb1,$xb0,$xb0
1363 vpunpckhdq $xb3,$xb2,$xb2
1364 vpunpcklqdq $xt3,$xt2,$xb1 # "b0"
1365 vpunpckhqdq $xt3,$xt2,$xt2 # "b1"
1366 vpunpcklqdq $xb2,$xb0,$xb3 # "b2"
1367 vpunpckhqdq $xb2,$xb0,$xb0 # "b3"
1369 ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2);
1370 my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1);
1372 vpaddd 0xc0-0x100(%rcx),$xc0,$xc0
1373 vpaddd 0xd0-0x100(%rcx),$xc1,$xc1
1374 vpaddd 0xe0-0x100(%rcx),$xc2,$xc2
1375 vpaddd 0xf0-0x100(%rcx),$xc3,$xc3
1377 vpunpckldq $xc1,$xc0,$xt2
1378 vpunpckldq $xc3,$xc2,$xt3
1379 vpunpckhdq $xc1,$xc0,$xc0
1380 vpunpckhdq $xc3,$xc2,$xc2
1381 vpunpcklqdq $xt3,$xt2,$xc1 # "c0"
1382 vpunpckhqdq $xt3,$xt2,$xt2 # "c1"
1383 vpunpcklqdq $xc2,$xc0,$xc3 # "c2"
1384 vpunpckhqdq $xc2,$xc0,$xc0 # "c3"
1386 ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2);
1388 vpaddd 0x100-0x100(%rcx),$xd0,$xd0
1389 vpaddd 0x110-0x100(%rcx),$xd1,$xd1
1390 vpaddd 0x120-0x100(%rcx),$xd2,$xd2
1391 vpaddd 0x130-0x100(%rcx),$xd3,$xd3
1393 vpunpckldq $xd1,$xd0,$xt2
1394 vpunpckldq $xd3,$xd2,$xt3
1395 vpunpckhdq $xd1,$xd0,$xd0
1396 vpunpckhdq $xd3,$xd2,$xd2
1397 vpunpcklqdq $xt3,$xt2,$xd1 # "d0"
1398 vpunpckhqdq $xt3,$xt2,$xt2 # "d1"
1399 vpunpcklqdq $xd2,$xd0,$xd3 # "d2"
1400 vpunpckhqdq $xd2,$xd0,$xd0 # "d3"
1402 ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2);
1403 ($xa0,$xa1)=($xt2,$xt3);
1405 vmovdqa 0x00(%rsp),$xa0 # restore $xa0,1
1406 vmovdqa 0x10(%rsp),$xa1
1411 vpxor 0x00($inp),$xa0,$xa0 # xor with input
1412 vpxor 0x10($inp),$xb0,$xb0
1413 vpxor 0x20($inp),$xc0,$xc0
1414 vpxor 0x30($inp),$xd0,$xd0
1415 vpxor 0x40($inp),$xa1,$xa1
1416 vpxor 0x50($inp),$xb1,$xb1
1417 vpxor 0x60($inp),$xc1,$xc1
1418 vpxor 0x70($inp),$xd1,$xd1
1419 lea 0x80($inp),$inp # size optimization
1420 vpxor 0x00($inp),$xa2,$xa2
1421 vpxor 0x10($inp),$xb2,$xb2
1422 vpxor 0x20($inp),$xc2,$xc2
1423 vpxor 0x30($inp),$xd2,$xd2
1424 vpxor 0x40($inp),$xa3,$xa3
1425 vpxor 0x50($inp),$xb3,$xb3
1426 vpxor 0x60($inp),$xc3,$xc3
1427 vpxor 0x70($inp),$xd3,$xd3
1428 lea 0x80($inp),$inp # inp+=64*4
1430 vmovdqu $xa0,0x00($out)
1431 vmovdqu $xb0,0x10($out)
1432 vmovdqu $xc0,0x20($out)
1433 vmovdqu $xd0,0x30($out)
1434 vmovdqu $xa1,0x40($out)
1435 vmovdqu $xb1,0x50($out)
1436 vmovdqu $xc1,0x60($out)
1437 vmovdqu $xd1,0x70($out)
1438 lea 0x80($out),$out # size optimization
1439 vmovdqu $xa2,0x00($out)
1440 vmovdqu $xb2,0x10($out)
1441 vmovdqu $xc2,0x20($out)
1442 vmovdqu $xd2,0x30($out)
1443 vmovdqu $xa3,0x40($out)
1444 vmovdqu $xb3,0x50($out)
1445 vmovdqu $xc3,0x60($out)
1446 vmovdqu $xd3,0x70($out)
1447 lea 0x80($out),$out # out+=64*4
1457 jae .L192_or_more4xop
1459 jae .L128_or_more4xop
1461 jae .L64_or_more4xop
1464 vmovdqa $xa0,0x00(%rsp)
1465 vmovdqa $xb0,0x10(%rsp)
1466 vmovdqa $xc0,0x20(%rsp)
1467 vmovdqa $xd0,0x30(%rsp)
1472 vpxor 0x00($inp),$xa0,$xa0 # xor with input
1473 vpxor 0x10($inp),$xb0,$xb0
1474 vpxor 0x20($inp),$xc0,$xc0
1475 vpxor 0x30($inp),$xd0,$xd0
1476 vmovdqu $xa0,0x00($out)
1477 vmovdqu $xb0,0x10($out)
1478 vmovdqu $xc0,0x20($out)
1479 vmovdqu $xd0,0x30($out)
1482 lea 0x40($inp),$inp # inp+=64*1
1483 vmovdqa $xa1,0x00(%rsp)
1485 vmovdqa $xb1,0x10(%rsp)
1486 lea 0x40($out),$out # out+=64*1
1487 vmovdqa $xc1,0x20(%rsp)
1488 sub \$64,$len # len-=64*1
1489 vmovdqa $xd1,0x30(%rsp)
1494 vpxor 0x00($inp),$xa0,$xa0 # xor with input
1495 vpxor 0x10($inp),$xb0,$xb0
1496 vpxor 0x20($inp),$xc0,$xc0
1497 vpxor 0x30($inp),$xd0,$xd0
1498 vpxor 0x40($inp),$xa1,$xa1
1499 vpxor 0x50($inp),$xb1,$xb1
1500 vpxor 0x60($inp),$xc1,$xc1
1501 vpxor 0x70($inp),$xd1,$xd1
1503 vmovdqu $xa0,0x00($out)
1504 vmovdqu $xb0,0x10($out)
1505 vmovdqu $xc0,0x20($out)
1506 vmovdqu $xd0,0x30($out)
1507 vmovdqu $xa1,0x40($out)
1508 vmovdqu $xb1,0x50($out)
1509 vmovdqu $xc1,0x60($out)
1510 vmovdqu $xd1,0x70($out)
1513 lea 0x80($inp),$inp # inp+=64*2
1514 vmovdqa $xa2,0x00(%rsp)
1516 vmovdqa $xb2,0x10(%rsp)
1517 lea 0x80($out),$out # out+=64*2
1518 vmovdqa $xc2,0x20(%rsp)
1519 sub \$128,$len # len-=64*2
1520 vmovdqa $xd2,0x30(%rsp)
1525 vpxor 0x00($inp),$xa0,$xa0 # xor with input
1526 vpxor 0x10($inp),$xb0,$xb0
1527 vpxor 0x20($inp),$xc0,$xc0
1528 vpxor 0x30($inp),$xd0,$xd0
1529 vpxor 0x40($inp),$xa1,$xa1
1530 vpxor 0x50($inp),$xb1,$xb1
1531 vpxor 0x60($inp),$xc1,$xc1
1532 vpxor 0x70($inp),$xd1,$xd1
1533 lea 0x80($inp),$inp # size optimization
1534 vpxor 0x00($inp),$xa2,$xa2
1535 vpxor 0x10($inp),$xb2,$xb2
1536 vpxor 0x20($inp),$xc2,$xc2
1537 vpxor 0x30($inp),$xd2,$xd2
1539 vmovdqu $xa0,0x00($out)
1540 vmovdqu $xb0,0x10($out)
1541 vmovdqu $xc0,0x20($out)
1542 vmovdqu $xd0,0x30($out)
1543 vmovdqu $xa1,0x40($out)
1544 vmovdqu $xb1,0x50($out)
1545 vmovdqu $xc1,0x60($out)
1546 vmovdqu $xd1,0x70($out)
1547 lea 0x80($out),$out # size optimization
1548 vmovdqu $xa2,0x00($out)
1549 vmovdqu $xb2,0x10($out)
1550 vmovdqu $xc2,0x20($out)
1551 vmovdqu $xd2,0x30($out)
1554 lea 0x40($inp),$inp # inp+=64*3
1555 vmovdqa $xa3,0x00(%rsp)
1557 vmovdqa $xb3,0x10(%rsp)
1558 lea 0x40($out),$out # out+=64*3
1559 vmovdqa $xc3,0x20(%rsp)
1560 sub \$192,$len # len-=64*3
1561 vmovdqa $xd3,0x30(%rsp)
1564 movzb ($inp,%r10),%eax
1565 movzb (%rsp,%r10),%ecx
1568 mov %al,-1($out,%r10)
1575 $code.=<<___ if ($win64);
1576 lea 0x140+0x30(%rsp),%r11
1577 movaps -0x30(%r11),%xmm6
1578 movaps -0x20(%r11),%xmm7
1579 movaps -0x10(%r11),%xmm8
1580 movaps 0x00(%r11),%xmm9
1581 movaps 0x10(%r11),%xmm10
1582 movaps 0x20(%r11),%xmm11
1583 movaps 0x30(%r11),%xmm12
1584 movaps 0x40(%r11),%xmm13
1585 movaps 0x50(%r11),%xmm14
1586 movaps 0x60(%r11),%xmm15
1589 add \$0x148+$xframe,%rsp
1591 .size ChaCha20_4xop,.-ChaCha20_4xop
1595 ########################################################################
1598 my ($xb0,$xb1,$xb2,$xb3, $xd0,$xd1,$xd2,$xd3,
1599 $xa0,$xa1,$xa2,$xa3, $xt0,$xt1,$xt2,$xt3)=map("%ymm$_",(0..15));
1600 my @xx=($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
1601 "%nox","%nox","%nox","%nox", $xd0,$xd1,$xd2,$xd3);
1603 sub AVX2_lane_ROUND {
1604 my ($a0,$b0,$c0,$d0)=@_;
1605 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
1606 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
1607 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
1608 my ($xc,$xc_,$t0,$t1)=map("\"$_\"",$xt0,$xt1,$xt2,$xt3);
1609 my @x=map("\"$_\"",@xx);
1611 # Consider order in which variables are addressed by their
1616 # 0 4 8 12 < even round
1620 # 0 5 10 15 < odd round
1625 # 'a', 'b' and 'd's are permanently allocated in registers,
1626 # @x[0..7,12..15], while 'c's are maintained in memory. If
1627 # you observe 'c' column, you'll notice that pair of 'c's is
1628 # invariant between rounds. This means that we have to reload
1629 # them once per round, in the middle. This is why you'll see
1630 # bunch of 'c' stores and loads in the middle, but none in
1631 # the beginning or end.
1634 "&vpaddd (@x[$a0],@x[$a0],@x[$b0])", # Q1
1635 "&vpxor (@x[$d0],@x[$a0],@x[$d0])",
1636 "&vpshufb (@x[$d0],@x[$d0],$t1)",
1637 "&vpaddd (@x[$a1],@x[$a1],@x[$b1])", # Q2
1638 "&vpxor (@x[$d1],@x[$a1],@x[$d1])",
1639 "&vpshufb (@x[$d1],@x[$d1],$t1)",
1641 "&vpaddd ($xc,$xc,@x[$d0])",
1642 "&vpxor (@x[$b0],$xc,@x[$b0])",
1643 "&vpslld ($t0,@x[$b0],12)",
1644 "&vpsrld (@x[$b0],@x[$b0],20)",
1645 "&vpor (@x[$b0],$t0,@x[$b0])",
1646 "&vbroadcasti128($t0,'(%r11)')", # .Lrot24(%rip)
1647 "&vpaddd ($xc_,$xc_,@x[$d1])",
1648 "&vpxor (@x[$b1],$xc_,@x[$b1])",
1649 "&vpslld ($t1,@x[$b1],12)",
1650 "&vpsrld (@x[$b1],@x[$b1],20)",
1651 "&vpor (@x[$b1],$t1,@x[$b1])",
1653 "&vpaddd (@x[$a0],@x[$a0],@x[$b0])",
1654 "&vpxor (@x[$d0],@x[$a0],@x[$d0])",
1655 "&vpshufb (@x[$d0],@x[$d0],$t0)",
1656 "&vpaddd (@x[$a1],@x[$a1],@x[$b1])",
1657 "&vpxor (@x[$d1],@x[$a1],@x[$d1])",
1658 "&vpshufb (@x[$d1],@x[$d1],$t0)",
1660 "&vpaddd ($xc,$xc,@x[$d0])",
1661 "&vpxor (@x[$b0],$xc,@x[$b0])",
1662 "&vpslld ($t1,@x[$b0],7)",
1663 "&vpsrld (@x[$b0],@x[$b0],25)",
1664 "&vpor (@x[$b0],$t1,@x[$b0])",
1665 "&vbroadcasti128($t1,'(%r10)')", # .Lrot16(%rip)
1666 "&vpaddd ($xc_,$xc_,@x[$d1])",
1667 "&vpxor (@x[$b1],$xc_,@x[$b1])",
1668 "&vpslld ($t0,@x[$b1],7)",
1669 "&vpsrld (@x[$b1],@x[$b1],25)",
1670 "&vpor (@x[$b1],$t0,@x[$b1])",
1672 "&vmovdqa (\"`32*($c0-8)`(%rsp)\",$xc)", # reload pair of 'c's
1673 "&vmovdqa (\"`32*($c1-8)`(%rsp)\",$xc_)",
1674 "&vmovdqa ($xc,\"`32*($c2-8)`(%rsp)\")",
1675 "&vmovdqa ($xc_,\"`32*($c3-8)`(%rsp)\")",
1677 "&vpaddd (@x[$a2],@x[$a2],@x[$b2])", # Q3
1678 "&vpxor (@x[$d2],@x[$a2],@x[$d2])",
1679 "&vpshufb (@x[$d2],@x[$d2],$t1)",
1680 "&vpaddd (@x[$a3],@x[$a3],@x[$b3])", # Q4
1681 "&vpxor (@x[$d3],@x[$a3],@x[$d3])",
1682 "&vpshufb (@x[$d3],@x[$d3],$t1)",
1684 "&vpaddd ($xc,$xc,@x[$d2])",
1685 "&vpxor (@x[$b2],$xc,@x[$b2])",
1686 "&vpslld ($t0,@x[$b2],12)",
1687 "&vpsrld (@x[$b2],@x[$b2],20)",
1688 "&vpor (@x[$b2],$t0,@x[$b2])",
1689 "&vbroadcasti128($t0,'(%r11)')", # .Lrot24(%rip)
1690 "&vpaddd ($xc_,$xc_,@x[$d3])",
1691 "&vpxor (@x[$b3],$xc_,@x[$b3])",
1692 "&vpslld ($t1,@x[$b3],12)",
1693 "&vpsrld (@x[$b3],@x[$b3],20)",
1694 "&vpor (@x[$b3],$t1,@x[$b3])",
1696 "&vpaddd (@x[$a2],@x[$a2],@x[$b2])",
1697 "&vpxor (@x[$d2],@x[$a2],@x[$d2])",
1698 "&vpshufb (@x[$d2],@x[$d2],$t0)",
1699 "&vpaddd (@x[$a3],@x[$a3],@x[$b3])",
1700 "&vpxor (@x[$d3],@x[$a3],@x[$d3])",
1701 "&vpshufb (@x[$d3],@x[$d3],$t0)",
1703 "&vpaddd ($xc,$xc,@x[$d2])",
1704 "&vpxor (@x[$b2],$xc,@x[$b2])",
1705 "&vpslld ($t1,@x[$b2],7)",
1706 "&vpsrld (@x[$b2],@x[$b2],25)",
1707 "&vpor (@x[$b2],$t1,@x[$b2])",
1708 "&vbroadcasti128($t1,'(%r10)')", # .Lrot16(%rip)
1709 "&vpaddd ($xc_,$xc_,@x[$d3])",
1710 "&vpxor (@x[$b3],$xc_,@x[$b3])",
1711 "&vpslld ($t0,@x[$b3],7)",
1712 "&vpsrld (@x[$b3],@x[$b3],25)",
1713 "&vpor (@x[$b3],$t0,@x[$b3])"
1717 my $xframe = $win64 ? 0xb0 : 8;
1720 .type ChaCha20_8x,\@function,5
1725 sub \$0x280+$xframe,%rsp
1728 $code.=<<___ if ($win64);
1729 lea 0x290+0x30(%rsp),%r11
1730 movaps %xmm6,-0x30(%r11)
1731 movaps %xmm7,-0x20(%r11)
1732 movaps %xmm8,-0x10(%r11)
1733 movaps %xmm9,0x00(%r11)
1734 movaps %xmm10,0x10(%r11)
1735 movaps %xmm11,0x20(%r11)
1736 movaps %xmm12,0x30(%r11)
1737 movaps %xmm13,0x40(%r11)
1738 movaps %xmm14,0x50(%r11)
1739 movaps %xmm15,0x60(%r11)
1743 mov %r10,0x280(%rsp)
1745 ################ stack layout
1746 # +0x00 SIMD equivalent of @x[8-12]
1748 # +0x80 constant copy of key[0-2] smashed by lanes
1750 # +0x200 SIMD counters (with nonce smashed by lanes)
1754 vbroadcasti128 .Lsigma(%rip),$xa3 # key[0]
1755 vbroadcasti128 ($key),$xb3 # key[1]
1756 vbroadcasti128 16($key),$xt3 # key[2]
1757 vbroadcasti128 ($counter),$xd3 # key[3]
1758 lea 0x100(%rsp),%rcx # size optimization
1759 lea 0x200(%rsp),%rax # size optimization
1760 lea .Lrot16(%rip),%r10
1761 lea .Lrot24(%rip),%r11
1763 vpshufd \$0x00,$xa3,$xa0 # smash key by lanes...
1764 vpshufd \$0x55,$xa3,$xa1
1765 vmovdqa $xa0,0x80-0x100(%rcx) # ... and offload
1766 vpshufd \$0xaa,$xa3,$xa2
1767 vmovdqa $xa1,0xa0-0x100(%rcx)
1768 vpshufd \$0xff,$xa3,$xa3
1769 vmovdqa $xa2,0xc0-0x100(%rcx)
1770 vmovdqa $xa3,0xe0-0x100(%rcx)
1772 vpshufd \$0x00,$xb3,$xb0
1773 vpshufd \$0x55,$xb3,$xb1
1774 vmovdqa $xb0,0x100-0x100(%rcx)
1775 vpshufd \$0xaa,$xb3,$xb2
1776 vmovdqa $xb1,0x120-0x100(%rcx)
1777 vpshufd \$0xff,$xb3,$xb3
1778 vmovdqa $xb2,0x140-0x100(%rcx)
1779 vmovdqa $xb3,0x160-0x100(%rcx)
1781 vpshufd \$0x00,$xt3,$xt0 # "xc0"
1782 vpshufd \$0x55,$xt3,$xt1 # "xc1"
1783 vmovdqa $xt0,0x180-0x200(%rax)
1784 vpshufd \$0xaa,$xt3,$xt2 # "xc2"
1785 vmovdqa $xt1,0x1a0-0x200(%rax)
1786 vpshufd \$0xff,$xt3,$xt3 # "xc3"
1787 vmovdqa $xt2,0x1c0-0x200(%rax)
1788 vmovdqa $xt3,0x1e0-0x200(%rax)
1790 vpshufd \$0x00,$xd3,$xd0
1791 vpshufd \$0x55,$xd3,$xd1
1792 vpaddd .Lincy(%rip),$xd0,$xd0 # don't save counters yet
1793 vpshufd \$0xaa,$xd3,$xd2
1794 vmovdqa $xd1,0x220-0x200(%rax)
1795 vpshufd \$0xff,$xd3,$xd3
1796 vmovdqa $xd2,0x240-0x200(%rax)
1797 vmovdqa $xd3,0x260-0x200(%rax)
1803 vmovdqa 0x80-0x100(%rcx),$xa0 # re-load smashed key
1804 vmovdqa 0xa0-0x100(%rcx),$xa1
1805 vmovdqa 0xc0-0x100(%rcx),$xa2
1806 vmovdqa 0xe0-0x100(%rcx),$xa3
1807 vmovdqa 0x100-0x100(%rcx),$xb0
1808 vmovdqa 0x120-0x100(%rcx),$xb1
1809 vmovdqa 0x140-0x100(%rcx),$xb2
1810 vmovdqa 0x160-0x100(%rcx),$xb3
1811 vmovdqa 0x180-0x200(%rax),$xt0 # "xc0"
1812 vmovdqa 0x1a0-0x200(%rax),$xt1 # "xc1"
1813 vmovdqa 0x1c0-0x200(%rax),$xt2 # "xc2"
1814 vmovdqa 0x1e0-0x200(%rax),$xt3 # "xc3"
1815 vmovdqa 0x200-0x200(%rax),$xd0
1816 vmovdqa 0x220-0x200(%rax),$xd1
1817 vmovdqa 0x240-0x200(%rax),$xd2
1818 vmovdqa 0x260-0x200(%rax),$xd3
1819 vpaddd .Leight(%rip),$xd0,$xd0 # next SIMD counters
1822 vmovdqa $xt2,0x40(%rsp) # SIMD equivalent of "@x[10]"
1823 vmovdqa $xt3,0x60(%rsp) # SIMD equivalent of "@x[11]"
1824 vbroadcasti128 (%r10),$xt3
1825 vmovdqa $xd0,0x200-0x200(%rax) # save SIMD counters
1832 foreach (&AVX2_lane_ROUND(0, 4, 8,12)) { eval; }
1833 foreach (&AVX2_lane_ROUND(0, 5,10,15)) { eval; }
1838 lea 0x200(%rsp),%rax # size optimization
1839 vpaddd 0x80-0x100(%rcx),$xa0,$xa0 # accumulate key
1840 vpaddd 0xa0-0x100(%rcx),$xa1,$xa1
1841 vpaddd 0xc0-0x100(%rcx),$xa2,$xa2
1842 vpaddd 0xe0-0x100(%rcx),$xa3,$xa3
1844 vpunpckldq $xa1,$xa0,$xt2 # "de-interlace" data
1845 vpunpckldq $xa3,$xa2,$xt3
1846 vpunpckhdq $xa1,$xa0,$xa0
1847 vpunpckhdq $xa3,$xa2,$xa2
1848 vpunpcklqdq $xt3,$xt2,$xa1 # "a0"
1849 vpunpckhqdq $xt3,$xt2,$xt2 # "a1"
1850 vpunpcklqdq $xa2,$xa0,$xa3 # "a2"
1851 vpunpckhqdq $xa2,$xa0,$xa0 # "a3"
1853 ($xa0,$xa1,$xa2,$xa3,$xt2)=($xa1,$xt2,$xa3,$xa0,$xa2);
1855 vpaddd 0x100-0x100(%rcx),$xb0,$xb0
1856 vpaddd 0x120-0x100(%rcx),$xb1,$xb1
1857 vpaddd 0x140-0x100(%rcx),$xb2,$xb2
1858 vpaddd 0x160-0x100(%rcx),$xb3,$xb3
1860 vpunpckldq $xb1,$xb0,$xt2
1861 vpunpckldq $xb3,$xb2,$xt3
1862 vpunpckhdq $xb1,$xb0,$xb0
1863 vpunpckhdq $xb3,$xb2,$xb2
1864 vpunpcklqdq $xt3,$xt2,$xb1 # "b0"
1865 vpunpckhqdq $xt3,$xt2,$xt2 # "b1"
1866 vpunpcklqdq $xb2,$xb0,$xb3 # "b2"
1867 vpunpckhqdq $xb2,$xb0,$xb0 # "b3"
1869 ($xb0,$xb1,$xb2,$xb3,$xt2)=($xb1,$xt2,$xb3,$xb0,$xb2);
1871 vperm2i128 \$0x20,$xb0,$xa0,$xt3 # "de-interlace" further
1872 vperm2i128 \$0x31,$xb0,$xa0,$xb0
1873 vperm2i128 \$0x20,$xb1,$xa1,$xa0
1874 vperm2i128 \$0x31,$xb1,$xa1,$xb1
1875 vperm2i128 \$0x20,$xb2,$xa2,$xa1
1876 vperm2i128 \$0x31,$xb2,$xa2,$xb2
1877 vperm2i128 \$0x20,$xb3,$xa3,$xa2
1878 vperm2i128 \$0x31,$xb3,$xa3,$xb3
1880 ($xa0,$xa1,$xa2,$xa3,$xt3)=($xt3,$xa0,$xa1,$xa2,$xa3);
1881 my ($xc0,$xc1,$xc2,$xc3)=($xt0,$xt1,$xa0,$xa1);
1883 vmovdqa $xa0,0x00(%rsp) # offload $xaN
1884 vmovdqa $xa1,0x20(%rsp)
1885 vmovdqa 0x40(%rsp),$xc2 # $xa0
1886 vmovdqa 0x60(%rsp),$xc3 # $xa1
1888 vpaddd 0x180-0x200(%rax),$xc0,$xc0
1889 vpaddd 0x1a0-0x200(%rax),$xc1,$xc1
1890 vpaddd 0x1c0-0x200(%rax),$xc2,$xc2
1891 vpaddd 0x1e0-0x200(%rax),$xc3,$xc3
1893 vpunpckldq $xc1,$xc0,$xt2
1894 vpunpckldq $xc3,$xc2,$xt3
1895 vpunpckhdq $xc1,$xc0,$xc0
1896 vpunpckhdq $xc3,$xc2,$xc2
1897 vpunpcklqdq $xt3,$xt2,$xc1 # "c0"
1898 vpunpckhqdq $xt3,$xt2,$xt2 # "c1"
1899 vpunpcklqdq $xc2,$xc0,$xc3 # "c2"
1900 vpunpckhqdq $xc2,$xc0,$xc0 # "c3"
1902 ($xc0,$xc1,$xc2,$xc3,$xt2)=($xc1,$xt2,$xc3,$xc0,$xc2);
1904 vpaddd 0x200-0x200(%rax),$xd0,$xd0
1905 vpaddd 0x220-0x200(%rax),$xd1,$xd1
1906 vpaddd 0x240-0x200(%rax),$xd2,$xd2
1907 vpaddd 0x260-0x200(%rax),$xd3,$xd3
1909 vpunpckldq $xd1,$xd0,$xt2
1910 vpunpckldq $xd3,$xd2,$xt3
1911 vpunpckhdq $xd1,$xd0,$xd0
1912 vpunpckhdq $xd3,$xd2,$xd2
1913 vpunpcklqdq $xt3,$xt2,$xd1 # "d0"
1914 vpunpckhqdq $xt3,$xt2,$xt2 # "d1"
1915 vpunpcklqdq $xd2,$xd0,$xd3 # "d2"
1916 vpunpckhqdq $xd2,$xd0,$xd0 # "d3"
1918 ($xd0,$xd1,$xd2,$xd3,$xt2)=($xd1,$xt2,$xd3,$xd0,$xd2);
1920 vperm2i128 \$0x20,$xd0,$xc0,$xt3 # "de-interlace" further
1921 vperm2i128 \$0x31,$xd0,$xc0,$xd0
1922 vperm2i128 \$0x20,$xd1,$xc1,$xc0
1923 vperm2i128 \$0x31,$xd1,$xc1,$xd1
1924 vperm2i128 \$0x20,$xd2,$xc2,$xc1
1925 vperm2i128 \$0x31,$xd2,$xc2,$xd2
1926 vperm2i128 \$0x20,$xd3,$xc3,$xc2
1927 vperm2i128 \$0x31,$xd3,$xc3,$xd3
1929 ($xc0,$xc1,$xc2,$xc3,$xt3)=($xt3,$xc0,$xc1,$xc2,$xc3);
1930 ($xb0,$xb1,$xb2,$xb3,$xc0,$xc1,$xc2,$xc3)=
1931 ($xc0,$xc1,$xc2,$xc3,$xb0,$xb1,$xb2,$xb3);
1932 ($xa0,$xa1)=($xt2,$xt3);
1934 vmovdqa 0x00(%rsp),$xa0 # $xaN was offloaded, remember?
1935 vmovdqa 0x20(%rsp),$xa1
1940 vpxor 0x00($inp),$xa0,$xa0 # xor with input
1941 vpxor 0x20($inp),$xb0,$xb0
1942 vpxor 0x40($inp),$xc0,$xc0
1943 vpxor 0x60($inp),$xd0,$xd0
1944 lea 0x80($inp),$inp # size optimization
1945 vmovdqu $xa0,0x00($out)
1946 vmovdqu $xb0,0x20($out)
1947 vmovdqu $xc0,0x40($out)
1948 vmovdqu $xd0,0x60($out)
1949 lea 0x80($out),$out # size optimization
1951 vpxor 0x00($inp),$xa1,$xa1
1952 vpxor 0x20($inp),$xb1,$xb1
1953 vpxor 0x40($inp),$xc1,$xc1
1954 vpxor 0x60($inp),$xd1,$xd1
1955 lea 0x80($inp),$inp # size optimization
1956 vmovdqu $xa1,0x00($out)
1957 vmovdqu $xb1,0x20($out)
1958 vmovdqu $xc1,0x40($out)
1959 vmovdqu $xd1,0x60($out)
1960 lea 0x80($out),$out # size optimization
1962 vpxor 0x00($inp),$xa2,$xa2
1963 vpxor 0x20($inp),$xb2,$xb2
1964 vpxor 0x40($inp),$xc2,$xc2
1965 vpxor 0x60($inp),$xd2,$xd2
1966 lea 0x80($inp),$inp # size optimization
1967 vmovdqu $xa2,0x00($out)
1968 vmovdqu $xb2,0x20($out)
1969 vmovdqu $xc2,0x40($out)
1970 vmovdqu $xd2,0x60($out)
1971 lea 0x80($out),$out # size optimization
1973 vpxor 0x00($inp),$xa3,$xa3
1974 vpxor 0x20($inp),$xb3,$xb3
1975 vpxor 0x40($inp),$xc3,$xc3
1976 vpxor 0x60($inp),$xd3,$xd3
1977 lea 0x80($inp),$inp # size optimization
1978 vmovdqu $xa3,0x00($out)
1979 vmovdqu $xb3,0x20($out)
1980 vmovdqu $xc3,0x40($out)
1981 vmovdqu $xd3,0x60($out)
1982 lea 0x80($out),$out # size optimization
2006 vmovdqa $xa0,0x00(%rsp)
2007 vmovdqa $xb0,0x20(%rsp)
2012 vpxor 0x00($inp),$xa0,$xa0 # xor with input
2013 vpxor 0x20($inp),$xb0,$xb0
2014 vmovdqu $xa0,0x00($out)
2015 vmovdqu $xb0,0x20($out)
2018 lea 0x40($inp),$inp # inp+=64*1
2020 vmovdqa $xc0,0x00(%rsp)
2021 lea 0x40($out),$out # out+=64*1
2022 sub \$64,$len # len-=64*1
2023 vmovdqa $xd0,0x20(%rsp)
2028 vpxor 0x00($inp),$xa0,$xa0 # xor with input
2029 vpxor 0x20($inp),$xb0,$xb0
2030 vpxor 0x40($inp),$xc0,$xc0
2031 vpxor 0x60($inp),$xd0,$xd0
2032 vmovdqu $xa0,0x00($out)
2033 vmovdqu $xb0,0x20($out)
2034 vmovdqu $xc0,0x40($out)
2035 vmovdqu $xd0,0x60($out)
2038 lea 0x80($inp),$inp # inp+=64*2
2040 vmovdqa $xa1,0x00(%rsp)
2041 lea 0x80($out),$out # out+=64*2
2042 sub \$128,$len # len-=64*2
2043 vmovdqa $xb1,0x20(%rsp)
2048 vpxor 0x00($inp),$xa0,$xa0 # xor with input
2049 vpxor 0x20($inp),$xb0,$xb0
2050 vpxor 0x40($inp),$xc0,$xc0
2051 vpxor 0x60($inp),$xd0,$xd0
2052 vpxor 0x80($inp),$xa1,$xa1
2053 vpxor 0xa0($inp),$xb1,$xb1
2054 vmovdqu $xa0,0x00($out)
2055 vmovdqu $xb0,0x20($out)
2056 vmovdqu $xc0,0x40($out)
2057 vmovdqu $xd0,0x60($out)
2058 vmovdqu $xa1,0x80($out)
2059 vmovdqu $xb1,0xa0($out)
2062 lea 0xc0($inp),$inp # inp+=64*3
2064 vmovdqa $xc1,0x00(%rsp)
2065 lea 0xc0($out),$out # out+=64*3
2066 sub \$192,$len # len-=64*3
2067 vmovdqa $xd1,0x20(%rsp)
2072 vpxor 0x00($inp),$xa0,$xa0 # xor with input
2073 vpxor 0x20($inp),$xb0,$xb0
2074 vpxor 0x40($inp),$xc0,$xc0
2075 vpxor 0x60($inp),$xd0,$xd0
2076 vpxor 0x80($inp),$xa1,$xa1
2077 vpxor 0xa0($inp),$xb1,$xb1
2078 vpxor 0xc0($inp),$xc1,$xc1
2079 vpxor 0xe0($inp),$xd1,$xd1
2080 vmovdqu $xa0,0x00($out)
2081 vmovdqu $xb0,0x20($out)
2082 vmovdqu $xc0,0x40($out)
2083 vmovdqu $xd0,0x60($out)
2084 vmovdqu $xa1,0x80($out)
2085 vmovdqu $xb1,0xa0($out)
2086 vmovdqu $xc1,0xc0($out)
2087 vmovdqu $xd1,0xe0($out)
2090 lea 0x100($inp),$inp # inp+=64*4
2092 vmovdqa $xa2,0x00(%rsp)
2093 lea 0x100($out),$out # out+=64*4
2094 sub \$256,$len # len-=64*4
2095 vmovdqa $xb2,0x20(%rsp)
2100 vpxor 0x00($inp),$xa0,$xa0 # xor with input
2101 vpxor 0x20($inp),$xb0,$xb0
2102 vpxor 0x40($inp),$xc0,$xc0
2103 vpxor 0x60($inp),$xd0,$xd0
2104 vpxor 0x80($inp),$xa1,$xa1
2105 vpxor 0xa0($inp),$xb1,$xb1
2106 vpxor 0xc0($inp),$xc1,$xc1
2107 vpxor 0xe0($inp),$xd1,$xd1
2108 vpxor 0x100($inp),$xa2,$xa2
2109 vpxor 0x120($inp),$xb2,$xb2
2110 vmovdqu $xa0,0x00($out)
2111 vmovdqu $xb0,0x20($out)
2112 vmovdqu $xc0,0x40($out)
2113 vmovdqu $xd0,0x60($out)
2114 vmovdqu $xa1,0x80($out)
2115 vmovdqu $xb1,0xa0($out)
2116 vmovdqu $xc1,0xc0($out)
2117 vmovdqu $xd1,0xe0($out)
2118 vmovdqu $xa2,0x100($out)
2119 vmovdqu $xb2,0x120($out)
2122 lea 0x140($inp),$inp # inp+=64*5
2124 vmovdqa $xc2,0x00(%rsp)
2125 lea 0x140($out),$out # out+=64*5
2126 sub \$320,$len # len-=64*5
2127 vmovdqa $xd2,0x20(%rsp)
2132 vpxor 0x00($inp),$xa0,$xa0 # xor with input
2133 vpxor 0x20($inp),$xb0,$xb0
2134 vpxor 0x40($inp),$xc0,$xc0
2135 vpxor 0x60($inp),$xd0,$xd0
2136 vpxor 0x80($inp),$xa1,$xa1
2137 vpxor 0xa0($inp),$xb1,$xb1
2138 vpxor 0xc0($inp),$xc1,$xc1
2139 vpxor 0xe0($inp),$xd1,$xd1
2140 vpxor 0x100($inp),$xa2,$xa2
2141 vpxor 0x120($inp),$xb2,$xb2
2142 vpxor 0x140($inp),$xc2,$xc2
2143 vpxor 0x160($inp),$xd2,$xd2
2144 vmovdqu $xa0,0x00($out)
2145 vmovdqu $xb0,0x20($out)
2146 vmovdqu $xc0,0x40($out)
2147 vmovdqu $xd0,0x60($out)
2148 vmovdqu $xa1,0x80($out)
2149 vmovdqu $xb1,0xa0($out)
2150 vmovdqu $xc1,0xc0($out)
2151 vmovdqu $xd1,0xe0($out)
2152 vmovdqu $xa2,0x100($out)
2153 vmovdqu $xb2,0x120($out)
2154 vmovdqu $xc2,0x140($out)
2155 vmovdqu $xd2,0x160($out)
2158 lea 0x180($inp),$inp # inp+=64*6
2160 vmovdqa $xa3,0x00(%rsp)
2161 lea 0x180($out),$out # out+=64*6
2162 sub \$384,$len # len-=64*6
2163 vmovdqa $xb3,0x20(%rsp)
2168 vpxor 0x00($inp),$xa0,$xa0 # xor with input
2169 vpxor 0x20($inp),$xb0,$xb0
2170 vpxor 0x40($inp),$xc0,$xc0
2171 vpxor 0x60($inp),$xd0,$xd0
2172 vpxor 0x80($inp),$xa1,$xa1
2173 vpxor 0xa0($inp),$xb1,$xb1
2174 vpxor 0xc0($inp),$xc1,$xc1
2175 vpxor 0xe0($inp),$xd1,$xd1
2176 vpxor 0x100($inp),$xa2,$xa2
2177 vpxor 0x120($inp),$xb2,$xb2
2178 vpxor 0x140($inp),$xc2,$xc2
2179 vpxor 0x160($inp),$xd2,$xd2
2180 vpxor 0x180($inp),$xa3,$xa3
2181 vpxor 0x1a0($inp),$xb3,$xb3
2182 vmovdqu $xa0,0x00($out)
2183 vmovdqu $xb0,0x20($out)
2184 vmovdqu $xc0,0x40($out)
2185 vmovdqu $xd0,0x60($out)
2186 vmovdqu $xa1,0x80($out)
2187 vmovdqu $xb1,0xa0($out)
2188 vmovdqu $xc1,0xc0($out)
2189 vmovdqu $xd1,0xe0($out)
2190 vmovdqu $xa2,0x100($out)
2191 vmovdqu $xb2,0x120($out)
2192 vmovdqu $xc2,0x140($out)
2193 vmovdqu $xd2,0x160($out)
2194 vmovdqu $xa3,0x180($out)
2195 vmovdqu $xb3,0x1a0($out)
2198 lea 0x1c0($inp),$inp # inp+=64*7
2200 vmovdqa $xc3,0x00(%rsp)
2201 lea 0x1c0($out),$out # out+=64*7
2202 sub \$448,$len # len-=64*7
2203 vmovdqa $xd3,0x20(%rsp)
2206 movzb ($inp,%r10),%eax
2207 movzb (%rsp,%r10),%ecx
2210 mov %al,-1($out,%r10)
2217 $code.=<<___ if ($win64);
2218 lea 0x290+0x30(%rsp),%r11
2219 movaps -0x30(%r11),%xmm6
2220 movaps -0x20(%r11),%xmm7
2221 movaps -0x10(%r11),%xmm8
2222 movaps 0x00(%r11),%xmm9
2223 movaps 0x10(%r11),%xmm10
2224 movaps 0x20(%r11),%xmm11
2225 movaps 0x30(%r11),%xmm12
2226 movaps 0x40(%r11),%xmm13
2227 movaps 0x50(%r11),%xmm14
2228 movaps 0x60(%r11),%xmm15
2231 mov 0x280(%rsp),%rsp
2233 .size ChaCha20_8x,.-ChaCha20_8x
2237 foreach (split("\n",$code)) {
2238 s/\`([^\`]*)\`/eval $1/geo;