2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
21 # Performance in cycles per byte out of large buffer.
23 # IALU/gcc-4.9 3xNEON+1xIALU 6xNEON+2xIALU
25 # Apple A7 5.50/+49% 3.33 1.70
26 # Cortex-A53 8.40/+80% 4.72 4.72(*)
27 # Cortex-A57 8.06/+43% 4.90 4.43(**)
28 # Denver 4.50/+82% 2.63 2.67(*)
29 # X-Gene 9.50/+46% 8.82 8.89(*)
30 # Mongoose 8.00/+44% 3.64 3.25
32 # (*) it's expected that doubling interleave factor doesn't help
33 # all processors, only those with higher NEON latency and
34 # higher instruction issue rate;
35 # (**) expected improvement was actually higher;
40 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
41 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
42 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
43 die "can't locate arm-xlate.pl";
45 open OUT,"| \"$^X\" $xlate $flavour $output";
48 sub AUTOLOAD() # thunk [simplified] x86-style perlasm
49 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
51 $arg = "#$arg" if ($arg*1 eq $arg);
52 $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
55 my ($out,$inp,$len,$key,$ctr) = map("x$_",(0..4));
57 my @x=map("x$_",(5..17,19..21));
58 my @d=map("x$_",(22..28,30));
61 my ($a0,$b0,$c0,$d0)=@_;
62 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
63 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
64 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
67 "&add_32 (@x[$a0],@x[$a0],@x[$b0])",
68 "&add_32 (@x[$a1],@x[$a1],@x[$b1])",
69 "&add_32 (@x[$a2],@x[$a2],@x[$b2])",
70 "&add_32 (@x[$a3],@x[$a3],@x[$b3])",
71 "&eor_32 (@x[$d0],@x[$d0],@x[$a0])",
72 "&eor_32 (@x[$d1],@x[$d1],@x[$a1])",
73 "&eor_32 (@x[$d2],@x[$d2],@x[$a2])",
74 "&eor_32 (@x[$d3],@x[$d3],@x[$a3])",
75 "&ror_32 (@x[$d0],@x[$d0],16)",
76 "&ror_32 (@x[$d1],@x[$d1],16)",
77 "&ror_32 (@x[$d2],@x[$d2],16)",
78 "&ror_32 (@x[$d3],@x[$d3],16)",
80 "&add_32 (@x[$c0],@x[$c0],@x[$d0])",
81 "&add_32 (@x[$c1],@x[$c1],@x[$d1])",
82 "&add_32 (@x[$c2],@x[$c2],@x[$d2])",
83 "&add_32 (@x[$c3],@x[$c3],@x[$d3])",
84 "&eor_32 (@x[$b0],@x[$b0],@x[$c0])",
85 "&eor_32 (@x[$b1],@x[$b1],@x[$c1])",
86 "&eor_32 (@x[$b2],@x[$b2],@x[$c2])",
87 "&eor_32 (@x[$b3],@x[$b3],@x[$c3])",
88 "&ror_32 (@x[$b0],@x[$b0],20)",
89 "&ror_32 (@x[$b1],@x[$b1],20)",
90 "&ror_32 (@x[$b2],@x[$b2],20)",
91 "&ror_32 (@x[$b3],@x[$b3],20)",
93 "&add_32 (@x[$a0],@x[$a0],@x[$b0])",
94 "&add_32 (@x[$a1],@x[$a1],@x[$b1])",
95 "&add_32 (@x[$a2],@x[$a2],@x[$b2])",
96 "&add_32 (@x[$a3],@x[$a3],@x[$b3])",
97 "&eor_32 (@x[$d0],@x[$d0],@x[$a0])",
98 "&eor_32 (@x[$d1],@x[$d1],@x[$a1])",
99 "&eor_32 (@x[$d2],@x[$d2],@x[$a2])",
100 "&eor_32 (@x[$d3],@x[$d3],@x[$a3])",
101 "&ror_32 (@x[$d0],@x[$d0],24)",
102 "&ror_32 (@x[$d1],@x[$d1],24)",
103 "&ror_32 (@x[$d2],@x[$d2],24)",
104 "&ror_32 (@x[$d3],@x[$d3],24)",
106 "&add_32 (@x[$c0],@x[$c0],@x[$d0])",
107 "&add_32 (@x[$c1],@x[$c1],@x[$d1])",
108 "&add_32 (@x[$c2],@x[$c2],@x[$d2])",
109 "&add_32 (@x[$c3],@x[$c3],@x[$d3])",
110 "&eor_32 (@x[$b0],@x[$b0],@x[$c0])",
111 "&eor_32 (@x[$b1],@x[$b1],@x[$c1])",
112 "&eor_32 (@x[$b2],@x[$b2],@x[$c2])",
113 "&eor_32 (@x[$b3],@x[$b3],@x[$c3])",
114 "&ror_32 (@x[$b0],@x[$b0],25)",
115 "&ror_32 (@x[$b1],@x[$b1],25)",
116 "&ror_32 (@x[$b2],@x[$b2],25)",
117 "&ror_32 (@x[$b3],@x[$b3],25)"
122 #include "arm_arch.h"
126 .extern OPENSSL_armcap_P
130 .quad 0x3320646e61707865,0x6b20657479622d32 // endian-neutral
135 .long OPENSSL_armcap_P-.
137 .quad OPENSSL_armcap_P-.
139 .asciz "ChaCha20 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
141 .globl ChaCha20_ctr32
142 .type ChaCha20_ctr32,%function
146 adr @x[0],.LOPENSSL_armcap_P
154 ldr w17,[@x[1],@x[0]]
159 stp x29,x30,[sp,#-96]!
170 ldp @d[0],@d[1],[@x[0]] // load sigma
171 ldp @d[2],@d[3],[$key] // load key
172 ldp @d[4],@d[5],[$key,#16]
173 ldp @d[6],@d[7],[$ctr] // load counter
184 mov.32 @x[0],@d[0] // unpack key block
206 foreach (&ROUND(0, 4, 8,12)) { eval; }
207 foreach (&ROUND(0, 5,10,15)) { eval; }
211 add.32 @x[0],@x[0],@d[0] // accumulate key block
212 add @x[1],@x[1],@d[0],lsr#32
213 add.32 @x[2],@x[2],@d[1]
214 add @x[3],@x[3],@d[1],lsr#32
215 add.32 @x[4],@x[4],@d[2]
216 add @x[5],@x[5],@d[2],lsr#32
217 add.32 @x[6],@x[6],@d[3]
218 add @x[7],@x[7],@d[3],lsr#32
219 add.32 @x[8],@x[8],@d[4]
220 add @x[9],@x[9],@d[4],lsr#32
221 add.32 @x[10],@x[10],@d[5]
222 add @x[11],@x[11],@d[5],lsr#32
223 add.32 @x[12],@x[12],@d[6]
224 add @x[13],@x[13],@d[6],lsr#32
225 add.32 @x[14],@x[14],@d[7]
226 add @x[15],@x[15],@d[7],lsr#32
230 add @x[0],@x[0],@x[1],lsl#32 // pack
231 add @x[2],@x[2],@x[3],lsl#32
232 ldp @x[1],@x[3],[$inp,#0] // load input
233 add @x[4],@x[4],@x[5],lsl#32
234 add @x[6],@x[6],@x[7],lsl#32
235 ldp @x[5],@x[7],[$inp,#16]
236 add @x[8],@x[8],@x[9],lsl#32
237 add @x[10],@x[10],@x[11],lsl#32
238 ldp @x[9],@x[11],[$inp,#32]
239 add @x[12],@x[12],@x[13],lsl#32
240 add @x[14],@x[14],@x[15],lsl#32
241 ldp @x[13],@x[15],[$inp,#48]
253 eor @x[0],@x[0],@x[1]
254 eor @x[2],@x[2],@x[3]
255 eor @x[4],@x[4],@x[5]
256 eor @x[6],@x[6],@x[7]
257 eor @x[8],@x[8],@x[9]
258 eor @x[10],@x[10],@x[11]
259 eor @x[12],@x[12],@x[13]
260 eor @x[14],@x[14],@x[15]
262 stp @x[0],@x[2],[$out,#0] // store output
263 add @d[6],@d[6],#1 // increment counter
264 stp @x[4],@x[6],[$out,#16]
265 stp @x[8],@x[10],[$out,#32]
266 stp @x[12],@x[14],[$out,#48]
271 ldp x19,x20,[x29,#16]
273 ldp x21,x22,[x29,#32]
274 ldp x23,x24,[x29,#48]
275 ldp x25,x26,[x29,#64]
276 ldp x27,x28,[x29,#80]
291 add @x[0],@x[0],@x[1],lsl#32 // pack
292 add @x[2],@x[2],@x[3],lsl#32
293 add @x[4],@x[4],@x[5],lsl#32
294 add @x[6],@x[6],@x[7],lsl#32
295 add @x[8],@x[8],@x[9],lsl#32
296 add @x[10],@x[10],@x[11],lsl#32
297 add @x[12],@x[12],@x[13],lsl#32
298 add @x[14],@x[14],@x[15],lsl#32
309 stp @x[0],@x[2],[sp,#0]
310 stp @x[4],@x[6],[sp,#16]
311 stp @x[8],@x[10],[sp,#32]
312 stp @x[12],@x[14],[sp,#48]
327 ldp x19,x20,[x29,#16]
329 ldp x21,x22,[x29,#32]
330 ldp x23,x24,[x29,#48]
331 ldp x25,x26,[x29,#64]
332 ldp x27,x28,[x29,#80]
335 .size ChaCha20_ctr32,.-ChaCha20_ctr32
339 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2,$T3) =
340 map("v$_.4s",(0..7,16..23));
341 my (@K)=map("v$_.4s",(24..30));
346 my ($a,$b,$c,$d,$t)=@_;
349 "&add ('$a','$a','$b')",
350 "&eor ('$d','$d','$a')",
351 "&rev32_16 ('$d','$d')", # vrot ($d,16)
353 "&add ('$c','$c','$d')",
354 "&eor ('$t','$b','$c')",
355 "&ushr ('$b','$t',20)",
356 "&sli ('$b','$t',12)",
358 "&add ('$a','$a','$b')",
359 "&eor ('$t','$d','$a')",
360 "&ushr ('$d','$t',24)",
361 "&sli ('$d','$t',8)",
363 "&add ('$c','$c','$d')",
364 "&eor ('$t','$b','$c')",
365 "&ushr ('$b','$t',25)",
366 "&sli ('$b','$t',7)",
368 "&ext ('$c','$c','$c',8)",
369 "&ext ('$d','$d','$d',$odd?4:12)",
370 "&ext ('$b','$b','$b',$odd?12:4)"
376 .type ChaCha20_neon,%function
379 stp x29,x30,[sp,#-96]!
389 b.hs .L512_or_more_neon
393 ldp @d[0],@d[1],[@x[0]] // load sigma
394 ld1 {@K[0]},[@x[0]],#16
395 ldp @d[2],@d[3],[$key] // load key
396 ldp @d[4],@d[5],[$key,#16]
397 ld1 {@K[1],@K[2]},[$key]
398 ldp @d[6],@d[7],[$ctr] // load counter
410 add @K[3],@K[3],$ONE // += 1
413 shl $ONE,$ONE,#2 // 1 -> 4
416 mov.32 @x[0],@d[0] // unpack key block
450 my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
451 my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
452 my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
453 my @thread3=&ROUND(0,4,8,12);
456 eval; eval(shift(@thread3));
457 eval(shift(@thread1)); eval(shift(@thread3));
458 eval(shift(@thread2)); eval(shift(@thread3));
461 @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
462 @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
463 @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
464 @thread3=&ROUND(0,5,10,15);
467 eval; eval(shift(@thread3));
468 eval(shift(@thread1)); eval(shift(@thread3));
469 eval(shift(@thread2)); eval(shift(@thread3));
474 add.32 @x[0],@x[0],@d[0] // accumulate key block
476 add @x[1],@x[1],@d[0],lsr#32
478 add.32 @x[2],@x[2],@d[1]
480 add @x[3],@x[3],@d[1],lsr#32
482 add.32 @x[4],@x[4],@d[2]
484 add @x[5],@x[5],@d[2],lsr#32
486 add.32 @x[6],@x[6],@d[3]
488 add @x[7],@x[7],@d[3],lsr#32
489 add.32 @x[8],@x[8],@d[4]
491 add @x[9],@x[9],@d[4],lsr#32
492 add.32 @x[10],@x[10],@d[5]
494 add @x[11],@x[11],@d[5],lsr#32
495 add.32 @x[12],@x[12],@d[6]
497 add @x[13],@x[13],@d[6],lsr#32
498 add.32 @x[14],@x[14],@d[7]
500 add @x[15],@x[15],@d[7],lsr#32
505 add @x[0],@x[0],@x[1],lsl#32 // pack
506 add @x[2],@x[2],@x[3],lsl#32
507 ldp @x[1],@x[3],[$inp,#0] // load input
508 add @x[4],@x[4],@x[5],lsl#32
509 add @x[6],@x[6],@x[7],lsl#32
510 ldp @x[5],@x[7],[$inp,#16]
511 add @x[8],@x[8],@x[9],lsl#32
512 add @x[10],@x[10],@x[11],lsl#32
513 ldp @x[9],@x[11],[$inp,#32]
514 add @x[12],@x[12],@x[13],lsl#32
515 add @x[14],@x[14],@x[15],lsl#32
516 ldp @x[13],@x[15],[$inp,#48]
528 ld1.8 {$T0-$T3},[$inp],#64
529 eor @x[0],@x[0],@x[1]
530 eor @x[2],@x[2],@x[3]
531 eor @x[4],@x[4],@x[5]
532 eor @x[6],@x[6],@x[7]
533 eor @x[8],@x[8],@x[9]
535 eor @x[10],@x[10],@x[11]
537 eor @x[12],@x[12],@x[13]
539 eor @x[14],@x[14],@x[15]
541 ld1.8 {$T0-$T3},[$inp],#64
543 stp @x[0],@x[2],[$out,#0] // store output
544 add @d[6],@d[6],#4 // increment counter
545 stp @x[4],@x[6],[$out,#16]
546 add @K[3],@K[3],$ONE // += 4
547 stp @x[8],@x[10],[$out,#32]
549 stp @x[12],@x[14],[$out,#48]
553 st1.8 {$A0-$D0},[$out],#64
554 ld1.8 {$A0-$D0},[$inp],#64
560 st1.8 {$A1-$D1},[$out],#64
566 st1.8 {$A2-$D2},[$out],#64
568 b.hi .Loop_outer_neon
570 ldp x19,x20,[x29,#16]
572 ldp x21,x22,[x29,#32]
573 ldp x23,x24,[x29,#48]
574 ldp x25,x26,[x29,#64]
575 ldp x27,x28,[x29,#80]
584 add @x[0],@x[0],@x[1],lsl#32 // pack
585 add @x[2],@x[2],@x[3],lsl#32
586 ldp @x[1],@x[3],[$inp,#0] // load input
587 add @x[4],@x[4],@x[5],lsl#32
588 add @x[6],@x[6],@x[7],lsl#32
589 ldp @x[5],@x[7],[$inp,#16]
590 add @x[8],@x[8],@x[9],lsl#32
591 add @x[10],@x[10],@x[11],lsl#32
592 ldp @x[9],@x[11],[$inp,#32]
593 add @x[12],@x[12],@x[13],lsl#32
594 add @x[14],@x[14],@x[15],lsl#32
595 ldp @x[13],@x[15],[$inp,#48]
607 eor @x[0],@x[0],@x[1]
608 eor @x[2],@x[2],@x[3]
609 eor @x[4],@x[4],@x[5]
610 eor @x[6],@x[6],@x[7]
611 eor @x[8],@x[8],@x[9]
612 eor @x[10],@x[10],@x[11]
613 eor @x[12],@x[12],@x[13]
614 eor @x[14],@x[14],@x[15]
616 stp @x[0],@x[2],[$out,#0] // store output
617 add @d[6],@d[6],#4 // increment counter
618 stp @x[4],@x[6],[$out,#16]
619 stp @x[8],@x[10],[$out,#32]
620 stp @x[12],@x[14],[$out,#48]
627 ld1.8 {$T0-$T3},[$inp],#64
632 st1.8 {$A0-$D0},[$out],#64
638 ld1.8 {$T0-$T3},[$inp],#64
643 st1.8 {$A1-$D1},[$out],#64
671 cbnz $len,.Loop_tail_neon
679 ldp x19,x20,[x29,#16]
681 ldp x21,x22,[x29,#32]
682 ldp x23,x24,[x29,#48]
683 ldp x25,x26,[x29,#64]
684 ldp x27,x28,[x29,#80]
687 .size ChaCha20_neon,.-ChaCha20_neon
690 my ($T0,$T1,$T2,$T3,$T4,$T5)=@K;
691 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,
692 $A3,$B3,$C3,$D3,$A4,$B4,$C4,$D4,$A5,$B5,$C5,$D5) = map("v$_.4s",(0..23));
695 .type ChaCha20_512_neon,%function
698 stp x29,x30,[sp,#-96]!
711 ldp @d[0],@d[1],[@x[0]] // load sigma
712 ld1 {@K[0]},[@x[0]],#16
713 ldp @d[2],@d[3],[$key] // load key
714 ldp @d[4],@d[5],[$key,#16]
715 ld1 {@K[1],@K[2]},[$key]
716 ldp @d[6],@d[7],[$ctr] // load counter
728 add @K[3],@K[3],$ONE // += 1
729 stp @K[0],@K[1],[sp,#0] // off-load key block, invariant part
730 add @K[3],@K[3],$ONE // not typo
735 shl $ONE,$ONE,#2 // 1 -> 4
737 stp d8,d9,[sp,#128+0] // meet ABI requirements
738 stp d10,d11,[sp,#128+16]
739 stp d12,d13,[sp,#128+32]
740 stp d14,d15,[sp,#128+48]
742 sub $len,$len,#512 // not typo
744 .Loop_outer_512_neon:
752 mov.32 @x[0],@d[0] // unpack key block
775 add $D4,$D0,$ONE // +4
777 add $D5,$D1,$ONE // +4
784 stp @K[3],@K[4],[sp,#48] // off-load key block, variable part
793 my @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
794 my @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
795 my @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
796 my @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
797 my @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
798 my @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
799 my @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
800 my $diff = ($#thread0+1)*6 - $#thread67 - 1;
804 eval; eval(shift(@thread67));
805 eval(shift(@thread1)); eval(shift(@thread67));
806 eval(shift(@thread2)); eval(shift(@thread67));
807 eval(shift(@thread3)); eval(shift(@thread67));
808 eval(shift(@thread4)); eval(shift(@thread67));
809 eval(shift(@thread5)); eval(shift(@thread67));
812 @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
813 @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
814 @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
815 @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
816 @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
817 @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
818 @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
821 eval; eval(shift(@thread67));
822 eval(shift(@thread1)); eval(shift(@thread67));
823 eval(shift(@thread2)); eval(shift(@thread67));
824 eval(shift(@thread3)); eval(shift(@thread67));
825 eval(shift(@thread4)); eval(shift(@thread67));
826 eval(shift(@thread5)); eval(shift(@thread67));
829 cbnz $ctr,.Loop_upper_neon
831 add.32 @x[0],@x[0],@d[0] // accumulate key block
832 add @x[1],@x[1],@d[0],lsr#32
833 add.32 @x[2],@x[2],@d[1]
834 add @x[3],@x[3],@d[1],lsr#32
835 add.32 @x[4],@x[4],@d[2]
836 add @x[5],@x[5],@d[2],lsr#32
837 add.32 @x[6],@x[6],@d[3]
838 add @x[7],@x[7],@d[3],lsr#32
839 add.32 @x[8],@x[8],@d[4]
840 add @x[9],@x[9],@d[4],lsr#32
841 add.32 @x[10],@x[10],@d[5]
842 add @x[11],@x[11],@d[5],lsr#32
843 add.32 @x[12],@x[12],@d[6]
844 add @x[13],@x[13],@d[6],lsr#32
845 add.32 @x[14],@x[14],@d[7]
846 add @x[15],@x[15],@d[7],lsr#32
848 add @x[0],@x[0],@x[1],lsl#32 // pack
849 add @x[2],@x[2],@x[3],lsl#32
850 ldp @x[1],@x[3],[$inp,#0] // load input
851 add @x[4],@x[4],@x[5],lsl#32
852 add @x[6],@x[6],@x[7],lsl#32
853 ldp @x[5],@x[7],[$inp,#16]
854 add @x[8],@x[8],@x[9],lsl#32
855 add @x[10],@x[10],@x[11],lsl#32
856 ldp @x[9],@x[11],[$inp,#32]
857 add @x[12],@x[12],@x[13],lsl#32
858 add @x[14],@x[14],@x[15],lsl#32
859 ldp @x[13],@x[15],[$inp,#48]
871 eor @x[0],@x[0],@x[1]
872 eor @x[2],@x[2],@x[3]
873 eor @x[4],@x[4],@x[5]
874 eor @x[6],@x[6],@x[7]
875 eor @x[8],@x[8],@x[9]
876 eor @x[10],@x[10],@x[11]
877 eor @x[12],@x[12],@x[13]
878 eor @x[14],@x[14],@x[15]
880 stp @x[0],@x[2],[$out,#0] // store output
881 add @d[6],@d[6],#1 // increment counter
882 mov.32 @x[0],@d[0] // unpack key block
884 stp @x[4],@x[6],[$out,#16]
887 stp @x[8],@x[10],[$out,#32]
890 stp @x[12],@x[14],[$out,#48]
907 @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,0);
908 @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,0);
909 @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,0);
910 @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,0);
911 @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,0);
912 @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,0);
913 @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
916 eval; eval(shift(@thread67));
917 eval(shift(@thread1)); eval(shift(@thread67));
918 eval(shift(@thread2)); eval(shift(@thread67));
919 eval(shift(@thread3)); eval(shift(@thread67));
920 eval(shift(@thread4)); eval(shift(@thread67));
921 eval(shift(@thread5)); eval(shift(@thread67));
924 @thread0=&NEONROUND($A0,$B0,$C0,$D0,$T0,1);
925 @thread1=&NEONROUND($A1,$B1,$C1,$D1,$T1,1);
926 @thread2=&NEONROUND($A2,$B2,$C2,$D2,$T2,1);
927 @thread3=&NEONROUND($A3,$B3,$C3,$D3,$T3,1);
928 @thread4=&NEONROUND($A4,$B4,$C4,$D4,$T4,1);
929 @thread5=&NEONROUND($A5,$B5,$C5,$D5,$T5,1);
930 @thread67=(&ROUND(0,4,8,12),&ROUND(0,5,10,15));
933 eval; eval(shift(@thread67));
934 eval(shift(@thread1)); eval(shift(@thread67));
935 eval(shift(@thread2)); eval(shift(@thread67));
936 eval(shift(@thread3)); eval(shift(@thread67));
937 eval(shift(@thread4)); eval(shift(@thread67));
938 eval(shift(@thread5)); eval(shift(@thread67));
941 cbnz $ctr,.Loop_lower_neon
943 add.32 @x[0],@x[0],@d[0] // accumulate key block
944 ldp @K[0],@K[1],[sp,#0]
945 add @x[1],@x[1],@d[0],lsr#32
946 ldp @K[2],@K[3],[sp,#32]
947 add.32 @x[2],@x[2],@d[1]
948 ldp @K[4],@K[5],[sp,#64]
949 add @x[3],@x[3],@d[1],lsr#32
951 add.32 @x[4],@x[4],@d[2]
953 add @x[5],@x[5],@d[2],lsr#32
955 add.32 @x[6],@x[6],@d[3]
957 add @x[7],@x[7],@d[3],lsr#32
959 add.32 @x[8],@x[8],@d[4]
961 add @x[9],@x[9],@d[4],lsr#32
963 add.32 @x[10],@x[10],@d[5]
965 add @x[11],@x[11],@d[5],lsr#32
967 add.32 @x[12],@x[12],@d[6]
969 add @x[13],@x[13],@d[6],lsr#32
971 add.32 @x[14],@x[14],@d[7]
973 add @x[15],@x[15],@d[7],lsr#32
974 add $D4,$D4,$ONE // +4
975 add @x[0],@x[0],@x[1],lsl#32 // pack
976 add $D5,$D5,$ONE // +4
977 add @x[2],@x[2],@x[3],lsl#32
979 ldp @x[1],@x[3],[$inp,#0] // load input
981 add @x[4],@x[4],@x[5],lsl#32
983 add @x[6],@x[6],@x[7],lsl#32
985 ldp @x[5],@x[7],[$inp,#16]
987 add @x[8],@x[8],@x[9],lsl#32
989 add @x[10],@x[10],@x[11],lsl#32
991 ldp @x[9],@x[11],[$inp,#32]
993 add @x[12],@x[12],@x[13],lsl#32
995 add @x[14],@x[14],@x[15],lsl#32
997 ldp @x[13],@x[15],[$inp,#48]
1012 ld1.8 {$T0-$T3},[$inp],#64
1013 eor @x[0],@x[0],@x[1]
1014 eor @x[2],@x[2],@x[3]
1015 eor @x[4],@x[4],@x[5]
1016 eor @x[6],@x[6],@x[7]
1017 eor @x[8],@x[8],@x[9]
1019 eor @x[10],@x[10],@x[11]
1021 eor @x[12],@x[12],@x[13]
1023 eor @x[14],@x[14],@x[15]
1025 ld1.8 {$T0-$T3},[$inp],#64
1027 stp @x[0],@x[2],[$out,#0] // store output
1028 add @d[6],@d[6],#7 // increment counter
1029 stp @x[4],@x[6],[$out,#16]
1030 stp @x[8],@x[10],[$out,#32]
1031 stp @x[12],@x[14],[$out,#48]
1033 st1.8 {$A0-$D0},[$out],#64
1035 ld1.8 {$A0-$D0},[$inp],#64
1040 st1.8 {$A1-$D1},[$out],#64
1042 ld1.8 {$A1-$D1},[$inp],#64
1044 ldp @K[0],@K[1],[sp,#0]
1046 ldp @K[2],@K[3],[sp,#32]
1049 st1.8 {$A2-$D2},[$out],#64
1051 ld1.8 {$A2-$D2},[$inp],#64
1056 st1.8 {$A3-$D3},[$out],#64
1058 ld1.8 {$A3-$D3},[$inp],#64
1063 st1.8 {$A4-$D4},[$out],#64
1065 shl $A0,$ONE,#1 // 4 -> 8
1070 st1.8 {$A5-$D5},[$out],#64
1072 add @K[3],@K[3],$A0 // += 8
1077 b.hs .Loop_outer_512_neon
1080 ushr $A0,$ONE,#2 // 4 -> 1
1082 ldp d8,d9,[sp,#128+0] // meet ABI requirements
1083 ldp d10,d11,[sp,#128+16]
1084 ldp d12,d13,[sp,#128+32]
1085 ldp d14,d15,[sp,#128+48]
1087 stp @K[0],$ONE,[sp,#0] // wipe off-load area
1088 stp @K[0],$ONE,[sp,#32]
1089 stp @K[0],$ONE,[sp,#64]
1091 b.eq .Ldone_512_neon
1094 sub @K[3],@K[3],$A0 // -= 1
1098 b.hs .Loop_outer_neon
1100 eor @K[1],@K[1],@K[1]
1101 eor @K[2],@K[2],@K[2]
1102 eor @K[3],@K[3],@K[3]
1103 eor @K[4],@K[4],@K[4]
1104 eor @K[5],@K[5],@K[5]
1105 eor @K[6],@K[6],@K[6]
1109 ldp x19,x20,[x29,#16]
1111 ldp x21,x22,[x29,#32]
1112 ldp x23,x24,[x29,#48]
1113 ldp x25,x26,[x29,#64]
1114 ldp x27,x28,[x29,#80]
1115 ldp x29,x30,[sp],#96
1117 .size ChaCha20_512_neon,.-ChaCha20_512_neon
1122 foreach (split("\n",$code)) {
1123 s/\`([^\`]*)\`/eval $1/geo;
1125 (s/\b([a-z]+)\.32\b/$1/ and (s/x([0-9]+)/w$1/g or 1)) or
1126 (m/\b(eor|ext|mov)\b/ and (s/\.4s/\.16b/g or 1)) or
1127 (s/\b((?:ld|st)1)\.8\b/$1/ and (s/\.4s/\.16b/g or 1)) or
1128 (m/\b(ld|st)[rp]\b/ and (s/v([0-9]+)\.4s/q$1/g or 1)) or
1129 (s/\brev32\.16\b/rev32/ and (s/\.4s/\.8h/g or 1));
1131 #s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;
1135 close STDOUT; # flush