2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
21 # Performance in cycles per byte out of large buffer.
23 # IALU/gcc-4.4 1xNEON 3xNEON+1xIALU
25 # Cortex-A5 19.3(*)/+95% 21.8 14.1
26 # Cortex-A8 10.5(*)/+160% 13.9 6.35
27 # Cortex-A9 12.9(**)/+110% 14.3 6.50
28 # Cortex-A15 11.0/+40% 16.0 5.00
29 # Snapdragon S4 11.5/+125% 13.6 4.90
31 # (*) most "favourable" result for aligned data on little-endian
32 # processor, result for misaligned data is 10-15% lower;
33 # (**) this result is a trade-off: it can be improved by 20%,
34 # but then Snapdragon S4 and Cortex-A8 results get
38 if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
39 else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
41 if ($flavour && $flavour ne "void") {
42 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
43 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
44 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
45 die "can't locate arm-xlate.pl";
47 open STDOUT,"| \"$^X\" $xlate $flavour $output";
49 open STDOUT,">$output";
52 sub AUTOLOAD() # thunk [simplified] x86-style perlasm
53 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
55 $arg = "#$arg" if ($arg*1 eq $arg);
56 $code .= "\t$opcode\t".join(',',@_,$arg)."\n";
59 my @x=map("r$_",(0..7,"x","x","x","x",12,"x",14,"x"));
60 my @t=map("r$_",(8..11));
63 my ($a0,$b0,$c0,$d0)=@_;
64 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
65 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
66 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
68 my ($xc,$xc_) = (@t[0..1]);
69 my ($xd,$xd_) = $odd ? (@t[2],@x[$d1]) : (@x[$d0],@t[2]);
72 # Consider order in which variables are addressed by their
77 # 0 4 8 12 < even round
81 # 0 5 10 15 < odd round
86 # 'a', 'b' are permanently allocated in registers, @x[0..7],
87 # while 'c's and pair of 'd's are maintained in memory. If
88 # you observe 'c' column, you'll notice that pair of 'c's is
89 # invariant between rounds. This means that we have to reload
90 # them once per round, in the middle. This is why you'll see
91 # bunch of 'c' stores and loads in the middle, but none in
92 # the beginning or end. If you observe 'd' column, you'll
93 # notice that 15 and 13 are reused in next pair of rounds.
94 # This is why these two are chosen for offloading to memory,
95 # to make loads count more.
97 "&add (@x[$a0],@x[$a0],@x[$b0])",
98 "&mov ($xd,$xd,'ror#16')",
99 "&add (@x[$a1],@x[$a1],@x[$b1])",
100 "&mov ($xd_,$xd_,'ror#16')",
101 "&eor ($xd,$xd,@x[$a0],'ror#16')",
102 "&eor ($xd_,$xd_,@x[$a1],'ror#16')",
104 "&add ($xc,$xc,$xd)",
105 "&mov (@x[$b0],@x[$b0],'ror#20')",
106 "&add ($xc_,$xc_,$xd_)",
107 "&mov (@x[$b1],@x[$b1],'ror#20')",
108 "&eor (@x[$b0],@x[$b0],$xc,'ror#20')",
109 "&eor (@x[$b1],@x[$b1],$xc_,'ror#20')",
111 "&add (@x[$a0],@x[$a0],@x[$b0])",
112 "&mov ($xd,$xd,'ror#24')",
113 "&add (@x[$a1],@x[$a1],@x[$b1])",
114 "&mov ($xd_,$xd_,'ror#24')",
115 "&eor ($xd,$xd,@x[$a0],'ror#24')",
116 "&eor ($xd_,$xd_,@x[$a1],'ror#24')",
118 "&add ($xc,$xc,$xd)",
119 "&mov (@x[$b0],@x[$b0],'ror#25')" );
121 "&str ($xd,'[sp,#4*(16+$d0)]')",
122 "&ldr ($xd,'[sp,#4*(16+$d2)]')" ) if ($odd);
124 "&add ($xc_,$xc_,$xd_)",
125 "&mov (@x[$b1],@x[$b1],'ror#25')" );
127 "&str ($xd_,'[sp,#4*(16+$d1)]')",
128 "&ldr ($xd_,'[sp,#4*(16+$d3)]')" ) if (!$odd);
130 "&eor (@x[$b0],@x[$b0],$xc,'ror#25')",
131 "&eor (@x[$b1],@x[$b1],$xc_,'ror#25')" );
133 $xd=@x[$d2] if (!$odd);
134 $xd_=@x[$d3] if ($odd);
136 "&str ($xc,'[sp,#4*(16+$c0)]')",
137 "&ldr ($xc,'[sp,#4*(16+$c2)]')",
138 "&add (@x[$a2],@x[$a2],@x[$b2])",
139 "&mov ($xd,$xd,'ror#16')",
140 "&str ($xc_,'[sp,#4*(16+$c1)]')",
141 "&ldr ($xc_,'[sp,#4*(16+$c3)]')",
142 "&add (@x[$a3],@x[$a3],@x[$b3])",
143 "&mov ($xd_,$xd_,'ror#16')",
144 "&eor ($xd,$xd,@x[$a2],'ror#16')",
145 "&eor ($xd_,$xd_,@x[$a3],'ror#16')",
147 "&add ($xc,$xc,$xd)",
148 "&mov (@x[$b2],@x[$b2],'ror#20')",
149 "&add ($xc_,$xc_,$xd_)",
150 "&mov (@x[$b3],@x[$b3],'ror#20')",
151 "&eor (@x[$b2],@x[$b2],$xc,'ror#20')",
152 "&eor (@x[$b3],@x[$b3],$xc_,'ror#20')",
154 "&add (@x[$a2],@x[$a2],@x[$b2])",
155 "&mov ($xd,$xd,'ror#24')",
156 "&add (@x[$a3],@x[$a3],@x[$b3])",
157 "&mov ($xd_,$xd_,'ror#24')",
158 "&eor ($xd,$xd,@x[$a2],'ror#24')",
159 "&eor ($xd_,$xd_,@x[$a3],'ror#24')",
161 "&add ($xc,$xc,$xd)",
162 "&mov (@x[$b2],@x[$b2],'ror#25')",
163 "&add ($xc_,$xc_,$xd_)",
164 "&mov (@x[$b3],@x[$b3],'ror#25')",
165 "&eor (@x[$b2],@x[$b2],$xc,'ror#25')",
166 "&eor (@x[$b3],@x[$b3],$xc_,'ror#25')" );
172 #include "arm_arch.h"
175 #if defined(__thumb2__)
182 #if defined(__thumb2__) || defined(__clang__)
183 #define ldrhsb ldrbhs
188 .long 0x61707865,0x3320646e,0x79622d32,0x6b206574 @ endian-neutral
191 #if __ARM_MAX_ARCH__>=7
193 .word OPENSSL_armcap_P-.LChaCha20_ctr32
198 .globl ChaCha20_ctr32
199 .type ChaCha20_ctr32,%function
203 ldr r12,[sp,#0] @ pull pointer to counter and nonce
204 stmdb sp!,{r0-r2,r4-r11,lr}
205 #if __ARM_ARCH__<7 && !defined(__thumb2__)
206 sub r14,pc,#16 @ ChaCha20_ctr32
208 adr r14,.LChaCha20_ctr32
216 #if __ARM_MAX_ARCH__>=7
217 cmp r2,#192 @ test len
228 ldmia r12,{r4-r7} @ load counter and nonce
229 sub sp,sp,#4*(16) @ off-load area
230 sub r14,r14,#64 @ .Lsigma
231 stmdb sp!,{r4-r7} @ copy counter and nonce
232 ldmia r3,{r4-r11} @ load key
233 ldmia r14,{r0-r3} @ load sigma
234 stmdb sp!,{r4-r11} @ copy key
235 stmdb sp!,{r0-r3} @ copy sigma
236 str r10,[sp,#4*(16+10)] @ off-load "@x[10]"
237 str r11,[sp,#4*(16+11)] @ off-load "@x[11]"
242 ldmia sp,{r0-r9} @ load key material
243 str @t[3],[sp,#4*(32+2)] @ save len
244 str r12, [sp,#4*(32+1)] @ save inp
245 str r14, [sp,#4*(32+0)] @ save out
247 ldr @t[3], [sp,#4*(15)]
248 ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load
249 ldr @t[2], [sp,#4*(13)]
250 ldr @x[14],[sp,#4*(14)]
251 str @t[3], [sp,#4*(16+15)]
259 foreach (&ROUND(0, 4, 8,12)) { eval; }
260 foreach (&ROUND(0, 5,10,15)) { eval; }
264 ldr @t[3],[sp,#4*(32+2)] @ load len
266 str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store
267 str @t[1], [sp,#4*(16+9)]
268 str @x[12],[sp,#4*(16+12)]
269 str @t[2], [sp,#4*(16+13)]
270 str @x[14],[sp,#4*(16+14)]
272 @ at this point we have first half of 512-bit result in
273 @ @x[0-7] and second half at sp+4*(16+8)
275 cmp @t[3],#64 @ done yet?
279 addlo r12,sp,#4*(0) @ shortcut or ...
280 ldrhs r12,[sp,#4*(32+1)] @ ... load inp
281 addlo r14,sp,#4*(0) @ shortcut or ...
282 ldrhs r14,[sp,#4*(32+0)] @ ... load out
284 ldr @t[0],[sp,#4*(0)] @ load key material
285 ldr @t[1],[sp,#4*(1)]
287 #if __ARM_ARCH__>=6 || !defined(__ARMEB__)
290 tst @t[2],#3 @ are input and output aligned?
291 ldr @t[2],[sp,#4*(2)]
293 cmp @t[3],#64 @ restore flags
295 ldr @t[2],[sp,#4*(2)]
297 ldr @t[3],[sp,#4*(3)]
299 add @x[0],@x[0],@t[0] @ accumulate key material
300 add @x[1],@x[1],@t[1]
304 ldrhs @t[0],[r12],#16 @ load input
305 ldrhs @t[1],[r12,#-12]
307 add @x[2],@x[2],@t[2]
308 add @x[3],@x[3],@t[3]
312 ldrhs @t[2],[r12,#-8]
313 ldrhs @t[3],[r12,#-4]
314 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
323 eorhs @x[0],@x[0],@t[0] @ xor with input
324 eorhs @x[1],@x[1],@t[1]
326 str @x[0],[r14],#16 @ store output
330 eorhs @x[2],@x[2],@t[2]
331 eorhs @x[3],@x[3],@t[3]
332 ldmia @t[0],{@t[0]-@t[3]} @ load key material
337 add @x[4],@x[4],@t[0] @ accumulate key material
338 add @x[5],@x[5],@t[1]
342 ldrhs @t[0],[r12],#16 @ load input
343 ldrhs @t[1],[r12,#-12]
344 add @x[6],@x[6],@t[2]
345 add @x[7],@x[7],@t[3]
349 ldrhs @t[2],[r12,#-8]
350 ldrhs @t[3],[r12,#-4]
351 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
360 eorhs @x[4],@x[4],@t[0]
361 eorhs @x[5],@x[5],@t[1]
363 str @x[4],[r14],#16 @ store output
367 eorhs @x[6],@x[6],@t[2]
368 eorhs @x[7],@x[7],@t[3]
370 ldmia @t[0],{@t[0]-@t[3]} @ load key material
372 add @x[0],sp,#4*(16+8)
375 ldmia @x[0],{@x[0]-@x[7]} @ load second half
377 add @x[0],@x[0],@t[0] @ accumulate key material
378 add @x[1],@x[1],@t[1]
382 ldrhs @t[0],[r12],#16 @ load input
383 ldrhs @t[1],[r12,#-12]
387 strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it
388 strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it
389 add @x[2],@x[2],@t[2]
390 add @x[3],@x[3],@t[3]
394 ldrhs @t[2],[r12,#-8]
395 ldrhs @t[3],[r12,#-4]
396 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
405 eorhs @x[0],@x[0],@t[0]
406 eorhs @x[1],@x[1],@t[1]
408 str @x[0],[r14],#16 @ store output
412 eorhs @x[2],@x[2],@t[2]
413 eorhs @x[3],@x[3],@t[3]
415 ldmia @t[0],{@t[0]-@t[3]} @ load key material
419 add @x[4],@x[4],@t[0] @ accumulate key material
420 add @x[5],@x[5],@t[1]
424 addhi @t[0],@t[0],#1 @ next counter value
425 strhi @t[0],[sp,#4*(12)] @ save next counter value
429 ldrhs @t[0],[r12],#16 @ load input
430 ldrhs @t[1],[r12,#-12]
431 add @x[6],@x[6],@t[2]
432 add @x[7],@x[7],@t[3]
436 ldrhs @t[2],[r12,#-8]
437 ldrhs @t[3],[r12,#-4]
438 # if __ARM_ARCH__>=6 && defined(__ARMEB__)
447 eorhs @x[4],@x[4],@t[0]
448 eorhs @x[5],@x[5],@t[1]
452 ldrne @t[0],[sp,#4*(32+2)] @ re-load len
456 eorhs @x[6],@x[6],@t[2]
457 eorhs @x[7],@x[7],@t[3]
458 str @x[4],[r14],#16 @ store output
463 subhs @t[3],@t[0],#64 @ len-=64
473 .Lunaligned: @ unaligned endian-neutral path
474 cmp @t[3],#64 @ restore flags
478 ldr @t[3],[sp,#4*(3)]
480 for ($i=0;$i<16;$i+=4) {
483 $code.=<<___ if ($i==4);
484 add @x[0],sp,#4*(16+8)
486 $code.=<<___ if ($i==8);
487 ldmia @x[0],{@x[0]-@x[7]} @ load second half
491 strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]"
492 strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]"
495 add @x[$j+0],@x[$j+0],@t[0] @ accumulate key material
497 $code.=<<___ if ($i==12);
501 addhi @t[0],@t[0],#1 @ next counter value
502 strhi @t[0],[sp,#4*(12)] @ save next counter value
505 add @x[$j+1],@x[$j+1],@t[1]
506 add @x[$j+2],@x[$j+2],@t[2]
510 eorlo @t[0],@t[0],@t[0] @ zero or ...
511 ldrhsb @t[0],[r12],#16 @ ... load input
512 eorlo @t[1],@t[1],@t[1]
513 ldrhsb @t[1],[r12,#-12]
515 add @x[$j+3],@x[$j+3],@t[3]
519 eorlo @t[2],@t[2],@t[2]
520 ldrhsb @t[2],[r12,#-8]
521 eorlo @t[3],@t[3],@t[3]
522 ldrhsb @t[3],[r12,#-4]
524 eor @x[$j+0],@t[0],@x[$j+0] @ xor with input (or zero)
525 eor @x[$j+1],@t[1],@x[$j+1]
529 ldrhsb @t[0],[r12,#-15] @ load more input
530 ldrhsb @t[1],[r12,#-11]
531 eor @x[$j+2],@t[2],@x[$j+2]
532 strb @x[$j+0],[r14],#16 @ store output
533 eor @x[$j+3],@t[3],@x[$j+3]
537 ldrhsb @t[2],[r12,#-7]
538 ldrhsb @t[3],[r12,#-3]
539 strb @x[$j+1],[r14,#-12]
540 eor @x[$j+0],@t[0],@x[$j+0],lsr#8
541 strb @x[$j+2],[r14,#-8]
542 eor @x[$j+1],@t[1],@x[$j+1],lsr#8
546 ldrhsb @t[0],[r12,#-14] @ load more input
547 ldrhsb @t[1],[r12,#-10]
548 strb @x[$j+3],[r14,#-4]
549 eor @x[$j+2],@t[2],@x[$j+2],lsr#8
550 strb @x[$j+0],[r14,#-15]
551 eor @x[$j+3],@t[3],@x[$j+3],lsr#8
555 ldrhsb @t[2],[r12,#-6]
556 ldrhsb @t[3],[r12,#-2]
557 strb @x[$j+1],[r14,#-11]
558 eor @x[$j+0],@t[0],@x[$j+0],lsr#8
559 strb @x[$j+2],[r14,#-7]
560 eor @x[$j+1],@t[1],@x[$j+1],lsr#8
564 ldrhsb @t[0],[r12,#-13] @ load more input
565 ldrhsb @t[1],[r12,#-9]
566 strb @x[$j+3],[r14,#-3]
567 eor @x[$j+2],@t[2],@x[$j+2],lsr#8
568 strb @x[$j+0],[r14,#-14]
569 eor @x[$j+3],@t[3],@x[$j+3],lsr#8
573 ldrhsb @t[2],[r12,#-5]
574 ldrhsb @t[3],[r12,#-1]
575 strb @x[$j+1],[r14,#-10]
576 strb @x[$j+2],[r14,#-6]
577 eor @x[$j+0],@t[0],@x[$j+0],lsr#8
578 strb @x[$j+3],[r14,#-2]
579 eor @x[$j+1],@t[1],@x[$j+1],lsr#8
580 strb @x[$j+0],[r14,#-13]
581 eor @x[$j+2],@t[2],@x[$j+2],lsr#8
582 strb @x[$j+1],[r14,#-9]
583 eor @x[$j+3],@t[3],@x[$j+3],lsr#8
584 strb @x[$j+2],[r14,#-5]
585 strb @x[$j+3],[r14,#-1]
587 $code.=<<___ if ($i<12);
588 add @t[0],sp,#4*(4+$i)
589 ldmia @t[0],{@t[0]-@t[3]} @ load key material
596 ldrne @t[0],[sp,#4*(32+2)] @ re-load len
600 subhs @t[3],@t[0],#64 @ len-=64
607 ldr r12,[sp,#4*(32+1)] @ load inp
609 ldr r14,[sp,#4*(32+0)] @ load out
612 ldrb @t[2],[@t[1]],#1 @ read buffer on stack
613 ldrb @t[3],[r12],#1 @ read input
615 eor @t[3],@t[3],@t[2]
616 strb @t[3],[r14],#1 @ store output
622 ldmia sp!,{r4-r11,pc}
623 .size ChaCha20_ctr32,.-ChaCha20_ctr32
627 my ($a0,$b0,$c0,$d0,$a1,$b1,$c1,$d1,$a2,$b2,$c2,$d2,$t0,$t1,$t2,$t3) =
632 my ($a,$b,$c,$d,$t)=@_;
635 "&vadd_i32 ($a,$a,$b)",
637 "&vrev32_16 ($d,$d)", # vrot ($d,16)
639 "&vadd_i32 ($c,$c,$d)",
641 "&vshr_u32 ($b,$t,20)",
642 "&vsli_32 ($b,$t,12)",
644 "&vadd_i32 ($a,$a,$b)",
646 "&vshr_u32 ($d,$t,24)",
647 "&vsli_32 ($d,$t,8)",
649 "&vadd_i32 ($c,$c,$d)",
651 "&vshr_u32 ($b,$t,25)",
652 "&vsli_32 ($b,$t,7)",
654 "&vext_8 ($c,$c,$c,8)",
655 "&vext_8 ($b,$b,$b,$odd?12:4)",
656 "&vext_8 ($d,$d,$d,$odd?4:12)"
661 #if __ARM_MAX_ARCH__>=7
665 .type ChaCha20_neon,%function
668 ldr r12,[sp,#0] @ pull pointer to counter and nonce
669 stmdb sp!,{r0-r2,r4-r11,lr}
672 vstmdb sp!,{d8-d15} @ ABI spec says so
675 vld1.32 {$b0-$c0},[r3] @ load key
676 ldmia r3,{r4-r11} @ load key
679 vld1.32 {$d0},[r12] @ load counter and nonce
681 ldmia r14,{r0-r3} @ load sigma
682 vld1.32 {$a0},[r14]! @ load sigma
683 vld1.32 {$t0},[r14] @ one
684 vst1.32 {$c0-$d0},[r12] @ copy 1/2key|counter|nonce
685 vst1.32 {$a0-$b0},[sp] @ copy sigma|1/2key
687 str r10,[sp,#4*(16+10)] @ off-load "@x[10]"
688 str r11,[sp,#4*(16+11)] @ off-load "@x[11]"
689 vshl.i32 $t1#lo,$t0#lo,#1 @ two
690 vstr $t0#lo,[sp,#4*(16+0)]
691 vshl.i32 $t2#lo,$t0#lo,#2 @ four
692 vstr $t1#lo,[sp,#4*(16+2)]
694 vstr $t2#lo,[sp,#4*(16+4)]
702 ldmia sp,{r0-r9} @ load key material
703 cmp @t[3],#64*2 @ if len<=64*2
704 bls .Lbreak_neon @ switch to integer-only
706 str @t[3],[sp,#4*(32+2)] @ save len
708 str r12, [sp,#4*(32+1)] @ save inp
710 str r14, [sp,#4*(32+0)] @ save out
713 ldr @t[3], [sp,#4*(15)]
714 vadd.i32 $d1,$d0,$t0 @ counter+1
715 ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load
717 ldr @t[2], [sp,#4*(13)]
719 ldr @x[14],[sp,#4*(14)]
720 vadd.i32 $d2,$d1,$t0 @ counter+2
721 str @t[3], [sp,#4*(16+15)]
723 add @x[12],@x[12],#3 @ counter+3
730 my @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,0);
731 my @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,0);
732 my @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,0);
733 my @thread3=&ROUND(0,4,8,12);
736 eval; eval(shift(@thread3));
737 eval(shift(@thread1)); eval(shift(@thread3));
738 eval(shift(@thread2)); eval(shift(@thread3));
741 @thread0=&NEONROUND($a0,$b0,$c0,$d0,$t0,1);
742 @thread1=&NEONROUND($a1,$b1,$c1,$d1,$t1,1);
743 @thread2=&NEONROUND($a2,$b2,$c2,$d2,$t2,1);
744 @thread3=&ROUND(0,5,10,15);
747 eval; eval(shift(@thread3));
748 eval(shift(@thread1)); eval(shift(@thread3));
749 eval(shift(@thread2)); eval(shift(@thread3));
755 vld1.32 {$t0-$t1},[sp] @ load key material
756 vld1.32 {$t2-$t3},[@t[3]]
758 ldr @t[3],[sp,#4*(32+2)] @ load len
760 str @t[0], [sp,#4*(16+8)] @ modulo-scheduled store
761 str @t[1], [sp,#4*(16+9)]
762 str @x[12],[sp,#4*(16+12)]
763 str @t[2], [sp,#4*(16+13)]
764 str @x[14],[sp,#4*(16+14)]
766 @ at this point we have first half of 512-bit result in
767 @ @x[0-7] and second half at sp+4*(16+8)
769 ldr r12,[sp,#4*(32+1)] @ load inp
770 ldr r14,[sp,#4*(32+0)] @ load out
772 vadd.i32 $a0,$a0,$t0 @ accumulate key material
775 vldr $t0#lo,[sp,#4*(16+0)] @ one
780 vldr $t1#lo,[sp,#4*(16+2)] @ two
785 vadd.i32 $d1#lo,$d1#lo,$t0#lo @ counter+1
786 vadd.i32 $d2#lo,$d2#lo,$t1#lo @ counter+2
795 vld1.8 {$t0-$t1},[r12]! @ load input
797 vld1.8 {$t2-$t3},[r12]!
798 veor $a0,$a0,$t0 @ xor with input
800 vld1.8 {$t0-$t1},[r12]!
803 vld1.8 {$t2-$t3},[r12]!
806 vst1.8 {$a0-$b0},[r14]! @ store output
808 vld1.8 {$t0-$t1},[r12]!
810 vst1.8 {$c0-$d0},[r14]!
812 vld1.8 {$t2-$t3},[r12]!
815 vld1.32 {$a0-$b0},[@t[3]]! @ load for next iteration
816 veor $t0#hi,$t0#hi,$t0#hi
817 vldr $t0#lo,[sp,#4*(16+4)] @ four
819 vld1.32 {$c0-$d0},[@t[3]]
821 vst1.8 {$a1-$b1},[r14]!
823 vst1.8 {$c1-$d1},[r14]!
825 vadd.i32 $d0#lo,$d0#lo,$t0#lo @ next counter value
826 vldr $t0#lo,[sp,#4*(16+0)] @ one
828 ldmia sp,{@t[0]-@t[3]} @ load key material
829 add @x[0],@x[0],@t[0] @ accumulate key material
830 ldr @t[0],[r12],#16 @ load input
831 vst1.8 {$a2-$b2},[r14]!
832 add @x[1],@x[1],@t[1]
834 vst1.8 {$c2-$d2},[r14]!
835 add @x[2],@x[2],@t[2]
837 add @x[3],@x[3],@t[3]
845 eor @x[0],@x[0],@t[0] @ xor with input
847 eor @x[1],@x[1],@t[1]
848 str @x[0],[r14],#16 @ store output
849 eor @x[2],@x[2],@t[2]
851 eor @x[3],@x[3],@t[3]
852 ldmia @t[0],{@t[0]-@t[3]} @ load key material
856 add @x[4],@x[4],@t[0] @ accumulate key material
857 ldr @t[0],[r12],#16 @ load input
858 add @x[5],@x[5],@t[1]
860 add @x[6],@x[6],@t[2]
862 add @x[7],@x[7],@t[3]
870 eor @x[4],@x[4],@t[0]
872 eor @x[5],@x[5],@t[1]
873 str @x[4],[r14],#16 @ store output
874 eor @x[6],@x[6],@t[2]
876 eor @x[7],@x[7],@t[3]
877 ldmia @t[0],{@t[0]-@t[3]} @ load key material
879 add @x[0],sp,#4*(16+8)
882 ldmia @x[0],{@x[0]-@x[7]} @ load second half
884 add @x[0],@x[0],@t[0] @ accumulate key material
885 ldr @t[0],[r12],#16 @ load input
886 add @x[1],@x[1],@t[1]
891 strhi @t[2],[sp,#4*(16+10)] @ copy "@x[10]" while at it
892 add @x[2],@x[2],@t[2]
897 strhi @t[3],[sp,#4*(16+11)] @ copy "@x[11]" while at it
898 add @x[3],@x[3],@t[3]
906 eor @x[0],@x[0],@t[0]
908 eor @x[1],@x[1],@t[1]
909 str @x[0],[r14],#16 @ store output
910 eor @x[2],@x[2],@t[2]
912 eor @x[3],@x[3],@t[3]
913 ldmia @t[0],{@t[0]-@t[3]} @ load key material
917 add @x[4],@x[4],@t[0] @ accumulate key material
918 add @t[0],@t[0],#4 @ next counter value
919 add @x[5],@x[5],@t[1]
920 str @t[0],[sp,#4*(12)] @ save next counter value
921 ldr @t[0],[r12],#16 @ load input
922 add @x[6],@x[6],@t[2]
923 add @x[4],@x[4],#3 @ counter+3
925 add @x[7],@x[7],@t[3]
934 eor @x[4],@x[4],@t[0]
938 ldrhi @t[0],[sp,#4*(32+2)] @ re-load len
939 eor @x[5],@x[5],@t[1]
940 eor @x[6],@x[6],@t[2]
941 str @x[4],[r14],#16 @ store output
942 eor @x[7],@x[7],@t[3]
944 sub @t[3],@t[0],#64*4 @ len-=64*4
953 @ harmonize NEON and integer-only stack frames: load data
954 @ from NEON frame, but save to integer-only one; distance
955 @ between the two is 4*(32+4+16-32)=4*(20).
957 str @t[3], [sp,#4*(20+32+2)] @ save len
958 add @t[3],sp,#4*(32+4)
959 str r12, [sp,#4*(20+32+1)] @ save inp
960 str r14, [sp,#4*(20+32+0)] @ save out
962 ldr @x[12],[sp,#4*(16+10)]
963 ldr @x[14],[sp,#4*(16+11)]
964 vldmia @t[3],{d8-d15} @ fulfill ABI requirement
965 str @x[12],[sp,#4*(20+16+10)] @ copy "@x[10]"
966 str @x[14],[sp,#4*(20+16+11)] @ copy "@x[11]"
968 ldr @t[3], [sp,#4*(15)]
969 ldr @x[12],[sp,#4*(12)] @ modulo-scheduled load
970 ldr @t[2], [sp,#4*(13)]
971 ldr @x[14],[sp,#4*(14)]
972 str @t[3], [sp,#4*(20+16+15)]
974 vst1.32 {$a0-$b0},[@t[3]]! @ copy key
975 add sp,sp,#4*(20) @ switch frame
976 vst1.32 {$c0-$d0},[@t[3]]
978 b .Loop @ go integer-only
983 bhs .L192_or_more_neon
985 bhs .L128_or_more_neon
987 bhs .L64_or_more_neon
990 vst1.8 {$a0-$b0},[sp]
992 vst1.8 {$c0-$d0},[@t[0]]
997 vld1.8 {$t0-$t1},[r12]!
998 vld1.8 {$t2-$t3},[r12]!
1003 vst1.8 {$a0-$b0},[r14]!
1004 vst1.8 {$c0-$d0},[r14]!
1009 vst1.8 {$a1-$b1},[sp]
1011 vst1.8 {$c1-$d1},[@t[0]]
1012 sub @t[3],@t[3],#64*1 @ len-=64*1
1017 vld1.8 {$t0-$t1},[r12]!
1018 vld1.8 {$t2-$t3},[r12]!
1021 vld1.8 {$t0-$t1},[r12]!
1024 vld1.8 {$t2-$t3},[r12]!
1028 vst1.8 {$a0-$b0},[r14]!
1030 vst1.8 {$c0-$d0},[r14]!
1032 vst1.8 {$a1-$b1},[r14]!
1033 vst1.8 {$c1-$d1},[r14]!
1038 vst1.8 {$a2-$b2},[sp]
1040 vst1.8 {$c2-$d2},[@t[0]]
1041 sub @t[3],@t[3],#64*2 @ len-=64*2
1046 vld1.8 {$t0-$t1},[r12]!
1047 vld1.8 {$t2-$t3},[r12]!
1050 vld1.8 {$t0-$t1},[r12]!
1053 vld1.8 {$t2-$t3},[r12]!
1057 vld1.8 {$t0-$t1},[r12]!
1059 vst1.8 {$a0-$b0},[r14]!
1061 vld1.8 {$t2-$t3},[r12]!
1064 vst1.8 {$c0-$d0},[r14]!
1066 vst1.8 {$a1-$b1},[r14]!
1068 vst1.8 {$c1-$d1},[r14]!
1070 vst1.8 {$a2-$b2},[r14]!
1071 vst1.8 {$c2-$d2},[r14]!
1075 ldmia sp,{@t[0]-@t[3]} @ load key material
1076 add @x[0],@x[0],@t[0] @ accumulate key material
1078 add @x[1],@x[1],@t[1]
1079 add @x[2],@x[2],@t[2]
1080 add @x[3],@x[3],@t[3]
1081 ldmia @t[0],{@t[0]-@t[3]} @ load key material
1083 add @x[4],@x[4],@t[0] @ accumulate key material
1085 add @x[5],@x[5],@t[1]
1086 add @x[6],@x[6],@t[2]
1087 add @x[7],@x[7],@t[3]
1088 ldmia @t[0],{@t[0]-@t[3]} @ load key material
1099 stmia sp,{@x[0]-@x[7]}
1100 add @x[0],sp,#4*(16+8)
1102 ldmia @x[0],{@x[0]-@x[7]} @ load second half
1104 add @x[0],@x[0],@t[0] @ accumulate key material
1105 add @t[0],sp,#4*(12)
1106 add @x[1],@x[1],@t[1]
1107 add @x[2],@x[2],@t[2]
1108 add @x[3],@x[3],@t[3]
1109 ldmia @t[0],{@t[0]-@t[3]} @ load key material
1111 add @x[4],@x[4],@t[0] @ accumulate key material
1113 add @x[5],@x[5],@t[1]
1114 add @x[4],@x[4],#3 @ counter+3
1115 add @x[6],@x[6],@t[2]
1116 add @x[7],@x[7],@t[3]
1117 ldr @t[3],[sp,#4*(32+2)] @ re-load len
1128 stmia @t[0],{@x[0]-@x[7]}
1130 sub @t[3],@t[3],#64*3 @ len-=64*3
1133 ldrb @t[0],[@t[2]],#1 @ read buffer on stack
1134 ldrb @t[1],[r12],#1 @ read input
1136 eor @t[0],@t[0],@t[1]
1137 strb @t[0],[r14],#1 @ store output
1144 ldmia sp!,{r4-r11,pc}
1145 .size ChaCha20_neon,.-ChaCha20_neon
1146 .comm OPENSSL_armcap_P,4,4
1151 foreach (split("\n",$code)) {
1152 s/\`([^\`]*)\`/eval $1/geo;
1154 s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo;