2 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
17 # IALU(*)/gcc-4.4 NEON
19 # ARM11xx(ARMv6) 7.78/+100% -
20 # Cortex-A5 6.35/+130% 3.00
21 # Cortex-A8 6.25/+115% 2.36
22 # Cortex-A9 5.10/+95% 2.55
23 # Cortex-A15 3.85/+85% 1.25(**)
24 # Snapdragon S4 5.70/+100% 1.48(**)
26 # (*) this is for -march=armv6, i.e. with bunch of ldrb loading data;
27 # (**) these are trade-off results, they can be improved by ~8% but at
28 # the cost of 15/12% regression on Cortex-A5/A7, it's even possible
29 # to improve Cortex-A9 result, but then A5/A7 loose more than 20%;
32 if ($flavour=~/\w[\w\-]*\.\w+$/) { $output=$flavour; undef $flavour; }
33 else { while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} }
35 if ($flavour && $flavour ne "void") {
36 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
37 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
38 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
39 die "can't locate arm-xlate.pl";
41 open STDOUT,"| \"$^X\" $xlate $flavour $output";
43 open STDOUT,">$output";
46 ($ctx,$inp,$len,$padbit)=map("r$_",(0..3));
51 #if defined(__thumb2__)
61 .globl poly1305_blocks
63 .type poly1305_init,%function
71 str r3,[$ctx,#0] @ zero hash value
76 str r3,[$ctx,#36] @ is_base2_26
85 #if __ARM_MAX_ARCH__>=7
86 adr r11,.Lpoly1305_init
87 ldr r12,.LOPENSSL_armcap
92 and r3,r10,#-4 @ 0x0ffffffc
103 #if __ARM_MAX_ARCH__>=7
104 # if !defined(_WIN32)
105 ldr r12,[r11,r12] @ OPENSSL_armcap_P
107 # if defined(__APPLE__) || defined(_WIN32)
120 #if __ARM_MAX_ARCH__>=7
121 tst r12,#ARMV7_NEON @ check for NEON
123 adr r9,.Lpoly1305_blocks_neon
124 adr r11,.Lpoly1305_blocks
125 adr r12,.Lpoly1305_emit
126 adr r10,.Lpoly1305_emit_neon
130 orr r11,r11,#1 @ thumb-ify address
133 addeq r12,r11,#(.Lpoly1305_emit-.Lpoly1305_init)
134 addne r12,r11,#(.Lpoly1305_emit_neon-.Lpoly1305_init)
135 addeq r11,r11,#(.Lpoly1305_blocks-.Lpoly1305_init)
136 addne r11,r11,#(.Lpoly1305_blocks_neon-.Lpoly1305_init)
157 #if __ARM_MAX_ARCH__>=7
158 stmia r2,{r11,r12} @ fill functions table
169 moveq pc,lr @ be binary compatible with V4, yet
170 bx lr @ interoperable with Thumb ISA:-)
172 .size poly1305_init,.-poly1305_init
175 my ($h0,$h1,$h2,$h3,$h4,$r0,$r1,$r2,$r3)=map("r$_",(4..12));
176 my ($s1,$s2,$s3)=($r1,$r2,$r3);
179 .type poly1305_blocks,%function
183 stmdb sp!,{r3-r11,lr}
189 add $len,$len,$inp @ end pointer
192 ldmia $ctx,{$h0-$r3} @ load context
194 str $ctx,[sp,#12] @ offload stuff
204 ldrb r0,[lr],#16 @ load input
208 addhi $h4,$h4,#1 @ 1<<128
218 adds $h0,$h0,r3 @ accumulate input
240 str lr,[sp,#8] @ offload input pointer
242 add $s1,$r1,$r1,lsr#2
245 ldr r0,[lr],#16 @ load input
249 addhi $h4,$h4,#1 @ padbit
259 adds $h0,$h0,r0 @ accumulate input
260 str lr,[sp,#8] @ offload input pointer
262 add $s1,$r1,$r1,lsr#2
265 add $s2,$r2,$r2,lsr#2
267 add $s3,$r3,$r3,lsr#2
274 ldr $r1,[sp,#20] @ reload $r1
280 str r0,[sp,#0] @ future $h0
282 ldr $r2,[sp,#24] @ reload $r2
283 adds r2,r2,r1 @ d1+=d0>>32
285 adc lr,r3,#0 @ future $h2
286 str r2,[sp,#4] @ future $h1
291 ldr $r3,[sp,#28] @ reload $r3
303 adds $h2,lr,r0 @ d2+=d1>>32
304 ldr lr,[sp,#8] @ reload input pointer
306 adds $h3,r2,r1 @ d3+=d2>>32
307 ldr r0,[sp,#16] @ reload end pointer
309 add $h4,$h4,r3 @ h4+=d3>>32
313 add r1,r1,r1,lsr#2 @ *=5
320 cmp r0,lr @ done yet?
325 stmia $ctx,{$h0-$h4} @ store the result
329 ldmia sp!,{r3-r11,pc}
331 ldmia sp!,{r3-r11,lr}
333 moveq pc,lr @ be binary compatible with V4, yet
334 bx lr @ interoperable with Thumb ISA:-)
336 .size poly1305_blocks,.-poly1305_blocks
340 my ($ctx,$mac,$nonce)=map("r$_",(0..2));
341 my ($h0,$h1,$h2,$h3,$h4,$g0,$g1,$g2,$g3)=map("r$_",(3..11));
345 .type poly1305_emit,%function
350 .Lpoly1305_emit_enter:
353 adds $g0,$h0,#5 @ compare to modulus
358 tst $g4,#4 @ did it carry/borrow?
435 moveq pc,lr @ be binary compatible with V4, yet
436 bx lr @ interoperable with Thumb ISA:-)
438 .size poly1305_emit,.-poly1305_emit
441 my ($R0,$R1,$S1,$R2,$S2,$R3,$S3,$R4,$S4) = map("d$_",(0..9));
442 my ($D0,$D1,$D2,$D3,$D4, $H0,$H1,$H2,$H3,$H4) = map("q$_",(5..14));
443 my ($T0,$T1,$MASK) = map("q$_",(15,4,0));
445 my ($in2,$zeros,$tbl0,$tbl1) = map("r$_",(4..7));
448 #if __ARM_MAX_ARCH__>=7
451 .type poly1305_init_neon,%function
454 ldr r4,[$ctx,#20] @ load key base 2^32
459 and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26
467 and r3,r3,#0x03ffffff
468 and r4,r4,#0x03ffffff
469 and r5,r5,#0x03ffffff
471 vdup.32 $R0,r2 @ r^1 in both lanes
472 add r2,r3,r3,lsl#2 @ *5
485 mov $zeros,#2 @ counter
488 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
489 @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
490 @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
491 @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
492 @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
493 @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
495 vmull.u32 $D0,$R0,${R0}[1]
496 vmull.u32 $D1,$R1,${R0}[1]
497 vmull.u32 $D2,$R2,${R0}[1]
498 vmull.u32 $D3,$R3,${R0}[1]
499 vmull.u32 $D4,$R4,${R0}[1]
501 vmlal.u32 $D0,$R4,${S1}[1]
502 vmlal.u32 $D1,$R0,${R1}[1]
503 vmlal.u32 $D2,$R1,${R1}[1]
504 vmlal.u32 $D3,$R2,${R1}[1]
505 vmlal.u32 $D4,$R3,${R1}[1]
507 vmlal.u32 $D0,$R3,${S2}[1]
508 vmlal.u32 $D1,$R4,${S2}[1]
509 vmlal.u32 $D3,$R1,${R2}[1]
510 vmlal.u32 $D2,$R0,${R2}[1]
511 vmlal.u32 $D4,$R2,${R2}[1]
513 vmlal.u32 $D0,$R2,${S3}[1]
514 vmlal.u32 $D3,$R0,${R3}[1]
515 vmlal.u32 $D1,$R3,${S3}[1]
516 vmlal.u32 $D2,$R4,${S3}[1]
517 vmlal.u32 $D4,$R1,${R3}[1]
519 vmlal.u32 $D3,$R4,${S4}[1]
520 vmlal.u32 $D0,$R1,${S4}[1]
521 vmlal.u32 $D1,$R2,${S4}[1]
522 vmlal.u32 $D2,$R3,${S4}[1]
523 vmlal.u32 $D4,$R0,${R4}[1]
525 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
526 @ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
529 @ H0>>+H1>>+H2>>+H3>>+H4
530 @ H3>>+H4>>*5+H0>>+H1
534 @ Result of multiplication of n-bit number by m-bit number is
535 @ n+m bits wide. However! Even though 2^n is a n+1-bit number,
536 @ m-bit number multiplied by 2^n is still n+m bits wide.
538 @ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
539 @ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
540 @ one is n+1 bits wide.
542 @ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
543 @ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
544 @ can be 27. However! In cases when their width exceeds 26 bits
545 @ they are limited by 2^26+2^6. This in turn means that *sum*
546 @ of the products with these values can still be viewed as sum
547 @ of 52-bit numbers as long as the amount of addends is not a
548 @ power of 2. For example,
550 @ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
552 @ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
553 @ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
554 @ 8 * (2^52) or 2^55. However, the value is then multiplied by
555 @ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
556 @ which is less than 32 * (2^52) or 2^57. And when processing
557 @ data we are looking at triple as many addends...
559 @ In key setup procedure pre-reduced H0 is limited by 5*4+1 and
560 @ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
561 @ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
562 @ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
563 @ instruction accepts 2x32-bit input and writes 2x64-bit result.
564 @ This means that result of reduction have to be compressed upon
565 @ loop wrap-around. This can be done in the process of reduction
566 @ to minimize amount of instructions [as well as amount of
567 @ 128-bit instructions, which benefits low-end processors], but
568 @ one has to watch for H2 (which is narrower than H0) and 5*H4
569 @ not being wider than 58 bits, so that result of right shift
570 @ by 26 bits fits in 32 bits. This is also useful on x86,
571 @ because it allows to use paddd in place for paddq, which
572 @ benefits Atom, where paddq is ridiculously slow.
578 vadd.i64 $D4,$D4,$T0 @ h3 -> h4
579 vbic.i32 $D3#lo,#0xfc000000 @ &=0x03ffffff
580 vadd.i64 $D1,$D1,$T1 @ h0 -> h1
581 vbic.i32 $D0#lo,#0xfc000000
583 vshrn.u64 $T0#lo,$D4,#26
587 vadd.i64 $D2,$D2,$T1 @ h1 -> h2
588 vbic.i32 $D4#lo,#0xfc000000
589 vbic.i32 $D1#lo,#0xfc000000
591 vadd.i32 $D0#lo,$D0#lo,$T0#lo
592 vshl.u32 $T0#lo,$T0#lo,#2
593 vshrn.u64 $T1#lo,$D2,#26
595 vadd.i32 $D0#lo,$D0#lo,$T0#lo @ h4 -> h0
596 vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3
597 vbic.i32 $D2#lo,#0xfc000000
599 vshr.u32 $T0#lo,$D0#lo,#26
600 vbic.i32 $D0#lo,#0xfc000000
601 vshr.u32 $T1#lo,$D3#lo,#26
602 vbic.i32 $D3#lo,#0xfc000000
603 vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1
604 vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4
606 subs $zeros,$zeros,#1
607 beq .Lsquare_break_neon
609 add $tbl0,$ctx,#(48+0*9*4)
610 add $tbl1,$ctx,#(48+1*9*4)
612 vtrn.32 $R0,$D0#lo @ r^2:r^1
618 vshl.u32 $S2,$R2,#2 @ *5
627 vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
628 vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
629 vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
630 vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
631 vst1.32 {${S4}[0]},[$tbl0,:32]
632 vst1.32 {${S4}[1]},[$tbl1,:32]
638 add $tbl0,$ctx,#(48+2*4*9)
639 add $tbl1,$ctx,#(48+3*4*9)
641 vmov $R0,$D0#lo @ r^4:r^3
642 vshl.u32 $S1,$D1#lo,#2 @ *5
644 vshl.u32 $S2,$D2#lo,#2
646 vshl.u32 $S3,$D3#lo,#2
648 vshl.u32 $S4,$D4#lo,#2
650 vadd.i32 $S1,$S1,$D1#lo
651 vadd.i32 $S2,$S2,$D2#lo
652 vadd.i32 $S3,$S3,$D3#lo
653 vadd.i32 $S4,$S4,$D4#lo
655 vst4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]!
656 vst4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]!
657 vst4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
658 vst4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
659 vst1.32 {${S4}[0]},[$tbl0]
660 vst1.32 {${S4}[1]},[$tbl1]
663 .size poly1305_init_neon,.-poly1305_init_neon
665 .type poly1305_blocks_neon,%function
667 poly1305_blocks_neon:
668 .Lpoly1305_blocks_neon:
669 ldr ip,[$ctx,#36] @ is_base2_26
675 tst ip,ip @ is_base2_26?
676 beq .Lpoly1305_blocks
680 vstmdb sp!,{d8-d15} @ ABI specification says so
682 tst ip,ip @ is_base2_26?
686 bl poly1305_init_neon
688 ldr r4,[$ctx,#0] @ load hash value base 2^32
694 and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26
696 veor $D0#lo,$D0#lo,$D0#lo
699 veor $D1#lo,$D1#lo,$D1#lo
702 veor $D2#lo,$D2#lo,$D2#lo
705 veor $D3#lo,$D3#lo,$D3#lo
706 and r3,r3,#0x03ffffff
708 veor $D4#lo,$D4#lo,$D4#lo
709 and r4,r4,#0x03ffffff
711 and r5,r5,#0x03ffffff
712 str r1,[$ctx,#36] @ is_base2_26
726 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
729 veor $D0#lo,$D0#lo,$D0#lo
730 veor $D1#lo,$D1#lo,$D1#lo
731 veor $D2#lo,$D2#lo,$D2#lo
732 veor $D3#lo,$D3#lo,$D3#lo
733 veor $D4#lo,$D4#lo,$D4#lo
734 vld4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
736 vld1.32 {$D4#lo[0]},[$ctx]
737 sub $ctx,$ctx,#16 @ rewind
741 mov $padbit,$padbit,lsl#24
745 vld4.32 {$H0#lo[0],$H1#lo[0],$H2#lo[0],$H3#lo[0]},[$inp]!
746 vmov.32 $H4#lo[0],$padbit
756 vsri.u32 $H4#lo,$H3#lo,#8 @ base 2^32 -> base 2^26
757 vshl.u32 $H3#lo,$H3#lo,#18
759 vsri.u32 $H3#lo,$H2#lo,#14
760 vshl.u32 $H2#lo,$H2#lo,#12
761 vadd.i32 $H4#hi,$H4#lo,$D4#lo @ add hash value and move to #hi
763 vbic.i32 $H3#lo,#0xfc000000
764 vsri.u32 $H2#lo,$H1#lo,#20
765 vshl.u32 $H1#lo,$H1#lo,#6
767 vbic.i32 $H2#lo,#0xfc000000
768 vsri.u32 $H1#lo,$H0#lo,#26
769 vadd.i32 $H3#hi,$H3#lo,$D3#lo
771 vbic.i32 $H0#lo,#0xfc000000
772 vbic.i32 $H1#lo,#0xfc000000
773 vadd.i32 $H2#hi,$H2#lo,$D2#lo
775 vadd.i32 $H0#hi,$H0#lo,$D0#lo
776 vadd.i32 $H1#hi,$H1#lo,$D1#lo
790 vmov.i32 $H4,#1<<24 @ padbit, yes, always
791 vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1]
793 vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0)
796 addhi $tbl1,$ctx,#(48+1*9*4)
797 addhi $tbl0,$ctx,#(48+3*9*4)
805 vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26
811 vbic.i32 $H3,#0xfc000000
815 vbic.i32 $H2,#0xfc000000
818 vbic.i32 $H0,#0xfc000000
819 vbic.i32 $H1,#0xfc000000
823 vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^2
824 vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4
825 vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
826 vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
831 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
832 @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
833 @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
834 @ \___________________/
835 @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
836 @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
837 @ \___________________/ \____________________/
839 @ Note that we start with inp[2:3]*r^2. This is because it
840 @ doesn't depend on reduction in previous iteration.
841 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
842 @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
843 @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
844 @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
845 @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
846 @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
848 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
851 vadd.i32 $H2#lo,$H2#lo,$D2#lo @ accumulate inp[0:1]
852 vmull.u32 $D2,$H2#hi,${R0}[1]
853 vadd.i32 $H0#lo,$H0#lo,$D0#lo
854 vmull.u32 $D0,$H0#hi,${R0}[1]
855 vadd.i32 $H3#lo,$H3#lo,$D3#lo
856 vmull.u32 $D3,$H3#hi,${R0}[1]
857 vmlal.u32 $D2,$H1#hi,${R1}[1]
858 vadd.i32 $H1#lo,$H1#lo,$D1#lo
859 vmull.u32 $D1,$H1#hi,${R0}[1]
861 vadd.i32 $H4#lo,$H4#lo,$D4#lo
862 vmull.u32 $D4,$H4#hi,${R0}[1]
864 vmlal.u32 $D0,$H4#hi,${S1}[1]
867 vmlal.u32 $D3,$H2#hi,${R1}[1]
868 vld1.32 ${S4}[1],[$tbl1,:32]
869 vmlal.u32 $D1,$H0#hi,${R1}[1]
870 vmlal.u32 $D4,$H3#hi,${R1}[1]
872 vmlal.u32 $D0,$H3#hi,${S2}[1]
873 vmlal.u32 $D3,$H1#hi,${R2}[1]
874 vmlal.u32 $D4,$H2#hi,${R2}[1]
875 vmlal.u32 $D1,$H4#hi,${S2}[1]
876 vmlal.u32 $D2,$H0#hi,${R2}[1]
878 vmlal.u32 $D3,$H0#hi,${R3}[1]
879 vmlal.u32 $D0,$H2#hi,${S3}[1]
880 vmlal.u32 $D4,$H1#hi,${R3}[1]
881 vmlal.u32 $D1,$H3#hi,${S3}[1]
882 vmlal.u32 $D2,$H4#hi,${S3}[1]
884 vmlal.u32 $D3,$H4#hi,${S4}[1]
885 vmlal.u32 $D0,$H1#hi,${S4}[1]
886 vmlal.u32 $D4,$H0#hi,${R4}[1]
887 vmlal.u32 $D1,$H2#hi,${S4}[1]
888 vmlal.u32 $D2,$H3#hi,${S4}[1]
890 vld4.32 {$H0#hi,$H1#hi,$H2#hi,$H3#hi},[$in2] @ inp[2:3] (or 0)
893 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
894 @ (hash+inp[0:1])*r^4 and accumulate
896 vmlal.u32 $D3,$H3#lo,${R0}[0]
897 vmlal.u32 $D0,$H0#lo,${R0}[0]
898 vmlal.u32 $D4,$H4#lo,${R0}[0]
899 vmlal.u32 $D1,$H1#lo,${R0}[0]
900 vmlal.u32 $D2,$H2#lo,${R0}[0]
901 vld1.32 ${S4}[0],[$tbl0,:32]
903 vmlal.u32 $D3,$H2#lo,${R1}[0]
904 vmlal.u32 $D0,$H4#lo,${S1}[0]
905 vmlal.u32 $D4,$H3#lo,${R1}[0]
906 vmlal.u32 $D1,$H0#lo,${R1}[0]
907 vmlal.u32 $D2,$H1#lo,${R1}[0]
909 vmlal.u32 $D3,$H1#lo,${R2}[0]
910 vmlal.u32 $D0,$H3#lo,${S2}[0]
911 vmlal.u32 $D4,$H2#lo,${R2}[0]
912 vmlal.u32 $D1,$H4#lo,${S2}[0]
913 vmlal.u32 $D2,$H0#lo,${R2}[0]
915 vmlal.u32 $D3,$H0#lo,${R3}[0]
916 vmlal.u32 $D0,$H2#lo,${S3}[0]
917 vmlal.u32 $D4,$H1#lo,${R3}[0]
918 vmlal.u32 $D1,$H3#lo,${S3}[0]
919 vmlal.u32 $D3,$H4#lo,${S4}[0]
921 vmlal.u32 $D2,$H4#lo,${S3}[0]
922 vmlal.u32 $D0,$H1#lo,${S4}[0]
923 vmlal.u32 $D4,$H0#lo,${R4}[0]
924 vmov.i32 $H4,#1<<24 @ padbit, yes, always
925 vmlal.u32 $D1,$H2#lo,${S4}[0]
926 vmlal.u32 $D2,$H3#lo,${S4}[0]
928 vld4.32 {$H0#lo,$H1#lo,$H2#lo,$H3#lo},[$inp] @ inp[0:1]
937 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
938 @ lazy reduction interleaved with base 2^32 -> base 2^26 of
939 @ inp[0:3] previously loaded to $H0-$H3 and smashed to $H0-$H4.
945 vadd.i64 $D4,$D4,$T0 @ h3 -> h4
946 vbic.i32 $D3#lo,#0xfc000000
947 vsri.u32 $H4,$H3,#8 @ base 2^32 -> base 2^26
948 vadd.i64 $D1,$D1,$T1 @ h0 -> h1
950 vbic.i32 $D0#lo,#0xfc000000
952 vshrn.u64 $T0#lo,$D4,#26
956 vadd.i64 $D2,$D2,$T1 @ h1 -> h2
958 vbic.i32 $D4#lo,#0xfc000000
960 vbic.i32 $D1#lo,#0xfc000000
962 vadd.i32 $D0#lo,$D0#lo,$T0#lo
963 vshl.u32 $T0#lo,$T0#lo,#2
964 vbic.i32 $H3,#0xfc000000
965 vshrn.u64 $T1#lo,$D2,#26
967 vaddl.u32 $D0,$D0#lo,$T0#lo @ h4 -> h0 [widen for a sec]
969 vadd.i32 $D3#lo,$D3#lo,$T1#lo @ h2 -> h3
971 vbic.i32 $D2#lo,#0xfc000000
972 vbic.i32 $H2,#0xfc000000
974 vshrn.u64 $T0#lo,$D0,#26 @ re-narrow
977 vbic.i32 $H0,#0xfc000000
978 vshr.u32 $T1#lo,$D3#lo,#26
979 vbic.i32 $D3#lo,#0xfc000000
980 vbic.i32 $D0#lo,#0xfc000000
981 vadd.i32 $D1#lo,$D1#lo,$T0#lo @ h0 -> h1
982 vadd.i32 $D4#lo,$D4#lo,$T1#lo @ h3 -> h4
983 vbic.i32 $H1,#0xfc000000
988 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
989 @ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
991 add $tbl1,$ctx,#(48+0*9*4)
992 add $tbl0,$ctx,#(48+1*9*4)
998 vadd.i32 $H2#hi,$H2#lo,$D2#lo @ add hash value and move to #hi
999 vadd.i32 $H0#hi,$H0#lo,$D0#lo
1000 vadd.i32 $H3#hi,$H3#lo,$D3#lo
1001 vadd.i32 $H1#hi,$H1#lo,$D1#lo
1002 vadd.i32 $H4#hi,$H4#lo,$D4#lo
1005 vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^1
1006 vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^2
1008 vadd.i32 $H2#lo,$H2#lo,$D2#lo @ can be redundant
1009 vmull.u32 $D2,$H2#hi,$R0
1010 vadd.i32 $H0#lo,$H0#lo,$D0#lo
1011 vmull.u32 $D0,$H0#hi,$R0
1012 vadd.i32 $H3#lo,$H3#lo,$D3#lo
1013 vmull.u32 $D3,$H3#hi,$R0
1014 vadd.i32 $H1#lo,$H1#lo,$D1#lo
1015 vmull.u32 $D1,$H1#hi,$R0
1016 vadd.i32 $H4#lo,$H4#lo,$D4#lo
1017 vmull.u32 $D4,$H4#hi,$R0
1019 vmlal.u32 $D0,$H4#hi,$S1
1020 vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
1021 vmlal.u32 $D3,$H2#hi,$R1
1022 vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
1023 vmlal.u32 $D1,$H0#hi,$R1
1024 vmlal.u32 $D4,$H3#hi,$R1
1025 vmlal.u32 $D2,$H1#hi,$R1
1027 vmlal.u32 $D3,$H1#hi,$R2
1028 vld1.32 ${S4}[1],[$tbl1,:32]
1029 vmlal.u32 $D0,$H3#hi,$S2
1030 vld1.32 ${S4}[0],[$tbl0,:32]
1031 vmlal.u32 $D4,$H2#hi,$R2
1032 vmlal.u32 $D1,$H4#hi,$S2
1033 vmlal.u32 $D2,$H0#hi,$R2
1035 vmlal.u32 $D3,$H0#hi,$R3
1037 addne $tbl1,$ctx,#(48+2*9*4)
1038 vmlal.u32 $D0,$H2#hi,$S3
1040 addne $tbl0,$ctx,#(48+3*9*4)
1041 vmlal.u32 $D4,$H1#hi,$R3
1042 vmlal.u32 $D1,$H3#hi,$S3
1043 vmlal.u32 $D2,$H4#hi,$S3
1045 vmlal.u32 $D3,$H4#hi,$S4
1046 vorn $MASK,$MASK,$MASK @ all-ones, can be redundant
1047 vmlal.u32 $D0,$H1#hi,$S4
1048 vshr.u64 $MASK,$MASK,#38
1049 vmlal.u32 $D4,$H0#hi,$R4
1050 vmlal.u32 $D1,$H2#hi,$S4
1051 vmlal.u32 $D2,$H3#hi,$S4
1055 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1056 @ (hash+inp[0:1])*r^4:r^3 and accumulate
1058 vld4.32 {${R0}[1],${R1}[1],${S1}[1],${R2}[1]},[$tbl1]! @ load r^3
1059 vld4.32 {${R0}[0],${R1}[0],${S1}[0],${R2}[0]},[$tbl0]! @ load r^4
1061 vmlal.u32 $D2,$H2#lo,$R0
1062 vmlal.u32 $D0,$H0#lo,$R0
1063 vmlal.u32 $D3,$H3#lo,$R0
1064 vmlal.u32 $D1,$H1#lo,$R0
1065 vmlal.u32 $D4,$H4#lo,$R0
1067 vmlal.u32 $D0,$H4#lo,$S1
1068 vld4.32 {${S2}[1],${R3}[1],${S3}[1],${R4}[1]},[$tbl1]!
1069 vmlal.u32 $D3,$H2#lo,$R1
1070 vld4.32 {${S2}[0],${R3}[0],${S3}[0],${R4}[0]},[$tbl0]!
1071 vmlal.u32 $D1,$H0#lo,$R1
1072 vmlal.u32 $D4,$H3#lo,$R1
1073 vmlal.u32 $D2,$H1#lo,$R1
1075 vmlal.u32 $D3,$H1#lo,$R2
1076 vld1.32 ${S4}[1],[$tbl1,:32]
1077 vmlal.u32 $D0,$H3#lo,$S2
1078 vld1.32 ${S4}[0],[$tbl0,:32]
1079 vmlal.u32 $D4,$H2#lo,$R2
1080 vmlal.u32 $D1,$H4#lo,$S2
1081 vmlal.u32 $D2,$H0#lo,$R2
1083 vmlal.u32 $D3,$H0#lo,$R3
1084 vmlal.u32 $D0,$H2#lo,$S3
1085 vmlal.u32 $D4,$H1#lo,$R3
1086 vmlal.u32 $D1,$H3#lo,$S3
1087 vmlal.u32 $D2,$H4#lo,$S3
1089 vmlal.u32 $D3,$H4#lo,$S4
1090 vorn $MASK,$MASK,$MASK @ all-ones
1091 vmlal.u32 $D0,$H1#lo,$S4
1092 vshr.u64 $MASK,$MASK,#38
1093 vmlal.u32 $D4,$H0#lo,$R4
1094 vmlal.u32 $D1,$H2#lo,$S4
1095 vmlal.u32 $D2,$H3#lo,$S4
1098 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1099 @ horizontal addition
1101 vadd.i64 $D3#lo,$D3#lo,$D3#hi
1102 vadd.i64 $D0#lo,$D0#lo,$D0#hi
1103 vadd.i64 $D4#lo,$D4#lo,$D4#hi
1104 vadd.i64 $D1#lo,$D1#lo,$D1#hi
1105 vadd.i64 $D2#lo,$D2#lo,$D2#hi
1107 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1108 @ lazy reduction, but without narrowing
1110 vshr.u64 $T0,$D3,#26
1111 vand.i64 $D3,$D3,$MASK
1112 vshr.u64 $T1,$D0,#26
1113 vand.i64 $D0,$D0,$MASK
1114 vadd.i64 $D4,$D4,$T0 @ h3 -> h4
1115 vadd.i64 $D1,$D1,$T1 @ h0 -> h1
1117 vshr.u64 $T0,$D4,#26
1118 vand.i64 $D4,$D4,$MASK
1119 vshr.u64 $T1,$D1,#26
1120 vand.i64 $D1,$D1,$MASK
1121 vadd.i64 $D2,$D2,$T1 @ h1 -> h2
1123 vadd.i64 $D0,$D0,$T0
1125 vshr.u64 $T1,$D2,#26
1126 vand.i64 $D2,$D2,$MASK
1127 vadd.i64 $D0,$D0,$T0 @ h4 -> h0
1128 vadd.i64 $D3,$D3,$T1 @ h2 -> h3
1130 vshr.u64 $T0,$D0,#26
1131 vand.i64 $D0,$D0,$MASK
1132 vshr.u64 $T1,$D3,#26
1133 vand.i64 $D3,$D3,$MASK
1134 vadd.i64 $D1,$D1,$T0 @ h0 -> h1
1135 vadd.i64 $D4,$D4,$T1 @ h3 -> h4
1140 @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
1143 vst4.32 {$D0#lo[0],$D1#lo[0],$D2#lo[0],$D3#lo[0]},[$ctx]!
1144 vst1.32 {$D4#lo[0]},[$ctx]
1146 vldmia sp!,{d8-d15} @ epilogue
1150 .size poly1305_blocks_neon,.-poly1305_blocks_neon
1152 .type poly1305_emit_neon,%function
1155 .Lpoly1305_emit_neon:
1156 ldr ip,[$ctx,#36] @ is_base2_26
1161 beq .Lpoly1305_emit_enter
1163 ldmia $ctx,{$h0-$h4}
1166 adds $h0,$h0,$h1,lsl#26 @ base 2^26 -> base 2^32
1168 adcs $h1,$h1,$h2,lsl#20
1170 adcs $h2,$h2,$h3,lsl#14
1172 adcs $h3,$h3,$h4,lsl#8
1173 adc $h4,$g0,$h4,lsr#24 @ can be partially reduced ...
1175 and $g0,$h4,#-4 @ ... so reduce
1177 add $g0,$g0,$g0,lsr#2 @ *= 5
1184 adds $g0,$h0,#5 @ compare to modulus
1189 tst $g4,#4 @ did it carry/borrow?
1202 ldr $g3,[$nonce,#12]
1204 adds $h0,$h0,$g0 @ accumulate nonce
1215 str $h0,[$mac,#0] @ store the result
1222 .size poly1305_emit_neon,.-poly1305_emit_neon
1226 .long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
1229 .word OPENSSL_armcap_P
1231 .word OPENSSL_armcap_P-.Lpoly1305_init
1237 .asciz "Poly1305 for ARMv4/NEON, CRYPTOGAMS by <appro\@openssl.org>"
1239 #if __ARM_MAX_ARCH__>=7
1240 .comm OPENSSL_armcap_P,4,4
1244 foreach (split("\n",$code)) {
1245 s/\`([^\`]*)\`/eval $1/geo;
1247 s/\bq([0-9]+)#(lo|hi)/sprintf "d%d",2*$1+($2 eq "hi")/geo or
1248 s/\bret\b/bx lr/go or
1249 s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4
1253 close STDOUT; # enforce flush