3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # ECP_NISTZ256 module for ARMv8.
14 # Original ECP_NISTZ256 submission targeting x86_64 is detailed in
15 # http://eprint.iacr.org/2013/816.
17 # with/without -DECP_NISTZ256_ASM(*)
19 # Cortex-A53 +135-720%
20 # Cortex-A57 +145-570%
24 # (*) comparison is not really "fair", because it's compared to C
25 # implementation, unlike other similar cases that is;
27 # Ranges denote minimum and maximum improvement coefficients depending
28 # on benchmark. Lower coefficients are for ECDSA sign, server-side
29 # operation. Keep in mind that +500% means 6x improvement.
32 while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
34 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
35 ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
36 ( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
37 die "can't locate arm-xlate.pl";
39 open OUT,"| \"$^X\" $xlate $flavour $output";
43 my ($rp,$ap,$bp,$bi,$a0,$a1,$a2,$a3,$t0,$t1,$t2,$t3,$poly1,$poly3,
44 $acc0,$acc1,$acc2,$acc3,$acc4,$acc5) =
45 map("x$_",(0..17,19,20));
47 my ($acc6,$acc7)=($ap,$bp); # used in __ecp_nistz256_sqr_mont
54 ########################################################################
55 # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
57 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
58 open TABLE,"<ecp_nistz256_table.c" or
59 open TABLE,"<${dir}../ecp_nistz256_table.c" or
60 die "failed to open ecp_nistz256_table.c:",$!;
65 s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
69 # See ecp_nistz256_table.c for explanation for why it's 64*16*37.
70 # 64*16*37-1 is because $#arr returns last valid index or @arr, not
72 die "insane number of elements" if ($#arr != 64*16*37-1);
75 .globl ecp_nistz256_precomputed
76 .type ecp_nistz256_precomputed,%object
78 ecp_nistz256_precomputed:
80 ########################################################################
81 # this conversion smashes P256_POINT_AFFINE by individual bytes with
82 # 64 byte interval, similar to
86 @tbl = splice(@arr,0,64*16);
87 for($i=0;$i<64;$i++) {
89 for($j=0;$j<64;$j++) {
90 push @line,(@tbl[$j*16+$i/4]>>(($i%4)*8))&0xff;
93 $code.=join(',',map { sprintf "0x%02x",$_} @line);
98 .size ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
101 .quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001
102 .LRR: // 2^512 mod P precomputed for NIST P256 polynomial
103 .quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd
105 .quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe
108 .asciz "ECP_NISTZ256 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
110 // void ecp_nistz256_to_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
111 .globl ecp_nistz256_to_mont
112 .type ecp_nistz256_to_mont,%function
114 ecp_nistz256_to_mont:
115 stp x29,x30,[sp,#-32]!
119 ldr $bi,.LRR // bp[0]
121 ldp $a2,$a3,[$ap,#16]
124 adr $bp,.LRR // &bp[0]
126 bl __ecp_nistz256_mul_mont
131 .size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont
133 // void ecp_nistz256_from_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
134 .globl ecp_nistz256_from_mont
135 .type ecp_nistz256_from_mont,%function
137 ecp_nistz256_from_mont:
138 stp x29,x30,[sp,#-32]!
144 ldp $a2,$a3,[$ap,#16]
147 adr $bp,.Lone // &bp[0]
149 bl __ecp_nistz256_mul_mont
154 .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
156 // void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4],
157 // const BN_ULONG x2[4]);
158 .globl ecp_nistz256_mul_mont
159 .type ecp_nistz256_mul_mont,%function
161 ecp_nistz256_mul_mont:
162 stp x29,x30,[sp,#-32]!
166 ldr $bi,[$bp] // bp[0]
168 ldp $a2,$a3,[$ap,#16]
172 bl __ecp_nistz256_mul_mont
177 .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
179 // void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]);
180 .globl ecp_nistz256_sqr_mont
181 .type ecp_nistz256_sqr_mont,%function
183 ecp_nistz256_sqr_mont:
184 stp x29,x30,[sp,#-32]!
189 ldp $a2,$a3,[$ap,#16]
193 bl __ecp_nistz256_sqr_mont
198 .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
200 // void ecp_nistz256_add(BN_ULONG x0[4],const BN_ULONG x1[4],
201 // const BN_ULONG x2[4]);
202 .globl ecp_nistz256_add
203 .type ecp_nistz256_add,%function
206 stp x29,x30,[sp,#-16]!
209 ldp $acc0,$acc1,[$ap]
211 ldp $acc2,$acc3,[$ap,#16]
212 ldp $t2,$t3,[$bp,#16]
216 bl __ecp_nistz256_add
220 .size ecp_nistz256_add,.-ecp_nistz256_add
222 // void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
223 .globl ecp_nistz256_div_by_2
224 .type ecp_nistz256_div_by_2,%function
226 ecp_nistz256_div_by_2:
227 stp x29,x30,[sp,#-16]!
230 ldp $acc0,$acc1,[$ap]
231 ldp $acc2,$acc3,[$ap,#16]
235 bl __ecp_nistz256_div_by_2
239 .size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
241 // void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]);
242 .globl ecp_nistz256_mul_by_2
243 .type ecp_nistz256_mul_by_2,%function
245 ecp_nistz256_mul_by_2:
246 stp x29,x30,[sp,#-16]!
249 ldp $acc0,$acc1,[$ap]
250 ldp $acc2,$acc3,[$ap,#16]
258 bl __ecp_nistz256_add // ret = a+a // 2*a
262 .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
264 // void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]);
265 .globl ecp_nistz256_mul_by_3
266 .type ecp_nistz256_mul_by_3,%function
268 ecp_nistz256_mul_by_3:
269 stp x29,x30,[sp,#-16]!
272 ldp $acc0,$acc1,[$ap]
273 ldp $acc2,$acc3,[$ap,#16]
285 bl __ecp_nistz256_add // ret = a+a // 2*a
292 bl __ecp_nistz256_add // ret += a // 2*a+a=3*a
296 .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
298 // void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4],
299 // const BN_ULONG x2[4]);
300 .globl ecp_nistz256_sub
301 .type ecp_nistz256_sub,%function
304 stp x29,x30,[sp,#-16]!
307 ldp $acc0,$acc1,[$ap]
308 ldp $acc2,$acc3,[$ap,#16]
312 bl __ecp_nistz256_sub_from
316 .size ecp_nistz256_sub,.-ecp_nistz256_sub
318 // void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]);
319 .globl ecp_nistz256_neg
320 .type ecp_nistz256_neg,%function
323 stp x29,x30,[sp,#-16]!
327 mov $acc0,xzr // a = 0
334 bl __ecp_nistz256_sub_from
338 .size ecp_nistz256_neg,.-ecp_nistz256_neg
340 // note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded
341 // to $a0-$a3 and b[0] - to $bi
342 .type __ecp_nistz256_mul_mont,%function
344 __ecp_nistz256_mul_mont:
345 mul $acc0,$a0,$bi // a[0]*b[0]
348 mul $acc1,$a1,$bi // a[1]*b[0]
351 mul $acc2,$a2,$bi // a[2]*b[0]
354 mul $acc3,$a3,$bi // a[3]*b[0]
356 ldr $bi,[$bp,#8] // b[1]
358 adds $acc1,$acc1,$t0 // accumulate high parts of multiplication
366 for($i=1;$i<4;$i++) {
367 # Reduction iteration is normally performed by accumulating
368 # result of multiplication of modulus by "magic" digit [and
369 # omitting least significant word, which is guaranteed to
370 # be 0], but thanks to special form of modulus and "magic"
371 # digit being equal to least significant word, it can be
372 # performed with additions and subtractions alone. Indeed:
374 # ffff0001.00000000.0000ffff.ffffffff
376 # + xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
378 # Now observing that ff..ff*x = (2^n-1)*x = 2^n*x-x, we
381 # xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.abcdefgh
382 # + abcdefgh.abcdefgh.0000abcd.efgh0000.00000000
383 # - 0000abcd.efgh0000.00000000.00000000.abcdefgh
385 # or marking redundant operations:
387 # xxxxxxxx.xxxxxxxx.xxxxxxxx.xxxxxxxx.--------
388 # + abcdefgh.abcdefgh.0000abcd.efgh0000.--------
389 # - 0000abcd.efgh0000.--------.--------.--------
392 subs $t2,$acc0,$t0 // "*0xffff0001"
394 adds $acc0,$acc1,$t0 // +=acc[0]<<96 and omit acc[0]
395 mul $t0,$a0,$bi // lo(a[0]*b[i])
397 mul $t1,$a1,$bi // lo(a[1]*b[i])
398 adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001
399 mul $t2,$a2,$bi // lo(a[2]*b[i])
401 mul $t3,$a3,$bi // lo(a[3]*b[i])
404 adds $acc0,$acc0,$t0 // accumulate low parts of multiplication
405 umulh $t0,$a0,$bi // hi(a[0]*b[i])
407 umulh $t1,$a1,$bi // hi(a[1]*b[i])
409 umulh $t2,$a2,$bi // hi(a[2]*b[i])
411 umulh $t3,$a3,$bi // hi(a[3]*b[i])
414 $code.=<<___ if ($i<3);
415 ldr $bi,[$bp,#8*($i+1)] // b[$i+1]
418 adds $acc1,$acc1,$t0 // accumulate high parts of multiplication
429 subs $t2,$acc0,$t0 // "*0xffff0001"
431 adds $acc0,$acc1,$t0 // +=acc[0]<<96 and omit acc[0]
433 adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001
437 adds $t0,$acc0,#1 // subs $t0,$acc0,#-1 // tmp = ret-modulus
438 sbcs $t1,$acc1,$poly1
440 sbcs $t3,$acc3,$poly3
441 sbcs xzr,$acc4,xzr // did it borrow?
443 csel $acc0,$acc0,$t0,lo // ret = borrow ? ret : ret-modulus
444 csel $acc1,$acc1,$t1,lo
445 csel $acc2,$acc2,$t2,lo
446 stp $acc0,$acc1,[$rp]
447 csel $acc3,$acc3,$t3,lo
448 stp $acc2,$acc3,[$rp,#16]
451 .size __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont
453 // note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded
455 .type __ecp_nistz256_sqr_mont,%function
457 __ecp_nistz256_sqr_mont:
458 // | | | | | |a1*a0| |
459 // | | | | |a2*a0| | |
460 // | |a3*a2|a3*a0| | | |
461 // | | | |a2*a1| | | |
462 // | | |a3*a1| | | | |
463 // *| | | | | | | | 2|
464 // +|a3*a3|a2*a2|a1*a1|a0*a0|
465 // |--+--+--+--+--+--+--+--|
466 // |A7|A6|A5|A4|A3|A2|A1|A0|, where Ax is $accx, i.e. follow $accx
468 // "can't overflow" below mark carrying into high part of
469 // multiplication result, which can't overflow, because it
470 // can never be all ones.
472 mul $acc1,$a1,$a0 // a[1]*a[0]
474 mul $acc2,$a2,$a0 // a[2]*a[0]
476 mul $acc3,$a3,$a0 // a[3]*a[0]
479 adds $acc2,$acc2,$t1 // accumulate high parts of multiplication
480 mul $t0,$a2,$a1 // a[2]*a[1]
483 mul $t2,$a3,$a1 // a[3]*a[1]
485 adc $acc4,$acc4,xzr // can't overflow
487 mul $acc5,$a3,$a2 // a[3]*a[2]
490 adds $t1,$t1,$t2 // accumulate high parts of multiplication
491 mul $acc0,$a0,$a0 // a[0]*a[0]
492 adc $t2,$t3,xzr // can't overflow
494 adds $acc3,$acc3,$t0 // accumulate low parts of multiplication
497 mul $t1,$a1,$a1 // a[1]*a[1]
500 adc $acc6,$acc6,xzr // can't overflow
502 adds $acc1,$acc1,$acc1 // acc[1-6]*=2
503 mul $t2,$a2,$a2 // a[2]*a[2]
504 adcs $acc2,$acc2,$acc2
506 adcs $acc3,$acc3,$acc3
507 mul $t3,$a3,$a3 // a[3]*a[3]
508 adcs $acc4,$acc4,$acc4
510 adcs $acc5,$acc5,$acc5
511 adcs $acc6,$acc6,$acc6
514 adds $acc1,$acc1,$a0 // +a[i]*a[i]
524 for($i=0;$i<3;$i++) { # reductions, see commentary in
525 # multiplication for details
527 subs $t2,$acc0,$t0 // "*0xffff0001"
529 adds $acc0,$acc1,$t0 // +=acc[0]<<96 and omit acc[0]
532 adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001
534 adc $acc3,$t3,xzr // can't overflow
538 subs $t2,$acc0,$t0 // "*0xffff0001"
540 adds $acc0,$acc1,$t0 // +=acc[0]<<96 and omit acc[0]
542 adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001
543 adc $acc3,$t3,xzr // can't overflow
545 adds $acc0,$acc0,$acc4 // accumulate upper half
546 adcs $acc1,$acc1,$acc5
547 adcs $acc2,$acc2,$acc6
548 adcs $acc3,$acc3,$acc7
551 adds $t0,$acc0,#1 // subs $t0,$acc0,#-1 // tmp = ret-modulus
552 sbcs $t1,$acc1,$poly1
554 sbcs $t3,$acc3,$poly3
555 sbcs xzr,$acc4,xzr // did it borrow?
557 csel $acc0,$acc0,$t0,lo // ret = borrow ? ret : ret-modulus
558 csel $acc1,$acc1,$t1,lo
559 csel $acc2,$acc2,$t2,lo
560 stp $acc0,$acc1,[$rp]
561 csel $acc3,$acc3,$t3,lo
562 stp $acc2,$acc3,[$rp,#16]
565 .size __ecp_nistz256_sqr_mont,.-__ecp_nistz256_sqr_mont
567 // Note that __ecp_nistz256_add expects both input vectors pre-loaded to
568 // $a0-$a3 and $t0-$t3. This is done because it's used in multiple
569 // contexts, e.g. in multiplication by 2 and 3...
570 .type __ecp_nistz256_add,%function
573 adds $acc0,$acc0,$t0 // ret = a+b
577 adc $ap,xzr,xzr // zap $ap
579 adds $t0,$acc0,#1 // subs $t0,$a0,#-1 // tmp = ret-modulus
580 sbcs $t1,$acc1,$poly1
583 cmp $ap,xzr // did addition carry?
585 csel $acc0,$acc0,$t0,eq // ret = carry ? ret-modulus : ret
586 csel $acc1,$acc1,$t1,eq
587 csel $acc2,$acc2,$t2,eq
588 stp $acc0,$acc1,[$rp]
589 csel $acc3,$acc3,$t3,eq
590 stp $acc2,$acc3,[$rp,#16]
593 .size __ecp_nistz256_add,.-__ecp_nistz256_add
595 .type __ecp_nistz256_sub_from,%function
597 __ecp_nistz256_sub_from:
599 ldp $t2,$t3,[$bp,#16]
600 subs $acc0,$acc0,$t0 // ret = a-b
604 sbc $ap,xzr,xzr // zap $ap
606 subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = ret+modulus
607 adcs $t1,$acc1,$poly1
610 cmp $ap,xzr // did subtraction borrow?
612 csel $acc0,$acc0,$t0,eq // ret = borrow ? ret+modulus : ret
613 csel $acc1,$acc1,$t1,eq
614 csel $acc2,$acc2,$t2,eq
615 stp $acc0,$acc1,[$rp]
616 csel $acc3,$acc3,$t3,eq
617 stp $acc2,$acc3,[$rp,#16]
620 .size __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from
622 .type __ecp_nistz256_sub_morf,%function
624 __ecp_nistz256_sub_morf:
626 ldp $t2,$t3,[$bp,#16]
627 subs $acc0,$t0,$acc0 // ret = b-a
631 sbc $ap,xzr,xzr // zap $ap
633 subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = ret+modulus
634 adcs $t1,$acc1,$poly1
637 cmp $ap,xzr // did subtraction borrow?
639 csel $acc0,$acc0,$t0,eq // ret = borrow ? ret+modulus : ret
640 csel $acc1,$acc1,$t1,eq
641 csel $acc2,$acc2,$t2,eq
642 stp $acc0,$acc1,[$rp]
643 csel $acc3,$acc3,$t3,eq
644 stp $acc2,$acc3,[$rp,#16]
647 .size __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf
649 .type __ecp_nistz256_div_by_2,%function
651 __ecp_nistz256_div_by_2:
652 subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = a+modulus
653 adcs $t1,$acc1,$poly1
655 adcs $t3,$acc3,$poly3
656 adc $ap,xzr,xzr // zap $ap
657 tst $acc0,#1 // is a even?
659 csel $acc0,$acc0,$t0,eq // ret = even ? a : a+modulus
660 csel $acc1,$acc1,$t1,eq
661 csel $acc2,$acc2,$t2,eq
662 csel $acc3,$acc3,$t3,eq
665 lsr $acc0,$acc0,#1 // ret >>= 1
666 orr $acc0,$acc0,$acc1,lsl#63
668 orr $acc1,$acc1,$acc2,lsl#63
670 orr $acc2,$acc2,$acc3,lsl#63
672 stp $acc0,$acc1,[$rp]
673 orr $acc3,$acc3,$ap,lsl#63
674 stp $acc2,$acc3,[$rp,#16]
677 .size __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2
679 ########################################################################
680 # following subroutines are "literal" implemetation of those found in
683 ########################################################################
684 # void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp);
687 my ($S,$M,$Zsqr,$tmp0)=map(32*$_,(0..3));
688 # above map() describes stack layout with 4 temporary
689 # 256-bit vectors on top.
690 my ($rp_real,$ap_real) = map("x$_",(21,22));
693 .globl ecp_nistz256_point_double
694 .type ecp_nistz256_point_double,%function
696 ecp_nistz256_point_double:
697 stp x29,x30,[sp,#-48]!
703 ldp $acc0,$acc1,[$ap,#32]
705 ldp $acc2,$acc3,[$ap,#48]
711 ldp $a0,$a1,[$ap_real,#64] // forward load for p256_sqr_mont
714 ldp $a2,$a3,[$ap_real,#64+16]
716 bl __ecp_nistz256_add // p256_mul_by_2(S, in_y);
719 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z);
721 ldp $t0,$t1,[$ap_real]
722 ldp $t2,$t3,[$ap_real,#16]
723 mov $a0,$acc0 // put Zsqr aside for p256_sub
728 bl __ecp_nistz256_add // p256_add(M, Zsqr, in_x);
731 mov $acc0,$a0 // restore Zsqr
733 ldp $a0,$a1,[sp,#$S] // forward load for p256_sqr_mont
736 ldp $a2,$a3,[sp,#$S+16]
738 bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr);
741 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S);
743 ldr $bi,[$ap_real,#32]
744 ldp $a0,$a1,[$ap_real,#64]
745 ldp $a2,$a3,[$ap_real,#64+16]
748 bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y);
752 ldp $a0,$a1,[sp,#$S] // forward load for p256_sqr_mont
755 ldp $a2,$a3,[sp,#$S+16]
757 bl __ecp_nistz256_add // p256_mul_by_2(res_z, tmp0);
760 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S);
762 ldr $bi,[sp,#$Zsqr] // forward load for p256_mul_mont
764 ldp $a2,$a3,[sp,#$M+16]
766 bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0);
770 bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr);
772 mov $t0,$acc0 // duplicate M
776 mov $a0,$acc0 // put M aside
781 bl __ecp_nistz256_add
782 mov $t0,$a0 // restore M
784 ldr $bi,[$ap_real] // forward load for p256_mul_mont
788 ldp $a2,$a3,[sp,#$S+16]
789 bl __ecp_nistz256_add // p256_mul_by_3(M, M);
793 bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x);
797 ldp $a0,$a1,[sp,#$M] // forward load for p256_sqr_mont
800 ldp $a2,$a3,[sp,#$M+16]
802 bl __ecp_nistz256_add // p256_mul_by_2(tmp0, S);
805 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M);
808 bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0);
812 bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x);
815 mov $a0,$acc0 // copy S
820 bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M);
824 bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y);
826 add sp,x29,#0 // destroy frame
827 ldp x19,x20,[x29,#16]
828 ldp x21,x22,[x29,#32]
831 .size ecp_nistz256_point_double,.-ecp_nistz256_point_double
835 ########################################################################
836 # void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1,
837 # const P256_POINT *in2);
839 my ($res_x,$res_y,$res_z,
840 $H,$Hsqr,$R,$Rsqr,$Hcub,
841 $U1,$U2,$S1,$S2)=map(32*$_,(0..11));
842 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
843 # above map() describes stack layout with 12 temporary
844 # 256-bit vectors on top.
845 my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("x$_",(21..26));
848 .globl ecp_nistz256_point_add
849 .type ecp_nistz256_point_add,%function
851 ecp_nistz256_point_add:
852 stp x29,x30,[sp,#-80]!
861 ldp $a2,$a3,[$bp,#16]
862 ldp $t0,$t1,[$bp,#32]
863 ldp $t2,$t3,[$bp,#48]
869 ldp $acc0,$acc1,[$ap]
872 ldp $acc2,$acc3,[$ap,#16]
875 ldp $t0,$t1,[$ap,#32]
876 orr $in2infty,$a0,$t2
878 ldp $t2,$t3,[$ap,#48]
879 csetm $in2infty,ne // !in2infty
881 ldp $a0,$a1,[$bp_real,#64] // forward load for p256_sqr_mont
882 orr $acc0,$acc0,$acc1
883 orr $acc2,$acc2,$acc3
884 ldp $a2,$a3,[$bp_real,#64+16]
887 orr $acc0,$acc0,$acc2
889 orr $in1infty,$acc0,$t0
893 csetm $in1infty,ne // !in1infty
896 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z);
898 ldp $a0,$a1,[$ap_real,#64]
899 ldp $a2,$a3,[$ap_real,#64+16]
901 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z);
903 ldr $bi,[$bp_real,#64]
904 ldp $a0,$a1,[sp,#$Z2sqr]
905 ldp $a2,$a3,[sp,#$Z2sqr+16]
908 bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z);
910 ldr $bi,[$ap_real,#64]
911 ldp $a0,$a1,[sp,#$Z1sqr]
912 ldp $a2,$a3,[sp,#$Z1sqr+16]
915 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z);
917 ldr $bi,[$ap_real,#32]
918 ldp $a0,$a1,[sp,#$S1]
919 ldp $a2,$a3,[sp,#$S1+16]
922 bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y);
924 ldr $bi,[$bp_real,#32]
925 ldp $a0,$a1,[sp,#$S2]
926 ldp $a2,$a3,[sp,#$S2+16]
929 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y);
932 ldr $bi,[sp,#$Z2sqr] // forward load for p256_mul_mont
933 ldp $a0,$a1,[$ap_real]
934 ldp $a2,$a3,[$ap_real,#16]
936 bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1);
938 orr $acc0,$acc0,$acc1 // see if result is zero
939 orr $acc2,$acc2,$acc3
940 orr $temp,$acc0,$acc2
944 bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr);
947 ldp $a0,$a1,[$bp_real]
948 ldp $a2,$a3,[$bp_real,#16]
951 bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr);
954 ldp $a0,$a1,[sp,#$R] // forward load for p256_sqr_mont
955 ldp $a2,$a3,[sp,#$R+16]
957 bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1);
959 orr $acc0,$acc0,$acc1 // see if result is zero
960 orr $acc2,$acc2,$acc3
961 orr $acc0,$acc0,$acc2
963 b.ne .Ladd_proceed // is_equal(U1,U2)?
965 tst $in1infty,$in2infty
966 b.eq .Ladd_proceed // (in1infty || in2infty)?
969 b.eq .Ladd_proceed // is_equal(S1,S2)?
973 stp $a0,$a1,[$rp_real]
974 stp $a0,$a1,[$rp_real,#16]
975 stp $a0,$a1,[$rp_real,#32]
976 stp $a0,$a1,[$rp_real,#48]
977 stp $a0,$a1,[$rp_real,#64]
978 stp $a0,$a1,[$rp_real,#80]
984 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R);
986 ldr $bi,[$ap_real,#64]
988 ldp $a2,$a3,[sp,#$H+16]
991 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z);
994 ldp $a2,$a3,[sp,#$H+16]
996 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H);
998 ldr $bi,[$bp_real,#64]
999 ldp $a0,$a1,[sp,#$res_z]
1000 ldp $a2,$a3,[sp,#$res_z+16]
1001 add $bp,$bp_real,#64
1003 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z);
1006 ldp $a0,$a1,[sp,#$Hsqr]
1007 ldp $a2,$a3,[sp,#$Hsqr+16]
1010 bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H);
1013 ldp $a0,$a1,[sp,#$U1]
1014 ldp $a2,$a3,[sp,#$U1+16]
1017 bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr);
1024 bl __ecp_nistz256_add // p256_mul_by_2(Hsqr, U2);
1028 bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr);
1031 bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub);
1034 ldr $bi,[sp,#$Hcub] // forward load for p256_mul_mont
1035 ldp $a0,$a1,[sp,#$S1]
1036 ldp $a2,$a3,[sp,#$S1+16]
1038 bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x);
1042 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub);
1045 ldp $a0,$a1,[sp,#$res_y]
1046 ldp $a2,$a3,[sp,#$res_y+16]
1049 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R);
1052 bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2);
1054 ldp $a0,$a1,[sp,#$res_x] // res
1055 ldp $a2,$a3,[sp,#$res_x+16]
1056 ldp $t0,$t1,[$bp_real] // in2
1057 ldp $t2,$t3,[$bp_real,#16]
1059 for($i=0;$i<64;$i+=32) { # conditional moves
1061 ldp $acc0,$acc1,[$ap_real,#$i] // in1
1062 cmp $in1infty,#0 // !$in1intfy, remember?
1063 ldp $acc2,$acc3,[$ap_real,#$i+16]
1066 ldp $a0,$a1,[sp,#$res_x+$i+32] // res
1069 cmp $in2infty,#0 // !$in2intfy, remember?
1070 ldp $a2,$a3,[sp,#$res_x+$i+48]
1071 csel $acc0,$t0,$acc0,ne
1072 csel $acc1,$t1,$acc1,ne
1073 ldp $t0,$t1,[$bp_real,#$i+32] // in2
1074 csel $acc2,$t2,$acc2,ne
1075 csel $acc3,$t3,$acc3,ne
1076 ldp $t2,$t3,[$bp_real,#$i+48]
1077 stp $acc0,$acc1,[$rp_real,#$i]
1078 stp $acc2,$acc3,[$rp_real,#$i+16]
1082 ldp $acc0,$acc1,[$ap_real,#$i] // in1
1083 cmp $in1infty,#0 // !$in1intfy, remember?
1084 ldp $acc2,$acc3,[$ap_real,#$i+16]
1089 cmp $in2infty,#0 // !$in2intfy, remember?
1090 csel $acc0,$t0,$acc0,ne
1091 csel $acc1,$t1,$acc1,ne
1092 csel $acc2,$t2,$acc2,ne
1093 csel $acc3,$t3,$acc3,ne
1094 stp $acc0,$acc1,[$rp_real,#$i]
1095 stp $acc2,$acc3,[$rp_real,#$i+16]
1098 add sp,x29,#0 // destroy frame
1099 ldp x19,x20,[x29,#16]
1100 ldp x21,x22,[x29,#32]
1101 ldp x23,x24,[x29,#48]
1102 ldp x25,x26,[x29,#64]
1103 ldp x29,x30,[sp],#80
1105 .size ecp_nistz256_point_add,.-ecp_nistz256_point_add
1109 ########################################################################
1110 # void ecp_nistz256_point_add_affine(P256_POINT *out,const P256_POINT *in1,
1111 # const P256_POINT_AFFINE *in2);
1113 my ($res_x,$res_y,$res_z,
1114 $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..9));
1116 # above map() describes stack layout with 10 temporary
1117 # 256-bit vectors on top.
1118 my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("x$_",(21..26));
1121 .globl ecp_nistz256_point_add_affine
1122 .type ecp_nistz256_point_add_affine,%function
1124 ecp_nistz256_point_add_affine:
1125 stp x29,x30,[sp,#-80]!
1127 stp x19,x20,[sp,#16]
1128 stp x21,x22,[sp,#32]
1129 stp x23,x24,[sp,#48]
1130 stp x25,x26,[sp,#64]
1137 ldr $poly3,.Lpoly+24
1140 ldp $a2,$a3,[$ap,#16]
1141 ldp $t0,$t1,[$ap,#32]
1142 ldp $t2,$t3,[$ap,#48]
1149 orr $in1infty,$a0,$t0
1151 csetm $in1infty,ne // !in1infty
1154 ldp $a2,$a3,[$bp,#16]
1155 ldp $t0,$t1,[$bp,#32]
1156 ldp $t2,$t3,[$bp,#48]
1163 orr $in2infty,$a0,$t0
1165 csetm $in2infty,ne // !in2infty
1167 ldp $a0,$a1,[$ap_real,#64]
1168 ldp $a2,$a3,[$ap_real,#64+16]
1170 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z);
1179 bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x);
1182 ldr $bi,[$ap_real,#64] // forward load for p256_mul_mont
1183 ldp $a0,$a1,[sp,#$Z1sqr]
1184 ldp $a2,$a3,[sp,#$Z1sqr+16]
1186 bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x);
1188 add $bp,$ap_real,#64
1190 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z);
1192 ldr $bi,[$ap_real,#64]
1193 ldp $a0,$a1,[sp,#$H]
1194 ldp $a2,$a3,[sp,#$H+16]
1195 add $bp,$ap_real,#64
1197 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z);
1199 ldr $bi,[$bp_real,#32]
1200 ldp $a0,$a1,[sp,#$S2]
1201 ldp $a2,$a3,[sp,#$S2+16]
1202 add $bp,$bp_real,#32
1204 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y);
1206 add $bp,$ap_real,#32
1207 ldp $a0,$a1,[sp,#$H] // forward load for p256_sqr_mont
1208 ldp $a2,$a3,[sp,#$H+16]
1210 bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y);
1213 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H);
1215 ldp $a0,$a1,[sp,#$R]
1216 ldp $a2,$a3,[sp,#$R+16]
1218 bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R);
1221 ldp $a0,$a1,[sp,#$Hsqr]
1222 ldp $a2,$a3,[sp,#$Hsqr+16]
1225 bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H);
1228 ldp $a0,$a1,[sp,#$Hsqr]
1229 ldp $a2,$a3,[sp,#$Hsqr+16]
1232 bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr);
1239 bl __ecp_nistz256_add // p256_mul_by_2(Hsqr, U2);
1243 bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr);
1246 bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub);
1249 ldr $bi,[$ap_real,#32] // forward load for p256_mul_mont
1250 ldp $a0,$a1,[sp,#$Hcub]
1251 ldp $a2,$a3,[sp,#$Hcub+16]
1253 bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x);
1255 add $bp,$ap_real,#32
1257 bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub);
1260 ldp $a0,$a1,[sp,#$res_y]
1261 ldp $a2,$a3,[sp,#$res_y+16]
1264 bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R);
1267 bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2);
1269 ldp $a0,$a1,[sp,#$res_x] // res
1270 ldp $a2,$a3,[sp,#$res_x+16]
1271 ldp $t0,$t1,[$bp_real] // in2
1272 ldp $t2,$t3,[$bp_real,#16]
1274 for($i=0;$i<64;$i+=32) { # conditional moves
1276 ldp $acc0,$acc1,[$ap_real,#$i] // in1
1277 cmp $in1infty,#0 // !$in1intfy, remember?
1278 ldp $acc2,$acc3,[$ap_real,#$i+16]
1281 ldp $a0,$a1,[sp,#$res_x+$i+32] // res
1284 cmp $in2infty,#0 // !$in2intfy, remember?
1285 ldp $a2,$a3,[sp,#$res_x+$i+48]
1286 csel $acc0,$t0,$acc0,ne
1287 csel $acc1,$t1,$acc1,ne
1288 ldp $t0,$t1,[$bp_real,#$i+32] // in2
1289 csel $acc2,$t2,$acc2,ne
1290 csel $acc3,$t3,$acc3,ne
1291 ldp $t2,$t3,[$bp_real,#$i+48]
1292 stp $acc0,$acc1,[$rp_real,#$i]
1293 stp $acc2,$acc3,[$rp_real,#$i+16]
1297 ldp $acc0,$acc1,[$ap_real,#$i] // in1
1298 cmp $in1infty,#0 // !$in1intfy, remember?
1299 ldp $acc2,$acc3,[$ap_real,#$i+16]
1304 cmp $in2infty,#0 // !$in2intfy, remember?
1305 csel $acc0,$t0,$acc0,ne
1306 csel $acc1,$t1,$acc1,ne
1307 csel $acc2,$t2,$acc2,ne
1308 csel $acc3,$t3,$acc3,ne
1309 stp $acc0,$acc1,[$rp_real,#$i]
1310 stp $acc2,$acc3,[$rp_real,#$i+16]
1312 add sp,x29,#0 // destroy frame
1313 ldp x19,x20,[x29,#16]
1314 ldp x21,x22,[x29,#32]
1315 ldp x23,x24,[x29,#48]
1316 ldp x25,x26,[x29,#64]
1317 ldp x29,x30,[sp],#80
1319 .size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine
1323 ########################################################################
1324 # scatter-gather subroutines
1326 my ($out,$inp,$index,$mask)=map("x$_",(0..3));
1328 // void ecp_nistz256_scatter_w5(void *x0,const P256_POINT *x1,
1330 .globl ecp_nistz256_scatter_w5
1331 .type ecp_nistz256_scatter_w5,%function
1333 ecp_nistz256_scatter_w5:
1334 stp x29,x30,[sp,#-16]!
1337 add $out,$out,$index,lsl#2
1339 ldp x4,x5,[$inp] // X
1340 ldp x6,x7,[$inp,#16]
1341 str w4,[$out,#64*0-4]
1343 str w5,[$out,#64*1-4]
1345 str w6,[$out,#64*2-4]
1347 str w7,[$out,#64*3-4]
1349 str w4,[$out,#64*4-4]
1350 str w5,[$out,#64*5-4]
1351 str w6,[$out,#64*6-4]
1352 str w7,[$out,#64*7-4]
1355 ldp x4,x5,[$inp,#32] // Y
1356 ldp x6,x7,[$inp,#48]
1357 str w4,[$out,#64*0-4]
1359 str w5,[$out,#64*1-4]
1361 str w6,[$out,#64*2-4]
1363 str w7,[$out,#64*3-4]
1365 str w4,[$out,#64*4-4]
1366 str w5,[$out,#64*5-4]
1367 str w6,[$out,#64*6-4]
1368 str w7,[$out,#64*7-4]
1371 ldp x4,x5,[$inp,#64] // Z
1372 ldp x6,x7,[$inp,#80]
1373 str w4,[$out,#64*0-4]
1375 str w5,[$out,#64*1-4]
1377 str w6,[$out,#64*2-4]
1379 str w7,[$out,#64*3-4]
1381 str w4,[$out,#64*4-4]
1382 str w5,[$out,#64*5-4]
1383 str w6,[$out,#64*6-4]
1384 str w7,[$out,#64*7-4]
1388 .size ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
1390 // void ecp_nistz256_gather_w5(P256_POINT *x0,const void *x1,
1392 .globl ecp_nistz256_gather_w5
1393 .type ecp_nistz256_gather_w5,%function
1395 ecp_nistz256_gather_w5:
1396 stp x29,x30,[sp,#-16]!
1401 add $index,$index,x3
1402 add $inp,$inp,$index,lsl#2
1410 ldr w10,[$inp,#64*6]
1411 ldr w11,[$inp,#64*7]
1415 orr x6,x6,x10,lsl#32
1416 orr x7,x7,x11,lsl#32
1421 stp x4,x5,[$out] // X
1422 stp x6,x7,[$out,#16]
1430 ldr w10,[$inp,#64*6]
1431 ldr w11,[$inp,#64*7]
1435 orr x6,x6,x10,lsl#32
1436 orr x7,x7,x11,lsl#32
1441 stp x4,x5,[$out,#32] // Y
1442 stp x6,x7,[$out,#48]
1450 ldr w10,[$inp,#64*6]
1451 ldr w11,[$inp,#64*7]
1454 orr x6,x6,x10,lsl#32
1455 orr x7,x7,x11,lsl#32
1460 stp x4,x5,[$out,#64] // Z
1461 stp x6,x7,[$out,#80]
1465 .size ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
1467 // void ecp_nistz256_scatter_w7(void *x0,const P256_POINT_AFFINE *x1,
1469 .globl ecp_nistz256_scatter_w7
1470 .type ecp_nistz256_scatter_w7,%function
1472 ecp_nistz256_scatter_w7:
1473 stp x29,x30,[sp,#-16]!
1476 add $out,$out,$index
1480 subs $index,$index,#1
1481 prfm pstl1strm,[$out,#4096+64*0]
1482 prfm pstl1strm,[$out,#4096+64*1]
1483 prfm pstl1strm,[$out,#4096+64*2]
1484 prfm pstl1strm,[$out,#4096+64*3]
1485 prfm pstl1strm,[$out,#4096+64*4]
1486 prfm pstl1strm,[$out,#4096+64*5]
1487 prfm pstl1strm,[$out,#4096+64*6]
1488 prfm pstl1strm,[$out,#4096+64*7]
1489 strb w3,[$out,#64*0-1]
1491 strb w3,[$out,#64*1-1]
1493 strb w3,[$out,#64*2-1]
1495 strb w3,[$out,#64*3-1]
1497 strb w3,[$out,#64*4-1]
1499 strb w3,[$out,#64*5-1]
1501 strb w3,[$out,#64*6-1]
1503 strb w3,[$out,#64*7-1]
1505 b.ne .Loop_scatter_w7
1509 .size ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
1511 // void ecp_nistz256_gather_w7(P256_POINT_AFFINE *x0,const void *x1,
1513 .globl ecp_nistz256_gather_w7
1514 .type ecp_nistz256_gather_w7,%function
1516 ecp_nistz256_gather_w7:
1517 stp x29,x30,[sp,#-16]!
1522 add $index,$index,x3
1523 add $inp,$inp,$index
1527 ldrb w4,[$inp,#64*0]
1528 prfm pldl1strm,[$inp,#4096+64*0]
1529 subs $index,$index,#1
1530 ldrb w5,[$inp,#64*1]
1531 prfm pldl1strm,[$inp,#4096+64*1]
1532 ldrb w6,[$inp,#64*2]
1533 prfm pldl1strm,[$inp,#4096+64*2]
1534 ldrb w7,[$inp,#64*3]
1535 prfm pldl1strm,[$inp,#4096+64*3]
1536 ldrb w8,[$inp,#64*4]
1537 prfm pldl1strm,[$inp,#4096+64*4]
1538 ldrb w9,[$inp,#64*5]
1539 prfm pldl1strm,[$inp,#4096+64*5]
1540 ldrb w10,[$inp,#64*6]
1541 prfm pldl1strm,[$inp,#4096+64*6]
1542 ldrb w11,[$inp,#64*7]
1543 prfm pldl1strm,[$inp,#4096+64*7]
1549 orr x10,x10,x11,lsl#8
1551 orr x4,x4,x10,lsl#48
1554 b.ne .Loop_gather_w7
1558 .size ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
1562 foreach (split("\n",$code)) {
1563 s/\`([^\`]*)\`/eval $1/ge;
1567 close STDOUT; # enforce flush