2 # Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 ##############################################################################
12 # Copyright 2014 Intel Corporation #
14 # Licensed under the Apache License, Version 2.0 (the "License"); #
15 # you may not use this file except in compliance with the License. #
16 # You may obtain a copy of the License at #
18 # http://www.apache.org/licenses/LICENSE-2.0 #
20 # Unless required by applicable law or agreed to in writing, software #
21 # distributed under the License is distributed on an "AS IS" BASIS, #
22 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
23 # See the License for the specific language governing permissions and #
24 # limitations under the License. #
26 ##############################################################################
28 # Developers and authors: #
29 # Shay Gueron (1, 2), and Vlad Krasnov (1) #
30 # (1) Intel Corporation, Israel Development Center #
31 # (2) University of Haifa #
33 # S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with#
36 ##############################################################################
38 # Further optimization by <appro@openssl.org>:
40 # this/original with/without -DECP_NISTZ256_ASM(*)
41 # Opteron +12-49% +110-150%
42 # Bulldozer +14-45% +175-210%
44 # Westmere +12-34% +80-87%
45 # Sandy Bridge +9-35% +110-120%
46 # Ivy Bridge +9-35% +110-125%
47 # Haswell +8-37% +140-160%
48 # Broadwell +18-58% +145-210%
49 # Atom +15-50% +130-180%
50 # VIA Nano +43-160% +300-480%
52 # (*) "without -DECP_NISTZ256_ASM" refers to build with
53 # "enable-ec_nistp_64_gcc_128";
55 # Ranges denote minimum and maximum improvement coefficients depending
56 # on benchmark. Lower coefficients are for ECDSA sign, relatively fastest
57 # server-side operation. Keep in mind that +100% means 2x improvement.
61 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
63 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
65 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
66 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
67 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
68 die "can't locate x86_64-xlate.pl";
70 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
73 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
74 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
75 $avx = ($1>=2.19) + ($1>=2.22);
79 if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
80 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
81 $avx = ($1>=2.09) + ($1>=2.10);
85 if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
86 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
87 $avx = ($1>=10) + ($1>=11);
91 if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9])\.([0-9]+)/) {
92 my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10
93 $avx = ($ver>=3.0) + ($ver>=3.01);
99 .extern OPENSSL_ia32cap_P
104 .quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001
106 # 2^512 mod P precomputed for NIST P256 polynomial
108 .quad 0x0000000000000003, 0xfffffffbffffffff, 0xfffffffffffffffe, 0x00000004fffffffd
111 .long 1,1,1,1,1,1,1,1
113 .long 2,2,2,2,2,2,2,2
115 .long 3,3,3,3,3,3,3,3
117 .quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe
121 ################################################################################
122 # void ecp_nistz256_mul_by_2(uint64_t res[4], uint64_t a[4]);
124 my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11));
125 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13");
126 my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx");
130 .globl ecp_nistz256_mul_by_2
131 .type ecp_nistz256_mul_by_2,\@function,2
133 ecp_nistz256_mul_by_2:
140 add $a0, $a0 # a0:a3+a0:a3
144 lea .Lpoly(%rip), $a_ptr
171 .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
173 ################################################################################
174 # void ecp_nistz256_div_by_2(uint64_t res[4], uint64_t a[4]);
175 .globl ecp_nistz256_div_by_2
176 .type ecp_nistz256_div_by_2,\@function,2
178 ecp_nistz256_div_by_2:
187 lea .Lpoly(%rip), $a_ptr
198 xor $a_ptr, $a_ptr # borrow $a_ptr
207 mov $a1, $t0 # a0:a3>>1
231 .size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
233 ################################################################################
234 # void ecp_nistz256_mul_by_3(uint64_t res[4], uint64_t a[4]);
235 .globl ecp_nistz256_mul_by_3
236 .type ecp_nistz256_mul_by_3,\@function,2
238 ecp_nistz256_mul_by_3:
245 add $a0, $a0 # a0:a3+a0:a3
257 sbb .Lpoly+8*1(%rip), $a1
260 sbb .Lpoly+8*3(%rip), $a3
269 add 8*0($a_ptr), $a0 # a0:a3+=a_ptr[0:3]
279 sbb .Lpoly+8*1(%rip), $a1
282 sbb .Lpoly+8*3(%rip), $a3
297 .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
299 ################################################################################
300 # void ecp_nistz256_add(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
301 .globl ecp_nistz256_add
302 .type ecp_nistz256_add,\@function,3
313 lea .Lpoly(%rip), $a_ptr
343 .size ecp_nistz256_add,.-ecp_nistz256_add
345 ################################################################################
346 # void ecp_nistz256_sub(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
347 .globl ecp_nistz256_sub
348 .type ecp_nistz256_sub,\@function,3
359 lea .Lpoly(%rip), $a_ptr
389 .size ecp_nistz256_sub,.-ecp_nistz256_sub
391 ################################################################################
392 # void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]);
393 .globl ecp_nistz256_neg
394 .type ecp_nistz256_neg,\@function,2
411 lea .Lpoly(%rip), $a_ptr
435 .size ecp_nistz256_neg,.-ecp_nistz256_neg
439 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
440 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
441 my ($t0,$t1,$t2,$t3,$t4)=("%rcx","%rbp","%rbx","%rdx","%rax");
442 my ($poly1,$poly3)=($acc6,$acc7);
445 ################################################################################
446 # void ecp_nistz256_to_mont(
449 .globl ecp_nistz256_to_mont
450 .type ecp_nistz256_to_mont,\@function,2
452 ecp_nistz256_to_mont:
454 $code.=<<___ if ($addx);
456 and OPENSSL_ia32cap_P+8(%rip), %ecx
459 lea .LRR(%rip), $b_org
461 .size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont
463 ################################################################################
464 # void ecp_nistz256_mul_mont(
469 .globl ecp_nistz256_mul_mont
470 .type ecp_nistz256_mul_mont,\@function,3
472 ecp_nistz256_mul_mont:
474 $code.=<<___ if ($addx);
476 and OPENSSL_ia32cap_P+8(%rip), %ecx
487 $code.=<<___ if ($addx);
493 mov 8*0($b_org), %rax
494 mov 8*0($a_ptr), $acc1
495 mov 8*1($a_ptr), $acc2
496 mov 8*2($a_ptr), $acc3
497 mov 8*3($a_ptr), $acc4
499 call __ecp_nistz256_mul_montq
501 $code.=<<___ if ($addx);
507 mov 8*0($b_org), %rdx
508 mov 8*0($a_ptr), $acc1
509 mov 8*1($a_ptr), $acc2
510 mov 8*2($a_ptr), $acc3
511 mov 8*3($a_ptr), $acc4
512 lea -128($a_ptr), $a_ptr # control u-op density
514 call __ecp_nistz256_mul_montx
525 .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
527 .type __ecp_nistz256_mul_montq,\@abi-omnipotent
529 __ecp_nistz256_mul_montq:
530 ########################################################################
534 mov .Lpoly+8*1(%rip),$poly1
540 mov .Lpoly+8*3(%rip),$poly3
559 ########################################################################
560 # First reduction step
561 # Basically now we want to multiply acc[0] by p256,
562 # and add the result to the acc.
563 # Due to the special form of p256 we do some optimizations
565 # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0]
566 # then we add acc[0] and get acc[0] x 2^96
572 add $acc0, $acc1 # +=acc[0]<<96
575 mov 8*1($b_ptr), %rax
580 ########################################################################
613 ########################################################################
614 # Second reduction step
622 mov 8*2($b_ptr), %rax
627 ########################################################################
660 ########################################################################
661 # Third reduction step
669 mov 8*3($b_ptr), %rax
674 ########################################################################
707 ########################################################################
708 # Final reduction step
721 ########################################################################
722 # Branch-less conditional subtraction of P
723 sub \$-1, $acc4 # .Lpoly[0]
725 sbb $poly1, $acc5 # .Lpoly[1]
726 sbb \$0, $acc0 # .Lpoly[2]
728 sbb $poly3, $acc1 # .Lpoly[3]
733 mov $acc4, 8*0($r_ptr)
735 mov $acc5, 8*1($r_ptr)
737 mov $acc0, 8*2($r_ptr)
738 mov $acc1, 8*3($r_ptr)
741 .size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq
743 ################################################################################
744 # void ecp_nistz256_sqr_mont(
748 # we optimize the square according to S.Gueron and V.Krasnov,
749 # "Speeding up Big-Number Squaring"
750 .globl ecp_nistz256_sqr_mont
751 .type ecp_nistz256_sqr_mont,\@function,2
753 ecp_nistz256_sqr_mont:
755 $code.=<<___ if ($addx);
757 and OPENSSL_ia32cap_P+8(%rip), %ecx
767 $code.=<<___ if ($addx);
772 mov 8*0($a_ptr), %rax
773 mov 8*1($a_ptr), $acc6
774 mov 8*2($a_ptr), $acc7
775 mov 8*3($a_ptr), $acc0
777 call __ecp_nistz256_sqr_montq
779 $code.=<<___ if ($addx);
784 mov 8*0($a_ptr), %rdx
785 mov 8*1($a_ptr), $acc6
786 mov 8*2($a_ptr), $acc7
787 mov 8*3($a_ptr), $acc0
788 lea -128($a_ptr), $a_ptr # control u-op density
790 call __ecp_nistz256_sqr_montx
801 .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
803 .type __ecp_nistz256_sqr_montq,\@abi-omnipotent
805 __ecp_nistz256_sqr_montq:
807 mulq $acc6 # a[1]*a[0]
812 mulq $acc5 # a[0]*a[2]
818 mulq $acc5 # a[0]*a[3]
824 #################################
825 mulq $acc6 # a[1]*a[2]
831 mulq $acc6 # a[1]*a[3]
839 #################################
840 mulq $acc7 # a[2]*a[3]
843 mov 8*0($a_ptr), %rax
847 add $acc1, $acc1 # acc1:6<<1
857 mov 8*1($a_ptr), %rax
863 mov 8*2($a_ptr), %rax
870 mov 8*3($a_ptr), %rax
880 mov .Lpoly+8*1(%rip), $a_ptr
881 mov .Lpoly+8*3(%rip), $t1
883 ##########################################
890 add $acc0, $acc1 # +=acc[0]<<96
896 ##########################################
909 ##########################################
922 ###########################################
935 ############################################
936 # Add the rest of the acc
945 sub \$-1, $acc4 # .Lpoly[0]
947 sbb $a_ptr, $acc5 # .Lpoly[1]
948 sbb \$0, $acc6 # .Lpoly[2]
950 sbb $t1, $acc7 # .Lpoly[3]
955 mov $acc4, 8*0($r_ptr)
957 mov $acc5, 8*1($r_ptr)
959 mov $acc6, 8*2($r_ptr)
960 mov $acc7, 8*3($r_ptr)
963 .size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq
968 .type __ecp_nistz256_mul_montx,\@abi-omnipotent
970 __ecp_nistz256_mul_montx:
971 ########################################################################
973 mulx $acc1, $acc0, $acc1
974 mulx $acc2, $t0, $acc2
976 xor $acc5, $acc5 # cf=0
977 mulx $acc3, $t1, $acc3
978 mov .Lpoly+8*3(%rip), $poly3
980 mulx $acc4, $t0, $acc4
983 shlx $poly1,$acc0,$t1
985 shrx $poly1,$acc0,$t0
988 ########################################################################
989 # First reduction step
993 mulx $poly3, $t0, $t1
994 mov 8*1($b_ptr), %rdx
998 xor $acc0, $acc0 # $acc0=0,cf=0,of=0
1000 ########################################################################
1002 mulx 8*0+128($a_ptr), $t0, $t1
1006 mulx 8*1+128($a_ptr), $t0, $t1
1010 mulx 8*2+128($a_ptr), $t0, $t1
1014 mulx 8*3+128($a_ptr), $t0, $t1
1017 shlx $poly1, $acc1, $t0
1019 shrx $poly1, $acc1, $t1
1025 ########################################################################
1026 # Second reduction step
1030 mulx $poly3, $t0, $t1
1031 mov 8*2($b_ptr), %rdx
1035 xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0
1037 ########################################################################
1039 mulx 8*0+128($a_ptr), $t0, $t1
1043 mulx 8*1+128($a_ptr), $t0, $t1
1047 mulx 8*2+128($a_ptr), $t0, $t1
1051 mulx 8*3+128($a_ptr), $t0, $t1
1054 shlx $poly1, $acc2, $t0
1056 shrx $poly1, $acc2, $t1
1062 ########################################################################
1063 # Third reduction step
1067 mulx $poly3, $t0, $t1
1068 mov 8*3($b_ptr), %rdx
1072 xor $acc2, $acc2 # $acc2=0,cf=0,of=0
1074 ########################################################################
1076 mulx 8*0+128($a_ptr), $t0, $t1
1080 mulx 8*1+128($a_ptr), $t0, $t1
1084 mulx 8*2+128($a_ptr), $t0, $t1
1088 mulx 8*3+128($a_ptr), $t0, $t1
1091 shlx $poly1, $acc3, $t0
1093 shrx $poly1, $acc3, $t1
1099 ########################################################################
1100 # Fourth reduction step
1104 mulx $poly3, $t0, $t1
1106 mov .Lpoly+8*1(%rip), $poly1
1112 ########################################################################
1113 # Branch-less conditional subtraction of P
1116 sbb \$-1, $acc4 # .Lpoly[0]
1117 sbb $poly1, $acc5 # .Lpoly[1]
1118 sbb \$0, $acc0 # .Lpoly[2]
1120 sbb $poly3, $acc1 # .Lpoly[3]
1125 mov $acc4, 8*0($r_ptr)
1127 mov $acc5, 8*1($r_ptr)
1129 mov $acc0, 8*2($r_ptr)
1130 mov $acc1, 8*3($r_ptr)
1133 .size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx
1135 .type __ecp_nistz256_sqr_montx,\@abi-omnipotent
1137 __ecp_nistz256_sqr_montx:
1138 mulx $acc6, $acc1, $acc2 # a[0]*a[1]
1139 mulx $acc7, $t0, $acc3 # a[0]*a[2]
1142 mulx $acc0, $t1, $acc4 # a[0]*a[3]
1146 xor $acc5, $acc5 # $acc5=0,cf=0,of=0
1148 #################################
1149 mulx $acc7, $t0, $t1 # a[1]*a[2]
1153 mulx $acc0, $t0, $t1 # a[1]*a[3]
1159 #################################
1160 mulx $acc0, $t0, $acc6 # a[2]*a[3]
1161 mov 8*0+128($a_ptr), %rdx
1162 xor $acc7, $acc7 # $acc7=0,cf=0,of=0
1163 adcx $acc1, $acc1 # acc1:6<<1
1166 adox $acc7, $acc6 # of=0
1168 mulx %rdx, $acc0, $t1
1169 mov 8*1+128($a_ptr), %rdx
1174 mov 8*2+128($a_ptr), %rdx
1180 mov 8*3+128($a_ptr), %rdx
1190 shlx $a_ptr, $acc0, $t0
1192 shrx $a_ptr, $acc0, $t4
1193 mov .Lpoly+8*3(%rip), $t1
1199 mulx $t1, $t0, $acc0
1202 shlx $a_ptr, $acc1, $t0
1204 shrx $a_ptr, $acc1, $t4
1210 mulx $t1, $t0, $acc1
1213 shlx $a_ptr, $acc2, $t0
1215 shrx $a_ptr, $acc2, $t4
1221 mulx $t1, $t0, $acc2
1224 shlx $a_ptr, $acc3, $t0
1226 shrx $a_ptr, $acc3, $t4
1232 mulx $t1, $t0, $acc3
1237 adc $acc0, $acc4 # accumulate upper half
1238 mov .Lpoly+8*1(%rip), $a_ptr
1246 xor %eax, %eax # cf=0
1247 sbb \$-1, $acc4 # .Lpoly[0]
1249 sbb $a_ptr, $acc5 # .Lpoly[1]
1250 sbb \$0, $acc6 # .Lpoly[2]
1252 sbb $t1, $acc7 # .Lpoly[3]
1257 mov $acc4, 8*0($r_ptr)
1259 mov $acc5, 8*1($r_ptr)
1261 mov $acc6, 8*2($r_ptr)
1262 mov $acc7, 8*3($r_ptr)
1265 .size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx
1270 my ($r_ptr,$in_ptr)=("%rdi","%rsi");
1271 my ($acc0,$acc1,$acc2,$acc3)=map("%r$_",(8..11));
1272 my ($t0,$t1,$t2)=("%rcx","%r12","%r13");
1275 ################################################################################
1276 # void ecp_nistz256_from_mont(
1279 # This one performs Montgomery multiplication by 1, so we only need the reduction
1281 .globl ecp_nistz256_from_mont
1282 .type ecp_nistz256_from_mont,\@function,2
1284 ecp_nistz256_from_mont:
1288 mov 8*0($in_ptr), %rax
1289 mov .Lpoly+8*3(%rip), $t2
1290 mov 8*1($in_ptr), $acc1
1291 mov 8*2($in_ptr), $acc2
1292 mov 8*3($in_ptr), $acc3
1294 mov .Lpoly+8*1(%rip), $t1
1296 #########################################
1308 #########################################
1321 ##########################################
1334 ###########################################
1348 ###########################################
1349 # Branch-less conditional subtraction
1359 cmovnz $in_ptr, $acc1
1360 mov $acc0, 8*0($r_ptr)
1362 mov $acc1, 8*1($r_ptr)
1364 mov $acc2, 8*2($r_ptr)
1365 mov $acc3, 8*3($r_ptr)
1370 .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
1374 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1375 my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7));
1376 my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15));
1377 my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15));
1380 ################################################################################
1381 # void ecp_nistz256_scatter_w5(uint64_t *val, uint64_t *in_t, int index);
1382 .globl ecp_nistz256_scatter_w5
1383 .type ecp_nistz256_scatter_w5,\@abi-omnipotent
1385 ecp_nistz256_scatter_w5:
1386 lea -3($index,$index,2), $index
1387 movdqa 0x00($in_t), %xmm0
1389 movdqa 0x10($in_t), %xmm1
1390 movdqa 0x20($in_t), %xmm2
1391 movdqa 0x30($in_t), %xmm3
1392 movdqa 0x40($in_t), %xmm4
1393 movdqa 0x50($in_t), %xmm5
1394 movdqa %xmm0, 0x00($val,$index)
1395 movdqa %xmm1, 0x10($val,$index)
1396 movdqa %xmm2, 0x20($val,$index)
1397 movdqa %xmm3, 0x30($val,$index)
1398 movdqa %xmm4, 0x40($val,$index)
1399 movdqa %xmm5, 0x50($val,$index)
1402 .size ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
1404 ################################################################################
1405 # void ecp_nistz256_gather_w5(uint64_t *val, uint64_t *in_t, int index);
1406 .globl ecp_nistz256_gather_w5
1407 .type ecp_nistz256_gather_w5,\@abi-omnipotent
1409 ecp_nistz256_gather_w5:
1411 $code.=<<___ if ($avx>1);
1412 mov OPENSSL_ia32cap_P+8(%rip), %eax
1414 jnz .Lavx2_gather_w5
1416 $code.=<<___ if ($win64);
1417 lea -0x88(%rsp), %rax
1418 .LSEH_begin_ecp_nistz256_gather_w5:
1419 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1420 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1421 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1422 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1423 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1424 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1425 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1426 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1427 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1428 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1429 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1432 movdqa .LOne(%rip), $ONE
1443 pshufd \$0, $INDEX, $INDEX
1446 .Lselect_loop_sse_w5:
1450 pcmpeqd $INDEX, $TMP0
1452 movdqa 16*0($in_t), $T0a
1453 movdqa 16*1($in_t), $T0b
1454 movdqa 16*2($in_t), $T0c
1455 movdqa 16*3($in_t), $T0d
1456 movdqa 16*4($in_t), $T0e
1457 movdqa 16*5($in_t), $T0f
1458 lea 16*6($in_t), $in_t
1474 jnz .Lselect_loop_sse_w5
1476 movdqu $Ra, 16*0($val)
1477 movdqu $Rb, 16*1($val)
1478 movdqu $Rc, 16*2($val)
1479 movdqu $Rd, 16*3($val)
1480 movdqu $Re, 16*4($val)
1481 movdqu $Rf, 16*5($val)
1483 $code.=<<___ if ($win64);
1484 movaps (%rsp), %xmm6
1485 movaps 0x10(%rsp), %xmm7
1486 movaps 0x20(%rsp), %xmm8
1487 movaps 0x30(%rsp), %xmm9
1488 movaps 0x40(%rsp), %xmm10
1489 movaps 0x50(%rsp), %xmm11
1490 movaps 0x60(%rsp), %xmm12
1491 movaps 0x70(%rsp), %xmm13
1492 movaps 0x80(%rsp), %xmm14
1493 movaps 0x90(%rsp), %xmm15
1494 lea 0xa8(%rsp), %rsp
1495 .LSEH_end_ecp_nistz256_gather_w5:
1499 .size ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
1501 ################################################################################
1502 # void ecp_nistz256_scatter_w7(uint64_t *val, uint64_t *in_t, int index);
1503 .globl ecp_nistz256_scatter_w7
1504 .type ecp_nistz256_scatter_w7,\@abi-omnipotent
1506 ecp_nistz256_scatter_w7:
1507 movdqu 0x00($in_t), %xmm0
1509 movdqu 0x10($in_t), %xmm1
1510 movdqu 0x20($in_t), %xmm2
1511 movdqu 0x30($in_t), %xmm3
1512 movdqa %xmm0, 0x00($val,$index)
1513 movdqa %xmm1, 0x10($val,$index)
1514 movdqa %xmm2, 0x20($val,$index)
1515 movdqa %xmm3, 0x30($val,$index)
1518 .size ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
1520 ################################################################################
1521 # void ecp_nistz256_gather_w7(uint64_t *val, uint64_t *in_t, int index);
1522 .globl ecp_nistz256_gather_w7
1523 .type ecp_nistz256_gather_w7,\@abi-omnipotent
1525 ecp_nistz256_gather_w7:
1527 $code.=<<___ if ($avx>1);
1528 mov OPENSSL_ia32cap_P+8(%rip), %eax
1530 jnz .Lavx2_gather_w7
1532 $code.=<<___ if ($win64);
1533 lea -0x88(%rsp), %rax
1534 .LSEH_begin_ecp_nistz256_gather_w7:
1535 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1536 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1537 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1538 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1539 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1540 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1541 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1542 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1543 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1544 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1545 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1548 movdqa .LOne(%rip), $M0
1557 pshufd \$0, $INDEX, $INDEX
1560 .Lselect_loop_sse_w7:
1563 movdqa 16*0($in_t), $T0a
1564 movdqa 16*1($in_t), $T0b
1565 pcmpeqd $INDEX, $TMP0
1566 movdqa 16*2($in_t), $T0c
1567 movdqa 16*3($in_t), $T0d
1568 lea 16*4($in_t), $in_t
1577 prefetcht0 255($in_t)
1581 jnz .Lselect_loop_sse_w7
1583 movdqu $Ra, 16*0($val)
1584 movdqu $Rb, 16*1($val)
1585 movdqu $Rc, 16*2($val)
1586 movdqu $Rd, 16*3($val)
1588 $code.=<<___ if ($win64);
1589 movaps (%rsp), %xmm6
1590 movaps 0x10(%rsp), %xmm7
1591 movaps 0x20(%rsp), %xmm8
1592 movaps 0x30(%rsp), %xmm9
1593 movaps 0x40(%rsp), %xmm10
1594 movaps 0x50(%rsp), %xmm11
1595 movaps 0x60(%rsp), %xmm12
1596 movaps 0x70(%rsp), %xmm13
1597 movaps 0x80(%rsp), %xmm14
1598 movaps 0x90(%rsp), %xmm15
1599 lea 0xa8(%rsp), %rsp
1600 .LSEH_end_ecp_nistz256_gather_w7:
1604 .size ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
1608 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1609 my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4));
1610 my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9));
1611 my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14));
1614 ################################################################################
1615 # void ecp_nistz256_avx2_gather_w5(uint64_t *val, uint64_t *in_t, int index);
1616 .type ecp_nistz256_avx2_gather_w5,\@abi-omnipotent
1618 ecp_nistz256_avx2_gather_w5:
1622 $code.=<<___ if ($win64);
1623 lea -0x88(%rsp), %rax
1624 .LSEH_begin_ecp_nistz256_avx2_gather_w5:
1625 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1626 .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
1627 .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
1628 .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
1629 .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
1630 .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
1631 .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
1632 .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
1633 .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
1634 .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
1635 .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
1638 vmovdqa .LTwo(%rip), $TWO
1644 vmovdqa .LOne(%rip), $M0
1645 vmovdqa .LTwo(%rip), $M1
1648 vpermd $INDEX, $Ra, $INDEX
1651 .Lselect_loop_avx2_w5:
1653 vmovdqa 32*0($in_t), $T0a
1654 vmovdqa 32*1($in_t), $T0b
1655 vmovdqa 32*2($in_t), $T0c
1657 vmovdqa 32*3($in_t), $T1a
1658 vmovdqa 32*4($in_t), $T1b
1659 vmovdqa 32*5($in_t), $T1c
1661 vpcmpeqd $INDEX, $M0, $TMP0
1662 vpcmpeqd $INDEX, $M1, $TMP1
1664 vpaddd $TWO, $M0, $M0
1665 vpaddd $TWO, $M1, $M1
1666 lea 32*6($in_t), $in_t
1668 vpand $TMP0, $T0a, $T0a
1669 vpand $TMP0, $T0b, $T0b
1670 vpand $TMP0, $T0c, $T0c
1671 vpand $TMP1, $T1a, $T1a
1672 vpand $TMP1, $T1b, $T1b
1673 vpand $TMP1, $T1c, $T1c
1675 vpxor $T0a, $Ra, $Ra
1676 vpxor $T0b, $Rb, $Rb
1677 vpxor $T0c, $Rc, $Rc
1678 vpxor $T1a, $Ra, $Ra
1679 vpxor $T1b, $Rb, $Rb
1680 vpxor $T1c, $Rc, $Rc
1683 jnz .Lselect_loop_avx2_w5
1685 vmovdqu $Ra, 32*0($val)
1686 vmovdqu $Rb, 32*1($val)
1687 vmovdqu $Rc, 32*2($val)
1690 $code.=<<___ if ($win64);
1691 movaps (%rsp), %xmm6
1692 movaps 0x10(%rsp), %xmm7
1693 movaps 0x20(%rsp), %xmm8
1694 movaps 0x30(%rsp), %xmm9
1695 movaps 0x40(%rsp), %xmm10
1696 movaps 0x50(%rsp), %xmm11
1697 movaps 0x60(%rsp), %xmm12
1698 movaps 0x70(%rsp), %xmm13
1699 movaps 0x80(%rsp), %xmm14
1700 movaps 0x90(%rsp), %xmm15
1701 lea 0xa8(%rsp), %rsp
1702 .LSEH_end_ecp_nistz256_avx2_gather_w5:
1706 .size ecp_nistz256_avx2_gather_w5,.-ecp_nistz256_avx2_gather_w5
1710 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1711 my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3));
1712 my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7));
1713 my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11));
1714 my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15));
1718 ################################################################################
1719 # void ecp_nistz256_avx2_gather_w7(uint64_t *val, uint64_t *in_t, int index);
1720 .globl ecp_nistz256_avx2_gather_w7
1721 .type ecp_nistz256_avx2_gather_w7,\@abi-omnipotent
1723 ecp_nistz256_avx2_gather_w7:
1727 $code.=<<___ if ($win64);
1728 lea -0x88(%rsp), %rax
1729 .LSEH_begin_ecp_nistz256_avx2_gather_w7:
1730 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1731 .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
1732 .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
1733 .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
1734 .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
1735 .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
1736 .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
1737 .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
1738 .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
1739 .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
1740 .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
1743 vmovdqa .LThree(%rip), $THREE
1748 vmovdqa .LOne(%rip), $M0
1749 vmovdqa .LTwo(%rip), $M1
1750 vmovdqa .LThree(%rip), $M2
1753 vpermd $INDEX, $Ra, $INDEX
1754 # Skip index = 0, because it is implicitly the point at infinity
1757 .Lselect_loop_avx2_w7:
1759 vmovdqa 32*0($in_t), $T0a
1760 vmovdqa 32*1($in_t), $T0b
1762 vmovdqa 32*2($in_t), $T1a
1763 vmovdqa 32*3($in_t), $T1b
1765 vmovdqa 32*4($in_t), $T2a
1766 vmovdqa 32*5($in_t), $T2b
1768 vpcmpeqd $INDEX, $M0, $TMP0
1769 vpcmpeqd $INDEX, $M1, $TMP1
1770 vpcmpeqd $INDEX, $M2, $TMP2
1772 vpaddd $THREE, $M0, $M0
1773 vpaddd $THREE, $M1, $M1
1774 vpaddd $THREE, $M2, $M2
1775 lea 32*6($in_t), $in_t
1777 vpand $TMP0, $T0a, $T0a
1778 vpand $TMP0, $T0b, $T0b
1779 vpand $TMP1, $T1a, $T1a
1780 vpand $TMP1, $T1b, $T1b
1781 vpand $TMP2, $T2a, $T2a
1782 vpand $TMP2, $T2b, $T2b
1784 vpxor $T0a, $Ra, $Ra
1785 vpxor $T0b, $Rb, $Rb
1786 vpxor $T1a, $Ra, $Ra
1787 vpxor $T1b, $Rb, $Rb
1788 vpxor $T2a, $Ra, $Ra
1789 vpxor $T2b, $Rb, $Rb
1792 jnz .Lselect_loop_avx2_w7
1795 vmovdqa 32*0($in_t), $T0a
1796 vmovdqa 32*1($in_t), $T0b
1798 vpcmpeqd $INDEX, $M0, $TMP0
1800 vpand $TMP0, $T0a, $T0a
1801 vpand $TMP0, $T0b, $T0b
1803 vpxor $T0a, $Ra, $Ra
1804 vpxor $T0b, $Rb, $Rb
1806 vmovdqu $Ra, 32*0($val)
1807 vmovdqu $Rb, 32*1($val)
1810 $code.=<<___ if ($win64);
1811 movaps (%rsp), %xmm6
1812 movaps 0x10(%rsp), %xmm7
1813 movaps 0x20(%rsp), %xmm8
1814 movaps 0x30(%rsp), %xmm9
1815 movaps 0x40(%rsp), %xmm10
1816 movaps 0x50(%rsp), %xmm11
1817 movaps 0x60(%rsp), %xmm12
1818 movaps 0x70(%rsp), %xmm13
1819 movaps 0x80(%rsp), %xmm14
1820 movaps 0x90(%rsp), %xmm15
1821 lea 0xa8(%rsp), %rsp
1822 .LSEH_end_ecp_nistz256_avx2_gather_w7:
1826 .size ecp_nistz256_avx2_gather_w7,.-ecp_nistz256_avx2_gather_w7
1830 .globl ecp_nistz256_avx2_gather_w7
1831 .type ecp_nistz256_avx2_gather_w7,\@function,3
1833 ecp_nistz256_avx2_gather_w7:
1834 .byte 0x0f,0x0b # ud2
1836 .size ecp_nistz256_avx2_gather_w7,.-ecp_nistz256_avx2_gather_w7
1840 ########################################################################
1841 # This block implements higher level point_double, point_add and
1842 # point_add_affine. The key to performance in this case is to allow
1843 # out-of-order execution logic to overlap computations from next step
1844 # with tail processing from current step. By using tailored calling
1845 # sequence we minimize inter-step overhead to give processor better
1846 # shot at overlapping operations...
1848 # You will notice that input data is copied to stack. Trouble is that
1849 # there are no registers to spare for holding original pointers and
1850 # reloading them, pointers, would create undesired dependencies on
1851 # effective addresses calculation paths. In other words it's too done
1852 # to favour out-of-order execution logic.
1853 # <appro@openssl.org>
1855 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
1856 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
1857 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4);
1858 my ($poly1,$poly3)=($acc6,$acc7);
1860 sub load_for_mul () {
1861 my ($a,$b,$src0) = @_;
1862 my $bias = $src0 eq "%rax" ? 0 : -128;
1868 lea $bias+$a, $a_ptr
1873 sub load_for_sqr () {
1875 my $bias = $src0 eq "%rax" ? 0 : -128;
1879 lea $bias+$a, $a_ptr
1885 ########################################################################
1886 # operate in 4-5-0-1 "name space" that matches multiplication output
1888 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
1891 .type __ecp_nistz256_add_toq,\@abi-omnipotent
1893 __ecp_nistz256_add_toq:
1895 add 8*0($b_ptr), $a0
1896 adc 8*1($b_ptr), $a1
1898 adc 8*2($b_ptr), $a2
1899 adc 8*3($b_ptr), $a3
1913 mov $a0, 8*0($r_ptr)
1915 mov $a1, 8*1($r_ptr)
1917 mov $a2, 8*2($r_ptr)
1918 mov $a3, 8*3($r_ptr)
1921 .size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq
1923 .type __ecp_nistz256_sub_fromq,\@abi-omnipotent
1925 __ecp_nistz256_sub_fromq:
1926 sub 8*0($b_ptr), $a0
1927 sbb 8*1($b_ptr), $a1
1929 sbb 8*2($b_ptr), $a2
1930 sbb 8*3($b_ptr), $a3
1944 mov $a0, 8*0($r_ptr)
1946 mov $a1, 8*1($r_ptr)
1948 mov $a2, 8*2($r_ptr)
1949 mov $a3, 8*3($r_ptr)
1952 .size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq
1954 .type __ecp_nistz256_subq,\@abi-omnipotent
1956 __ecp_nistz256_subq:
1979 .size __ecp_nistz256_subq,.-__ecp_nistz256_subq
1981 .type __ecp_nistz256_mul_by_2q,\@abi-omnipotent
1983 __ecp_nistz256_mul_by_2q:
1985 add $a0, $a0 # a0:a3+a0:a3
2003 mov $a0, 8*0($r_ptr)
2005 mov $a1, 8*1($r_ptr)
2007 mov $a2, 8*2($r_ptr)
2008 mov $a3, 8*3($r_ptr)
2011 .size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q
2016 my ($src0,$sfx,$bias);
2017 my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
2025 .globl ecp_nistz256_point_double
2026 .type ecp_nistz256_point_double,\@function,2
2028 ecp_nistz256_point_double:
2030 $code.=<<___ if ($addx);
2032 and OPENSSL_ia32cap_P+8(%rip), %ecx
2042 .type ecp_nistz256_point_doublex,\@function,2
2044 ecp_nistz256_point_doublex:
2057 .Lpoint_double_shortcut$x:
2058 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x
2059 mov $a_ptr, $b_ptr # backup copy
2060 movdqu 0x10($a_ptr), %xmm1
2061 mov 0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order
2062 mov 0x20+8*1($a_ptr), $acc5
2063 mov 0x20+8*2($a_ptr), $acc0
2064 mov 0x20+8*3($a_ptr), $acc1
2065 mov .Lpoly+8*1(%rip), $poly1
2066 mov .Lpoly+8*3(%rip), $poly3
2067 movdqa %xmm0, $in_x(%rsp)
2068 movdqa %xmm1, $in_x+0x10(%rsp)
2069 lea 0x20($r_ptr), $acc2
2070 lea 0x40($r_ptr), $acc3
2075 lea $S(%rsp), $r_ptr
2076 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(S, in_y);
2078 mov 0x40+8*0($a_ptr), $src0
2079 mov 0x40+8*1($a_ptr), $acc6
2080 mov 0x40+8*2($a_ptr), $acc7
2081 mov 0x40+8*3($a_ptr), $acc0
2082 lea 0x40-$bias($a_ptr), $a_ptr
2083 lea $Zsqr(%rsp), $r_ptr
2084 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Zsqr, in_z);
2086 `&load_for_sqr("$S(%rsp)", "$src0")`
2087 lea $S(%rsp), $r_ptr
2088 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(S, S);
2090 mov 0x20($b_ptr), $src0 # $b_ptr is still valid
2091 mov 0x40+8*0($b_ptr), $acc1
2092 mov 0x40+8*1($b_ptr), $acc2
2093 mov 0x40+8*2($b_ptr), $acc3
2094 mov 0x40+8*3($b_ptr), $acc4
2095 lea 0x40-$bias($b_ptr), $a_ptr
2096 lea 0x20($b_ptr), $b_ptr
2098 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, in_z, in_y);
2099 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(res_z, res_z);
2101 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
2102 mov $in_x+8*1(%rsp), $acc5
2103 lea $Zsqr(%rsp), $b_ptr
2104 mov $in_x+8*2(%rsp), $acc0
2105 mov $in_x+8*3(%rsp), $acc1
2106 lea $M(%rsp), $r_ptr
2107 call __ecp_nistz256_add_to$x # p256_add(M, in_x, Zsqr);
2109 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
2110 mov $in_x+8*1(%rsp), $acc5
2111 lea $Zsqr(%rsp), $b_ptr
2112 mov $in_x+8*2(%rsp), $acc0
2113 mov $in_x+8*3(%rsp), $acc1
2114 lea $Zsqr(%rsp), $r_ptr
2115 call __ecp_nistz256_sub_from$x # p256_sub(Zsqr, in_x, Zsqr);
2117 `&load_for_sqr("$S(%rsp)", "$src0")`
2119 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_y, S);
2122 ######## ecp_nistz256_div_by_2(res_y, res_y); ##########################
2123 # operate in 4-5-6-7 "name space" that matches squaring output
2125 my ($poly1,$poly3)=($a_ptr,$t1);
2126 my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2);
2139 xor $a_ptr, $a_ptr # borrow $a_ptr
2148 mov $a1, $t0 # a0:a3>>1
2159 mov $a0, 8*0($r_ptr)
2161 mov $a1, 8*1($r_ptr)
2165 mov $a2, 8*2($r_ptr)
2166 mov $a3, 8*3($r_ptr)
2170 `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")`
2171 lea $M(%rsp), $r_ptr
2172 call __ecp_nistz256_mul_mont$x # p256_mul_mont(M, M, Zsqr);
2174 lea $tmp0(%rsp), $r_ptr
2175 call __ecp_nistz256_mul_by_2$x
2177 lea $M(%rsp), $b_ptr
2178 lea $M(%rsp), $r_ptr
2179 call __ecp_nistz256_add_to$x # p256_mul_by_3(M, M);
2181 `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")`
2182 lea $S(%rsp), $r_ptr
2183 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, in_x);
2185 lea $tmp0(%rsp), $r_ptr
2186 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(tmp0, S);
2188 `&load_for_sqr("$M(%rsp)", "$src0")`
2190 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_x, M);
2192 lea $tmp0(%rsp), $b_ptr
2193 mov $acc6, $acc0 # harmonize sqr output and sub input
2197 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, tmp0);
2199 mov $S+8*0(%rsp), $t0
2200 mov $S+8*1(%rsp), $t1
2201 mov $S+8*2(%rsp), $t2
2202 mov $S+8*3(%rsp), $acc2 # "4-5-0-1" order
2203 lea $S(%rsp), $r_ptr
2204 call __ecp_nistz256_sub$x # p256_sub(S, S, res_x);
2207 lea $M(%rsp), $b_ptr
2208 mov $acc4, $acc6 # harmonize sub output and mul input
2210 mov $acc4, $S+8*0(%rsp) # have to save:-(
2212 mov $acc5, $S+8*1(%rsp)
2214 mov $acc0, $S+8*2(%rsp)
2215 lea $S-$bias(%rsp), $a_ptr
2217 mov $acc1, $S+8*3(%rsp)
2219 lea $S(%rsp), $r_ptr
2220 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, M);
2224 call __ecp_nistz256_sub_from$x # p256_sub(res_y, S, res_y);
2234 .size ecp_nistz256_point_double$sfx,.-ecp_nistz256_point_double$sfx
2241 my ($src0,$sfx,$bias);
2242 my ($H,$Hsqr,$R,$Rsqr,$Hcub,
2244 $res_x,$res_y,$res_z,
2245 $in1_x,$in1_y,$in1_z,
2246 $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17));
2247 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
2255 .globl ecp_nistz256_point_add
2256 .type ecp_nistz256_point_add,\@function,3
2258 ecp_nistz256_point_add:
2260 $code.=<<___ if ($addx);
2262 and OPENSSL_ia32cap_P+8(%rip), %ecx
2272 .type ecp_nistz256_point_addx,\@function,3
2274 ecp_nistz256_point_addx:
2287 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
2288 movdqu 0x10($a_ptr), %xmm1
2289 movdqu 0x20($a_ptr), %xmm2
2290 movdqu 0x30($a_ptr), %xmm3
2291 movdqu 0x40($a_ptr), %xmm4
2292 movdqu 0x50($a_ptr), %xmm5
2293 mov $a_ptr, $b_ptr # reassign
2294 mov $b_org, $a_ptr # reassign
2295 movdqa %xmm0, $in1_x(%rsp)
2296 movdqa %xmm1, $in1_x+0x10(%rsp)
2297 movdqa %xmm2, $in1_y(%rsp)
2298 movdqa %xmm3, $in1_y+0x10(%rsp)
2299 movdqa %xmm4, $in1_z(%rsp)
2300 movdqa %xmm5, $in1_z+0x10(%rsp)
2303 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr
2304 pshufd \$0xb1, %xmm5, %xmm3
2305 movdqu 0x10($a_ptr), %xmm1
2306 movdqu 0x20($a_ptr), %xmm2
2308 movdqu 0x30($a_ptr), %xmm3
2309 mov 0x40+8*0($a_ptr), $src0 # load original in2_z
2310 mov 0x40+8*1($a_ptr), $acc6
2311 mov 0x40+8*2($a_ptr), $acc7
2312 mov 0x40+8*3($a_ptr), $acc0
2313 movdqa %xmm0, $in2_x(%rsp)
2314 pshufd \$0x1e, %xmm5, %xmm4
2315 movdqa %xmm1, $in2_x+0x10(%rsp)
2316 movdqu 0x40($a_ptr),%xmm0 # in2_z again
2317 movdqu 0x50($a_ptr),%xmm1
2318 movdqa %xmm2, $in2_y(%rsp)
2319 movdqa %xmm3, $in2_y+0x10(%rsp)
2323 movq $r_ptr, %xmm0 # save $r_ptr
2325 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2326 mov $src0, $in2_z+8*0(%rsp) # make in2_z copy
2327 mov $acc6, $in2_z+8*1(%rsp)
2328 mov $acc7, $in2_z+8*2(%rsp)
2329 mov $acc0, $in2_z+8*3(%rsp)
2330 lea $Z2sqr(%rsp), $r_ptr # Z2^2
2331 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z2sqr, in2_z);
2333 pcmpeqd %xmm4, %xmm5
2334 pshufd \$0xb1, %xmm1, %xmm4
2336 pshufd \$0, %xmm5, %xmm5 # in1infty
2337 pshufd \$0x1e, %xmm4, %xmm3
2340 pcmpeqd %xmm3, %xmm4
2341 pshufd \$0, %xmm4, %xmm4 # in2infty
2342 mov 0x40+8*0($b_ptr), $src0 # load original in1_z
2343 mov 0x40+8*1($b_ptr), $acc6
2344 mov 0x40+8*2($b_ptr), $acc7
2345 mov 0x40+8*3($b_ptr), $acc0
2348 lea 0x40-$bias($b_ptr), $a_ptr
2349 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2350 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2352 `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")`
2353 lea $S1(%rsp), $r_ptr # S1 = Z2^3
2354 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, Z2sqr, in2_z);
2356 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2357 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2358 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2360 `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")`
2361 lea $S1(%rsp), $r_ptr # S1 = Y1*Z2^3
2362 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, S1, in1_y);
2364 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2365 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2366 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2368 lea $S1(%rsp), $b_ptr
2369 lea $R(%rsp), $r_ptr # R = S2 - S1
2370 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, S1);
2372 or $acc5, $acc4 # see if result is zero
2376 por %xmm5, %xmm2 # in1infty || in2infty
2379 `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2380 lea $U1(%rsp), $r_ptr # U1 = X1*Z2^2
2381 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U1, in1_x, Z2sqr);
2383 `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")`
2384 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2385 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in2_x, Z1sqr);
2387 lea $U1(%rsp), $b_ptr
2388 lea $H(%rsp), $r_ptr # H = U2 - U1
2389 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, U1);
2391 or $acc5, $acc4 # see if result is zero
2395 .byte 0x3e # predict taken
2396 jnz .Ladd_proceed$x # is_equal(U1,U2)?
2400 jnz .Ladd_proceed$x # (in1infty || in2infty)?
2402 jz .Ladd_double$x # is_equal(S1,S2)?
2404 movq %xmm0, $r_ptr # restore $r_ptr
2406 movdqu %xmm0, 0x00($r_ptr)
2407 movdqu %xmm0, 0x10($r_ptr)
2408 movdqu %xmm0, 0x20($r_ptr)
2409 movdqu %xmm0, 0x30($r_ptr)
2410 movdqu %xmm0, 0x40($r_ptr)
2411 movdqu %xmm0, 0x50($r_ptr)
2416 movq %xmm1, $a_ptr # restore $a_ptr
2417 movq %xmm0, $r_ptr # restore $r_ptr
2418 add \$`32*(18-5)`, %rsp # difference in frame sizes
2419 jmp .Lpoint_double_shortcut$x
2423 `&load_for_sqr("$R(%rsp)", "$src0")`
2424 lea $Rsqr(%rsp), $r_ptr # R^2
2425 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2427 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2428 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2429 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2431 `&load_for_sqr("$H(%rsp)", "$src0")`
2432 lea $Hsqr(%rsp), $r_ptr # H^2
2433 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2435 `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")`
2436 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2437 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, res_z, in2_z);
2439 `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")`
2440 lea $Hcub(%rsp), $r_ptr # H^3
2441 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2443 `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")`
2444 lea $U2(%rsp), $r_ptr # U1*H^2
2445 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, U1, Hsqr);
2448 #######################################################################
2449 # operate in 4-5-0-1 "name space" that matches multiplication output
2451 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2452 my ($poly1, $poly3)=($acc6,$acc7);
2455 #lea $U2(%rsp), $a_ptr
2456 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2457 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2460 add $acc0, $acc0 # a0:a3+a0:a3
2461 lea $Rsqr(%rsp), $a_ptr
2478 mov 8*0($a_ptr), $t0
2480 mov 8*1($a_ptr), $t1
2482 mov 8*2($a_ptr), $t2
2484 mov 8*3($a_ptr), $t3
2486 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2488 lea $Hcub(%rsp), $b_ptr
2489 lea $res_x(%rsp), $r_ptr
2490 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2492 mov $U2+8*0(%rsp), $t0
2493 mov $U2+8*1(%rsp), $t1
2494 mov $U2+8*2(%rsp), $t2
2495 mov $U2+8*3(%rsp), $t3
2496 lea $res_y(%rsp), $r_ptr
2498 call __ecp_nistz256_sub$x # p256_sub(res_y, U2, res_x);
2500 mov $acc0, 8*0($r_ptr) # save the result, as
2501 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2502 mov $acc2, 8*2($r_ptr)
2503 mov $acc3, 8*3($r_ptr)
2507 `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")`
2508 lea $S2(%rsp), $r_ptr
2509 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S1, Hcub);
2511 `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")`
2512 lea $res_y(%rsp), $r_ptr
2513 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_y, R, res_y);
2515 lea $S2(%rsp), $b_ptr
2516 lea $res_y(%rsp), $r_ptr
2517 call __ecp_nistz256_sub_from$x # p256_sub(res_y, res_y, S2);
2519 movq %xmm0, $r_ptr # restore $r_ptr
2521 movdqa %xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty);
2523 pandn $res_z(%rsp), %xmm0
2525 pandn $res_z+0x10(%rsp), %xmm1
2527 pand $in2_z(%rsp), %xmm2
2528 pand $in2_z+0x10(%rsp), %xmm3
2532 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2538 pand $in1_z(%rsp), %xmm2
2539 pand $in1_z+0x10(%rsp), %xmm3
2542 movdqu %xmm2, 0x40($r_ptr)
2543 movdqu %xmm3, 0x50($r_ptr)
2545 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
2547 pandn $res_x(%rsp), %xmm0
2549 pandn $res_x+0x10(%rsp), %xmm1
2551 pand $in2_x(%rsp), %xmm2
2552 pand $in2_x+0x10(%rsp), %xmm3
2556 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
2562 pand $in1_x(%rsp), %xmm2
2563 pand $in1_x+0x10(%rsp), %xmm3
2566 movdqu %xmm2, 0x00($r_ptr)
2567 movdqu %xmm3, 0x10($r_ptr)
2569 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
2571 pandn $res_y(%rsp), %xmm0
2573 pandn $res_y+0x10(%rsp), %xmm1
2575 pand $in2_y(%rsp), %xmm2
2576 pand $in2_y+0x10(%rsp), %xmm3
2580 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
2586 pand $in1_y(%rsp), %xmm2
2587 pand $in1_y+0x10(%rsp), %xmm3
2590 movdqu %xmm2, 0x20($r_ptr)
2591 movdqu %xmm3, 0x30($r_ptr)
2602 .size ecp_nistz256_point_add$sfx,.-ecp_nistz256_point_add$sfx
2607 sub gen_add_affine () {
2609 my ($src0,$sfx,$bias);
2610 my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr,
2611 $res_x,$res_y,$res_z,
2612 $in1_x,$in1_y,$in1_z,
2613 $in2_x,$in2_y)=map(32*$_,(0..14));
2622 .globl ecp_nistz256_point_add_affine
2623 .type ecp_nistz256_point_add_affine,\@function,3
2625 ecp_nistz256_point_add_affine:
2627 $code.=<<___ if ($addx);
2629 and OPENSSL_ia32cap_P+8(%rip), %ecx
2631 je .Lpoint_add_affinex
2639 .type ecp_nistz256_point_add_affinex,\@function,3
2641 ecp_nistz256_point_add_affinex:
2642 .Lpoint_add_affinex:
2654 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
2655 mov $b_org, $b_ptr # reassign
2656 movdqu 0x10($a_ptr), %xmm1
2657 movdqu 0x20($a_ptr), %xmm2
2658 movdqu 0x30($a_ptr), %xmm3
2659 movdqu 0x40($a_ptr), %xmm4
2660 movdqu 0x50($a_ptr), %xmm5
2661 mov 0x40+8*0($a_ptr), $src0 # load original in1_z
2662 mov 0x40+8*1($a_ptr), $acc6
2663 mov 0x40+8*2($a_ptr), $acc7
2664 mov 0x40+8*3($a_ptr), $acc0
2665 movdqa %xmm0, $in1_x(%rsp)
2666 movdqa %xmm1, $in1_x+0x10(%rsp)
2667 movdqa %xmm2, $in1_y(%rsp)
2668 movdqa %xmm3, $in1_y+0x10(%rsp)
2669 movdqa %xmm4, $in1_z(%rsp)
2670 movdqa %xmm5, $in1_z+0x10(%rsp)
2673 movdqu 0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr
2674 pshufd \$0xb1, %xmm5, %xmm3
2675 movdqu 0x10($b_ptr), %xmm1
2676 movdqu 0x20($b_ptr), %xmm2
2678 movdqu 0x30($b_ptr), %xmm3
2679 movdqa %xmm0, $in2_x(%rsp)
2680 pshufd \$0x1e, %xmm5, %xmm4
2681 movdqa %xmm1, $in2_x+0x10(%rsp)
2683 movq $r_ptr, %xmm0 # save $r_ptr
2684 movdqa %xmm2, $in2_y(%rsp)
2685 movdqa %xmm3, $in2_y+0x10(%rsp)
2691 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2692 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2693 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2695 pcmpeqd %xmm4, %xmm5
2696 pshufd \$0xb1, %xmm3, %xmm4
2697 mov 0x00($b_ptr), $src0 # $b_ptr is still valid
2698 #lea 0x00($b_ptr), $b_ptr
2699 mov $acc4, $acc1 # harmonize sqr output and mul input
2701 pshufd \$0, %xmm5, %xmm5 # in1infty
2702 pshufd \$0x1e, %xmm4, %xmm3
2707 pcmpeqd %xmm3, %xmm4
2708 pshufd \$0, %xmm4, %xmm4 # in2infty
2710 lea $Z1sqr-$bias(%rsp), $a_ptr
2712 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2713 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, Z1sqr, in2_x);
2715 lea $in1_x(%rsp), $b_ptr
2716 lea $H(%rsp), $r_ptr # H = U2 - U1
2717 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, in1_x);
2719 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2720 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2721 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2723 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2724 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2725 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2727 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2728 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2729 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2731 lea $in1_y(%rsp), $b_ptr
2732 lea $R(%rsp), $r_ptr # R = S2 - S1
2733 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, in1_y);
2735 `&load_for_sqr("$H(%rsp)", "$src0")`
2736 lea $Hsqr(%rsp), $r_ptr # H^2
2737 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2739 `&load_for_sqr("$R(%rsp)", "$src0")`
2740 lea $Rsqr(%rsp), $r_ptr # R^2
2741 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2743 `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")`
2744 lea $Hcub(%rsp), $r_ptr # H^3
2745 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2747 `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2748 lea $U2(%rsp), $r_ptr # U1*H^2
2749 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in1_x, Hsqr);
2752 #######################################################################
2753 # operate in 4-5-0-1 "name space" that matches multiplication output
2755 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2756 my ($poly1, $poly3)=($acc6,$acc7);
2759 #lea $U2(%rsp), $a_ptr
2760 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2761 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2764 add $acc0, $acc0 # a0:a3+a0:a3
2765 lea $Rsqr(%rsp), $a_ptr
2782 mov 8*0($a_ptr), $t0
2784 mov 8*1($a_ptr), $t1
2786 mov 8*2($a_ptr), $t2
2788 mov 8*3($a_ptr), $t3
2790 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2792 lea $Hcub(%rsp), $b_ptr
2793 lea $res_x(%rsp), $r_ptr
2794 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2796 mov $U2+8*0(%rsp), $t0
2797 mov $U2+8*1(%rsp), $t1
2798 mov $U2+8*2(%rsp), $t2
2799 mov $U2+8*3(%rsp), $t3
2800 lea $H(%rsp), $r_ptr
2802 call __ecp_nistz256_sub$x # p256_sub(H, U2, res_x);
2804 mov $acc0, 8*0($r_ptr) # save the result, as
2805 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2806 mov $acc2, 8*2($r_ptr)
2807 mov $acc3, 8*3($r_ptr)
2811 `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")`
2812 lea $S2(%rsp), $r_ptr
2813 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Hcub, in1_y);
2815 `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")`
2816 lea $H(%rsp), $r_ptr
2817 call __ecp_nistz256_mul_mont$x # p256_mul_mont(H, H, R);
2819 lea $S2(%rsp), $b_ptr
2820 lea $res_y(%rsp), $r_ptr
2821 call __ecp_nistz256_sub_from$x # p256_sub(res_y, H, S2);
2823 movq %xmm0, $r_ptr # restore $r_ptr
2825 movdqa %xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty);
2827 pandn $res_z(%rsp), %xmm0
2829 pandn $res_z+0x10(%rsp), %xmm1
2831 pand .LONE_mont(%rip), %xmm2
2832 pand .LONE_mont+0x10(%rip), %xmm3
2836 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2842 pand $in1_z(%rsp), %xmm2
2843 pand $in1_z+0x10(%rsp), %xmm3
2846 movdqu %xmm2, 0x40($r_ptr)
2847 movdqu %xmm3, 0x50($r_ptr)
2849 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
2851 pandn $res_x(%rsp), %xmm0
2853 pandn $res_x+0x10(%rsp), %xmm1
2855 pand $in2_x(%rsp), %xmm2
2856 pand $in2_x+0x10(%rsp), %xmm3
2860 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
2866 pand $in1_x(%rsp), %xmm2
2867 pand $in1_x+0x10(%rsp), %xmm3
2870 movdqu %xmm2, 0x00($r_ptr)
2871 movdqu %xmm3, 0x10($r_ptr)
2873 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
2875 pandn $res_y(%rsp), %xmm0
2877 pandn $res_y+0x10(%rsp), %xmm1
2879 pand $in2_y(%rsp), %xmm2
2880 pand $in2_y+0x10(%rsp), %xmm3
2884 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
2890 pand $in1_y(%rsp), %xmm2
2891 pand $in1_y+0x10(%rsp), %xmm3
2894 movdqu %xmm2, 0x20($r_ptr)
2895 movdqu %xmm3, 0x30($r_ptr)
2905 .size ecp_nistz256_point_add_affine$sfx,.-ecp_nistz256_point_add_affine$sfx
2908 &gen_add_affine("q");
2910 ########################################################################
2914 ########################################################################
2915 # operate in 4-5-0-1 "name space" that matches multiplication output
2917 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2920 .type __ecp_nistz256_add_tox,\@abi-omnipotent
2922 __ecp_nistz256_add_tox:
2924 adc 8*0($b_ptr), $a0
2925 adc 8*1($b_ptr), $a1
2927 adc 8*2($b_ptr), $a2
2928 adc 8*3($b_ptr), $a3
2943 mov $a0, 8*0($r_ptr)
2945 mov $a1, 8*1($r_ptr)
2947 mov $a2, 8*2($r_ptr)
2948 mov $a3, 8*3($r_ptr)
2951 .size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox
2953 .type __ecp_nistz256_sub_fromx,\@abi-omnipotent
2955 __ecp_nistz256_sub_fromx:
2957 sbb 8*0($b_ptr), $a0
2958 sbb 8*1($b_ptr), $a1
2960 sbb 8*2($b_ptr), $a2
2961 sbb 8*3($b_ptr), $a3
2976 mov $a0, 8*0($r_ptr)
2978 mov $a1, 8*1($r_ptr)
2980 mov $a2, 8*2($r_ptr)
2981 mov $a3, 8*3($r_ptr)
2984 .size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx
2986 .type __ecp_nistz256_subx,\@abi-omnipotent
2988 __ecp_nistz256_subx:
3013 .size __ecp_nistz256_subx,.-__ecp_nistz256_subx
3015 .type __ecp_nistz256_mul_by_2x,\@abi-omnipotent
3017 __ecp_nistz256_mul_by_2x:
3019 adc $a0, $a0 # a0:a3+a0:a3
3038 mov $a0, 8*0($r_ptr)
3040 mov $a1, 8*1($r_ptr)
3042 mov $a2, 8*2($r_ptr)
3043 mov $a3, 8*3($r_ptr)
3046 .size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x
3051 &gen_add_affine("x");
3055 ########################################################################
3056 # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
3058 open TABLE,"<ecp_nistz256_table.c" or
3059 open TABLE,"<${dir}../ecp_nistz256_table.c" or
3060 die "failed to open ecp_nistz256_table.c:",$!;
3065 s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
3069 die "insane number of elements" if ($#arr != 64*16*37-1);
3073 .globl ecp_nistz256_precomputed
3074 .type ecp_nistz256_precomputed,\@object
3076 ecp_nistz256_precomputed:
3078 while (@line=splice(@arr,0,16)) {
3079 print ".long\t",join(',',map { sprintf "0x%08x",$_} @line),"\n";
3082 .size ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
3085 $code =~ s/\`([^\`]*)\`/eval $1/gem;