3 ##############################################################################
5 # Copyright 2014 Intel Corporation #
7 # Licensed under the Apache License, Version 2.0 (the "License"); #
8 # you may not use this file except in compliance with the License. #
9 # You may obtain a copy of the License at #
11 # http://www.apache.org/licenses/LICENSE-2.0 #
13 # Unless required by applicable law or agreed to in writing, software #
14 # distributed under the License is distributed on an "AS IS" BASIS, #
15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
16 # See the License for the specific language governing permissions and #
17 # limitations under the License. #
19 ##############################################################################
21 # Developers and authors: #
22 # Shay Gueron (1, 2), and Vlad Krasnov (1) #
23 # (1) Intel Corporation, Israel Development Center #
24 # (2) University of Haifa #
26 # S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with#
29 ##############################################################################
31 # Further optimization by <appro@openssl.org>:
33 # this/original with/without -DECP_NISTZ256_ASM(*)
34 # Opteron +12-49% +110-150%
35 # Bulldozer +14-45% +175-210%
37 # Westmere +12-34% +80-87%
38 # Sandy Bridge +9-35% +110-120%
39 # Ivy Bridge +9-35% +110-125%
40 # Haswell +8-37% +140-160%
41 # Broadwell +18-58% +145-210%
42 # Atom +15-50% +130-180%
43 # VIA Nano +43-160% +300-480%
45 # (*) "without -DECP_NISTZ256_ASM" refers to build with
46 # "enable-ec_nistp_64_gcc_128";
48 # Ranges denote minimum and maximum improvement coefficients depending
49 # on benchmark. Lower coefficients are for ECDSA sign, relatively fastest
50 # server-side operation. Keep in mind that +100% means 2x improvement.
54 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
56 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
58 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
59 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
60 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
61 die "can't locate x86_64-xlate.pl";
63 open OUT,"| \"$^X\" $xlate $flavour $output";
66 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
67 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
68 $avx = ($1>=2.19) + ($1>=2.22);
72 if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
73 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
74 $avx = ($1>=2.09) + ($1>=2.10);
78 if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
79 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
80 $avx = ($1>=10) + ($1>=11);
84 if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9])\.([0-9]+)/) {
85 my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10
86 $avx = ($ver>=3.0) + ($ver>=3.01);
92 .extern OPENSSL_ia32cap_P
97 .quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001
99 # 2^512 mod P precomputed for NIST P256 polynomial
101 .quad 0x0000000000000003, 0xfffffffbffffffff, 0xfffffffffffffffe, 0x00000004fffffffd
104 .long 1,1,1,1,1,1,1,1
106 .long 2,2,2,2,2,2,2,2
108 .long 3,3,3,3,3,3,3,3
110 .quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe
114 ################################################################################
115 # void ecp_nistz256_mul_by_2(uint64_t res[4], uint64_t a[4]);
117 my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11));
118 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13");
119 my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx");
123 .globl ecp_nistz256_mul_by_2
124 .type ecp_nistz256_mul_by_2,\@function,2
126 ecp_nistz256_mul_by_2:
132 add $a0, $a0 # a0:a3+a0:a3
136 lea .Lpoly(%rip), $a_ptr
163 .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
165 ################################################################################
166 # void ecp_nistz256_div_by_2(uint64_t res[4], uint64_t a[4]);
167 .globl ecp_nistz256_div_by_2
168 .type ecp_nistz256_div_by_2,\@function,2
170 ecp_nistz256_div_by_2:
179 lea .Lpoly(%rip), $a_ptr
190 xor $a_ptr, $a_ptr # borrow $a_ptr
199 mov $a1, $t0 # a0:a3>>1
223 .size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
225 ################################################################################
226 # void ecp_nistz256_mul_by_3(uint64_t res[4], uint64_t a[4]);
227 .globl ecp_nistz256_mul_by_3
228 .type ecp_nistz256_mul_by_3,\@function,2
230 ecp_nistz256_mul_by_3:
237 add $a0, $a0 # a0:a3+a0:a3
249 sbb .Lpoly+8*1(%rip), $a1
252 sbb .Lpoly+8*3(%rip), $a3
261 add 8*0($a_ptr), $a0 # a0:a3+=a_ptr[0:3]
271 sbb .Lpoly+8*1(%rip), $a1
274 sbb .Lpoly+8*3(%rip), $a3
289 .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
291 ################################################################################
292 # void ecp_nistz256_add(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
293 .globl ecp_nistz256_add
294 .type ecp_nistz256_add,\@function,3
305 lea .Lpoly(%rip), $a_ptr
335 .size ecp_nistz256_add,.-ecp_nistz256_add
337 ################################################################################
338 # void ecp_nistz256_sub(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
339 .globl ecp_nistz256_sub
340 .type ecp_nistz256_sub,\@function,3
351 lea .Lpoly(%rip), $a_ptr
381 .size ecp_nistz256_sub,.-ecp_nistz256_sub
383 ################################################################################
384 # void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]);
385 .globl ecp_nistz256_neg
386 .type ecp_nistz256_neg,\@function,2
403 lea .Lpoly(%rip), $a_ptr
427 .size ecp_nistz256_neg,.-ecp_nistz256_neg
431 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
432 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
433 my ($t0,$t1,$t2,$t3,$t4)=("%rcx","%rbp","%rbx","%rdx","%rax");
434 my ($poly1,$poly3)=($acc6,$acc7);
437 ################################################################################
438 # void ecp_nistz256_to_mont(
441 .globl ecp_nistz256_to_mont
442 .type ecp_nistz256_to_mont,\@function,2
444 ecp_nistz256_to_mont:
446 $code.=<<___ if ($addx);
448 and OPENSSL_ia32cap_P+8(%rip), %ecx
451 lea .LRR(%rip), $b_org
453 .size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont
455 ################################################################################
456 # void ecp_nistz256_mul_mont(
461 .globl ecp_nistz256_mul_mont
462 .type ecp_nistz256_mul_mont,\@function,3
464 ecp_nistz256_mul_mont:
466 $code.=<<___ if ($addx);
468 and OPENSSL_ia32cap_P+8(%rip), %ecx
479 $code.=<<___ if ($addx);
485 mov 8*0($b_org), %rax
486 mov 8*0($a_ptr), $acc1
487 mov 8*1($a_ptr), $acc2
488 mov 8*2($a_ptr), $acc3
489 mov 8*3($a_ptr), $acc4
491 call __ecp_nistz256_mul_montq
493 $code.=<<___ if ($addx);
499 mov 8*0($b_org), %rdx
500 mov 8*0($a_ptr), $acc1
501 mov 8*1($a_ptr), $acc2
502 mov 8*2($a_ptr), $acc3
503 mov 8*3($a_ptr), $acc4
504 lea -128($a_ptr), $a_ptr # control u-op density
506 call __ecp_nistz256_mul_montx
517 .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
519 .type __ecp_nistz256_mul_montq,\@abi-omnipotent
521 __ecp_nistz256_mul_montq:
522 ########################################################################
526 mov .Lpoly+8*1(%rip),$poly1
532 mov .Lpoly+8*3(%rip),$poly3
551 ########################################################################
552 # First reduction step
553 # Basically now we want to multiply acc[0] by p256,
554 # and add the result to the acc.
555 # Due to the special form of p256 we do some optimizations
557 # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0]
558 # then we add acc[0] and get acc[0] x 2^96
564 add $acc0, $acc1 # +=acc[0]<<96
567 mov 8*1($b_ptr), %rax
572 ########################################################################
605 ########################################################################
606 # Second reduction step
614 mov 8*2($b_ptr), %rax
619 ########################################################################
652 ########################################################################
653 # Third reduction step
661 mov 8*3($b_ptr), %rax
666 ########################################################################
699 ########################################################################
700 # Final reduction step
713 ########################################################################
714 # Branch-less conditional subtraction of P
715 sub \$-1, $acc4 # .Lpoly[0]
717 sbb $poly1, $acc5 # .Lpoly[1]
718 sbb \$0, $acc0 # .Lpoly[2]
720 sbb $poly3, $acc1 # .Lpoly[3]
725 mov $acc4, 8*0($r_ptr)
727 mov $acc5, 8*1($r_ptr)
729 mov $acc0, 8*2($r_ptr)
730 mov $acc1, 8*3($r_ptr)
733 .size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq
735 ################################################################################
736 # void ecp_nistz256_sqr_mont(
740 # we optimize the square according to S.Gueron and V.Krasnov,
741 # "Speeding up Big-Number Squaring"
742 .globl ecp_nistz256_sqr_mont
743 .type ecp_nistz256_sqr_mont,\@function,2
745 ecp_nistz256_sqr_mont:
747 $code.=<<___ if ($addx);
749 and OPENSSL_ia32cap_P+8(%rip), %ecx
759 $code.=<<___ if ($addx);
764 mov 8*0($a_ptr), %rax
765 mov 8*1($a_ptr), $acc6
766 mov 8*2($a_ptr), $acc7
767 mov 8*3($a_ptr), $acc0
769 call __ecp_nistz256_sqr_montq
771 $code.=<<___ if ($addx);
776 mov 8*0($a_ptr), %rdx
777 mov 8*1($a_ptr), $acc6
778 mov 8*2($a_ptr), $acc7
779 mov 8*3($a_ptr), $acc0
780 lea -128($a_ptr), $a_ptr # control u-op density
782 call __ecp_nistz256_sqr_montx
793 .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
795 .type __ecp_nistz256_sqr_montq,\@abi-omnipotent
797 __ecp_nistz256_sqr_montq:
799 mulq $acc6 # a[1]*a[0]
804 mulq $acc5 # a[0]*a[2]
810 mulq $acc5 # a[0]*a[3]
816 #################################
817 mulq $acc6 # a[1]*a[2]
823 mulq $acc6 # a[1]*a[3]
831 #################################
832 mulq $acc7 # a[2]*a[3]
835 mov 8*0($a_ptr), %rax
839 add $acc1, $acc1 # acc1:6<<1
849 mov 8*1($a_ptr), %rax
855 mov 8*2($a_ptr), %rax
862 mov 8*3($a_ptr), %rax
872 mov .Lpoly+8*1(%rip), $a_ptr
873 mov .Lpoly+8*3(%rip), $t1
875 ##########################################
882 add $acc0, $acc1 # +=acc[0]<<96
888 ##########################################
901 ##########################################
914 ###########################################
927 ############################################
928 # Add the rest of the acc
937 sub \$-1, $acc4 # .Lpoly[0]
939 sbb $a_ptr, $acc5 # .Lpoly[1]
940 sbb \$0, $acc6 # .Lpoly[2]
942 sbb $t1, $acc7 # .Lpoly[3]
947 mov $acc4, 8*0($r_ptr)
949 mov $acc5, 8*1($r_ptr)
951 mov $acc6, 8*2($r_ptr)
952 mov $acc7, 8*3($r_ptr)
955 .size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq
960 .type __ecp_nistz256_mul_montx,\@abi-omnipotent
962 __ecp_nistz256_mul_montx:
963 ########################################################################
965 mulx $acc1, $acc0, $acc1
966 mulx $acc2, $t0, $acc2
968 xor $acc5, $acc5 # cf=0
969 mulx $acc3, $t1, $acc3
970 mov .Lpoly+8*3(%rip), $poly3
972 mulx $acc4, $t0, $acc4
975 shlx $poly1,$acc0,$t1
977 shrx $poly1,$acc0,$t0
980 ########################################################################
981 # First reduction step
985 mulx $poly3, $t0, $t1
986 mov 8*1($b_ptr), %rdx
990 xor $acc0, $acc0 # $acc0=0,cf=0,of=0
992 ########################################################################
994 mulx 8*0+128($a_ptr), $t0, $t1
998 mulx 8*1+128($a_ptr), $t0, $t1
1002 mulx 8*2+128($a_ptr), $t0, $t1
1006 mulx 8*3+128($a_ptr), $t0, $t1
1009 shlx $poly1, $acc1, $t0
1011 shrx $poly1, $acc1, $t1
1017 ########################################################################
1018 # Second reduction step
1022 mulx $poly3, $t0, $t1
1023 mov 8*2($b_ptr), %rdx
1027 xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0
1029 ########################################################################
1031 mulx 8*0+128($a_ptr), $t0, $t1
1035 mulx 8*1+128($a_ptr), $t0, $t1
1039 mulx 8*2+128($a_ptr), $t0, $t1
1043 mulx 8*3+128($a_ptr), $t0, $t1
1046 shlx $poly1, $acc2, $t0
1048 shrx $poly1, $acc2, $t1
1054 ########################################################################
1055 # Third reduction step
1059 mulx $poly3, $t0, $t1
1060 mov 8*3($b_ptr), %rdx
1064 xor $acc2, $acc2 # $acc2=0,cf=0,of=0
1066 ########################################################################
1068 mulx 8*0+128($a_ptr), $t0, $t1
1072 mulx 8*1+128($a_ptr), $t0, $t1
1076 mulx 8*2+128($a_ptr), $t0, $t1
1080 mulx 8*3+128($a_ptr), $t0, $t1
1083 shlx $poly1, $acc3, $t0
1085 shrx $poly1, $acc3, $t1
1091 ########################################################################
1092 # Fourth reduction step
1096 mulx $poly3, $t0, $t1
1098 mov .Lpoly+8*1(%rip), $poly1
1104 ########################################################################
1105 # Branch-less conditional subtraction of P
1108 sbb \$-1, $acc4 # .Lpoly[0]
1109 sbb $poly1, $acc5 # .Lpoly[1]
1110 sbb \$0, $acc0 # .Lpoly[2]
1112 sbb $poly3, $acc1 # .Lpoly[3]
1117 mov $acc4, 8*0($r_ptr)
1119 mov $acc5, 8*1($r_ptr)
1121 mov $acc0, 8*2($r_ptr)
1122 mov $acc1, 8*3($r_ptr)
1125 .size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx
1127 .type __ecp_nistz256_sqr_montx,\@abi-omnipotent
1129 __ecp_nistz256_sqr_montx:
1130 mulx $acc6, $acc1, $acc2 # a[0]*a[1]
1131 mulx $acc7, $t0, $acc3 # a[0]*a[2]
1134 mulx $acc0, $t1, $acc4 # a[0]*a[3]
1138 xor $acc5, $acc5 # $acc5=0,cf=0,of=0
1140 #################################
1141 mulx $acc7, $t0, $t1 # a[1]*a[2]
1145 mulx $acc0, $t0, $t1 # a[1]*a[3]
1151 #################################
1152 mulx $acc0, $t0, $acc6 # a[2]*a[3]
1153 mov 8*0+128($a_ptr), %rdx
1154 xor $acc7, $acc7 # $acc7=0,cf=0,of=0
1155 adcx $acc1, $acc1 # acc1:6<<1
1158 adox $acc7, $acc6 # of=0
1160 mulx %rdx, $acc0, $t1
1161 mov 8*1+128($a_ptr), %rdx
1166 mov 8*2+128($a_ptr), %rdx
1172 mov 8*3+128($a_ptr), %rdx
1182 shlx $a_ptr, $acc0, $t0
1184 shrx $a_ptr, $acc0, $t4
1185 mov .Lpoly+8*3(%rip), $t1
1191 mulx $t1, $t0, $acc0
1194 shlx $a_ptr, $acc1, $t0
1196 shrx $a_ptr, $acc1, $t4
1202 mulx $t1, $t0, $acc1
1205 shlx $a_ptr, $acc2, $t0
1207 shrx $a_ptr, $acc2, $t4
1213 mulx $t1, $t0, $acc2
1216 shlx $a_ptr, $acc3, $t0
1218 shrx $a_ptr, $acc3, $t4
1224 mulx $t1, $t0, $acc3
1229 adc $acc0, $acc4 # accumulate upper half
1230 mov .Lpoly+8*1(%rip), $a_ptr
1238 xor %eax, %eax # cf=0
1239 sbb \$-1, $acc4 # .Lpoly[0]
1241 sbb $a_ptr, $acc5 # .Lpoly[1]
1242 sbb \$0, $acc6 # .Lpoly[2]
1244 sbb $t1, $acc7 # .Lpoly[3]
1249 mov $acc4, 8*0($r_ptr)
1251 mov $acc5, 8*1($r_ptr)
1253 mov $acc6, 8*2($r_ptr)
1254 mov $acc7, 8*3($r_ptr)
1257 .size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx
1262 my ($r_ptr,$in_ptr)=("%rdi","%rsi");
1263 my ($acc0,$acc1,$acc2,$acc3)=map("%r$_",(8..11));
1264 my ($t0,$t1,$t2)=("%rcx","%r12","%r13");
1267 ################################################################################
1268 # void ecp_nistz256_from_mont(
1271 # This one performs Montgomery multiplication by 1, so we only need the reduction
1273 .globl ecp_nistz256_from_mont
1274 .type ecp_nistz256_from_mont,\@function,2
1276 ecp_nistz256_from_mont:
1280 mov 8*0($in_ptr), %rax
1281 mov .Lpoly+8*3(%rip), $t2
1282 mov 8*1($in_ptr), $acc1
1283 mov 8*2($in_ptr), $acc2
1284 mov 8*3($in_ptr), $acc3
1286 mov .Lpoly+8*1(%rip), $t1
1288 #########################################
1300 #########################################
1313 ##########################################
1326 ###########################################
1340 ###########################################
1341 # Branch-less conditional subtraction
1351 cmovnz $in_ptr, $acc1
1352 mov $acc0, 8*0($r_ptr)
1354 mov $acc1, 8*1($r_ptr)
1356 mov $acc2, 8*2($r_ptr)
1357 mov $acc3, 8*3($r_ptr)
1362 .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
1366 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1367 my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7));
1368 my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15));
1369 my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15));
1372 ################################################################################
1373 # void ecp_nistz256_scatter_w5(uint64_t *val, uint64_t *in_t, int index);
1374 .globl ecp_nistz256_scatter_w5
1375 .type ecp_nistz256_scatter_w5,\@abi-omnipotent
1377 ecp_nistz256_scatter_w5:
1378 lea -3($index,$index,2), $index
1379 movdqa 0x00($in_t), %xmm0
1381 movdqa 0x10($in_t), %xmm1
1382 movdqa 0x20($in_t), %xmm2
1383 movdqa 0x30($in_t), %xmm3
1384 movdqa 0x40($in_t), %xmm4
1385 movdqa 0x50($in_t), %xmm5
1386 movdqa %xmm0, 0x00($val,$index)
1387 movdqa %xmm1, 0x10($val,$index)
1388 movdqa %xmm2, 0x20($val,$index)
1389 movdqa %xmm3, 0x30($val,$index)
1390 movdqa %xmm4, 0x40($val,$index)
1391 movdqa %xmm5, 0x50($val,$index)
1394 .size ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
1396 ################################################################################
1397 # void ecp_nistz256_gather_w5(uint64_t *val, uint64_t *in_t, int index);
1398 .globl ecp_nistz256_gather_w5
1399 .type ecp_nistz256_gather_w5,\@abi-omnipotent
1401 ecp_nistz256_gather_w5:
1403 $code.=<<___ if ($avx>1);
1404 mov OPENSSL_ia32cap_P+8(%rip), %eax
1406 jnz .Lavx2_gather_w5
1408 $code.=<<___ if ($win64);
1409 lea -0x88(%rsp), %rax
1410 .LSEH_begin_ecp_nistz256_gather_w5:
1411 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1412 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1413 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1414 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1415 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1416 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1417 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1418 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1419 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1420 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1421 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1424 movdqa .LOne(%rip), $ONE
1435 pshufd \$0, $INDEX, $INDEX
1438 .Lselect_loop_sse_w5:
1442 pcmpeqd $INDEX, $TMP0
1444 movdqa 16*0($in_t), $T0a
1445 movdqa 16*1($in_t), $T0b
1446 movdqa 16*2($in_t), $T0c
1447 movdqa 16*3($in_t), $T0d
1448 movdqa 16*4($in_t), $T0e
1449 movdqa 16*5($in_t), $T0f
1450 lea 16*6($in_t), $in_t
1466 jnz .Lselect_loop_sse_w5
1468 movdqu $Ra, 16*0($val)
1469 movdqu $Rb, 16*1($val)
1470 movdqu $Rc, 16*2($val)
1471 movdqu $Rd, 16*3($val)
1472 movdqu $Re, 16*4($val)
1473 movdqu $Rf, 16*5($val)
1475 $code.=<<___ if ($win64);
1476 movaps (%rsp), %xmm6
1477 movaps 0x10(%rsp), %xmm7
1478 movaps 0x20(%rsp), %xmm8
1479 movaps 0x30(%rsp), %xmm9
1480 movaps 0x40(%rsp), %xmm10
1481 movaps 0x50(%rsp), %xmm11
1482 movaps 0x60(%rsp), %xmm12
1483 movaps 0x70(%rsp), %xmm13
1484 movaps 0x80(%rsp), %xmm14
1485 movaps 0x90(%rsp), %xmm15
1486 lea 0xa8(%rsp), %rsp
1487 .LSEH_end_ecp_nistz256_gather_w5:
1491 .size ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
1493 ################################################################################
1494 # void ecp_nistz256_scatter_w7(uint64_t *val, uint64_t *in_t, int index);
1495 .globl ecp_nistz256_scatter_w7
1496 .type ecp_nistz256_scatter_w7,\@abi-omnipotent
1498 ecp_nistz256_scatter_w7:
1499 movdqu 0x00($in_t), %xmm0
1501 movdqu 0x10($in_t), %xmm1
1502 movdqu 0x20($in_t), %xmm2
1503 movdqu 0x30($in_t), %xmm3
1504 movdqa %xmm0, 0x00($val,$index)
1505 movdqa %xmm1, 0x10($val,$index)
1506 movdqa %xmm2, 0x20($val,$index)
1507 movdqa %xmm3, 0x30($val,$index)
1510 .size ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
1512 ################################################################################
1513 # void ecp_nistz256_gather_w7(uint64_t *val, uint64_t *in_t, int index);
1514 .globl ecp_nistz256_gather_w7
1515 .type ecp_nistz256_gather_w7,\@abi-omnipotent
1517 ecp_nistz256_gather_w7:
1519 $code.=<<___ if ($avx>1);
1520 mov OPENSSL_ia32cap_P+8(%rip), %eax
1522 jnz .Lavx2_gather_w7
1524 $code.=<<___ if ($win64);
1525 lea -0x88(%rsp), %rax
1526 .LSEH_begin_ecp_nistz256_gather_w7:
1527 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1528 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1529 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1530 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1531 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1532 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1533 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1534 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1535 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1536 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1537 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1540 movdqa .LOne(%rip), $M0
1549 pshufd \$0, $INDEX, $INDEX
1552 .Lselect_loop_sse_w7:
1555 movdqa 16*0($in_t), $T0a
1556 movdqa 16*1($in_t), $T0b
1557 pcmpeqd $INDEX, $TMP0
1558 movdqa 16*2($in_t), $T0c
1559 movdqa 16*3($in_t), $T0d
1560 lea 16*4($in_t), $in_t
1569 prefetcht0 255($in_t)
1573 jnz .Lselect_loop_sse_w7
1575 movdqu $Ra, 16*0($val)
1576 movdqu $Rb, 16*1($val)
1577 movdqu $Rc, 16*2($val)
1578 movdqu $Rd, 16*3($val)
1580 $code.=<<___ if ($win64);
1581 movaps (%rsp), %xmm6
1582 movaps 0x10(%rsp), %xmm7
1583 movaps 0x20(%rsp), %xmm8
1584 movaps 0x30(%rsp), %xmm9
1585 movaps 0x40(%rsp), %xmm10
1586 movaps 0x50(%rsp), %xmm11
1587 movaps 0x60(%rsp), %xmm12
1588 movaps 0x70(%rsp), %xmm13
1589 movaps 0x80(%rsp), %xmm14
1590 movaps 0x90(%rsp), %xmm15
1591 lea 0xa8(%rsp), %rsp
1592 .LSEH_end_ecp_nistz256_gather_w7:
1596 .size ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
1600 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1601 my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4));
1602 my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9));
1603 my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14));
1606 ################################################################################
1607 # void ecp_nistz256_avx2_gather_w5(uint64_t *val, uint64_t *in_t, int index);
1608 .type ecp_nistz256_avx2_gather_w5,\@abi-omnipotent
1610 ecp_nistz256_avx2_gather_w5:
1614 $code.=<<___ if ($win64);
1615 lea -0x88(%rsp), %rax
1616 .LSEH_begin_ecp_nistz256_avx2_gather_w5:
1617 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1618 .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
1619 .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
1620 .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
1621 .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
1622 .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
1623 .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
1624 .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
1625 .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
1626 .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
1627 .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
1630 vmovdqa .LTwo(%rip), $TWO
1636 vmovdqa .LOne(%rip), $M0
1637 vmovdqa .LTwo(%rip), $M1
1640 vpermd $INDEX, $Ra, $INDEX
1643 .Lselect_loop_avx2_w5:
1645 vmovdqa 32*0($in_t), $T0a
1646 vmovdqa 32*1($in_t), $T0b
1647 vmovdqa 32*2($in_t), $T0c
1649 vmovdqa 32*3($in_t), $T1a
1650 vmovdqa 32*4($in_t), $T1b
1651 vmovdqa 32*5($in_t), $T1c
1653 vpcmpeqd $INDEX, $M0, $TMP0
1654 vpcmpeqd $INDEX, $M1, $TMP1
1656 vpaddd $TWO, $M0, $M0
1657 vpaddd $TWO, $M1, $M1
1658 lea 32*6($in_t), $in_t
1660 vpand $TMP0, $T0a, $T0a
1661 vpand $TMP0, $T0b, $T0b
1662 vpand $TMP0, $T0c, $T0c
1663 vpand $TMP1, $T1a, $T1a
1664 vpand $TMP1, $T1b, $T1b
1665 vpand $TMP1, $T1c, $T1c
1667 vpxor $T0a, $Ra, $Ra
1668 vpxor $T0b, $Rb, $Rb
1669 vpxor $T0c, $Rc, $Rc
1670 vpxor $T1a, $Ra, $Ra
1671 vpxor $T1b, $Rb, $Rb
1672 vpxor $T1c, $Rc, $Rc
1675 jnz .Lselect_loop_avx2_w5
1677 vmovdqu $Ra, 32*0($val)
1678 vmovdqu $Rb, 32*1($val)
1679 vmovdqu $Rc, 32*2($val)
1682 $code.=<<___ if ($win64);
1683 movaps (%rsp), %xmm6
1684 movaps 0x10(%rsp), %xmm7
1685 movaps 0x20(%rsp), %xmm8
1686 movaps 0x30(%rsp), %xmm9
1687 movaps 0x40(%rsp), %xmm10
1688 movaps 0x50(%rsp), %xmm11
1689 movaps 0x60(%rsp), %xmm12
1690 movaps 0x70(%rsp), %xmm13
1691 movaps 0x80(%rsp), %xmm14
1692 movaps 0x90(%rsp), %xmm15
1693 lea 0xa8(%rsp), %rsp
1694 .LSEH_end_ecp_nistz256_avx2_gather_w5:
1698 .size ecp_nistz256_avx2_gather_w5,.-ecp_nistz256_avx2_gather_w5
1702 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1703 my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3));
1704 my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7));
1705 my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11));
1706 my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15));
1710 ################################################################################
1711 # void ecp_nistz256_avx2_gather_w7(uint64_t *val, uint64_t *in_t, int index);
1712 .globl ecp_nistz256_avx2_gather_w7
1713 .type ecp_nistz256_avx2_gather_w7,\@abi-omnipotent
1715 ecp_nistz256_avx2_gather_w7:
1719 $code.=<<___ if ($win64);
1720 lea -0x88(%rsp), %rax
1721 .LSEH_begin_ecp_nistz256_avx2_gather_w7:
1722 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1723 .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
1724 .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
1725 .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
1726 .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
1727 .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
1728 .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
1729 .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
1730 .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
1731 .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
1732 .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
1735 vmovdqa .LThree(%rip), $THREE
1740 vmovdqa .LOne(%rip), $M0
1741 vmovdqa .LTwo(%rip), $M1
1742 vmovdqa .LThree(%rip), $M2
1745 vpermd $INDEX, $Ra, $INDEX
1746 # Skip index = 0, because it is implicitly the point at infinity
1749 .Lselect_loop_avx2_w7:
1751 vmovdqa 32*0($in_t), $T0a
1752 vmovdqa 32*1($in_t), $T0b
1754 vmovdqa 32*2($in_t), $T1a
1755 vmovdqa 32*3($in_t), $T1b
1757 vmovdqa 32*4($in_t), $T2a
1758 vmovdqa 32*5($in_t), $T2b
1760 vpcmpeqd $INDEX, $M0, $TMP0
1761 vpcmpeqd $INDEX, $M1, $TMP1
1762 vpcmpeqd $INDEX, $M2, $TMP2
1764 vpaddd $THREE, $M0, $M0
1765 vpaddd $THREE, $M1, $M1
1766 vpaddd $THREE, $M2, $M2
1767 lea 32*6($in_t), $in_t
1769 vpand $TMP0, $T0a, $T0a
1770 vpand $TMP0, $T0b, $T0b
1771 vpand $TMP1, $T1a, $T1a
1772 vpand $TMP1, $T1b, $T1b
1773 vpand $TMP2, $T2a, $T2a
1774 vpand $TMP2, $T2b, $T2b
1776 vpxor $T0a, $Ra, $Ra
1777 vpxor $T0b, $Rb, $Rb
1778 vpxor $T1a, $Ra, $Ra
1779 vpxor $T1b, $Rb, $Rb
1780 vpxor $T2a, $Ra, $Ra
1781 vpxor $T2b, $Rb, $Rb
1784 jnz .Lselect_loop_avx2_w7
1787 vmovdqa 32*0($in_t), $T0a
1788 vmovdqa 32*1($in_t), $T0b
1790 vpcmpeqd $INDEX, $M0, $TMP0
1792 vpand $TMP0, $T0a, $T0a
1793 vpand $TMP0, $T0b, $T0b
1795 vpxor $T0a, $Ra, $Ra
1796 vpxor $T0b, $Rb, $Rb
1798 vmovdqu $Ra, 32*0($val)
1799 vmovdqu $Rb, 32*1($val)
1802 $code.=<<___ if ($win64);
1803 movaps (%rsp), %xmm6
1804 movaps 0x10(%rsp), %xmm7
1805 movaps 0x20(%rsp), %xmm8
1806 movaps 0x30(%rsp), %xmm9
1807 movaps 0x40(%rsp), %xmm10
1808 movaps 0x50(%rsp), %xmm11
1809 movaps 0x60(%rsp), %xmm12
1810 movaps 0x70(%rsp), %xmm13
1811 movaps 0x80(%rsp), %xmm14
1812 movaps 0x90(%rsp), %xmm15
1813 lea 0xa8(%rsp), %rsp
1814 .LSEH_end_ecp_nistz256_avx2_gather_w7:
1818 .size ecp_nistz256_avx2_gather_w7,.-ecp_nistz256_avx2_gather_w7
1822 .globl ecp_nistz256_avx2_gather_w7
1823 .type ecp_nistz256_avx2_gather_w7,\@function,3
1825 ecp_nistz256_avx2_gather_w7:
1826 .byte 0x0f,0x0b # ud2
1828 .size ecp_nistz256_avx2_gather_w7,.-ecp_nistz256_avx2_gather_w7
1832 ########################################################################
1833 # This block implements higher level point_double, point_add and
1834 # point_add_affine. The key to performance in this case is to allow
1835 # out-of-order execution logic to overlap computations from next step
1836 # with tail processing from current step. By using tailored calling
1837 # sequence we minimize inter-step overhead to give processor better
1838 # shot at overlapping operations...
1840 # You will notice that input data is copied to stack. Trouble is that
1841 # there are no registers to spare for holding original pointers and
1842 # reloading them, pointers, would create undesired dependencies on
1843 # effective addresses calculation paths. In other words it's too done
1844 # to favour out-of-order execution logic.
1845 # <appro@openssl.org>
1847 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
1848 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
1849 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4);
1850 my ($poly1,$poly3)=($acc6,$acc7);
1852 sub load_for_mul () {
1853 my ($a,$b,$src0) = @_;
1854 my $bias = $src0 eq "%rax" ? 0 : -128;
1860 lea $bias+$a, $a_ptr
1865 sub load_for_sqr () {
1867 my $bias = $src0 eq "%rax" ? 0 : -128;
1871 lea $bias+$a, $a_ptr
1877 ########################################################################
1878 # operate in 4-5-0-1 "name space" that matches multiplication output
1880 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
1883 .type __ecp_nistz256_add_toq,\@abi-omnipotent
1885 __ecp_nistz256_add_toq:
1886 add 8*0($b_ptr), $a0
1887 adc 8*1($b_ptr), $a1
1889 adc 8*2($b_ptr), $a2
1890 adc 8*3($b_ptr), $a3
1904 mov $a0, 8*0($r_ptr)
1906 mov $a1, 8*1($r_ptr)
1908 mov $a2, 8*2($r_ptr)
1909 mov $a3, 8*3($r_ptr)
1912 .size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq
1914 .type __ecp_nistz256_sub_fromq,\@abi-omnipotent
1916 __ecp_nistz256_sub_fromq:
1917 sub 8*0($b_ptr), $a0
1918 sbb 8*1($b_ptr), $a1
1920 sbb 8*2($b_ptr), $a2
1921 sbb 8*3($b_ptr), $a3
1935 mov $a0, 8*0($r_ptr)
1937 mov $a1, 8*1($r_ptr)
1939 mov $a2, 8*2($r_ptr)
1940 mov $a3, 8*3($r_ptr)
1943 .size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq
1945 .type __ecp_nistz256_subq,\@abi-omnipotent
1947 __ecp_nistz256_subq:
1970 .size __ecp_nistz256_subq,.-__ecp_nistz256_subq
1972 .type __ecp_nistz256_mul_by_2q,\@abi-omnipotent
1974 __ecp_nistz256_mul_by_2q:
1975 add $a0, $a0 # a0:a3+a0:a3
1993 mov $a0, 8*0($r_ptr)
1995 mov $a1, 8*1($r_ptr)
1997 mov $a2, 8*2($r_ptr)
1998 mov $a3, 8*3($r_ptr)
2001 .size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q
2006 my ($src0,$sfx,$bias);
2007 my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
2015 .globl ecp_nistz256_point_double
2016 .type ecp_nistz256_point_double,\@function,2
2018 ecp_nistz256_point_double:
2020 $code.=<<___ if ($addx);
2022 and OPENSSL_ia32cap_P+8(%rip), %ecx
2032 .type ecp_nistz256_point_doublex,\@function,2
2034 ecp_nistz256_point_doublex:
2047 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x
2048 mov $a_ptr, $b_ptr # backup copy
2049 movdqu 0x10($a_ptr), %xmm1
2050 mov 0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order
2051 mov 0x20+8*1($a_ptr), $acc5
2052 mov 0x20+8*2($a_ptr), $acc0
2053 mov 0x20+8*3($a_ptr), $acc1
2054 mov .Lpoly+8*1(%rip), $poly1
2055 mov .Lpoly+8*3(%rip), $poly3
2056 movdqa %xmm0, $in_x(%rsp)
2057 movdqa %xmm1, $in_x+0x10(%rsp)
2058 lea 0x20($r_ptr), $acc2
2059 lea 0x40($r_ptr), $acc3
2064 lea $S(%rsp), $r_ptr
2065 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(S, in_y);
2067 mov 0x40+8*0($a_ptr), $src0
2068 mov 0x40+8*1($a_ptr), $acc6
2069 mov 0x40+8*2($a_ptr), $acc7
2070 mov 0x40+8*3($a_ptr), $acc0
2071 lea 0x40-$bias($a_ptr), $a_ptr
2072 lea $Zsqr(%rsp), $r_ptr
2073 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Zsqr, in_z);
2075 `&load_for_sqr("$S(%rsp)", "$src0")`
2076 lea $S(%rsp), $r_ptr
2077 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(S, S);
2079 mov 0x20($b_ptr), $src0 # $b_ptr is still valid
2080 mov 0x40+8*0($b_ptr), $acc1
2081 mov 0x40+8*1($b_ptr), $acc2
2082 mov 0x40+8*2($b_ptr), $acc3
2083 mov 0x40+8*3($b_ptr), $acc4
2084 lea 0x40-$bias($b_ptr), $a_ptr
2085 lea 0x20($b_ptr), $b_ptr
2087 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, in_z, in_y);
2088 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(res_z, res_z);
2090 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
2091 mov $in_x+8*1(%rsp), $acc5
2092 lea $Zsqr(%rsp), $b_ptr
2093 mov $in_x+8*2(%rsp), $acc0
2094 mov $in_x+8*3(%rsp), $acc1
2095 lea $M(%rsp), $r_ptr
2096 call __ecp_nistz256_add_to$x # p256_add(M, in_x, Zsqr);
2098 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
2099 mov $in_x+8*1(%rsp), $acc5
2100 lea $Zsqr(%rsp), $b_ptr
2101 mov $in_x+8*2(%rsp), $acc0
2102 mov $in_x+8*3(%rsp), $acc1
2103 lea $Zsqr(%rsp), $r_ptr
2104 call __ecp_nistz256_sub_from$x # p256_sub(Zsqr, in_x, Zsqr);
2106 `&load_for_sqr("$S(%rsp)", "$src0")`
2108 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_y, S);
2111 ######## ecp_nistz256_div_by_2(res_y, res_y); ##########################
2112 # operate in 4-5-6-7 "name space" that matches squaring output
2114 my ($poly1,$poly3)=($a_ptr,$t1);
2115 my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2);
2128 xor $a_ptr, $a_ptr # borrow $a_ptr
2137 mov $a1, $t0 # a0:a3>>1
2148 mov $a0, 8*0($r_ptr)
2150 mov $a1, 8*1($r_ptr)
2154 mov $a2, 8*2($r_ptr)
2155 mov $a3, 8*3($r_ptr)
2159 `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")`
2160 lea $M(%rsp), $r_ptr
2161 call __ecp_nistz256_mul_mont$x # p256_mul_mont(M, M, Zsqr);
2163 lea $tmp0(%rsp), $r_ptr
2164 call __ecp_nistz256_mul_by_2$x
2166 lea $M(%rsp), $b_ptr
2167 lea $M(%rsp), $r_ptr
2168 call __ecp_nistz256_add_to$x # p256_mul_by_3(M, M);
2170 `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")`
2171 lea $S(%rsp), $r_ptr
2172 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, in_x);
2174 lea $tmp0(%rsp), $r_ptr
2175 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(tmp0, S);
2177 `&load_for_sqr("$M(%rsp)", "$src0")`
2179 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_x, M);
2181 lea $tmp0(%rsp), $b_ptr
2182 mov $acc6, $acc0 # harmonize sqr output and sub input
2186 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, tmp0);
2188 mov $S+8*0(%rsp), $t0
2189 mov $S+8*1(%rsp), $t1
2190 mov $S+8*2(%rsp), $t2
2191 mov $S+8*3(%rsp), $acc2 # "4-5-0-1" order
2192 lea $S(%rsp), $r_ptr
2193 call __ecp_nistz256_sub$x # p256_sub(S, S, res_x);
2196 lea $M(%rsp), $b_ptr
2197 mov $acc4, $acc6 # harmonize sub output and mul input
2199 mov $acc4, $S+8*0(%rsp) # have to save:-(
2201 mov $acc5, $S+8*1(%rsp)
2203 mov $acc0, $S+8*2(%rsp)
2204 lea $S-$bias(%rsp), $a_ptr
2206 mov $acc1, $S+8*3(%rsp)
2208 lea $S(%rsp), $r_ptr
2209 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, M);
2213 call __ecp_nistz256_sub_from$x # p256_sub(res_y, S, res_y);
2223 .size ecp_nistz256_point_double$sfx,.-ecp_nistz256_point_double$sfx
2230 my ($src0,$sfx,$bias);
2231 my ($H,$Hsqr,$R,$Rsqr,$Hcub,
2233 $res_x,$res_y,$res_z,
2234 $in1_x,$in1_y,$in1_z,
2235 $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17));
2236 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
2244 .globl ecp_nistz256_point_add
2245 .type ecp_nistz256_point_add,\@function,3
2247 ecp_nistz256_point_add:
2249 $code.=<<___ if ($addx);
2251 and OPENSSL_ia32cap_P+8(%rip), %ecx
2261 .type ecp_nistz256_point_addx,\@function,3
2263 ecp_nistz256_point_addx:
2276 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
2277 movdqu 0x10($a_ptr), %xmm1
2278 movdqu 0x20($a_ptr), %xmm2
2279 movdqu 0x30($a_ptr), %xmm3
2280 movdqu 0x40($a_ptr), %xmm4
2281 movdqu 0x50($a_ptr), %xmm5
2282 mov $a_ptr, $b_ptr # reassign
2283 mov $b_org, $a_ptr # reassign
2284 movdqa %xmm0, $in1_x(%rsp)
2285 movdqa %xmm1, $in1_x+0x10(%rsp)
2287 movdqa %xmm2, $in1_y(%rsp)
2288 movdqa %xmm3, $in1_y+0x10(%rsp)
2290 movdqa %xmm4, $in1_z(%rsp)
2291 movdqa %xmm5, $in1_z+0x10(%rsp)
2294 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr
2295 pshufd \$0xb1, %xmm3, %xmm5
2296 movdqu 0x10($a_ptr), %xmm1
2297 movdqu 0x20($a_ptr), %xmm2
2299 movdqu 0x30($a_ptr), %xmm3
2300 mov 0x40+8*0($a_ptr), $src0 # load original in2_z
2301 mov 0x40+8*1($a_ptr), $acc6
2302 mov 0x40+8*2($a_ptr), $acc7
2303 mov 0x40+8*3($a_ptr), $acc0
2304 movdqa %xmm0, $in2_x(%rsp)
2305 pshufd \$0x1e, %xmm5, %xmm4
2306 movdqa %xmm1, $in2_x+0x10(%rsp)
2308 movq $r_ptr, %xmm0 # save $r_ptr
2309 movdqa %xmm2, $in2_y(%rsp)
2310 movdqa %xmm3, $in2_y+0x10(%rsp)
2316 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2317 mov $src0, $in2_z+8*0(%rsp) # make in2_z copy
2318 mov $acc6, $in2_z+8*1(%rsp)
2319 mov $acc7, $in2_z+8*2(%rsp)
2320 mov $acc0, $in2_z+8*3(%rsp)
2321 lea $Z2sqr(%rsp), $r_ptr # Z2^2
2322 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z2sqr, in2_z);
2324 pcmpeqd %xmm4, %xmm5
2325 pshufd \$0xb1, %xmm3, %xmm4
2327 pshufd \$0, %xmm5, %xmm5 # in1infty
2328 pshufd \$0x1e, %xmm4, %xmm3
2331 pcmpeqd %xmm3, %xmm4
2332 pshufd \$0, %xmm4, %xmm4 # in2infty
2333 mov 0x40+8*0($b_ptr), $src0 # load original in1_z
2334 mov 0x40+8*1($b_ptr), $acc6
2335 mov 0x40+8*2($b_ptr), $acc7
2336 mov 0x40+8*3($b_ptr), $acc0
2338 lea 0x40-$bias($b_ptr), $a_ptr
2339 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2340 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2342 `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")`
2343 lea $S1(%rsp), $r_ptr # S1 = Z2^3
2344 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, Z2sqr, in2_z);
2346 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2347 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2348 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2350 `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")`
2351 lea $S1(%rsp), $r_ptr # S1 = Y1*Z2^3
2352 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, S1, in1_y);
2354 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2355 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2356 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2358 lea $S1(%rsp), $b_ptr
2359 lea $R(%rsp), $r_ptr # R = S2 - S1
2360 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, S1);
2362 or $acc5, $acc4 # see if result is zero
2366 por %xmm5, %xmm2 # in1infty || in2infty
2369 `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2370 lea $U1(%rsp), $r_ptr # U1 = X1*Z2^2
2371 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U1, in1_x, Z2sqr);
2373 `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")`
2374 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2375 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in2_x, Z1sqr);
2377 lea $U1(%rsp), $b_ptr
2378 lea $H(%rsp), $r_ptr # H = U2 - U1
2379 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, U1);
2381 or $acc5, $acc4 # see if result is zero
2385 .byte 0x3e # predict taken
2386 jnz .Ladd_proceed$x # is_equal(U1,U2)?
2390 jnz .Ladd_proceed$x # (in1infty || in2infty)?
2392 jz .Ladd_proceed$x # is_equal(S1,S2)?
2394 movq %xmm0, $r_ptr # restore $r_ptr
2396 movdqu %xmm0, 0x00($r_ptr)
2397 movdqu %xmm0, 0x10($r_ptr)
2398 movdqu %xmm0, 0x20($r_ptr)
2399 movdqu %xmm0, 0x30($r_ptr)
2400 movdqu %xmm0, 0x40($r_ptr)
2401 movdqu %xmm0, 0x50($r_ptr)
2406 `&load_for_sqr("$R(%rsp)", "$src0")`
2407 lea $Rsqr(%rsp), $r_ptr # R^2
2408 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2410 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2411 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2412 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2414 `&load_for_sqr("$H(%rsp)", "$src0")`
2415 lea $Hsqr(%rsp), $r_ptr # H^2
2416 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2418 `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")`
2419 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2420 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, res_z, in2_z);
2422 `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")`
2423 lea $Hcub(%rsp), $r_ptr # H^3
2424 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2426 `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")`
2427 lea $U2(%rsp), $r_ptr # U1*H^2
2428 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, U1, Hsqr);
2431 #######################################################################
2432 # operate in 4-5-0-1 "name space" that matches multiplication output
2434 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2435 my ($poly1, $poly3)=($acc6,$acc7);
2438 #lea $U2(%rsp), $a_ptr
2439 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2440 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2442 add $acc0, $acc0 # a0:a3+a0:a3
2443 lea $Rsqr(%rsp), $a_ptr
2460 mov 8*0($a_ptr), $t0
2462 mov 8*1($a_ptr), $t1
2464 mov 8*2($a_ptr), $t2
2466 mov 8*3($a_ptr), $t3
2468 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2470 lea $Hcub(%rsp), $b_ptr
2471 lea $res_x(%rsp), $r_ptr
2472 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2474 mov $U2+8*0(%rsp), $t0
2475 mov $U2+8*1(%rsp), $t1
2476 mov $U2+8*2(%rsp), $t2
2477 mov $U2+8*3(%rsp), $t3
2478 lea $res_y(%rsp), $r_ptr
2480 call __ecp_nistz256_sub$x # p256_sub(res_y, U2, res_x);
2482 mov $acc0, 8*0($r_ptr) # save the result, as
2483 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2484 mov $acc2, 8*2($r_ptr)
2485 mov $acc3, 8*3($r_ptr)
2489 `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")`
2490 lea $S2(%rsp), $r_ptr
2491 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S1, Hcub);
2493 `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")`
2494 lea $res_y(%rsp), $r_ptr
2495 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_y, R, res_y);
2497 lea $S2(%rsp), $b_ptr
2498 lea $res_y(%rsp), $r_ptr
2499 call __ecp_nistz256_sub_from$x # p256_sub(res_y, res_y, S2);
2501 movq %xmm0, $r_ptr # restore $r_ptr
2503 movdqa %xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty);
2505 pandn $res_z(%rsp), %xmm0
2507 pandn $res_z+0x10(%rsp), %xmm1
2509 pand $in2_z(%rsp), %xmm2
2510 pand $in2_z+0x10(%rsp), %xmm3
2514 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2520 pand $in1_z(%rsp), %xmm2
2521 pand $in1_z+0x10(%rsp), %xmm3
2524 movdqu %xmm2, 0x40($r_ptr)
2525 movdqu %xmm3, 0x50($r_ptr)
2527 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
2529 pandn $res_x(%rsp), %xmm0
2531 pandn $res_x+0x10(%rsp), %xmm1
2533 pand $in2_x(%rsp), %xmm2
2534 pand $in2_x+0x10(%rsp), %xmm3
2538 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
2544 pand $in1_x(%rsp), %xmm2
2545 pand $in1_x+0x10(%rsp), %xmm3
2548 movdqu %xmm2, 0x00($r_ptr)
2549 movdqu %xmm3, 0x10($r_ptr)
2551 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
2553 pandn $res_y(%rsp), %xmm0
2555 pandn $res_y+0x10(%rsp), %xmm1
2557 pand $in2_y(%rsp), %xmm2
2558 pand $in2_y+0x10(%rsp), %xmm3
2562 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
2568 pand $in1_y(%rsp), %xmm2
2569 pand $in1_y+0x10(%rsp), %xmm3
2572 movdqu %xmm2, 0x20($r_ptr)
2573 movdqu %xmm3, 0x30($r_ptr)
2584 .size ecp_nistz256_point_add$sfx,.-ecp_nistz256_point_add$sfx
2589 sub gen_add_affine () {
2591 my ($src0,$sfx,$bias);
2592 my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr,
2593 $res_x,$res_y,$res_z,
2594 $in1_x,$in1_y,$in1_z,
2595 $in2_x,$in2_y)=map(32*$_,(0..14));
2604 .globl ecp_nistz256_point_add_affine
2605 .type ecp_nistz256_point_add_affine,\@function,3
2607 ecp_nistz256_point_add_affine:
2609 $code.=<<___ if ($addx);
2611 and OPENSSL_ia32cap_P+8(%rip), %ecx
2613 je .Lpoint_add_affinex
2621 .type ecp_nistz256_point_add_affinex,\@function,3
2623 ecp_nistz256_point_add_affinex:
2624 .Lpoint_add_affinex:
2636 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
2637 mov $b_org, $b_ptr # reassign
2638 movdqu 0x10($a_ptr), %xmm1
2639 movdqu 0x20($a_ptr), %xmm2
2640 movdqu 0x30($a_ptr), %xmm3
2641 movdqu 0x40($a_ptr), %xmm4
2642 movdqu 0x50($a_ptr), %xmm5
2643 mov 0x40+8*0($a_ptr), $src0 # load original in1_z
2644 mov 0x40+8*1($a_ptr), $acc6
2645 mov 0x40+8*2($a_ptr), $acc7
2646 mov 0x40+8*3($a_ptr), $acc0
2647 movdqa %xmm0, $in1_x(%rsp)
2648 movdqa %xmm1, $in1_x+0x10(%rsp)
2650 movdqa %xmm2, $in1_y(%rsp)
2651 movdqa %xmm3, $in1_y+0x10(%rsp)
2653 movdqa %xmm4, $in1_z(%rsp)
2654 movdqa %xmm5, $in1_z+0x10(%rsp)
2657 movdqu 0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr
2658 pshufd \$0xb1, %xmm3, %xmm5
2659 movdqu 0x10($b_ptr), %xmm1
2660 movdqu 0x20($b_ptr), %xmm2
2662 movdqu 0x30($b_ptr), %xmm3
2663 movdqa %xmm0, $in2_x(%rsp)
2664 pshufd \$0x1e, %xmm5, %xmm4
2665 movdqa %xmm1, $in2_x+0x10(%rsp)
2667 movq $r_ptr, %xmm0 # save $r_ptr
2668 movdqa %xmm2, $in2_y(%rsp)
2669 movdqa %xmm3, $in2_y+0x10(%rsp)
2675 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2676 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2677 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2679 pcmpeqd %xmm4, %xmm5
2680 pshufd \$0xb1, %xmm3, %xmm4
2681 mov 0x00($b_ptr), $src0 # $b_ptr is still valid
2682 #lea 0x00($b_ptr), $b_ptr
2683 mov $acc4, $acc1 # harmonize sqr output and mul input
2685 pshufd \$0, %xmm5, %xmm5 # in1infty
2686 pshufd \$0x1e, %xmm4, %xmm3
2691 pcmpeqd %xmm3, %xmm4
2692 pshufd \$0, %xmm4, %xmm4 # in2infty
2694 lea $Z1sqr-$bias(%rsp), $a_ptr
2696 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2697 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, Z1sqr, in2_x);
2699 lea $in1_x(%rsp), $b_ptr
2700 lea $H(%rsp), $r_ptr # H = U2 - U1
2701 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, in1_x);
2703 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2704 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2705 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2707 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2708 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2709 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2711 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2712 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2713 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2715 lea $in1_y(%rsp), $b_ptr
2716 lea $R(%rsp), $r_ptr # R = S2 - S1
2717 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, in1_y);
2719 `&load_for_sqr("$H(%rsp)", "$src0")`
2720 lea $Hsqr(%rsp), $r_ptr # H^2
2721 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2723 `&load_for_sqr("$R(%rsp)", "$src0")`
2724 lea $Rsqr(%rsp), $r_ptr # R^2
2725 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2727 `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")`
2728 lea $Hcub(%rsp), $r_ptr # H^3
2729 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2731 `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2732 lea $U2(%rsp), $r_ptr # U1*H^2
2733 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in1_x, Hsqr);
2736 #######################################################################
2737 # operate in 4-5-0-1 "name space" that matches multiplication output
2739 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2740 my ($poly1, $poly3)=($acc6,$acc7);
2743 #lea $U2(%rsp), $a_ptr
2744 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2745 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2747 add $acc0, $acc0 # a0:a3+a0:a3
2748 lea $Rsqr(%rsp), $a_ptr
2765 mov 8*0($a_ptr), $t0
2767 mov 8*1($a_ptr), $t1
2769 mov 8*2($a_ptr), $t2
2771 mov 8*3($a_ptr), $t3
2773 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2775 lea $Hcub(%rsp), $b_ptr
2776 lea $res_x(%rsp), $r_ptr
2777 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2779 mov $U2+8*0(%rsp), $t0
2780 mov $U2+8*1(%rsp), $t1
2781 mov $U2+8*2(%rsp), $t2
2782 mov $U2+8*3(%rsp), $t3
2783 lea $H(%rsp), $r_ptr
2785 call __ecp_nistz256_sub$x # p256_sub(H, U2, res_x);
2787 mov $acc0, 8*0($r_ptr) # save the result, as
2788 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2789 mov $acc2, 8*2($r_ptr)
2790 mov $acc3, 8*3($r_ptr)
2794 `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")`
2795 lea $S2(%rsp), $r_ptr
2796 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Hcub, in1_y);
2798 `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")`
2799 lea $H(%rsp), $r_ptr
2800 call __ecp_nistz256_mul_mont$x # p256_mul_mont(H, H, R);
2802 lea $S2(%rsp), $b_ptr
2803 lea $res_y(%rsp), $r_ptr
2804 call __ecp_nistz256_sub_from$x # p256_sub(res_y, H, S2);
2806 movq %xmm0, $r_ptr # restore $r_ptr
2808 movdqa %xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty);
2810 pandn $res_z(%rsp), %xmm0
2812 pandn $res_z+0x10(%rsp), %xmm1
2814 pand .LONE_mont(%rip), %xmm2
2815 pand .LONE_mont+0x10(%rip), %xmm3
2819 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2825 pand $in1_z(%rsp), %xmm2
2826 pand $in1_z+0x10(%rsp), %xmm3
2829 movdqu %xmm2, 0x40($r_ptr)
2830 movdqu %xmm3, 0x50($r_ptr)
2832 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
2834 pandn $res_x(%rsp), %xmm0
2836 pandn $res_x+0x10(%rsp), %xmm1
2838 pand $in2_x(%rsp), %xmm2
2839 pand $in2_x+0x10(%rsp), %xmm3
2843 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
2849 pand $in1_x(%rsp), %xmm2
2850 pand $in1_x+0x10(%rsp), %xmm3
2853 movdqu %xmm2, 0x00($r_ptr)
2854 movdqu %xmm3, 0x10($r_ptr)
2856 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
2858 pandn $res_y(%rsp), %xmm0
2860 pandn $res_y+0x10(%rsp), %xmm1
2862 pand $in2_y(%rsp), %xmm2
2863 pand $in2_y+0x10(%rsp), %xmm3
2867 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
2873 pand $in1_y(%rsp), %xmm2
2874 pand $in1_y+0x10(%rsp), %xmm3
2877 movdqu %xmm2, 0x20($r_ptr)
2878 movdqu %xmm3, 0x30($r_ptr)
2888 .size ecp_nistz256_point_add_affine$sfx,.-ecp_nistz256_point_add_affine$sfx
2891 &gen_add_affine("q");
2893 ########################################################################
2897 ########################################################################
2898 # operate in 4-5-0-1 "name space" that matches multiplication output
2900 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2903 .type __ecp_nistz256_add_tox,\@abi-omnipotent
2905 __ecp_nistz256_add_tox:
2907 adc 8*0($b_ptr), $a0
2908 adc 8*1($b_ptr), $a1
2910 adc 8*2($b_ptr), $a2
2911 adc 8*3($b_ptr), $a3
2926 mov $a0, 8*0($r_ptr)
2928 mov $a1, 8*1($r_ptr)
2930 mov $a2, 8*2($r_ptr)
2931 mov $a3, 8*3($r_ptr)
2934 .size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox
2936 .type __ecp_nistz256_sub_fromx,\@abi-omnipotent
2938 __ecp_nistz256_sub_fromx:
2940 sbb 8*0($b_ptr), $a0
2941 sbb 8*1($b_ptr), $a1
2943 sbb 8*2($b_ptr), $a2
2944 sbb 8*3($b_ptr), $a3
2959 mov $a0, 8*0($r_ptr)
2961 mov $a1, 8*1($r_ptr)
2963 mov $a2, 8*2($r_ptr)
2964 mov $a3, 8*3($r_ptr)
2967 .size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx
2969 .type __ecp_nistz256_subx,\@abi-omnipotent
2971 __ecp_nistz256_subx:
2996 .size __ecp_nistz256_subx,.-__ecp_nistz256_subx
2998 .type __ecp_nistz256_mul_by_2x,\@abi-omnipotent
3000 __ecp_nistz256_mul_by_2x:
3002 adc $a0, $a0 # a0:a3+a0:a3
3021 mov $a0, 8*0($r_ptr)
3023 mov $a1, 8*1($r_ptr)
3025 mov $a2, 8*2($r_ptr)
3026 mov $a3, 8*3($r_ptr)
3029 .size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x
3034 &gen_add_affine("x");
3038 ########################################################################
3039 # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
3041 open TABLE,"<ecp_nistz256_table.c" or
3042 open TABLE,"<${dir}../ecp_nistz256_table.c" or
3043 die "failed to open ecp_nistz256_table.c:",$!;
3048 s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
3052 die "insane number of elements" if ($#arr != 64*16*37-1);
3056 .globl ecp_nistz256_precomputed
3057 .type ecp_nistz256_precomputed,\@object
3059 ecp_nistz256_precomputed:
3061 while (@line=splice(@arr,0,16)) {
3062 print ".long\t",join(',',map { sprintf "0x%08x",$_} @line),"\n";
3065 .size ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
3068 $code =~ s/\`([^\`]*)\`/eval $1/gem;