2 # Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 ##############################################################################
12 # Copyright 2014 Intel Corporation #
14 # Licensed under the Apache License, Version 2.0 (the "License"); #
15 # you may not use this file except in compliance with the License. #
16 # You may obtain a copy of the License at #
18 # http://www.apache.org/licenses/LICENSE-2.0 #
20 # Unless required by applicable law or agreed to in writing, software #
21 # distributed under the License is distributed on an "AS IS" BASIS, #
22 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
23 # See the License for the specific language governing permissions and #
24 # limitations under the License. #
26 ##############################################################################
28 # Developers and authors: #
29 # Shay Gueron (1, 2), and Vlad Krasnov (1) #
30 # (1) Intel Corporation, Israel Development Center #
31 # (2) University of Haifa #
33 # S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with#
36 ##############################################################################
38 # Further optimization by <appro@openssl.org>:
40 # this/original with/without -DECP_NISTZ256_ASM(*)
41 # Opteron +12-49% +110-150%
42 # Bulldozer +14-45% +175-210%
44 # Westmere +12-34% +80-87%
45 # Sandy Bridge +9-35% +110-120%
46 # Ivy Bridge +9-35% +110-125%
47 # Haswell +8-37% +140-160%
48 # Broadwell +18-58% +145-210%
49 # Atom +15-50% +130-180%
50 # VIA Nano +43-160% +300-480%
52 # (*) "without -DECP_NISTZ256_ASM" refers to build with
53 # "enable-ec_nistp_64_gcc_128";
55 # Ranges denote minimum and maximum improvement coefficients depending
56 # on benchmark. Lower coefficients are for ECDSA sign, relatively fastest
57 # server-side operation. Keep in mind that +100% means 2x improvement.
61 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
63 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
65 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
66 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
67 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
68 die "can't locate x86_64-xlate.pl";
70 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
73 if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`
74 =~ /GNU assembler version ([2-9]\.[0-9]+)/) {
75 $avx = ($1>=2.19) + ($1>=2.22);
79 if (!$addx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&
80 `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {
81 $avx = ($1>=2.09) + ($1>=2.10);
85 if (!$addx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&
86 `ml64 2>&1` =~ /Version ([0-9]+)\./) {
87 $avx = ($1>=10) + ($1>=11);
91 if (!$addx && `$ENV{CC} -v 2>&1` =~ /((?:^clang|LLVM) version|.*based on LLVM) ([3-9])\.([0-9]+)/) {
92 my $ver = $2 + $3/100.0; # 3.1->3.01, 3.10->3.10
93 $avx = ($ver>=3.0) + ($ver>=3.01);
99 .extern OPENSSL_ia32cap_P
104 .quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001
106 # 2^512 mod P precomputed for NIST P256 polynomial
108 .quad 0x0000000000000003, 0xfffffffbffffffff, 0xfffffffffffffffe, 0x00000004fffffffd
111 .long 1,1,1,1,1,1,1,1
113 .long 2,2,2,2,2,2,2,2
115 .long 3,3,3,3,3,3,3,3
117 .quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe
121 ################################################################################
122 # void ecp_nistz256_mul_by_2(uint64_t res[4], uint64_t a[4]);
124 my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11));
125 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13");
126 my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx");
130 .globl ecp_nistz256_mul_by_2
131 .type ecp_nistz256_mul_by_2,\@function,2
133 ecp_nistz256_mul_by_2:
144 add $a0, $a0 # a0:a3+a0:a3
148 lea .Lpoly(%rip), $a_ptr
177 .cfi_adjust_cfa_offset -16
181 .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
183 ################################################################################
184 # void ecp_nistz256_div_by_2(uint64_t res[4], uint64_t a[4]);
185 .globl ecp_nistz256_div_by_2
186 .type ecp_nistz256_div_by_2,\@function,2
188 ecp_nistz256_div_by_2:
201 lea .Lpoly(%rip), $a_ptr
212 xor $a_ptr, $a_ptr # borrow $a_ptr
221 mov $a1, $t0 # a0:a3>>1
247 .cfi_adjust_cfa_offset -16
251 .size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2
253 ################################################################################
254 # void ecp_nistz256_mul_by_3(uint64_t res[4], uint64_t a[4]);
255 .globl ecp_nistz256_mul_by_3
256 .type ecp_nistz256_mul_by_3,\@function,2
258 ecp_nistz256_mul_by_3:
269 add $a0, $a0 # a0:a3+a0:a3
281 sbb .Lpoly+8*1(%rip), $a1
284 sbb .Lpoly+8*3(%rip), $a3
293 add 8*0($a_ptr), $a0 # a0:a3+=a_ptr[0:3]
303 sbb .Lpoly+8*1(%rip), $a1
306 sbb .Lpoly+8*3(%rip), $a3
323 .cfi_adjust_cfa_offset -16
327 .size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3
329 ################################################################################
330 # void ecp_nistz256_add(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
331 .globl ecp_nistz256_add
332 .type ecp_nistz256_add,\@function,3
347 lea .Lpoly(%rip), $a_ptr
379 .cfi_adjust_cfa_offset -16
383 .size ecp_nistz256_add,.-ecp_nistz256_add
385 ################################################################################
386 # void ecp_nistz256_sub(uint64_t res[4], uint64_t a[4], uint64_t b[4]);
387 .globl ecp_nistz256_sub
388 .type ecp_nistz256_sub,\@function,3
403 lea .Lpoly(%rip), $a_ptr
435 .cfi_adjust_cfa_offset -16
439 .size ecp_nistz256_sub,.-ecp_nistz256_sub
441 ################################################################################
442 # void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]);
443 .globl ecp_nistz256_neg
444 .type ecp_nistz256_neg,\@function,2
465 lea .Lpoly(%rip), $a_ptr
491 .cfi_adjust_cfa_offset -16
495 .size ecp_nistz256_neg,.-ecp_nistz256_neg
499 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
500 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
501 my ($t0,$t1,$t2,$t3,$t4)=("%rcx","%rbp","%rbx","%rdx","%rax");
502 my ($poly1,$poly3)=($acc6,$acc7);
505 ################################################################################
506 # void ecp_nistz256_to_mont(
509 .globl ecp_nistz256_to_mont
510 .type ecp_nistz256_to_mont,\@function,2
512 ecp_nistz256_to_mont:
514 $code.=<<___ if ($addx);
516 and OPENSSL_ia32cap_P+8(%rip), %ecx
519 lea .LRR(%rip), $b_org
521 .size ecp_nistz256_to_mont,.-ecp_nistz256_to_mont
523 ################################################################################
524 # void ecp_nistz256_mul_mont(
529 .globl ecp_nistz256_mul_mont
530 .type ecp_nistz256_mul_mont,\@function,3
532 ecp_nistz256_mul_mont:
535 $code.=<<___ if ($addx);
537 and OPENSSL_ia32cap_P+8(%rip), %ecx
555 $code.=<<___ if ($addx);
561 mov 8*0($b_org), %rax
562 mov 8*0($a_ptr), $acc1
563 mov 8*1($a_ptr), $acc2
564 mov 8*2($a_ptr), $acc3
565 mov 8*3($a_ptr), $acc4
567 call __ecp_nistz256_mul_montq
569 $code.=<<___ if ($addx);
575 mov 8*0($b_org), %rdx
576 mov 8*0($a_ptr), $acc1
577 mov 8*1($a_ptr), $acc2
578 mov 8*2($a_ptr), $acc3
579 mov 8*3($a_ptr), $acc4
580 lea -128($a_ptr), $a_ptr # control u-op density
582 call __ecp_nistz256_mul_montx
599 .cfi_adjust_cfa_offset -48
603 .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
605 .type __ecp_nistz256_mul_montq,\@abi-omnipotent
607 __ecp_nistz256_mul_montq:
608 ########################################################################
612 mov .Lpoly+8*1(%rip),$poly1
618 mov .Lpoly+8*3(%rip),$poly3
637 ########################################################################
638 # First reduction step
639 # Basically now we want to multiply acc[0] by p256,
640 # and add the result to the acc.
641 # Due to the special form of p256 we do some optimizations
643 # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0]
644 # then we add acc[0] and get acc[0] x 2^96
650 add $acc0, $acc1 # +=acc[0]<<96
653 mov 8*1($b_ptr), %rax
658 ########################################################################
691 ########################################################################
692 # Second reduction step
700 mov 8*2($b_ptr), %rax
705 ########################################################################
738 ########################################################################
739 # Third reduction step
747 mov 8*3($b_ptr), %rax
752 ########################################################################
785 ########################################################################
786 # Final reduction step
799 ########################################################################
800 # Branch-less conditional subtraction of P
801 sub \$-1, $acc4 # .Lpoly[0]
803 sbb $poly1, $acc5 # .Lpoly[1]
804 sbb \$0, $acc0 # .Lpoly[2]
806 sbb $poly3, $acc1 # .Lpoly[3]
811 mov $acc4, 8*0($r_ptr)
813 mov $acc5, 8*1($r_ptr)
815 mov $acc0, 8*2($r_ptr)
816 mov $acc1, 8*3($r_ptr)
819 .size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq
821 ################################################################################
822 # void ecp_nistz256_sqr_mont(
826 # we optimize the square according to S.Gueron and V.Krasnov,
827 # "Speeding up Big-Number Squaring"
828 .globl ecp_nistz256_sqr_mont
829 .type ecp_nistz256_sqr_mont,\@function,2
831 ecp_nistz256_sqr_mont:
834 $code.=<<___ if ($addx);
836 and OPENSSL_ia32cap_P+8(%rip), %ecx
853 $code.=<<___ if ($addx);
858 mov 8*0($a_ptr), %rax
859 mov 8*1($a_ptr), $acc6
860 mov 8*2($a_ptr), $acc7
861 mov 8*3($a_ptr), $acc0
863 call __ecp_nistz256_sqr_montq
865 $code.=<<___ if ($addx);
870 mov 8*0($a_ptr), %rdx
871 mov 8*1($a_ptr), $acc6
872 mov 8*2($a_ptr), $acc7
873 mov 8*3($a_ptr), $acc0
874 lea -128($a_ptr), $a_ptr # control u-op density
876 call __ecp_nistz256_sqr_montx
893 .cfi_adjust_cfa_offset -48
897 .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
899 .type __ecp_nistz256_sqr_montq,\@abi-omnipotent
901 __ecp_nistz256_sqr_montq:
903 mulq $acc6 # a[1]*a[0]
908 mulq $acc5 # a[0]*a[2]
914 mulq $acc5 # a[0]*a[3]
920 #################################
921 mulq $acc6 # a[1]*a[2]
927 mulq $acc6 # a[1]*a[3]
935 #################################
936 mulq $acc7 # a[2]*a[3]
939 mov 8*0($a_ptr), %rax
943 add $acc1, $acc1 # acc1:6<<1
953 mov 8*1($a_ptr), %rax
959 mov 8*2($a_ptr), %rax
966 mov 8*3($a_ptr), %rax
976 mov .Lpoly+8*1(%rip), $a_ptr
977 mov .Lpoly+8*3(%rip), $t1
979 ##########################################
986 add $acc0, $acc1 # +=acc[0]<<96
992 ##########################################
1005 ##########################################
1018 ###########################################
1031 ############################################
1032 # Add the rest of the acc
1041 sub \$-1, $acc4 # .Lpoly[0]
1043 sbb $a_ptr, $acc5 # .Lpoly[1]
1044 sbb \$0, $acc6 # .Lpoly[2]
1046 sbb $t1, $acc7 # .Lpoly[3]
1051 mov $acc4, 8*0($r_ptr)
1053 mov $acc5, 8*1($r_ptr)
1055 mov $acc6, 8*2($r_ptr)
1056 mov $acc7, 8*3($r_ptr)
1059 .size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq
1064 .type __ecp_nistz256_mul_montx,\@abi-omnipotent
1066 __ecp_nistz256_mul_montx:
1067 ########################################################################
1069 mulx $acc1, $acc0, $acc1
1070 mulx $acc2, $t0, $acc2
1072 xor $acc5, $acc5 # cf=0
1073 mulx $acc3, $t1, $acc3
1074 mov .Lpoly+8*3(%rip), $poly3
1076 mulx $acc4, $t0, $acc4
1079 shlx $poly1,$acc0,$t1
1081 shrx $poly1,$acc0,$t0
1084 ########################################################################
1085 # First reduction step
1089 mulx $poly3, $t0, $t1
1090 mov 8*1($b_ptr), %rdx
1094 xor $acc0, $acc0 # $acc0=0,cf=0,of=0
1096 ########################################################################
1098 mulx 8*0+128($a_ptr), $t0, $t1
1102 mulx 8*1+128($a_ptr), $t0, $t1
1106 mulx 8*2+128($a_ptr), $t0, $t1
1110 mulx 8*3+128($a_ptr), $t0, $t1
1113 shlx $poly1, $acc1, $t0
1115 shrx $poly1, $acc1, $t1
1121 ########################################################################
1122 # Second reduction step
1126 mulx $poly3, $t0, $t1
1127 mov 8*2($b_ptr), %rdx
1131 xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0
1133 ########################################################################
1135 mulx 8*0+128($a_ptr), $t0, $t1
1139 mulx 8*1+128($a_ptr), $t0, $t1
1143 mulx 8*2+128($a_ptr), $t0, $t1
1147 mulx 8*3+128($a_ptr), $t0, $t1
1150 shlx $poly1, $acc2, $t0
1152 shrx $poly1, $acc2, $t1
1158 ########################################################################
1159 # Third reduction step
1163 mulx $poly3, $t0, $t1
1164 mov 8*3($b_ptr), %rdx
1168 xor $acc2, $acc2 # $acc2=0,cf=0,of=0
1170 ########################################################################
1172 mulx 8*0+128($a_ptr), $t0, $t1
1176 mulx 8*1+128($a_ptr), $t0, $t1
1180 mulx 8*2+128($a_ptr), $t0, $t1
1184 mulx 8*3+128($a_ptr), $t0, $t1
1187 shlx $poly1, $acc3, $t0
1189 shrx $poly1, $acc3, $t1
1195 ########################################################################
1196 # Fourth reduction step
1200 mulx $poly3, $t0, $t1
1202 mov .Lpoly+8*1(%rip), $poly1
1208 ########################################################################
1209 # Branch-less conditional subtraction of P
1212 sbb \$-1, $acc4 # .Lpoly[0]
1213 sbb $poly1, $acc5 # .Lpoly[1]
1214 sbb \$0, $acc0 # .Lpoly[2]
1216 sbb $poly3, $acc1 # .Lpoly[3]
1221 mov $acc4, 8*0($r_ptr)
1223 mov $acc5, 8*1($r_ptr)
1225 mov $acc0, 8*2($r_ptr)
1226 mov $acc1, 8*3($r_ptr)
1229 .size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx
1231 .type __ecp_nistz256_sqr_montx,\@abi-omnipotent
1233 __ecp_nistz256_sqr_montx:
1234 mulx $acc6, $acc1, $acc2 # a[0]*a[1]
1235 mulx $acc7, $t0, $acc3 # a[0]*a[2]
1238 mulx $acc0, $t1, $acc4 # a[0]*a[3]
1242 xor $acc5, $acc5 # $acc5=0,cf=0,of=0
1244 #################################
1245 mulx $acc7, $t0, $t1 # a[1]*a[2]
1249 mulx $acc0, $t0, $t1 # a[1]*a[3]
1255 #################################
1256 mulx $acc0, $t0, $acc6 # a[2]*a[3]
1257 mov 8*0+128($a_ptr), %rdx
1258 xor $acc7, $acc7 # $acc7=0,cf=0,of=0
1259 adcx $acc1, $acc1 # acc1:6<<1
1262 adox $acc7, $acc6 # of=0
1264 mulx %rdx, $acc0, $t1
1265 mov 8*1+128($a_ptr), %rdx
1270 mov 8*2+128($a_ptr), %rdx
1276 mov 8*3+128($a_ptr), %rdx
1284 mov .Lpoly+8*3(%rip), %rdx
1286 shlx $a_ptr, $acc0, $t0
1288 shrx $a_ptr, $acc0, $t4
1295 mulx $acc0, $t0, $acc0
1297 shlx $a_ptr, $acc1, $t0
1299 shrx $a_ptr, $acc1, $t4
1305 mulx $acc1, $t0, $acc1
1307 shlx $a_ptr, $acc2, $t0
1309 shrx $a_ptr, $acc2, $t4
1315 mulx $acc2, $t0, $acc2
1317 shlx $a_ptr, $acc3, $t0
1319 shrx $a_ptr, $acc3, $t4
1325 mulx $acc3, $t0, $acc3
1330 add $acc0, $acc4 # accumulate upper half
1331 mov .Lpoly+8*1(%rip), $a_ptr
1339 sub \$-1, $acc4 # .Lpoly[0]
1341 sbb $a_ptr, $acc5 # .Lpoly[1]
1342 sbb \$0, $acc6 # .Lpoly[2]
1344 sbb $t1, $acc7 # .Lpoly[3]
1349 mov $acc4, 8*0($r_ptr)
1351 mov $acc5, 8*1($r_ptr)
1353 mov $acc6, 8*2($r_ptr)
1354 mov $acc7, 8*3($r_ptr)
1357 .size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx
1362 my ($r_ptr,$in_ptr)=("%rdi","%rsi");
1363 my ($acc0,$acc1,$acc2,$acc3)=map("%r$_",(8..11));
1364 my ($t0,$t1,$t2)=("%rcx","%r12","%r13");
1367 ################################################################################
1368 # void ecp_nistz256_from_mont(
1371 # This one performs Montgomery multiplication by 1, so we only need the reduction
1373 .globl ecp_nistz256_from_mont
1374 .type ecp_nistz256_from_mont,\@function,2
1376 ecp_nistz256_from_mont:
1384 mov 8*0($in_ptr), %rax
1385 mov .Lpoly+8*3(%rip), $t2
1386 mov 8*1($in_ptr), $acc1
1387 mov 8*2($in_ptr), $acc2
1388 mov 8*3($in_ptr), $acc3
1390 mov .Lpoly+8*1(%rip), $t1
1392 #########################################
1404 #########################################
1417 ##########################################
1430 ###########################################
1444 ###########################################
1445 # Branch-less conditional subtraction
1455 cmovnz $in_ptr, $acc1
1456 mov $acc0, 8*0($r_ptr)
1458 mov $acc1, 8*1($r_ptr)
1460 mov $acc2, 8*2($r_ptr)
1461 mov $acc3, 8*3($r_ptr)
1468 .cfi_adjust_cfa_offset -16
1472 .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
1476 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1477 my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7));
1478 my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15));
1479 my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15));
1482 ################################################################################
1483 # void ecp_nistz256_scatter_w5(uint64_t *val, uint64_t *in_t, int index);
1484 .globl ecp_nistz256_scatter_w5
1485 .type ecp_nistz256_scatter_w5,\@abi-omnipotent
1487 ecp_nistz256_scatter_w5:
1488 lea -3($index,$index,2), $index
1489 movdqa 0x00($in_t), %xmm0
1491 movdqa 0x10($in_t), %xmm1
1492 movdqa 0x20($in_t), %xmm2
1493 movdqa 0x30($in_t), %xmm3
1494 movdqa 0x40($in_t), %xmm4
1495 movdqa 0x50($in_t), %xmm5
1496 movdqa %xmm0, 0x00($val,$index)
1497 movdqa %xmm1, 0x10($val,$index)
1498 movdqa %xmm2, 0x20($val,$index)
1499 movdqa %xmm3, 0x30($val,$index)
1500 movdqa %xmm4, 0x40($val,$index)
1501 movdqa %xmm5, 0x50($val,$index)
1504 .size ecp_nistz256_scatter_w5,.-ecp_nistz256_scatter_w5
1506 ################################################################################
1507 # void ecp_nistz256_gather_w5(uint64_t *val, uint64_t *in_t, int index);
1508 .globl ecp_nistz256_gather_w5
1509 .type ecp_nistz256_gather_w5,\@abi-omnipotent
1511 ecp_nistz256_gather_w5:
1513 $code.=<<___ if ($avx>1);
1514 mov OPENSSL_ia32cap_P+8(%rip), %eax
1516 jnz .Lavx2_gather_w5
1518 $code.=<<___ if ($win64);
1519 lea -0x88(%rsp), %rax
1520 .LSEH_begin_ecp_nistz256_gather_w5:
1521 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1522 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1523 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1524 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1525 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1526 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1527 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1528 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1529 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1530 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1531 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1534 movdqa .LOne(%rip), $ONE
1545 pshufd \$0, $INDEX, $INDEX
1548 .Lselect_loop_sse_w5:
1552 pcmpeqd $INDEX, $TMP0
1554 movdqa 16*0($in_t), $T0a
1555 movdqa 16*1($in_t), $T0b
1556 movdqa 16*2($in_t), $T0c
1557 movdqa 16*3($in_t), $T0d
1558 movdqa 16*4($in_t), $T0e
1559 movdqa 16*5($in_t), $T0f
1560 lea 16*6($in_t), $in_t
1576 jnz .Lselect_loop_sse_w5
1578 movdqu $Ra, 16*0($val)
1579 movdqu $Rb, 16*1($val)
1580 movdqu $Rc, 16*2($val)
1581 movdqu $Rd, 16*3($val)
1582 movdqu $Re, 16*4($val)
1583 movdqu $Rf, 16*5($val)
1585 $code.=<<___ if ($win64);
1586 movaps (%rsp), %xmm6
1587 movaps 0x10(%rsp), %xmm7
1588 movaps 0x20(%rsp), %xmm8
1589 movaps 0x30(%rsp), %xmm9
1590 movaps 0x40(%rsp), %xmm10
1591 movaps 0x50(%rsp), %xmm11
1592 movaps 0x60(%rsp), %xmm12
1593 movaps 0x70(%rsp), %xmm13
1594 movaps 0x80(%rsp), %xmm14
1595 movaps 0x90(%rsp), %xmm15
1596 lea 0xa8(%rsp), %rsp
1600 .LSEH_end_ecp_nistz256_gather_w5:
1601 .size ecp_nistz256_gather_w5,.-ecp_nistz256_gather_w5
1603 ################################################################################
1604 # void ecp_nistz256_scatter_w7(uint64_t *val, uint64_t *in_t, int index);
1605 .globl ecp_nistz256_scatter_w7
1606 .type ecp_nistz256_scatter_w7,\@abi-omnipotent
1608 ecp_nistz256_scatter_w7:
1609 movdqu 0x00($in_t), %xmm0
1611 movdqu 0x10($in_t), %xmm1
1612 movdqu 0x20($in_t), %xmm2
1613 movdqu 0x30($in_t), %xmm3
1614 movdqa %xmm0, 0x00($val,$index)
1615 movdqa %xmm1, 0x10($val,$index)
1616 movdqa %xmm2, 0x20($val,$index)
1617 movdqa %xmm3, 0x30($val,$index)
1620 .size ecp_nistz256_scatter_w7,.-ecp_nistz256_scatter_w7
1622 ################################################################################
1623 # void ecp_nistz256_gather_w7(uint64_t *val, uint64_t *in_t, int index);
1624 .globl ecp_nistz256_gather_w7
1625 .type ecp_nistz256_gather_w7,\@abi-omnipotent
1627 ecp_nistz256_gather_w7:
1629 $code.=<<___ if ($avx>1);
1630 mov OPENSSL_ia32cap_P+8(%rip), %eax
1632 jnz .Lavx2_gather_w7
1634 $code.=<<___ if ($win64);
1635 lea -0x88(%rsp), %rax
1636 .LSEH_begin_ecp_nistz256_gather_w7:
1637 .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
1638 .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
1639 .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
1640 .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
1641 .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
1642 .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
1643 .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
1644 .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
1645 .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
1646 .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
1647 .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
1650 movdqa .LOne(%rip), $M0
1659 pshufd \$0, $INDEX, $INDEX
1662 .Lselect_loop_sse_w7:
1665 movdqa 16*0($in_t), $T0a
1666 movdqa 16*1($in_t), $T0b
1667 pcmpeqd $INDEX, $TMP0
1668 movdqa 16*2($in_t), $T0c
1669 movdqa 16*3($in_t), $T0d
1670 lea 16*4($in_t), $in_t
1679 prefetcht0 255($in_t)
1683 jnz .Lselect_loop_sse_w7
1685 movdqu $Ra, 16*0($val)
1686 movdqu $Rb, 16*1($val)
1687 movdqu $Rc, 16*2($val)
1688 movdqu $Rd, 16*3($val)
1690 $code.=<<___ if ($win64);
1691 movaps (%rsp), %xmm6
1692 movaps 0x10(%rsp), %xmm7
1693 movaps 0x20(%rsp), %xmm8
1694 movaps 0x30(%rsp), %xmm9
1695 movaps 0x40(%rsp), %xmm10
1696 movaps 0x50(%rsp), %xmm11
1697 movaps 0x60(%rsp), %xmm12
1698 movaps 0x70(%rsp), %xmm13
1699 movaps 0x80(%rsp), %xmm14
1700 movaps 0x90(%rsp), %xmm15
1701 lea 0xa8(%rsp), %rsp
1705 .LSEH_end_ecp_nistz256_gather_w7:
1706 .size ecp_nistz256_gather_w7,.-ecp_nistz256_gather_w7
1710 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1711 my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4));
1712 my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9));
1713 my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14));
1716 ################################################################################
1717 # void ecp_nistz256_avx2_gather_w5(uint64_t *val, uint64_t *in_t, int index);
1718 .type ecp_nistz256_avx2_gather_w5,\@abi-omnipotent
1720 ecp_nistz256_avx2_gather_w5:
1724 $code.=<<___ if ($win64);
1725 lea -0x88(%rsp), %rax
1727 .LSEH_begin_ecp_nistz256_avx2_gather_w5:
1728 .byte 0x48,0x8d,0x60,0xe0 # lea -0x20(%rax), %rsp
1729 .byte 0xc5,0xf8,0x29,0x70,0xe0 # vmovaps %xmm6, -0x20(%rax)
1730 .byte 0xc5,0xf8,0x29,0x78,0xf0 # vmovaps %xmm7, -0x10(%rax)
1731 .byte 0xc5,0x78,0x29,0x40,0x00 # vmovaps %xmm8, 8(%rax)
1732 .byte 0xc5,0x78,0x29,0x48,0x10 # vmovaps %xmm9, 0x10(%rax)
1733 .byte 0xc5,0x78,0x29,0x50,0x20 # vmovaps %xmm10, 0x20(%rax)
1734 .byte 0xc5,0x78,0x29,0x58,0x30 # vmovaps %xmm11, 0x30(%rax)
1735 .byte 0xc5,0x78,0x29,0x60,0x40 # vmovaps %xmm12, 0x40(%rax)
1736 .byte 0xc5,0x78,0x29,0x68,0x50 # vmovaps %xmm13, 0x50(%rax)
1737 .byte 0xc5,0x78,0x29,0x70,0x60 # vmovaps %xmm14, 0x60(%rax)
1738 .byte 0xc5,0x78,0x29,0x78,0x70 # vmovaps %xmm15, 0x70(%rax)
1741 vmovdqa .LTwo(%rip), $TWO
1747 vmovdqa .LOne(%rip), $M0
1748 vmovdqa .LTwo(%rip), $M1
1751 vpermd $INDEX, $Ra, $INDEX
1754 .Lselect_loop_avx2_w5:
1756 vmovdqa 32*0($in_t), $T0a
1757 vmovdqa 32*1($in_t), $T0b
1758 vmovdqa 32*2($in_t), $T0c
1760 vmovdqa 32*3($in_t), $T1a
1761 vmovdqa 32*4($in_t), $T1b
1762 vmovdqa 32*5($in_t), $T1c
1764 vpcmpeqd $INDEX, $M0, $TMP0
1765 vpcmpeqd $INDEX, $M1, $TMP1
1767 vpaddd $TWO, $M0, $M0
1768 vpaddd $TWO, $M1, $M1
1769 lea 32*6($in_t), $in_t
1771 vpand $TMP0, $T0a, $T0a
1772 vpand $TMP0, $T0b, $T0b
1773 vpand $TMP0, $T0c, $T0c
1774 vpand $TMP1, $T1a, $T1a
1775 vpand $TMP1, $T1b, $T1b
1776 vpand $TMP1, $T1c, $T1c
1778 vpxor $T0a, $Ra, $Ra
1779 vpxor $T0b, $Rb, $Rb
1780 vpxor $T0c, $Rc, $Rc
1781 vpxor $T1a, $Ra, $Ra
1782 vpxor $T1b, $Rb, $Rb
1783 vpxor $T1c, $Rc, $Rc
1786 jnz .Lselect_loop_avx2_w5
1788 vmovdqu $Ra, 32*0($val)
1789 vmovdqu $Rb, 32*1($val)
1790 vmovdqu $Rc, 32*2($val)
1793 $code.=<<___ if ($win64);
1794 movaps (%rsp), %xmm6
1795 movaps 0x10(%rsp), %xmm7
1796 movaps 0x20(%rsp), %xmm8
1797 movaps 0x30(%rsp), %xmm9
1798 movaps 0x40(%rsp), %xmm10
1799 movaps 0x50(%rsp), %xmm11
1800 movaps 0x60(%rsp), %xmm12
1801 movaps 0x70(%rsp), %xmm13
1802 movaps 0x80(%rsp), %xmm14
1803 movaps 0x90(%rsp), %xmm15
1808 .LSEH_end_ecp_nistz256_avx2_gather_w5:
1809 .size ecp_nistz256_avx2_gather_w5,.-ecp_nistz256_avx2_gather_w5
1813 my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
1814 my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3));
1815 my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7));
1816 my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11));
1817 my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15));
1821 ################################################################################
1822 # void ecp_nistz256_avx2_gather_w7(uint64_t *val, uint64_t *in_t, int index);
1823 .globl ecp_nistz256_avx2_gather_w7
1824 .type ecp_nistz256_avx2_gather_w7,\@abi-omnipotent
1826 ecp_nistz256_avx2_gather_w7:
1830 $code.=<<___ if ($win64);
1832 lea -0x88(%rsp), %rax
1833 .LSEH_begin_ecp_nistz256_avx2_gather_w7:
1834 .byte 0x48,0x8d,0x60,0xe0 # lea -0x20(%rax), %rsp
1835 .byte 0xc5,0xf8,0x29,0x70,0xe0 # vmovaps %xmm6, -0x20(%rax)
1836 .byte 0xc5,0xf8,0x29,0x78,0xf0 # vmovaps %xmm7, -0x10(%rax)
1837 .byte 0xc5,0x78,0x29,0x40,0x00 # vmovaps %xmm8, 8(%rax)
1838 .byte 0xc5,0x78,0x29,0x48,0x10 # vmovaps %xmm9, 0x10(%rax)
1839 .byte 0xc5,0x78,0x29,0x50,0x20 # vmovaps %xmm10, 0x20(%rax)
1840 .byte 0xc5,0x78,0x29,0x58,0x30 # vmovaps %xmm11, 0x30(%rax)
1841 .byte 0xc5,0x78,0x29,0x60,0x40 # vmovaps %xmm12, 0x40(%rax)
1842 .byte 0xc5,0x78,0x29,0x68,0x50 # vmovaps %xmm13, 0x50(%rax)
1843 .byte 0xc5,0x78,0x29,0x70,0x60 # vmovaps %xmm14, 0x60(%rax)
1844 .byte 0xc5,0x78,0x29,0x78,0x70 # vmovaps %xmm15, 0x70(%rax)
1847 vmovdqa .LThree(%rip), $THREE
1852 vmovdqa .LOne(%rip), $M0
1853 vmovdqa .LTwo(%rip), $M1
1854 vmovdqa .LThree(%rip), $M2
1857 vpermd $INDEX, $Ra, $INDEX
1858 # Skip index = 0, because it is implicitly the point at infinity
1861 .Lselect_loop_avx2_w7:
1863 vmovdqa 32*0($in_t), $T0a
1864 vmovdqa 32*1($in_t), $T0b
1866 vmovdqa 32*2($in_t), $T1a
1867 vmovdqa 32*3($in_t), $T1b
1869 vmovdqa 32*4($in_t), $T2a
1870 vmovdqa 32*5($in_t), $T2b
1872 vpcmpeqd $INDEX, $M0, $TMP0
1873 vpcmpeqd $INDEX, $M1, $TMP1
1874 vpcmpeqd $INDEX, $M2, $TMP2
1876 vpaddd $THREE, $M0, $M0
1877 vpaddd $THREE, $M1, $M1
1878 vpaddd $THREE, $M2, $M2
1879 lea 32*6($in_t), $in_t
1881 vpand $TMP0, $T0a, $T0a
1882 vpand $TMP0, $T0b, $T0b
1883 vpand $TMP1, $T1a, $T1a
1884 vpand $TMP1, $T1b, $T1b
1885 vpand $TMP2, $T2a, $T2a
1886 vpand $TMP2, $T2b, $T2b
1888 vpxor $T0a, $Ra, $Ra
1889 vpxor $T0b, $Rb, $Rb
1890 vpxor $T1a, $Ra, $Ra
1891 vpxor $T1b, $Rb, $Rb
1892 vpxor $T2a, $Ra, $Ra
1893 vpxor $T2b, $Rb, $Rb
1896 jnz .Lselect_loop_avx2_w7
1899 vmovdqa 32*0($in_t), $T0a
1900 vmovdqa 32*1($in_t), $T0b
1902 vpcmpeqd $INDEX, $M0, $TMP0
1904 vpand $TMP0, $T0a, $T0a
1905 vpand $TMP0, $T0b, $T0b
1907 vpxor $T0a, $Ra, $Ra
1908 vpxor $T0b, $Rb, $Rb
1910 vmovdqu $Ra, 32*0($val)
1911 vmovdqu $Rb, 32*1($val)
1914 $code.=<<___ if ($win64);
1915 movaps (%rsp), %xmm6
1916 movaps 0x10(%rsp), %xmm7
1917 movaps 0x20(%rsp), %xmm8
1918 movaps 0x30(%rsp), %xmm9
1919 movaps 0x40(%rsp), %xmm10
1920 movaps 0x50(%rsp), %xmm11
1921 movaps 0x60(%rsp), %xmm12
1922 movaps 0x70(%rsp), %xmm13
1923 movaps 0x80(%rsp), %xmm14
1924 movaps 0x90(%rsp), %xmm15
1929 .LSEH_end_ecp_nistz256_avx2_gather_w7:
1930 .size ecp_nistz256_avx2_gather_w7,.-ecp_nistz256_avx2_gather_w7
1934 .globl ecp_nistz256_avx2_gather_w7
1935 .type ecp_nistz256_avx2_gather_w7,\@function,3
1937 ecp_nistz256_avx2_gather_w7:
1938 .byte 0x0f,0x0b # ud2
1940 .size ecp_nistz256_avx2_gather_w7,.-ecp_nistz256_avx2_gather_w7
1944 ########################################################################
1945 # This block implements higher level point_double, point_add and
1946 # point_add_affine. The key to performance in this case is to allow
1947 # out-of-order execution logic to overlap computations from next step
1948 # with tail processing from current step. By using tailored calling
1949 # sequence we minimize inter-step overhead to give processor better
1950 # shot at overlapping operations...
1952 # You will notice that input data is copied to stack. Trouble is that
1953 # there are no registers to spare for holding original pointers and
1954 # reloading them, pointers, would create undesired dependencies on
1955 # effective addresses calculation paths. In other words it's too done
1956 # to favour out-of-order execution logic.
1957 # <appro@openssl.org>
1959 my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
1960 my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
1961 my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4);
1962 my ($poly1,$poly3)=($acc6,$acc7);
1964 sub load_for_mul () {
1965 my ($a,$b,$src0) = @_;
1966 my $bias = $src0 eq "%rax" ? 0 : -128;
1972 lea $bias+$a, $a_ptr
1977 sub load_for_sqr () {
1979 my $bias = $src0 eq "%rax" ? 0 : -128;
1983 lea $bias+$a, $a_ptr
1989 ########################################################################
1990 # operate in 4-5-0-1 "name space" that matches multiplication output
1992 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
1995 .type __ecp_nistz256_add_toq,\@abi-omnipotent
1997 __ecp_nistz256_add_toq:
1999 add 8*0($b_ptr), $a0
2000 adc 8*1($b_ptr), $a1
2002 adc 8*2($b_ptr), $a2
2003 adc 8*3($b_ptr), $a3
2017 mov $a0, 8*0($r_ptr)
2019 mov $a1, 8*1($r_ptr)
2021 mov $a2, 8*2($r_ptr)
2022 mov $a3, 8*3($r_ptr)
2025 .size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq
2027 .type __ecp_nistz256_sub_fromq,\@abi-omnipotent
2029 __ecp_nistz256_sub_fromq:
2030 sub 8*0($b_ptr), $a0
2031 sbb 8*1($b_ptr), $a1
2033 sbb 8*2($b_ptr), $a2
2034 sbb 8*3($b_ptr), $a3
2048 mov $a0, 8*0($r_ptr)
2050 mov $a1, 8*1($r_ptr)
2052 mov $a2, 8*2($r_ptr)
2053 mov $a3, 8*3($r_ptr)
2056 .size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq
2058 .type __ecp_nistz256_subq,\@abi-omnipotent
2060 __ecp_nistz256_subq:
2083 .size __ecp_nistz256_subq,.-__ecp_nistz256_subq
2085 .type __ecp_nistz256_mul_by_2q,\@abi-omnipotent
2087 __ecp_nistz256_mul_by_2q:
2089 add $a0, $a0 # a0:a3+a0:a3
2107 mov $a0, 8*0($r_ptr)
2109 mov $a1, 8*1($r_ptr)
2111 mov $a2, 8*2($r_ptr)
2112 mov $a3, 8*3($r_ptr)
2115 .size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q
2120 my ($src0,$sfx,$bias);
2121 my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
2129 .globl ecp_nistz256_point_double
2130 .type ecp_nistz256_point_double,\@function,2
2132 ecp_nistz256_point_double:
2135 $code.=<<___ if ($addx);
2137 and OPENSSL_ia32cap_P+8(%rip), %ecx
2147 .type ecp_nistz256_point_doublex,\@function,2
2149 ecp_nistz256_point_doublex:
2168 .cfi_adjust_cfa_offset 32*5+8
2169 .Lpoint_double${x}_body:
2171 .Lpoint_double_shortcut$x:
2172 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x
2173 mov $a_ptr, $b_ptr # backup copy
2174 movdqu 0x10($a_ptr), %xmm1
2175 mov 0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order
2176 mov 0x20+8*1($a_ptr), $acc5
2177 mov 0x20+8*2($a_ptr), $acc0
2178 mov 0x20+8*3($a_ptr), $acc1
2179 mov .Lpoly+8*1(%rip), $poly1
2180 mov .Lpoly+8*3(%rip), $poly3
2181 movdqa %xmm0, $in_x(%rsp)
2182 movdqa %xmm1, $in_x+0x10(%rsp)
2183 lea 0x20($r_ptr), $acc2
2184 lea 0x40($r_ptr), $acc3
2189 lea $S(%rsp), $r_ptr
2190 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(S, in_y);
2192 mov 0x40+8*0($a_ptr), $src0
2193 mov 0x40+8*1($a_ptr), $acc6
2194 mov 0x40+8*2($a_ptr), $acc7
2195 mov 0x40+8*3($a_ptr), $acc0
2196 lea 0x40-$bias($a_ptr), $a_ptr
2197 lea $Zsqr(%rsp), $r_ptr
2198 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Zsqr, in_z);
2200 `&load_for_sqr("$S(%rsp)", "$src0")`
2201 lea $S(%rsp), $r_ptr
2202 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(S, S);
2204 mov 0x20($b_ptr), $src0 # $b_ptr is still valid
2205 mov 0x40+8*0($b_ptr), $acc1
2206 mov 0x40+8*1($b_ptr), $acc2
2207 mov 0x40+8*2($b_ptr), $acc3
2208 mov 0x40+8*3($b_ptr), $acc4
2209 lea 0x40-$bias($b_ptr), $a_ptr
2210 lea 0x20($b_ptr), $b_ptr
2212 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, in_z, in_y);
2213 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(res_z, res_z);
2215 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
2216 mov $in_x+8*1(%rsp), $acc5
2217 lea $Zsqr(%rsp), $b_ptr
2218 mov $in_x+8*2(%rsp), $acc0
2219 mov $in_x+8*3(%rsp), $acc1
2220 lea $M(%rsp), $r_ptr
2221 call __ecp_nistz256_add_to$x # p256_add(M, in_x, Zsqr);
2223 mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
2224 mov $in_x+8*1(%rsp), $acc5
2225 lea $Zsqr(%rsp), $b_ptr
2226 mov $in_x+8*2(%rsp), $acc0
2227 mov $in_x+8*3(%rsp), $acc1
2228 lea $Zsqr(%rsp), $r_ptr
2229 call __ecp_nistz256_sub_from$x # p256_sub(Zsqr, in_x, Zsqr);
2231 `&load_for_sqr("$S(%rsp)", "$src0")`
2233 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_y, S);
2236 ######## ecp_nistz256_div_by_2(res_y, res_y); ##########################
2237 # operate in 4-5-6-7 "name space" that matches squaring output
2239 my ($poly1,$poly3)=($a_ptr,$t1);
2240 my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2);
2253 xor $a_ptr, $a_ptr # borrow $a_ptr
2262 mov $a1, $t0 # a0:a3>>1
2273 mov $a0, 8*0($r_ptr)
2275 mov $a1, 8*1($r_ptr)
2279 mov $a2, 8*2($r_ptr)
2280 mov $a3, 8*3($r_ptr)
2284 `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")`
2285 lea $M(%rsp), $r_ptr
2286 call __ecp_nistz256_mul_mont$x # p256_mul_mont(M, M, Zsqr);
2288 lea $tmp0(%rsp), $r_ptr
2289 call __ecp_nistz256_mul_by_2$x
2291 lea $M(%rsp), $b_ptr
2292 lea $M(%rsp), $r_ptr
2293 call __ecp_nistz256_add_to$x # p256_mul_by_3(M, M);
2295 `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")`
2296 lea $S(%rsp), $r_ptr
2297 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, in_x);
2299 lea $tmp0(%rsp), $r_ptr
2300 call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(tmp0, S);
2302 `&load_for_sqr("$M(%rsp)", "$src0")`
2304 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_x, M);
2306 lea $tmp0(%rsp), $b_ptr
2307 mov $acc6, $acc0 # harmonize sqr output and sub input
2311 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, tmp0);
2313 mov $S+8*0(%rsp), $t0
2314 mov $S+8*1(%rsp), $t1
2315 mov $S+8*2(%rsp), $t2
2316 mov $S+8*3(%rsp), $acc2 # "4-5-0-1" order
2317 lea $S(%rsp), $r_ptr
2318 call __ecp_nistz256_sub$x # p256_sub(S, S, res_x);
2321 lea $M(%rsp), $b_ptr
2322 mov $acc4, $acc6 # harmonize sub output and mul input
2324 mov $acc4, $S+8*0(%rsp) # have to save:-(
2326 mov $acc5, $S+8*1(%rsp)
2328 mov $acc0, $S+8*2(%rsp)
2329 lea $S-$bias(%rsp), $a_ptr
2331 mov $acc1, $S+8*3(%rsp)
2333 lea $S(%rsp), $r_ptr
2334 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, M);
2338 call __ecp_nistz256_sub_from$x # p256_sub(res_y, S, res_y);
2340 lea 32*5+56(%rsp), %rsi
2355 .cfi_def_cfa_register %rsp
2356 .Lpoint_double${x}_epilogue:
2359 .size ecp_nistz256_point_double$sfx,.-ecp_nistz256_point_double$sfx
2366 my ($src0,$sfx,$bias);
2367 my ($H,$Hsqr,$R,$Rsqr,$Hcub,
2369 $res_x,$res_y,$res_z,
2370 $in1_x,$in1_y,$in1_z,
2371 $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17));
2372 my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
2380 .globl ecp_nistz256_point_add
2381 .type ecp_nistz256_point_add,\@function,3
2383 ecp_nistz256_point_add:
2386 $code.=<<___ if ($addx);
2388 and OPENSSL_ia32cap_P+8(%rip), %ecx
2398 .type ecp_nistz256_point_addx,\@function,3
2400 ecp_nistz256_point_addx:
2419 .cfi_adjust_cfa_offset 32*18+8
2420 .Lpoint_add${x}_body:
2422 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
2423 movdqu 0x10($a_ptr), %xmm1
2424 movdqu 0x20($a_ptr), %xmm2
2425 movdqu 0x30($a_ptr), %xmm3
2426 movdqu 0x40($a_ptr), %xmm4
2427 movdqu 0x50($a_ptr), %xmm5
2428 mov $a_ptr, $b_ptr # reassign
2429 mov $b_org, $a_ptr # reassign
2430 movdqa %xmm0, $in1_x(%rsp)
2431 movdqa %xmm1, $in1_x+0x10(%rsp)
2432 movdqa %xmm2, $in1_y(%rsp)
2433 movdqa %xmm3, $in1_y+0x10(%rsp)
2434 movdqa %xmm4, $in1_z(%rsp)
2435 movdqa %xmm5, $in1_z+0x10(%rsp)
2438 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr
2439 pshufd \$0xb1, %xmm5, %xmm3
2440 movdqu 0x10($a_ptr), %xmm1
2441 movdqu 0x20($a_ptr), %xmm2
2443 movdqu 0x30($a_ptr), %xmm3
2444 mov 0x40+8*0($a_ptr), $src0 # load original in2_z
2445 mov 0x40+8*1($a_ptr), $acc6
2446 mov 0x40+8*2($a_ptr), $acc7
2447 mov 0x40+8*3($a_ptr), $acc0
2448 movdqa %xmm0, $in2_x(%rsp)
2449 pshufd \$0x1e, %xmm5, %xmm4
2450 movdqa %xmm1, $in2_x+0x10(%rsp)
2451 movdqu 0x40($a_ptr),%xmm0 # in2_z again
2452 movdqu 0x50($a_ptr),%xmm1
2453 movdqa %xmm2, $in2_y(%rsp)
2454 movdqa %xmm3, $in2_y+0x10(%rsp)
2458 movq $r_ptr, %xmm0 # save $r_ptr
2460 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2461 mov $src0, $in2_z+8*0(%rsp) # make in2_z copy
2462 mov $acc6, $in2_z+8*1(%rsp)
2463 mov $acc7, $in2_z+8*2(%rsp)
2464 mov $acc0, $in2_z+8*3(%rsp)
2465 lea $Z2sqr(%rsp), $r_ptr # Z2^2
2466 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z2sqr, in2_z);
2468 pcmpeqd %xmm4, %xmm5
2469 pshufd \$0xb1, %xmm1, %xmm4
2471 pshufd \$0, %xmm5, %xmm5 # in1infty
2472 pshufd \$0x1e, %xmm4, %xmm3
2475 pcmpeqd %xmm3, %xmm4
2476 pshufd \$0, %xmm4, %xmm4 # in2infty
2477 mov 0x40+8*0($b_ptr), $src0 # load original in1_z
2478 mov 0x40+8*1($b_ptr), $acc6
2479 mov 0x40+8*2($b_ptr), $acc7
2480 mov 0x40+8*3($b_ptr), $acc0
2483 lea 0x40-$bias($b_ptr), $a_ptr
2484 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2485 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2487 `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")`
2488 lea $S1(%rsp), $r_ptr # S1 = Z2^3
2489 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, Z2sqr, in2_z);
2491 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2492 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2493 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2495 `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")`
2496 lea $S1(%rsp), $r_ptr # S1 = Y1*Z2^3
2497 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, S1, in1_y);
2499 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2500 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2501 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2503 lea $S1(%rsp), $b_ptr
2504 lea $R(%rsp), $r_ptr # R = S2 - S1
2505 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, S1);
2507 or $acc5, $acc4 # see if result is zero
2511 por %xmm5, %xmm2 # in1infty || in2infty
2514 `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2515 lea $U1(%rsp), $r_ptr # U1 = X1*Z2^2
2516 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U1, in1_x, Z2sqr);
2518 `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")`
2519 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2520 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in2_x, Z1sqr);
2522 lea $U1(%rsp), $b_ptr
2523 lea $H(%rsp), $r_ptr # H = U2 - U1
2524 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, U1);
2526 or $acc5, $acc4 # see if result is zero
2530 .byte 0x3e # predict taken
2531 jnz .Ladd_proceed$x # is_equal(U1,U2)?
2535 jnz .Ladd_proceed$x # (in1infty || in2infty)?
2537 jz .Ladd_double$x # is_equal(S1,S2)?
2539 movq %xmm0, $r_ptr # restore $r_ptr
2541 movdqu %xmm0, 0x00($r_ptr)
2542 movdqu %xmm0, 0x10($r_ptr)
2543 movdqu %xmm0, 0x20($r_ptr)
2544 movdqu %xmm0, 0x30($r_ptr)
2545 movdqu %xmm0, 0x40($r_ptr)
2546 movdqu %xmm0, 0x50($r_ptr)
2551 movq %xmm1, $a_ptr # restore $a_ptr
2552 movq %xmm0, $r_ptr # restore $r_ptr
2553 add \$`32*(18-5)`, %rsp # difference in frame sizes
2554 jmp .Lpoint_double_shortcut$x
2558 `&load_for_sqr("$R(%rsp)", "$src0")`
2559 lea $Rsqr(%rsp), $r_ptr # R^2
2560 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2562 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2563 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2564 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2566 `&load_for_sqr("$H(%rsp)", "$src0")`
2567 lea $Hsqr(%rsp), $r_ptr # H^2
2568 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2570 `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")`
2571 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2572 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, res_z, in2_z);
2574 `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")`
2575 lea $Hcub(%rsp), $r_ptr # H^3
2576 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2578 `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")`
2579 lea $U2(%rsp), $r_ptr # U1*H^2
2580 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, U1, Hsqr);
2583 #######################################################################
2584 # operate in 4-5-0-1 "name space" that matches multiplication output
2586 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2587 my ($poly1, $poly3)=($acc6,$acc7);
2590 #lea $U2(%rsp), $a_ptr
2591 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2592 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2595 add $acc0, $acc0 # a0:a3+a0:a3
2596 lea $Rsqr(%rsp), $a_ptr
2613 mov 8*0($a_ptr), $t0
2615 mov 8*1($a_ptr), $t1
2617 mov 8*2($a_ptr), $t2
2619 mov 8*3($a_ptr), $t3
2621 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2623 lea $Hcub(%rsp), $b_ptr
2624 lea $res_x(%rsp), $r_ptr
2625 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2627 mov $U2+8*0(%rsp), $t0
2628 mov $U2+8*1(%rsp), $t1
2629 mov $U2+8*2(%rsp), $t2
2630 mov $U2+8*3(%rsp), $t3
2631 lea $res_y(%rsp), $r_ptr
2633 call __ecp_nistz256_sub$x # p256_sub(res_y, U2, res_x);
2635 mov $acc0, 8*0($r_ptr) # save the result, as
2636 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2637 mov $acc2, 8*2($r_ptr)
2638 mov $acc3, 8*3($r_ptr)
2642 `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")`
2643 lea $S2(%rsp), $r_ptr
2644 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S1, Hcub);
2646 `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")`
2647 lea $res_y(%rsp), $r_ptr
2648 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_y, R, res_y);
2650 lea $S2(%rsp), $b_ptr
2651 lea $res_y(%rsp), $r_ptr
2652 call __ecp_nistz256_sub_from$x # p256_sub(res_y, res_y, S2);
2654 movq %xmm0, $r_ptr # restore $r_ptr
2656 movdqa %xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty);
2658 pandn $res_z(%rsp), %xmm0
2660 pandn $res_z+0x10(%rsp), %xmm1
2662 pand $in2_z(%rsp), %xmm2
2663 pand $in2_z+0x10(%rsp), %xmm3
2667 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2673 pand $in1_z(%rsp), %xmm2
2674 pand $in1_z+0x10(%rsp), %xmm3
2677 movdqu %xmm2, 0x40($r_ptr)
2678 movdqu %xmm3, 0x50($r_ptr)
2680 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
2682 pandn $res_x(%rsp), %xmm0
2684 pandn $res_x+0x10(%rsp), %xmm1
2686 pand $in2_x(%rsp), %xmm2
2687 pand $in2_x+0x10(%rsp), %xmm3
2691 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
2697 pand $in1_x(%rsp), %xmm2
2698 pand $in1_x+0x10(%rsp), %xmm3
2701 movdqu %xmm2, 0x00($r_ptr)
2702 movdqu %xmm3, 0x10($r_ptr)
2704 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
2706 pandn $res_y(%rsp), %xmm0
2708 pandn $res_y+0x10(%rsp), %xmm1
2710 pand $in2_y(%rsp), %xmm2
2711 pand $in2_y+0x10(%rsp), %xmm3
2715 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
2721 pand $in1_y(%rsp), %xmm2
2722 pand $in1_y+0x10(%rsp), %xmm3
2725 movdqu %xmm2, 0x20($r_ptr)
2726 movdqu %xmm3, 0x30($r_ptr)
2729 lea 32*18+56(%rsp), %rsi
2744 .cfi_def_cfa_register %rsp
2745 .Lpoint_add${x}_epilogue:
2748 .size ecp_nistz256_point_add$sfx,.-ecp_nistz256_point_add$sfx
2753 sub gen_add_affine () {
2755 my ($src0,$sfx,$bias);
2756 my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr,
2757 $res_x,$res_y,$res_z,
2758 $in1_x,$in1_y,$in1_z,
2759 $in2_x,$in2_y)=map(32*$_,(0..14));
2768 .globl ecp_nistz256_point_add_affine
2769 .type ecp_nistz256_point_add_affine,\@function,3
2771 ecp_nistz256_point_add_affine:
2774 $code.=<<___ if ($addx);
2776 and OPENSSL_ia32cap_P+8(%rip), %ecx
2778 je .Lpoint_add_affinex
2786 .type ecp_nistz256_point_add_affinex,\@function,3
2788 ecp_nistz256_point_add_affinex:
2790 .Lpoint_add_affinex:
2807 .cfi_adjust_cfa_offset 32*15+8
2808 .Ladd_affine${x}_body:
2810 movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
2811 mov $b_org, $b_ptr # reassign
2812 movdqu 0x10($a_ptr), %xmm1
2813 movdqu 0x20($a_ptr), %xmm2
2814 movdqu 0x30($a_ptr), %xmm3
2815 movdqu 0x40($a_ptr), %xmm4
2816 movdqu 0x50($a_ptr), %xmm5
2817 mov 0x40+8*0($a_ptr), $src0 # load original in1_z
2818 mov 0x40+8*1($a_ptr), $acc6
2819 mov 0x40+8*2($a_ptr), $acc7
2820 mov 0x40+8*3($a_ptr), $acc0
2821 movdqa %xmm0, $in1_x(%rsp)
2822 movdqa %xmm1, $in1_x+0x10(%rsp)
2823 movdqa %xmm2, $in1_y(%rsp)
2824 movdqa %xmm3, $in1_y+0x10(%rsp)
2825 movdqa %xmm4, $in1_z(%rsp)
2826 movdqa %xmm5, $in1_z+0x10(%rsp)
2829 movdqu 0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr
2830 pshufd \$0xb1, %xmm5, %xmm3
2831 movdqu 0x10($b_ptr), %xmm1
2832 movdqu 0x20($b_ptr), %xmm2
2834 movdqu 0x30($b_ptr), %xmm3
2835 movdqa %xmm0, $in2_x(%rsp)
2836 pshufd \$0x1e, %xmm5, %xmm4
2837 movdqa %xmm1, $in2_x+0x10(%rsp)
2839 movq $r_ptr, %xmm0 # save $r_ptr
2840 movdqa %xmm2, $in2_y(%rsp)
2841 movdqa %xmm3, $in2_y+0x10(%rsp)
2847 lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
2848 lea $Z1sqr(%rsp), $r_ptr # Z1^2
2849 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
2851 pcmpeqd %xmm4, %xmm5
2852 pshufd \$0xb1, %xmm3, %xmm4
2853 mov 0x00($b_ptr), $src0 # $b_ptr is still valid
2854 #lea 0x00($b_ptr), $b_ptr
2855 mov $acc4, $acc1 # harmonize sqr output and mul input
2857 pshufd \$0, %xmm5, %xmm5 # in1infty
2858 pshufd \$0x1e, %xmm4, %xmm3
2863 pcmpeqd %xmm3, %xmm4
2864 pshufd \$0, %xmm4, %xmm4 # in2infty
2866 lea $Z1sqr-$bias(%rsp), $a_ptr
2868 lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
2869 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, Z1sqr, in2_x);
2871 lea $in1_x(%rsp), $b_ptr
2872 lea $H(%rsp), $r_ptr # H = U2 - U1
2873 call __ecp_nistz256_sub_from$x # p256_sub(H, U2, in1_x);
2875 `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
2876 lea $S2(%rsp), $r_ptr # S2 = Z1^3
2877 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
2879 `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
2880 lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
2881 call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
2883 `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
2884 lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
2885 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
2887 lea $in1_y(%rsp), $b_ptr
2888 lea $R(%rsp), $r_ptr # R = S2 - S1
2889 call __ecp_nistz256_sub_from$x # p256_sub(R, S2, in1_y);
2891 `&load_for_sqr("$H(%rsp)", "$src0")`
2892 lea $Hsqr(%rsp), $r_ptr # H^2
2893 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
2895 `&load_for_sqr("$R(%rsp)", "$src0")`
2896 lea $Rsqr(%rsp), $r_ptr # R^2
2897 call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
2899 `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")`
2900 lea $Hcub(%rsp), $r_ptr # H^3
2901 call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
2903 `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")`
2904 lea $U2(%rsp), $r_ptr # U1*H^2
2905 call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in1_x, Hsqr);
2908 #######################################################################
2909 # operate in 4-5-0-1 "name space" that matches multiplication output
2911 my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
2912 my ($poly1, $poly3)=($acc6,$acc7);
2915 #lea $U2(%rsp), $a_ptr
2916 #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
2917 #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
2920 add $acc0, $acc0 # a0:a3+a0:a3
2921 lea $Rsqr(%rsp), $a_ptr
2938 mov 8*0($a_ptr), $t0
2940 mov 8*1($a_ptr), $t1
2942 mov 8*2($a_ptr), $t2
2944 mov 8*3($a_ptr), $t3
2946 call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
2948 lea $Hcub(%rsp), $b_ptr
2949 lea $res_x(%rsp), $r_ptr
2950 call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
2952 mov $U2+8*0(%rsp), $t0
2953 mov $U2+8*1(%rsp), $t1
2954 mov $U2+8*2(%rsp), $t2
2955 mov $U2+8*3(%rsp), $t3
2956 lea $H(%rsp), $r_ptr
2958 call __ecp_nistz256_sub$x # p256_sub(H, U2, res_x);
2960 mov $acc0, 8*0($r_ptr) # save the result, as
2961 mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
2962 mov $acc2, 8*2($r_ptr)
2963 mov $acc3, 8*3($r_ptr)
2967 `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")`
2968 lea $S2(%rsp), $r_ptr
2969 call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Hcub, in1_y);
2971 `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")`
2972 lea $H(%rsp), $r_ptr
2973 call __ecp_nistz256_mul_mont$x # p256_mul_mont(H, H, R);
2975 lea $S2(%rsp), $b_ptr
2976 lea $res_y(%rsp), $r_ptr
2977 call __ecp_nistz256_sub_from$x # p256_sub(res_y, H, S2);
2979 movq %xmm0, $r_ptr # restore $r_ptr
2981 movdqa %xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty);
2983 pandn $res_z(%rsp), %xmm0
2985 pandn $res_z+0x10(%rsp), %xmm1
2987 pand .LONE_mont(%rip), %xmm2
2988 pand .LONE_mont+0x10(%rip), %xmm3
2992 movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
2998 pand $in1_z(%rsp), %xmm2
2999 pand $in1_z+0x10(%rsp), %xmm3
3002 movdqu %xmm2, 0x40($r_ptr)
3003 movdqu %xmm3, 0x50($r_ptr)
3005 movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
3007 pandn $res_x(%rsp), %xmm0
3009 pandn $res_x+0x10(%rsp), %xmm1
3011 pand $in2_x(%rsp), %xmm2
3012 pand $in2_x+0x10(%rsp), %xmm3
3016 movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
3022 pand $in1_x(%rsp), %xmm2
3023 pand $in1_x+0x10(%rsp), %xmm3
3026 movdqu %xmm2, 0x00($r_ptr)
3027 movdqu %xmm3, 0x10($r_ptr)
3029 movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
3031 pandn $res_y(%rsp), %xmm0
3033 pandn $res_y+0x10(%rsp), %xmm1
3035 pand $in2_y(%rsp), %xmm2
3036 pand $in2_y+0x10(%rsp), %xmm3
3040 movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
3046 pand $in1_y(%rsp), %xmm2
3047 pand $in1_y+0x10(%rsp), %xmm3
3050 movdqu %xmm2, 0x20($r_ptr)
3051 movdqu %xmm3, 0x30($r_ptr)
3053 lea 32*15+56(%rsp), %rsi
3068 .cfi_def_cfa_register %rsp
3069 .Ladd_affine${x}_epilogue:
3072 .size ecp_nistz256_point_add_affine$sfx,.-ecp_nistz256_point_add_affine$sfx
3075 &gen_add_affine("q");
3077 ########################################################################
3081 ########################################################################
3082 # operate in 4-5-0-1 "name space" that matches multiplication output
3084 my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
3087 .type __ecp_nistz256_add_tox,\@abi-omnipotent
3089 __ecp_nistz256_add_tox:
3091 adc 8*0($b_ptr), $a0
3092 adc 8*1($b_ptr), $a1
3094 adc 8*2($b_ptr), $a2
3095 adc 8*3($b_ptr), $a3
3110 mov $a0, 8*0($r_ptr)
3112 mov $a1, 8*1($r_ptr)
3114 mov $a2, 8*2($r_ptr)
3115 mov $a3, 8*3($r_ptr)
3118 .size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox
3120 .type __ecp_nistz256_sub_fromx,\@abi-omnipotent
3122 __ecp_nistz256_sub_fromx:
3124 sbb 8*0($b_ptr), $a0
3125 sbb 8*1($b_ptr), $a1
3127 sbb 8*2($b_ptr), $a2
3128 sbb 8*3($b_ptr), $a3
3143 mov $a0, 8*0($r_ptr)
3145 mov $a1, 8*1($r_ptr)
3147 mov $a2, 8*2($r_ptr)
3148 mov $a3, 8*3($r_ptr)
3151 .size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx
3153 .type __ecp_nistz256_subx,\@abi-omnipotent
3155 __ecp_nistz256_subx:
3180 .size __ecp_nistz256_subx,.-__ecp_nistz256_subx
3182 .type __ecp_nistz256_mul_by_2x,\@abi-omnipotent
3184 __ecp_nistz256_mul_by_2x:
3186 adc $a0, $a0 # a0:a3+a0:a3
3205 mov $a0, 8*0($r_ptr)
3207 mov $a1, 8*1($r_ptr)
3209 mov $a2, 8*2($r_ptr)
3210 mov $a3, 8*3($r_ptr)
3213 .size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x
3218 &gen_add_affine("x");
3222 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
3223 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
3231 .extern __imp_RtlVirtualUnwind
3233 .type short_handler,\@abi-omnipotent
3247 mov 120($context),%rax # pull context->Rax
3248 mov 248($context),%rbx # pull context->Rip
3250 mov 8($disp),%rsi # disp->ImageBase
3251 mov 56($disp),%r11 # disp->HandlerData
3253 mov 0(%r11),%r10d # HandlerData[0]
3254 lea (%rsi,%r10),%r10 # end of prologue label
3255 cmp %r10,%rbx # context->Rip<end of prologue label
3256 jb .Lcommon_seh_tail
3258 mov 152($context),%rax # pull context->Rsp
3260 mov 4(%r11),%r10d # HandlerData[1]
3261 lea (%rsi,%r10),%r10 # epilogue label
3262 cmp %r10,%rbx # context->Rip>=epilogue label
3263 jae .Lcommon_seh_tail
3269 mov %r12,216($context) # restore context->R12
3270 mov %r13,224($context) # restore context->R13
3272 jmp .Lcommon_seh_tail
3273 .size short_handler,.-short_handler
3275 .type full_handler,\@abi-omnipotent
3289 mov 120($context),%rax # pull context->Rax
3290 mov 248($context),%rbx # pull context->Rip
3292 mov 8($disp),%rsi # disp->ImageBase
3293 mov 56($disp),%r11 # disp->HandlerData
3295 mov 0(%r11),%r10d # HandlerData[0]
3296 lea (%rsi,%r10),%r10 # end of prologue label
3297 cmp %r10,%rbx # context->Rip<end of prologue label
3298 jb .Lcommon_seh_tail
3300 mov 152($context),%rax # pull context->Rsp
3302 mov 4(%r11),%r10d # HandlerData[1]
3303 lea (%rsi,%r10),%r10 # epilogue label
3304 cmp %r10,%rbx # context->Rip>=epilogue label
3305 jae .Lcommon_seh_tail
3307 mov 8(%r11),%r10d # HandlerData[2]
3308 lea (%rax,%r10),%rax
3316 mov %rbx,144($context) # restore context->Rbx
3317 mov %rbp,160($context) # restore context->Rbp
3318 mov %r12,216($context) # restore context->R12
3319 mov %r13,224($context) # restore context->R13
3320 mov %r14,232($context) # restore context->R14
3321 mov %r15,240($context) # restore context->R15
3326 mov %rax,152($context) # restore context->Rsp
3327 mov %rsi,168($context) # restore context->Rsi
3328 mov %rdi,176($context) # restore context->Rdi
3330 mov 40($disp),%rdi # disp->ContextRecord
3331 mov $context,%rsi # context
3332 mov \$154,%ecx # sizeof(CONTEXT)
3333 .long 0xa548f3fc # cld; rep movsq
3336 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
3337 mov 8(%rsi),%rdx # arg2, disp->ImageBase
3338 mov 0(%rsi),%r8 # arg3, disp->ControlPc
3339 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
3340 mov 40(%rsi),%r10 # disp->ContextRecord
3341 lea 56(%rsi),%r11 # &disp->HandlerData
3342 lea 24(%rsi),%r12 # &disp->EstablisherFrame
3343 mov %r10,32(%rsp) # arg5
3344 mov %r11,40(%rsp) # arg6
3345 mov %r12,48(%rsp) # arg7
3346 mov %rcx,56(%rsp) # arg8, (NULL)
3347 call *__imp_RtlVirtualUnwind(%rip)
3349 mov \$1,%eax # ExceptionContinueSearch
3361 .size full_handler,.-full_handler
3365 .rva .LSEH_begin_ecp_nistz256_mul_by_2
3366 .rva .LSEH_end_ecp_nistz256_mul_by_2
3367 .rva .LSEH_info_ecp_nistz256_mul_by_2
3369 .rva .LSEH_begin_ecp_nistz256_div_by_2
3370 .rva .LSEH_end_ecp_nistz256_div_by_2
3371 .rva .LSEH_info_ecp_nistz256_div_by_2
3373 .rva .LSEH_begin_ecp_nistz256_mul_by_3
3374 .rva .LSEH_end_ecp_nistz256_mul_by_3
3375 .rva .LSEH_info_ecp_nistz256_mul_by_3
3377 .rva .LSEH_begin_ecp_nistz256_add
3378 .rva .LSEH_end_ecp_nistz256_add
3379 .rva .LSEH_info_ecp_nistz256_add
3381 .rva .LSEH_begin_ecp_nistz256_sub
3382 .rva .LSEH_end_ecp_nistz256_sub
3383 .rva .LSEH_info_ecp_nistz256_sub
3385 .rva .LSEH_begin_ecp_nistz256_neg
3386 .rva .LSEH_end_ecp_nistz256_neg
3387 .rva .LSEH_info_ecp_nistz256_neg
3389 .rva .LSEH_begin_ecp_nistz256_to_mont
3390 .rva .LSEH_end_ecp_nistz256_to_mont
3391 .rva .LSEH_info_ecp_nistz256_to_mont
3393 .rva .LSEH_begin_ecp_nistz256_mul_mont
3394 .rva .LSEH_end_ecp_nistz256_mul_mont
3395 .rva .LSEH_info_ecp_nistz256_mul_mont
3397 .rva .LSEH_begin_ecp_nistz256_sqr_mont
3398 .rva .LSEH_end_ecp_nistz256_sqr_mont
3399 .rva .LSEH_info_ecp_nistz256_sqr_mont
3401 .rva .LSEH_begin_ecp_nistz256_from_mont
3402 .rva .LSEH_end_ecp_nistz256_from_mont
3403 .rva .LSEH_info_ecp_nistz256_from_mont
3405 .rva .LSEH_begin_ecp_nistz256_gather_w5
3406 .rva .LSEH_end_ecp_nistz256_gather_w5
3407 .rva .LSEH_info_ecp_nistz256_gather_wX
3409 .rva .LSEH_begin_ecp_nistz256_gather_w7
3410 .rva .LSEH_end_ecp_nistz256_gather_w7
3411 .rva .LSEH_info_ecp_nistz256_gather_wX
3413 $code.=<<___ if ($avx>1);
3414 .rva .LSEH_begin_ecp_nistz256_avx2_gather_w5
3415 .rva .LSEH_end_ecp_nistz256_avx2_gather_w5
3416 .rva .LSEH_info_ecp_nistz256_avx2_gather_wX
3418 .rva .LSEH_begin_ecp_nistz256_avx2_gather_w7
3419 .rva .LSEH_end_ecp_nistz256_avx2_gather_w7
3420 .rva .LSEH_info_ecp_nistz256_avx2_gather_wX
3423 .rva .LSEH_begin_ecp_nistz256_point_double
3424 .rva .LSEH_end_ecp_nistz256_point_double
3425 .rva .LSEH_info_ecp_nistz256_point_double
3427 .rva .LSEH_begin_ecp_nistz256_point_add
3428 .rva .LSEH_end_ecp_nistz256_point_add
3429 .rva .LSEH_info_ecp_nistz256_point_add
3431 .rva .LSEH_begin_ecp_nistz256_point_add_affine
3432 .rva .LSEH_end_ecp_nistz256_point_add_affine
3433 .rva .LSEH_info_ecp_nistz256_point_add_affine
3435 $code.=<<___ if ($addx);
3436 .rva .LSEH_begin_ecp_nistz256_point_doublex
3437 .rva .LSEH_end_ecp_nistz256_point_doublex
3438 .rva .LSEH_info_ecp_nistz256_point_doublex
3440 .rva .LSEH_begin_ecp_nistz256_point_addx
3441 .rva .LSEH_end_ecp_nistz256_point_addx
3442 .rva .LSEH_info_ecp_nistz256_point_addx
3444 .rva .LSEH_begin_ecp_nistz256_point_add_affinex
3445 .rva .LSEH_end_ecp_nistz256_point_add_affinex
3446 .rva .LSEH_info_ecp_nistz256_point_add_affinex
3452 .LSEH_info_ecp_nistz256_mul_by_2:
3455 .rva .Lmul_by_2_body,.Lmul_by_2_epilogue # HandlerData[]
3456 .LSEH_info_ecp_nistz256_div_by_2:
3459 .rva .Ldiv_by_2_body,.Ldiv_by_2_epilogue # HandlerData[]
3460 .LSEH_info_ecp_nistz256_mul_by_3:
3463 .rva .Lmul_by_3_body,.Lmul_by_3_epilogue # HandlerData[]
3464 .LSEH_info_ecp_nistz256_add:
3467 .rva .Ladd_body,.Ladd_epilogue # HandlerData[]
3468 .LSEH_info_ecp_nistz256_sub:
3471 .rva .Lsub_body,.Lsub_epilogue # HandlerData[]
3472 .LSEH_info_ecp_nistz256_neg:
3475 .rva .Lneg_body,.Lneg_epilogue # HandlerData[]
3476 .LSEH_info_ecp_nistz256_to_mont:
3479 .rva .Lmul_body,.Lmul_epilogue # HandlerData[]
3481 .LSEH_info_ecp_nistz256_mul_mont:
3484 .rva .Lmul_body,.Lmul_epilogue # HandlerData[]
3486 .LSEH_info_ecp_nistz256_sqr_mont:
3489 .rva .Lsqr_body,.Lsqr_epilogue # HandlerData[]
3491 .LSEH_info_ecp_nistz256_from_mont:
3494 .rva .Lfrom_body,.Lfrom_epilogue # HandlerData[]
3495 .LSEH_info_ecp_nistz256_gather_wX:
3496 .byte 0x01,0x33,0x16,0x00
3497 .byte 0x33,0xf8,0x09,0x00 #movaps 0x90(rsp),xmm15
3498 .byte 0x2e,0xe8,0x08,0x00 #movaps 0x80(rsp),xmm14
3499 .byte 0x29,0xd8,0x07,0x00 #movaps 0x70(rsp),xmm13
3500 .byte 0x24,0xc8,0x06,0x00 #movaps 0x60(rsp),xmm12
3501 .byte 0x1f,0xb8,0x05,0x00 #movaps 0x50(rsp),xmm11
3502 .byte 0x1a,0xa8,0x04,0x00 #movaps 0x40(rsp),xmm10
3503 .byte 0x15,0x98,0x03,0x00 #movaps 0x30(rsp),xmm9
3504 .byte 0x10,0x88,0x02,0x00 #movaps 0x20(rsp),xmm8
3505 .byte 0x0c,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7
3506 .byte 0x08,0x68,0x00,0x00 #movaps 0x00(rsp),xmm6
3507 .byte 0x04,0x01,0x15,0x00 #sub rsp,0xa8
3510 $code.=<<___ if ($avx>1);
3511 .LSEH_info_ecp_nistz256_avx2_gather_wX:
3512 .byte 0x01,0x36,0x17,0x0b
3513 .byte 0x36,0xf8,0x09,0x00 # vmovaps 0x90(rsp),xmm15
3514 .byte 0x31,0xe8,0x08,0x00 # vmovaps 0x80(rsp),xmm14
3515 .byte 0x2c,0xd8,0x07,0x00 # vmovaps 0x70(rsp),xmm13
3516 .byte 0x27,0xc8,0x06,0x00 # vmovaps 0x60(rsp),xmm12
3517 .byte 0x22,0xb8,0x05,0x00 # vmovaps 0x50(rsp),xmm11
3518 .byte 0x1d,0xa8,0x04,0x00 # vmovaps 0x40(rsp),xmm10
3519 .byte 0x18,0x98,0x03,0x00 # vmovaps 0x30(rsp),xmm9
3520 .byte 0x13,0x88,0x02,0x00 # vmovaps 0x20(rsp),xmm8
3521 .byte 0x0e,0x78,0x01,0x00 # vmovaps 0x10(rsp),xmm7
3522 .byte 0x09,0x68,0x00,0x00 # vmovaps 0x00(rsp),xmm6
3523 .byte 0x04,0x01,0x15,0x00 # sub rsp,0xa8
3524 .byte 0x00,0xb3,0x00,0x00 # set_frame r11
3528 .LSEH_info_ecp_nistz256_point_double:
3531 .rva .Lpoint_doubleq_body,.Lpoint_doubleq_epilogue # HandlerData[]
3533 .LSEH_info_ecp_nistz256_point_add:
3536 .rva .Lpoint_addq_body,.Lpoint_addq_epilogue # HandlerData[]
3538 .LSEH_info_ecp_nistz256_point_add_affine:
3541 .rva .Ladd_affineq_body,.Ladd_affineq_epilogue # HandlerData[]
3544 $code.=<<___ if ($addx);
3546 .LSEH_info_ecp_nistz256_point_doublex:
3549 .rva .Lpoint_doublex_body,.Lpoint_doublex_epilogue # HandlerData[]
3551 .LSEH_info_ecp_nistz256_point_addx:
3554 .rva .Lpoint_addx_body,.Lpoint_addx_epilogue # HandlerData[]
3556 .LSEH_info_ecp_nistz256_point_add_affinex:
3559 .rva .Ladd_affinex_body,.Ladd_affinex_epilogue # HandlerData[]
3564 ########################################################################
3565 # Convert ecp_nistz256_table.c to layout expected by ecp_nistz_gather_w7
3567 open TABLE,"<ecp_nistz256_table.c" or
3568 open TABLE,"<${dir}../ecp_nistz256_table.c" or
3569 die "failed to open ecp_nistz256_table.c:",$!;
3574 s/TOBN\(\s*(0x[0-9a-f]+),\s*(0x[0-9a-f]+)\s*\)/push @arr,hex($2),hex($1)/geo;
3578 die "insane number of elements" if ($#arr != 64*16*37-1);
3582 .globl ecp_nistz256_precomputed
3583 .type ecp_nistz256_precomputed,\@object
3585 ecp_nistz256_precomputed:
3587 while (@line=splice(@arr,0,16)) {
3588 print ".long\t",join(',',map { sprintf "0x%08x",$_} @line),"\n";
3591 .size ecp_nistz256_precomputed,.-ecp_nistz256_precomputed
3594 $code =~ s/\`([^\`]*)\`/eval $1/gem;