3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
12 # The module implements bn_GF2m_mul_2x2 polynomial multiplication used
13 # in bn_gf2m.c. It's kind of low-hanging mechanical port from C for
14 # the time being... Except that it has two code paths: code suitable
15 # for any x86_64 CPU and PCLMULQDQ one suitable for Westmere and
16 # later. Improvement varies from one benchmark and µ-arch to another.
17 # Vanilla code path is at most 20% faster than compiler-generated code
18 # [not very impressive], while PCLMULQDQ - whole 85%-160% better on
19 # 163- and 571-bit ECDH benchmarks on Intel CPUs. Keep in mind that
20 # these coefficients are not ones for bn_GF2m_mul_2x2 itself, as not
21 # all CPU time is burnt in it...
25 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
27 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
29 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
30 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
31 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
32 die "can't locate x86_64-xlate.pl";
34 open STDOUT,"| $^X $xlate $flavour $output";
36 ($lo,$hi)=("%rax","%rdx"); $a=$lo;
37 ($i0,$i1)=("%rsi","%rdi");
38 ($t0,$t1)=("%rbx","%rcx");
39 ($b,$mask)=("%rbp","%r8");
40 ($a1,$a2,$a4,$a8,$a12,$a48)=map("%r$_",(9..15));
41 ($R,$Tx)=("%xmm0","%xmm1");
46 .type _mul_1x1,\@abi-omnipotent
54 and $a,$a1 # a1=a&0x1fffffffffffffff
56 sar \$63,$a # broadcast 63rd bit
58 sar \$63,$i0 # broadcast 62nd bit
61 sar \$63,$i1 # boardcast 61st bit
62 mov $a,$hi # $a is $lo
79 movq \$0,0(%rsp) # tab[0]=0
81 mov $a1,8(%rsp) # tab[1]=a1
83 mov $a2,16(%rsp) # tab[2]=a2
85 mov $a12,24(%rsp) # tab[3]=a1^a2
88 mov $a4,32(%rsp) # tab[4]=a4
90 mov $a1,40(%rsp) # tab[5]=a1^a4
92 mov $a2,48(%rsp) # tab[6]=a2^a4
93 xor $a48,$a1 # a1^a4^a4^a8=a1^a8
94 mov $a12,56(%rsp) # tab[7]=a1^a2^a4
95 xor $a48,$a2 # a2^a4^a4^a8=a1^a8
97 mov $a8,64(%rsp) # tab[8]=a8
98 xor $a48,$a12 # a1^a2^a4^a4^a8=a1^a2^a8
99 mov $a1,72(%rsp) # tab[9]=a1^a8
100 xor $a4,$a1 # a1^a8^a4
101 mov $a2,80(%rsp) # tab[10]=a2^a8
102 xor $a4,$a2 # a2^a8^a4
103 mov $a12,88(%rsp) # tab[11]=a1^a2^a8
105 xor $a4,$a12 # a1^a2^a8^a4
106 mov $a48,96(%rsp) # tab[12]=a4^a8
108 mov $a1,104(%rsp) # tab[13]=a1^a4^a8
110 mov $a2,112(%rsp) # tab[14]=a2^a4^a8
112 mov $a12,120(%rsp) # tab[15]=a1^a2^a4^a8
117 movq (%rsp,$i0,8),$R # half of calculations is done in SSE2
122 for ($n=1;$n<8;$n++) {
129 movq (%rsp,$i0,8),$Tx
130 shr \$`64-(8*$n-4)`,$t0
146 shr \$`64-(8*$n-4)`,$t0
156 .size _mul_1x1,.-_mul_1x1
159 ($rp,$a1,$a0,$b1,$b0) = $win64? ("%rcx","%rdx","%r8", "%r9","%r10") : # Win64 order
160 ("%rdi","%rsi","%rdx","%rcx","%r8"); # Unix order
163 .extern OPENSSL_ia32cap_P
164 .globl bn_GF2m_mul_2x2
165 .type bn_GF2m_mul_2x2,\@abi-omnipotent
168 mov OPENSSL_ia32cap_P(%rip),%rax
176 $code.=<<___ if ($win64);
179 $code.=<<___ if (!$win64);
185 pclmulqdq \$0,%xmm1,%xmm0 # a1·b1
188 pclmulqdq \$0,%xmm3,%xmm2 # a0·b0
189 pclmulqdq \$0,%xmm5,%xmm4 # (a0+a1)·(b0+b1)
205 $code.=<<___ if ($win64);
206 mov `8*17+40`(%rsp),$b0
217 mov $rp,32(%rsp) # save the arguments
226 call _mul_1x1 # a1·b1
232 call _mul_1x1 # a0·b0
240 call _mul_1x1 # (a0+a1)·(b0+b1)
242 @r=("%rbx","%rcx","%rdi","%rsi");
268 $code.=<<___ if ($win64);
275 .size bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2
276 .asciz "GF(2^m) Multiplication for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
279 $code =~ s/\`([^\`]*)\`/eval($1)/gem;