3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
12 # The module implements bn_GF2m_mul_2x2 polynomial multiplication used
13 # in bn_gf2m.c. It's kind of low-hanging mechanical port from C for
14 # the time being... Except that it has two code paths: code suitable
15 # for any x86_64 CPU and PCLMULQDQ one suitable for Westmere and
16 # later. Improvement varies from one benchmark and µ-arch to another.
17 # Vanilla code path is at most 20% faster than compiler-generated code
18 # [not very impressive], while PCLMULQDQ - whole 85%-160% better on
19 # 163- and 571-bit ECDH benchmarks on Intel CPUs. Keep in mind that
20 # these coefficients are not ones for bn_GF2m_mul_2x2 itself, as not
21 # all CPU time is burnt in it...
25 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
27 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
29 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
30 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
31 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
32 die "can't locate x86_64-xlate.pl";
34 open OUT,"| \"$^X\" $xlate $flavour $output";
37 ($lo,$hi)=("%rax","%rdx"); $a=$lo;
38 ($i0,$i1)=("%rsi","%rdi");
39 ($t0,$t1)=("%rbx","%rcx");
40 ($b,$mask)=("%rbp","%r8");
41 ($a1,$a2,$a4,$a8,$a12,$a48)=map("%r$_",(9..15));
42 ($R,$Tx)=("%xmm0","%xmm1");
47 .type _mul_1x1,\@abi-omnipotent
55 and $a,$a1 # a1=a&0x1fffffffffffffff
57 sar \$63,$a # broadcast 63rd bit
59 sar \$63,$i0 # broadcast 62nd bit
62 sar \$63,$i1 # boardcast 61st bit
63 mov $a,$hi # $a is $lo
80 movq \$0,0(%rsp) # tab[0]=0
82 mov $a1,8(%rsp) # tab[1]=a1
84 mov $a2,16(%rsp) # tab[2]=a2
86 mov $a12,24(%rsp) # tab[3]=a1^a2
89 mov $a4,32(%rsp) # tab[4]=a4
91 mov $a1,40(%rsp) # tab[5]=a1^a4
93 mov $a2,48(%rsp) # tab[6]=a2^a4
94 xor $a48,$a1 # a1^a4^a4^a8=a1^a8
95 mov $a12,56(%rsp) # tab[7]=a1^a2^a4
96 xor $a48,$a2 # a2^a4^a4^a8=a1^a8
98 mov $a8,64(%rsp) # tab[8]=a8
99 xor $a48,$a12 # a1^a2^a4^a4^a8=a1^a2^a8
100 mov $a1,72(%rsp) # tab[9]=a1^a8
101 xor $a4,$a1 # a1^a8^a4
102 mov $a2,80(%rsp) # tab[10]=a2^a8
103 xor $a4,$a2 # a2^a8^a4
104 mov $a12,88(%rsp) # tab[11]=a1^a2^a8
106 xor $a4,$a12 # a1^a2^a8^a4
107 mov $a48,96(%rsp) # tab[12]=a4^a8
109 mov $a1,104(%rsp) # tab[13]=a1^a4^a8
111 mov $a2,112(%rsp) # tab[14]=a2^a4^a8
113 mov $a12,120(%rsp) # tab[15]=a1^a2^a4^a8
118 movq (%rsp,$i0,8),$R # half of calculations is done in SSE2
123 for ($n=1;$n<8;$n++) {
130 movq (%rsp,$i0,8),$Tx
131 shr \$`64-(8*$n-4)`,$t0
147 shr \$`64-(8*$n-4)`,$t0
158 .size _mul_1x1,.-_mul_1x1
161 ($rp,$a1,$a0,$b1,$b0) = $win64? ("%rcx","%rdx","%r8", "%r9","%r10") : # Win64 order
162 ("%rdi","%rsi","%rdx","%rcx","%r8"); # Unix order
165 .extern OPENSSL_ia32cap_P
166 .globl bn_GF2m_mul_2x2
167 .type bn_GF2m_mul_2x2,\@abi-omnipotent
170 mov OPENSSL_ia32cap_P(%rip),%rax
172 jnc .Lvanilla_mul_2x2
178 $code.=<<___ if ($win64);
181 $code.=<<___ if (!$win64);
187 pclmulqdq \$0,%xmm1,%xmm0 # a1·b1
190 pclmulqdq \$0,%xmm3,%xmm2 # a0·b0
191 pclmulqdq \$0,%xmm5,%xmm4 # (a0+a1)·(b0+b1)
193 xorps %xmm2,%xmm4 # (a0+a1)·(b0+b1)-a0·b0-a1·b1
207 $code.=<<___ if ($win64);
208 mov `8*17+40`(%rsp),$b0
219 mov $rp,32(%rsp) # save the arguments
228 call _mul_1x1 # a1·b1
234 call _mul_1x1 # a0·b0
242 call _mul_1x1 # (a0+a1)·(b0+b1)
244 @r=("%rbx","%rcx","%rdi","%rsi");
270 $code.=<<___ if ($win64);
278 .size bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2
279 .asciz "GF(2^m) Multiplication for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
283 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
284 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
292 .extern __imp_RtlVirtualUnwind
294 .type se_handler,\@abi-omnipotent
308 mov 152($context),%rax # pull context->Rsp
309 mov 248($context),%rbx # pull context->Rip
311 lea .Lbody_mul_2x2(%rip),%r10
312 cmp %r10,%rbx # context->Rip<"prologue" label
315 mov 8*10(%rax),%r14 # mimic epilogue
323 mov %rbx,144($context) # restore context->Rbx
324 mov %rbp,160($context) # restore context->Rbp
325 mov %rsi,168($context) # restore context->Rsi
326 mov %rdi,176($context) # restore context->Rdi
327 mov %r12,216($context) # restore context->R12
328 mov %r13,224($context) # restore context->R13
329 mov %r14,232($context) # restore context->R14
333 mov %rax,152($context) # restore context->Rsp
335 mov 40($disp),%rdi # disp->ContextRecord
336 mov $context,%rsi # context
337 mov \$154,%ecx # sizeof(CONTEXT)
338 .long 0xa548f3fc # cld; rep movsq
341 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
342 mov 8(%rsi),%rdx # arg2, disp->ImageBase
343 mov 0(%rsi),%r8 # arg3, disp->ControlPc
344 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
345 mov 40(%rsi),%r10 # disp->ContextRecord
346 lea 56(%rsi),%r11 # &disp->HandlerData
347 lea 24(%rsi),%r12 # &disp->EstablisherFrame
348 mov %r10,32(%rsp) # arg5
349 mov %r11,40(%rsp) # arg6
350 mov %r12,48(%rsp) # arg7
351 mov %rcx,56(%rsp) # arg8, (NULL)
352 call *__imp_RtlVirtualUnwind(%rip)
354 mov \$1,%eax # ExceptionContinueSearch
366 .size se_handler,.-se_handler
374 .rva .Lvanilla_mul_2x2
380 .byte 0x01,0x07,0x02,0x00
381 .byte 0x07,0x01,0x11,0x00 # sub rsp,128+8
388 $code =~ s/\`([^\`]*)\`/eval($1)/gem;