2 # Copyright 2011-2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
19 # The module implements bn_GF2m_mul_2x2 polynomial multiplication used
20 # in bn_gf2m.c. It's kind of low-hanging mechanical port from C for
21 # the time being... Except that it has two code paths: code suitable
22 # for any x86_64 CPU and PCLMULQDQ one suitable for Westmere and
23 # later. Improvement varies from one benchmark and µ-arch to another.
24 # Vanilla code path is at most 20% faster than compiler-generated code
25 # [not very impressive], while PCLMULQDQ - whole 85%-160% better on
26 # 163- and 571-bit ECDH benchmarks on Intel CPUs. Keep in mind that
27 # these coefficients are not ones for bn_GF2m_mul_2x2 itself, as not
28 # all CPU time is burnt in it...
32 if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
34 $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
36 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
37 ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
38 ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
39 die "can't locate x86_64-xlate.pl";
41 open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
44 ($lo,$hi)=("%rax","%rdx"); $a=$lo;
45 ($i0,$i1)=("%rsi","%rdi");
46 ($t0,$t1)=("%rbx","%rcx");
47 ($b,$mask)=("%rbp","%r8");
48 ($a1,$a2,$a4,$a8,$a12,$a48)=map("%r$_",(9..15));
49 ($R,$Tx)=("%xmm0","%xmm1");
54 .type _mul_1x1,\@abi-omnipotent
62 and $a,$a1 # a1=a&0x1fffffffffffffff
64 sar \$63,$a # broadcast 63rd bit
66 sar \$63,$i0 # broadcast 62nd bit
69 sar \$63,$i1 # boardcast 61st bit
70 mov $a,$hi # $a is $lo
87 movq \$0,0(%rsp) # tab[0]=0
89 mov $a1,8(%rsp) # tab[1]=a1
91 mov $a2,16(%rsp) # tab[2]=a2
93 mov $a12,24(%rsp) # tab[3]=a1^a2
96 mov $a4,32(%rsp) # tab[4]=a4
98 mov $a1,40(%rsp) # tab[5]=a1^a4
100 mov $a2,48(%rsp) # tab[6]=a2^a4
101 xor $a48,$a1 # a1^a4^a4^a8=a1^a8
102 mov $a12,56(%rsp) # tab[7]=a1^a2^a4
103 xor $a48,$a2 # a2^a4^a4^a8=a1^a8
105 mov $a8,64(%rsp) # tab[8]=a8
106 xor $a48,$a12 # a1^a2^a4^a4^a8=a1^a2^a8
107 mov $a1,72(%rsp) # tab[9]=a1^a8
108 xor $a4,$a1 # a1^a8^a4
109 mov $a2,80(%rsp) # tab[10]=a2^a8
110 xor $a4,$a2 # a2^a8^a4
111 mov $a12,88(%rsp) # tab[11]=a1^a2^a8
113 xor $a4,$a12 # a1^a2^a8^a4
114 mov $a48,96(%rsp) # tab[12]=a4^a8
116 mov $a1,104(%rsp) # tab[13]=a1^a4^a8
118 mov $a2,112(%rsp) # tab[14]=a2^a4^a8
120 mov $a12,120(%rsp) # tab[15]=a1^a2^a4^a8
125 movq (%rsp,$i0,8),$R # half of calculations is done in SSE2
130 for ($n=1;$n<8;$n++) {
137 movq (%rsp,$i0,8),$Tx
138 shr \$`64-(8*$n-4)`,$t0
154 shr \$`64-(8*$n-4)`,$t0
165 .size _mul_1x1,.-_mul_1x1
168 ($rp,$a1,$a0,$b1,$b0) = $win64? ("%rcx","%rdx","%r8", "%r9","%r10") : # Win64 order
169 ("%rdi","%rsi","%rdx","%rcx","%r8"); # Unix order
172 .extern OPENSSL_ia32cap_P
173 .globl bn_GF2m_mul_2x2
174 .type bn_GF2m_mul_2x2,\@abi-omnipotent
177 mov OPENSSL_ia32cap_P(%rip),%rax
179 jnc .Lvanilla_mul_2x2
185 $code.=<<___ if ($win64);
188 $code.=<<___ if (!$win64);
194 pclmulqdq \$0,%xmm1,%xmm0 # a1·b1
197 pclmulqdq \$0,%xmm3,%xmm2 # a0·b0
198 pclmulqdq \$0,%xmm5,%xmm4 # (a0+a1)·(b0+b1)
200 xorps %xmm2,%xmm4 # (a0+a1)·(b0+b1)-a0·b0-a1·b1
214 $code.=<<___ if ($win64);
215 mov `8*17+40`(%rsp),$b0
226 mov $rp,32(%rsp) # save the arguments
235 call _mul_1x1 # a1·b1
241 call _mul_1x1 # a0·b0
249 call _mul_1x1 # (a0+a1)·(b0+b1)
251 @r=("%rbx","%rcx","%rdi","%rsi");
277 $code.=<<___ if ($win64);
285 .size bn_GF2m_mul_2x2,.-bn_GF2m_mul_2x2
286 .asciz "GF(2^m) Multiplication for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
290 # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,
291 # CONTEXT *context,DISPATCHER_CONTEXT *disp)
299 .extern __imp_RtlVirtualUnwind
301 .type se_handler,\@abi-omnipotent
315 mov 152($context),%rax # pull context->Rsp
316 mov 248($context),%rbx # pull context->Rip
318 lea .Lbody_mul_2x2(%rip),%r10
319 cmp %r10,%rbx # context->Rip<"prologue" label
322 mov 8*10(%rax),%r14 # mimic epilogue
330 mov %rbx,144($context) # restore context->Rbx
331 mov %rbp,160($context) # restore context->Rbp
332 mov %rsi,168($context) # restore context->Rsi
333 mov %rdi,176($context) # restore context->Rdi
334 mov %r12,216($context) # restore context->R12
335 mov %r13,224($context) # restore context->R13
336 mov %r14,232($context) # restore context->R14
340 mov %rax,152($context) # restore context->Rsp
342 mov 40($disp),%rdi # disp->ContextRecord
343 mov $context,%rsi # context
344 mov \$154,%ecx # sizeof(CONTEXT)
345 .long 0xa548f3fc # cld; rep movsq
348 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER
349 mov 8(%rsi),%rdx # arg2, disp->ImageBase
350 mov 0(%rsi),%r8 # arg3, disp->ControlPc
351 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry
352 mov 40(%rsi),%r10 # disp->ContextRecord
353 lea 56(%rsi),%r11 # &disp->HandlerData
354 lea 24(%rsi),%r12 # &disp->EstablisherFrame
355 mov %r10,32(%rsp) # arg5
356 mov %r11,40(%rsp) # arg6
357 mov %r12,48(%rsp) # arg7
358 mov %rcx,56(%rsp) # arg8, (NULL)
359 call *__imp_RtlVirtualUnwind(%rip)
361 mov \$1,%eax # ExceptionContinueSearch
373 .size se_handler,.-se_handler
381 .rva .Lvanilla_mul_2x2
387 .byte 0x01,0x07,0x02,0x00
388 .byte 0x07,0x01,0x11,0x00 # sub rsp,128+8
395 $code =~ s/\`([^\`]*)\`/eval($1)/gem;