2 # Copyright 2007-2018 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
19 # Performance improvement over vanilla C code varies from 85% to 45%
20 # depending on key length and benchmark. Unfortunately in this context
21 # these are not very impressive results [for code that utilizes "wide"
22 # 64x64=128-bit multiplication, which is not commonly available to C
23 # programmers], at least hand-coded bn_asm.c replacement is known to
24 # provide 30-40% better results for longest keys. Well, on a second
25 # thought it's not very surprising, because z-CPUs are single-issue
26 # and _strictly_ in-order execution, while bn_mul_mont is more or less
27 # dependent on CPU ability to pipe-line instructions and have several
28 # of them "in-flight" at the same time. I mean while other methods,
29 # for example Karatsuba, aim to minimize amount of multiplications at
30 # the cost of other operations increase, bn_mul_mont aim to neatly
31 # "overlap" multiplications and the other operations [and on most
32 # platforms even minimize the amount of the other operations, in
33 # particular references to memory]. But it's possible to improve this
34 # module performance by implementing dedicated squaring code-path and
35 # possibly by unrolling loops...
39 # Reschedule to minimize/avoid Address Generation Interlock hazard,
40 # make inner loops counter-based.
44 # Adapt for -m31 build. If kernel supports what's called "highgprs"
45 # feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
46 # instructions and achieve "64-bit" performance even in 31-bit legacy
47 # application context. The feature is not specific to any particular
48 # processor, as long as it's "z-CPU". Latter implies that the code
49 # remains z/Architecture specific. Compatibility with 32-bit BN_ULONG
50 # is achieved by swapping words after 64-bit loads, follow _dswap-s.
51 # On z990 it was measured to perform 2.6-2.2 times better than
52 # compiler-generated code, less for longer keys...
54 # $output is the last argument if it looks like a file (it has an extension)
55 # $flavour is the first argument if it doesn't look like a file
56 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
57 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
59 if ($flavour =~ /3[12]/) {
67 $output and open STDOUT,">$output";
69 $stdframe=16*$SIZE_T+4*8;
75 $rp="%r2"; # BN_ULONG *rp,
76 $ap="%r3"; # const BN_ULONG *ap,
77 $bp="%r4"; # const BN_ULONG *bp,
78 $np="%r5"; # const BN_ULONG *np,
79 $n0="%r6"; # const BN_ULONG *n0,
80 #$num="160(%r15)" # int num);
97 .type bn_mul_mont,\@function
99 lgf $num,`$stdframe+$SIZE_T-4`($sp) # pull $num
100 sla $num,`log($SIZE_T)/log(2)` # $num to enumerate bytes
103 st${g} %r2,2*$SIZE_T($sp)
107 blr %r14 # if($num<16) return 0;
109 $code.=<<___ if ($flavour =~ /3[12]/);
111 bnzr %r14 # if ($num&1) return 0;
113 $code.=<<___ if ($flavour !~ /3[12]/);
115 bhr %r14 # if($num>96) return 0;
118 stm${g} %r3,%r15,3*$SIZE_T($sp)
120 lghi $rp,-$stdframe-8 # leave room for carry bit
124 la $sp,0($j,$rp) # alloca
125 st${g} %r0,0($sp) # back chain
127 sra $num,3 # restore $num
128 la $bp,0($j,$bp) # restore $bp
129 ahi $num,-1 # adjust $num for inner loop
130 lg $n0,0($n0) # pull n0
137 mlgr $ahi,$bi # ap[0]*bp[0]
140 lgr $mn0,$alo # "tp[0]"*n0
145 mlgr $nhi,$mn0 # np[0]*m1
146 algr $nlo,$alo # +="tp[0]"
157 mlgr $ahi,$bi # ap[j]*bp[0]
164 mlgr $nhi,$mn0 # np[j]*m1
167 alcgr $nhi,$NHI # +="tp[j]"
171 stg $nlo,$stdframe-8($j,$sp) # tp[j-1]=
177 alcgr $AHI,$AHI # upmost overflow bit
178 stg $NHI,$stdframe-8($j,$sp)
179 stg $AHI,$stdframe($j,$sp)
183 lg $bi,0($bp) # bp[i]
187 mlgr $ahi,$bi # ap[0]*bp[i]
188 alg $alo,$stdframe($sp) # +=tp[0]
193 msgr $mn0,$n0 # tp[0]*n0
195 lg $nlo,0($np) # np[0]
197 mlgr $nhi,$mn0 # np[0]*m1
198 algr $nlo,$alo # +="tp[0]"
209 mlgr $ahi,$bi # ap[j]*bp[i]
213 alg $alo,$stdframe($j,$sp)# +=tp[j]
218 mlgr $nhi,$mn0 # np[j]*m1
222 algr $nlo,$alo # +="tp[j]"
225 stg $nlo,$stdframe-8($j,$sp) # tp[j-1]=
232 alg $NHI,$stdframe($j,$sp)# accumulate previous upmost overflow bit
234 alcgr $AHI,$ahi # new upmost overflow bit
235 stg $NHI,$stdframe-8($j,$sp)
236 stg $AHI,$stdframe($j,$sp)
239 cl${g} $bp,`$stdframe+8+4*$SIZE_T`($j,$sp) # compare to &bp[num]
242 l${g} $rp,`$stdframe+8+2*$SIZE_T`($j,$sp) # reincarnate rp
243 la $ap,$stdframe($sp)
244 ahi $num,1 # restore $num, incidentally clears "borrow"
248 .Lsub: lg $alo,0($j,$ap)
256 slbgr $AHI,$ahi # handle upmost carry
262 .Lcopy: lg $ahi,$stdframe($j,$sp) # conditional copy
268 stg $j,$stdframe($j,$sp) # zap tp
273 la %r1,`$stdframe+8+6*$SIZE_T`($j,$sp)
274 lm${g} %r6,%r15,0(%r1)
275 lghi %r2,1 # signal "processed"
277 .size bn_mul_mont,.-bn_mul_mont
278 .string "Montgomery Multiplication for s390x, CRYPTOGAMS by <appro\@openssl.org>"
281 foreach (split("\n",$code)) {
282 s/\`([^\`]*)\`/eval $1/ge;
283 s/_dswap\s+(%r[0-9]+)/sprintf("rllg\t%s,%s,32",$1,$1) if($SIZE_T==4)/e;
286 close STDOUT or die "error closing STDOUT: $!";