2 # Copyright 2009-2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
19 # Provided that UltraSPARC VIS instructions are pipe-lined(*) and
20 # pairable(*) with IALU ones, offloading of Xupdate to the UltraSPARC
21 # Graphic Unit would make it possible to achieve higher instruction-
22 # level parallelism, ILP, and thus higher performance. It should be
23 # explicitly noted that ILP is the keyword, and it means that this
24 # code would be unsuitable for cores like UltraSPARC-Tx. The idea is
25 # not really novel, Sun had VIS-powered implementation for a while.
26 # Unlike Sun's implementation this one can process multiple unaligned
27 # input blocks, and as such works as drop-in replacement for OpenSSL
28 # sha1_block_data_order. Performance improvement was measured to be
29 # 40% over pure IALU sha1-sparcv9.pl on UltraSPARC-IIi, but 12% on
30 # UltraSPARC-III. See below for discussion...
32 # The module does not present direct interest for OpenSSL, because
33 # it doesn't provide better performance on contemporary SPARCv9 CPUs,
34 # UltraSPARC-Tx and SPARC64-V[II] to be specific. Those who feel they
35 # absolutely must score on UltraSPARC-I-IV can simply replace
36 # crypto/sha/asm/sha1-sparcv9.pl with this module.
38 # (*) "Pipe-lined" means that even if it takes several cycles to
39 # complete, next instruction using same functional unit [but not
40 # depending on the result of the current instruction] can start
41 # execution without having to wait for the unit. "Pairable"
42 # means that two [or more] independent instructions can be
43 # issued at the very same time.
46 for (@ARGV) { $bits=64 if (/\-m64/ || /\-xarch\=v9/); }
47 if ($bits==64) { $bias=2047; $frame=192; }
48 else { $bias=0; $frame=112; }
50 $output=pop and open STDOUT,">$output";
84 @VK=($VK_00_19,$VK_20_39,$VK_40_59,$VK_60_79);
85 @X=("%f0", "%f1", "%f2", "%f3", "%f4", "%f5", "%f6", "%f7",
86 "%f8", "%f9","%f10","%f11","%f12","%f13","%f14","%f15","%f16");
88 # This is reference 2x-parallelized VIS-powered Xupdate procedure. It
89 # covers even K_NN_MM addition...
92 my $K=@VK[($i+16)/20];
95 # [ provided that GSR.alignaddr_offset is 5, $mul contains
96 # 0x100ULL<<32|0x100 value and K_NN_MM are pre-loaded to
97 # chosen registers... ]
99 fxors @X[($j+13)%16],@X[$j],@X[$j] !-1/-1/-1:X[0]^=X[13]
100 fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14]
101 fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9]
102 fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9]
103 faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24
104 fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1
105 fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1
106 ![fxors %f15,%f2,%f2]
107 for %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp
108 ![fxors %f0,%f3,%f3] !10/17/12:X[0] dependency
109 fpadd32 $K,@X[$j],%f20
110 std %f20,[$Xfer+`4*$j`]
112 # The numbers delimited with slash are the earliest possible dispatch
113 # cycles for given instruction assuming 1 cycle latency for simple VIS
114 # instructions, such as on UltraSPARC-I&II, 3 cycles latency, such as
115 # on UltraSPARC-III&IV, and 2 cycles latency(*), respectively. Being
116 # 2x-parallelized the procedure is "worth" 5, 8.5 or 6 ticks per SHA1
117 # round. As [long as] FPU/VIS instructions are perfectly pairable with
118 # IALU ones, the round timing is defined by the maximum between VIS
119 # and IALU timings. The latter varies from round to round and averages
120 # out at 6.25 ticks. This means that USI&II should operate at IALU
121 # rate, while USIII&IV - at VIS rate. This explains why performance
122 # improvement varies among processors. Well, given that pure IALU
123 # sha1-sparcv9.pl module exhibits virtually uniform performance of
124 # ~9.3 cycles per SHA1 round. Timings mentioned above are theoretical
125 # lower limits. Real-life performance was measured to be 6.6 cycles
126 # per SHA1 round on USIIi and 8.3 on USIII. The latter is lower than
127 # half-round VIS timing, because there are 16 Xupdate-free rounds,
128 # which "push down" average theoretical timing to 8 cycles...
130 # (*) SPARC64-V[II] was originally believed to have 2 cycles VIS
131 # latency. Well, it might have, but it doesn't have dedicated
132 # VIS-unit. Instead, VIS instructions are executed by other
133 # functional units, ones used here - by IALU. This doesn't
134 # improve effective ILP...
137 # The reference Xupdate procedure is then "strained" over *pairs* of
138 # BODY_NN_MM and kind of modulo-scheduled in respect to X[n]^=X[n+13]
139 # and K_NN_MM addition. It's "running" 15 rounds ahead, which leaves
140 # plenty of room to amortize for read-after-write hazard, as well as
141 # to fetch and align input for the next spin. The VIS instructions are
142 # scheduled for latency of 2 cycles, because there are not enough IALU
143 # instructions to schedule for latency of 3, while scheduling for 1
144 # would give no gain on USI&II anyway.
147 my ($i,$a,$b,$c,$d,$e)=@_;
149 my $k=($j+16+2)%16; # ahead reference
150 my $l=($j+16-2)%16; # behind reference
151 my $K=@VK[($j+16-2)/20];
155 $code.=<<___ if (!($i&1));
158 ld [$Xfer+`4*($i%16)`],$Xi
159 fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14]
162 fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9]
167 fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9]
172 faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24
174 $code.=<<___ if ($i&1);
177 ld [$Xfer+`4*($i%16)`],$Xi
178 fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1
181 fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1
184 fpadd32 $K,@X[$l],%f20 !
187 fxors @X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X[0]^=X[13]
190 fxor %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp
194 $code.=<<___ if ($i&1 && $i>=2);
195 std %f20,[$Xfer+`4*$l`] !
200 my ($i,$a,$b,$c,$d,$e)=@_;
202 my $k=($j+16+2)%16; # ahead reference
203 my $l=($j+16-2)%16; # behind reference
204 my $K=@VK[($j+16-2)/20];
208 $code.=<<___ if (!($i&1) && $i<64);
210 ld [$Xfer+`4*($i%16)`],$Xi
211 fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14]
214 fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9]
219 fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9]
224 faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24
226 $code.=<<___ if ($i&1 && $i<64);
228 ld [$Xfer+`4*($i%16)`],$Xi
229 fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1
232 fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1
235 fpadd32 $K,@X[$l],%f20 !
238 fxors @X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X[0]^=X[13]
241 fxor %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp
244 std %f20,[$Xfer+`4*$l`] !
246 $code.=<<___ if ($i==64);
248 ld [$Xfer+`4*($i%16)`],$Xi
249 fpadd32 $K,@X[$l],%f20
256 std %f20,[$Xfer+`4*$l`]
262 $code.=<<___ if ($i>64);
264 ld [$Xfer+`4*($i%16)`],$Xi
279 my ($i,$a,$b,$c,$d,$e)=@_;
281 my $k=($j+16+2)%16; # ahead reference
282 my $l=($j+16-2)%16; # behind reference
283 my $K=@VK[($j+16-2)/20];
287 $code.=<<___ if (!($i&1));
289 ld [$Xfer+`4*($i%16)`],$Xi
290 fxors @X[($j+14)%16],@X[$j+1],@X[$j+1]! 0/ 0/ 0:X[1]^=X[14]
293 fxor @X[($j+2)%16],@X[($j+8)%16],%f18! 1/ 1/ 1:Tmp=X[2,3]^X[8,9]
298 fxor %f18,@X[$j],@X[$j] ! 2/ 4/ 3:X[0,1]^=X[2,3]^X[8,9]
303 faligndata @X[$j],@X[$j],%f18 ! 3/ 7/ 5:Tmp=X[0,1]>>>24
306 fpadd32 @X[$j],@X[$j],@X[$j] ! 4/ 8/ 6:X[0,1]<<=1
308 $code.=<<___ if ($i&1);
310 ld [$Xfer+`4*($i%16)`],$Xi
313 fmul8ulx16 %f18,$fmul,%f18 ! 5/10/ 7:Tmp>>=7, Tmp&=1
316 fpadd32 $K,@X[$l],%f20 !
319 fxors @X[($k+13)%16],@X[$k],@X[$k] !-1/-1/-1:X[0]^=X[13]
322 fxor %f18,@X[$j],@X[$j] ! 8/14/10:X[0,1]|=Tmp
327 std %f20,[$Xfer+`4*$l`] !
331 # If there is more data to process, then we pre-fetch the data for
332 # next iteration in last ten rounds...
334 my ($i,$a,$b,$c,$d,$e)=@_;
340 $code.=<<___ if ($i==70);
342 ld [$Xfer+`4*($i%16)`],$Xi
357 and $nXfer,255,$nXfer
358 alignaddr %g0,$align,%g0
359 add $base,$nXfer,$nXfer
361 $code.=<<___ if ($i==71);
363 ld [$Xfer+`4*($i%16)`],$Xi
375 $code.=<<___ if ($i>=72);
376 faligndata @X[$m],@X[$m+2],@X[$m]
378 ld [$Xfer+`4*($i%16)`],$Xi
383 fpadd32 $VK_00_19,@X[$m],%f20
391 $code.=<<___ if ($i<77);
392 ldd [$inp+`8*($i+1-70)`],@X[2*($i+1-70)]
394 $code.=<<___ if ($i==77); # redundant if $inp was aligned
397 ldd [$inp+$tmp0],@X[16]
399 $code.=<<___ if ($i>=72);
400 std %f20,[$nXfer+`4*$m`]
405 .section ".text",#alloc,#execinstr
409 .long 0x5a827999,0x5a827999 ! K_00_19
410 .long 0x6ed9eba1,0x6ed9eba1 ! K_20_39
411 .long 0x8f1bbcdc,0x8f1bbcdc ! K_40_59
412 .long 0xca62c1d6,0xca62c1d6 ! K_60_79
413 .long 0x00000100,0x00000100
415 .type vis_const,#object
416 .size vis_const,(.-vis_const)
418 .globl sha1_block_data_order
419 sha1_block_data_order:
421 add %fp,$bias-256,$base
424 add %o7,vis_const-1b,$tmp0
426 ldd [$tmp0+0],$VK_00_19
427 ldd [$tmp0+8],$VK_20_39
428 ldd [$tmp0+16],$VK_40_59
429 ldd [$tmp0+24],$VK_60_79
435 sub $base,$bias+$frame,%sp
442 ! X[16] is maintained in FP register bank
443 alignaddr %g0,$align,%g0
451 add $base,$Xfer,$Xfer
455 brz,pt $align,.Laligned
459 faligndata @X[0],@X[2],@X[0]
460 faligndata @X[2],@X[4],@X[2]
461 faligndata @X[4],@X[6],@X[4]
462 faligndata @X[6],@X[8],@X[6]
463 faligndata @X[8],@X[10],@X[8]
464 faligndata @X[10],@X[12],@X[10]
465 faligndata @X[12],@X[14],@X[12]
466 faligndata @X[14],@X[16],@X[14]
471 alignaddr %g0,$tmp0,%g0
472 fpadd32 $VK_00_19,@X[0],%f16
473 fpadd32 $VK_00_19,@X[2],%f18
474 fpadd32 $VK_00_19,@X[4],%f20
475 fpadd32 $VK_00_19,@X[6],%f22
476 fpadd32 $VK_00_19,@X[8],%f24
477 fpadd32 $VK_00_19,@X[10],%f26
478 fpadd32 $VK_00_19,@X[12],%f28
479 fpadd32 $VK_00_19,@X[14],%f30
491 fxors @X[13],@X[0],@X[0]
498 for ($i=0;$i<20;$i++) { &BODY_00_19($i,@V); unshift(@V,pop(@V)); }
499 for (;$i<40;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
500 for (;$i<60;$i++) { &BODY_40_59($i,@V); unshift(@V,pop(@V)); }
501 for (;$i<70;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
504 bz,pn `$bits==32?"%icc":"%xcc"`,.Ltail
507 for (;$i<80;$i++) { &BODY_70_79($i,@V); unshift(@V,pop(@V)); }
515 fxors @X[13],@X[0],@X[0]
521 alignaddr %g0,$tmp0,%g0
529 for($i=70;$i<80;$i++) { &BODY_20_39($i,@V); unshift(@V,pop(@V)); }
545 .type sha1_block_data_order,#function
546 .size sha1_block_data_order,(.-sha1_block_data_order)
547 .asciz "SHA1 block transform for SPARCv9a, CRYPTOGAMS by <appro\@openssl.org>"
551 # Purpose of these subroutines is to explicitly encode VIS instructions,
552 # so that one can compile the module without having to specify VIS
553 # extensions on compiler command line, e.g. -xarch=v9 vs. -xarch=v9a.
554 # Idea is to reserve for option to produce "universal" binary and let
555 # programmer detect if current CPU is VIS capable at run-time.
557 my ($mnemonic,$rs1,$rs2,$rd)=@_;
559 my %visopf = ( "fmul8ulx16" => 0x037,
560 "faligndata" => 0x048,
565 $ref = "$mnemonic\t$rs1,$rs2,$rd";
567 if ($opf=$visopf{$mnemonic}) {
568 foreach ($rs1,$rs2,$rd) {
569 return $ref if (!/%f([0-9]{1,2})/);
572 return $ref if ($1&1);
573 # re-encode for upper double register addressing
578 return sprintf ".word\t0x%08x !%s",
579 0x81b00000|$rd<<25|$rs1<<14|$opf<<5|$rs2,
586 my ($mnemonic,$rs1,$rs2,$rd)=@_;
587 my %bias = ( "g" => 0, "o" => 8, "l" => 16, "i" => 24 );
588 my $ref="$mnemonic\t$rs1,$rs2,$rd";
590 foreach ($rs1,$rs2,$rd) {
591 if (/%([goli])([0-7])/) { $_=$bias{$1}+$2; }
592 else { return $ref; }
594 return sprintf ".word\t0x%08x !%s",
595 0x81b00300|$rd<<25|$rs1<<14|$rs2,
599 $code =~ s/\`([^\`]*)\`/eval $1/gem;
600 $code =~ s/\b(f[^\s]*)\s+(%f[0-9]{1,2}),(%f[0-9]{1,2}),(%f[0-9]{1,2})/
603 $code =~ s/\b(alignaddr)\s+(%[goli][0-7]),(%[goli][0-7]),(%[goli][0-7])/
604 &unalignaddr($1,$2,$3,$4)