3 # ====================================================================
4 # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
12 # The reason for undertaken effort is basically following. Even though
13 # Power 6 CPU operates at incredible 4.7GHz clock frequency, its PKI
14 # performance was observed to be less than impressive, essentially as
15 # fast as 1.8GHz PPC970, or 2.6 times(!) slower than one would hope.
16 # Well, it's not surprising that IBM had to make some sacrifices to
17 # boost the clock frequency that much, but no overall improvement?
18 # Having observed how much difference did switching to FPU make on
19 # UltraSPARC, playing same stunt on Power 6 appeared appropriate...
20 # Unfortunately the resulting performance improvement is not as
21 # impressive, ~30%, and in absolute terms is still very far from what
22 # one would expect from 4.7GHz CPU. There is a chance that I'm doing
23 # something wrong, but in the lack of assembler level micro-profiling
24 # data or at least decent platform guide I can't tell... Or better
25 # results might be achieved with VMX... Anyway, this module provides
26 # *worse* performance on other PowerPC implementations, ~40-15% slower
27 # on PPC970 depending on key length and ~40% slower on Power 5 for all
28 # key lengths. As it's obviously inappropriate as "best all-round"
29 # alternative, it has to be complemented with run-time CPU family
30 # detection. Oh! It should also be noted that unlike other PowerPC
31 # implementation IALU ppc-mont.pl module performs *suboptimaly* on
32 # >=1024-bit key lengths on Power 6. It should also be noted that
33 # *everything* said so far applies to 64-bit builds! As far as 32-bit
34 # application executed on 64-bit CPU goes, this module is likely to
35 # become preferred choice, because it's easy to adapt it for such
36 # case and *is* faster than 32-bit ppc-mont.pl on *all* processors.
40 # Micro-profiling assisted optimization results in ~15% improvement
41 # over original ppc64-mont.pl version, or overall ~50% improvement
42 # over ppc.pl module on Power 6. If compared to ppc-mont.pl on same
43 # Power 6 CPU, this module is 5-150% faster depending on key length,
44 # [hereafter] more for longer keys. But if compared to ppc-mont.pl
45 # on 1.8GHz PPC970, it's only 5-55% faster. Still far from impressive
46 # in absolute terms, but it's apparently the way Power 6 is...
50 # Adapted for 32-bit build this module delivers 25-120%, yes, more
51 # than *twice* for longer keys, performance improvement over 32-bit
52 # ppc-mont.pl on 1.8GHz PPC970. However! This implementation utilizes
53 # even 64-bit integer operations and the trouble is that most PPC
54 # operating systems don't preserve upper halves of general purpose
55 # registers upon 32-bit signal delivery. They do preserve them upon
56 # context switch, but not signalling:-( This means that asynchronous
57 # signals have to be blocked upon entry to this subroutine. Signal
58 # masking (and of course complementary unmasking) has quite an impact
59 # on performance, naturally larger for shorter keys. It's so severe
60 # that 512-bit key performance can be as low as 1/3 of expected one.
61 # This is why this routine can be engaged for longer key operations
62 # only on these OSes, see crypto/ppccap.c for further details. MacOS X
63 # is an exception from this and doesn't require signal masking, and
64 # that's where above improvement coefficients were collected. For
65 # others alternative would be to break dependence on upper halves of
66 # GPRs by sticking to 32-bit integer operations...
70 if ($flavour =~ /32/) {
73 $FRAME= $SIZE_T*12+8*12;
74 $fname= "bn_mul_mont_fpu64";
76 $STUX= "stwux"; # store indexed and update
79 } elsif ($flavour =~ /64/) {
82 $FRAME= $SIZE_T*12+8*12;
83 $fname= "bn_mul_mont_fpu64";
85 # same as above, but 64-bit mnemonics...
86 $STUX= "stdux"; # store indexed and update
89 } else { die "nonsense $flavour"; }
91 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
92 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
93 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
94 die "can't locate ppc-xlate.pl";
96 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
98 $FRAME=($FRAME+63)&~63;
110 $rp="r9"; # $rp is reassigned
114 # non-volatile registers
115 $nap_d="r14"; # interleaved ap and np in double format
117 $t0="r16"; # temporary registers
126 # PPC offers enough register bank capacity to unroll inner loops twice
150 $ba="f0"; $bb="f1"; $bc="f2"; $bd="f3";
151 $na="f4"; $nb="f5"; $nc="f6"; $nd="f7";
152 $dota="f8"; $dotb="f9";
153 $A0="f10"; $A1="f11"; $A2="f12"; $A3="f13";
154 $N0="f14"; $N1="f15"; $N2="f16"; $N3="f17";
155 $T0a="f18"; $T0b="f19";
156 $T1a="f20"; $T1b="f21";
157 $T2a="f22"; $T2b="f23";
158 $T3a="f24"; $T3b="f25";
160 # sp----------->+-------------------------------+
162 # +-------------------------------+
164 # +-------------------------------+
165 # | 10 saved gpr, r14-r23 |
168 # +12*size_t +-------------------------------+
169 # | 12 saved fpr, f14-f25 |
172 # +12*8 +-------------------------------+
173 # | padding to 64 byte boundary |
175 # +X +-------------------------------+
176 # | 16 gpr<->fpr transfer zone |
179 # +16*8 +-------------------------------+
180 # | __int64 tmp[-1] |
181 # +-------------------------------+
182 # | __int64 tmp[num] |
186 # +(num+1)*8 +-------------------------------+
187 # | padding to 64 byte boundary |
189 # +X +-------------------------------+
190 # | double nap_d[4*num] |
194 # +-------------------------------+
203 cmpwi $num,`3*8/$SIZE_T`
204 mr $rp,r3 ; $rp is reassigned
205 li r3,0 ; possible "not handled" return code
207 andi. r0,$num,`16/$SIZE_T-1` ; $num has to be "even"
210 slwi $num,$num,`log($SIZE_T)/log(2)` ; num*=sizeof(BN_LONG)
212 slwi $tp,$num,2 ; place for {an}p_{lh}[num], i.e. 4*num
213 add $tp,$tp,$num ; place for tp[num+1]
214 addi $tp,$tp,`$FRAME+$TRANSFER+8+64+$RZONE`
215 subf $tp,$tp,$sp ; $sp-$tp
216 and $tp,$tp,$i ; minimize TLB usage
217 subf $tp,$sp,$tp ; $tp-$sp
218 $STUX $sp,$sp,$tp ; alloca
220 $PUSH r14,`2*$SIZE_T`($sp)
221 $PUSH r15,`3*$SIZE_T`($sp)
222 $PUSH r16,`4*$SIZE_T`($sp)
223 $PUSH r17,`5*$SIZE_T`($sp)
224 $PUSH r18,`6*$SIZE_T`($sp)
225 $PUSH r19,`7*$SIZE_T`($sp)
226 $PUSH r20,`8*$SIZE_T`($sp)
227 $PUSH r21,`9*$SIZE_T`($sp)
228 $PUSH r22,`10*$SIZE_T`($sp)
229 $PUSH r23,`11*$SIZE_T`($sp)
230 stfd f14,`12*$SIZE_T+0`($sp)
231 stfd f15,`12*$SIZE_T+8`($sp)
232 stfd f16,`12*$SIZE_T+16`($sp)
233 stfd f17,`12*$SIZE_T+24`($sp)
234 stfd f18,`12*$SIZE_T+32`($sp)
235 stfd f19,`12*$SIZE_T+40`($sp)
236 stfd f20,`12*$SIZE_T+48`($sp)
237 stfd f21,`12*$SIZE_T+56`($sp)
238 stfd f22,`12*$SIZE_T+64`($sp)
239 stfd f23,`12*$SIZE_T+72`($sp)
240 stfd f24,`12*$SIZE_T+80`($sp)
241 stfd f25,`12*$SIZE_T+88`($sp)
243 $code.=<<___ if ($SIZE_T==8);
244 ld $a0,0($ap) ; pull ap[0] value
245 ld $n0,0($n0) ; pull n0[0] value
246 ld $t3,0($bp) ; bp[0]
248 $code.=<<___ if ($SIZE_T==4);
250 lwz $a0,0($ap) ; pull ap[0,1] value
252 lwz $n0,0($t1) ; pull n0[0,1] value
254 lwz $t3,0($bp) ; bp[0,1]
261 addi $tp,$sp,`$FRAME+$TRANSFER+8+64`
264 and $nap_d,$nap_d,$i ; align to 64 bytes
266 mulld $t7,$a0,$t3 ; ap[0]*bp[0]
267 ; nap_d is off by 1, because it's used with stfdu/lfdu
268 addi $nap_d,$nap_d,-8
269 srwi $j,$num,`3+1` ; counter register, num/2
270 mulld $t7,$t7,$n0 ; tp[0]*n0
272 addi $tp,$sp,`$FRAME+$TRANSFER-8`
276 ; transfer bp[0] to FPU as 4x16-bit values
281 std $t0,`$FRAME+0`($sp)
282 std $t1,`$FRAME+8`($sp)
283 std $t2,`$FRAME+16`($sp)
284 std $t3,`$FRAME+24`($sp)
285 ; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values
290 std $t4,`$FRAME+32`($sp)
291 std $t5,`$FRAME+40`($sp)
292 std $t6,`$FRAME+48`($sp)
293 std $t7,`$FRAME+56`($sp)
295 $code.=<<___ if ($SIZE_T==8);
296 lwz $t0,4($ap) ; load a[j] as 32-bit word pair
298 lwz $t2,12($ap) ; load a[j+1] as 32-bit word pair
300 lwz $t4,4($np) ; load n[j] as 32-bit word pair
302 lwz $t6,12($np) ; load n[j+1] as 32-bit word pair
305 $code.=<<___ if ($SIZE_T==4);
306 lwz $t0,0($ap) ; load a[j..j+3] as 32-bit word pairs
310 lwz $t4,0($np) ; load n[j..j+3] as 32-bit word pairs
316 lfd $ba,`$FRAME+0`($sp)
317 lfd $bb,`$FRAME+8`($sp)
318 lfd $bc,`$FRAME+16`($sp)
319 lfd $bd,`$FRAME+24`($sp)
320 lfd $na,`$FRAME+32`($sp)
321 lfd $nb,`$FRAME+40`($sp)
322 lfd $nc,`$FRAME+48`($sp)
323 lfd $nd,`$FRAME+56`($sp)
324 std $t0,`$FRAME+64`($sp)
325 std $t1,`$FRAME+72`($sp)
326 std $t2,`$FRAME+80`($sp)
327 std $t3,`$FRAME+88`($sp)
328 std $t4,`$FRAME+96`($sp)
329 std $t5,`$FRAME+104`($sp)
330 std $t6,`$FRAME+112`($sp)
331 std $t7,`$FRAME+120`($sp)
341 lfd $A0,`$FRAME+64`($sp)
342 lfd $A1,`$FRAME+72`($sp)
343 lfd $A2,`$FRAME+80`($sp)
344 lfd $A3,`$FRAME+88`($sp)
345 lfd $N0,`$FRAME+96`($sp)
346 lfd $N1,`$FRAME+104`($sp)
347 lfd $N2,`$FRAME+112`($sp)
348 lfd $N3,`$FRAME+120`($sp)
362 stfd $A0,8($nap_d) ; save a[j] in double format
366 stfd $A2,24($nap_d) ; save a[j+1] in double format
370 stfd $N0,40($nap_d) ; save n[j] in double format
374 stfd $N2,56($nap_d) ; save n[j+1] in double format
377 fmadd $T1a,$A0,$bc,$T1a
378 fmadd $T1b,$A0,$bd,$T1b
379 fmadd $T2a,$A1,$bc,$T2a
380 fmadd $T2b,$A1,$bd,$T2b
381 fmadd $T3a,$A2,$bc,$T3a
382 fmadd $T3b,$A2,$bd,$T3b
386 fmadd $T1a,$N1,$na,$T1a
387 fmadd $T1b,$N1,$nb,$T1b
388 fmadd $T2a,$N2,$na,$T2a
389 fmadd $T2b,$N2,$nb,$T2b
390 fmadd $T3a,$N3,$na,$T3a
391 fmadd $T3b,$N3,$nb,$T3b
392 fmadd $T0a,$N0,$na,$T0a
393 fmadd $T0b,$N0,$nb,$T0b
395 fmadd $T1a,$N0,$nc,$T1a
396 fmadd $T1b,$N0,$nd,$T1b
397 fmadd $T2a,$N1,$nc,$T2a
398 fmadd $T2b,$N1,$nd,$T2b
399 fmadd $T3a,$N2,$nc,$T3a
400 fmadd $T3b,$N2,$nd,$T3b
401 fmadd $dota,$N3,$nc,$dota
402 fmadd $dotb,$N3,$nd,$dotb
413 stfd $T0a,`$FRAME+0`($sp)
414 stfd $T0b,`$FRAME+8`($sp)
415 stfd $T1a,`$FRAME+16`($sp)
416 stfd $T1b,`$FRAME+24`($sp)
417 stfd $T2a,`$FRAME+32`($sp)
418 stfd $T2b,`$FRAME+40`($sp)
419 stfd $T3a,`$FRAME+48`($sp)
420 stfd $T3b,`$FRAME+56`($sp)
425 $code.=<<___ if ($SIZE_T==8);
426 lwz $t0,4($ap) ; load a[j] as 32-bit word pair
428 lwz $t2,12($ap) ; load a[j+1] as 32-bit word pair
430 lwz $t4,4($np) ; load n[j] as 32-bit word pair
432 lwz $t6,12($np) ; load n[j+1] as 32-bit word pair
435 $code.=<<___ if ($SIZE_T==4);
436 lwz $t0,0($ap) ; load a[j..j+3] as 32-bit word pairs
440 lwz $t4,0($np) ; load n[j..j+3] as 32-bit word pairs
446 std $t0,`$FRAME+64`($sp)
447 std $t1,`$FRAME+72`($sp)
448 std $t2,`$FRAME+80`($sp)
449 std $t3,`$FRAME+88`($sp)
450 std $t4,`$FRAME+96`($sp)
451 std $t5,`$FRAME+104`($sp)
452 std $t6,`$FRAME+112`($sp)
453 std $t7,`$FRAME+120`($sp)
454 ld $t0,`$FRAME+0`($sp)
455 ld $t1,`$FRAME+8`($sp)
456 ld $t2,`$FRAME+16`($sp)
457 ld $t3,`$FRAME+24`($sp)
458 ld $t4,`$FRAME+32`($sp)
459 ld $t5,`$FRAME+40`($sp)
460 ld $t6,`$FRAME+48`($sp)
461 ld $t7,`$FRAME+56`($sp)
462 lfd $A0,`$FRAME+64`($sp)
463 lfd $A1,`$FRAME+72`($sp)
464 lfd $A2,`$FRAME+80`($sp)
465 lfd $A3,`$FRAME+88`($sp)
466 lfd $N0,`$FRAME+96`($sp)
467 lfd $N1,`$FRAME+104`($sp)
468 lfd $N2,`$FRAME+112`($sp)
469 lfd $N3,`$FRAME+120`($sp)
485 stfd $A0,8($nap_d) ; save a[j] in double format
489 fmadd $T0a,$A0,$ba,$dota
490 fmadd $T0b,$A0,$bb,$dotb
491 stfd $A2,24($nap_d) ; save a[j+1] in double format
494 fmadd $T1a,$A0,$bc,$T1a
495 fmadd $T1b,$A0,$bd,$T1b
496 fmadd $T2a,$A1,$bc,$T2a
497 fmadd $T2b,$A1,$bd,$T2b
498 stfd $N0,40($nap_d) ; save n[j] in double format
500 fmadd $T3a,$A2,$bc,$T3a
501 fmadd $T3b,$A2,$bd,$T3b
502 add $t0,$t0,$carry ; can not overflow
505 stfd $N2,56($nap_d) ; save n[j+1] in double format
511 fmadd $T1a,$N1,$na,$T1a
512 fmadd $T1b,$N1,$nb,$T1b
514 fmadd $T2a,$N2,$na,$T2a
515 fmadd $T2b,$N2,$nb,$T2b
517 fmadd $T3a,$N3,$na,$T3a
518 fmadd $T3b,$N3,$nb,$T3b
520 fmadd $T0a,$N0,$na,$T0a
521 fmadd $T0b,$N0,$nb,$T0b
526 fmadd $T1a,$N0,$nc,$T1a
527 fmadd $T1b,$N0,$nd,$T1b
528 insrdi $t0,$t3,16,0 ; 0..63 bits
529 fmadd $T2a,$N1,$nc,$T2a
530 fmadd $T2b,$N1,$nd,$T2b
532 fmadd $T3a,$N2,$nc,$T3a
533 fmadd $T3b,$N2,$nd,$T3b
535 fmadd $dota,$N3,$nc,$dota
536 fmadd $dotb,$N3,$nd,$dotb
553 insrdi $t4,$t7,16,0 ; 64..127 bits
554 srdi $carry,$t7,16 ; upper 33 bits
556 stfd $T0a,`$FRAME+0`($sp)
557 stfd $T0b,`$FRAME+8`($sp)
558 stfd $T1a,`$FRAME+16`($sp)
559 stfd $T1b,`$FRAME+24`($sp)
560 stfd $T2a,`$FRAME+32`($sp)
561 stfd $T2b,`$FRAME+40`($sp)
562 stfd $T3a,`$FRAME+48`($sp)
563 stfd $T3b,`$FRAME+56`($sp)
564 std $t0,8($tp) ; tp[j-1]
565 stdu $t4,16($tp) ; tp[j]
571 ld $t0,`$FRAME+0`($sp)
572 ld $t1,`$FRAME+8`($sp)
573 ld $t2,`$FRAME+16`($sp)
574 ld $t3,`$FRAME+24`($sp)
575 ld $t4,`$FRAME+32`($sp)
576 ld $t5,`$FRAME+40`($sp)
577 ld $t6,`$FRAME+48`($sp)
578 ld $t7,`$FRAME+56`($sp)
579 stfd $dota,`$FRAME+64`($sp)
580 stfd $dotb,`$FRAME+72`($sp)
582 add $t0,$t0,$carry ; can not overflow
592 insrdi $t0,$t3,16,0 ; 0..63 bits
602 insrdi $t4,$t7,16,0 ; 64..127 bits
603 srdi $carry,$t7,16 ; upper 33 bits
604 ld $t6,`$FRAME+64`($sp)
605 ld $t7,`$FRAME+72`($sp)
607 std $t0,8($tp) ; tp[j-1]
608 stdu $t4,16($tp) ; tp[j]
610 add $t6,$t6,$carry ; can not overflow
615 std $t6,8($tp) ; tp[num-1]
618 subf $nap_d,$t7,$nap_d ; rewind pointer
624 $code.=<<___ if ($SIZE_T==8);
625 ldx $t3,$bp,$i ; bp[i]
627 $code.=<<___ if ($SIZE_T==4);
629 lwz $t3,0($t0) ; bp[i,i+1]
634 ld $t6,`$FRAME+$TRANSFER+8`($sp) ; tp[0]
635 mulld $t7,$a0,$t3 ; ap[0]*bp[i]
637 addi $tp,$sp,`$FRAME+$TRANSFER`
638 add $t7,$t7,$t6 ; ap[0]*bp[i]+tp[0]
640 mulld $t7,$t7,$n0 ; tp[0]*n0
643 ; transfer bp[i] to FPU as 4x16-bit values
648 std $t0,`$FRAME+0`($sp)
649 std $t1,`$FRAME+8`($sp)
650 std $t2,`$FRAME+16`($sp)
651 std $t3,`$FRAME+24`($sp)
652 ; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values
657 std $t4,`$FRAME+32`($sp)
658 std $t5,`$FRAME+40`($sp)
659 std $t6,`$FRAME+48`($sp)
660 std $t7,`$FRAME+56`($sp)
662 lfd $A0,8($nap_d) ; load a[j] in double format
664 lfd $A2,24($nap_d) ; load a[j+1] in double format
666 lfd $N0,40($nap_d) ; load n[j] in double format
668 lfd $N2,56($nap_d) ; load n[j+1] in double format
671 lfd $ba,`$FRAME+0`($sp)
672 lfd $bb,`$FRAME+8`($sp)
673 lfd $bc,`$FRAME+16`($sp)
674 lfd $bd,`$FRAME+24`($sp)
675 lfd $na,`$FRAME+32`($sp)
676 lfd $nb,`$FRAME+40`($sp)
677 lfd $nc,`$FRAME+48`($sp)
678 lfd $nd,`$FRAME+56`($sp)
698 fmadd $T1a,$A0,$bc,$T1a
699 fmadd $T1b,$A0,$bd,$T1b
700 fmadd $T2a,$A1,$bc,$T2a
701 fmadd $T2b,$A1,$bd,$T2b
702 fmadd $T3a,$A2,$bc,$T3a
703 fmadd $T3b,$A2,$bd,$T3b
707 fmadd $T1a,$N1,$na,$T1a
708 fmadd $T1b,$N1,$nb,$T1b
709 lfd $A0,8($nap_d) ; load a[j] in double format
711 fmadd $T2a,$N2,$na,$T2a
712 fmadd $T2b,$N2,$nb,$T2b
713 lfd $A2,24($nap_d) ; load a[j+1] in double format
715 fmadd $T3a,$N3,$na,$T3a
716 fmadd $T3b,$N3,$nb,$T3b
717 fmadd $T0a,$N0,$na,$T0a
718 fmadd $T0b,$N0,$nb,$T0b
720 fmadd $T1a,$N0,$nc,$T1a
721 fmadd $T1b,$N0,$nd,$T1b
722 fmadd $T2a,$N1,$nc,$T2a
723 fmadd $T2b,$N1,$nd,$T2b
724 fmadd $T3a,$N2,$nc,$T3a
725 fmadd $T3b,$N2,$nd,$T3b
726 fmadd $dota,$N3,$nc,$dota
727 fmadd $dotb,$N3,$nd,$dotb
738 stfd $T0a,`$FRAME+0`($sp)
739 stfd $T0b,`$FRAME+8`($sp)
740 stfd $T1a,`$FRAME+16`($sp)
741 stfd $T1b,`$FRAME+24`($sp)
742 stfd $T2a,`$FRAME+32`($sp)
743 stfd $T2b,`$FRAME+40`($sp)
744 stfd $T3a,`$FRAME+48`($sp)
745 stfd $T3b,`$FRAME+56`($sp)
753 lfd $N0,40($nap_d) ; load n[j] in double format
757 fmadd $T0a,$A0,$ba,$dota
758 fmadd $T0b,$A0,$bb,$dotb
759 lfd $N2,56($nap_d) ; load n[j+1] in double format
762 fmadd $T1a,$A0,$bc,$T1a
763 fmadd $T1b,$A0,$bd,$T1b
764 fmadd $T2a,$A1,$bc,$T2a
765 fmadd $T2b,$A1,$bd,$T2b
766 lfd $A0,8($nap_d) ; load a[j] in double format
768 fmadd $T3a,$A2,$bc,$T3a
769 fmadd $T3b,$A2,$bd,$T3b
772 lfd $A2,24($nap_d) ; load a[j+1] in double format
775 fmadd $T1a,$N1,$na,$T1a
776 fmadd $T1b,$N1,$nb,$T1b
777 ld $t0,`$FRAME+0`($sp)
778 ld $t1,`$FRAME+8`($sp)
779 fmadd $T2a,$N2,$na,$T2a
780 fmadd $T2b,$N2,$nb,$T2b
781 ld $t2,`$FRAME+16`($sp)
782 ld $t3,`$FRAME+24`($sp)
783 fmadd $T3a,$N3,$na,$T3a
784 fmadd $T3b,$N3,$nb,$T3b
785 add $t0,$t0,$carry ; can not overflow
786 ld $t4,`$FRAME+32`($sp)
787 ld $t5,`$FRAME+40`($sp)
788 fmadd $T0a,$N0,$na,$T0a
789 fmadd $T0b,$N0,$nb,$T0b
793 ld $t6,`$FRAME+48`($sp)
794 ld $t7,`$FRAME+56`($sp)
796 fmadd $T1a,$N0,$nc,$T1a
797 fmadd $T1b,$N0,$nd,$T1b
799 ld $t1,8($tp) ; tp[j]
800 fmadd $T2a,$N1,$nc,$T2a
801 fmadd $T2b,$N1,$nd,$T2b
803 fmadd $T3a,$N2,$nc,$T3a
804 fmadd $T3b,$N2,$nd,$T3b
807 fmadd $dota,$N3,$nc,$dota
808 fmadd $dotb,$N3,$nd,$dotb
810 ldu $t2,16($tp) ; tp[j+1]
812 insrdi $t0,$t3,16,0 ; 0..63 bits
831 stfd $T0a,`$FRAME+0`($sp)
832 stfd $T0b,`$FRAME+8`($sp)
836 $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
842 stfd $T1a,`$FRAME+16`($sp)
843 stfd $T1b,`$FRAME+24`($sp)
844 insrdi $t4,$t7,16,0 ; 64..127 bits
845 srdi $carry,$t7,16 ; upper 33 bits
846 stfd $T2a,`$FRAME+32`($sp)
847 stfd $T2b,`$FRAME+40`($sp)
850 $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
856 stfd $T3a,`$FRAME+48`($sp)
857 stfd $T3b,`$FRAME+56`($sp)
859 std $t3,-16($tp) ; tp[j-1]
860 std $t5,-8($tp) ; tp[j]
865 ld $t0,`$FRAME+0`($sp)
866 ld $t1,`$FRAME+8`($sp)
867 ld $t2,`$FRAME+16`($sp)
868 ld $t3,`$FRAME+24`($sp)
869 ld $t4,`$FRAME+32`($sp)
870 ld $t5,`$FRAME+40`($sp)
871 ld $t6,`$FRAME+48`($sp)
872 ld $t7,`$FRAME+56`($sp)
873 stfd $dota,`$FRAME+64`($sp)
874 stfd $dotb,`$FRAME+72`($sp)
876 add $t0,$t0,$carry ; can not overflow
882 ld $t1,8($tp) ; tp[j]
886 ldu $t2,16($tp) ; tp[j+1]
888 insrdi $t0,$t3,16,0 ; 0..63 bits
898 insrdi $t4,$t7,16,0 ; 64..127 bits
899 srdi $carry,$t7,16 ; upper 33 bits
900 ld $t6,`$FRAME+64`($sp)
901 ld $t7,`$FRAME+72`($sp)
905 $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
913 $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
921 std $t3,-16($tp) ; tp[j-1]
922 std $t5,-8($tp) ; tp[j]
924 add $carry,$carry,$ovf ; comsume upmost overflow
925 add $t6,$t6,$carry ; can not overflow
930 std $t6,0($tp) ; tp[num-1]
934 subf $nap_d,$t7,$nap_d ; rewind pointer
939 $code.=<<___ if ($SIZE_T==8);
940 subf $np,$num,$np ; rewind np
941 addi $j,$j,1 ; restore counter
942 subfc $i,$i,$i ; j=0 and "clear" XER[CA]
943 addi $tp,$sp,`$FRAME+$TRANSFER+8`
944 addi $t4,$sp,`$FRAME+$TRANSFER+16`
954 subfe $t0,$t1,$t0 ; tp[j]-np[j]
955 subfe $t2,$t3,$t2 ; tp[j+1]-np[j+1]
962 subfe $ovf,$i,$ovf ; handle upmost overflow bit
965 or $ap,$ap,$np ; ap=borrow?tp:rp
970 Lcopy: ; copy or in-place refresh
973 std $i,8($nap_d) ; zap nap_d
983 stdx $i,$tp,$i ; zap tp at once
988 $code.=<<___ if ($SIZE_T==4);
989 subf $np,$num,$np ; rewind np
990 addi $j,$j,1 ; restore counter
991 subfc $i,$i,$i ; j=0 and "clear" XER[CA]
992 addi $tp,$sp,`$FRAME+$TRANSFER`
995 addi $ap,$sp,`$FRAME+$TRANSFER+4`
999 Lsub: ld $t0,8($tp) ; load tp[j..j+3] in 64-bit word order
1001 lwz $t4,4($np) ; load np[j..j+3] in 32-bit word order
1007 subfe $t4,$t4,$t0 ; tp[j]-np[j]
1008 stw $t0,4($ap) ; save tp[j..j+3] in 32-bit word order
1009 subfe $t5,$t5,$t1 ; tp[j+1]-np[j+1]
1011 subfe $t6,$t6,$t2 ; tp[j+2]-np[j+2]
1013 subfe $t7,$t7,$t3 ; tp[j+3]-np[j+3]
1022 subfe $ovf,$i,$ovf ; handle upmost overflow bit
1023 addi $tp,$sp,`$FRAME+$TRANSFER+4`
1024 subf $rp,$num,$rp ; rewind rp
1027 or $ap,$ap,$np ; ap=borrow?tp:rp
1028 addi $tp,$sp,`$FRAME+$TRANSFER`
1032 Lcopy: ; copy or in-place refresh
1037 std $i,8($nap_d) ; zap nap_d
1049 std $i,8($tp) ; zap tp at once
1055 $POP r14,`2*$SIZE_T`($sp)
1056 $POP r15,`3*$SIZE_T`($sp)
1057 $POP r16,`4*$SIZE_T`($sp)
1058 $POP r17,`5*$SIZE_T`($sp)
1059 $POP r18,`6*$SIZE_T`($sp)
1060 $POP r19,`7*$SIZE_T`($sp)
1061 $POP r20,`8*$SIZE_T`($sp)
1062 $POP r21,`9*$SIZE_T`($sp)
1063 $POP r22,`10*$SIZE_T`($sp)
1064 $POP r23,`11*$SIZE_T`($sp)
1065 lfd f14,`12*$SIZE_T+0`($sp)
1066 lfd f15,`12*$SIZE_T+8`($sp)
1067 lfd f16,`12*$SIZE_T+16`($sp)
1068 lfd f17,`12*$SIZE_T+24`($sp)
1069 lfd f18,`12*$SIZE_T+32`($sp)
1070 lfd f19,`12*$SIZE_T+40`($sp)
1071 lfd f20,`12*$SIZE_T+48`($sp)
1072 lfd f21,`12*$SIZE_T+56`($sp)
1073 lfd f22,`12*$SIZE_T+64`($sp)
1074 lfd f23,`12*$SIZE_T+72`($sp)
1075 lfd f24,`12*$SIZE_T+80`($sp)
1076 lfd f25,`12*$SIZE_T+88`($sp)
1078 li r3,1 ; signal "handled"
1081 .asciz "Montgomery Multiplication for PPC64, CRYPTOGAMS by <appro\@fy.chalmers.se>"
1084 $code =~ s/\`([^\`]*)\`/eval $1/gem;