3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
12 # The reason for undertaken effort is basically following. Even though
13 # Power 6 CPU operates at incredible 4.7GHz clock frequency, its PKI
14 # performance was observed to be less than impressive, essentially as
15 # fast as 1.8GHz PPC970, or 2.6 times(!) slower than one would hope.
16 # Well, it's not surprising that IBM had to make some sacrifices to
17 # boost the clock frequency that much, but no overall improvement?
18 # Having observed how much difference did switching to FPU make on
19 # UltraSPARC, playing same stunt on Power 6 appeared appropriate...
20 # Unfortunately the resulting performance improvement is not as
21 # impressive, ~30%, and in absolute terms is still very far from what
22 # one would expect from 4.7GHz CPU. There is a chance that I'm doing
23 # something wrong, but in the lack of assembler level micro-profiling
24 # data or at least decent platform guide I can't tell... Or better
25 # results might be achieved with VMX... Anyway, this module provides
26 # *worse* performance on other PowerPC implementations, ~40-15% slower
27 # on PPC970 depending on key length and ~40% slower on Power 5 for all
28 # key lengths. As it's obviously inappropriate as "best all-round"
29 # alternative, it has to be complemented with run-time CPU family
30 # detection. Oh! It should also be noted that unlike other PowerPC
31 # implementation IALU ppc-mont.pl module performs *suboptimaly* on
32 # >=1024-bit key lengths on Power 6. It should also be noted that
33 # *everything* said so far applies to 64-bit builds! As far as 32-bit
34 # application executed on 64-bit CPU goes, this module is likely to
35 # become preferred choice, because it's easy to adapt it for such
36 # case and *is* faster than 32-bit ppc-mont.pl on *all* processors.
40 # Micro-profiling assisted optimization results in ~15% improvement
41 # over original ppc64-mont.pl version, or overall ~50% improvement
42 # over ppc.pl module on Power 6. If compared to ppc-mont.pl on same
43 # Power 6 CPU, this module is 5-150% faster depending on key length,
44 # [hereafter] more for longer keys. But if compared to ppc-mont.pl
45 # on 1.8GHz PPC970, it's only 5-55% faster. Still far from impressive
46 # in absolute terms, but it's apparently the way Power 6 is...
50 # Adapted for 32-bit build this module delivers 25-120%, yes, more
51 # than *twice* for longer keys, performance improvement over 32-bit
52 # ppc-mont.pl on 1.8GHz PPC970. However! This implementation utilizes
53 # even 64-bit integer operations and the trouble is that most PPC
54 # operating systems don't preserve upper halves of general purpose
55 # registers upon 32-bit signal delivery. They do preserve them upon
56 # context switch, but not signalling:-( This means that asynchronous
57 # signals have to be blocked upon entry to this subroutine. Signal
58 # masking (and of course complementary unmasking) has quite an impact
59 # on performance, naturally larger for shorter keys. It's so severe
60 # that 512-bit key performance can be as low as 1/3 of expected one.
61 # This is why this routine can be engaged for longer key operations
62 # only on these OSes, see crypto/ppccap.c for further details. MacOS X
63 # is an exception from this and doesn't require signal masking, and
64 # that's where above improvement coefficients were collected. For
65 # others alternative would be to break dependence on upper halves of
66 # GPRs by sticking to 32-bit integer operations...
70 # Remove above mentioned dependence on GPRs' upper halves in 32-bit
71 # build. No signal masking overhead, but integer instructions are
72 # *more* numerous... It's still "universally" faster than 32-bit
73 # ppc-mont.pl, but improvement coefficient is not as impressive
78 if ($flavour =~ /32/) {
81 $fname= "bn_mul_mont_fpu64";
83 $STUX= "stwux"; # store indexed and update
86 } elsif ($flavour =~ /64/) {
89 $fname= "bn_mul_mont_fpu64";
91 # same as above, but 64-bit mnemonics...
92 $STUX= "stdux"; # store indexed and update
95 } else { die "nonsense $flavour"; }
97 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 4 : 0;
99 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
100 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
101 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
102 die "can't locate ppc-xlate.pl";
104 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
106 $FRAME=64; # padded frame header
118 $rp="r9"; # $rp is reassigned
122 # non-volatile registers
126 $nap_d="r22"; # interleaved ap and np in double format
128 $t0="r24"; # temporary registers
137 # PPC offers enough register bank capacity to unroll inner loops twice
161 $ba="f0"; $bb="f1"; $bc="f2"; $bd="f3";
162 $na="f4"; $nb="f5"; $nc="f6"; $nd="f7";
163 $dota="f8"; $dotb="f9";
164 $A0="f10"; $A1="f11"; $A2="f12"; $A3="f13";
165 $N0="f20"; $N1="f21"; $N2="f22"; $N3="f23";
166 $T0a="f24"; $T0b="f25";
167 $T1a="f26"; $T1b="f27";
168 $T2a="f28"; $T2b="f29";
169 $T3a="f30"; $T3b="f31";
171 # sp----------->+-------------------------------+
173 # +-------------------------------+
175 # +64 +-------------------------------+
176 # | 16 gpr<->fpr transfer zone |
179 # +16*8 +-------------------------------+
180 # | __int64 tmp[-1] |
181 # +-------------------------------+
182 # | __int64 tmp[num] |
186 # +(num+1)*8 +-------------------------------+
187 # | padding to 64 byte boundary |
189 # +X +-------------------------------+
190 # | double nap_d[4*num] |
194 # +-------------------------------+
196 # -13*size_t +-------------------------------+
197 # | 13 saved gpr, r19-r31 |
200 # -12*8 +-------------------------------+
201 # | 12 saved fpr, f20-f31 |
204 # +-------------------------------+
213 cmpwi $num,`3*8/$SIZE_T`
214 mr $rp,r3 ; $rp is reassigned
215 li r3,0 ; possible "not handled" return code
217 andi. r0,$num,`16/$SIZE_T-1` ; $num has to be "even"
220 slwi $num,$num,`log($SIZE_T)/log(2)` ; num*=sizeof(BN_LONG)
222 slwi $tp,$num,2 ; place for {an}p_{lh}[num], i.e. 4*num
223 add $tp,$tp,$num ; place for tp[num+1]
224 addi $tp,$tp,`$FRAME+$TRANSFER+8+64+$RZONE`
225 subf $tp,$tp,$sp ; $sp-$tp
226 and $tp,$tp,$i ; minimize TLB usage
227 subf $tp,$sp,$tp ; $tp-$sp
229 $STUX $sp,$sp,$tp ; alloca
231 $PUSH r19,`-12*8-13*$SIZE_T`($i)
232 $PUSH r20,`-12*8-12*$SIZE_T`($i)
233 $PUSH r21,`-12*8-11*$SIZE_T`($i)
234 $PUSH r22,`-12*8-10*$SIZE_T`($i)
235 $PUSH r23,`-12*8-9*$SIZE_T`($i)
236 $PUSH r24,`-12*8-8*$SIZE_T`($i)
237 $PUSH r25,`-12*8-7*$SIZE_T`($i)
238 $PUSH r26,`-12*8-6*$SIZE_T`($i)
239 $PUSH r27,`-12*8-5*$SIZE_T`($i)
240 $PUSH r28,`-12*8-4*$SIZE_T`($i)
241 $PUSH r29,`-12*8-3*$SIZE_T`($i)
242 $PUSH r30,`-12*8-2*$SIZE_T`($i)
243 $PUSH r31,`-12*8-1*$SIZE_T`($i)
257 addi $tp,$sp,`$FRAME+$TRANSFER+8+64`
260 and $nap_d,$nap_d,$i ; align to 64 bytes
261 ; nap_d is off by 1, because it's used with stfdu/lfdu
262 addi $nap_d,$nap_d,-8
263 srwi $j,$num,`3+1` ; counter register, num/2
265 addi $tp,$sp,`$FRAME+$TRANSFER-8`
270 $code.=<<___ if ($SIZE_T==8);
271 ld $a0,0($ap) ; pull ap[0] value
272 ld $t3,0($bp) ; bp[0]
273 ld $n0,0($n0) ; pull n0[0] value
275 mulld $t7,$a0,$t3 ; ap[0]*bp[0]
276 ; transfer bp[0] to FPU as 4x16-bit values
281 std $t0,`$FRAME+0`($sp)
282 std $t1,`$FRAME+8`($sp)
283 std $t2,`$FRAME+16`($sp)
284 std $t3,`$FRAME+24`($sp)
286 mulld $t7,$t7,$n0 ; tp[0]*n0
287 ; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values
292 std $t4,`$FRAME+32`($sp)
293 std $t5,`$FRAME+40`($sp)
294 std $t6,`$FRAME+48`($sp)
295 std $t7,`$FRAME+56`($sp)
297 extrdi $t0,$a0,32,32 ; lwz $t0,4($ap)
298 extrdi $t1,$a0,32,0 ; lwz $t1,0($ap)
299 lwz $t2,`12^$LITTLE_ENDIAN`($ap) ; load a[1] as 32-bit word pair
300 lwz $t3,`8^$LITTLE_ENDIAN`($ap)
301 lwz $t4,`4^$LITTLE_ENDIAN`($np) ; load n[0] as 32-bit word pair
302 lwz $t5,`0^$LITTLE_ENDIAN`($np)
303 lwz $t6,`12^$LITTLE_ENDIAN`($np) ; load n[1] as 32-bit word pair
304 lwz $t7,`8^$LITTLE_ENDIAN`($np)
306 $code.=<<___ if ($SIZE_T==4);
307 lwz $a0,0($ap) ; pull ap[0,1] value
311 lwz $t1,0($bp) ; bp[0,1]
313 lwz $n0,0($n1) ; pull n0[0,1] value
316 mullw $t4,$a0,$t1 ; mulld ap[0]*bp[0]
322 ; transfer bp[0] to FPU as 4x16-bit values
327 std $t0,`$FRAME+0`($sp) ; yes, std in 32-bit build
328 std $t1,`$FRAME+8`($sp)
329 std $t2,`$FRAME+16`($sp)
330 std $t3,`$FRAME+24`($sp)
332 mullw $t0,$t4,$n0 ; mulld tp[0]*n0
338 ; transfer (ap[0]*bp[0])*n0 to FPU as 4x16-bit values
343 std $t4,`$FRAME+32`($sp) ; yes, std in 32-bit build
344 std $t5,`$FRAME+40`($sp)
345 std $t6,`$FRAME+48`($sp)
346 std $t7,`$FRAME+56`($sp)
348 mr $t0,$a0 ; lwz $t0,0($ap)
349 mr $t1,$a1 ; lwz $t1,4($ap)
350 lwz $t2,8($ap) ; load a[j..j+3] as 32-bit word pairs
352 lwz $t4,0($np) ; load n[j..j+3] as 32-bit word pairs
358 lfd $ba,`$FRAME+0`($sp)
359 lfd $bb,`$FRAME+8`($sp)
360 lfd $bc,`$FRAME+16`($sp)
361 lfd $bd,`$FRAME+24`($sp)
362 lfd $na,`$FRAME+32`($sp)
363 lfd $nb,`$FRAME+40`($sp)
364 lfd $nc,`$FRAME+48`($sp)
365 lfd $nd,`$FRAME+56`($sp)
366 std $t0,`$FRAME+64`($sp) ; yes, std even in 32-bit build
367 std $t1,`$FRAME+72`($sp)
368 std $t2,`$FRAME+80`($sp)
369 std $t3,`$FRAME+88`($sp)
370 std $t4,`$FRAME+96`($sp)
371 std $t5,`$FRAME+104`($sp)
372 std $t6,`$FRAME+112`($sp)
373 std $t7,`$FRAME+120`($sp)
383 lfd $A0,`$FRAME+64`($sp)
384 lfd $A1,`$FRAME+72`($sp)
385 lfd $A2,`$FRAME+80`($sp)
386 lfd $A3,`$FRAME+88`($sp)
387 lfd $N0,`$FRAME+96`($sp)
388 lfd $N1,`$FRAME+104`($sp)
389 lfd $N2,`$FRAME+112`($sp)
390 lfd $N3,`$FRAME+120`($sp)
404 stfd $A0,8($nap_d) ; save a[j] in double format
408 stfd $A2,24($nap_d) ; save a[j+1] in double format
412 stfd $N0,40($nap_d) ; save n[j] in double format
416 stfd $N2,56($nap_d) ; save n[j+1] in double format
419 fmadd $T1a,$A0,$bc,$T1a
420 fmadd $T1b,$A0,$bd,$T1b
421 fmadd $T2a,$A1,$bc,$T2a
422 fmadd $T2b,$A1,$bd,$T2b
423 fmadd $T3a,$A2,$bc,$T3a
424 fmadd $T3b,$A2,$bd,$T3b
428 fmadd $T1a,$N1,$na,$T1a
429 fmadd $T1b,$N1,$nb,$T1b
430 fmadd $T2a,$N2,$na,$T2a
431 fmadd $T2b,$N2,$nb,$T2b
432 fmadd $T3a,$N3,$na,$T3a
433 fmadd $T3b,$N3,$nb,$T3b
434 fmadd $T0a,$N0,$na,$T0a
435 fmadd $T0b,$N0,$nb,$T0b
437 fmadd $T1a,$N0,$nc,$T1a
438 fmadd $T1b,$N0,$nd,$T1b
439 fmadd $T2a,$N1,$nc,$T2a
440 fmadd $T2b,$N1,$nd,$T2b
441 fmadd $T3a,$N2,$nc,$T3a
442 fmadd $T3b,$N2,$nd,$T3b
443 fmadd $dota,$N3,$nc,$dota
444 fmadd $dotb,$N3,$nd,$dotb
455 stfd $T0a,`$FRAME+0`($sp)
456 stfd $T0b,`$FRAME+8`($sp)
457 stfd $T1a,`$FRAME+16`($sp)
458 stfd $T1b,`$FRAME+24`($sp)
459 stfd $T2a,`$FRAME+32`($sp)
460 stfd $T2b,`$FRAME+40`($sp)
461 stfd $T3a,`$FRAME+48`($sp)
462 stfd $T3b,`$FRAME+56`($sp)
467 $code.=<<___ if ($SIZE_T==8);
468 lwz $t0,`4^$LITTLE_ENDIAN`($ap) ; load a[j] as 32-bit word pair
469 lwz $t1,`0^$LITTLE_ENDIAN`($ap)
470 lwz $t2,`12^$LITTLE_ENDIAN`($ap) ; load a[j+1] as 32-bit word pair
471 lwz $t3,`8^$LITTLE_ENDIAN`($ap)
472 lwz $t4,`4^$LITTLE_ENDIAN`($np) ; load n[j] as 32-bit word pair
473 lwz $t5,`0^$LITTLE_ENDIAN`($np)
474 lwz $t6,`12^$LITTLE_ENDIAN`($np) ; load n[j+1] as 32-bit word pair
475 lwz $t7,`8^$LITTLE_ENDIAN`($np)
477 $code.=<<___ if ($SIZE_T==4);
478 lwz $t0,0($ap) ; load a[j..j+3] as 32-bit word pairs
482 lwz $t4,0($np) ; load n[j..j+3] as 32-bit word pairs
488 std $t0,`$FRAME+64`($sp) ; yes, std even in 32-bit build
489 std $t1,`$FRAME+72`($sp)
490 std $t2,`$FRAME+80`($sp)
491 std $t3,`$FRAME+88`($sp)
492 std $t4,`$FRAME+96`($sp)
493 std $t5,`$FRAME+104`($sp)
494 std $t6,`$FRAME+112`($sp)
495 std $t7,`$FRAME+120`($sp)
497 if ($SIZE_T==8 or $flavour =~ /osx/) {
499 ld $t0,`$FRAME+0`($sp)
500 ld $t1,`$FRAME+8`($sp)
501 ld $t2,`$FRAME+16`($sp)
502 ld $t3,`$FRAME+24`($sp)
503 ld $t4,`$FRAME+32`($sp)
504 ld $t5,`$FRAME+40`($sp)
505 ld $t6,`$FRAME+48`($sp)
506 ld $t7,`$FRAME+56`($sp)
510 lwz $t1,`$FRAME+0^$LITTLE_ENDIAN`($sp)
511 lwz $t0,`$FRAME+4^$LITTLE_ENDIAN`($sp)
512 lwz $t3,`$FRAME+8^$LITTLE_ENDIAN`($sp)
513 lwz $t2,`$FRAME+12^$LITTLE_ENDIAN`($sp)
514 lwz $t5,`$FRAME+16^$LITTLE_ENDIAN`($sp)
515 lwz $t4,`$FRAME+20^$LITTLE_ENDIAN`($sp)
516 lwz $t7,`$FRAME+24^$LITTLE_ENDIAN`($sp)
517 lwz $t6,`$FRAME+28^$LITTLE_ENDIAN`($sp)
521 lfd $A0,`$FRAME+64`($sp)
522 lfd $A1,`$FRAME+72`($sp)
523 lfd $A2,`$FRAME+80`($sp)
524 lfd $A3,`$FRAME+88`($sp)
525 lfd $N0,`$FRAME+96`($sp)
526 lfd $N1,`$FRAME+104`($sp)
527 lfd $N2,`$FRAME+112`($sp)
528 lfd $N3,`$FRAME+120`($sp)
544 stfd $A0,8($nap_d) ; save a[j] in double format
548 fmadd $T0a,$A0,$ba,$dota
549 fmadd $T0b,$A0,$bb,$dotb
550 stfd $A2,24($nap_d) ; save a[j+1] in double format
553 if ($SIZE_T==8 or $flavour =~ /osx/) {
555 fmadd $T1a,$A0,$bc,$T1a
556 fmadd $T1b,$A0,$bd,$T1b
557 fmadd $T2a,$A1,$bc,$T2a
558 fmadd $T2b,$A1,$bd,$T2b
559 stfd $N0,40($nap_d) ; save n[j] in double format
561 fmadd $T3a,$A2,$bc,$T3a
562 fmadd $T3b,$A2,$bd,$T3b
563 add $t0,$t0,$carry ; can not overflow
566 stfd $N2,56($nap_d) ; save n[j+1] in double format
572 fmadd $T1a,$N1,$na,$T1a
573 fmadd $T1b,$N1,$nb,$T1b
575 fmadd $T2a,$N2,$na,$T2a
576 fmadd $T2b,$N2,$nb,$T2b
578 fmadd $T3a,$N3,$na,$T3a
579 fmadd $T3b,$N3,$nb,$T3b
581 fmadd $T0a,$N0,$na,$T0a
582 fmadd $T0b,$N0,$nb,$T0b
587 fmadd $T1a,$N0,$nc,$T1a
588 fmadd $T1b,$N0,$nd,$T1b
589 insrdi $t0,$t3,16,0 ; 0..63 bits
590 fmadd $T2a,$N1,$nc,$T2a
591 fmadd $T2b,$N1,$nd,$T2b
593 fmadd $T3a,$N2,$nc,$T3a
594 fmadd $T3b,$N2,$nd,$T3b
596 fmadd $dota,$N3,$nc,$dota
597 fmadd $dotb,$N3,$nd,$dotb
614 insrdi $t4,$t7,16,0 ; 64..127 bits
615 srdi $carry,$t7,16 ; upper 33 bits
617 stfd $T0a,`$FRAME+0`($sp)
618 stfd $T0b,`$FRAME+8`($sp)
619 stfd $T1a,`$FRAME+16`($sp)
620 stfd $T1b,`$FRAME+24`($sp)
621 stfd $T2a,`$FRAME+32`($sp)
622 stfd $T2b,`$FRAME+40`($sp)
623 stfd $T3a,`$FRAME+48`($sp)
624 stfd $T3b,`$FRAME+56`($sp)
625 std $t0,8($tp) ; tp[j-1]
626 stdu $t4,16($tp) ; tp[j]
630 fmadd $T1a,$A0,$bc,$T1a
631 fmadd $T1b,$A0,$bd,$T1b
635 fmadd $T2a,$A1,$bc,$T2a
636 fmadd $T2b,$A1,$bd,$T2b
637 stfd $N0,40($nap_d) ; save n[j] in double format
640 insrwi $carry,$t1,16,0
641 fmadd $T3a,$A2,$bc,$T3a
642 fmadd $T3b,$A2,$bd,$T3b
648 stfd $N2,56($nap_d) ; save n[j+1] in double format
650 insrwi $t0,$t2,16,0 ; 0..31 bits
652 insrwi $carry,$t3,16,0
654 fmadd $T1a,$N1,$na,$T1a
655 fmadd $T1b,$N1,$nb,$T1b
656 lwz $t3,`$FRAME+32^$LITTLE_ENDIAN`($sp) ; permuted $t1
657 lwz $t2,`$FRAME+36^$LITTLE_ENDIAN`($sp) ; permuted $t0
661 fmadd $T2a,$N2,$na,$T2a
662 fmadd $T2b,$N2,$nb,$T2b
664 insrwi $carry,$t5,16,0
665 fmadd $T3a,$N3,$na,$T3a
666 fmadd $T3b,$N3,$nb,$T3b
670 fmadd $T0a,$N0,$na,$T0a
671 fmadd $T0b,$N0,$nb,$T0b
672 insrwi $t4,$t6,16,0 ; 32..63 bits
674 insrwi $carry,$t7,16,0
676 fmadd $T1a,$N0,$nc,$T1a
677 fmadd $T1b,$N0,$nd,$T1b
678 lwz $t7,`$FRAME+40^$LITTLE_ENDIAN`($sp) ; permuted $t3
679 lwz $t6,`$FRAME+44^$LITTLE_ENDIAN`($sp) ; permuted $t2
683 fmadd $T2a,$N1,$nc,$T2a
684 fmadd $T2b,$N1,$nd,$T2b
685 stw $t0,12($tp) ; tp[j-1]
688 insrwi $carry,$t3,16,0
689 fmadd $T3a,$N2,$nc,$T3a
690 fmadd $T3b,$N2,$nd,$T3b
691 lwz $t1,`$FRAME+48^$LITTLE_ENDIAN`($sp) ; permuted $t5
692 lwz $t0,`$FRAME+52^$LITTLE_ENDIAN`($sp) ; permuted $t4
696 fmadd $dota,$N3,$nc,$dota
697 fmadd $dotb,$N3,$nd,$dotb
698 insrwi $t2,$t6,16,0 ; 64..95 bits
700 insrwi $carry,$t7,16,0
704 lwz $t5,`$FRAME+56^$LITTLE_ENDIAN`($sp) ; permuted $t7
705 lwz $t4,`$FRAME+60^$LITTLE_ENDIAN`($sp) ; permuted $t6
712 insrwi $carry,$t1,16,0
720 insrwi $t0,$t4,16,0 ; 96..127 bits
722 insrwi $carry,$t5,16,0
724 stfd $T0a,`$FRAME+0`($sp)
725 stfd $T0b,`$FRAME+8`($sp)
726 stfd $T1a,`$FRAME+16`($sp)
727 stfd $T1b,`$FRAME+24`($sp)
728 stfd $T2a,`$FRAME+32`($sp)
729 stfd $T2b,`$FRAME+40`($sp)
730 stfd $T3a,`$FRAME+48`($sp)
731 stfd $T3b,`$FRAME+56`($sp)
732 stw $t2,20($tp) ; tp[j]
742 if ($SIZE_T==8 or $flavour =~ /osx/) {
744 ld $t0,`$FRAME+0`($sp)
745 ld $t1,`$FRAME+8`($sp)
746 ld $t2,`$FRAME+16`($sp)
747 ld $t3,`$FRAME+24`($sp)
748 ld $t4,`$FRAME+32`($sp)
749 ld $t5,`$FRAME+40`($sp)
750 ld $t6,`$FRAME+48`($sp)
751 ld $t7,`$FRAME+56`($sp)
752 stfd $dota,`$FRAME+64`($sp)
753 stfd $dotb,`$FRAME+72`($sp)
755 add $t0,$t0,$carry ; can not overflow
765 insrdi $t0,$t3,16,0 ; 0..63 bits
775 insrdi $t4,$t7,16,0 ; 64..127 bits
776 srdi $carry,$t7,16 ; upper 33 bits
777 ld $t6,`$FRAME+64`($sp)
778 ld $t7,`$FRAME+72`($sp)
780 std $t0,8($tp) ; tp[j-1]
781 stdu $t4,16($tp) ; tp[j]
783 add $t6,$t6,$carry ; can not overflow
788 std $t6,8($tp) ; tp[num-1]
792 lwz $t1,`$FRAME+0^$LITTLE_ENDIAN`($sp)
793 lwz $t0,`$FRAME+4^$LITTLE_ENDIAN`($sp)
794 lwz $t3,`$FRAME+8^$LITTLE_ENDIAN`($sp)
795 lwz $t2,`$FRAME+12^$LITTLE_ENDIAN`($sp)
796 lwz $t5,`$FRAME+16^$LITTLE_ENDIAN`($sp)
797 lwz $t4,`$FRAME+20^$LITTLE_ENDIAN`($sp)
798 lwz $t7,`$FRAME+24^$LITTLE_ENDIAN`($sp)
799 lwz $t6,`$FRAME+28^$LITTLE_ENDIAN`($sp)
800 stfd $dota,`$FRAME+64`($sp)
801 stfd $dotb,`$FRAME+72`($sp)
806 insrwi $carry,$t1,16,0
811 insrwi $t0,$t2,16,0 ; 0..31 bits
812 insrwi $carry,$t3,16,0
817 insrwi $carry,$t5,16,0
822 insrwi $t4,$t6,16,0 ; 32..63 bits
823 insrwi $carry,$t7,16,0
825 stw $t0,12($tp) ; tp[j-1]
828 lwz $t3,`$FRAME+32^$LITTLE_ENDIAN`($sp) ; permuted $t1
829 lwz $t2,`$FRAME+36^$LITTLE_ENDIAN`($sp) ; permuted $t0
830 lwz $t7,`$FRAME+40^$LITTLE_ENDIAN`($sp) ; permuted $t3
831 lwz $t6,`$FRAME+44^$LITTLE_ENDIAN`($sp) ; permuted $t2
832 lwz $t1,`$FRAME+48^$LITTLE_ENDIAN`($sp) ; permuted $t5
833 lwz $t0,`$FRAME+52^$LITTLE_ENDIAN`($sp) ; permuted $t4
834 lwz $t5,`$FRAME+56^$LITTLE_ENDIAN`($sp) ; permuted $t7
835 lwz $t4,`$FRAME+60^$LITTLE_ENDIAN`($sp) ; permuted $t6
840 insrwi $carry,$t3,16,0
845 insrwi $t2,$t6,16,0 ; 64..95 bits
846 insrwi $carry,$t7,16,0
851 insrwi $carry,$t1,16,0
856 insrwi $t0,$t4,16,0 ; 96..127 bits
857 insrwi $carry,$t5,16,0
859 stw $t2,20($tp) ; tp[j]
862 lwz $t7,`$FRAME+64^$LITTLE_ENDIAN`($sp)
863 lwz $t6,`$FRAME+68^$LITTLE_ENDIAN`($sp)
864 lwz $t5,`$FRAME+72^$LITTLE_ENDIAN`($sp)
865 lwz $t4,`$FRAME+76^$LITTLE_ENDIAN`($sp)
870 insrwi $carry,$t7,16,0
879 stw $t6,12($tp) ; tp[num-1]
885 subf $nap_d,$t7,$nap_d ; rewind pointer
890 addi $tp,$sp,`$FRAME+$TRANSFER`
894 $code.=<<___ if ($SIZE_T==8);
895 ldx $t3,$bp,$i ; bp[i]
897 ld $t6,`$FRAME+$TRANSFER+8`($sp) ; tp[0]
898 mulld $t7,$a0,$t3 ; ap[0]*bp[i]
899 add $t7,$t7,$t6 ; ap[0]*bp[i]+tp[0]
900 ; transfer bp[i] to FPU as 4x16-bit values
905 std $t0,`$FRAME+0`($sp)
906 std $t1,`$FRAME+8`($sp)
907 std $t2,`$FRAME+16`($sp)
908 std $t3,`$FRAME+24`($sp)
910 mulld $t7,$t7,$n0 ; tp[0]*n0
911 ; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values
916 std $t4,`$FRAME+32`($sp)
917 std $t5,`$FRAME+40`($sp)
918 std $t6,`$FRAME+48`($sp)
919 std $t7,`$FRAME+56`($sp)
921 $code.=<<___ if ($SIZE_T==4);
924 lwz $t1,0($t0) ; bp[i,i+1]
927 mullw $t4,$a0,$t1 ; ap[0]*bp[i]
928 lwz $t0,`$FRAME+$TRANSFER+8+4`($sp) ; tp[0]
930 lwz $t2,`$FRAME+$TRANSFER+8`($sp) ; tp[0]
935 addc $t4,$t4,$t0 ; ap[0]*bp[i]+tp[0]
937 ; transfer bp[i] to FPU as 4x16-bit values
942 std $t0,`$FRAME+0`($sp) ; yes, std in 32-bit build
943 std $t1,`$FRAME+8`($sp)
944 std $t2,`$FRAME+16`($sp)
945 std $t3,`$FRAME+24`($sp)
947 mullw $t0,$t4,$n0 ; mulld tp[0]*n0
953 ; transfer (ap[0]*bp[i]+tp[0])*n0 to FPU as 4x16-bit values
958 std $t4,`$FRAME+32`($sp) ; yes, std in 32-bit build
959 std $t5,`$FRAME+40`($sp)
960 std $t6,`$FRAME+48`($sp)
961 std $t7,`$FRAME+56`($sp)
964 lfd $A0,8($nap_d) ; load a[j] in double format
966 lfd $A2,24($nap_d) ; load a[j+1] in double format
968 lfd $N0,40($nap_d) ; load n[j] in double format
970 lfd $N2,56($nap_d) ; load n[j+1] in double format
973 lfd $ba,`$FRAME+0`($sp)
974 lfd $bb,`$FRAME+8`($sp)
975 lfd $bc,`$FRAME+16`($sp)
976 lfd $bd,`$FRAME+24`($sp)
977 lfd $na,`$FRAME+32`($sp)
978 lfd $nb,`$FRAME+40`($sp)
979 lfd $nc,`$FRAME+48`($sp)
980 lfd $nd,`$FRAME+56`($sp)
1000 fmadd $T1a,$A0,$bc,$T1a
1001 fmadd $T1b,$A0,$bd,$T1b
1002 fmadd $T2a,$A1,$bc,$T2a
1003 fmadd $T2b,$A1,$bd,$T2b
1004 fmadd $T3a,$A2,$bc,$T3a
1005 fmadd $T3b,$A2,$bd,$T3b
1009 fmadd $T1a,$N1,$na,$T1a
1010 fmadd $T1b,$N1,$nb,$T1b
1011 lfd $A0,8($nap_d) ; load a[j] in double format
1013 fmadd $T2a,$N2,$na,$T2a
1014 fmadd $T2b,$N2,$nb,$T2b
1015 lfd $A2,24($nap_d) ; load a[j+1] in double format
1017 fmadd $T3a,$N3,$na,$T3a
1018 fmadd $T3b,$N3,$nb,$T3b
1019 fmadd $T0a,$N0,$na,$T0a
1020 fmadd $T0b,$N0,$nb,$T0b
1022 fmadd $T1a,$N0,$nc,$T1a
1023 fmadd $T1b,$N0,$nd,$T1b
1024 fmadd $T2a,$N1,$nc,$T2a
1025 fmadd $T2b,$N1,$nd,$T2b
1026 fmadd $T3a,$N2,$nc,$T3a
1027 fmadd $T3b,$N2,$nd,$T3b
1028 fmadd $dota,$N3,$nc,$dota
1029 fmadd $dotb,$N3,$nd,$dotb
1040 stfd $T0a,`$FRAME+0`($sp)
1041 stfd $T0b,`$FRAME+8`($sp)
1042 stfd $T1a,`$FRAME+16`($sp)
1043 stfd $T1b,`$FRAME+24`($sp)
1044 stfd $T2a,`$FRAME+32`($sp)
1045 stfd $T2b,`$FRAME+40`($sp)
1046 stfd $T3a,`$FRAME+48`($sp)
1047 stfd $T3b,`$FRAME+56`($sp)
1055 lfd $N0,40($nap_d) ; load n[j] in double format
1059 fmadd $T0a,$A0,$ba,$dota
1060 fmadd $T0b,$A0,$bb,$dotb
1061 lfd $N2,56($nap_d) ; load n[j+1] in double format
1064 fmadd $T1a,$A0,$bc,$T1a
1065 fmadd $T1b,$A0,$bd,$T1b
1066 fmadd $T2a,$A1,$bc,$T2a
1067 fmadd $T2b,$A1,$bd,$T2b
1068 lfd $A0,8($nap_d) ; load a[j] in double format
1070 fmadd $T3a,$A2,$bc,$T3a
1071 fmadd $T3b,$A2,$bd,$T3b
1074 lfd $A2,24($nap_d) ; load a[j+1] in double format
1077 if ($SIZE_T==8 or $flavour =~ /osx/) {
1079 fmadd $T1a,$N1,$na,$T1a
1080 fmadd $T1b,$N1,$nb,$T1b
1081 ld $t0,`$FRAME+0`($sp)
1082 ld $t1,`$FRAME+8`($sp)
1083 fmadd $T2a,$N2,$na,$T2a
1084 fmadd $T2b,$N2,$nb,$T2b
1085 ld $t2,`$FRAME+16`($sp)
1086 ld $t3,`$FRAME+24`($sp)
1087 fmadd $T3a,$N3,$na,$T3a
1088 fmadd $T3b,$N3,$nb,$T3b
1089 add $t0,$t0,$carry ; can not overflow
1090 ld $t4,`$FRAME+32`($sp)
1091 ld $t5,`$FRAME+40`($sp)
1092 fmadd $T0a,$N0,$na,$T0a
1093 fmadd $T0b,$N0,$nb,$T0b
1097 ld $t6,`$FRAME+48`($sp)
1098 ld $t7,`$FRAME+56`($sp)
1100 fmadd $T1a,$N0,$nc,$T1a
1101 fmadd $T1b,$N0,$nd,$T1b
1102 insrdi $t0,$t1,16,32
1103 ld $t1,8($tp) ; tp[j]
1104 fmadd $T2a,$N1,$nc,$T2a
1105 fmadd $T2b,$N1,$nd,$T2b
1107 fmadd $T3a,$N2,$nc,$T3a
1108 fmadd $T3b,$N2,$nd,$T3b
1110 insrdi $t0,$t2,16,16
1111 fmadd $dota,$N3,$nc,$dota
1112 fmadd $dotb,$N3,$nd,$dotb
1114 ldu $t2,16($tp) ; tp[j+1]
1116 insrdi $t0,$t3,16,0 ; 0..63 bits
1128 insrdi $t4,$t5,16,32
1133 insrdi $t4,$t6,16,16
1135 stfd $T0a,`$FRAME+0`($sp)
1136 stfd $T0b,`$FRAME+8`($sp)
1140 $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
1146 stfd $T1a,`$FRAME+16`($sp)
1147 stfd $T1b,`$FRAME+24`($sp)
1148 insrdi $t4,$t7,16,0 ; 64..127 bits
1149 srdi $carry,$t7,16 ; upper 33 bits
1150 stfd $T2a,`$FRAME+32`($sp)
1151 stfd $T2b,`$FRAME+40`($sp)
1154 $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
1160 stfd $T3a,`$FRAME+48`($sp)
1161 stfd $T3b,`$FRAME+56`($sp)
1163 std $t3,-16($tp) ; tp[j-1]
1164 std $t5,-8($tp) ; tp[j]
1168 fmadd $T1a,$N1,$na,$T1a
1169 fmadd $T1b,$N1,$nb,$T1b
1170 lwz $t1,`$FRAME+0^$LITTLE_ENDIAN`($sp)
1171 lwz $t0,`$FRAME+4^$LITTLE_ENDIAN`($sp)
1172 fmadd $T2a,$N2,$na,$T2a
1173 fmadd $T2b,$N2,$nb,$T2b
1174 lwz $t3,`$FRAME+8^$LITTLE_ENDIAN`($sp)
1175 lwz $t2,`$FRAME+12^$LITTLE_ENDIAN`($sp)
1176 fmadd $T3a,$N3,$na,$T3a
1177 fmadd $T3b,$N3,$nb,$T3b
1178 lwz $t5,`$FRAME+16^$LITTLE_ENDIAN`($sp)
1179 lwz $t4,`$FRAME+20^$LITTLE_ENDIAN`($sp)
1183 fmadd $T0a,$N0,$na,$T0a
1184 fmadd $T0b,$N0,$nb,$T0b
1185 lwz $t7,`$FRAME+24^$LITTLE_ENDIAN`($sp)
1186 lwz $t6,`$FRAME+28^$LITTLE_ENDIAN`($sp)
1188 insrwi $carry,$t1,16,0
1190 fmadd $T1a,$N0,$nc,$T1a
1191 fmadd $T1b,$N0,$nd,$T1b
1195 fmadd $T2a,$N1,$nc,$T2a
1196 fmadd $T2b,$N1,$nd,$T2b
1197 insrwi $t0,$t2,16,0 ; 0..31 bits
1199 insrwi $carry,$t3,16,0
1200 fmadd $T3a,$N2,$nc,$T3a
1201 fmadd $T3b,$N2,$nd,$T3b
1202 lwz $t2,12($tp) ; tp[j]
1207 fmadd $dota,$N3,$nc,$dota
1208 fmadd $dotb,$N3,$nd,$dotb
1210 insrwi $carry,$t5,16,0
1217 insrwi $t4,$t6,16,0 ; 32..63 bits
1219 insrwi $carry,$t7,16,0
1223 lwz $t3,`$FRAME+32^$LITTLE_ENDIAN`($sp) ; permuted $t1
1224 lwz $t2,`$FRAME+36^$LITTLE_ENDIAN`($sp) ; permuted $t0
1228 stw $t0,4($tp) ; tp[j-1]
1234 lwz $t7,`$FRAME+40^$LITTLE_ENDIAN`($sp) ; permuted $t3
1235 lwz $t6,`$FRAME+44^$LITTLE_ENDIAN`($sp) ; permuted $t2
1238 insrwi $carry,$t3,16,0
1239 lwz $t1,`$FRAME+48^$LITTLE_ENDIAN`($sp) ; permuted $t5
1240 lwz $t0,`$FRAME+52^$LITTLE_ENDIAN`($sp) ; permuted $t4
1245 lwz $t5,`$FRAME+56^$LITTLE_ENDIAN`($sp) ; permuted $t7
1246 lwz $t4,`$FRAME+60^$LITTLE_ENDIAN`($sp) ; permuted $t6
1249 insrwi $t2,$t6,16,0 ; 64..95 bits
1250 insrwi $carry,$t7,16,0
1255 stfd $T0a,`$FRAME+0`($sp)
1258 stfd $T0b,`$FRAME+8`($sp)
1259 insrwi $carry,$t1,16,0
1262 stfd $T1a,`$FRAME+16`($sp)
1265 insrwi $t0,$t4,16,0 ; 96..127 bits
1266 stfd $T1b,`$FRAME+24`($sp)
1267 insrwi $carry,$t5,16,0
1271 stfd $T2a,`$FRAME+32`($sp)
1273 stfd $T2b,`$FRAME+40`($sp)
1275 stfd $T3a,`$FRAME+48`($sp)
1277 stfd $T3b,`$FRAME+56`($sp)
1278 stw $t2,-4($tp) ; tp[j]
1288 if ($SIZE_T==8 or $flavour =~ /osx/) {
1290 ld $t0,`$FRAME+0`($sp)
1291 ld $t1,`$FRAME+8`($sp)
1292 ld $t2,`$FRAME+16`($sp)
1293 ld $t3,`$FRAME+24`($sp)
1294 ld $t4,`$FRAME+32`($sp)
1295 ld $t5,`$FRAME+40`($sp)
1296 ld $t6,`$FRAME+48`($sp)
1297 ld $t7,`$FRAME+56`($sp)
1298 stfd $dota,`$FRAME+64`($sp)
1299 stfd $dotb,`$FRAME+72`($sp)
1301 add $t0,$t0,$carry ; can not overflow
1305 insrdi $t0,$t1,16,32
1307 ld $t1,8($tp) ; tp[j]
1309 insrdi $t0,$t2,16,16
1311 ldu $t2,16($tp) ; tp[j+1]
1313 insrdi $t0,$t3,16,0 ; 0..63 bits
1318 insrdi $t4,$t5,16,32
1321 insrdi $t4,$t6,16,16
1323 insrdi $t4,$t7,16,0 ; 64..127 bits
1324 srdi $carry,$t7,16 ; upper 33 bits
1325 ld $t6,`$FRAME+64`($sp)
1326 ld $t7,`$FRAME+72`($sp)
1330 $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
1338 $code.=<<___ if ($SIZE_T==4); # adjust XER[CA]
1346 std $t3,-16($tp) ; tp[j-1]
1347 std $t5,-8($tp) ; tp[j]
1349 add $carry,$carry,$ovf ; comsume upmost overflow
1350 add $t6,$t6,$carry ; can not overflow
1355 std $t6,0($tp) ; tp[num-1]
1359 lwz $t1,`$FRAME+0^$LITTLE_ENDIAN`($sp)
1360 lwz $t0,`$FRAME+4^$LITTLE_ENDIAN`($sp)
1361 lwz $t3,`$FRAME+8^$LITTLE_ENDIAN`($sp)
1362 lwz $t2,`$FRAME+12^$LITTLE_ENDIAN`($sp)
1363 lwz $t5,`$FRAME+16^$LITTLE_ENDIAN`($sp)
1364 lwz $t4,`$FRAME+20^$LITTLE_ENDIAN`($sp)
1365 lwz $t7,`$FRAME+24^$LITTLE_ENDIAN`($sp)
1366 lwz $t6,`$FRAME+28^$LITTLE_ENDIAN`($sp)
1367 stfd $dota,`$FRAME+64`($sp)
1368 stfd $dotb,`$FRAME+72`($sp)
1373 insrwi $carry,$t1,16,0
1378 insrwi $t0,$t2,16,0 ; 0..31 bits
1379 lwz $t2,12($tp) ; tp[j]
1380 insrwi $carry,$t3,16,0
1386 insrwi $carry,$t5,16,0
1391 insrwi $t4,$t6,16,0 ; 32..63 bits
1392 insrwi $carry,$t7,16,0
1399 stw $t0,4($tp) ; tp[j-1]
1402 lwz $t3,`$FRAME+32^$LITTLE_ENDIAN`($sp) ; permuted $t1
1403 lwz $t2,`$FRAME+36^$LITTLE_ENDIAN`($sp) ; permuted $t0
1404 lwz $t7,`$FRAME+40^$LITTLE_ENDIAN`($sp) ; permuted $t3
1405 lwz $t6,`$FRAME+44^$LITTLE_ENDIAN`($sp) ; permuted $t2
1406 lwz $t1,`$FRAME+48^$LITTLE_ENDIAN`($sp) ; permuted $t5
1407 lwz $t0,`$FRAME+52^$LITTLE_ENDIAN`($sp) ; permuted $t4
1408 lwz $t5,`$FRAME+56^$LITTLE_ENDIAN`($sp) ; permuted $t7
1409 lwz $t4,`$FRAME+60^$LITTLE_ENDIAN`($sp) ; permuted $t6
1414 insrwi $carry,$t3,16,0
1419 insrwi $t2,$t6,16,0 ; 64..95 bits
1421 insrwi $carry,$t7,16,0
1427 insrwi $carry,$t1,16,0
1432 insrwi $t0,$t4,16,0 ; 96..127 bits
1433 insrwi $carry,$t5,16,0
1438 lwz $t7,`$FRAME+64^$LITTLE_ENDIAN`($sp)
1439 lwz $t6,`$FRAME+68^$LITTLE_ENDIAN`($sp)
1442 lwz $t5,`$FRAME+72^$LITTLE_ENDIAN`($sp)
1443 lwz $t4,`$FRAME+76^$LITTLE_ENDIAN`($sp)
1447 stw $t2,-4($tp) ; tp[j]
1452 insrwi $carry,$t7,16,0
1461 stw $t6,4($tp) ; tp[num-1]
1468 subf $nap_d,$t7,$nap_d ; rewind pointer
1473 $code.=<<___ if ($SIZE_T==8);
1474 subf $np,$num,$np ; rewind np
1475 addi $j,$j,1 ; restore counter
1476 subfc $i,$i,$i ; j=0 and "clear" XER[CA]
1477 addi $tp,$sp,`$FRAME+$TRANSFER+8`
1478 addi $t4,$sp,`$FRAME+$TRANSFER+16`
1484 Lsub: ldx $t0,$tp,$i
1488 subfe $t0,$t1,$t0 ; tp[j]-np[j]
1489 subfe $t2,$t3,$t2 ; tp[j+1]-np[j+1]
1496 subfe $ovf,$i,$ovf ; handle upmost overflow bit
1499 or $ap,$ap,$np ; ap=borrow?tp:rp
1504 Lcopy: ; copy or in-place refresh
1507 std $i,8($nap_d) ; zap nap_d
1517 stdx $i,$tp,$i ; zap tp at once
1522 $code.=<<___ if ($SIZE_T==4);
1523 subf $np,$num,$np ; rewind np
1524 addi $j,$j,1 ; restore counter
1525 subfc $i,$i,$i ; j=0 and "clear" XER[CA]
1526 addi $tp,$sp,`$FRAME+$TRANSFER`
1529 addi $ap,$sp,`$FRAME+$TRANSFER+4`
1533 Lsub: lwz $t0,12($tp) ; load tp[j..j+3] in 64-bit word order
1537 lwz $t4,4($np) ; load np[j..j+3] in 32-bit word order
1541 subfe $t4,$t4,$t0 ; tp[j]-np[j]
1542 stw $t0,4($ap) ; save tp[j..j+3] in 32-bit word order
1543 subfe $t5,$t5,$t1 ; tp[j+1]-np[j+1]
1545 subfe $t6,$t6,$t2 ; tp[j+2]-np[j+2]
1547 subfe $t7,$t7,$t3 ; tp[j+3]-np[j+3]
1556 subfe $ovf,$i,$ovf ; handle upmost overflow bit
1557 addi $tp,$sp,`$FRAME+$TRANSFER+4`
1558 subf $rp,$num,$rp ; rewind rp
1561 or $ap,$ap,$np ; ap=borrow?tp:rp
1562 addi $tp,$sp,`$FRAME+$TRANSFER`
1566 Lcopy: ; copy or in-place refresh
1571 std $i,8($nap_d) ; zap nap_d
1583 std $i,8($tp) ; zap tp at once
1590 li r3,1 ; signal "handled"
1591 $POP r19,`-12*8-13*$SIZE_T`($i)
1592 $POP r20,`-12*8-12*$SIZE_T`($i)
1593 $POP r21,`-12*8-11*$SIZE_T`($i)
1594 $POP r22,`-12*8-10*$SIZE_T`($i)
1595 $POP r23,`-12*8-9*$SIZE_T`($i)
1596 $POP r24,`-12*8-8*$SIZE_T`($i)
1597 $POP r25,`-12*8-7*$SIZE_T`($i)
1598 $POP r26,`-12*8-6*$SIZE_T`($i)
1599 $POP r27,`-12*8-5*$SIZE_T`($i)
1600 $POP r28,`-12*8-4*$SIZE_T`($i)
1601 $POP r29,`-12*8-3*$SIZE_T`($i)
1602 $POP r30,`-12*8-2*$SIZE_T`($i)
1603 $POP r31,`-12*8-1*$SIZE_T`($i)
1619 .byte 0,12,4,0,0x8c,13,6,0
1621 .size .$fname,.-.$fname
1623 .asciz "Montgomery Multiplication for PPC64, CRYPTOGAMS by <appro\@openssl.org>"
1626 $code =~ s/\`([^\`]*)\`/eval $1/gem;