3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
10 # This module implements Poly1305 hash for PowerPC FPU.
14 # Numbers are cycles per processed byte with poly1305_blocks alone,
15 # and improvement coefficients relative to gcc-generated code.
17 # Freescale e300 9.78/+30%
25 if ($flavour =~ /64/) {
32 } elsif ($flavour =~ /32/) {
39 } else { die "nonsense $flavour"; }
41 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 4 : 0;
43 $LWXLE = $LITTLE_ENDIAN ? "lwzx" : "lwbrx";
45 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
46 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
47 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
48 die "can't locate ppc-xlate.pl";
50 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
53 $FRAME=$LOCALS+6*8+18*8;
57 my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
58 my ($in0,$in1,$in2,$in3,$i1,$i2,$i3) = map("r$_",(7..12,6));
60 my ($h0lo,$h0hi,$h1lo,$h1hi,$h2lo,$h2hi,$h3lo,$h3hi,
61 $two0,$two32,$two64,$two96,$two130,$five_two130,
62 $r0lo,$r0hi,$r1lo,$r1hi,$r2lo,$r2hi,
63 $s2lo,$s2hi,$s3lo,$s3hi,
64 $c0lo,$c0hi,$c1lo,$c1hi,$c2lo,$c2hi,$c3lo,$c3hi) = map("f$_",(0..31));
66 my ($r3lo,$r3hi,$s1lo,$s1hi) = ($c0lo,$c0hi,$c1lo,$c1hi);
67 my ($x0,$x1,$x2,$x3) = ($c2lo,$c2hi,$c3lo,$c3hi);
68 my ($y0,$y1,$y2,$y3) = ($c3lo,$c3hi,$c1lo,$c1hi);
74 .globl .poly1305_init_fpu
77 $STU $sp,-$LOCALS($sp) # minimal frame
79 $PUSH $padbit,`$LOCALS+$LRSAVE`($sp)
84 mtlr $padbit # restore lr
86 lfd $two0,8*0($len) # load constants
91 lfd $five_two130,8*5($len)
93 stfd $two0,8*0($ctx) # initial hash value, biased 0
101 lfd $h3lo,8*13($len) # new fpscr
102 mffs $h3hi # old fpscr
104 stfd $two0,8*4($ctx) # key "template"
105 stfd $two32,8*5($ctx)
106 stfd $two64,8*6($ctx)
107 stfd $two96,8*7($ctx)
112 $LWXLE $in0,0,$inp # load key
113 $LWXLE $in1,$in1,$inp
114 $LWXLE $in2,$in2,$inp
115 $LWXLE $in3,$in3,$inp
117 lis $i1,0xf000 # 0xf0000000
118 ori $i2,$i1,3 # 0xf0000003
119 andc $in0,$in0,$i1 # &=0x0fffffff
120 andc $in1,$in1,$i2 # &=0x0ffffffc
124 stw $in0,`8*4+(4^$LITTLE_ENDIAN)`($ctx) # fill "template"
125 stw $in1,`8*5+(4^$LITTLE_ENDIAN)`($ctx)
126 stw $in2,`8*6+(4^$LITTLE_ENDIAN)`($ctx)
127 stw $in3,`8*7+(4^$LITTLE_ENDIAN)`($ctx)
129 mtfsf 255,$h3lo # fpscr
130 stfd $two0,8*18($ctx) # copy constants to context
131 stfd $two32,8*19($ctx)
132 stfd $two64,8*20($ctx)
133 stfd $two96,8*21($ctx)
134 stfd $two130,8*22($ctx)
135 stfd $five_two130,8*23($ctx)
137 lfd $h0lo,8*4($ctx) # load [biased] key
142 fsub $h0lo,$h0lo,$two0 # r0
143 fsub $h1lo,$h1lo,$two32 # r1
144 fsub $h2lo,$h2lo,$two64 # r2
145 fsub $h3lo,$h3lo,$two96 # r3
147 lfd $two0,8*6($len) # more constants
152 fmul $h1hi,$h1lo,$five_two130 # s1
153 fmul $h2hi,$h2lo,$five_two130 # s2
154 stfd $h3hi,8*15($ctx) # borrow slot for original fpscr
155 fmul $h3hi,$h3lo,$five_two130 # s3
157 fadd $h0hi,$h0lo,$two0
158 stfd $h1hi,8*12($ctx) # put aside for now
159 fadd $h1hi,$h1lo,$two32
160 stfd $h2hi,8*13($ctx)
161 fadd $h2hi,$h2lo,$two64
162 stfd $h3hi,8*14($ctx)
163 fadd $h3hi,$h3lo,$two96
165 fsub $h0hi,$h0hi,$two0
166 fsub $h1hi,$h1hi,$two32
167 fsub $h2hi,$h2hi,$two64
168 fsub $h3hi,$h3hi,$two96
170 lfd $two0,8*10($len) # more constants
171 lfd $two32,8*11($len)
172 lfd $two64,8*12($len)
174 fsub $h0lo,$h0lo,$h0hi
175 fsub $h1lo,$h1lo,$h1hi
176 fsub $h2lo,$h2lo,$h2hi
177 fsub $h3lo,$h3lo,$h3hi
179 stfd $h0hi,8*5($ctx) # r0hi
180 stfd $h1hi,8*7($ctx) # r1hi
181 stfd $h2hi,8*9($ctx) # r2hi
182 stfd $h3hi,8*11($ctx) # r3hi
184 stfd $h0lo,8*4($ctx) # r0lo
185 stfd $h1lo,8*6($ctx) # r1lo
186 stfd $h2lo,8*8($ctx) # r2lo
187 stfd $h3lo,8*10($ctx) # r3lo
189 lfd $h1lo,8*12($ctx) # s1
190 lfd $h2lo,8*13($ctx) # s2
191 lfd $h3lo,8*14($ctx) # s3
192 lfd $h0lo,8*15($ctx) # pull original fpscr
194 fadd $h1hi,$h1lo,$two0
195 fadd $h2hi,$h2lo,$two32
196 fadd $h3hi,$h3lo,$two64
198 fsub $h1hi,$h1hi,$two0
199 fsub $h2hi,$h2hi,$two32
200 fsub $h3hi,$h3hi,$two64
202 fsub $h1lo,$h1lo,$h1hi
203 fsub $h2lo,$h2lo,$h2hi
204 fsub $h3lo,$h3lo,$h3hi
206 stfd $h1hi,8*13($ctx) # s1hi
207 stfd $h2hi,8*15($ctx) # s2hi
208 stfd $h3hi,8*17($ctx) # s3hi
210 stfd $h1lo,8*12($ctx) # s1lo
211 stfd $h2lo,8*14($ctx) # s2lo
212 stfd $h3lo,8*16($ctx) # s3lo
214 mtfsf 255,$h0lo # restore fpscr
220 .byte 0,12,4,1,0x80,0,2,0
221 .size .poly1305_init_fpu,.-.poly1305_init_fpu
223 .globl .poly1305_blocks_fpu
225 .poly1305_blocks_fpu:
229 $STU $sp,-$FRAME($sp)
231 stfd f14,`$FRAME-8*18`($sp)
232 stfd f15,`$FRAME-8*17`($sp)
233 stfd f16,`$FRAME-8*16`($sp)
234 stfd f17,`$FRAME-8*15`($sp)
235 stfd f18,`$FRAME-8*14`($sp)
236 stfd f19,`$FRAME-8*13`($sp)
237 stfd f20,`$FRAME-8*12`($sp)
238 stfd f21,`$FRAME-8*11`($sp)
239 stfd f22,`$FRAME-8*10`($sp)
240 stfd f23,`$FRAME-8*9`($sp)
241 stfd f24,`$FRAME-8*8`($sp)
242 stfd f25,`$FRAME-8*7`($sp)
243 stfd f26,`$FRAME-8*6`($sp)
244 stfd f27,`$FRAME-8*5`($sp)
245 stfd f28,`$FRAME-8*4`($sp)
246 stfd f29,`$FRAME-8*3`($sp)
247 stfd f30,`$FRAME-8*2`($sp)
248 stfd f31,`$FRAME-8*1`($sp)
249 $PUSH r0,`$FRAME+$LRSAVE`($sp)
255 stw r0,`$LOCALS+8*4+(0^$LITTLE_ENDIAN)`($sp)
256 stw $in3,`$LOCALS+8*4+(4^$LITTLE_ENDIAN)`($sp)
258 lfd $two0,8*18($ctx) # load constants
259 lfd $two32,8*19($ctx)
260 lfd $two64,8*20($ctx)
261 lfd $two96,8*21($ctx)
262 lfd $two130,8*22($ctx)
263 lfd $five_two130,8*23($ctx)
265 lfd $h0lo,8*0($ctx) # load [biased] hash value
270 stfd $two0,`$LOCALS+8*0`($sp) # input "template"
271 oris $in3,$padbit,`(1023+52+96)<<4`
272 stfd $two32,`$LOCALS+8*1`($sp)
273 stfd $two64,`$LOCALS+8*2`($sp)
274 stw $in3,`$LOCALS+8*3+(0^$LITTLE_ENDIAN)`($sp)
279 $LWXLE $in0,0,$inp # load input
285 stw $in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp) # fill "template"
286 stw $in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
287 stw $in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
288 stw $in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
290 mffs $x0 # original fpscr
291 lfd $x1,`$LOCALS+8*4`($sp) # new fpscr
292 lfd $r0lo,8*4($ctx) # load key
307 stfd $x0,`$LOCALS+8*4`($sp) # save original fpscr
313 sub $inp,$inp,r0 # conditional rewind
315 lfd $x0,`$LOCALS+8*0`($sp)
316 lfd $x1,`$LOCALS+8*1`($sp)
317 lfd $x2,`$LOCALS+8*2`($sp)
318 lfd $x3,`$LOCALS+8*3`($sp)
320 fsub $h0lo,$h0lo,$two0 # de-bias hash value
321 $LWXLE $in0,0,$inp # modulo-scheduled input load
322 fsub $h1lo,$h1lo,$two32
324 fsub $h2lo,$h2lo,$two64
326 fsub $h3lo,$h3lo,$two96
329 fsub $x0,$x0,$two0 # de-bias input
335 fadd $x0,$x0,$h0lo # accumulate input
336 stw $in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp)
338 stw $in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
340 stw $in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
342 stw $in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
348 fsub $y0,$y0,$two0 # de-bias input
355 sub $inp,$inp,r0 # conditional rewind
357 fadd $h0lo,$h0lo,$y0 # accumulate input
362 ######################################### base 2^48 -> base 2^32
363 fadd $c1lo,$h1lo,$two64
364 $LWXLE $in0,0,$inp # modulo-scheduled input load
365 fadd $c1hi,$h1hi,$two64
367 fadd $c3lo,$h3lo,$two130
369 fadd $c3hi,$h3hi,$two130
371 fadd $c0lo,$h0lo,$two32
373 fadd $c0hi,$h0hi,$two32
374 fadd $c2lo,$h2lo,$two96
375 fadd $c2hi,$h2hi,$two96
377 fsub $c1lo,$c1lo,$two64
378 stw $in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp) # fill "template"
379 fsub $c1hi,$c1hi,$two64
380 stw $in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
381 fsub $c3lo,$c3lo,$two130
382 stw $in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
383 fsub $c3hi,$c3hi,$two130
384 stw $in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
385 fsub $c0lo,$c0lo,$two32
386 fsub $c0hi,$c0hi,$two32
387 fsub $c2lo,$c2lo,$two96
388 fsub $c2hi,$c2hi,$two96
390 fsub $h1lo,$h1lo,$c1lo
391 fsub $h1hi,$h1hi,$c1hi
392 fsub $h3lo,$h3lo,$c3lo
393 fsub $h3hi,$h3hi,$c3hi
394 fsub $h2lo,$h2lo,$c2lo
395 fsub $h2hi,$h2hi,$c2hi
396 fsub $h0lo,$h0lo,$c0lo
397 fsub $h0hi,$h0hi,$c0hi
399 fadd $h1lo,$h1lo,$c0lo
400 fadd $h1hi,$h1hi,$c0hi
401 fadd $h3lo,$h3lo,$c2lo
402 fadd $h3hi,$h3hi,$c2hi
403 fadd $h2lo,$h2lo,$c1lo
404 fadd $h2hi,$h2hi,$c1hi
405 fmadd $h0lo,$c3lo,$five_two130,$h0lo
406 fmadd $h0hi,$c3hi,$five_two130,$h0hi
409 lfd $s1lo,8*12($ctx) # reload constants
426 fmadd $h0lo,$s1lo,$x3,$h0lo
427 fmadd $h0hi,$s1hi,$x3,$h0hi
428 fmadd $h2lo,$s3lo,$x3,$h2lo
429 fmadd $h2hi,$s3hi,$x3,$h2hi
430 fmadd $h1lo,$s2lo,$x3,$h1lo
431 fmadd $h1hi,$s2hi,$x3,$h1hi
432 fmadd $h3lo,$r0lo,$x3,$h3lo
433 fmadd $h3hi,$r0hi,$x3,$h3hi
435 fmadd $h0lo,$s2lo,$x2,$h0lo
436 fmadd $h0hi,$s2hi,$x2,$h0hi
437 fmadd $h2lo,$r0lo,$x2,$h2lo
438 fmadd $h2hi,$r0hi,$x2,$h2hi
439 fmadd $h1lo,$s3lo,$x2,$h1lo
440 fmadd $h1hi,$s3hi,$x2,$h1hi
441 fmadd $h3lo,$r1lo,$x2,$h3lo
442 fmadd $h3hi,$r1hi,$x2,$h3hi
444 fmadd $h0lo,$r0lo,$x0,$h0lo
445 lfd $y0,`$LOCALS+8*0`($sp) # load [biased] input
446 fmadd $h0hi,$r0hi,$x0,$h0hi
447 lfd $y1,`$LOCALS+8*1`($sp)
448 fmadd $h2lo,$r2lo,$x0,$h2lo
449 lfd $y2,`$LOCALS+8*2`($sp)
450 fmadd $h2hi,$r2hi,$x0,$h2hi
451 lfd $y3,`$LOCALS+8*3`($sp)
452 fmadd $h1lo,$r1lo,$x0,$h1lo
453 fmadd $h1hi,$r1hi,$x0,$h1hi
454 fmadd $h3lo,$r3lo,$x0,$h3lo
455 fmadd $h3hi,$r3hi,$x0,$h3hi
459 ######################################### base 2^48 -> base 2^32
460 fadd $c0lo,$h0lo,$two32
461 fadd $c0hi,$h0hi,$two32
462 fadd $c2lo,$h2lo,$two96
463 fadd $c2hi,$h2hi,$two96
464 fadd $c1lo,$h1lo,$two64
465 fadd $c1hi,$h1hi,$two64
466 fadd $c3lo,$h3lo,$two130
467 fadd $c3hi,$h3hi,$two130
469 fsub $c0lo,$c0lo,$two32
470 fsub $c0hi,$c0hi,$two32
471 fsub $c2lo,$c2lo,$two96
472 fsub $c2hi,$c2hi,$two96
473 fsub $c1lo,$c1lo,$two64
474 fsub $c1hi,$c1hi,$two64
475 fsub $c3lo,$c3lo,$two130
476 fsub $c3hi,$c3hi,$two130
478 fsub $h1lo,$h1lo,$c1lo
479 fsub $h1hi,$h1hi,$c1hi
480 fsub $h3lo,$h3lo,$c3lo
481 fsub $h3hi,$h3hi,$c3hi
482 fsub $h2lo,$h2lo,$c2lo
483 fsub $h2hi,$h2hi,$c2hi
484 fsub $h0lo,$h0lo,$c0lo
485 fsub $h0hi,$h0hi,$c0hi
487 fadd $h1lo,$h1lo,$c0lo
488 fadd $h1hi,$h1hi,$c0hi
489 fadd $h3lo,$h3lo,$c2lo
490 fadd $h3hi,$h3hi,$c2hi
491 fadd $h2lo,$h2lo,$c1lo
492 fadd $h2hi,$h2hi,$c1hi
493 fmadd $h0lo,$c3lo,$five_two130,$h0lo
494 fmadd $h0hi,$c3hi,$five_two130,$h0hi
501 lfd $h0lo,`$LOCALS+8*4`($sp) # pull saved fpscr
502 fadd $x1,$x1,$two32 # bias
507 stfd $x1,8*1($ctx) # store [biased] hash value
512 mtfsf 255,$h0lo # restore original fpscr
513 lfd f14,`$FRAME-8*18`($sp)
514 lfd f15,`$FRAME-8*17`($sp)
515 lfd f16,`$FRAME-8*16`($sp)
516 lfd f17,`$FRAME-8*15`($sp)
517 lfd f18,`$FRAME-8*14`($sp)
518 lfd f19,`$FRAME-8*13`($sp)
519 lfd f20,`$FRAME-8*12`($sp)
520 lfd f21,`$FRAME-8*11`($sp)
521 lfd f22,`$FRAME-8*10`($sp)
522 lfd f23,`$FRAME-8*9`($sp)
523 lfd f24,`$FRAME-8*8`($sp)
524 lfd f25,`$FRAME-8*7`($sp)
525 lfd f26,`$FRAME-8*6`($sp)
526 lfd f27,`$FRAME-8*5`($sp)
527 lfd f28,`$FRAME-8*4`($sp)
528 lfd f29,`$FRAME-8*3`($sp)
529 lfd f30,`$FRAME-8*2`($sp)
530 lfd f31,`$FRAME-8*1`($sp)
535 .byte 0,12,4,1,0x80,0,4,0
536 .size .poly1305_blocks_fpu,.-.poly1305_blocks_fpu
539 my ($mac,$nonce)=($inp,$len);
541 my ($h0,$h1,$h2,$h3,$h4, $d0,$d1,$d2,$d3
542 ) = map("r$_",(7..11,28..31));
544 my $FRAME = (6+4)*$SIZE_T;
547 .globl .poly1305_emit_fpu
550 $STU $sp,-$FRAME($sp)
552 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
553 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
554 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
555 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
556 $PUSH r0,`$FRAME+$LRSAVE`($sp)
558 lwz $d0,`8*0+(0^$LITTLE_ENDIAN)`($ctx) # load hash
559 lwz $h0,`8*0+(4^$LITTLE_ENDIAN)`($ctx)
560 lwz $d1,`8*1+(0^$LITTLE_ENDIAN)`($ctx)
561 lwz $h1,`8*1+(4^$LITTLE_ENDIAN)`($ctx)
562 lwz $d2,`8*2+(0^$LITTLE_ENDIAN)`($ctx)
563 lwz $h2,`8*2+(4^$LITTLE_ENDIAN)`($ctx)
564 lwz $d3,`8*3+(0^$LITTLE_ENDIAN)`($ctx)
565 lwz $h3,`8*3+(4^$LITTLE_ENDIAN)`($ctx)
568 andc $d0,$d0,$mask # mask exponent
571 andc $d3,$d3,$mask # can be partially reduced...
574 srwi $padbit,$d3,2 # ... so reduce
587 addic $d0,$h0,5 # compare to modulus
593 srwi $mask,$mask,2 # did it carry/borrow?
595 srawi $mask,$mask,31 # mask
602 lwz $d0,0($nonce) # load nonce
614 addc $h0,$h0,$d0 # accumulate nonce
638 addic $d0,$h0,5 # compare to modulus
642 srdi $mask,$d2,2 # did it carry/borrow?
644 sradi $mask,$mask,63 # mask
645 ld $d2,0($nonce) # load nonce
655 $code.=<<___ if (!$LITTLE_ENDIAN);
656 rotldi $d2,$d2,32 # flip nonce words
660 addc $h0,$h0,$d2 # accumulate nonce
667 $code.=<<___ if ($LITTLE_ENDIAN);
668 stw $h0,0($mac) # write result
673 $code.=<<___ if (!$LITTLE_ENDIAN);
675 stwbrx $h0,0,$mac # write result
683 $POP r28,`$FRAME-$SIZE_T*4`($sp)
684 $POP r29,`$FRAME-$SIZE_T*3`($sp)
685 $POP r30,`$FRAME-$SIZE_T*2`($sp)
686 $POP r31,`$FRAME-$SIZE_T*1`($sp)
690 .byte 0,12,4,1,0x80,4,3,0
691 .size .poly1305_emit_fpu,.-.poly1305_emit_fpu
694 # Ugly hack here, because PPC assembler syntax seem to vary too
695 # much from platforms to platform...
701 mflr $len # vvvvvv "distance" between . and 1st data entry
702 addi $len,$len,`64-8` # borrow $len
706 .byte 0,12,0x14,0,0,0,0,0
709 .quad 0x4330000000000000 # 2^(52+0)
710 .quad 0x4530000000000000 # 2^(52+32)
711 .quad 0x4730000000000000 # 2^(52+64)
712 .quad 0x4930000000000000 # 2^(52+96)
713 .quad 0x4b50000000000000 # 2^(52+130)
715 .quad 0x37f4000000000000 # 5/2^130
717 .quad 0x4430000000000000 # 2^(52+16+0)
718 .quad 0x4630000000000000 # 2^(52+16+32)
719 .quad 0x4830000000000000 # 2^(52+16+64)
720 .quad 0x4a30000000000000 # 2^(52+16+96)
721 .quad 0x3e30000000000000 # 2^(52+16+0-96)
722 .quad 0x4030000000000000 # 2^(52+16+32-96)
723 .quad 0x4230000000000000 # 2^(52+16+64-96)
725 .quad 0x0000000000000001 # fpscr: truncate, no exceptions
726 .asciz "Poly1305 for PPC FPU, CRYPTOGAMS by <appro\@openssl.org>"
730 $code =~ s/\`([^\`]*)\`/eval $1/gem;