2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
17 # This module implements Poly1305 hash for PowerPC FPU.
21 # Numbers are cycles per processed byte with poly1305_blocks alone,
22 # and improvement coefficients relative to gcc-generated code.
24 # Freescale e300 9.78/+30%
32 if ($flavour =~ /64/) {
39 } elsif ($flavour =~ /32/) {
46 } else { die "nonsense $flavour"; }
48 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 4 : 0;
50 $LWXLE = $LITTLE_ENDIAN ? "lwzx" : "lwbrx";
52 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
53 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
54 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
55 die "can't locate ppc-xlate.pl";
57 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
60 $FRAME=$LOCALS+6*8+18*8;
64 my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
65 my ($in0,$in1,$in2,$in3,$i1,$i2,$i3) = map("r$_",(7..12,6));
67 my ($h0lo,$h0hi,$h1lo,$h1hi,$h2lo,$h2hi,$h3lo,$h3hi,
68 $two0,$two32,$two64,$two96,$two130,$five_two130,
69 $r0lo,$r0hi,$r1lo,$r1hi,$r2lo,$r2hi,
70 $s2lo,$s2hi,$s3lo,$s3hi,
71 $c0lo,$c0hi,$c1lo,$c1hi,$c2lo,$c2hi,$c3lo,$c3hi) = map("f$_",(0..31));
73 my ($r3lo,$r3hi,$s1lo,$s1hi) = ($c0lo,$c0hi,$c1lo,$c1hi);
74 my ($x0,$x1,$x2,$x3) = ($c2lo,$c2hi,$c3lo,$c3hi);
75 my ($y0,$y1,$y2,$y3) = ($c3lo,$c3hi,$c1lo,$c1hi);
81 .globl .poly1305_init_fpu
84 $STU $sp,-$LOCALS($sp) # minimal frame
86 $PUSH $padbit,`$LOCALS+$LRSAVE`($sp)
91 mtlr $padbit # restore lr
93 lfd $two0,8*0($len) # load constants
98 lfd $five_two130,8*5($len)
100 stfd $two0,8*0($ctx) # initial hash value, biased 0
101 stfd $two32,8*1($ctx)
102 stfd $two64,8*2($ctx)
103 stfd $two96,8*3($ctx)
108 lfd $h3lo,8*13($len) # new fpscr
109 mffs $h3hi # old fpscr
111 stfd $two0,8*4($ctx) # key "template"
112 stfd $two32,8*5($ctx)
113 stfd $two64,8*6($ctx)
114 stfd $two96,8*7($ctx)
119 $LWXLE $in0,0,$inp # load key
120 $LWXLE $in1,$in1,$inp
121 $LWXLE $in2,$in2,$inp
122 $LWXLE $in3,$in3,$inp
124 lis $i1,0xf000 # 0xf0000000
125 ori $i2,$i1,3 # 0xf0000003
126 andc $in0,$in0,$i1 # &=0x0fffffff
127 andc $in1,$in1,$i2 # &=0x0ffffffc
131 stw $in0,`8*4+(4^$LITTLE_ENDIAN)`($ctx) # fill "template"
132 stw $in1,`8*5+(4^$LITTLE_ENDIAN)`($ctx)
133 stw $in2,`8*6+(4^$LITTLE_ENDIAN)`($ctx)
134 stw $in3,`8*7+(4^$LITTLE_ENDIAN)`($ctx)
136 mtfsf 255,$h3lo # fpscr
137 stfd $two0,8*18($ctx) # copy constants to context
138 stfd $two32,8*19($ctx)
139 stfd $two64,8*20($ctx)
140 stfd $two96,8*21($ctx)
141 stfd $two130,8*22($ctx)
142 stfd $five_two130,8*23($ctx)
144 lfd $h0lo,8*4($ctx) # load [biased] key
149 fsub $h0lo,$h0lo,$two0 # r0
150 fsub $h1lo,$h1lo,$two32 # r1
151 fsub $h2lo,$h2lo,$two64 # r2
152 fsub $h3lo,$h3lo,$two96 # r3
154 lfd $two0,8*6($len) # more constants
159 fmul $h1hi,$h1lo,$five_two130 # s1
160 fmul $h2hi,$h2lo,$five_two130 # s2
161 stfd $h3hi,8*15($ctx) # borrow slot for original fpscr
162 fmul $h3hi,$h3lo,$five_two130 # s3
164 fadd $h0hi,$h0lo,$two0
165 stfd $h1hi,8*12($ctx) # put aside for now
166 fadd $h1hi,$h1lo,$two32
167 stfd $h2hi,8*13($ctx)
168 fadd $h2hi,$h2lo,$two64
169 stfd $h3hi,8*14($ctx)
170 fadd $h3hi,$h3lo,$two96
172 fsub $h0hi,$h0hi,$two0
173 fsub $h1hi,$h1hi,$two32
174 fsub $h2hi,$h2hi,$two64
175 fsub $h3hi,$h3hi,$two96
177 lfd $two0,8*10($len) # more constants
178 lfd $two32,8*11($len)
179 lfd $two64,8*12($len)
181 fsub $h0lo,$h0lo,$h0hi
182 fsub $h1lo,$h1lo,$h1hi
183 fsub $h2lo,$h2lo,$h2hi
184 fsub $h3lo,$h3lo,$h3hi
186 stfd $h0hi,8*5($ctx) # r0hi
187 stfd $h1hi,8*7($ctx) # r1hi
188 stfd $h2hi,8*9($ctx) # r2hi
189 stfd $h3hi,8*11($ctx) # r3hi
191 stfd $h0lo,8*4($ctx) # r0lo
192 stfd $h1lo,8*6($ctx) # r1lo
193 stfd $h2lo,8*8($ctx) # r2lo
194 stfd $h3lo,8*10($ctx) # r3lo
196 lfd $h1lo,8*12($ctx) # s1
197 lfd $h2lo,8*13($ctx) # s2
198 lfd $h3lo,8*14($ctx) # s3
199 lfd $h0lo,8*15($ctx) # pull original fpscr
201 fadd $h1hi,$h1lo,$two0
202 fadd $h2hi,$h2lo,$two32
203 fadd $h3hi,$h3lo,$two64
205 fsub $h1hi,$h1hi,$two0
206 fsub $h2hi,$h2hi,$two32
207 fsub $h3hi,$h3hi,$two64
209 fsub $h1lo,$h1lo,$h1hi
210 fsub $h2lo,$h2lo,$h2hi
211 fsub $h3lo,$h3lo,$h3hi
213 stfd $h1hi,8*13($ctx) # s1hi
214 stfd $h2hi,8*15($ctx) # s2hi
215 stfd $h3hi,8*17($ctx) # s3hi
217 stfd $h1lo,8*12($ctx) # s1lo
218 stfd $h2lo,8*14($ctx) # s2lo
219 stfd $h3lo,8*16($ctx) # s3lo
221 mtfsf 255,$h0lo # restore fpscr
227 .byte 0,12,4,1,0x80,0,2,0
228 .size .poly1305_init_fpu,.-.poly1305_init_fpu
230 .globl .poly1305_blocks_fpu
232 .poly1305_blocks_fpu:
236 $STU $sp,-$FRAME($sp)
238 stfd f14,`$FRAME-8*18`($sp)
239 stfd f15,`$FRAME-8*17`($sp)
240 stfd f16,`$FRAME-8*16`($sp)
241 stfd f17,`$FRAME-8*15`($sp)
242 stfd f18,`$FRAME-8*14`($sp)
243 stfd f19,`$FRAME-8*13`($sp)
244 stfd f20,`$FRAME-8*12`($sp)
245 stfd f21,`$FRAME-8*11`($sp)
246 stfd f22,`$FRAME-8*10`($sp)
247 stfd f23,`$FRAME-8*9`($sp)
248 stfd f24,`$FRAME-8*8`($sp)
249 stfd f25,`$FRAME-8*7`($sp)
250 stfd f26,`$FRAME-8*6`($sp)
251 stfd f27,`$FRAME-8*5`($sp)
252 stfd f28,`$FRAME-8*4`($sp)
253 stfd f29,`$FRAME-8*3`($sp)
254 stfd f30,`$FRAME-8*2`($sp)
255 stfd f31,`$FRAME-8*1`($sp)
256 $PUSH r0,`$FRAME+$LRSAVE`($sp)
262 stw r0,`$LOCALS+8*4+(0^$LITTLE_ENDIAN)`($sp)
263 stw $in3,`$LOCALS+8*4+(4^$LITTLE_ENDIAN)`($sp)
265 lfd $two0,8*18($ctx) # load constants
266 lfd $two32,8*19($ctx)
267 lfd $two64,8*20($ctx)
268 lfd $two96,8*21($ctx)
269 lfd $two130,8*22($ctx)
270 lfd $five_two130,8*23($ctx)
272 lfd $h0lo,8*0($ctx) # load [biased] hash value
277 stfd $two0,`$LOCALS+8*0`($sp) # input "template"
278 oris $in3,$padbit,`(1023+52+96)<<4`
279 stfd $two32,`$LOCALS+8*1`($sp)
280 stfd $two64,`$LOCALS+8*2`($sp)
281 stw $in3,`$LOCALS+8*3+(0^$LITTLE_ENDIAN)`($sp)
286 $LWXLE $in0,0,$inp # load input
292 stw $in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp) # fill "template"
293 stw $in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
294 stw $in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
295 stw $in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
297 mffs $x0 # original fpscr
298 lfd $x1,`$LOCALS+8*4`($sp) # new fpscr
299 lfd $r0lo,8*4($ctx) # load key
314 stfd $x0,`$LOCALS+8*4`($sp) # save original fpscr
320 sub $inp,$inp,r0 # conditional rewind
322 lfd $x0,`$LOCALS+8*0`($sp)
323 lfd $x1,`$LOCALS+8*1`($sp)
324 lfd $x2,`$LOCALS+8*2`($sp)
325 lfd $x3,`$LOCALS+8*3`($sp)
327 fsub $h0lo,$h0lo,$two0 # de-bias hash value
328 $LWXLE $in0,0,$inp # modulo-scheduled input load
329 fsub $h1lo,$h1lo,$two32
331 fsub $h2lo,$h2lo,$two64
333 fsub $h3lo,$h3lo,$two96
336 fsub $x0,$x0,$two0 # de-bias input
342 fadd $x0,$x0,$h0lo # accumulate input
343 stw $in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp)
345 stw $in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
347 stw $in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
349 stw $in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
355 fsub $y0,$y0,$two0 # de-bias input
362 sub $inp,$inp,r0 # conditional rewind
364 fadd $h0lo,$h0lo,$y0 # accumulate input
369 ######################################### base 2^48 -> base 2^32
370 fadd $c1lo,$h1lo,$two64
371 $LWXLE $in0,0,$inp # modulo-scheduled input load
372 fadd $c1hi,$h1hi,$two64
374 fadd $c3lo,$h3lo,$two130
376 fadd $c3hi,$h3hi,$two130
378 fadd $c0lo,$h0lo,$two32
380 fadd $c0hi,$h0hi,$two32
381 fadd $c2lo,$h2lo,$two96
382 fadd $c2hi,$h2hi,$two96
384 fsub $c1lo,$c1lo,$two64
385 stw $in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp) # fill "template"
386 fsub $c1hi,$c1hi,$two64
387 stw $in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
388 fsub $c3lo,$c3lo,$two130
389 stw $in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
390 fsub $c3hi,$c3hi,$two130
391 stw $in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
392 fsub $c0lo,$c0lo,$two32
393 fsub $c0hi,$c0hi,$two32
394 fsub $c2lo,$c2lo,$two96
395 fsub $c2hi,$c2hi,$two96
397 fsub $h1lo,$h1lo,$c1lo
398 fsub $h1hi,$h1hi,$c1hi
399 fsub $h3lo,$h3lo,$c3lo
400 fsub $h3hi,$h3hi,$c3hi
401 fsub $h2lo,$h2lo,$c2lo
402 fsub $h2hi,$h2hi,$c2hi
403 fsub $h0lo,$h0lo,$c0lo
404 fsub $h0hi,$h0hi,$c0hi
406 fadd $h1lo,$h1lo,$c0lo
407 fadd $h1hi,$h1hi,$c0hi
408 fadd $h3lo,$h3lo,$c2lo
409 fadd $h3hi,$h3hi,$c2hi
410 fadd $h2lo,$h2lo,$c1lo
411 fadd $h2hi,$h2hi,$c1hi
412 fmadd $h0lo,$c3lo,$five_two130,$h0lo
413 fmadd $h0hi,$c3hi,$five_two130,$h0hi
416 lfd $s1lo,8*12($ctx) # reload constants
433 fmadd $h0lo,$s1lo,$x3,$h0lo
434 fmadd $h0hi,$s1hi,$x3,$h0hi
435 fmadd $h2lo,$s3lo,$x3,$h2lo
436 fmadd $h2hi,$s3hi,$x3,$h2hi
437 fmadd $h1lo,$s2lo,$x3,$h1lo
438 fmadd $h1hi,$s2hi,$x3,$h1hi
439 fmadd $h3lo,$r0lo,$x3,$h3lo
440 fmadd $h3hi,$r0hi,$x3,$h3hi
442 fmadd $h0lo,$s2lo,$x2,$h0lo
443 fmadd $h0hi,$s2hi,$x2,$h0hi
444 fmadd $h2lo,$r0lo,$x2,$h2lo
445 fmadd $h2hi,$r0hi,$x2,$h2hi
446 fmadd $h1lo,$s3lo,$x2,$h1lo
447 fmadd $h1hi,$s3hi,$x2,$h1hi
448 fmadd $h3lo,$r1lo,$x2,$h3lo
449 fmadd $h3hi,$r1hi,$x2,$h3hi
451 fmadd $h0lo,$r0lo,$x0,$h0lo
452 lfd $y0,`$LOCALS+8*0`($sp) # load [biased] input
453 fmadd $h0hi,$r0hi,$x0,$h0hi
454 lfd $y1,`$LOCALS+8*1`($sp)
455 fmadd $h2lo,$r2lo,$x0,$h2lo
456 lfd $y2,`$LOCALS+8*2`($sp)
457 fmadd $h2hi,$r2hi,$x0,$h2hi
458 lfd $y3,`$LOCALS+8*3`($sp)
459 fmadd $h1lo,$r1lo,$x0,$h1lo
460 fmadd $h1hi,$r1hi,$x0,$h1hi
461 fmadd $h3lo,$r3lo,$x0,$h3lo
462 fmadd $h3hi,$r3hi,$x0,$h3hi
466 ######################################### base 2^48 -> base 2^32
467 fadd $c0lo,$h0lo,$two32
468 fadd $c0hi,$h0hi,$two32
469 fadd $c2lo,$h2lo,$two96
470 fadd $c2hi,$h2hi,$two96
471 fadd $c1lo,$h1lo,$two64
472 fadd $c1hi,$h1hi,$two64
473 fadd $c3lo,$h3lo,$two130
474 fadd $c3hi,$h3hi,$two130
476 fsub $c0lo,$c0lo,$two32
477 fsub $c0hi,$c0hi,$two32
478 fsub $c2lo,$c2lo,$two96
479 fsub $c2hi,$c2hi,$two96
480 fsub $c1lo,$c1lo,$two64
481 fsub $c1hi,$c1hi,$two64
482 fsub $c3lo,$c3lo,$two130
483 fsub $c3hi,$c3hi,$two130
485 fsub $h1lo,$h1lo,$c1lo
486 fsub $h1hi,$h1hi,$c1hi
487 fsub $h3lo,$h3lo,$c3lo
488 fsub $h3hi,$h3hi,$c3hi
489 fsub $h2lo,$h2lo,$c2lo
490 fsub $h2hi,$h2hi,$c2hi
491 fsub $h0lo,$h0lo,$c0lo
492 fsub $h0hi,$h0hi,$c0hi
494 fadd $h1lo,$h1lo,$c0lo
495 fadd $h1hi,$h1hi,$c0hi
496 fadd $h3lo,$h3lo,$c2lo
497 fadd $h3hi,$h3hi,$c2hi
498 fadd $h2lo,$h2lo,$c1lo
499 fadd $h2hi,$h2hi,$c1hi
500 fmadd $h0lo,$c3lo,$five_two130,$h0lo
501 fmadd $h0hi,$c3hi,$five_two130,$h0hi
508 lfd $h0lo,`$LOCALS+8*4`($sp) # pull saved fpscr
509 fadd $x1,$x1,$two32 # bias
514 stfd $x1,8*1($ctx) # store [biased] hash value
519 mtfsf 255,$h0lo # restore original fpscr
520 lfd f14,`$FRAME-8*18`($sp)
521 lfd f15,`$FRAME-8*17`($sp)
522 lfd f16,`$FRAME-8*16`($sp)
523 lfd f17,`$FRAME-8*15`($sp)
524 lfd f18,`$FRAME-8*14`($sp)
525 lfd f19,`$FRAME-8*13`($sp)
526 lfd f20,`$FRAME-8*12`($sp)
527 lfd f21,`$FRAME-8*11`($sp)
528 lfd f22,`$FRAME-8*10`($sp)
529 lfd f23,`$FRAME-8*9`($sp)
530 lfd f24,`$FRAME-8*8`($sp)
531 lfd f25,`$FRAME-8*7`($sp)
532 lfd f26,`$FRAME-8*6`($sp)
533 lfd f27,`$FRAME-8*5`($sp)
534 lfd f28,`$FRAME-8*4`($sp)
535 lfd f29,`$FRAME-8*3`($sp)
536 lfd f30,`$FRAME-8*2`($sp)
537 lfd f31,`$FRAME-8*1`($sp)
542 .byte 0,12,4,1,0x80,0,4,0
543 .size .poly1305_blocks_fpu,.-.poly1305_blocks_fpu
546 my ($mac,$nonce)=($inp,$len);
548 my ($h0,$h1,$h2,$h3,$h4, $d0,$d1,$d2,$d3
549 ) = map("r$_",(7..11,28..31));
551 my $FRAME = (6+4)*$SIZE_T;
554 .globl .poly1305_emit_fpu
557 $STU $sp,-$FRAME($sp)
559 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
560 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
561 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
562 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
563 $PUSH r0,`$FRAME+$LRSAVE`($sp)
565 lwz $d0,`8*0+(0^$LITTLE_ENDIAN)`($ctx) # load hash
566 lwz $h0,`8*0+(4^$LITTLE_ENDIAN)`($ctx)
567 lwz $d1,`8*1+(0^$LITTLE_ENDIAN)`($ctx)
568 lwz $h1,`8*1+(4^$LITTLE_ENDIAN)`($ctx)
569 lwz $d2,`8*2+(0^$LITTLE_ENDIAN)`($ctx)
570 lwz $h2,`8*2+(4^$LITTLE_ENDIAN)`($ctx)
571 lwz $d3,`8*3+(0^$LITTLE_ENDIAN)`($ctx)
572 lwz $h3,`8*3+(4^$LITTLE_ENDIAN)`($ctx)
575 andc $d0,$d0,$mask # mask exponent
578 andc $d3,$d3,$mask # can be partially reduced...
581 srwi $padbit,$d3,2 # ... so reduce
594 addic $d0,$h0,5 # compare to modulus
600 srwi $mask,$mask,2 # did it carry/borrow?
602 srawi $mask,$mask,31 # mask
609 lwz $d0,0($nonce) # load nonce
621 addc $h0,$h0,$d0 # accumulate nonce
645 addic $d0,$h0,5 # compare to modulus
649 srdi $mask,$d2,2 # did it carry/borrow?
651 sradi $mask,$mask,63 # mask
652 ld $d2,0($nonce) # load nonce
662 $code.=<<___ if (!$LITTLE_ENDIAN);
663 rotldi $d2,$d2,32 # flip nonce words
667 addc $h0,$h0,$d2 # accumulate nonce
674 $code.=<<___ if ($LITTLE_ENDIAN);
675 stw $h0,0($mac) # write result
680 $code.=<<___ if (!$LITTLE_ENDIAN);
682 stwbrx $h0,0,$mac # write result
690 $POP r28,`$FRAME-$SIZE_T*4`($sp)
691 $POP r29,`$FRAME-$SIZE_T*3`($sp)
692 $POP r30,`$FRAME-$SIZE_T*2`($sp)
693 $POP r31,`$FRAME-$SIZE_T*1`($sp)
697 .byte 0,12,4,1,0x80,4,3,0
698 .size .poly1305_emit_fpu,.-.poly1305_emit_fpu
701 # Ugly hack here, because PPC assembler syntax seem to vary too
702 # much from platforms to platform...
708 mflr $len # vvvvvv "distance" between . and 1st data entry
709 addi $len,$len,`64-8` # borrow $len
713 .byte 0,12,0x14,0,0,0,0,0
716 .quad 0x4330000000000000 # 2^(52+0)
717 .quad 0x4530000000000000 # 2^(52+32)
718 .quad 0x4730000000000000 # 2^(52+64)
719 .quad 0x4930000000000000 # 2^(52+96)
720 .quad 0x4b50000000000000 # 2^(52+130)
722 .quad 0x37f4000000000000 # 5/2^130
724 .quad 0x4430000000000000 # 2^(52+16+0)
725 .quad 0x4630000000000000 # 2^(52+16+32)
726 .quad 0x4830000000000000 # 2^(52+16+64)
727 .quad 0x4a30000000000000 # 2^(52+16+96)
728 .quad 0x3e30000000000000 # 2^(52+16+0-96)
729 .quad 0x4030000000000000 # 2^(52+16+32-96)
730 .quad 0x4230000000000000 # 2^(52+16+64-96)
732 .quad 0x0000000000000001 # fpscr: truncate, no exceptions
733 .asciz "Poly1305 for PPC FPU, CRYPTOGAMS by <appro\@openssl.org>"
737 $code =~ s/\`([^\`]*)\`/eval $1/gem;