2 # Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
17 # This module implements Poly1305 hash for PowerPC FPU.
21 # Numbers are cycles per processed byte with poly1305_blocks alone,
22 # and improvement coefficients relative to gcc-generated code.
24 # Freescale e300 9.78/+30%
30 # $output is the last argument if it looks like a file (it has an extension)
31 # $flavour is the first argument if it doesn't look like a file
32 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
33 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
35 if ($flavour =~ /64/) {
42 } elsif ($flavour =~ /32/) {
49 } else { die "nonsense $flavour"; }
51 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 4 : 0;
53 $LWXLE = $LITTLE_ENDIAN ? "lwzx" : "lwbrx";
55 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
56 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
57 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
58 die "can't locate ppc-xlate.pl";
60 open STDOUT,"| $^X $xlate $flavour \"$output\""
61 or die "can't call $xlate: $!";
64 $FRAME=$LOCALS+6*8+18*8;
68 my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
69 my ($in0,$in1,$in2,$in3,$i1,$i2,$i3) = map("r$_",(7..12,6));
71 my ($h0lo,$h0hi,$h1lo,$h1hi,$h2lo,$h2hi,$h3lo,$h3hi,
72 $two0,$two32,$two64,$two96,$two130,$five_two130,
73 $r0lo,$r0hi,$r1lo,$r1hi,$r2lo,$r2hi,
74 $s2lo,$s2hi,$s3lo,$s3hi,
75 $c0lo,$c0hi,$c1lo,$c1hi,$c2lo,$c2hi,$c3lo,$c3hi) = map("f$_",(0..31));
77 my ($r3lo,$r3hi,$s1lo,$s1hi) = ($c0lo,$c0hi,$c1lo,$c1hi);
78 my ($x0,$x1,$x2,$x3) = ($c2lo,$c2hi,$c3lo,$c3hi);
79 my ($y0,$y1,$y2,$y3) = ($c3lo,$c3hi,$c1lo,$c1hi);
85 .globl .poly1305_init_fpu
88 $STU $sp,-$LOCALS($sp) # minimal frame
90 $PUSH $padbit,`$LOCALS+$LRSAVE`($sp)
95 mtlr $padbit # restore lr
97 lfd $two0,8*0($len) # load constants
101 lfd $two130,8*4($len)
102 lfd $five_two130,8*5($len)
104 stfd $two0,8*0($ctx) # initial hash value, biased 0
105 stfd $two32,8*1($ctx)
106 stfd $two64,8*2($ctx)
107 stfd $two96,8*3($ctx)
112 lfd $h3lo,8*13($len) # new fpscr
113 mffs $h3hi # old fpscr
115 stfd $two0,8*4($ctx) # key "template"
116 stfd $two32,8*5($ctx)
117 stfd $two64,8*6($ctx)
118 stfd $two96,8*7($ctx)
123 $LWXLE $in0,0,$inp # load key
124 $LWXLE $in1,$in1,$inp
125 $LWXLE $in2,$in2,$inp
126 $LWXLE $in3,$in3,$inp
128 lis $i1,0xf000 # 0xf0000000
129 ori $i2,$i1,3 # 0xf0000003
130 andc $in0,$in0,$i1 # &=0x0fffffff
131 andc $in1,$in1,$i2 # &=0x0ffffffc
135 stw $in0,`8*4+(4^$LITTLE_ENDIAN)`($ctx) # fill "template"
136 stw $in1,`8*5+(4^$LITTLE_ENDIAN)`($ctx)
137 stw $in2,`8*6+(4^$LITTLE_ENDIAN)`($ctx)
138 stw $in3,`8*7+(4^$LITTLE_ENDIAN)`($ctx)
140 mtfsf 255,$h3lo # fpscr
141 stfd $two0,8*18($ctx) # copy constants to context
142 stfd $two32,8*19($ctx)
143 stfd $two64,8*20($ctx)
144 stfd $two96,8*21($ctx)
145 stfd $two130,8*22($ctx)
146 stfd $five_two130,8*23($ctx)
148 lfd $h0lo,8*4($ctx) # load [biased] key
153 fsub $h0lo,$h0lo,$two0 # r0
154 fsub $h1lo,$h1lo,$two32 # r1
155 fsub $h2lo,$h2lo,$two64 # r2
156 fsub $h3lo,$h3lo,$two96 # r3
158 lfd $two0,8*6($len) # more constants
163 fmul $h1hi,$h1lo,$five_two130 # s1
164 fmul $h2hi,$h2lo,$five_two130 # s2
165 stfd $h3hi,8*15($ctx) # borrow slot for original fpscr
166 fmul $h3hi,$h3lo,$five_two130 # s3
168 fadd $h0hi,$h0lo,$two0
169 stfd $h1hi,8*12($ctx) # put aside for now
170 fadd $h1hi,$h1lo,$two32
171 stfd $h2hi,8*13($ctx)
172 fadd $h2hi,$h2lo,$two64
173 stfd $h3hi,8*14($ctx)
174 fadd $h3hi,$h3lo,$two96
176 fsub $h0hi,$h0hi,$two0
177 fsub $h1hi,$h1hi,$two32
178 fsub $h2hi,$h2hi,$two64
179 fsub $h3hi,$h3hi,$two96
181 lfd $two0,8*10($len) # more constants
182 lfd $two32,8*11($len)
183 lfd $two64,8*12($len)
185 fsub $h0lo,$h0lo,$h0hi
186 fsub $h1lo,$h1lo,$h1hi
187 fsub $h2lo,$h2lo,$h2hi
188 fsub $h3lo,$h3lo,$h3hi
190 stfd $h0hi,8*5($ctx) # r0hi
191 stfd $h1hi,8*7($ctx) # r1hi
192 stfd $h2hi,8*9($ctx) # r2hi
193 stfd $h3hi,8*11($ctx) # r3hi
195 stfd $h0lo,8*4($ctx) # r0lo
196 stfd $h1lo,8*6($ctx) # r1lo
197 stfd $h2lo,8*8($ctx) # r2lo
198 stfd $h3lo,8*10($ctx) # r3lo
200 lfd $h1lo,8*12($ctx) # s1
201 lfd $h2lo,8*13($ctx) # s2
202 lfd $h3lo,8*14($ctx) # s3
203 lfd $h0lo,8*15($ctx) # pull original fpscr
205 fadd $h1hi,$h1lo,$two0
206 fadd $h2hi,$h2lo,$two32
207 fadd $h3hi,$h3lo,$two64
209 fsub $h1hi,$h1hi,$two0
210 fsub $h2hi,$h2hi,$two32
211 fsub $h3hi,$h3hi,$two64
213 fsub $h1lo,$h1lo,$h1hi
214 fsub $h2lo,$h2lo,$h2hi
215 fsub $h3lo,$h3lo,$h3hi
217 stfd $h1hi,8*13($ctx) # s1hi
218 stfd $h2hi,8*15($ctx) # s2hi
219 stfd $h3hi,8*17($ctx) # s3hi
221 stfd $h1lo,8*12($ctx) # s1lo
222 stfd $h2lo,8*14($ctx) # s2lo
223 stfd $h3lo,8*16($ctx) # s3lo
225 mtfsf 255,$h0lo # restore fpscr
231 .byte 0,12,4,1,0x80,0,2,0
232 .size .poly1305_init_fpu,.-.poly1305_init_fpu
234 .globl .poly1305_blocks_fpu
236 .poly1305_blocks_fpu:
240 $STU $sp,-$FRAME($sp)
242 stfd f14,`$FRAME-8*18`($sp)
243 stfd f15,`$FRAME-8*17`($sp)
244 stfd f16,`$FRAME-8*16`($sp)
245 stfd f17,`$FRAME-8*15`($sp)
246 stfd f18,`$FRAME-8*14`($sp)
247 stfd f19,`$FRAME-8*13`($sp)
248 stfd f20,`$FRAME-8*12`($sp)
249 stfd f21,`$FRAME-8*11`($sp)
250 stfd f22,`$FRAME-8*10`($sp)
251 stfd f23,`$FRAME-8*9`($sp)
252 stfd f24,`$FRAME-8*8`($sp)
253 stfd f25,`$FRAME-8*7`($sp)
254 stfd f26,`$FRAME-8*6`($sp)
255 stfd f27,`$FRAME-8*5`($sp)
256 stfd f28,`$FRAME-8*4`($sp)
257 stfd f29,`$FRAME-8*3`($sp)
258 stfd f30,`$FRAME-8*2`($sp)
259 stfd f31,`$FRAME-8*1`($sp)
260 $PUSH r0,`$FRAME+$LRSAVE`($sp)
266 stw r0,`$LOCALS+8*4+(0^$LITTLE_ENDIAN)`($sp)
267 stw $in3,`$LOCALS+8*4+(4^$LITTLE_ENDIAN)`($sp)
269 lfd $two0,8*18($ctx) # load constants
270 lfd $two32,8*19($ctx)
271 lfd $two64,8*20($ctx)
272 lfd $two96,8*21($ctx)
273 lfd $two130,8*22($ctx)
274 lfd $five_two130,8*23($ctx)
276 lfd $h0lo,8*0($ctx) # load [biased] hash value
281 stfd $two0,`$LOCALS+8*0`($sp) # input "template"
282 oris $in3,$padbit,`(1023+52+96)<<4`
283 stfd $two32,`$LOCALS+8*1`($sp)
284 stfd $two64,`$LOCALS+8*2`($sp)
285 stw $in3,`$LOCALS+8*3+(0^$LITTLE_ENDIAN)`($sp)
290 $LWXLE $in0,0,$inp # load input
296 stw $in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp) # fill "template"
297 stw $in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
298 stw $in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
299 stw $in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
301 mffs $x0 # original fpscr
302 lfd $x1,`$LOCALS+8*4`($sp) # new fpscr
303 lfd $r0lo,8*4($ctx) # load key
318 stfd $x0,`$LOCALS+8*4`($sp) # save original fpscr
324 sub $inp,$inp,r0 # conditional rewind
326 lfd $x0,`$LOCALS+8*0`($sp)
327 lfd $x1,`$LOCALS+8*1`($sp)
328 lfd $x2,`$LOCALS+8*2`($sp)
329 lfd $x3,`$LOCALS+8*3`($sp)
331 fsub $h0lo,$h0lo,$two0 # de-bias hash value
332 $LWXLE $in0,0,$inp # modulo-scheduled input load
333 fsub $h1lo,$h1lo,$two32
335 fsub $h2lo,$h2lo,$two64
337 fsub $h3lo,$h3lo,$two96
340 fsub $x0,$x0,$two0 # de-bias input
346 fadd $x0,$x0,$h0lo # accumulate input
347 stw $in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp)
349 stw $in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
351 stw $in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
353 stw $in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
359 fsub $y0,$y0,$two0 # de-bias input
366 sub $inp,$inp,r0 # conditional rewind
368 fadd $h0lo,$h0lo,$y0 # accumulate input
373 ######################################### base 2^48 -> base 2^32
374 fadd $c1lo,$h1lo,$two64
375 $LWXLE $in0,0,$inp # modulo-scheduled input load
376 fadd $c1hi,$h1hi,$two64
378 fadd $c3lo,$h3lo,$two130
380 fadd $c3hi,$h3hi,$two130
382 fadd $c0lo,$h0lo,$two32
384 fadd $c0hi,$h0hi,$two32
385 fadd $c2lo,$h2lo,$two96
386 fadd $c2hi,$h2hi,$two96
388 fsub $c1lo,$c1lo,$two64
389 stw $in0,`$LOCALS+8*0+(4^$LITTLE_ENDIAN)`($sp) # fill "template"
390 fsub $c1hi,$c1hi,$two64
391 stw $in1,`$LOCALS+8*1+(4^$LITTLE_ENDIAN)`($sp)
392 fsub $c3lo,$c3lo,$two130
393 stw $in2,`$LOCALS+8*2+(4^$LITTLE_ENDIAN)`($sp)
394 fsub $c3hi,$c3hi,$two130
395 stw $in3,`$LOCALS+8*3+(4^$LITTLE_ENDIAN)`($sp)
396 fsub $c0lo,$c0lo,$two32
397 fsub $c0hi,$c0hi,$two32
398 fsub $c2lo,$c2lo,$two96
399 fsub $c2hi,$c2hi,$two96
401 fsub $h1lo,$h1lo,$c1lo
402 fsub $h1hi,$h1hi,$c1hi
403 fsub $h3lo,$h3lo,$c3lo
404 fsub $h3hi,$h3hi,$c3hi
405 fsub $h2lo,$h2lo,$c2lo
406 fsub $h2hi,$h2hi,$c2hi
407 fsub $h0lo,$h0lo,$c0lo
408 fsub $h0hi,$h0hi,$c0hi
410 fadd $h1lo,$h1lo,$c0lo
411 fadd $h1hi,$h1hi,$c0hi
412 fadd $h3lo,$h3lo,$c2lo
413 fadd $h3hi,$h3hi,$c2hi
414 fadd $h2lo,$h2lo,$c1lo
415 fadd $h2hi,$h2hi,$c1hi
416 fmadd $h0lo,$c3lo,$five_two130,$h0lo
417 fmadd $h0hi,$c3hi,$five_two130,$h0hi
420 lfd $s1lo,8*12($ctx) # reload constants
437 fmadd $h0lo,$s1lo,$x3,$h0lo
438 fmadd $h0hi,$s1hi,$x3,$h0hi
439 fmadd $h2lo,$s3lo,$x3,$h2lo
440 fmadd $h2hi,$s3hi,$x3,$h2hi
441 fmadd $h1lo,$s2lo,$x3,$h1lo
442 fmadd $h1hi,$s2hi,$x3,$h1hi
443 fmadd $h3lo,$r0lo,$x3,$h3lo
444 fmadd $h3hi,$r0hi,$x3,$h3hi
446 fmadd $h0lo,$s2lo,$x2,$h0lo
447 fmadd $h0hi,$s2hi,$x2,$h0hi
448 fmadd $h2lo,$r0lo,$x2,$h2lo
449 fmadd $h2hi,$r0hi,$x2,$h2hi
450 fmadd $h1lo,$s3lo,$x2,$h1lo
451 fmadd $h1hi,$s3hi,$x2,$h1hi
452 fmadd $h3lo,$r1lo,$x2,$h3lo
453 fmadd $h3hi,$r1hi,$x2,$h3hi
455 fmadd $h0lo,$r0lo,$x0,$h0lo
456 lfd $y0,`$LOCALS+8*0`($sp) # load [biased] input
457 fmadd $h0hi,$r0hi,$x0,$h0hi
458 lfd $y1,`$LOCALS+8*1`($sp)
459 fmadd $h2lo,$r2lo,$x0,$h2lo
460 lfd $y2,`$LOCALS+8*2`($sp)
461 fmadd $h2hi,$r2hi,$x0,$h2hi
462 lfd $y3,`$LOCALS+8*3`($sp)
463 fmadd $h1lo,$r1lo,$x0,$h1lo
464 fmadd $h1hi,$r1hi,$x0,$h1hi
465 fmadd $h3lo,$r3lo,$x0,$h3lo
466 fmadd $h3hi,$r3hi,$x0,$h3hi
470 ######################################### base 2^48 -> base 2^32
471 fadd $c0lo,$h0lo,$two32
472 fadd $c0hi,$h0hi,$two32
473 fadd $c2lo,$h2lo,$two96
474 fadd $c2hi,$h2hi,$two96
475 fadd $c1lo,$h1lo,$two64
476 fadd $c1hi,$h1hi,$two64
477 fadd $c3lo,$h3lo,$two130
478 fadd $c3hi,$h3hi,$two130
480 fsub $c0lo,$c0lo,$two32
481 fsub $c0hi,$c0hi,$two32
482 fsub $c2lo,$c2lo,$two96
483 fsub $c2hi,$c2hi,$two96
484 fsub $c1lo,$c1lo,$two64
485 fsub $c1hi,$c1hi,$two64
486 fsub $c3lo,$c3lo,$two130
487 fsub $c3hi,$c3hi,$two130
489 fsub $h1lo,$h1lo,$c1lo
490 fsub $h1hi,$h1hi,$c1hi
491 fsub $h3lo,$h3lo,$c3lo
492 fsub $h3hi,$h3hi,$c3hi
493 fsub $h2lo,$h2lo,$c2lo
494 fsub $h2hi,$h2hi,$c2hi
495 fsub $h0lo,$h0lo,$c0lo
496 fsub $h0hi,$h0hi,$c0hi
498 fadd $h1lo,$h1lo,$c0lo
499 fadd $h1hi,$h1hi,$c0hi
500 fadd $h3lo,$h3lo,$c2lo
501 fadd $h3hi,$h3hi,$c2hi
502 fadd $h2lo,$h2lo,$c1lo
503 fadd $h2hi,$h2hi,$c1hi
504 fmadd $h0lo,$c3lo,$five_two130,$h0lo
505 fmadd $h0hi,$c3hi,$five_two130,$h0hi
512 lfd $h0lo,`$LOCALS+8*4`($sp) # pull saved fpscr
513 fadd $x1,$x1,$two32 # bias
518 stfd $x1,8*1($ctx) # store [biased] hash value
523 mtfsf 255,$h0lo # restore original fpscr
524 lfd f14,`$FRAME-8*18`($sp)
525 lfd f15,`$FRAME-8*17`($sp)
526 lfd f16,`$FRAME-8*16`($sp)
527 lfd f17,`$FRAME-8*15`($sp)
528 lfd f18,`$FRAME-8*14`($sp)
529 lfd f19,`$FRAME-8*13`($sp)
530 lfd f20,`$FRAME-8*12`($sp)
531 lfd f21,`$FRAME-8*11`($sp)
532 lfd f22,`$FRAME-8*10`($sp)
533 lfd f23,`$FRAME-8*9`($sp)
534 lfd f24,`$FRAME-8*8`($sp)
535 lfd f25,`$FRAME-8*7`($sp)
536 lfd f26,`$FRAME-8*6`($sp)
537 lfd f27,`$FRAME-8*5`($sp)
538 lfd f28,`$FRAME-8*4`($sp)
539 lfd f29,`$FRAME-8*3`($sp)
540 lfd f30,`$FRAME-8*2`($sp)
541 lfd f31,`$FRAME-8*1`($sp)
546 .byte 0,12,4,1,0x80,0,4,0
547 .size .poly1305_blocks_fpu,.-.poly1305_blocks_fpu
550 my ($mac,$nonce)=($inp,$len);
552 my ($h0,$h1,$h2,$h3,$h4, $d0,$d1,$d2,$d3
553 ) = map("r$_",(7..11,28..31));
555 my $FRAME = (6+4)*$SIZE_T;
558 .globl .poly1305_emit_fpu
561 $STU $sp,-$FRAME($sp)
563 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
564 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
565 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
566 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
567 $PUSH r0,`$FRAME+$LRSAVE`($sp)
569 lwz $d0,`8*0+(0^$LITTLE_ENDIAN)`($ctx) # load hash
570 lwz $h0,`8*0+(4^$LITTLE_ENDIAN)`($ctx)
571 lwz $d1,`8*1+(0^$LITTLE_ENDIAN)`($ctx)
572 lwz $h1,`8*1+(4^$LITTLE_ENDIAN)`($ctx)
573 lwz $d2,`8*2+(0^$LITTLE_ENDIAN)`($ctx)
574 lwz $h2,`8*2+(4^$LITTLE_ENDIAN)`($ctx)
575 lwz $d3,`8*3+(0^$LITTLE_ENDIAN)`($ctx)
576 lwz $h3,`8*3+(4^$LITTLE_ENDIAN)`($ctx)
579 andc $d0,$d0,$mask # mask exponent
582 andc $d3,$d3,$mask # can be partially reduced...
585 srwi $padbit,$d3,2 # ... so reduce
598 addic $d0,$h0,5 # compare to modulus
604 srwi $mask,$mask,2 # did it carry/borrow?
606 srawi $mask,$mask,31 # mask
613 lwz $d0,0($nonce) # load nonce
625 addc $h0,$h0,$d0 # accumulate nonce
649 addic $d0,$h0,5 # compare to modulus
653 srdi $mask,$d2,2 # did it carry/borrow?
655 sradi $mask,$mask,63 # mask
656 ld $d2,0($nonce) # load nonce
666 $code.=<<___ if (!$LITTLE_ENDIAN);
667 rotldi $d2,$d2,32 # flip nonce words
671 addc $h0,$h0,$d2 # accumulate nonce
678 $code.=<<___ if ($LITTLE_ENDIAN);
679 stw $h0,0($mac) # write result
684 $code.=<<___ if (!$LITTLE_ENDIAN);
686 stwbrx $h0,0,$mac # write result
694 $POP r28,`$FRAME-$SIZE_T*4`($sp)
695 $POP r29,`$FRAME-$SIZE_T*3`($sp)
696 $POP r30,`$FRAME-$SIZE_T*2`($sp)
697 $POP r31,`$FRAME-$SIZE_T*1`($sp)
701 .byte 0,12,4,1,0x80,4,3,0
702 .size .poly1305_emit_fpu,.-.poly1305_emit_fpu
705 # Ugly hack here, because PPC assembler syntax seem to vary too
706 # much from platforms to platform...
712 mflr $len # vvvvvv "distance" between . and 1st data entry
713 addi $len,$len,`64-8` # borrow $len
717 .byte 0,12,0x14,0,0,0,0,0
720 .quad 0x4330000000000000 # 2^(52+0)
721 .quad 0x4530000000000000 # 2^(52+32)
722 .quad 0x4730000000000000 # 2^(52+64)
723 .quad 0x4930000000000000 # 2^(52+96)
724 .quad 0x4b50000000000000 # 2^(52+130)
726 .quad 0x37f4000000000000 # 5/2^130
728 .quad 0x4430000000000000 # 2^(52+16+0)
729 .quad 0x4630000000000000 # 2^(52+16+32)
730 .quad 0x4830000000000000 # 2^(52+16+64)
731 .quad 0x4a30000000000000 # 2^(52+16+96)
732 .quad 0x3e30000000000000 # 2^(52+16+0-96)
733 .quad 0x4030000000000000 # 2^(52+16+32-96)
734 .quad 0x4230000000000000 # 2^(52+16+64-96)
736 .quad 0x0000000000000001 # fpscr: truncate, no exceptions
737 .asciz "Poly1305 for PPC FPU, CRYPTOGAMS by <appro\@openssl.org>"
741 $code =~ s/\`([^\`]*)\`/eval $1/gem;
743 close STDOUT or die "error closing STDOUT";