2 # Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov, @dot-asm, initially for use in the OpenSSL
12 # project. The module is dual licensed under OpenSSL and CRYPTOGAMS
13 # licenses depending on where you obtain it. For further details see
14 # https://github.com/dot-asm/cryptogams/.
15 # ====================================================================
17 # This module implements Poly1305 hash for PowerPC.
21 # Numbers are cycles per processed byte with poly1305_blocks alone,
22 # and improvement coefficients relative to gcc-generated code.
26 # Freescale e300 14.8/+80% -
28 # PPC970 7.00/+114% 3.51/+205%
29 # POWER7 3.75/+260% 1.93/+100%
33 # Do we need floating-point implementation for PPC? Results presented
34 # in poly1305_ieee754.c are tricky to compare to, because they are for
35 # compiler-generated code. On the other hand it's known that floating-
36 # point performance can be dominated by FPU latency, which means that
37 # there is limit even for ideally optimized (and even vectorized) code.
38 # And this limit is estimated to be higher than above -m64 results. Or
39 # in other words floating-point implementation can be meaningful to
40 # consider only in 32-bit application context. We probably have to
41 # recognize that 32-bit builds are getting less popular on high-end
42 # systems and therefore tend to target embedded ones, which might not
45 # On side note, Power ISA 2.07 enables vector base 2^26 implementation,
46 # and POWER8 might have capacity to break 1.0 cycle per byte barrier...
50 # ... Unfortunately not:-( Estimate was a projection of ARM result,
51 # but ARM has vector multiply-n-add instruction, while PowerISA does
52 # not, not one usable in the context. Improvement is ~40% over -m64
53 # result above and is ~1.43 on little-endian systems.
55 # $output is the last argument if it looks like a file (it has an extension)
56 # $flavour is the first argument if it doesn't look like a file
57 $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
58 $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
60 if ($flavour =~ /64/) {
67 } elsif ($flavour =~ /32/) {
74 } else { die "nonsense $flavour"; }
76 # Define endianness based on flavour
78 $LITTLE_ENDIAN = ($flavour=~/le$/) ? $SIZE_T : 0;
80 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
81 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
82 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
83 die "can't locate ppc-xlate.pl";
85 open STDOUT,"| $^X $xlate $flavour \"$output\""
86 or die "can't call $xlate: $!";
91 my ($ctx,$inp,$len,$padbit) = map("r$_",(3..6));
92 my ($mac,$nonce)=($inp,$len);
99 if ($flavour =~ /64/) {
100 ###############################################################################
101 # base 2^64 implementation
103 my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(7..12,27..31));
106 .globl .poly1305_init_int
110 std r0,0($ctx) # zero hash value
113 stw r0,24($ctx) # clear is_base2_26
118 $code.=<<___ if ($LITTLE_ENDIAN);
119 ld $d0,0($inp) # load key material
122 $code.=<<___ if (!$LITTLE_ENDIAN);
124 lwbrx $d0,0,$inp # load key material
134 lis $h1,0xfff # 0x0fff0000
135 ori $h1,$h1,0xfffc # 0x0ffffffc
136 insrdi $h1,$h1,32,0 # 0x0ffffffc0ffffffc
137 ori $h0,$h1,3 # 0x0ffffffc0fffffff
142 std $d0,32($ctx) # store key
149 .byte 0,12,0x14,0,0,0,2,0
150 .size .poly1305_init_int,.-.poly1305_init_int
152 .globl .poly1305_blocks
159 $STU $sp,-$FRAME($sp)
161 $PUSH r27,`$FRAME-$SIZE_T*5`($sp)
162 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
163 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
164 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
165 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
166 $PUSH r0,`$FRAME+$LRSAVE`($sp)
168 ld $r0,32($ctx) # load key
171 ld $h0,0($ctx) # load hash value
177 add $s1,$s1,$r1 # s1 = r1 + r1>>2
184 $code.=<<___ if ($LITTLE_ENDIAN);
185 ld $t0,0($inp) # load input
188 $code.=<<___ if (!$LITTLE_ENDIAN);
190 lwbrx $t0,0,$inp # load input
202 addc $h0,$h0,$t0 # accumulate input
205 mulld $d0,$h0,$r0 # h0*r0
209 mulld $t0,$h1,$s1 # h1*5*r1
214 mulld $t0,$h0,$r1 # h0*r1
219 mulld $t0,$h1,$r0 # h1*r0
224 mulld $t0,$h2,$s1 # h2*5*r1
225 mulld $t1,$h2,$r0 # h2*r0
229 andc $t0,$d2,$mask # final reduction step
239 std $h0,0($ctx) # store hash value
243 $POP r27,`$FRAME-$SIZE_T*5`($sp)
244 $POP r28,`$FRAME-$SIZE_T*4`($sp)
245 $POP r29,`$FRAME-$SIZE_T*3`($sp)
246 $POP r30,`$FRAME-$SIZE_T*2`($sp)
247 $POP r31,`$FRAME-$SIZE_T*1`($sp)
252 .byte 0,12,4,1,0x80,5,4,0
253 .size .poly1305_blocks,.-.poly1305_blocks
256 my ($h0,$h1,$h2,$h3,$h4,$t0) = map("r$_",(7..12));
259 .globl .poly1305_emit
262 lwz $h0,0($ctx) # load hash value base 2^26
267 lwz r0,24($ctx) # is_base2_26
269 sldi $h1,$h1,26 # base 2^26 -> base 2^64
281 ld $h3,0($ctx) # load hash value base 2^64
286 xor $h0,$h0,$h3 # choose between radixes
296 addic $h3,$h0,5 # compare to modulus
300 srdi $t0,$t0,2 # see if it carried/borrowed
318 addc $h0,$h0,$h3 # accumulate nonce
324 stbu $h0,1($ctx) # write [little-endian] result
364 .byte 0,12,0x14,0,0,0,3,0
365 .size .poly1305_emit,.-.poly1305_emit
368 ###############################################################################
369 # base 2^32 implementation
371 my ($h0,$h1,$h2,$h3,$h4, $r0,$r1,$r2,$r3, $s1,$s2,$s3,
372 $t0,$t1,$t2,$t3, $D0,$D1,$D2,$D3, $d0,$d1,$d2,$d3
373 ) = map("r$_",(7..12,14..31));
376 .globl .poly1305_init_int
380 stw r0,0($ctx) # zero hash value
385 stw r0,24($ctx) # clear is_base2_26
390 $code.=<<___ if ($LITTLE_ENDIAN);
391 lw $h0,0($inp) # load key material
396 $code.=<<___ if (!$LITTLE_ENDIAN);
398 lwbrx $h0,0,$inp # load key material
406 lis $mask,0xf000 # 0xf0000000
408 andc $r0,$r0,$mask # 0x0ffffffc
415 stw $h0,32($ctx) # store key
424 .byte 0,12,0x14,0,0,0,2,0
425 .size .poly1305_init_int,.-.poly1305_init_int
427 .globl .poly1305_blocks
434 $STU $sp,-$FRAME($sp)
436 $PUSH r14,`$FRAME-$SIZE_T*18`($sp)
437 $PUSH r15,`$FRAME-$SIZE_T*17`($sp)
438 $PUSH r16,`$FRAME-$SIZE_T*16`($sp)
439 $PUSH r17,`$FRAME-$SIZE_T*15`($sp)
440 $PUSH r18,`$FRAME-$SIZE_T*14`($sp)
441 $PUSH r19,`$FRAME-$SIZE_T*13`($sp)
442 $PUSH r20,`$FRAME-$SIZE_T*12`($sp)
443 $PUSH r21,`$FRAME-$SIZE_T*11`($sp)
444 $PUSH r22,`$FRAME-$SIZE_T*10`($sp)
445 $PUSH r23,`$FRAME-$SIZE_T*9`($sp)
446 $PUSH r24,`$FRAME-$SIZE_T*8`($sp)
447 $PUSH r25,`$FRAME-$SIZE_T*7`($sp)
448 $PUSH r26,`$FRAME-$SIZE_T*6`($sp)
449 $PUSH r27,`$FRAME-$SIZE_T*5`($sp)
450 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
451 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
452 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
453 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
454 $PUSH r0,`$FRAME+$LRSAVE`($sp)
456 lwz $r0,32($ctx) # load key
461 lwz $h0,0($ctx) # load hash value
470 add $s1,$s1,$r1 # si = ri + ri>>2
480 $code.=<<___ if ($LITTLE_ENDIAN);
481 lwz $d0,0($inp) # load input
486 $code.=<<___ if (!$LITTLE_ENDIAN);
488 lwbrx $d0,0,$inp # load input
498 addc $h0,$h0,$d0 # accumulate input
502 mullw $d0,$h0,$r0 # h0*r0
505 mullw $d1,$h0,$r1 # h0*r1
508 mullw $d2,$h0,$r2 # h0*r2
514 mullw $d3,$h0,$r3 # h0*r3
517 mullw $t0,$h1,$s3 # h1*s3
520 mullw $t2,$h1,$r0 # h1*r0
525 mullw $t0,$h1,$r1 # h1*r1
530 mullw $t2,$h1,$r2 # h1*r2
535 mullw $t0,$h2,$s2 # h2*s2
540 mullw $t2,$h2,$s3 # h2*s3
545 mullw $t0,$h2,$r0 # h2*r0
550 mullw $t2,$h2,$r1 # h2*r1
555 mullw $t0,$h3,$s1 # h3*s1
560 mullw $t2,$h3,$s2 # h3*s2
565 mullw $t0,$h3,$s3 # h3*s3
570 mullw $t2,$h3,$r0 # h3*r0
575 mullw $t0,$h4,$s1 # h4*s1
580 mullw $t1,$h4,$s2 # h4*s2
585 mullw $t2,$h4,$s3 # h4*s3
589 mullw $h4,$h4,$r0 # h4*r0
596 andc $D0,$h4,$mask # final reduction step
608 stw $h0,0($ctx) # store hash value
614 $POP r14,`$FRAME-$SIZE_T*18`($sp)
615 $POP r15,`$FRAME-$SIZE_T*17`($sp)
616 $POP r16,`$FRAME-$SIZE_T*16`($sp)
617 $POP r17,`$FRAME-$SIZE_T*15`($sp)
618 $POP r18,`$FRAME-$SIZE_T*14`($sp)
619 $POP r19,`$FRAME-$SIZE_T*13`($sp)
620 $POP r20,`$FRAME-$SIZE_T*12`($sp)
621 $POP r21,`$FRAME-$SIZE_T*11`($sp)
622 $POP r22,`$FRAME-$SIZE_T*10`($sp)
623 $POP r23,`$FRAME-$SIZE_T*9`($sp)
624 $POP r24,`$FRAME-$SIZE_T*8`($sp)
625 $POP r25,`$FRAME-$SIZE_T*7`($sp)
626 $POP r26,`$FRAME-$SIZE_T*6`($sp)
627 $POP r27,`$FRAME-$SIZE_T*5`($sp)
628 $POP r28,`$FRAME-$SIZE_T*4`($sp)
629 $POP r29,`$FRAME-$SIZE_T*3`($sp)
630 $POP r30,`$FRAME-$SIZE_T*2`($sp)
631 $POP r31,`$FRAME-$SIZE_T*1`($sp)
636 .byte 0,12,4,1,0x80,18,4,0
637 .size .poly1305_blocks,.-.poly1305_blocks
640 my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(6..12));
643 .globl .poly1305_emit
646 lwz r0,24($ctx) # is_base2_26
647 lwz $h0,0($ctx) # load hash value
655 slwi $t0,$h1,26 # base 2^26 -> base 2^32
670 addic r0,$h0,5 # compare to modulus
676 srwi r0,r0,2 # see if it carried/borrowed
689 addc $h0,$h0,r0 # accumulate nonce
697 stbu $h0,1($ctx) # write [little-endian] result
735 .byte 0,12,0x14,0,0,0,3,0
736 .size .poly1305_emit,.-.poly1305_emit
740 ########################################################################
741 # PowerISA 2.07/VSX section #
742 ########################################################################
744 my $LOCALS= 6*$SIZE_T;
745 my $VSXFRAME = $LOCALS + 6*$SIZE_T;
746 $VSXFRAME += 128; # local variables
747 $VSXFRAME += 13*16; # v20-v31 offload
749 my $BIG_ENDIAN = ($flavour !~ /le/) ? 4 : 0;
751 ########################################################################
752 # Layout of opaque area is following:
754 # unsigned __int32 h[5]; # current hash value base 2^26
755 # unsigned __int32 pad;
756 # unsigned __int32 is_base2_26, pad;
757 # unsigned __int64 r[2]; # key value base 2^64
758 # struct { unsigned __int32 r^2, r^4, r^1, r^3; } r[9];
760 # where r^n are base 2^26 digits of powers of multiplier key. There are
761 # 5 digits, but last four are interleaved with multiples of 5, totalling
762 # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4. Order of
763 # powers is as they appear in register, not memory.
765 my ($H0, $H1, $H2, $H3, $H4) = map("v$_",(0..4));
766 my ($I0, $I1, $I2, $I3, $I4) = map("v$_",(5..9));
767 my ($R0, $R1, $S1, $R2, $S2) = map("v$_",(10..14));
768 my ($R3, $S3, $R4, $S4) = ($R1, $S1, $R2, $S2);
769 my ($ACC0, $ACC1, $ACC2, $ACC3, $ACC4) = map("v$_",(15..19));
770 my ($T0, $T1, $T2, $T3, $T4) = map("v$_",(20..24));
771 my ($_26,$_4,$_40,$_14,$mask26,$padbits,$I2perm) = map("v$_",(25..31));
772 my ($x00,$x60,$x70,$x10,$x20,$x30,$x40,$x50) = (0, map("r$_",(7,8,27..31)));
773 my ($ctx_,$_ctx,$const) = map("r$_",(10..12));
775 if ($flavour =~ /64/) {
776 ###############################################################################
777 # setup phase of poly1305_blocks_vsx is different on 32- and 64-bit platforms,
778 # but the base 2^26 computational part is same...
780 my ($h0,$h1,$h2,$d0,$d1,$d2, $r0,$r1,$s1, $t0,$t1) = map("r$_",(6..11,27..31));
784 .globl .poly1305_blocks_vsx
786 .poly1305_blocks_vsx:
787 lwz r7,24($ctx) # is_base2_26
789 bge __poly1305_blocks_vsx
791 neg r0,r7 # is_base2_26 as mask
792 lwz r7,0($ctx) # load hash base 2^26
798 sldi r8,r8,26 # base 2^26 -> base 2^64
810 ld r8,0($ctx) # load hash base 2^64
814 xor r7,r7,r8 # select between radixes
825 std r7,0($ctx) # store hash base 2^64
828 stw r0,24($ctx) # clear is_base2_26
832 .byte 0,12,0x14,0,0,0,4,0
833 .size .poly1305_blocks_vsx,.-.poly1305_blocks_vsx
837 mulld $d0,$h0,$r0 # h0*r0
840 mulld $t0,$h1,$s1 # h1*5*r1
845 mulld $t0,$h0,$r1 # h0*r1
850 mulld $t0,$h1,$r0 # h1*r0
855 mulld $t0,$h2,$s1 # h2*5*r1
856 mulld $t1,$h2,$r0 # h2*r0
860 andc $t0,$d2,$mask # final reduction step
870 .byte 0,12,0x14,0,0,0,0,0
871 .size __poly1305_mul,.-__poly1305_mul
882 add $d0,$d0,$d1 # * 5
888 add $d0,$d0,$d2 # * 5
895 add $d0,$d0,$d1 # * 5
901 add $d0,$d0,$d2 # * 5
906 .byte 0,12,0x14,0,0,0,0,0
907 .size __poly1305_splat,.-__poly1305_splat
910 __poly1305_blocks_vsx:
911 $STU $sp,-$VSXFRAME($sp)
913 li r10,`15+$LOCALS+128`
914 li r11,`31+$LOCALS+128`
938 stw r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
940 mtspr 256,r12 # preserve all AltiVec registers
941 $PUSH r27,`$VSXFRAME-$SIZE_T*5`($sp)
942 $PUSH r28,`$VSXFRAME-$SIZE_T*4`($sp)
943 $PUSH r29,`$VSXFRAME-$SIZE_T*3`($sp)
944 $PUSH r30,`$VSXFRAME-$SIZE_T*2`($sp)
945 $PUSH r31,`$VSXFRAME-$SIZE_T*1`($sp)
946 $PUSH r0,`$VSXFRAME+$LRSAVE`($sp)
955 lvx_u $mask26,$x00,$const
956 lvx_u $_26,$x10,$const
957 lvx_u $_40,$x20,$const
958 lvx_u $I2perm,$x30,$const
959 lvx_u $padbits,$x40,$const
961 cmplwi r7,0 # is_base2_26?
964 ld $r0,32($ctx) # load key base 2^64
968 add $s1,$s1,$r1 # s1 = r1 + r1>>2
970 mr $h0,$r0 # "calculate" r^1
973 addi $t1,$ctx,`48+(12^$BIG_ENDIAN)`
976 bl __poly1305_mul # calculate r^2
977 addi $t1,$ctx,`48+(4^$BIG_ENDIAN)`
980 bl __poly1305_mul # calculate r^3
981 addi $t1,$ctx,`48+(8^$BIG_ENDIAN)`
984 bl __poly1305_mul # calculate r^4
985 addi $t1,$ctx,`48+(0^$BIG_ENDIAN)`
988 ld $h0,0($ctx) # load hash
992 extrdi $d0,$h0,26,38 # base 2^64 -> base 2^26
1006 ###############################################################################
1007 # 32-bit initialization
1009 my ($h0,$h1,$h2,$h3,$h4,$t0,$t1) = map("r$_",(7..11,0,12));
1010 my ($R3,$S3,$R4,$S4)=($I1,$I2,$I3,$I4);
1013 .globl .poly1305_blocks_vsx
1015 .poly1305_blocks_vsx:
1016 lwz r7,24($ctx) # is_base2_26
1018 bge __poly1305_blocks_vsx
1020 beq Lpoly1305_blocks
1022 lwz $h0,0($ctx) # load hash
1028 slwi $t0,$h1,26 # base 2^26 -> base 2^32
1043 stw $h0,0($ctx) # store hash base 2^32
1048 stw $t0,24($ctx) # clear is_base2_26
1052 .byte 0,12,0x14,0,0,0,4,0
1053 .size .poly1305_blocks_vsx,.-.poly1305_blocks_vsx
1057 vmulouw $ACC0,$H0,$R0
1058 vmulouw $ACC1,$H1,$R0
1059 vmulouw $ACC2,$H2,$R0
1060 vmulouw $ACC3,$H3,$R0
1061 vmulouw $ACC4,$H4,$R0
1064 vaddudm $ACC0,$ACC0,$T0
1066 vaddudm $ACC1,$ACC1,$T0
1068 vaddudm $ACC2,$ACC2,$T0
1070 vaddudm $ACC3,$ACC3,$T0
1072 vaddudm $ACC4,$ACC4,$T0
1075 vaddudm $ACC0,$ACC0,$T0
1077 vaddudm $ACC1,$ACC1,$T0
1079 vaddudm $ACC2,$ACC2,$T0
1081 vaddudm $ACC3,$ACC3,$T0
1083 vaddudm $ACC4,$ACC4,$T0
1086 vaddudm $ACC0,$ACC0,$T0
1088 vaddudm $ACC1,$ACC1,$T0
1090 vaddudm $ACC2,$ACC2,$T0
1092 vaddudm $ACC3,$ACC3,$T0
1094 vaddudm $ACC4,$ACC4,$T0
1097 vaddudm $ACC0,$ACC0,$T0
1099 vaddudm $ACC1,$ACC1,$T0
1101 vaddudm $ACC2,$ACC2,$T0
1103 vaddudm $ACC3,$ACC3,$T0
1105 vaddudm $ACC4,$ACC4,$T0
1107 ################################################################
1113 vand $H3,$ACC3,$mask26
1114 vand $H0,$ACC0,$mask26
1115 vaddudm $H4,$H4,$ACC4 # h3 -> h4
1116 vaddudm $H1,$H1,$ACC1 # h0 -> h1
1120 vand $H4,$H4,$mask26
1121 vand $H1,$H1,$mask26
1122 vaddudm $H0,$H0,$ACC4
1123 vaddudm $H2,$ACC2,$ACC1 # h1 -> h2
1125 vsld $ACC4,$ACC4,$T0 # <<2
1127 vand $H2,$H2,$mask26
1128 vaddudm $H0,$H0,$ACC4 # h4 -> h0
1129 vaddudm $H3,$H3,$ACC2 # h2 -> h3
1133 vand $H0,$H0,$mask26
1134 vand $H3,$H3,$mask26
1135 vaddudm $H1,$H1,$ACC0 # h0 -> h1
1136 vaddudm $H4,$H4,$ACC3 # h3 -> h4
1140 .byte 0,12,0x14,0,0,0,0,0
1141 .size __poly1305_mul,.-__poly1305_mul
1144 __poly1305_blocks_vsx:
1145 $STU $sp,-$VSXFRAME($sp)
1147 li r10,`15+$LOCALS+128`
1148 li r11,`31+$LOCALS+128`
1172 stw r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# save vrsave
1174 mtspr 256,r12 # preserve all AltiVec registers
1175 $PUSH r27,`$VSXFRAME-$SIZE_T*5`($sp)
1176 $PUSH r28,`$VSXFRAME-$SIZE_T*4`($sp)
1177 $PUSH r29,`$VSXFRAME-$SIZE_T*3`($sp)
1178 $PUSH r30,`$VSXFRAME-$SIZE_T*2`($sp)
1179 $PUSH r31,`$VSXFRAME-$SIZE_T*1`($sp)
1180 $PUSH r0,`$VSXFRAME+$LRSAVE`($sp)
1189 lvx_u $mask26,$x00,$const
1190 lvx_u $_26,$x10,$const
1191 lvx_u $_40,$x20,$const
1192 lvx_u $I2perm,$x30,$const
1193 lvx_u $padbits,$x40,$const
1195 cmplwi r7,0 # is_base2_26?
1198 lwz $h1,32($ctx) # load key base 2^32
1203 extrwi $h0,$h1,26,6 # base 2^32 -> base 2^26
1236 bl __poly1305_mul # r^1:- * r^1:-
1238 vpermdi $R0,$H0,$R0,0b00
1239 vpermdi $R1,$H1,$R1,0b00
1240 vpermdi $R2,$H2,$R2,0b00
1241 vpermdi $R3,$H3,$R3,0b00
1242 vpermdi $R4,$H4,$R4,0b00
1243 vpermdi $H0,$H0,$H0,0b00
1244 vpermdi $H1,$H1,$H1,0b00
1245 vpermdi $H2,$H2,$H2,0b00
1246 vpermdi $H3,$H3,$H3,0b00
1247 vpermdi $H4,$H4,$H4,0b00
1248 vsld $S1,$R1,$T0 # <<2
1257 bl __poly1305_mul # r^2:r^2 * r^2:r^1
1260 lwz $h1,0($ctx) # load hash
1266 vmrgow $R0,$R0,$H0 # r^2:r^4:r^1:r^3
1271 vslw $S1,$R1,$T0 # <<2
1280 stvx_u $R0,$x30,$ctx
1281 stvx_u $R1,$x40,$ctx
1282 stvx_u $S1,$x50,$ctx
1290 extrwi $h0,$h1,26,6 # base 2^32 -> base 2^26
1308 stw r0,24($ctx) # set is_base2_26
1317 lvwzx_u $H0,$x00,$ctx
1318 lvwzx_u $H1,$x10,$ctx
1319 lvwzx_u $H2,$x20,$ctx
1320 lvwzx_u $H3,$x30,$ctx
1321 lvwzx_u $H4,$x40,$ctx
1331 addi $ctx_,$ctx,64 # &ctx->r[1]
1332 addi $_ctx,$sp,`$LOCALS+15` # &ctx->r[1], r^2:r^4 shadow
1334 vxor $T0,$T0,$T0 # ensure second half is zero
1335 vpermdi $H0,$H0,$T0,0b00
1336 vpermdi $H1,$H1,$T0,0b00
1337 vpermdi $H2,$H2,$T0,0b00
1338 vpermdi $H3,$H3,$T0,0b00
1339 vpermdi $H4,$H4,$T0,0b00
1341 be?lvx_u $_4,$x50,$const # byte swap mask
1342 lvx_u $T1,$x00,$inp # load first input block
1346 be?vperm $T1,$T1,$T1,$_4
1347 be?vperm $T2,$T2,$T2,$_4
1348 be?vperm $T3,$T3,$T3,$_4
1349 be?vperm $T4,$T4,$T4,$_4
1351 vpermdi $I0,$T1,$T2,0b00 # smash input to base 2^26
1353 vperm $I2,$T1,$T2,$I2perm # 0x...0e0f0001...1e1f1011
1355 vpermdi $I3,$T1,$T2,0b11
1361 vand $I0,$I0,$mask26
1362 vand $I1,$I1,$mask26
1363 vand $I2,$I2,$mask26
1364 vand $I3,$I3,$mask26
1366 vpermdi $T1,$T3,$T4,0b00
1367 vperm $T2,$T3,$T4,$I2perm # 0x...0e0f0001...1e1f1011
1368 vpermdi $T3,$T3,$T4,0b11
1374 vand $T1,$T1,$mask26
1375 vand $T0,$T0,$mask26
1376 vand $T2,$T2,$mask26
1377 vand $T3,$T3,$mask26
1379 # inp[2]:inp[0]:inp[3]:inp[1]
1385 vor $I4,$I4,$padbits
1387 lvx_splt $R0,$x30,$ctx # taking lvx_vsplt out of loop
1388 lvx_splt $R1,$x00,$ctx_ # gives ~8% improvement
1389 lvx_splt $S1,$x10,$ctx_
1390 lvx_splt $R2,$x20,$ctx_
1391 lvx_splt $S2,$x30,$ctx_
1392 lvx_splt $T1,$x40,$ctx_
1393 lvx_splt $T2,$x50,$ctx_
1394 lvx_splt $T3,$x60,$ctx_
1395 lvx_splt $T4,$x70,$ctx_
1406 addi $const,$const,0x50
1414 ################################################################
1415 ## ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
1416 ## ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
1417 ## \___________________/
1419 ## Note that we start with inp[2:3]*r^2. This is because it
1420 ## doesn't depend on reduction in previous iteration.
1421 ################################################################
1422 ## d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
1423 ## d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
1424 ## d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
1425 ## d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
1426 ## d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
1428 vmuleuw $ACC0,$I0,$R0
1429 vmuleuw $ACC1,$I0,$R1
1430 vmuleuw $ACC2,$I0,$R2
1431 vmuleuw $ACC3,$I1,$R2
1434 vaddudm $ACC1,$ACC1,$T0
1436 vaddudm $ACC2,$ACC2,$T0
1437 vmuleuw $ACC4,$I2,$R2
1439 vaddudm $ACC0,$ACC0,$T0
1441 vaddudm $ACC3,$ACC3,$T0
1444 vaddudm $ACC4,$ACC4,$T0
1454 vaddudm $ACC0,$ACC0,$T0
1456 vaddudm $ACC1,$ACC1,$T0
1458 vaddudm $ACC2,$ACC2,$T0
1460 vaddudm $ACC3,$ACC3,$T0
1463 vaddudm $ACC4,$ACC4,$T0
1467 vaddudm $ACC0,$ACC0,$T0
1469 vaddudm $ACC1,$ACC1,$T0
1471 vaddudm $ACC2,$ACC2,$T0
1473 vaddudm $ACC3,$ACC3,$T0
1475 vaddudm $ACC4,$ACC4,$T0
1477 be?lvx_u $_4,$x00,$const # byte swap mask
1478 lvx_u $T1,$x00,$inp # load next input block
1482 be?vperm $T1,$T1,$T1,$_4
1483 be?vperm $T2,$T2,$T2,$_4
1484 be?vperm $T3,$T3,$T3,$_4
1485 be?vperm $T4,$T4,$T4,$_4
1488 vaddudm $ACC0,$ACC0,$T0
1490 vaddudm $ACC1,$ACC1,$T0
1492 vaddudm $ACC2,$ACC2,$T0
1494 vaddudm $ACC3,$ACC3,$T0
1496 vaddudm $ACC4,$ACC4,$T0
1498 vpermdi $I0,$T1,$T2,0b00 # smash input to base 2^26
1500 vperm $I2,$T1,$T2,$I2perm # 0x...0e0f0001...1e1f1011
1501 vpermdi $I3,$T1,$T2,0b11
1503 # (hash + inp[0:1]) * r^4
1505 vaddudm $ACC0,$ACC0,$T0
1507 vaddudm $ACC1,$ACC1,$T0
1509 vaddudm $ACC2,$ACC2,$T0
1511 vaddudm $ACC3,$ACC3,$T0
1513 vaddudm $ACC4,$ACC4,$T0
1515 vpermdi $T1,$T3,$T4,0b00
1516 vperm $T2,$T3,$T4,$I2perm # 0x...0e0f0001...1e1f1011
1517 vpermdi $T3,$T3,$T4,0b11
1520 vaddudm $ACC0,$ACC0,$T0
1522 vaddudm $ACC1,$ACC1,$T0
1524 vaddudm $ACC2,$ACC2,$T0
1526 vaddudm $ACC3,$ACC3,$T0
1529 vaddudm $ACC4,$ACC4,$T0
1538 vaddudm $ACC0,$ACC0,$T0
1540 vaddudm $ACC1,$ACC1,$T0
1542 vaddudm $ACC2,$ACC2,$T0
1544 vaddudm $ACC3,$ACC3,$T0
1547 vaddudm $ACC4,$ACC4,$T0
1550 vand $I0,$I0,$mask26
1551 vand $I1,$I1,$mask26
1552 vand $I2,$I2,$mask26
1553 vand $I3,$I3,$mask26
1556 vaddudm $ACC0,$ACC0,$T0
1558 vaddudm $ACC1,$ACC1,$T0
1560 vaddudm $ACC2,$ACC2,$T0
1562 vaddudm $ACC3,$ACC3,$T0
1564 vaddudm $ACC4,$ACC4,$T0
1572 vaddudm $ACC0,$ACC0,$T0
1574 vaddudm $ACC1,$ACC1,$T0
1576 vaddudm $ACC2,$ACC2,$T0
1578 vaddudm $ACC3,$ACC3,$T0
1580 vaddudm $ACC4,$ACC4,$T0
1582 vand $T1,$T1,$mask26
1583 vand $_4,$_4,$mask26
1584 vand $T2,$T2,$mask26
1585 vand $T3,$T3,$mask26
1587 ################################################################
1588 # lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
1594 vand $H3,$ACC3,$mask26
1595 vand $H0,$ACC0,$mask26
1596 vaddudm $H4,$H4,$ACC4 # h3 -> h4
1597 vaddudm $H1,$H1,$ACC1 # h0 -> h1
1604 vor $I4,$I4,$padbits
1608 vand $H4,$H4,$mask26
1609 vand $H1,$H1,$mask26
1610 vaddudm $H0,$H0,$ACC4
1611 vaddudm $H2,$ACC2,$ACC1 # h1 -> h2
1613 vsld $ACC4,$ACC4,$T0 # <<2
1615 vand $H2,$H2,$mask26
1616 vaddudm $H0,$H0,$ACC4 # h4 -> h0
1617 vaddudm $H3,$H3,$ACC2 # h2 -> h3
1621 vand $H0,$H0,$mask26
1622 vand $H3,$H3,$mask26
1623 vaddudm $H1,$H1,$ACC0 # h0 -> h1
1624 vaddudm $H4,$H4,$ACC3 # h3 -> h4
1630 andi. $len,$len,0x30
1633 lvx_u $R0,$x30,$ctx # load all powers
1634 lvx_u $R1,$x00,$ctx_
1635 lvx_u $S1,$x10,$ctx_
1636 lvx_u $R2,$x20,$ctx_
1637 lvx_u $S2,$x30,$ctx_
1640 vmuleuw $ACC0,$I0,$R0
1641 vmuleuw $ACC1,$I1,$R0
1642 vmuleuw $ACC2,$I2,$R0
1643 vmuleuw $ACC3,$I3,$R0
1644 vmuleuw $ACC4,$I4,$R0
1647 vaddudm $ACC0,$ACC0,$T0
1649 vaddudm $ACC1,$ACC1,$T0
1651 vaddudm $ACC2,$ACC2,$T0
1653 vaddudm $ACC3,$ACC3,$T0
1654 lvx_u $S3,$x50,$ctx_
1656 vaddudm $ACC4,$ACC4,$T0
1657 lvx_u $R3,$x40,$ctx_
1666 vaddudm $ACC0,$ACC0,$T0
1668 vaddudm $ACC1,$ACC1,$T0
1670 vaddudm $ACC2,$ACC2,$T0
1672 vaddudm $ACC3,$ACC3,$T0
1673 lvx_u $S4,$x70,$ctx_
1675 vaddudm $ACC4,$ACC4,$T0
1676 lvx_u $R4,$x60,$ctx_
1679 vaddudm $ACC0,$ACC0,$T0
1681 vaddudm $ACC1,$ACC1,$T0
1683 vaddudm $ACC2,$ACC2,$T0
1685 vaddudm $ACC3,$ACC3,$T0
1687 vaddudm $ACC4,$ACC4,$T0
1690 vaddudm $ACC0,$ACC0,$T0
1692 vaddudm $ACC1,$ACC1,$T0
1694 vaddudm $ACC2,$ACC2,$T0
1696 vaddudm $ACC3,$ACC3,$T0
1698 vaddudm $ACC4,$ACC4,$T0
1700 # (hash + inp[0:1]) * r^4
1702 vaddudm $ACC0,$ACC0,$T0
1704 vaddudm $ACC1,$ACC1,$T0
1706 vaddudm $ACC2,$ACC2,$T0
1708 vaddudm $ACC3,$ACC3,$T0
1710 vaddudm $ACC4,$ACC4,$T0
1713 vaddudm $ACC0,$ACC0,$T0
1715 vaddudm $ACC1,$ACC1,$T0
1717 vaddudm $ACC2,$ACC2,$T0
1719 vaddudm $ACC3,$ACC3,$T0
1720 lvx_u $S1,$x10,$ctx_
1722 vaddudm $ACC4,$ACC4,$T0
1723 lvx_u $R1,$x00,$ctx_
1726 vaddudm $ACC0,$ACC0,$T0
1728 vaddudm $ACC1,$ACC1,$T0
1730 vaddudm $ACC2,$ACC2,$T0
1732 vaddudm $ACC3,$ACC3,$T0
1733 lvx_u $S2,$x30,$ctx_
1735 vaddudm $ACC4,$ACC4,$T0
1736 lvx_u $R2,$x20,$ctx_
1739 vaddudm $ACC0,$ACC0,$T0
1741 vaddudm $ACC1,$ACC1,$T0
1743 vaddudm $ACC2,$ACC2,$T0
1745 vaddudm $ACC3,$ACC3,$T0
1747 vaddudm $ACC4,$ACC4,$T0
1750 vaddudm $ACC0,$ACC0,$T0
1752 vaddudm $ACC1,$ACC1,$T0
1754 vaddudm $ACC2,$ACC2,$T0
1756 vaddudm $ACC3,$ACC3,$T0
1758 vaddudm $ACC4,$ACC4,$T0
1760 ################################################################
1761 # horizontal addition
1763 vpermdi $H0,$ACC0,$ACC0,0b10
1764 vpermdi $H1,$ACC1,$ACC1,0b10
1765 vpermdi $H2,$ACC2,$ACC2,0b10
1766 vpermdi $H3,$ACC3,$ACC3,0b10
1767 vpermdi $H4,$ACC4,$ACC4,0b10
1768 vaddudm $ACC0,$ACC0,$H0
1769 vaddudm $ACC1,$ACC1,$H1
1770 vaddudm $ACC2,$ACC2,$H2
1771 vaddudm $ACC3,$ACC3,$H3
1772 vaddudm $ACC4,$ACC4,$H4
1774 ################################################################
1780 vand $H3,$ACC3,$mask26
1781 vand $H0,$ACC0,$mask26
1782 vaddudm $H4,$H4,$ACC4 # h3 -> h4
1783 vaddudm $H1,$H1,$ACC1 # h0 -> h1
1787 vand $H4,$H4,$mask26
1788 vand $H1,$H1,$mask26
1789 vaddudm $H0,$H0,$ACC4
1790 vaddudm $H2,$ACC2,$ACC1 # h1 -> h2
1792 vsld $ACC4,$ACC4,$T0 # <<2
1794 vand $H2,$H2,$mask26
1795 vaddudm $H0,$H0,$ACC4 # h4 -> h0
1796 vaddudm $H3,$H3,$ACC2 # h2 -> h3
1800 vand $H0,$H0,$mask26
1801 vand $H3,$H3,$mask26
1802 vaddudm $H1,$H1,$ACC0 # h0 -> h1
1803 vaddudm $H4,$H4,$ACC3 # h3 -> h4
1809 be?lvx_u $_4,$x00,$const # byte swap mask
1810 lvx_u $T1,$x00,$inp # load last partial input block
1814 be?vperm $T1,$T1,$T1,$_4
1815 be?vperm $T2,$T2,$T2,$_4
1816 be?vperm $T3,$T3,$T3,$_4
1817 be?vperm $T4,$T4,$T4,$_4
1819 vpermdi $I0,$T1,$T2,0b00 # smash input to base 2^26
1821 vperm $I2,$T1,$T2,$I2perm # 0x...0e0f0001...1e1f1011
1822 vpermdi $I3,$T1,$T2,0b11
1828 vand $I0,$I0,$mask26
1829 vand $I1,$I1,$mask26
1830 vand $I2,$I2,$mask26
1831 vand $I3,$I3,$mask26
1833 vpermdi $T0,$T3,$T4,0b00
1834 vperm $T1,$T3,$T4,$I2perm # 0x...0e0f0001...1e1f1011
1835 vpermdi $T2,$T3,$T4,0b11
1844 vand $T0,$T0,$mask26
1845 vand $T3,$T3,$mask26
1846 vand $T1,$T1,$mask26
1847 vand $T2,$T2,$mask26
1849 # inp[2]:inp[0]:inp[3]:inp[1]
1855 vor $I4,$I4,$padbits
1857 vperm $H0,$H0,$H0,$ACC0 # move hash to right lane
1858 vand $I0,$I0, $ACC1 # mask redundant input lane[s]
1859 vperm $H1,$H1,$H1,$ACC0
1861 vperm $H2,$H2,$H2,$ACC0
1863 vperm $H3,$H3,$H3,$ACC0
1865 vperm $H4,$H4,$H4,$ACC0
1868 vaddudm $I0,$I0,$H0 # accumulate hash
1869 vxor $H0,$H0,$H0 # wipe hash value
1884 $POP r0,`$VSXFRAME+$LRSAVE`($sp)
1889 stvwx_u $H0,$x00,$ctx # store hash
1890 stvwx_u $H1,$x10,$ctx
1891 stvwx_u $H2,$x20,$ctx
1892 stvwx_u $H3,$x30,$ctx
1893 stvwx_u $H4,$x40,$ctx
1895 lwz r12,`$VSXFRAME-$SIZE_T*5-4`($sp)# pull vrsave
1897 li r10,`15+$LOCALS+128`
1898 li r11,`31+$LOCALS+128`
1899 mtspr 256,r12 # restore vrsave
1922 $POP r27,`$VSXFRAME-$SIZE_T*5`($sp)
1923 $POP r28,`$VSXFRAME-$SIZE_T*4`($sp)
1924 $POP r29,`$VSXFRAME-$SIZE_T*3`($sp)
1925 $POP r30,`$VSXFRAME-$SIZE_T*2`($sp)
1926 $POP r31,`$VSXFRAME-$SIZE_T*1`($sp)
1927 addi $sp,$sp,$VSXFRAME
1930 .byte 0,12,0x04,1,0x80,5,4,0
1932 .size __poly1305_blocks_vsx,.-__poly1305_blocks_vsx
1938 mflr $const # vvvvvv "distance" between . and 1st data entry
1939 addi $const,$const,`64-8`
1943 .byte 0,12,0x14,0,0,0,0,0
1946 .quad 0x0000000003ffffff,0x0000000003ffffff # mask26
1947 .quad 0x000000000000001a,0x000000000000001a # _26
1948 .quad 0x0000000000000028,0x0000000000000028 # _40
1949 .quad 0x000000000e0f0001,0x000000001e1f1011 # I2perm
1950 .quad 0x0100000001000000,0x0100000001000000 # padbits
1951 .quad 0x0706050403020100,0x0f0e0d0c0b0a0908 # byte swap for big-endian
1953 .quad 0x0000000000000000,0x0000000004050607 # magic tail masks
1954 .quad 0x0405060700000000,0x0000000000000000
1955 .quad 0x0000000000000000,0x0405060700000000
1957 .quad 0xffffffff00000000,0xffffffffffffffff
1958 .quad 0xffffffff00000000,0xffffffff00000000
1959 .quad 0x0000000000000000,0xffffffff00000000
1963 .asciz "Poly1305 for PPC, CRYPTOGAMS by \@dot-asm"
1966 foreach (split("\n",$code)) {
1967 s/\`([^\`]*)\`/eval($1)/ge;
1969 # instructions prefixed with '?' are endian-specific and need
1970 # to be adjusted accordingly...
1971 if ($flavour !~ /le$/) { # big-endian
1974 } else { # little-endian
1981 close STDOUT or die "error closing STDOUT: $!";