2 # Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the OpenSSL license (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
19 # ChaCha20 for PowerPC/AltiVec.
21 # Performance in cycles per byte out of large buffer.
23 # IALU/gcc-4.x 3xAltiVec+1xIALU
25 # Freescale e300 13.6/+115% -
26 # PPC74x0 6.81/+310% 4.66
27 # POWER7 8.62/+61% 4.27
28 # POWER8 8.70/+51% 3.96
32 if ($flavour =~ /64/) {
39 } elsif ($flavour =~ /32/) {
46 } else { die "nonsense $flavour"; }
48 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
50 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
51 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
52 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
53 die "can't locate ppc-xlate.pl";
55 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
58 $FRAME=$LOCALS+64+18*$SIZE_T; # 64 is for local variables
60 sub AUTOLOAD() # thunk [simplified] x86-style perlasm
61 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
62 $code .= "\t$opcode\t".join(',',@_)."\n";
67 my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
69 my @x=map("r$_",(16..31));
70 my @d=map("r$_",(11,12,14,15));
71 my @t=map("r$_",(7..10));
74 my ($a0,$b0,$c0,$d0)=@_;
75 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
76 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
77 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
80 "&add (@x[$a0],@x[$a0],@x[$b0])",
81 "&add (@x[$a1],@x[$a1],@x[$b1])",
82 "&add (@x[$a2],@x[$a2],@x[$b2])",
83 "&add (@x[$a3],@x[$a3],@x[$b3])",
84 "&xor (@x[$d0],@x[$d0],@x[$a0])",
85 "&xor (@x[$d1],@x[$d1],@x[$a1])",
86 "&xor (@x[$d2],@x[$d2],@x[$a2])",
87 "&xor (@x[$d3],@x[$d3],@x[$a3])",
88 "&rotlwi (@x[$d0],@x[$d0],16)",
89 "&rotlwi (@x[$d1],@x[$d1],16)",
90 "&rotlwi (@x[$d2],@x[$d2],16)",
91 "&rotlwi (@x[$d3],@x[$d3],16)",
93 "&add (@x[$c0],@x[$c0],@x[$d0])",
94 "&add (@x[$c1],@x[$c1],@x[$d1])",
95 "&add (@x[$c2],@x[$c2],@x[$d2])",
96 "&add (@x[$c3],@x[$c3],@x[$d3])",
97 "&xor (@x[$b0],@x[$b0],@x[$c0])",
98 "&xor (@x[$b1],@x[$b1],@x[$c1])",
99 "&xor (@x[$b2],@x[$b2],@x[$c2])",
100 "&xor (@x[$b3],@x[$b3],@x[$c3])",
101 "&rotlwi (@x[$b0],@x[$b0],12)",
102 "&rotlwi (@x[$b1],@x[$b1],12)",
103 "&rotlwi (@x[$b2],@x[$b2],12)",
104 "&rotlwi (@x[$b3],@x[$b3],12)",
106 "&add (@x[$a0],@x[$a0],@x[$b0])",
107 "&add (@x[$a1],@x[$a1],@x[$b1])",
108 "&add (@x[$a2],@x[$a2],@x[$b2])",
109 "&add (@x[$a3],@x[$a3],@x[$b3])",
110 "&xor (@x[$d0],@x[$d0],@x[$a0])",
111 "&xor (@x[$d1],@x[$d1],@x[$a1])",
112 "&xor (@x[$d2],@x[$d2],@x[$a2])",
113 "&xor (@x[$d3],@x[$d3],@x[$a3])",
114 "&rotlwi (@x[$d0],@x[$d0],8)",
115 "&rotlwi (@x[$d1],@x[$d1],8)",
116 "&rotlwi (@x[$d2],@x[$d2],8)",
117 "&rotlwi (@x[$d3],@x[$d3],8)",
119 "&add (@x[$c0],@x[$c0],@x[$d0])",
120 "&add (@x[$c1],@x[$c1],@x[$d1])",
121 "&add (@x[$c2],@x[$c2],@x[$d2])",
122 "&add (@x[$c3],@x[$c3],@x[$d3])",
123 "&xor (@x[$b0],@x[$b0],@x[$c0])",
124 "&xor (@x[$b1],@x[$b1],@x[$c1])",
125 "&xor (@x[$b2],@x[$b2],@x[$c2])",
126 "&xor (@x[$b3],@x[$b3],@x[$c3])",
127 "&rotlwi (@x[$b0],@x[$b0],7)",
128 "&rotlwi (@x[$b1],@x[$b1],7)",
129 "&rotlwi (@x[$b2],@x[$b2],7)",
130 "&rotlwi (@x[$b3],@x[$b3],7)"
137 .globl .ChaCha20_ctr32_int
140 __ChaCha20_ctr32_int:
144 $STU $sp,-$FRAME($sp)
147 $PUSH r14,`$FRAME-$SIZE_T*18`($sp)
148 $PUSH r15,`$FRAME-$SIZE_T*17`($sp)
149 $PUSH r16,`$FRAME-$SIZE_T*16`($sp)
150 $PUSH r17,`$FRAME-$SIZE_T*15`($sp)
151 $PUSH r18,`$FRAME-$SIZE_T*14`($sp)
152 $PUSH r19,`$FRAME-$SIZE_T*13`($sp)
153 $PUSH r20,`$FRAME-$SIZE_T*12`($sp)
154 $PUSH r21,`$FRAME-$SIZE_T*11`($sp)
155 $PUSH r22,`$FRAME-$SIZE_T*10`($sp)
156 $PUSH r23,`$FRAME-$SIZE_T*9`($sp)
157 $PUSH r24,`$FRAME-$SIZE_T*8`($sp)
158 $PUSH r25,`$FRAME-$SIZE_T*7`($sp)
159 $PUSH r26,`$FRAME-$SIZE_T*6`($sp)
160 $PUSH r27,`$FRAME-$SIZE_T*5`($sp)
161 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
162 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
163 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
164 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
165 $PUSH r0,`$FRAME+$LRSAVE`($sp)
167 lwz @d[0],0($ctr) # load counter
174 $POP r0,`$FRAME+$LRSAVE`($sp)
175 $POP r14,`$FRAME-$SIZE_T*18`($sp)
176 $POP r15,`$FRAME-$SIZE_T*17`($sp)
177 $POP r16,`$FRAME-$SIZE_T*16`($sp)
178 $POP r17,`$FRAME-$SIZE_T*15`($sp)
179 $POP r18,`$FRAME-$SIZE_T*14`($sp)
180 $POP r19,`$FRAME-$SIZE_T*13`($sp)
181 $POP r20,`$FRAME-$SIZE_T*12`($sp)
182 $POP r21,`$FRAME-$SIZE_T*11`($sp)
183 $POP r22,`$FRAME-$SIZE_T*10`($sp)
184 $POP r23,`$FRAME-$SIZE_T*9`($sp)
185 $POP r24,`$FRAME-$SIZE_T*8`($sp)
186 $POP r25,`$FRAME-$SIZE_T*7`($sp)
187 $POP r26,`$FRAME-$SIZE_T*6`($sp)
188 $POP r27,`$FRAME-$SIZE_T*5`($sp)
189 $POP r28,`$FRAME-$SIZE_T*4`($sp)
190 $POP r29,`$FRAME-$SIZE_T*3`($sp)
191 $POP r30,`$FRAME-$SIZE_T*2`($sp)
192 $POP r31,`$FRAME-$SIZE_T*1`($sp)
197 .byte 0,12,4,1,0x80,18,5,0
199 .size .ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
204 lis @x[0],0x6170 # synthesize sigma
208 ori @x[0],@x[0],0x7865
209 ori @x[1],@x[1],0x646e
210 ori @x[2],@x[2],0x2d32
211 ori @x[3],@x[3],0x6574
213 li r0,10 # inner loop counter
214 lwz @x[4],0($key) # load key
219 mr @x[12],@d[0] # copy counter
235 foreach (&ROUND(0, 4, 8,12)) { eval; }
236 foreach (&ROUND(0, 5,10,15)) { eval; }
240 subic $len,$len,64 # $len-=64
241 addi @x[0],@x[0],0x7865 # accumulate key block
242 addi @x[1],@x[1],0x646e
243 addi @x[2],@x[2],0x2d32
244 addi @x[3],@x[3],0x6574
245 addis @x[0],@x[0],0x6170
246 addis @x[1],@x[1],0x3320
247 addis @x[2],@x[2],0x7962
248 addis @x[3],@x[3],0x6b20
250 subfe. r0,r0,r0 # borrow?-1:0
251 add @x[4],@x[4],@t[0]
253 add @x[5],@x[5],@t[1]
255 add @x[6],@x[6],@t[2]
257 add @x[7],@x[7],@t[3]
259 add @x[8],@x[8],@t[0]
260 add @x[9],@x[9],@t[1]
261 add @x[10],@x[10],@t[2]
262 add @x[11],@x[11],@t[3]
264 add @x[12],@x[12],@d[0]
265 add @x[13],@x[13],@d[1]
266 add @x[14],@x[14],@d[2]
267 add @x[15],@x[15],@d[3]
268 addi @d[0],@d[0],1 # increment counter
270 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) { # flip byte order
273 rotlwi @x[$i],@x[$i],8
274 rlwimi @x[$i],@t[$i&3],24,0,7
275 rlwimi @x[$i],@t[$i&3],24,16,23
279 bne Ltail # $len-=64 borrowed
281 lwz @t[0],0($inp) # load input, aligned or not
283 ${UCMP}i $len,0 # done already?
286 xor @x[0],@x[0],@t[0] # xor with input
288 xor @x[1],@x[1],@t[1]
290 xor @x[2],@x[2],@t[2]
292 xor @x[3],@x[3],@t[3]
294 xor @x[4],@x[4],@t[0]
296 xor @x[5],@x[5],@t[1]
298 xor @x[6],@x[6],@t[2]
300 xor @x[7],@x[7],@t[3]
302 xor @x[8],@x[8],@t[0]
304 xor @x[9],@x[9],@t[1]
306 xor @x[10],@x[10],@t[2]
308 xor @x[11],@x[11],@t[3]
310 xor @x[12],@x[12],@t[0]
311 stw @x[0],0($out) # store output, aligned or not
312 xor @x[13],@x[13],@t[1]
314 xor @x[14],@x[14],@t[2]
316 xor @x[15],@x[15],@t[3]
339 addi $len,$len,64 # restore tail length
340 subi $inp,$inp,1 # prepare for *++ptr
342 addi @t[0],$sp,$LOCALS-1
345 stw @x[0],`$LOCALS+0`($sp) # save whole block to stack
346 stw @x[1],`$LOCALS+4`($sp)
347 stw @x[2],`$LOCALS+8`($sp)
348 stw @x[3],`$LOCALS+12`($sp)
349 stw @x[4],`$LOCALS+16`($sp)
350 stw @x[5],`$LOCALS+20`($sp)
351 stw @x[6],`$LOCALS+24`($sp)
352 stw @x[7],`$LOCALS+28`($sp)
353 stw @x[8],`$LOCALS+32`($sp)
354 stw @x[9],`$LOCALS+36`($sp)
355 stw @x[10],`$LOCALS+40`($sp)
356 stw @x[11],`$LOCALS+44`($sp)
357 stw @x[12],`$LOCALS+48`($sp)
358 stw @x[13],`$LOCALS+52`($sp)
359 stw @x[14],`$LOCALS+56`($sp)
360 stw @x[15],`$LOCALS+60`($sp)
362 Loop_tail: # byte-by-byte loop
365 xor @d[1],@d[0],@x[0]
369 stw $sp,`$LOCALS+0`($sp) # wipe block on stack
370 stw $sp,`$LOCALS+4`($sp)
371 stw $sp,`$LOCALS+8`($sp)
372 stw $sp,`$LOCALS+12`($sp)
373 stw $sp,`$LOCALS+16`($sp)
374 stw $sp,`$LOCALS+20`($sp)
375 stw $sp,`$LOCALS+24`($sp)
376 stw $sp,`$LOCALS+28`($sp)
377 stw $sp,`$LOCALS+32`($sp)
378 stw $sp,`$LOCALS+36`($sp)
379 stw $sp,`$LOCALS+40`($sp)
380 stw $sp,`$LOCALS+44`($sp)
381 stw $sp,`$LOCALS+48`($sp)
382 stw $sp,`$LOCALS+52`($sp)
383 stw $sp,`$LOCALS+56`($sp)
384 stw $sp,`$LOCALS+60`($sp)
388 .byte 0,12,0x14,0,0,0,0,0
392 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2) =
394 my (@K)=map("v$_",(15..20));
395 my ($FOUR,$sixteen,$twenty4,$twenty,$twelve,$twenty5,$seven) =
397 my ($inpperm,$outperm,$outmask) = map("v$_",(28..30));
398 my @D=("v31",$seven,$T0,$T1,$T2);
400 my $FRAME=$LOCALS+64+13*16+18*$SIZE_T; # 13*16 is for v20-v31 offload
404 my ($a,$b,$c,$d,$t)=@_;
407 "&vadduwm ('$a','$a','$b')",
408 "&vxor ('$d','$d','$a')",
409 "&vperm ('$d','$d','$d','$sixteen')",
411 "&vadduwm ('$c','$c','$d')",
412 "&vxor ('$t','$b','$c')",
413 "&vsrw ('$b','$t','$twenty')",
414 "&vslw ('$t','$t','$twelve')",
415 "&vor ('$b','$b','$t')",
417 "&vadduwm ('$a','$a','$b')",
418 "&vxor ('$d','$d','$a')",
419 "&vperm ('$d','$d','$d','$twenty4')",
421 "&vadduwm ('$c','$c','$d')",
422 "&vxor ('$t','$b','$c')",
423 "&vsrw ('$b','$t','$twenty5')",
424 "&vslw ('$t','$t','$seven')",
425 "&vor ('$b','$b','$t')",
427 "&vsldoi ('$c','$c','$c',8)",
428 "&vsldoi ('$b','$b','$b',$odd?4:12)",
429 "&vsldoi ('$d','$d','$d',$odd?12:4)"
435 .globl .ChaCha20_ctr32_vmx
439 blt __ChaCha20_ctr32_int
441 $STU $sp,-$FRAME($sp)
443 li r10,`15+$LOCALS+64`
444 li r11,`31+$LOCALS+64`
468 stw r12,`$FRAME-$SIZE_T*18-4`($sp) # save vrsave
469 $PUSH r14,`$FRAME-$SIZE_T*18`($sp)
470 $PUSH r15,`$FRAME-$SIZE_T*17`($sp)
471 $PUSH r16,`$FRAME-$SIZE_T*16`($sp)
472 $PUSH r17,`$FRAME-$SIZE_T*15`($sp)
473 $PUSH r18,`$FRAME-$SIZE_T*14`($sp)
474 $PUSH r19,`$FRAME-$SIZE_T*13`($sp)
475 $PUSH r20,`$FRAME-$SIZE_T*12`($sp)
476 $PUSH r21,`$FRAME-$SIZE_T*11`($sp)
477 $PUSH r22,`$FRAME-$SIZE_T*10`($sp)
478 $PUSH r23,`$FRAME-$SIZE_T*9`($sp)
479 $PUSH r24,`$FRAME-$SIZE_T*8`($sp)
480 $PUSH r25,`$FRAME-$SIZE_T*7`($sp)
481 $PUSH r26,`$FRAME-$SIZE_T*6`($sp)
482 $PUSH r27,`$FRAME-$SIZE_T*5`($sp)
483 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
484 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
485 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
486 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
488 $PUSH r0, `$FRAME+$LRSAVE`($sp)
489 mtspr 256,r12 # preserve all AltiVec registers
491 bl Lconsts # returns pointer Lsigma in r12
496 li @x[4],31 # 31 is not a typo
497 li @x[5],15 # nor is 15
499 lvx @K[1],0,$key # load key
500 ?lvsr $T0,0,$key # prepare unaligned load
504 lvx @K[3],0,$ctr # load counter
505 ?lvsr $T1,0,$ctr # prepare unaligned load
508 lvx @K[0],0,r12 # load constants
509 lvx @K[5],@x[0],r12 # one
511 lvx $sixteen,@x[2],r12
512 lvx $twenty4,@x[3],r12
514 ?vperm @K[1],@K[2],@K[1],$T0 # align key
515 ?vperm @K[2],@D[0],@K[2],$T0
516 ?vperm @K[3],@D[1],@K[3],$T1 # align counter
518 lwz @d[0],0($ctr) # load counter to GPR
520 vadduwm @K[3],@K[3],@K[5] # adjust AltiVec counter
522 vadduwm @K[4],@K[3],@K[5]
524 vadduwm @K[5],@K[4],@K[5]
526 vspltisw $twenty,-12 # synthesize constants
529 #vspltisw $seven,7 # synthesized in the loop
531 vxor $T0,$T0,$T0 # 0x00..00
532 vspltisw $outmask,-1 # 0xff..ff
533 ?lvsr $inpperm,0,$inp # prepare for unaligned load
534 ?lvsl $outperm,0,$out # prepare for unaligned store
535 ?vperm $outmask,$outmask,$T0,$outperm
537 be?lvsl $T0,0,@x[0] # 0x00..0f
538 be?vspltisb $T1,3 # 0x03..03
539 be?vxor $T0,$T0,$T1 # swap bytes within words
540 be?vxor $outperm,$outperm,$T1
541 be?vperm $inpperm,$inpperm,$inpperm,$T0
547 lis @x[0],0x6170 # synthesize sigma
553 ori @x[0],@x[0],0x7865
554 ori @x[1],@x[1],0x646e
556 ori @x[2],@x[2],0x2d32
557 ori @x[3],@x[3],0x6574
560 li r0,10 # inner loop counter
561 lwz @x[4],0($key) # load key to GPR
571 mr @x[12],@d[0] # copy GPR counter
592 my @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,0);
593 my @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,0);
594 my @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,0);
595 my @thread3=&ROUND(0,4,8,12);
598 eval; eval(shift(@thread3));
599 eval(shift(@thread1)); eval(shift(@thread3));
600 eval(shift(@thread2)); eval(shift(@thread3));
603 @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,1);
604 @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,1);
605 @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,1);
606 @thread3=&ROUND(0,5,10,15);
609 eval; eval(shift(@thread3));
610 eval(shift(@thread1)); eval(shift(@thread3));
611 eval(shift(@thread2)); eval(shift(@thread3));
616 subi $len,$len,256 # $len-=256
617 addi @x[0],@x[0],0x7865 # accumulate key block
618 addi @x[1],@x[1],0x646e
619 addi @x[2],@x[2],0x2d32
620 addi @x[3],@x[3],0x6574
621 addis @x[0],@x[0],0x6170
622 addis @x[1],@x[1],0x3320
623 addis @x[2],@x[2],0x7962
624 addis @x[3],@x[3],0x6b20
625 add @x[4],@x[4],@t[0]
627 add @x[5],@x[5],@t[1]
629 add @x[6],@x[6],@t[2]
631 add @x[7],@x[7],@t[3]
633 add @x[8],@x[8],@t[0]
634 add @x[9],@x[9],@t[1]
635 add @x[10],@x[10],@t[2]
636 add @x[11],@x[11],@t[3]
637 add @x[12],@x[12],@d[0]
638 add @x[13],@x[13],@d[1]
639 add @x[14],@x[14],@d[2]
640 add @x[15],@x[15],@d[3]
642 vadduwm $A0,$A0,@K[0] # accumulate key block
643 vadduwm $A1,$A1,@K[0]
644 vadduwm $A2,$A2,@K[0]
645 vadduwm $B0,$B0,@K[1]
646 vadduwm $B1,$B1,@K[1]
647 vadduwm $B2,$B2,@K[1]
648 vadduwm $C0,$C0,@K[2]
649 vadduwm $C1,$C1,@K[2]
650 vadduwm $C2,$C2,@K[2]
651 vadduwm $D0,$D0,@K[3]
652 vadduwm $D1,$D1,@K[4]
653 vadduwm $D2,$D2,@K[5]
655 addi @d[0],@d[0],4 # increment counter
656 vadduwm @K[3],@K[3],$FOUR
657 vadduwm @K[4],@K[4],$FOUR
658 vadduwm @K[5],@K[5],$FOUR
661 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) { # flip byte order
664 rotlwi @x[$i],@x[$i],8
665 rlwimi @x[$i],@t[$i&3],24,0,7
666 rlwimi @x[$i],@t[$i&3],24,16,23
670 lwz @t[0],0($inp) # load input, aligned or not
674 xor @x[0],@x[0],@t[0] # xor with input
676 xor @x[1],@x[1],@t[1]
678 xor @x[2],@x[2],@t[2]
680 xor @x[3],@x[3],@t[3]
682 xor @x[4],@x[4],@t[0]
684 xor @x[5],@x[5],@t[1]
686 xor @x[6],@x[6],@t[2]
688 xor @x[7],@x[7],@t[3]
690 xor @x[8],@x[8],@t[0]
692 xor @x[9],@x[9],@t[1]
694 xor @x[10],@x[10],@t[2]
696 xor @x[11],@x[11],@t[3]
698 xor @x[12],@x[12],@t[0]
699 stw @x[0],0($out) # store output, aligned or not
700 xor @x[13],@x[13],@t[1]
702 xor @x[14],@x[14],@t[2]
704 xor @x[15],@x[15],@t[3]
725 lvx @D[0],0,$inp # load input
732 ?vperm @D[0],@D[1],@D[0],$inpperm # align input
733 ?vperm @D[1],@D[2],@D[1],$inpperm
734 ?vperm @D[2],@D[3],@D[2],$inpperm
735 ?vperm @D[3],@D[4],@D[3],$inpperm
736 vxor $A0,$A0,@D[0] # xor with input
738 lvx @D[1],@t[0],$inp # keep loading input
745 li @t[3],63 # 63 is not a typo
746 vperm $A0,$A0,$A0,$outperm # pre-misalign output
747 vperm $B0,$B0,$B0,$outperm
748 vperm $C0,$C0,$C0,$outperm
749 vperm $D0,$D0,$D0,$outperm
751 ?vperm @D[4],@D[1],@D[4],$inpperm # align input
752 ?vperm @D[1],@D[2],@D[1],$inpperm
753 ?vperm @D[2],@D[3],@D[2],$inpperm
754 ?vperm @D[3],@D[0],@D[3],$inpperm
757 lvx @D[1],@t[0],$inp # keep loading input
762 lvx @D[4],@t[3],$inp # redundant in aligned case
764 vperm $A1,$A1,$A1,$outperm # pre-misalign output
765 vperm $B1,$B1,$B1,$outperm
766 vperm $C1,$C1,$C1,$outperm
767 vperm $D1,$D1,$D1,$outperm
769 ?vperm @D[0],@D[1],@D[0],$inpperm # align input
770 ?vperm @D[1],@D[2],@D[1],$inpperm
771 ?vperm @D[2],@D[3],@D[2],$inpperm
772 ?vperm @D[3],@D[4],@D[3],$inpperm
777 vperm $A2,$A2,$A2,$outperm # pre-misalign output
778 vperm $B2,$B2,$B2,$outperm
779 vperm $C2,$C2,$C2,$outperm
780 vperm $D2,$D2,$D2,$outperm
782 andi. @x[1],$out,15 # is $out aligned?
785 vsel @D[0],$A0,$B0,$outmask # collect pre-misaligned output
786 vsel @D[1],$B0,$C0,$outmask
787 vsel @D[2],$C0,$D0,$outmask
788 vsel @D[3],$D0,$A1,$outmask
789 vsel $B0,$A1,$B1,$outmask
790 vsel $C0,$B1,$C1,$outmask
791 vsel $D0,$C1,$D1,$outmask
792 vsel $A1,$D1,$A2,$outmask
793 vsel $B1,$A2,$B2,$outmask
794 vsel $C1,$B2,$C2,$outmask
795 vsel $D1,$C2,$D2,$outmask
797 #stvx $A0,0,$out # take it easy on the edges
798 stvx @D[0],@t[0],$out # store output
799 stvx @D[1],@t[1],$out
800 stvx @D[2],@t[2],$out
815 sub @x[2],$out,@x[1] # in misaligned case edges
816 li @x[3],0 # are written byte-by-byte
818 stvebx $D2,@x[3],@x[2]
821 bne Lunaligned_tail_vmx
823 sub @x[2],@x[0],@x[1]
825 stvebx $A0,@x[1],@x[2]
828 bne Lunaligned_head_vmx
830 ${UCMP}i $len,255 # done with 256-byte blocks yet?
837 stvx $A0,0,@x[0] # head hexaword was not stored
839 ${UCMP}i $len,255 # done with 256-byte blocks yet?
844 ${UCMP}i $len,0 # done yet?
847 lwz r12,`$FRAME-$SIZE_T*18-4`($sp) # pull vrsave
848 li r10,`15+$LOCALS+64`
849 li r11,`31+$LOCALS+64`
850 mtspr 256,r12 # restore vrsave
873 $POP r0, `$FRAME+$LRSAVE`($sp)
874 $POP r14,`$FRAME-$SIZE_T*18`($sp)
875 $POP r15,`$FRAME-$SIZE_T*17`($sp)
876 $POP r16,`$FRAME-$SIZE_T*16`($sp)
877 $POP r17,`$FRAME-$SIZE_T*15`($sp)
878 $POP r18,`$FRAME-$SIZE_T*14`($sp)
879 $POP r19,`$FRAME-$SIZE_T*13`($sp)
880 $POP r20,`$FRAME-$SIZE_T*12`($sp)
881 $POP r21,`$FRAME-$SIZE_T*11`($sp)
882 $POP r22,`$FRAME-$SIZE_T*10`($sp)
883 $POP r23,`$FRAME-$SIZE_T*9`($sp)
884 $POP r24,`$FRAME-$SIZE_T*8`($sp)
885 $POP r25,`$FRAME-$SIZE_T*7`($sp)
886 $POP r26,`$FRAME-$SIZE_T*6`($sp)
887 $POP r27,`$FRAME-$SIZE_T*5`($sp)
888 $POP r28,`$FRAME-$SIZE_T*4`($sp)
889 $POP r29,`$FRAME-$SIZE_T*3`($sp)
890 $POP r30,`$FRAME-$SIZE_T*2`($sp)
891 $POP r31,`$FRAME-$SIZE_T*1`($sp)
896 .byte 0,12,0x04,1,0x80,18,5,0
898 .size .ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
904 mflr r12 #vvvvv "distance between . and _vpaes_consts
909 .byte 0,12,0x14,0,0,0,0,0
912 .long 0x61707865,0x3320646e,0x79622d32,0x6b206574
916 $code.=<<___ if ($LITTLE_ENDIAN);
917 .long 0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
918 .long 0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
920 $code.=<<___ if (!$LITTLE_ENDIAN); # flipped words
921 .long 0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
922 .long 0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
925 .asciz "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
930 foreach (split("\n",$code)) {
931 s/\`([^\`]*)\`/eval $1/ge;
933 # instructions prefixed with '?' are endian-specific and need
934 # to be adjusted accordingly...
935 if ($flavour !~ /le$/) { # big-endian
940 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
941 s/(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/;
942 } else { # little-endian