3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
12 # ChaCha20 for PowerPC/AltiVec.
14 # Performance in cycles per byte out of large buffer.
16 # IALU/gcc-4.x 3xAltiVec+1xIALU
18 # Freescale e300 13.6/+115% -
19 # PPC74x0 6.81/+310% 4.66
20 # POWER7 8.62/+61% 4.27
21 # POWER8 8.70/+51% 3.96
25 if ($flavour =~ /64/) {
32 } elsif ($flavour =~ /32/) {
39 } else { die "nonsense $flavour"; }
41 $LITTLE_ENDIAN = ($flavour=~/le$/) ? 1 : 0;
43 $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
44 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
45 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
46 die "can't locate ppc-xlate.pl";
48 open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
51 $FRAME=$LOCALS+64+18*$SIZE_T; # 64 is for local variables
53 sub AUTOLOAD() # thunk [simplified] x86-style perlasm
54 { my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./;
55 $code .= "\t$opcode\t".join(',',@_)."\n";
60 my ($out,$inp,$len,$key,$ctr) = map("r$_",(3..7));
62 my @x=map("r$_",(16..31));
63 my @d=map("r$_",(11,12,14,15));
64 my @t=map("r$_",(7..10));
67 my ($a0,$b0,$c0,$d0)=@_;
68 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
69 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
70 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
73 "&add (@x[$a0],@x[$a0],@x[$b0])",
74 "&add (@x[$a1],@x[$a1],@x[$b1])",
75 "&add (@x[$a2],@x[$a2],@x[$b2])",
76 "&add (@x[$a3],@x[$a3],@x[$b3])",
77 "&xor (@x[$d0],@x[$d0],@x[$a0])",
78 "&xor (@x[$d1],@x[$d1],@x[$a1])",
79 "&xor (@x[$d2],@x[$d2],@x[$a2])",
80 "&xor (@x[$d3],@x[$d3],@x[$a3])",
81 "&rotlwi (@x[$d0],@x[$d0],16)",
82 "&rotlwi (@x[$d1],@x[$d1],16)",
83 "&rotlwi (@x[$d2],@x[$d2],16)",
84 "&rotlwi (@x[$d3],@x[$d3],16)",
86 "&add (@x[$c0],@x[$c0],@x[$d0])",
87 "&add (@x[$c1],@x[$c1],@x[$d1])",
88 "&add (@x[$c2],@x[$c2],@x[$d2])",
89 "&add (@x[$c3],@x[$c3],@x[$d3])",
90 "&xor (@x[$b0],@x[$b0],@x[$c0])",
91 "&xor (@x[$b1],@x[$b1],@x[$c1])",
92 "&xor (@x[$b2],@x[$b2],@x[$c2])",
93 "&xor (@x[$b3],@x[$b3],@x[$c3])",
94 "&rotlwi (@x[$b0],@x[$b0],12)",
95 "&rotlwi (@x[$b1],@x[$b1],12)",
96 "&rotlwi (@x[$b2],@x[$b2],12)",
97 "&rotlwi (@x[$b3],@x[$b3],12)",
99 "&add (@x[$a0],@x[$a0],@x[$b0])",
100 "&add (@x[$a1],@x[$a1],@x[$b1])",
101 "&add (@x[$a2],@x[$a2],@x[$b2])",
102 "&add (@x[$a3],@x[$a3],@x[$b3])",
103 "&xor (@x[$d0],@x[$d0],@x[$a0])",
104 "&xor (@x[$d1],@x[$d1],@x[$a1])",
105 "&xor (@x[$d2],@x[$d2],@x[$a2])",
106 "&xor (@x[$d3],@x[$d3],@x[$a3])",
107 "&rotlwi (@x[$d0],@x[$d0],8)",
108 "&rotlwi (@x[$d1],@x[$d1],8)",
109 "&rotlwi (@x[$d2],@x[$d2],8)",
110 "&rotlwi (@x[$d3],@x[$d3],8)",
112 "&add (@x[$c0],@x[$c0],@x[$d0])",
113 "&add (@x[$c1],@x[$c1],@x[$d1])",
114 "&add (@x[$c2],@x[$c2],@x[$d2])",
115 "&add (@x[$c3],@x[$c3],@x[$d3])",
116 "&xor (@x[$b0],@x[$b0],@x[$c0])",
117 "&xor (@x[$b1],@x[$b1],@x[$c1])",
118 "&xor (@x[$b2],@x[$b2],@x[$c2])",
119 "&xor (@x[$b3],@x[$b3],@x[$c3])",
120 "&rotlwi (@x[$b0],@x[$b0],7)",
121 "&rotlwi (@x[$b1],@x[$b1],7)",
122 "&rotlwi (@x[$b2],@x[$b2],7)",
123 "&rotlwi (@x[$b3],@x[$b3],7)"
130 .globl .ChaCha20_ctr32_int
133 __ChaCha20_ctr32_int:
137 $STU $sp,-$FRAME($sp)
140 $PUSH r14,`$FRAME-$SIZE_T*18`($sp)
141 $PUSH r15,`$FRAME-$SIZE_T*17`($sp)
142 $PUSH r16,`$FRAME-$SIZE_T*16`($sp)
143 $PUSH r17,`$FRAME-$SIZE_T*15`($sp)
144 $PUSH r18,`$FRAME-$SIZE_T*14`($sp)
145 $PUSH r19,`$FRAME-$SIZE_T*13`($sp)
146 $PUSH r20,`$FRAME-$SIZE_T*12`($sp)
147 $PUSH r21,`$FRAME-$SIZE_T*11`($sp)
148 $PUSH r22,`$FRAME-$SIZE_T*10`($sp)
149 $PUSH r23,`$FRAME-$SIZE_T*9`($sp)
150 $PUSH r24,`$FRAME-$SIZE_T*8`($sp)
151 $PUSH r25,`$FRAME-$SIZE_T*7`($sp)
152 $PUSH r26,`$FRAME-$SIZE_T*6`($sp)
153 $PUSH r27,`$FRAME-$SIZE_T*5`($sp)
154 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
155 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
156 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
157 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
158 $PUSH r0,`$FRAME+$LRSAVE`($sp)
160 lwz @d[0],0($ctr) # load counter
167 $POP r0,`$FRAME+$LRSAVE`($sp)
168 $POP r14,`$FRAME-$SIZE_T*18`($sp)
169 $POP r15,`$FRAME-$SIZE_T*17`($sp)
170 $POP r16,`$FRAME-$SIZE_T*16`($sp)
171 $POP r17,`$FRAME-$SIZE_T*15`($sp)
172 $POP r18,`$FRAME-$SIZE_T*14`($sp)
173 $POP r19,`$FRAME-$SIZE_T*13`($sp)
174 $POP r20,`$FRAME-$SIZE_T*12`($sp)
175 $POP r21,`$FRAME-$SIZE_T*11`($sp)
176 $POP r22,`$FRAME-$SIZE_T*10`($sp)
177 $POP r23,`$FRAME-$SIZE_T*9`($sp)
178 $POP r24,`$FRAME-$SIZE_T*8`($sp)
179 $POP r25,`$FRAME-$SIZE_T*7`($sp)
180 $POP r26,`$FRAME-$SIZE_T*6`($sp)
181 $POP r27,`$FRAME-$SIZE_T*5`($sp)
182 $POP r28,`$FRAME-$SIZE_T*4`($sp)
183 $POP r29,`$FRAME-$SIZE_T*3`($sp)
184 $POP r30,`$FRAME-$SIZE_T*2`($sp)
185 $POP r31,`$FRAME-$SIZE_T*1`($sp)
190 .byte 0,12,4,1,0x80,18,5,0
192 .size .ChaCha20_ctr32_int,.-.ChaCha20_ctr32_int
197 lis @x[0],0x6170 # synthesize sigma
201 ori @x[0],@x[0],0x7865
202 ori @x[1],@x[1],0x646e
203 ori @x[2],@x[2],0x2d32
204 ori @x[3],@x[3],0x6574
206 li r0,10 # inner loop counter
207 lwz @x[4],0($key) # load key
212 mr @x[12],@d[0] # copy counter
228 foreach (&ROUND(0, 4, 8,12)) { eval; }
229 foreach (&ROUND(0, 5,10,15)) { eval; }
233 subic $len,$len,64 # $len-=64
234 addi @x[0],@x[0],0x7865 # accumulate key block
235 addi @x[1],@x[1],0x646e
236 addi @x[2],@x[2],0x2d32
237 addi @x[3],@x[3],0x6574
238 addis @x[0],@x[0],0x6170
239 addis @x[1],@x[1],0x3320
240 addis @x[2],@x[2],0x7962
241 addis @x[3],@x[3],0x6b20
243 subfe. r0,r0,r0 # borrow?-1:0
244 add @x[4],@x[4],@t[0]
246 add @x[5],@x[5],@t[1]
248 add @x[6],@x[6],@t[2]
250 add @x[7],@x[7],@t[3]
252 add @x[8],@x[8],@t[0]
253 add @x[9],@x[9],@t[1]
254 add @x[10],@x[10],@t[2]
255 add @x[11],@x[11],@t[3]
257 add @x[12],@x[12],@d[0]
258 add @x[13],@x[13],@d[1]
259 add @x[14],@x[14],@d[2]
260 add @x[15],@x[15],@d[3]
261 addi @d[0],@d[0],1 # increment counter
263 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) { # flip byte order
266 rotlwi @x[$i],@x[$i],8
267 rlwimi @x[$i],@t[$i&3],24,0,7
268 rlwimi @x[$i],@t[$i&3],24,16,23
272 bne Ltail # $len-=64 borrowed
274 lwz @t[0],0($inp) # load input, aligned or not
276 ${UCMP}i $len,0 # done already?
279 xor @x[0],@x[0],@t[0] # xor with input
281 xor @x[1],@x[1],@t[1]
283 xor @x[2],@x[2],@t[2]
285 xor @x[3],@x[3],@t[3]
287 xor @x[4],@x[4],@t[0]
289 xor @x[5],@x[5],@t[1]
291 xor @x[6],@x[6],@t[2]
293 xor @x[7],@x[7],@t[3]
295 xor @x[8],@x[8],@t[0]
297 xor @x[9],@x[9],@t[1]
299 xor @x[10],@x[10],@t[2]
301 xor @x[11],@x[11],@t[3]
303 xor @x[12],@x[12],@t[0]
304 stw @x[0],0($out) # store output, aligned or not
305 xor @x[13],@x[13],@t[1]
307 xor @x[14],@x[14],@t[2]
309 xor @x[15],@x[15],@t[3]
332 addi $len,$len,64 # restore tail length
333 subi $inp,$inp,1 # prepare for *++ptr
335 addi @t[0],$sp,$LOCALS-1
338 stw @x[0],`$LOCALS+0`($sp) # save whole block to stack
339 stw @x[1],`$LOCALS+4`($sp)
340 stw @x[2],`$LOCALS+8`($sp)
341 stw @x[3],`$LOCALS+12`($sp)
342 stw @x[4],`$LOCALS+16`($sp)
343 stw @x[5],`$LOCALS+20`($sp)
344 stw @x[6],`$LOCALS+24`($sp)
345 stw @x[7],`$LOCALS+28`($sp)
346 stw @x[8],`$LOCALS+32`($sp)
347 stw @x[9],`$LOCALS+36`($sp)
348 stw @x[10],`$LOCALS+40`($sp)
349 stw @x[11],`$LOCALS+44`($sp)
350 stw @x[12],`$LOCALS+48`($sp)
351 stw @x[13],`$LOCALS+52`($sp)
352 stw @x[14],`$LOCALS+56`($sp)
353 stw @x[15],`$LOCALS+60`($sp)
355 Loop_tail: # byte-by-byte loop
358 xor @d[1],@d[0],@x[0]
362 stw $sp,`$LOCALS+0`($sp) # wipe block on stack
363 stw $sp,`$LOCALS+4`($sp)
364 stw $sp,`$LOCALS+8`($sp)
365 stw $sp,`$LOCALS+12`($sp)
366 stw $sp,`$LOCALS+16`($sp)
367 stw $sp,`$LOCALS+20`($sp)
368 stw $sp,`$LOCALS+24`($sp)
369 stw $sp,`$LOCALS+28`($sp)
370 stw $sp,`$LOCALS+32`($sp)
371 stw $sp,`$LOCALS+36`($sp)
372 stw $sp,`$LOCALS+40`($sp)
373 stw $sp,`$LOCALS+44`($sp)
374 stw $sp,`$LOCALS+48`($sp)
375 stw $sp,`$LOCALS+52`($sp)
376 stw $sp,`$LOCALS+56`($sp)
377 stw $sp,`$LOCALS+60`($sp)
381 .byte 0,12,0x14,0,0,0,0,0
385 my ($A0,$B0,$C0,$D0,$A1,$B1,$C1,$D1,$A2,$B2,$C2,$D2,$T0,$T1,$T2) =
387 my (@K)=map("v$_",(15..20));
388 my ($FOUR,$sixteen,$twenty4,$twenty,$twelve,$twenty5,$seven) =
390 my ($inpperm,$outperm,$outmask) = map("v$_",(28..30));
391 my @D=("v31",$seven,$T0,$T1,$T2);
393 my $FRAME=$LOCALS+64+13*16+18*$SIZE_T; # 13*16 is for v20-v31 offload
397 my ($a,$b,$c,$d,$t)=@_;
400 "&vadduwm ('$a','$a','$b')",
401 "&vxor ('$d','$d','$a')",
402 "&vperm ('$d','$d','$d','$sixteen')",
404 "&vadduwm ('$c','$c','$d')",
405 "&vxor ('$t','$b','$c')",
406 "&vsrw ('$b','$t','$twenty')",
407 "&vslw ('$t','$t','$twelve')",
408 "&vor ('$b','$b','$t')",
410 "&vadduwm ('$a','$a','$b')",
411 "&vxor ('$d','$d','$a')",
412 "&vperm ('$d','$d','$d','$twenty4')",
414 "&vadduwm ('$c','$c','$d')",
415 "&vxor ('$t','$b','$c')",
416 "&vsrw ('$b','$t','$twenty5')",
417 "&vslw ('$t','$t','$seven')",
418 "&vor ('$b','$b','$t')",
420 "&vsldoi ('$c','$c','$c',8)",
421 "&vsldoi ('$b','$b','$b',$odd?4:12)",
422 "&vsldoi ('$d','$d','$d',$odd?12:4)"
428 .globl .ChaCha20_ctr32_vmx
432 blt __ChaCha20_ctr32_int
434 $STU $sp,-$FRAME($sp)
436 li r10,`15+$LOCALS+64`
437 li r11,`31+$LOCALS+64`
461 stw r12,`$FRAME-$SIZE_T*18-4`($sp) # save vrsave
462 $PUSH r14,`$FRAME-$SIZE_T*18`($sp)
463 $PUSH r15,`$FRAME-$SIZE_T*17`($sp)
464 $PUSH r16,`$FRAME-$SIZE_T*16`($sp)
465 $PUSH r17,`$FRAME-$SIZE_T*15`($sp)
466 $PUSH r18,`$FRAME-$SIZE_T*14`($sp)
467 $PUSH r19,`$FRAME-$SIZE_T*13`($sp)
468 $PUSH r20,`$FRAME-$SIZE_T*12`($sp)
469 $PUSH r21,`$FRAME-$SIZE_T*11`($sp)
470 $PUSH r22,`$FRAME-$SIZE_T*10`($sp)
471 $PUSH r23,`$FRAME-$SIZE_T*9`($sp)
472 $PUSH r24,`$FRAME-$SIZE_T*8`($sp)
473 $PUSH r25,`$FRAME-$SIZE_T*7`($sp)
474 $PUSH r26,`$FRAME-$SIZE_T*6`($sp)
475 $PUSH r27,`$FRAME-$SIZE_T*5`($sp)
476 $PUSH r28,`$FRAME-$SIZE_T*4`($sp)
477 $PUSH r29,`$FRAME-$SIZE_T*3`($sp)
478 $PUSH r30,`$FRAME-$SIZE_T*2`($sp)
479 $PUSH r31,`$FRAME-$SIZE_T*1`($sp)
481 $PUSH r0, `$FRAME+$LRSAVE`($sp)
482 mtspr 256,r12 # preserve all AltiVec registers
484 bl Lconsts # returns pointer Lsigma in r12
489 li @x[4],31 # 31 is not a typo
490 li @x[5],15 # nor is 15
492 lvx @K[1],0,$key # load key
493 ?lvsr $T0,0,$key # prepare unaligned load
497 lvx @K[3],0,$ctr # load counter
498 ?lvsr $T1,0,$ctr # prepare unaligned load
501 lvx @K[0],0,r12 # load constants
502 lvx @K[5],@x[0],r12 # one
504 lvx $sixteen,@x[2],r12
505 lvx $twenty4,@x[3],r12
507 ?vperm @K[1],@K[2],@K[1],$T0 # align key
508 ?vperm @K[2],@D[0],@K[2],$T0
509 ?vperm @K[3],@D[1],@K[3],$T1 # align counter
511 lwz @d[0],0($ctr) # load counter to GPR
513 vadduwm @K[3],@K[3],@K[5] # adjust AltiVec counter
515 vadduwm @K[4],@K[3],@K[5]
517 vadduwm @K[5],@K[4],@K[5]
519 vspltisw $twenty,-12 # synthesize constants
522 #vspltisw $seven,7 # synthesized in the loop
524 vxor $T0,$T0,$T0 # 0x00..00
525 vspltisw $outmask,-1 # 0xff..ff
526 ?lvsr $inpperm,0,$inp # prepare for unaligned load
527 ?lvsl $outperm,0,$out # prepare for unaligned store
528 ?vperm $outmask,$outmask,$T0,$outperm
530 be?vspltisb $T1,3 # 0x03..03
531 be?vxor $inpperm,$inpperm,$T1 # swap bytes within words
532 be?vxor $outperm,$outperm,$T1
538 lis @x[0],0x6170 # synthesize sigma
544 ori @x[0],@x[0],0x7865
545 ori @x[1],@x[1],0x646e
547 ori @x[2],@x[2],0x2d32
548 ori @x[3],@x[3],0x6574
551 li r0,10 # inner loop counter
552 lwz @x[4],0($key) # load key to GPR
562 mr @x[12],@d[0] # copy GPR counter
583 my @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,0);
584 my @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,0);
585 my @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,0);
586 my @thread3=&ROUND(0,4,8,12);
589 eval; eval(shift(@thread3));
590 eval(shift(@thread1)); eval(shift(@thread3));
591 eval(shift(@thread2)); eval(shift(@thread3));
594 @thread0=&VMXROUND($A0,$B0,$C0,$D0,$T0,1);
595 @thread1=&VMXROUND($A1,$B1,$C1,$D1,$T1,1);
596 @thread2=&VMXROUND($A2,$B2,$C2,$D2,$T2,1);
597 @thread3=&ROUND(0,5,10,15);
600 eval; eval(shift(@thread3));
601 eval(shift(@thread1)); eval(shift(@thread3));
602 eval(shift(@thread2)); eval(shift(@thread3));
607 subi $len,$len,256 # $len-=256
608 addi @x[0],@x[0],0x7865 # accumulate key block
609 addi @x[1],@x[1],0x646e
610 addi @x[2],@x[2],0x2d32
611 addi @x[3],@x[3],0x6574
612 addis @x[0],@x[0],0x6170
613 addis @x[1],@x[1],0x3320
614 addis @x[2],@x[2],0x7962
615 addis @x[3],@x[3],0x6b20
616 add @x[4],@x[4],@t[0]
618 add @x[5],@x[5],@t[1]
620 add @x[6],@x[6],@t[2]
622 add @x[7],@x[7],@t[3]
624 add @x[8],@x[8],@t[0]
625 add @x[9],@x[9],@t[1]
626 add @x[10],@x[10],@t[2]
627 add @x[11],@x[11],@t[3]
628 add @x[12],@x[12],@d[0]
629 add @x[13],@x[13],@d[1]
630 add @x[14],@x[14],@d[2]
631 add @x[15],@x[15],@d[3]
633 vadduwm $A0,$A0,@K[0] # accumulate key block
634 vadduwm $A1,$A1,@K[0]
635 vadduwm $A2,$A2,@K[0]
636 vadduwm $B0,$B0,@K[1]
637 vadduwm $B1,$B1,@K[1]
638 vadduwm $B2,$B2,@K[1]
639 vadduwm $C0,$C0,@K[2]
640 vadduwm $C1,$C1,@K[2]
641 vadduwm $C2,$C2,@K[2]
642 vadduwm $D0,$D0,@K[3]
643 vadduwm $D1,$D1,@K[4]
644 vadduwm $D2,$D2,@K[5]
646 addi @d[0],@d[0],4 # increment counter
647 vadduwm @K[3],@K[3],$FOUR
648 vadduwm @K[4],@K[4],$FOUR
649 vadduwm @K[5],@K[5],$FOUR
652 if (!$LITTLE_ENDIAN) { for($i=0;$i<16;$i++) { # flip byte order
655 rotlwi @x[$i],@x[$i],8
656 rlwimi @x[$i],@t[$i&3],24,0,7
657 rlwimi @x[$i],@t[$i&3],24,16,23
661 lwz @t[0],0($inp) # load input, aligned or not
665 xor @x[0],@x[0],@t[0] # xor with input
667 xor @x[1],@x[1],@t[1]
669 xor @x[2],@x[2],@t[2]
671 xor @x[3],@x[3],@t[3]
673 xor @x[4],@x[4],@t[0]
675 xor @x[5],@x[5],@t[1]
677 xor @x[6],@x[6],@t[2]
679 xor @x[7],@x[7],@t[3]
681 xor @x[8],@x[8],@t[0]
683 xor @x[9],@x[9],@t[1]
685 xor @x[10],@x[10],@t[2]
687 xor @x[11],@x[11],@t[3]
689 xor @x[12],@x[12],@t[0]
690 stw @x[0],0($out) # store output, aligned or not
691 xor @x[13],@x[13],@t[1]
693 xor @x[14],@x[14],@t[2]
695 xor @x[15],@x[15],@t[3]
716 lvx @D[0],0,$inp # load input
723 ?vperm @D[0],@D[1],@D[0],$inpperm # align input
724 ?vperm @D[1],@D[2],@D[1],$inpperm
725 ?vperm @D[2],@D[3],@D[2],$inpperm
726 ?vperm @D[3],@D[4],@D[3],$inpperm
727 vxor $A0,$A0,@D[0] # xor with input
729 lvx @D[1],@t[0],$inp # keep loading input
736 li @t[3],63 # 63 is not a typo
737 vperm $A0,$A0,$A0,$outperm # pre-misalign output
738 vperm $B0,$B0,$B0,$outperm
739 vperm $C0,$C0,$C0,$outperm
740 vperm $D0,$D0,$D0,$outperm
742 ?vperm @D[4],@D[1],@D[4],$inpperm # align input
743 ?vperm @D[1],@D[2],@D[1],$inpperm
744 ?vperm @D[2],@D[3],@D[2],$inpperm
745 ?vperm @D[3],@D[0],@D[3],$inpperm
748 lvx @D[1],@t[0],$inp # keep loading input
753 lvx @D[4],@t[3],$inp # redundant in aligned case
755 vperm $A1,$A1,$A1,$outperm # pre-misalign output
756 vperm $B1,$B1,$B1,$outperm
757 vperm $C1,$C1,$C1,$outperm
758 vperm $D1,$D1,$D1,$outperm
760 ?vperm @D[0],@D[1],@D[0],$inpperm # align input
761 ?vperm @D[1],@D[2],@D[1],$inpperm
762 ?vperm @D[2],@D[3],@D[2],$inpperm
763 ?vperm @D[3],@D[4],@D[3],$inpperm
768 vperm $A2,$A2,$A2,$outperm # pre-misalign output
769 vperm $B2,$B2,$B2,$outperm
770 vperm $C2,$C2,$C2,$outperm
771 vperm $D2,$D2,$D2,$outperm
773 andi. @x[1],$out,15 # is $out aligned?
776 vsel @D[0],$A0,$B0,$outmask # collect pre-misaligned output
777 vsel @D[1],$B0,$C0,$outmask
778 vsel @D[2],$C0,$D0,$outmask
779 vsel @D[3],$D0,$A1,$outmask
780 vsel $B0,$A1,$B1,$outmask
781 vsel $C0,$B1,$C1,$outmask
782 vsel $D0,$C1,$D1,$outmask
783 vsel $A1,$D1,$A2,$outmask
784 vsel $B1,$A2,$B2,$outmask
785 vsel $C1,$B2,$C2,$outmask
786 vsel $D1,$C2,$D2,$outmask
788 #stvx $A0,0,$out # take it easy on the edges
789 stvx @D[0],@t[0],$out # store output
790 stvx @D[1],@t[1],$out
791 stvx @D[2],@t[2],$out
806 sub @x[2],$out,@x[1] # in misaligned case edges
807 li @x[3],0 # are written byte-by-byte
809 stvebx $D2,@x[3],@x[2]
812 bne Lunaligned_tail_vmx
814 sub @x[2],@x[0],@x[1]
816 stvebx $A0,@x[1],@x[2]
819 bne Lunaligned_head_vmx
821 ${UCMP}i $len,255 # done with 256-byte blocks yet?
828 stvx $A0,0,@x[0] # head hexaword was not stored
830 ${UCMP}i $len,255 # done with 256-byte blocks yet?
835 ${UCMP}i $len,0 # done yet?
838 lwz r12,`$FRAME-$SIZE_T*18-4`($sp) # pull vrsave
839 li r10,`15+$LOCALS+64`
840 li r11,`31+$LOCALS+64`
841 mtspr 256,r12 # restore vrsave
864 $POP r0, `$FRAME+$LRSAVE`($sp)
865 $POP r14,`$FRAME-$SIZE_T*18`($sp)
866 $POP r15,`$FRAME-$SIZE_T*17`($sp)
867 $POP r16,`$FRAME-$SIZE_T*16`($sp)
868 $POP r17,`$FRAME-$SIZE_T*15`($sp)
869 $POP r18,`$FRAME-$SIZE_T*14`($sp)
870 $POP r19,`$FRAME-$SIZE_T*13`($sp)
871 $POP r20,`$FRAME-$SIZE_T*12`($sp)
872 $POP r21,`$FRAME-$SIZE_T*11`($sp)
873 $POP r22,`$FRAME-$SIZE_T*10`($sp)
874 $POP r23,`$FRAME-$SIZE_T*9`($sp)
875 $POP r24,`$FRAME-$SIZE_T*8`($sp)
876 $POP r25,`$FRAME-$SIZE_T*7`($sp)
877 $POP r26,`$FRAME-$SIZE_T*6`($sp)
878 $POP r27,`$FRAME-$SIZE_T*5`($sp)
879 $POP r28,`$FRAME-$SIZE_T*4`($sp)
880 $POP r29,`$FRAME-$SIZE_T*3`($sp)
881 $POP r30,`$FRAME-$SIZE_T*2`($sp)
882 $POP r31,`$FRAME-$SIZE_T*1`($sp)
887 .byte 0,12,0x04,1,0x80,18,5,0
889 .size .ChaCha20_ctr32_vmx,.-.ChaCha20_ctr32_vmx
895 mflr r12 #vvvvv "distance between . and _vpaes_consts
900 .byte 0,12,0x14,0,0,0,0,0
903 .long 0x61707865,0x3320646e,0x79622d32,0x6b206574
907 $code.=<<___ if ($LITTLE_ENDIAN);
908 .long 0x0e0f0c0d,0x0a0b0809,0x06070405,0x02030001
909 .long 0x0d0e0f0c,0x090a0b08,0x05060704,0x01020300
911 $code.=<<___ if (!$LITTLE_ENDIAN); # flipped words
912 .long 0x02030001,0x06070405,0x0a0b0809,0x0e0f0c0d
913 .long 0x01020300,0x05060704,0x090a0b08,0x0d0e0f0c
916 .asciz "ChaCha20 for PowerPC/AltiVec, CRYPTOGAMS by <appro\@openssl.org>"
921 foreach (split("\n",$code)) {
922 s/\`([^\`]*)\`/eval $1/ge;
924 # instructions prefixed with '?' are endian-specific and need
925 # to be adjusted accordingly...
926 if ($flavour !~ /le$/) { # big-endian
931 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/ or
932 s/(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/;
933 } else { # little-endian