2 # Copyright 2016-2019 The OpenSSL Project Authors. All Rights Reserved.
4 # Licensed under the Apache License 2.0 (the "License"). You may not use
5 # this file except in compliance with the License. You can obtain a copy
6 # in the file LICENSE in the source distribution or at
7 # https://www.openssl.org/source/license.html
10 # ====================================================================
11 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12 # project. The module is, however, dual licensed under OpenSSL and
13 # CRYPTOGAMS licenses depending on where you obtain it. For further
14 # details see http://www.openssl.org/~appro/cryptogams/.
15 # ====================================================================
21 # 3 times faster than compiler-generated code.
26 # Add vx code path: 4x"vertical".
28 # Copyright IBM Corp. 2018
29 # Author: Patrick Steuer <patrick.steuer@de.ibm.com>
34 # Add 6x"horizontal" VX implementation. It's ~25% faster than IBM's
35 # 4x"vertical" submission [on z13] and >3 faster than scalar code.
36 # But to harness overheads revert to transliteration of VSX code path
37 # from chacha-ppc module, which is also 4x"vertical", to handle inputs
38 # not longer than 256 bytes.
43 use perlasm::s390x qw(:DEFAULT :VX :LD AUTOLOAD LABEL INCLUDE);
45 # $output is the last argument if it looks like a file (it has an extension)
46 # $flavour is the first argument if it doesn't look like a file
47 my $output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
48 my $flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
51 if ($flavour =~ /3[12]/) {
60 my $stdframe=16*$SIZE_T+4*8;
63 my @x=map("%r$_",(0..7,"x","x","x","x",(10..13)));
64 my @t=map("%r$_",(8,9));
65 my ($a0,$b0,$c0,$d0)=@_;
66 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
67 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
68 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
69 my ($xc,$xc_)=map("$_",@t);
71 # Consider order in which variables are addressed by their
76 # 0 4 8 12 < even round
80 # 0 5 10 15 < odd round
85 # 'a', 'b' and 'd's are permanently allocated in registers,
86 # @x[0..7,12..15], while 'c's are maintained in memory. If
87 # you observe 'c' column, you'll notice that pair of 'c's is
88 # invariant between rounds. This means that we have to reload
89 # them once per round, in the middle. This is why you'll see
90 # 'c' stores and loads in the middle, but none in the beginning
93 alr (@x[$a0],@x[$b0]); # Q1
94 alr (@x[$a1],@x[$b1]); # Q2
97 rll (@x[$d0],@x[$d0],16);
98 rll (@x[$d1],@x[$d1],16);
104 rll (@x[$b0],@x[$b0],12);
105 rll (@x[$b1],@x[$b1],12);
107 alr (@x[$a0],@x[$b0]);
108 alr (@x[$a1],@x[$b1]);
109 xr (@x[$d0],@x[$a0]);
110 xr (@x[$d1],@x[$a1]);
111 rll (@x[$d0],@x[$d0],8);
112 rll (@x[$d1],@x[$d1],8);
118 rll (@x[$b0],@x[$b0],7);
119 rll (@x[$b1],@x[$b1],7);
121 stm ($xc,$xc_,"$stdframe+4*8+4*$c0($sp)"); # reload pair of 'c's
122 lm ($xc,$xc_,"$stdframe+4*8+4*$c2($sp)");
124 alr (@x[$a2],@x[$b2]); # Q3
125 alr (@x[$a3],@x[$b3]); # Q4
126 xr (@x[$d2],@x[$a2]);
127 xr (@x[$d3],@x[$a3]);
128 rll (@x[$d2],@x[$d2],16);
129 rll (@x[$d3],@x[$d3],16);
135 rll (@x[$b2],@x[$b2],12);
136 rll (@x[$b3],@x[$b3],12);
138 alr (@x[$a2],@x[$b2]);
139 alr (@x[$a3],@x[$b3]);
140 xr (@x[$d2],@x[$a2]);
141 xr (@x[$d3],@x[$a3]);
142 rll (@x[$d2],@x[$d2],8);
143 rll (@x[$d3],@x[$d3],8);
149 rll (@x[$b2],@x[$b2],7);
150 rll (@x[$b3],@x[$b3],7);
154 my ($a0,$b0,$c0,$d0)=@_;
155 my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
156 my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
157 my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
158 my @x=map("%v$_",(0..15));
160 vaf (@x[$a0],@x[$a0],@x[$b0]); # Q1
161 vx (@x[$d0],@x[$d0],@x[$a0]);
162 verllf (@x[$d0],@x[$d0],16);
163 vaf (@x[$a1],@x[$a1],@x[$b1]); # Q2
164 vx (@x[$d1],@x[$d1],@x[$a1]);
165 verllf (@x[$d1],@x[$d1],16);
166 vaf (@x[$a2],@x[$a2],@x[$b2]); # Q3
167 vx (@x[$d2],@x[$d2],@x[$a2]);
168 verllf (@x[$d2],@x[$d2],16);
169 vaf (@x[$a3],@x[$a3],@x[$b3]); # Q4
170 vx (@x[$d3],@x[$d3],@x[$a3]);
171 verllf (@x[$d3],@x[$d3],16);
173 vaf (@x[$c0],@x[$c0],@x[$d0]);
174 vx (@x[$b0],@x[$b0],@x[$c0]);
175 verllf (@x[$b0],@x[$b0],12);
176 vaf (@x[$c1],@x[$c1],@x[$d1]);
177 vx (@x[$b1],@x[$b1],@x[$c1]);
178 verllf (@x[$b1],@x[$b1],12);
179 vaf (@x[$c2],@x[$c2],@x[$d2]);
180 vx (@x[$b2],@x[$b2],@x[$c2]);
181 verllf (@x[$b2],@x[$b2],12);
182 vaf (@x[$c3],@x[$c3],@x[$d3]);
183 vx (@x[$b3],@x[$b3],@x[$c3]);
184 verllf (@x[$b3],@x[$b3],12);
186 vaf (@x[$a0],@x[$a0],@x[$b0]);
187 vx (@x[$d0],@x[$d0],@x[$a0]);
188 verllf (@x[$d0],@x[$d0],8);
189 vaf (@x[$a1],@x[$a1],@x[$b1]);
190 vx (@x[$d1],@x[$d1],@x[$a1]);
191 verllf (@x[$d1],@x[$d1],8);
192 vaf (@x[$a2],@x[$a2],@x[$b2]);
193 vx (@x[$d2],@x[$d2],@x[$a2]);
194 verllf (@x[$d2],@x[$d2],8);
195 vaf (@x[$a3],@x[$a3],@x[$b3]);
196 vx (@x[$d3],@x[$d3],@x[$a3]);
197 verllf (@x[$d3],@x[$d3],8);
199 vaf (@x[$c0],@x[$c0],@x[$d0]);
200 vx (@x[$b0],@x[$b0],@x[$c0]);
201 verllf (@x[$b0],@x[$b0],7);
202 vaf (@x[$c1],@x[$c1],@x[$d1]);
203 vx (@x[$b1],@x[$b1],@x[$c1]);
204 verllf (@x[$b1],@x[$b1],7);
205 vaf (@x[$c2],@x[$c2],@x[$d2]);
206 vx (@x[$b2],@x[$b2],@x[$c2]);
207 verllf (@x[$b2],@x[$b2],7);
208 vaf (@x[$c3],@x[$c3],@x[$d3]);
209 vx (@x[$b3],@x[$b3],@x[$c3]);
210 verllf (@x[$b3],@x[$b3],7);
220 vaf (@a[$_],@a[$_],@b[$_]) for (0..5);
221 vx (@d[$_],@d[$_],@a[$_]) for (0..5);
222 verllf (@d[$_],@d[$_],16) for (0..5);
224 vaf (@c[$_],@c[$_],@d[$_]) for (0..5);
225 vx (@b[$_],@b[$_],@c[$_]) for (0..5);
226 verllf (@b[$_],@b[$_],12) for (0..5);
228 vaf (@a[$_],@a[$_],@b[$_]) for (0..5);
229 vx (@d[$_],@d[$_],@a[$_]) for (0..5);
230 verllf (@d[$_],@d[$_],8) for (0..5);
232 vaf (@c[$_],@c[$_],@d[$_]) for (0..5);
233 vx (@b[$_],@b[$_],@c[$_]) for (0..5);
234 verllf (@b[$_],@b[$_],7) for (0..5);
236 vsldb (@c[$_],@c[$_],@c[$_],8) for (0..5);
237 vsldb (@b[$_],@b[$_],@b[$_],$odd?12:4) for (0..5);
238 vsldb (@d[$_],@d[$_],@d[$_],$odd?4:12) for (0..5);
241 PERLASM_BEGIN($output);
243 INCLUDE ("s390x_arch.h");
247 # void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len,
248 # const unsigned int key[8], const unsigned int counter[4])
249 my ($out,$inp,$len,$key,$counter)=map("%r$_",(2..6));
251 my $frame=$stdframe+4*20;
252 my @x=map("%r$_",(0..7,"x","x","x","x",(10..13)));
253 my @t=map("%r$_",(8,9));
255 GLOBL ("ChaCha20_ctr32");
256 TYPE ("ChaCha20_ctr32","\@function");
258 LABEL ("ChaCha20_ctr32");
259 larl ("%r1","OPENSSL_s390xcap_P");
262 &{$z? \<gr:\<r} ($len,$len); # len==0?
264 lg ("%r1","S390X_STFLE+16(%r1)");
265 &{$z? \&clgr:\&clr} ($len,"%r0");
268 tmhh ("%r1",0x4000); # check for vx bit
269 jnz (".LChaCha20_ctr32_vx");
272 &{$z? \&aghi:\&ahi} ($len,-64);
273 &{$z? \&lghi:\&lhi} ("%r1",-$frame);
274 &{$z? \&stmg:\&stm} ("%r6","%r15","6*$SIZE_T($sp)");
275 &{$z? \&slgr:\&slr} ($out,$inp); # difference
276 la ($len,"0($inp,$len)"); # end of input minus 64
277 larl ("%r7",".Lsigma");
279 la ($sp,"0(%r1,$sp)");
280 &{$z? \&stg:\&st} ("%r0","0($sp)");
282 lmg ("%r8","%r11","0($key)"); # load key
283 lmg ("%r12","%r13","0($counter)"); # load counter
284 lmg ("%r6","%r7","0(%r7)"); # load sigma constant
286 la ("%r14","0($inp)");
287 &{$z? \&stg:\&st} ($out,"$frame+3*$SIZE_T($sp)");
288 &{$z? \&stg:\&st} ($len,"$frame+4*$SIZE_T($sp)");
289 stmg ("%r6","%r13","$stdframe($sp)");# copy key schedule to stack
290 srlg (@x[12],"%r12",32); # 32-bit counter value
294 LABEL (".Loop_outer");
295 lm (@x[0],@x[7],"$stdframe+4*0($sp)"); # load x[0]-x[7]
296 lm (@t[0],@t[1],"$stdframe+4*10($sp)"); # load x[10]-x[11]
297 lm (@x[13],@x[15],"$stdframe+4*13($sp)"); # load x[13]-x[15]
298 stm (@t[0],@t[1],"$stdframe+4*8+4*10($sp)");# offload x[10]-x[11]
299 lm (@t[0],@t[1],"$stdframe+4*8($sp)"); # load x[8]-x[9]
300 st (@x[12],"$stdframe+4*12($sp)"); # save counter
301 &{$z? \&stg:\&st} ("%r14","$frame+2*$SIZE_T($sp)");# save input pointer
309 brct ("%r14",".Loop");
311 &{$z? \&lg:\&l} ("%r14","$frame+2*$SIZE_T($sp)");# pull input pointer
312 stm (@t[0],@t[1],"$stdframe+4*8+4*8($sp)"); # offload x[8]-x[9]
313 &{$z? \&lmg:\&lm} (@t[0],@t[1],"$frame+3*$SIZE_T($sp)");
315 al (@x[0],"$stdframe+4*0($sp)"); # accumulate key schedule
316 al (@x[1],"$stdframe+4*1($sp)");
317 al (@x[2],"$stdframe+4*2($sp)");
318 al (@x[3],"$stdframe+4*3($sp)");
319 al (@x[4],"$stdframe+4*4($sp)");
320 al (@x[5],"$stdframe+4*5($sp)");
321 al (@x[6],"$stdframe+4*6($sp)");
322 al (@x[7],"$stdframe+4*7($sp)");
331 al (@x[12],"$stdframe+4*12($sp)");
332 al (@x[13],"$stdframe+4*13($sp)");
333 al (@x[14],"$stdframe+4*14($sp)");
334 al (@x[15],"$stdframe+4*15($sp)");
335 lrvr (@x[12],@x[12]);
336 lrvr (@x[13],@x[13]);
337 lrvr (@x[14],@x[14]);
338 lrvr (@x[15],@x[15]);
340 la (@t[0],"0(@t[0],%r14)"); # reconstruct output pointer
341 &{$z? \&clgr:\&clr} ("%r14",@t[1]);
344 x (@x[0],"4*0(%r14)"); # xor with input
345 x (@x[1],"4*1(%r14)");
346 st (@x[0],"4*0(@t[0])"); # store output
347 x (@x[2],"4*2(%r14)");
348 st (@x[1],"4*1(@t[0])");
349 x (@x[3],"4*3(%r14)");
350 st (@x[2],"4*2(@t[0])");
351 x (@x[4],"4*4(%r14)");
352 st (@x[3],"4*3(@t[0])");
353 lm (@x[0],@x[3],"$stdframe+4*8+4*8($sp)"); # load x[8]-x[11]
354 x (@x[5],"4*5(%r14)");
355 st (@x[4],"4*4(@t[0])");
356 x (@x[6],"4*6(%r14)");
357 al (@x[0],"$stdframe+4*8($sp)");
358 st (@x[5],"4*5(@t[0])");
359 x (@x[7],"4*7(%r14)");
360 al (@x[1],"$stdframe+4*9($sp)");
361 st (@x[6],"4*6(@t[0])");
362 x (@x[12],"4*12(%r14)");
363 al (@x[2],"$stdframe+4*10($sp)");
364 st (@x[7],"4*7(@t[0])");
365 x (@x[13],"4*13(%r14)");
366 al (@x[3],"$stdframe+4*11($sp)");
367 st (@x[12],"4*12(@t[0])");
368 x (@x[14],"4*14(%r14)");
369 st (@x[13],"4*13(@t[0])");
370 x (@x[15],"4*15(%r14)");
371 st (@x[14],"4*14(@t[0])");
373 st (@x[15],"4*15(@t[0])");
378 x (@x[0],"4*8(%r14)");
379 al (@x[12],"$stdframe+4*12($sp)"); # increment counter
380 x (@x[1],"4*9(%r14)");
381 st (@x[0],"4*8(@t[0])");
382 x (@x[2],"4*10(%r14)");
383 st (@x[1],"4*9(@t[0])");
384 x (@x[3],"4*11(%r14)");
385 st (@x[2],"4*10(@t[0])");
386 st (@x[3],"4*11(@t[0])");
388 &{$z? \&clgr:\&clr} ("%r14",@t[1]); # done yet?
389 la ("%r14","64(%r14)");
397 stmg ("%r0","%r3","$stdframe+4*4($sp)"); # wipe key copy
398 stmg ("%r0","%r3","$stdframe+4*12($sp)");
400 &{$z? \&lmg:\&lm} ("%r6","%r15","$frame+6*$SIZE_T($sp)");
405 la (@t[1],"64($t[1])");
406 stm (@x[0],@x[7],"$stdframe+4*0($sp)");
407 &{$z? \&slgr:\&slr} (@t[1],"%r14");
408 lm (@x[0],@x[3],"$stdframe+4*8+4*8($sp)");
409 &{$z? \&lghi:\&lhi} (@x[6],0);
410 stm (@x[12],@x[15],"$stdframe+4*12($sp)");
411 al (@x[0],"$stdframe+4*8($sp)");
412 al (@x[1],"$stdframe+4*9($sp)");
413 al (@x[2],"$stdframe+4*10($sp)");
414 al (@x[3],"$stdframe+4*11($sp)");
419 stm (@x[0],@x[3],"$stdframe+4*8($sp)");
421 LABEL (".Loop_tail");
422 llgc (@x[4],"0(@x[6],%r14)");
423 llgc (@x[5],"$stdframe(@x[6],$sp)");
425 stc (@x[5],"0(@x[6],@t[0])");
426 la (@x[6],"1(@x[6])");
427 brct (@t[1],".Loop_tail");
430 SIZE ("ChaCha20_ctr32",".-ChaCha20_ctr32");
433 ########################################################################
434 # 4x"vertical" layout minimizes amount of instructions, but pipeline
435 # runs underutilized [because of vector instructions' high latency].
436 # On the other hand minimum amount of data it takes to fully utilize
437 # the pipeline is higher, so that effectively, short inputs would be
438 # processed slower. Hence this code path targeting <=256 bytes lengths.
441 my ($xa0,$xa1,$xa2,$xa3, $xb0,$xb1,$xb2,$xb3,
442 $xc0,$xc1,$xc2,$xc3, $xd0,$xd1,$xd2,$xd3)=map("%v$_",(0..15));
443 my @K=map("%v$_",(16..19));
445 my ($xt0,$xt1,$xt2,$xt3)=map("%v$_",(27..30));
447 my ($x00,$x10,$x20,$x30)=(0,map("r$_",(8..10)));
448 my $FRAME=$stdframe+4*16;
451 LABEL ("ChaCha20_ctr32_4x");
452 LABEL (".LChaCha20_ctr32_4x");
453 &{$z? \&stmg:\&stm} ("%r6","%r7","6*$SIZE_T($sp)");
455 std ("%f4","16*$SIZE_T+2*8($sp)");
456 std ("%f6","16*$SIZE_T+3*8($sp)");
458 &{$z? \&lghi:\&lhi} ("%r1",-$FRAME);
460 la ($sp,"0(%r1,$sp)");
461 &{$z? \&stg:\&st} ("%r0","0($sp)"); # back-chain
463 std ("%f8","$stdframe+8*0($sp)");
464 std ("%f9","$stdframe+8*1($sp)");
465 std ("%f10","$stdframe+8*2($sp)");
466 std ("%f11","$stdframe+8*3($sp)");
467 std ("%f12","$stdframe+8*4($sp)");
468 std ("%f13","$stdframe+8*5($sp)");
469 std ("%f14","$stdframe+8*6($sp)");
470 std ("%f15","$stdframe+8*7($sp)");
472 larl ("%r7",".Lsigma");
476 vl (@K[0],"0(%r7)"); # load sigma
477 vl (@K[1],"0($key)"); # load key
478 vl (@K[2],"16($key)");
479 vl (@K[3],"0($counter)"); # load counter
481 vl ($beperm,"0x40(%r7)");
482 vl ($xt1,"0x50(%r7)");
483 vrepf ($CTR,@K[3],0);
484 vlvgf (@K[3],"%r1",0); # clear @K[3].word[0]
485 vaf ($CTR,$CTR,$xt1);
487 #LABEL (".Loop_outer_4x");
488 vlm ($xa0,$xa3,"0x60(%r7)"); # load [smashed] sigma
490 vrepf ($xb0,@K[1],0); # smash the key
491 vrepf ($xb1,@K[1],1);
492 vrepf ($xb2,@K[1],2);
493 vrepf ($xb3,@K[1],3);
495 vrepf ($xc0,@K[2],0);
496 vrepf ($xc1,@K[2],1);
497 vrepf ($xc2,@K[2],2);
498 vrepf ($xc3,@K[2],3);
501 vrepf ($xd1,@K[3],1);
502 vrepf ($xd2,@K[3],2);
503 vrepf ($xd3,@K[3],3);
506 VX_lane_ROUND(0, 4, 8,12);
507 VX_lane_ROUND(0, 5,10,15);
508 brct ("%r0",".Loop_4x");
510 vaf ($xd0,$xd0,$CTR);
512 vmrhf ($xt0,$xa0,$xa1); # transpose data
513 vmrhf ($xt1,$xa2,$xa3);
514 vmrlf ($xt2,$xa0,$xa1);
515 vmrlf ($xt3,$xa2,$xa3);
516 vpdi ($xa0,$xt0,$xt1,0b0000);
517 vpdi ($xa1,$xt0,$xt1,0b0101);
518 vpdi ($xa2,$xt2,$xt3,0b0000);
519 vpdi ($xa3,$xt2,$xt3,0b0101);
521 vmrhf ($xt0,$xb0,$xb1);
522 vmrhf ($xt1,$xb2,$xb3);
523 vmrlf ($xt2,$xb0,$xb1);
524 vmrlf ($xt3,$xb2,$xb3);
525 vpdi ($xb0,$xt0,$xt1,0b0000);
526 vpdi ($xb1,$xt0,$xt1,0b0101);
527 vpdi ($xb2,$xt2,$xt3,0b0000);
528 vpdi ($xb3,$xt2,$xt3,0b0101);
530 vmrhf ($xt0,$xc0,$xc1);
531 vmrhf ($xt1,$xc2,$xc3);
532 vmrlf ($xt2,$xc0,$xc1);
533 vmrlf ($xt3,$xc2,$xc3);
534 vpdi ($xc0,$xt0,$xt1,0b0000);
535 vpdi ($xc1,$xt0,$xt1,0b0101);
536 vpdi ($xc2,$xt2,$xt3,0b0000);
537 vpdi ($xc3,$xt2,$xt3,0b0101);
539 vmrhf ($xt0,$xd0,$xd1);
540 vmrhf ($xt1,$xd2,$xd3);
541 vmrlf ($xt2,$xd0,$xd1);
542 vmrlf ($xt3,$xd2,$xd3);
543 vpdi ($xd0,$xt0,$xt1,0b0000);
544 vpdi ($xd1,$xt0,$xt1,0b0101);
545 vpdi ($xd2,$xt2,$xt3,0b0000);
546 vpdi ($xd3,$xt2,$xt3,0b0101);
549 #vaf ($CTR,$CTR,$xt0); # next counter value
551 vaf ($xa0,$xa0,@K[0]);
552 vaf ($xb0,$xb0,@K[1]);
553 vaf ($xc0,$xc0,@K[2]);
554 vaf ($xd0,$xd0,@K[3]);
556 vperm ($xa0,$xa0,$xa0,$beperm);
557 vperm ($xb0,$xb0,$xb0,$beperm);
558 vperm ($xc0,$xc0,$xc0,$beperm);
559 vperm ($xd0,$xd0,$xd0,$beperm);
561 #&{$z? \&clgfi:\&clfi} ($len,0x40);
564 vlm ($xt0,$xt3,"0($inp)");
571 vstm ($xt0,$xt3,"0($out)");
573 la ($inp,"0x40($inp)");
574 la ($out,"0x40($out)");
575 &{$z? \&aghi:\&ahi} ($len,-0x40);
578 vaf ($xa0,$xa1,@K[0]);
579 vaf ($xb0,$xb1,@K[1]);
580 vaf ($xc0,$xc1,@K[2]);
581 vaf ($xd0,$xd1,@K[3]);
583 vperm ($xa0,$xa0,$xa0,$beperm);
584 vperm ($xb0,$xb0,$xb0,$beperm);
585 vperm ($xc0,$xc0,$xc0,$beperm);
586 vperm ($xd0,$xd0,$xd0,$beperm);
588 &{$z? \&clgfi:\&clfi} ($len,0x40);
591 vlm ($xt0,$xt3,"0($inp)");
598 vstm ($xt0,$xt3,"0($out)");
600 la ($inp,"0x40($inp)");
601 la ($out,"0x40($out)");
602 &{$z? \&aghi:\&ahi} ($len,-0x40);
605 vaf ($xa0,$xa2,@K[0]);
606 vaf ($xb0,$xb2,@K[1]);
607 vaf ($xc0,$xc2,@K[2]);
608 vaf ($xd0,$xd2,@K[3]);
610 vperm ($xa0,$xa0,$xa0,$beperm);
611 vperm ($xb0,$xb0,$xb0,$beperm);
612 vperm ($xc0,$xc0,$xc0,$beperm);
613 vperm ($xd0,$xd0,$xd0,$beperm);
615 &{$z? \&clgfi:\&clfi} ($len,0x40);
618 vlm ($xt0,$xt3,"0($inp)");
625 vstm ($xt0,$xt3,"0($out)");
627 la ($inp,"0x40($inp)");
628 la ($out,"0x40($out)");
629 &{$z? \&aghi:\&ahi} ($len,-0x40);
632 vaf ($xa0,$xa3,@K[0]);
633 vaf ($xb0,$xb3,@K[1]);
634 vaf ($xc0,$xc3,@K[2]);
635 vaf ($xd0,$xd3,@K[3]);
637 vperm ($xa0,$xa0,$xa0,$beperm);
638 vperm ($xb0,$xb0,$xb0,$beperm);
639 vperm ($xc0,$xc0,$xc0,$beperm);
640 vperm ($xd0,$xd0,$xd0,$beperm);
642 &{$z? \&clgfi:\&clfi} ($len,0x40);
645 vlm ($xt0,$xt3,"0($inp)");
652 vstm ($xt0,$xt3,"0($out)");
654 #la $inp,0x40($inp));
655 #la $out,0x40($out));
657 #&{$z? \&aghi:\&ahi} $len,-0x40);
658 #jne .Loop_outer_4x);
662 ld ("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
663 ld ("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
665 ld ("%f8","$stdframe+8*0($sp)");
666 ld ("%f9","$stdframe+8*1($sp)");
667 ld ("%f10","$stdframe+8*2($sp)");
668 ld ("%f11","$stdframe+8*3($sp)");
669 ld ("%f12","$stdframe+8*4($sp)");
670 ld ("%f13","$stdframe+8*5($sp)");
671 ld ("%f14","$stdframe+8*6($sp)");
672 ld ("%f15","$stdframe+8*7($sp)");
674 &{$z? \&lmg:\&lm} ("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
675 la ($sp,"$FRAME($sp)");
682 ld ("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
683 ld ("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
685 vst ($xa0,"$stdframe+0x00($sp)");
686 vst ($xt0,"$stdframe+0x10($sp)");
687 vst ($xc0,"$stdframe+0x20($sp)");
688 vst ($xd0,"$stdframe+0x30($sp)");
691 ld ("%f8","$stdframe+8*0($sp)");
692 ld ("%f9","$stdframe+8*1($sp)");
693 ld ("%f10","$stdframe+8*2($sp)");
694 ld ("%f11","$stdframe+8*3($sp)");
696 ld ("%f12","$stdframe+8*4($sp)");
697 ld ("%f13","$stdframe+8*5($sp)");
698 ld ("%f14","$stdframe+8*6($sp)");
699 ld ("%f15","$stdframe+8*7($sp)");
701 vst ($xa0,"$stdframe+0x00($sp)");
702 vst ($xb0,"$stdframe+0x10($sp)");
703 vst ($xt0,"$stdframe+0x20($sp)");
704 vst ($xt1,"$stdframe+0x30($sp)");
708 LABEL (".Loop_tail_4x");
709 llgc ("%r5","0(%r1,$inp)");
710 llgc ("%r6","$stdframe(%r1,$sp)");
712 stc ("%r6","0(%r1,$out)");
714 brct ($len,".Loop_tail_4x");
716 &{$z? \&lmg:\&lm} ("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
717 la ($sp,"$FRAME($sp)");
719 SIZE ("ChaCha20_ctr32_4x",".-ChaCha20_ctr32_4x");
722 ########################################################################
723 # 6x"horizontal" layout is optimal fit for the platform in its current
724 # shape, more specifically for given vector instructions' latency. Well,
725 # computational part of 8x"vertical" would be faster, but it consumes
726 # all registers and dealing with that will diminish the return...
729 my ($a0,$b0,$c0,$d0, $a1,$b1,$c1,$d1,
730 $a2,$b2,$c2,$d2, $a3,$b3,$c3,$d3,
731 $a4,$b4,$c4,$d4, $a5,$b5,$c5,$d5)=map("%v$_",(0..23));
732 my @K=map("%v$_",(27,24..26));
733 my ($t0,$t1,$t2,$t3)=map("%v$_",27..30);
735 my $FRAME=$stdframe + 4*16;
737 GLOBL ("ChaCha20_ctr32_vx");
739 LABEL ("ChaCha20_ctr32_vx");
740 LABEL (".LChaCha20_ctr32_vx");
741 &{$z? \&clgfi:\&clfi} ($len,256);
742 jle (".LChaCha20_ctr32_4x");
743 &{$z? \&stmg:\&stm} ("%r6","%r7","6*$SIZE_T($sp)");
745 std ("%f4","16*$SIZE_T+2*8($sp)");
746 std ("%f6","16*$SIZE_T+3*8($sp)");
748 &{$z? \&lghi:\&lhi} ("%r1",-$FRAME);
750 la ($sp,"0(%r1,$sp)");
751 &{$z? \&stg:\&st} ("%r0","0($sp)"); # back-chain
753 std ("%f8","$FRAME-8*8($sp)");
754 std ("%f9","$FRAME-8*7($sp)");
755 std ("%f10","$FRAME-8*6($sp)");
756 std ("%f11","$FRAME-8*5($sp)");
757 std ("%f12","$FRAME-8*4($sp)");
758 std ("%f13","$FRAME-8*3($sp)");
759 std ("%f14","$FRAME-8*2($sp)");
760 std ("%f15","$FRAME-8*1($sp)");
762 larl ("%r7",".Lsigma");
765 vlm (@K[1],@K[2],"0($key)"); # load key
766 vl (@K[3],"0($counter)"); # load counter
768 vlm (@K[0],"$beperm","0(%r7)"); # load sigma, increments, ...
770 LABEL (".Loop_outer_vx");
785 vaf ($d1,@K[3],$t1); # K[3]+1
786 vaf ($d2,@K[3],$t2); # K[3]+2
787 vaf ($d3,@K[3],$t3); # K[3]+3
788 vaf ($d4,$d2,$t2); # K[3]+4
789 vaf ($d5,$d2,$t3); # K[3]+5
805 VX_ROUND($a0,$a1,$a2,$a3,$a4,$a5,
806 $b0,$b1,$b2,$b3,$b4,$b5,
807 $c0,$c1,$c2,$c3,$c4,$c5,
808 $d0,$d1,$d2,$d3,$d4,$d5,
811 VX_ROUND($a0,$a1,$a2,$a3,$a4,$a5,
812 $b0,$b1,$b2,$b3,$b4,$b5,
813 $c0,$c1,$c2,$c3,$c4,$c5,
814 $d0,$d1,$d2,$d3,$d4,$d5,
817 brct ("%r0",".Loop_vx");
824 vaf ($d1,$d1,$t1); # +K[3]+1
826 vperm ($a0,$a0,$a0,$beperm);
827 vperm ($b0,$b0,$b0,$beperm);
828 vperm ($c0,$c0,$c0,$beperm);
829 vperm ($d0,$d0,$d0,$beperm);
831 &{$z? \&clgfi:\&clfi} ($len,0x40);
834 vaf ($d2,$d2,$t2); # +K[3]+2
835 vaf ($d3,$d3,$t3); # +K[3]+3
836 vlm ($t0,$t3,"0($inp)");
843 vlm (@K[0],$t3,"0(%r7)"); # re-load sigma and increments
845 vstm ($a0,$d0,"0($out)");
847 la ($inp,"0x40($inp)");
848 la ($out,"0x40($out)");
849 &{$z? \&aghi:\&ahi} ($len,-0x40);
855 vperm ($a0,$a1,$a1,$beperm);
856 vperm ($b0,$b1,$b1,$beperm);
857 vperm ($c0,$c1,$c1,$beperm);
858 vperm ($d0,$d1,$d1,$beperm);
860 &{$z? \&clgfi:\&clfi} ($len,0x40);
863 vlm ($a1,$d1,"0($inp)");
870 vstm ($a0,$d0,"0($out)");
872 la ($inp,"0x40($inp)");
873 la ($out,"0x40($out)");
874 &{$z? \&aghi:\&ahi} ($len,-0x40);
881 vperm ($a0,$a2,$a2,$beperm);
882 vperm ($b0,$b2,$b2,$beperm);
883 vperm ($c0,$c2,$c2,$beperm);
884 vperm ($d0,$d2,$d2,$beperm);
886 &{$z? \&clgfi:\&clfi} ($len,0x40);
889 vlm ($a1,$d1,"0($inp)");
896 vstm ($a0,$d0,"0($out)");
898 la ($inp,"0x40($inp)");
899 la ($out,"0x40($out)");
900 &{$z? \&aghi:\&ahi} ($len,-0x40);
906 vaf ($d2,@K[3],$t3); # K[3]+3
908 vperm ($a0,$a3,$a3,$beperm);
909 vperm ($b0,$b3,$b3,$beperm);
910 vperm ($c0,$c3,$c3,$beperm);
911 vperm ($d0,$d3,$d3,$beperm);
913 &{$z? \&clgfi:\&clfi} ($len,0x40);
916 vaf ($d3,$d2,$t1); # K[3]+4
917 vlm ($a1,$d1,"0($inp)");
924 vstm ($a0,$d0,"0($out)");
926 la ($inp,"0x40($inp)");
927 la ($out,"0x40($out)");
928 &{$z? \&aghi:\&ahi} ($len,-0x40);
934 vaf ($d4,$d4,$d3); # +K[3]+4
935 vaf ($d3,$d3,$t1); # K[3]+5
936 vaf (@K[3],$d2,$t3); # K[3]+=6
938 vperm ($a0,$a4,$a4,$beperm);
939 vperm ($b0,$b4,$b4,$beperm);
940 vperm ($c0,$c4,$c4,$beperm);
941 vperm ($d0,$d4,$d4,$beperm);
943 &{$z? \&clgfi:\&clfi} ($len,0x40);
946 vlm ($a1,$d1,"0($inp)");
953 vstm ($a0,$d0,"0($out)");
955 la ($inp,"0x40($inp)");
956 la ($out,"0x40($out)");
957 &{$z? \&aghi:\&ahi} ($len,-0x40);
963 vaf ($d5,$d5,$d3); # +K[3]+5
965 vperm ($a0,$a5,$a5,$beperm);
966 vperm ($b0,$b5,$b5,$beperm);
967 vperm ($c0,$c5,$c5,$beperm);
968 vperm ($d0,$d5,$d5,$beperm);
970 &{$z? \&clgfi:\&clfi} ($len,0x40);
973 vlm ($a1,$d1,"0($inp)");
980 vstm ($a0,$d0,"0($out)");
982 la ($inp,"0x40($inp)");
983 la ($out,"0x40($out)");
985 &{$z? \&aghi:\&ahi} ($len,-0x40);
986 jne (".Loop_outer_vx");
990 ld ("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
991 ld ("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
993 ld ("%f8","$FRAME-8*8($sp)");
994 ld ("%f9","$FRAME-8*7($sp)");
995 ld ("%f10","$FRAME-8*6($sp)");
996 ld ("%f11","$FRAME-8*5($sp)");
997 ld ("%f12","$FRAME-8*4($sp)");
998 ld ("%f13","$FRAME-8*3($sp)");
999 ld ("%f14","$FRAME-8*2($sp)");
1000 ld ("%f15","$FRAME-8*1($sp)");
1002 &{$z? \&lmg:\&lm} ("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
1003 la ($sp,"$FRAME($sp)");
1007 LABEL (".Ltail_vx");
1009 ld ("%f4","$FRAME+16*$SIZE_T+2*8($sp)");
1010 ld ("%f6","$FRAME+16*$SIZE_T+3*8($sp)");
1012 ld ("%f8","$FRAME-8*8($sp)");
1013 ld ("%f9","$FRAME-8*7($sp)");
1014 ld ("%f10","$FRAME-8*6($sp)");
1015 ld ("%f11","$FRAME-8*5($sp)");
1016 ld ("%f12","$FRAME-8*4($sp)");
1017 ld ("%f13","$FRAME-8*3($sp)");
1018 ld ("%f14","$FRAME-8*2($sp)");
1019 ld ("%f15","$FRAME-8*1($sp)");
1021 vstm ($a0,$d0,"$stdframe($sp)");
1024 LABEL (".Loop_tail_vx");
1025 llgc ("%r5","0(%r1,$inp)");
1026 llgc ("%r6","$stdframe(%r1,$sp)");
1028 stc ("%r6","0(%r1,$out)");
1029 la ("%r1","1(%r1)");
1030 brct ($len,".Loop_tail_vx");
1032 &{$z? \&lmg:\&lm} ("%r6","%r7","$FRAME+6*$SIZE_T($sp)");
1033 la ($sp,"$FRAME($sp)");
1035 SIZE ("ChaCha20_ctr32_vx",".-ChaCha20_ctr32_vx");
1041 LONG (0x61707865,0x3320646e,0x79622d32,0x6b206574); # endian-neutral sigma
1045 LONG (0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c); # byte swap
1048 LONG (0x61707865,0x61707865,0x61707865,0x61707865); # smashed sigma
1049 LONG (0x3320646e,0x3320646e,0x3320646e,0x3320646e);
1050 LONG (0x79622d32,0x79622d32,0x79622d32,0x79622d32);
1051 LONG (0x6b206574,0x6b206574,0x6b206574,0x6b206574);
1053 ASCIZ ("\"ChaCha20 for s390x, CRYPTOGAMS by <appro\@openssl.org>\"");