3 # ====================================================================
4 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
14 # Performance is 3.54 cycles per processed byte, which is ~4.3 times
15 # faster than code generated by TI compiler. Compiler also disables
16 # interrupts for some reason, thus making interrupt response time
17 # dependent on input length. This module on the other hand is free
18 # from such limiation.
20 ($OUT,$INP,$LEN,$KEYB,$COUNTERA)=("A4","B4","A6","B6","A8");
21 ($KEYA,$COUNTERB,$STEP)=("A7","B7","A3");
23 @X= ("A16","B16","A17","B17","A18","B18","A19","B19",
24 "A20","B20","A21","B21","A22","B22","A23","B23");
25 @Y= ("A24","B24","A25","B25","A26","B26","A27","B27",
26 "A28","B28","A29","B29","A30","B30","A31","B31");
27 @DAT=("A6", "A7", "B6", "B7", "A8", "A9", "B8", "B9",
28 "A10","A11","B10","B11","A12","A13","B12","B13");
30 # yes, overlaps with @DAT, used only in 2x interleave code path...
31 @K2x=("A6", "B6", "A7", "B7", "A8", "B8", "A9", "B9",
32 "A10","B10","A11","B11","A2", "B2", "A13","B13");
37 .if .ASSEMBLER_VERSION<7000000
41 .asg ChaCha20_ctr32,_ChaCha20_ctr32
48 .global _ChaCha20_ctr32
51 .asmfunc stack_usage(40+64)
53 [!A0] BNOP RA ; no data
54 || [A0] STW FP,*SP--(40+64) ; save frame pointer and alloca(40+64)
56 [A0] STDW B13:B12,*SP[4+8] ; ABI says so
57 || [A0] MV $KEYB,$KEYA
58 || [A0] MV $COUNTERA,$COUNTERB
59 [A0] STDW B11:B10,*SP[3+8]
60 || [A0] STDW A13:A12,*FP[-3]
61 [A0] STDW A11:A10,*FP[-4]
62 || [A0] MVK 128,$STEP ; 2 * input block size
64 [A0] LDW *${KEYA}[0],@Y[4] ; load key
65 || [A0] LDW *${KEYB}[1],@Y[5]
66 || [A0] MVK 0x00007865,@Y[0] ; synthesize sigma
67 || [A0] MVK 0x0000646e,@Y[1]
68 [A0] LDW *${KEYA}[2],@Y[6]
69 || [A0] LDW *${KEYB}[3],@Y[7]
70 || [A0] MVKH 0x61700000,@Y[0]
71 || [A0] MVKH 0x33200000,@Y[1]
73 || LDW *${KEYB}[5],@Y[9]
74 || MVK 0x00002d32,@Y[2]
75 || MVK 0x00006574,@Y[3]
76 LDW *${KEYA}[6],@Y[10]
77 || LDW *${KEYB}[7],@Y[11]
78 || MVKH 0x79620000,@Y[2]
79 || MVKH 0x6b200000,@Y[3]
80 LDW *${COUNTERA}[0],@Y[12] ; load counter||nonce
81 || LDW *${COUNTERB}[1],@Y[13]
82 || CMPLTU A0,$STEP,A1 ; is length < 2*blocks?
83 LDW *${COUNTERA}[2],@Y[14]
84 || LDW *${COUNTERB}[3],@Y[15]
86 [A1] MVK 64,$STEP ; input block size
87 || MVK 10,B0 ; inner loop counter
89 DMV @Y[2],@Y[0],@X[2]:@X[0] ; copy block
90 || DMV @Y[3],@Y[1],@X[3]:@X[1]
91 ||[!A1] STDW @Y[2]:@Y[0],*FP[-12] ; offload key material to stack
92 ||[!A1] STDW @Y[3]:@Y[1],*SP[2]
93 DMV @Y[6],@Y[4],@X[6]:@X[4]
94 || DMV @Y[7],@Y[5],@X[7]:@X[5]
95 ||[!A1] STDW @Y[6]:@Y[4],*FP[-10]
96 ||[!A1] STDW @Y[7]:@Y[5],*SP[4]
97 DMV @Y[10],@Y[8],@X[10]:@X[8]
98 || DMV @Y[11],@Y[9],@X[11]:@X[9]
99 ||[!A1] STDW @Y[10]:@Y[8],*FP[-8]
100 ||[!A1] STDW @Y[11]:@Y[9],*SP[6]
101 DMV @Y[14],@Y[12],@X[14]:@X[12]
102 || DMV @Y[15],@Y[13],@X[15]:@X[13]
103 ||[!A1] MV @Y[12],@K2x[12] ; counter
104 ||[!A1] MV @Y[13],@K2x[13]
105 ||[!A1] STW @Y[14],*FP[-6*2]
106 ||[!A1] STW @Y[15],*SP[8*2]
108 { ################################################################
109 # 2x interleave gives 50% performance improvement
111 my ($a0,$a1,$a2,$a3) = (0..3);
112 my ($b0,$b1,$b2,$b3) = (4..7);
113 my ($c0,$c1,$c2,$c3) = (8..11);
114 my ($d0,$d1,$d2,$d3) = (12..15);
118 ADD @X[$b1],@X[$a1],@X[$a1]
119 || ADD @X[$b2],@X[$a2],@X[$a2]
120 || ADD @X[$b0],@X[$a0],@X[$a0]
121 || ADD @X[$b3],@X[$a3],@X[$a3]
122 || DMV @Y[2],@Y[0],@K2x[2]:@K2x[0]
123 || DMV @Y[3],@Y[1],@K2x[3]:@K2x[1]
124 XOR @X[$a1],@X[$d1],@X[$d1]
125 || XOR @X[$a2],@X[$d2],@X[$d2]
126 || XOR @X[$a0],@X[$d0],@X[$d0]
127 || XOR @X[$a3],@X[$d3],@X[$d3]
128 || DMV @Y[6],@Y[4],@K2x[6]:@K2x[4]
129 || DMV @Y[7],@Y[5],@K2x[7]:@K2x[5]
130 SWAP2 @X[$d1],@X[$d1] ; rotate by 16
131 || SWAP2 @X[$d2],@X[$d2]
132 || SWAP2 @X[$d0],@X[$d0]
133 || SWAP2 @X[$d3],@X[$d3]
135 ADD @X[$d1],@X[$c1],@X[$c1]
136 || ADD @X[$d2],@X[$c2],@X[$c2]
137 || ADD @X[$d0],@X[$c0],@X[$c0]
138 || ADD @X[$d3],@X[$c3],@X[$c3]
139 || DMV @Y[10],@Y[8],@K2x[10]:@K2x[8]
140 || DMV @Y[11],@Y[9],@K2x[11]:@K2x[9]
141 XOR @X[$c1],@X[$b1],@X[$b1]
142 || XOR @X[$c2],@X[$b2],@X[$b2]
143 || XOR @X[$c0],@X[$b0],@X[$b0]
144 || XOR @X[$c3],@X[$b3],@X[$b3]
145 || ADD 1,@Y[12],@Y[12] ; adjust counter for 2nd block
146 ROTL @X[$b1],12,@X[$b1]
147 || ROTL @X[$b2],12,@X[$b2]
148 || MV @Y[14],@K2x[14]
149 || MV @Y[15],@K2x[15]
151 ROTL @X[$b0],12,@X[$b0]
152 || ROTL @X[$b3],12,@X[$b3]
153 || ADD @Y[$b1],@Y[$a1],@Y[$a1]
154 || ADD @Y[$b2],@Y[$a2],@Y[$a2]
155 ADD @Y[$b0],@Y[$a0],@Y[$a0]
156 || ADD @Y[$b3],@Y[$a3],@Y[$a3]
158 || ADD @X[$b1],@X[$a1],@X[$a1]
159 || ADD @X[$b2],@X[$a2],@X[$a2]
160 || XOR @Y[$a1],@Y[$d1],@Y[$d1]
161 || XOR @Y[$a2],@Y[$d2],@Y[$d2]
162 XOR @Y[$a0],@Y[$d0],@Y[$d0]
163 || XOR @Y[$a3],@Y[$d3],@Y[$d3]
164 || ADD @X[$b0],@X[$a0],@X[$a0]
165 || ADD @X[$b3],@X[$a3],@X[$a3]
166 || XOR @X[$a1],@X[$d1],@X[$d1]
167 || XOR @X[$a2],@X[$d2],@X[$d2]
168 XOR @X[$a0],@X[$d0],@X[$d0]
169 || XOR @X[$a3],@X[$d3],@X[$d3]
170 || ROTL @X[$d1],8,@X[$d1]
171 || ROTL @X[$d2],8,@X[$d2]
172 || SWAP2 @Y[$d1],@Y[$d1] ; rotate by 16
173 || SWAP2 @Y[$d2],@Y[$d2]
174 || SWAP2 @Y[$d0],@Y[$d0]
175 || SWAP2 @Y[$d3],@Y[$d3]
176 ROTL @X[$d0],8,@X[$d0]
177 || ROTL @X[$d3],8,@X[$d3]
178 || ADD @Y[$d1],@Y[$c1],@Y[$c1]
179 || ADD @Y[$d2],@Y[$c2],@Y[$c2]
180 || ADD @Y[$d0],@Y[$c0],@Y[$c0]
181 || ADD @Y[$d3],@Y[$c3],@Y[$c3]
182 || BNOP middle2x1? ; protect from interrupt
184 ADD @X[$d1],@X[$c1],@X[$c1]
185 || ADD @X[$d2],@X[$c2],@X[$c2]
186 || XOR @Y[$c1],@Y[$b1],@Y[$b1]
187 || XOR @Y[$c2],@Y[$b2],@Y[$b2]
188 || XOR @Y[$c0],@Y[$b0],@Y[$b0]
189 || XOR @Y[$c3],@Y[$b3],@Y[$b3]
190 ADD @X[$d0],@X[$c0],@X[$c0]
191 || ADD @X[$d3],@X[$c3],@X[$c3]
192 || XOR @X[$c1],@X[$b1],@X[$b1]
193 || XOR @X[$c2],@X[$b2],@X[$b2]
194 || ROTL @X[$d1],0,@X[$d2] ; moved to avoid cross-path stall
195 || ROTL @X[$d2],0,@X[$d3]
196 XOR @X[$c0],@X[$b0],@X[$b0]
197 || XOR @X[$c3],@X[$b3],@X[$b3]
198 || MV @X[$d0],@X[$d1]
199 || MV @X[$d3],@X[$d0]
200 || ROTL @Y[$b1],12,@Y[$b1]
201 || ROTL @Y[$b2],12,@Y[$b2]
202 ROTL @X[$b1],7,@X[$b0] ; avoided cross-path stall
203 || ROTL @X[$b2],7,@X[$b1]
204 ROTL @X[$b0],7,@X[$b3]
205 || ROTL @X[$b3],7,@X[$b2]
208 ROTL @Y[$b0],12,@Y[$b0]
209 || ROTL @Y[$b3],12,@Y[$b3]
210 || ADD @X[$b0],@X[$a0],@X[$a0]
211 || ADD @X[$b1],@X[$a1],@X[$a1]
212 ADD @X[$b2],@X[$a2],@X[$a2]
213 || ADD @X[$b3],@X[$a3],@X[$a3]
215 || ADD @Y[$b1],@Y[$a1],@Y[$a1]
216 || ADD @Y[$b2],@Y[$a2],@Y[$a2]
217 || XOR @X[$a0],@X[$d0],@X[$d0]
218 || XOR @X[$a1],@X[$d1],@X[$d1]
219 XOR @X[$a2],@X[$d2],@X[$d2]
220 || XOR @X[$a3],@X[$d3],@X[$d3]
221 || ADD @Y[$b0],@Y[$a0],@Y[$a0]
222 || ADD @Y[$b3],@Y[$a3],@Y[$a3]
223 || XOR @Y[$a1],@Y[$d1],@Y[$d1]
224 || XOR @Y[$a2],@Y[$d2],@Y[$d2]
225 XOR @Y[$a0],@Y[$d0],@Y[$d0]
226 || XOR @Y[$a3],@Y[$d3],@Y[$d3]
227 || ROTL @Y[$d1],8,@Y[$d1]
228 || ROTL @Y[$d2],8,@Y[$d2]
229 || SWAP2 @X[$d0],@X[$d0] ; rotate by 16
230 || SWAP2 @X[$d1],@X[$d1]
231 || SWAP2 @X[$d2],@X[$d2]
232 || SWAP2 @X[$d3],@X[$d3]
233 ROTL @Y[$d0],8,@Y[$d0]
234 || ROTL @Y[$d3],8,@Y[$d3]
235 || ADD @X[$d0],@X[$c2],@X[$c2]
236 || ADD @X[$d1],@X[$c3],@X[$c3]
237 || ADD @X[$d2],@X[$c0],@X[$c0]
238 || ADD @X[$d3],@X[$c1],@X[$c1]
239 || BNOP middle2x2? ; protect from interrupt
241 ADD @Y[$d1],@Y[$c1],@Y[$c1]
242 || ADD @Y[$d2],@Y[$c2],@Y[$c2]
243 || XOR @X[$c2],@X[$b0],@X[$b0]
244 || XOR @X[$c3],@X[$b1],@X[$b1]
245 || XOR @X[$c0],@X[$b2],@X[$b2]
246 || XOR @X[$c1],@X[$b3],@X[$b3]
247 ADD @Y[$d0],@Y[$c0],@Y[$c0]
248 || ADD @Y[$d3],@Y[$c3],@Y[$c3]
249 || XOR @Y[$c1],@Y[$b1],@Y[$b1]
250 || XOR @Y[$c2],@Y[$b2],@Y[$b2]
251 || ROTL @Y[$d1],0,@Y[$d2] ; moved to avoid cross-path stall
252 || ROTL @Y[$d2],0,@Y[$d3]
253 XOR @Y[$c0],@Y[$b0],@Y[$b0]
254 || XOR @Y[$c3],@Y[$b3],@Y[$b3]
255 || MV @Y[$d0],@Y[$d1]
256 || MV @Y[$d3],@Y[$d0]
257 || ROTL @X[$b0],12,@X[$b0]
258 || ROTL @X[$b1],12,@X[$b1]
259 ROTL @Y[$b1],7,@Y[$b0] ; avoided cross-path stall
260 || ROTL @Y[$b2],7,@Y[$b1]
261 ROTL @Y[$b0],7,@Y[$b3]
262 || ROTL @Y[$b3],7,@Y[$b2]
265 ROTL @X[$b2],12,@X[$b2]
266 || ROTL @X[$b3],12,@X[$b3]
267 || ADD @Y[$b0],@Y[$a0],@Y[$a0]
268 || ADD @Y[$b1],@Y[$a1],@Y[$a1]
269 ADD @Y[$b2],@Y[$a2],@Y[$a2]
270 || ADD @Y[$b3],@Y[$a3],@Y[$a3]
272 || ADD @X[$b0],@X[$a0],@X[$a0]
273 || ADD @X[$b1],@X[$a1],@X[$a1]
274 || XOR @Y[$a0],@Y[$d0],@Y[$d0]
275 || XOR @Y[$a1],@Y[$d1],@Y[$d1]
276 XOR @Y[$a2],@Y[$d2],@Y[$d2]
277 || XOR @Y[$a3],@Y[$d3],@Y[$d3]
278 || ADD @X[$b2],@X[$a2],@X[$a2]
279 || ADD @X[$b3],@X[$a3],@X[$a3]
280 || XOR @X[$a0],@X[$d0],@X[$d0]
281 || XOR @X[$a1],@X[$d1],@X[$d1]
282 XOR @X[$a2],@X[$d2],@X[$d2]
283 || XOR @X[$a3],@X[$d3],@X[$d3]
284 || ROTL @X[$d0],8,@X[$d0]
285 || ROTL @X[$d1],8,@X[$d1]
286 || SWAP2 @Y[$d0],@Y[$d0] ; rotate by 16
287 || SWAP2 @Y[$d1],@Y[$d1]
288 || SWAP2 @Y[$d2],@Y[$d2]
289 || SWAP2 @Y[$d3],@Y[$d3]
290 ROTL @X[$d2],8,@X[$d2]
291 || ROTL @X[$d3],8,@X[$d3]
292 || ADD @Y[$d0],@Y[$c2],@Y[$c2]
293 || ADD @Y[$d1],@Y[$c3],@Y[$c3]
294 || ADD @Y[$d2],@Y[$c0],@Y[$c0]
295 || ADD @Y[$d3],@Y[$c1],@Y[$c1]
296 || BNOP bottom2x1? ; protect from interrupt
298 ADD @X[$d0],@X[$c2],@X[$c2]
299 || ADD @X[$d1],@X[$c3],@X[$c3]
300 || XOR @Y[$c2],@Y[$b0],@Y[$b0]
301 || XOR @Y[$c3],@Y[$b1],@Y[$b1]
302 || XOR @Y[$c0],@Y[$b2],@Y[$b2]
303 || XOR @Y[$c1],@Y[$b3],@Y[$b3]
304 ADD @X[$d2],@X[$c0],@X[$c0]
305 || ADD @X[$d3],@X[$c1],@X[$c1]
306 || XOR @X[$c2],@X[$b0],@X[$b0]
307 || XOR @X[$c3],@X[$b1],@X[$b1]
308 || ROTL @X[$d0],0,@X[$d3] ; moved to avoid cross-path stall
309 || ROTL @X[$d1],0,@X[$d0]
310 XOR @X[$c0],@X[$b2],@X[$b2]
311 || XOR @X[$c1],@X[$b3],@X[$b3]
312 || MV @X[$d2],@X[$d1]
313 || MV @X[$d3],@X[$d2]
314 || ROTL @Y[$b0],12,@Y[$b0]
315 || ROTL @Y[$b1],12,@Y[$b1]
316 ROTL @X[$b0],7,@X[$b1] ; avoided cross-path stall
317 || ROTL @X[$b1],7,@X[$b2]
318 ROTL @X[$b2],7,@X[$b3]
319 || ROTL @X[$b3],7,@X[$b0]
320 || [B0] SUB B0,1,B0 ; decrement inner loop counter
323 ROTL @Y[$b2],12,@Y[$b2]
324 || ROTL @Y[$b3],12,@Y[$b3]
325 || [B0] ADD @X[$b1],@X[$a1],@X[$a1] ; modulo-scheduled
326 || [B0] ADD @X[$b2],@X[$a2],@X[$a2]
327 [B0] ADD @X[$b0],@X[$a0],@X[$a0]
328 || [B0] ADD @X[$b3],@X[$a3],@X[$a3]
330 || ADD @Y[$b0],@Y[$a0],@Y[$a0]
331 || ADD @Y[$b1],@Y[$a1],@Y[$a1]
332 || [B0] XOR @X[$a1],@X[$d1],@X[$d1]
333 || [B0] XOR @X[$a2],@X[$d2],@X[$d2]
334 [B0] XOR @X[$a0],@X[$d0],@X[$d0]
335 || [B0] XOR @X[$a3],@X[$d3],@X[$d3]
336 || ADD @Y[$b2],@Y[$a2],@Y[$a2]
337 || ADD @Y[$b3],@Y[$a3],@Y[$a3]
338 || XOR @Y[$a0],@Y[$d0],@Y[$d0]
339 || XOR @Y[$a1],@Y[$d1],@Y[$d1]
340 XOR @Y[$a2],@Y[$d2],@Y[$d2]
341 || XOR @Y[$a3],@Y[$d3],@Y[$d3]
342 || ROTL @Y[$d0],8,@Y[$d0]
343 || ROTL @Y[$d1],8,@Y[$d1]
344 || [B0] SWAP2 @X[$d1],@X[$d1] ; rotate by 16
345 || [B0] SWAP2 @X[$d2],@X[$d2]
346 || [B0] SWAP2 @X[$d0],@X[$d0]
347 || [B0] SWAP2 @X[$d3],@X[$d3]
348 ROTL @Y[$d2],8,@Y[$d2]
349 || ROTL @Y[$d3],8,@Y[$d3]
350 || [B0] ADD @X[$d1],@X[$c1],@X[$c1]
351 || [B0] ADD @X[$d2],@X[$c2],@X[$c2]
352 || [B0] ADD @X[$d0],@X[$c0],@X[$c0]
353 || [B0] ADD @X[$d3],@X[$c3],@X[$c3]
354 || [B0] BNOP top2x? ; even protects from interrupt
356 ADD @Y[$d0],@Y[$c2],@Y[$c2]
357 || ADD @Y[$d1],@Y[$c3],@Y[$c3]
358 || [B0] XOR @X[$c1],@X[$b1],@X[$b1]
359 || [B0] XOR @X[$c2],@X[$b2],@X[$b2]
360 || [B0] XOR @X[$c0],@X[$b0],@X[$b0]
361 || [B0] XOR @X[$c3],@X[$b3],@X[$b3]
362 ADD @Y[$d2],@Y[$c0],@Y[$c0]
363 || ADD @Y[$d3],@Y[$c1],@Y[$c1]
364 || XOR @Y[$c2],@Y[$b0],@Y[$b0]
365 || XOR @Y[$c3],@Y[$b1],@Y[$b1]
366 || ROTL @Y[$d0],0,@Y[$d3] ; moved to avoid cross-path stall
367 || ROTL @Y[$d1],0,@Y[$d0]
368 XOR @Y[$c0],@Y[$b2],@Y[$b2]
369 || XOR @Y[$c1],@Y[$b3],@Y[$b3]
370 || MV @Y[$d2],@Y[$d1]
371 || MV @Y[$d3],@Y[$d2]
372 || [B0] ROTL @X[$b1],12,@X[$b1]
373 || [B0] ROTL @X[$b2],12,@X[$b2]
374 ROTL @Y[$b0],7,@Y[$b1] ; avoided cross-path stall
375 || ROTL @Y[$b1],7,@Y[$b2]
376 ROTL @Y[$b2],7,@Y[$b3]
377 || ROTL @Y[$b3],7,@Y[$b0]
383 ADD @K2x[0],@X[0],@X[0] ; accumulate key material
384 || ADD @K2x[1],@X[1],@X[1]
385 || ADD @K2x[2],@X[2],@X[2]
386 || ADD @K2x[3],@X[3],@X[3]
387 ADD @K2x[0],@Y[0],@Y[0]
388 || ADD @K2x[1],@Y[1],@Y[1]
389 || ADD @K2x[2],@Y[2],@Y[2]
390 || ADD @K2x[3],@Y[3],@Y[3]
391 || LDNDW *${INP}++[8],@DAT[1]:@DAT[0]
392 ADD @K2x[4],@X[4],@X[4]
393 || ADD @K2x[5],@X[5],@X[5]
394 || ADD @K2x[6],@X[6],@X[6]
395 || ADD @K2x[7],@X[7],@X[7]
396 || LDNDW *${INP}[-7],@DAT[3]:@DAT[2]
397 ADD @K2x[4],@Y[4],@Y[4]
398 || ADD @K2x[5],@Y[5],@Y[5]
399 || ADD @K2x[6],@Y[6],@Y[6]
400 || ADD @K2x[7],@Y[7],@Y[7]
401 || LDNDW *${INP}[-6],@DAT[5]:@DAT[4]
402 ADD @K2x[8],@X[8],@X[8]
403 || ADD @K2x[9],@X[9],@X[9]
404 || ADD @K2x[10],@X[10],@X[10]
405 || ADD @K2x[11],@X[11],@X[11]
406 || LDNDW *${INP}[-5],@DAT[7]:@DAT[6]
407 ADD @K2x[8],@Y[8],@Y[8]
408 || ADD @K2x[9],@Y[9],@Y[9]
409 || ADD @K2x[10],@Y[10],@Y[10]
410 || ADD @K2x[11],@Y[11],@Y[11]
411 || LDNDW *${INP}[-4],@DAT[9]:@DAT[8]
412 ADD @K2x[12],@X[12],@X[12]
413 || ADD @K2x[13],@X[13],@X[13]
414 || ADD @K2x[14],@X[14],@X[14]
415 || ADD @K2x[15],@X[15],@X[15]
416 || LDNDW *${INP}[-3],@DAT[11]:@DAT[10]
417 ADD @K2x[12],@Y[12],@Y[12]
418 || ADD @K2x[13],@Y[13],@Y[13]
419 || ADD @K2x[14],@Y[14],@Y[14]
420 || ADD @K2x[15],@Y[15],@Y[15]
421 || LDNDW *${INP}[-2],@DAT[13]:@DAT[12]
422 ADD 1,@Y[12],@Y[12] ; adjust counter for 2nd block
423 || ADD 2,@K2x[12],@K2x[12] ; increment counter
424 || LDNDW *${INP}[-1],@DAT[15]:@DAT[14]
440 || SWAP2 @X[11],@X[11]
444 || SWAP2 @X[13],@X[13]
448 || SWAP2 @X[15],@X[15]
456 || SWAP4 @X[11],@X[10]
460 || SWAP4 @X[13],@X[12]
464 || SWAP4 @X[15],@X[14]
472 || SWAP2 @Y[11],@Y[11]
476 || SWAP2 @Y[13],@Y[13]
480 || SWAP2 @Y[15],@Y[15]
486 || SWAP4 @Y[11],@Y[10]
488 || SWAP4 @Y[13],@Y[12]
490 || SWAP4 @Y[15],@Y[14]
493 XOR @DAT[0],@X[0],@X[0] ; xor 1st block
494 || XOR @DAT[3],@X[3],@X[3]
495 || XOR @DAT[2],@X[2],@X[1]
496 || XOR @DAT[1],@X[1],@X[2]
497 || LDNDW *${INP}++[8],@DAT[1]:@DAT[0]
498 XOR @DAT[4],@X[4],@X[4]
499 || XOR @DAT[7],@X[7],@X[7]
500 || LDNDW *${INP}[-7],@DAT[3]:@DAT[2]
501 XOR @DAT[6],@X[6],@X[5]
502 || XOR @DAT[5],@X[5],@X[6]
503 || LDNDW *${INP}[-6],@DAT[5]:@DAT[4]
504 XOR @DAT[8],@X[8],@X[8]
505 || XOR @DAT[11],@X[11],@X[11]
506 || LDNDW *${INP}[-5],@DAT[7]:@DAT[6]
507 XOR @DAT[10],@X[10],@X[9]
508 || XOR @DAT[9],@X[9],@X[10]
509 || LDNDW *${INP}[-4],@DAT[9]:@DAT[8]
510 XOR @DAT[12],@X[12],@X[12]
511 || XOR @DAT[15],@X[15],@X[15]
512 || LDNDW *${INP}[-3],@DAT[11]:@DAT[10]
513 XOR @DAT[14],@X[14],@X[13]
514 || XOR @DAT[13],@X[13],@X[14]
515 || LDNDW *${INP}[-2],@DAT[13]:@DAT[12]
516 [A0] SUB A0,$STEP,A0 ; SUB A0,128,A0
517 || LDNDW *${INP}[-1],@DAT[15]:@DAT[14]
519 XOR @Y[0],@DAT[0],@DAT[0] ; xor 2nd block
520 || XOR @Y[1],@DAT[1],@DAT[1]
521 || STNDW @X[2]:@X[0],*${OUT}++[8]
522 XOR @Y[2],@DAT[2],@DAT[2]
523 || XOR @Y[3],@DAT[3],@DAT[3]
524 || STNDW @X[3]:@X[1],*${OUT}[-7]
525 XOR @Y[4],@DAT[4],@DAT[4]
526 || [A0] LDDW *FP[-12],@X[2]:@X[0] ; re-load key material from stack
527 || [A0] LDDW *SP[2], @X[3]:@X[1]
528 XOR @Y[5],@DAT[5],@DAT[5]
529 || STNDW @X[6]:@X[4],*${OUT}[-6]
530 XOR @Y[6],@DAT[6],@DAT[6]
531 || XOR @Y[7],@DAT[7],@DAT[7]
532 || STNDW @X[7]:@X[5],*${OUT}[-5]
533 XOR @Y[8],@DAT[8],@DAT[8]
534 || [A0] LDDW *FP[-10],@X[6]:@X[4]
535 || [A0] LDDW *SP[4], @X[7]:@X[5]
536 XOR @Y[9],@DAT[9],@DAT[9]
537 || STNDW @X[10]:@X[8],*${OUT}[-4]
538 XOR @Y[10],@DAT[10],@DAT[10]
539 || XOR @Y[11],@DAT[11],@DAT[11]
540 || STNDW @X[11]:@X[9],*${OUT}[-3]
541 XOR @Y[12],@DAT[12],@DAT[12]
542 || [A0] LDDW *FP[-8], @X[10]:@X[8]
543 || [A0] LDDW *SP[6], @X[11]:@X[9]
544 XOR @Y[13],@DAT[13],@DAT[13]
545 || STNDW @X[14]:@X[12],*${OUT}[-2]
546 XOR @Y[14],@DAT[14],@DAT[14]
547 || XOR @Y[15],@DAT[15],@DAT[15]
548 || STNDW @X[15]:@X[13],*${OUT}[-1]
550 [A0] MV @K2x[12],@X[12]
551 || [A0] MV @K2x[13],@X[13]
552 || [A0] LDW *FP[-6*2], @X[14]
553 || [A0] LDW *SP[8*2], @X[15]
555 [A0] DMV @X[2],@X[0],@Y[2]:@Y[0] ; duplicate key material
556 || STNDW @DAT[1]:@DAT[0],*${OUT}++[8]
557 [A0] DMV @X[3],@X[1],@Y[3]:@Y[1]
558 || STNDW @DAT[3]:@DAT[2],*${OUT}[-7]
559 [A0] DMV @X[6],@X[4],@Y[6]:@Y[4]
560 || STNDW @DAT[5]:@DAT[4],*${OUT}[-6]
561 || CMPLTU A0,$STEP,A1 ; is remaining length < 2*blocks?
562 ||[!A0] BNOP epilogue?
563 [A0] DMV @X[7],@X[5],@Y[7]:@Y[5]
564 || STNDW @DAT[7]:@DAT[6],*${OUT}[-5]
565 ||[!A1] BNOP outer2x?
566 [A0] DMV @X[10],@X[8],@Y[10]:@Y[8]
567 || STNDW @DAT[9]:@DAT[8],*${OUT}[-4]
568 [A0] DMV @X[11],@X[9],@Y[11]:@Y[9]
569 || STNDW @DAT[11]:@DAT[10],*${OUT}[-3]
570 [A0] DMV @X[14],@X[12],@Y[14]:@Y[12]
571 || STNDW @DAT[13]:@DAT[12],*${OUT}[-2]
572 [A0] DMV @X[15],@X[13],@Y[15]:@Y[13]
573 || STNDW @DAT[15]:@DAT[14],*${OUT}[-1]
574 ;;===== branch to epilogue? is taken here
576 || [A0] MVK 10,B0 ; inner loop counter
577 ;;===== branch to outer2x? is taken here
580 my ($a0,$a1,$a2,$a3) = (0..3);
581 my ($b0,$b1,$b2,$b3) = (4..7);
582 my ($c0,$c1,$c2,$c3) = (8..11);
583 my ($d0,$d1,$d2,$d3) = (12..15);
587 ADD @X[$b1],@X[$a1],@X[$a1]
588 || ADD @X[$b2],@X[$a2],@X[$a2]
589 ADD @X[$b0],@X[$a0],@X[$a0]
590 || ADD @X[$b3],@X[$a3],@X[$a3]
591 || XOR @X[$a1],@X[$d1],@X[$d1]
592 || XOR @X[$a2],@X[$d2],@X[$d2]
593 XOR @X[$a0],@X[$d0],@X[$d0]
594 || XOR @X[$a3],@X[$d3],@X[$d3]
595 || SWAP2 @X[$d1],@X[$d1] ; rotate by 16
596 || SWAP2 @X[$d2],@X[$d2]
597 SWAP2 @X[$d0],@X[$d0]
598 || SWAP2 @X[$d3],@X[$d3]
600 || ADD @X[$d1],@X[$c1],@X[$c1]
601 || ADD @X[$d2],@X[$c2],@X[$c2]
602 ADD @X[$d0],@X[$c0],@X[$c0]
603 || ADD @X[$d3],@X[$c3],@X[$c3]
604 || XOR @X[$c1],@X[$b1],@X[$b1]
605 || XOR @X[$c2],@X[$b2],@X[$b2]
606 XOR @X[$c0],@X[$b0],@X[$b0]
607 || XOR @X[$c3],@X[$b3],@X[$b3]
608 || ROTL @X[$b1],12,@X[$b1]
609 || ROTL @X[$b2],12,@X[$b2]
610 ROTL @X[$b0],12,@X[$b0]
611 || ROTL @X[$b3],12,@X[$b3]
613 ADD @X[$b1],@X[$a1],@X[$a1]
614 || ADD @X[$b2],@X[$a2],@X[$a2]
615 ADD @X[$b0],@X[$a0],@X[$a0]
616 || ADD @X[$b3],@X[$a3],@X[$a3]
617 || XOR @X[$a1],@X[$d1],@X[$d1]
618 || XOR @X[$a2],@X[$d2],@X[$d2]
619 XOR @X[$a0],@X[$d0],@X[$d0]
620 || XOR @X[$a3],@X[$d3],@X[$d3]
621 || ROTL @X[$d1],8,@X[$d1]
622 || ROTL @X[$d2],8,@X[$d2]
623 ROTL @X[$d0],8,@X[$d0]
624 || ROTL @X[$d3],8,@X[$d3]
625 || BNOP middle1x? ; protect from interrupt
627 ADD @X[$d1],@X[$c1],@X[$c1]
628 || ADD @X[$d2],@X[$c2],@X[$c2]
629 ADD @X[$d0],@X[$c0],@X[$c0]
630 || ADD @X[$d3],@X[$c3],@X[$c3]
631 || XOR @X[$c1],@X[$b1],@X[$b1]
632 || XOR @X[$c2],@X[$b2],@X[$b2]
633 || ROTL @X[$d1],0,@X[$d2] ; moved to avoid cross-path stall
634 || ROTL @X[$d2],0,@X[$d3]
635 XOR @X[$c0],@X[$b0],@X[$b0]
636 || XOR @X[$c3],@X[$b3],@X[$b3]
637 || ROTL @X[$d0],0,@X[$d1]
638 || ROTL @X[$d3],0,@X[$d0]
639 ROTL @X[$b1],7,@X[$b0] ; avoided cross-path stall
640 || ROTL @X[$b2],7,@X[$b1]
641 ROTL @X[$b0],7,@X[$b3]
642 || ROTL @X[$b3],7,@X[$b2]
645 ADD @X[$b0],@X[$a0],@X[$a0]
646 || ADD @X[$b1],@X[$a1],@X[$a1]
647 ADD @X[$b2],@X[$a2],@X[$a2]
648 || ADD @X[$b3],@X[$a3],@X[$a3]
649 || XOR @X[$a0],@X[$d0],@X[$d0]
650 || XOR @X[$a1],@X[$d1],@X[$d1]
651 XOR @X[$a2],@X[$d2],@X[$d2]
652 || XOR @X[$a3],@X[$d3],@X[$d3]
653 || SWAP2 @X[$d0],@X[$d0] ; rotate by 16
654 || SWAP2 @X[$d1],@X[$d1]
655 SWAP2 @X[$d2],@X[$d2]
656 || SWAP2 @X[$d3],@X[$d3]
658 || ADD @X[$d0],@X[$c2],@X[$c2]
659 || ADD @X[$d1],@X[$c3],@X[$c3]
660 ADD @X[$d2],@X[$c0],@X[$c0]
661 || ADD @X[$d3],@X[$c1],@X[$c1]
662 || XOR @X[$c2],@X[$b0],@X[$b0]
663 || XOR @X[$c3],@X[$b1],@X[$b1]
664 XOR @X[$c0],@X[$b2],@X[$b2]
665 || XOR @X[$c1],@X[$b3],@X[$b3]
666 || ROTL @X[$b0],12,@X[$b0]
667 || ROTL @X[$b1],12,@X[$b1]
668 ROTL @X[$b2],12,@X[$b2]
669 || ROTL @X[$b3],12,@X[$b3]
671 ADD @X[$b0],@X[$a0],@X[$a0]
672 || ADD @X[$b1],@X[$a1],@X[$a1]
673 || [B0] SUB B0,1,B0 ; decrement inner loop counter
674 ADD @X[$b2],@X[$a2],@X[$a2]
675 || ADD @X[$b3],@X[$a3],@X[$a3]
676 || XOR @X[$a0],@X[$d0],@X[$d0]
677 || XOR @X[$a1],@X[$d1],@X[$d1]
678 XOR @X[$a2],@X[$d2],@X[$d2]
679 || XOR @X[$a3],@X[$d3],@X[$d3]
680 || ROTL @X[$d0],8,@X[$d0]
681 || ROTL @X[$d1],8,@X[$d1]
682 ROTL @X[$d2],8,@X[$d2]
683 || ROTL @X[$d3],8,@X[$d3]
684 || [B0] BNOP top1x? ; even protects from interrupt
686 ADD @X[$d0],@X[$c2],@X[$c2]
687 || ADD @X[$d1],@X[$c3],@X[$c3]
688 ADD @X[$d2],@X[$c0],@X[$c0]
689 || ADD @X[$d3],@X[$c1],@X[$c1]
690 || XOR @X[$c2],@X[$b0],@X[$b0]
691 || XOR @X[$c3],@X[$b1],@X[$b1]
692 || ROTL @X[$d0],0,@X[$d3] ; moved to avoid cross-path stall
693 || ROTL @X[$d1],0,@X[$d0]
694 XOR @X[$c0],@X[$b2],@X[$b2]
695 || XOR @X[$c1],@X[$b3],@X[$b3]
696 || ROTL @X[$d2],0,@X[$d1]
697 || ROTL @X[$d3],0,@X[$d2]
698 ROTL @X[$b0],7,@X[$b1] ; avoided cross-path stall
699 || ROTL @X[$b1],7,@X[$b2]
700 ROTL @X[$b2],7,@X[$b3]
701 || ROTL @X[$b3],7,@X[$b0]
702 ||[!B0] CMPLTU A0,$STEP,A1 ; less than 64 bytes left?
708 ADD @Y[0],@X[0],@X[0] ; accumulate key material
709 || ADD @Y[1],@X[1],@X[1]
710 || ADD @Y[2],@X[2],@X[2]
711 || ADD @Y[3],@X[3],@X[3]
712 ||[!A1] LDNDW *${INP}++[8],@DAT[1]:@DAT[0]
714 ADD @Y[4],@X[4],@X[4]
715 || ADD @Y[5],@X[5],@X[5]
716 || ADD @Y[6],@X[6],@X[6]
717 || ADD @Y[7],@X[7],@X[7]
718 ||[!A1] LDNDW *${INP}[-7],@DAT[3]:@DAT[2]
719 ADD @Y[8],@X[8],@X[8]
720 || ADD @Y[9],@X[9],@X[9]
721 || ADD @Y[10],@X[10],@X[10]
722 || ADD @Y[11],@X[11],@X[11]
723 ||[!A1] LDNDW *${INP}[-6],@DAT[5]:@DAT[4]
724 ADD @Y[12],@X[12],@X[12]
725 || ADD @Y[13],@X[13],@X[13]
726 || ADD @Y[14],@X[14],@X[14]
727 || ADD @Y[15],@X[15],@X[15]
728 ||[!A1] LDNDW *${INP}[-5],@DAT[7]:@DAT[6]
729 [!A1] LDNDW *${INP}[-4],@DAT[9]:@DAT[8]
730 [!A1] LDNDW *${INP}[-3],@DAT[11]:@DAT[10]
731 LDNDW *${INP}[-2],@DAT[13]:@DAT[12]
732 LDNDW *${INP}[-1],@DAT[15]:@DAT[14]
748 || SWAP2 @X[11],@X[11]
752 || SWAP2 @X[13],@X[13]
756 || SWAP2 @X[15],@X[15]
762 || SWAP4 @X[11],@X[10]
764 || SWAP4 @X[13],@X[12]
766 || SWAP4 @X[15],@X[14]
771 XOR @X[0],@DAT[0],@DAT[0] ; xor with input
772 || XOR @X[1],@DAT[1],@DAT[1]
773 || XOR @X[2],@DAT[2],@DAT[2]
774 || XOR @X[3],@DAT[3],@DAT[3]
775 || [A0] SUB A0,$STEP,A0 ; SUB A0,64,A0
776 XOR @X[4],@DAT[4],@DAT[4]
777 || XOR @X[5],@DAT[5],@DAT[5]
778 || XOR @X[6],@DAT[6],@DAT[6]
779 || XOR @X[7],@DAT[7],@DAT[7]
780 || STNDW @DAT[1]:@DAT[0],*${OUT}++[8]
781 XOR @X[8],@DAT[8],@DAT[8]
782 || XOR @X[9],@DAT[9],@DAT[9]
783 || XOR @X[10],@DAT[10],@DAT[10]
784 || XOR @X[11],@DAT[11],@DAT[11]
785 || STNDW @DAT[3]:@DAT[2],*${OUT}[-7]
786 XOR @X[12],@DAT[12],@DAT[12]
787 || XOR @X[13],@DAT[13],@DAT[13]
788 || XOR @X[14],@DAT[14],@DAT[14]
789 || XOR @X[15],@DAT[15],@DAT[15]
790 || STNDW @DAT[5]:@DAT[4],*${OUT}[-6]
792 [A0] DMV @Y[2],@Y[0],@X[2]:@X[0] ; duplicate key material
793 || [A0] DMV @Y[3],@Y[1],@X[3]:@X[1]
794 || STNDW @DAT[7]:@DAT[6],*${OUT}[-5]
795 [A0] DMV @Y[6],@Y[4],@X[6]:@X[4]
796 || [A0] DMV @Y[7],@Y[5],@X[7]:@X[5]
797 || STNDW @DAT[9]:@DAT[8],*${OUT}[-4]
798 [A0] DMV @Y[10],@Y[8],@X[10]:@X[8]
799 || [A0] DMV @Y[11],@Y[9],@X[11]:@X[9]
800 || [A0] ADD 1,@Y[12],@Y[12] ; increment counter
801 || STNDW @DAT[11]:@DAT[10],*${OUT}[-3]
802 [A0] DMV @Y[14],@Y[12],@X[14]:@X[12]
803 || [A0] DMV @Y[15],@Y[13],@X[15]:@X[13]
804 || STNDW @DAT[13]:@DAT[12],*${OUT}[-2]
805 [A0] MVK 10,B0 ; inner loop counter
806 || STNDW @DAT[15]:@DAT[14],*${OUT}[-1]
807 ;;===== branch to top1x? is taken here
810 LDDW *FP[-4],A11:A10 ; ABI says so
812 || LDDW *SP[3+8],B11:B10
813 LDDW *SP[4+8],B13:B12
815 LDW *++SP(40+64),FP ; restore frame pointer
819 LDBU *${INP}++[1],B24 ; load byte by byte
822 [!B1] BNOP epilogue? ; interrupts are disabled for whole time
823 || [A0] LDBU *${INP}++[1],B24
827 || [A0] LDBU *${INP}++[1],B24
832 || [A0] LDBU *${INP}++[1],B24
837 || [A0] LDBU *${INP}++[1],A24
842 || [A0] LDBU *${INP}++[1],A24
846 STB B25,*${OUT}++[1] ; store byte by byte
847 ||[!B1] BNOP epilogue?
849 || [A0] LDBU *${INP}++[1],A24
857 my $T = ($Xi=~/^B/?"B24":"A24"); # match @X[i] to avoid cross path
858 my $D = $T; $D=~tr/AB/BA/;
859 my $O = $D; $O=~s/24/25/;
862 ||[!B1] BNOP epilogue?
864 || [A0] LDBU *${INP}++[1],$D
869 ||[!B1] BNOP epilogue?
871 || [A0] LDBU *${INP}++[1],$T
876 ||[!B1] BNOP epilogue?
878 || [A0] LDBU *${INP}++[1],$T
883 ||[!B1] BNOP epilogue?
885 || [A0] LDBU *${INP}++[1],$T
892 foreach (1..14) { TAIL_STEP(@X[$_]); }
894 ||[!B1] BNOP epilogue?
898 || ROTL @X[15],24,B24
901 || ROTL @X[15],16,B24
911 .cstring "ChaCha20 for C64x+, CRYPTOGAMS by <appro\@openssl.org>"