From 367c55279016527515b237aeaae71bddb82ed316 Mon Sep 17 00:00:00 2001 From: Andy Polyakov Date: Mon, 5 Jun 2017 11:09:37 +0200 Subject: [PATCH] sha/asm/keccak1600-armv4.pl: add NEON code path. Reviewed-by: Rich Salz --- crypto/sha/asm/keccak1600-armv4.pl | 550 +++++++++++++++++++++++++++-- 1 file changed, 530 insertions(+), 20 deletions(-) mode change 100644 => 100755 crypto/sha/asm/keccak1600-armv4.pl diff --git a/crypto/sha/asm/keccak1600-armv4.pl b/crypto/sha/asm/keccak1600-armv4.pl old mode 100644 new mode 100755 index a560aa7c31..b4598269b5 --- a/crypto/sha/asm/keccak1600-armv4.pl +++ b/crypto/sha/asm/keccak1600-armv4.pl @@ -17,25 +17,31 @@ # # June 2017. # -# This is KECCAK_1X variant (see keccak1600.c) with bit interleaving. -# How does it compare to Keccak Code Package? It's as fast, but several -# times smaller, and is endian- and ISA-neutral. ISA neutrality means -# that minimum ISA requirement is ARMv4, yet it can be assembled even -# as ARMv7 Thumb-2. +# Non-NEON code is KECCAK_1X variant (see sha/keccak1600.c) with bit +# interleaving. How does it compare to Keccak Code Package? It's as +# fast, but several times smaller, and is endian- and ISA-neutral. ISA +# neutrality means that minimum ISA requirement is ARMv4, yet it can +# be assembled even as Thumb-2. NEON code path is KECCAK_1X_ALT with +# register layout taken from Keccak Code Package. It's also as fast, +# in fact faster by 10-15% on some processors, and endian-neutral. # ######################################################################## -# Numbers are cycles per processed byte accounting even for input bit -# interleaving. +# Numbers are cycles per processed byte. Non-NEON results account even +# for input bit interleaving [which takes ~1/4-1/3 of time]. # -# r=1600(*) r=1024 +# r=1600(*),NEON r=1088(**),NEON # -# Cortex-A7 71/+180% 103 -# Cortex-A8 48/+290% 69 -# Cortex-A15 34/+210% 49 +# Cortex-A5 80/+220%, 24 110, 36 +# Cortex-A7 71/+180%, 23 99, 34 +# Cortex-A8 48/+290%, 20 67, 30 +# Cortex-A9 48/+290%, 17 66, 26 +# Cortex-A15 34/+210%, 12 47, 18 +# Snapdragon S4 44/+230%, 16 59, 24 # -# (*) Not used in real life, meaningful as estimate for single sponge -# operation performance. Numbers after slash are improvement over -# compiler-generated KECCAK_1X reference code. +# (*) Not used in real life, meaningful as estimate for single absorb +# operation performance. Percentage after slash is improvement +# over compiler-generated KECCAK_1X reference code. +# (**) Corresponds to SHA3-256, 8KB message size. my @C = map("r$_",(0..9)); my @E = map("r$_",(10..12,14)); @@ -72,9 +78,9 @@ $code.=<<___; .code 32 #endif -.type iotas,%object +.type iotas32, %object .align 5 -iotas: +iotas32: .long 0x00000001, 0x00000000 .long 0x00000000, 0x00000089 .long 0x00000000, 0x8000008b @@ -99,7 +105,7 @@ iotas: .long 0x00000000, 0x80000088 .long 0x00000001, 0x00008000 .long 0x00000000, 0x80008082 -.size iotas,.-iotas +.size iotas32,.-iotas32 .type KeccakF1600_int, %function .align 5 @@ -254,7 +260,7 @@ KeccakF1600_enter: ldr @E[2],[sp,#324] @ load counter eor @C[2],@C[2],@E[0] ror @C[4],@E[3],#32-22 - adr @E[3],iotas + adr @E[3],iotas32 eor @C[3],@C[3],@E[1] ror @C[2],@C[2],#32-22 @ C[1] = ROL64(A[1][1] ^ E[0], rhotates[1][1]); /* D[1] */ add @E[3],@E[3],@E[2] @@ -791,11 +797,515 @@ SHA3_squeeze: .Lsqueeze_done: ldmia sp!,{r4-r10,pc} .size SHA3_squeeze,.-SHA3_squeeze -.asciz "Keccak-1600 absorb and squeeze for ARMv4, CRYPTOGAMS by " -.align 2 ___ } +$code.=<<___; +.fpu neon + +.type iotas64, %object +.align 5 +iotas64: + .quad 0x0000000000000001 + .quad 0x0000000000008082 + .quad 0x800000000000808a + .quad 0x8000000080008000 + .quad 0x000000000000808b + .quad 0x0000000080000001 + .quad 0x8000000080008081 + .quad 0x8000000000008009 + .quad 0x000000000000008a + .quad 0x0000000000000088 + .quad 0x0000000080008009 + .quad 0x000000008000000a + .quad 0x000000008000808b + .quad 0x800000000000008b + .quad 0x8000000000008089 + .quad 0x8000000000008003 + .quad 0x8000000000008002 + .quad 0x8000000000000080 + .quad 0x000000000000800a + .quad 0x800000008000000a + .quad 0x8000000080008081 + .quad 0x8000000000008080 + .quad 0x0000000080000001 + .quad 0x8000000080008008 +.size iotas64,.-iotas64 + +.type KeccakF1600_neon, %function +.align 5 +KeccakF1600_neon: + add r1, r0, #16 + adr r2, iotas64 + mov r3, #24 @ loop counter + b .Loop_neon + +.align 4 +.Loop_neon: + @ Theta + vst1.64 {q4}, [r0:64] @ offload A[0..1][4] + veor q13, q0, q5 @ A[0..1][0]^A[2..3][0] + vst1.64 {d18}, [r1:64] @ offload A[2][4] + veor q14, q1, q6 @ A[0..1][1]^A[2..3][1] + veor q15, q2, q7 @ A[0..1][2]^A[2..3][2] + veor d26, d26, d27 @ C[0]=A[0][0]^A[1][0]^A[2][0]^A[3][0] + veor d27, d28, d29 @ C[1]=A[0][1]^A[1][1]^A[2][1]^A[3][1] + veor q14, q3, q8 @ A[0..1][3]^A[2..3][3] + veor q4, q4, q9 @ A[0..1][4]^A[2..3][4] + veor d30, d30, d31 @ C[2]=A[0][2]^A[1][2]^A[2][2]^A[3][2] + veor d31, d28, d29 @ C[3]=A[0][3]^A[1][3]^A[2][3]^A[3][3] + veor d25, d8, d9 @ C[4]=A[0][4]^A[1][4]^A[2][4]^A[3][4] + veor q13, q13, q10 @ C[0..1]^=A[4][0..1] + veor q14, q15, q11 @ C[2..3]^=A[4][2..3] + veor d25, d25, d24 @ C[4]^=A[4][4] + + vadd.u64 q4, q13, q13 @ C[0..1]<<1 + vadd.u64 q15, q14, q14 @ C[2..3]<<1 + vadd.u64 d18, d25, d25 @ C[4]<<1 + vsri.u64 q4, q13, #63 @ ROL64(C[0..1],1) + vsri.u64 q15, q14, #63 @ ROL64(C[2..3],1) + vsri.u64 d18, d25, #63 @ ROL64(C[4],1) + veor d25, d25, d9 @ D[0] = C[4] ^= ROL64(C[1],1) + veor q13, q13, q15 @ D[1..2] = C[0..1] ^ ROL64(C[2..3],1) + veor d28, d28, d18 @ D[3] = C[2] ^= ROL64(C[4],1) + veor d29, d29, d8 @ D[4] = C[3] ^= ROL64(C[0],1) + + veor d0, d0, d25 @ A[0][0] ^= C[4] + veor d1, d1, d25 @ A[1][0] ^= C[4] + veor d10, d10, d25 @ A[2][0] ^= C[4] + veor d11, d11, d25 @ A[3][0] ^= C[4] + veor d20, d20, d25 @ A[4][0] ^= C[4] + + veor d2, d2, d26 @ A[0][1] ^= D[1] + veor d3, d3, d26 @ A[1][1] ^= D[1] + veor d12, d12, d26 @ A[2][1] ^= D[1] + veor d13, d13, d26 @ A[3][1] ^= D[1] + veor d21, d21, d26 @ A[4][1] ^= D[1] + vmov d26, d27 + + veor d6, d6, d28 @ A[0][3] ^= C[2] + veor d7, d7, d28 @ A[1][3] ^= C[2] + veor d16, d16, d28 @ A[2][3] ^= C[2] + veor d17, d17, d28 @ A[3][3] ^= C[2] + veor d23, d23, d28 @ A[4][3] ^= C[2] + vld1.64 {q4}, [r0:64] @ restore A[0..1][4] + vmov d28, d29 + + vld1.64 {d18}, [r1:64] @ restore A[2][4] + veor q2, q2, q13 @ A[0..1][2] ^= D[2] + veor q7, q7, q13 @ A[2..3][2] ^= D[2] + veor d22, d22, d27 @ A[4][2] ^= D[2] + + veor q4, q4, q14 @ A[0..1][4] ^= C[3] + veor q9, q9, q14 @ A[2..3][4] ^= C[3] + veor d24, d24, d29 @ A[4][4] ^= C[3] + + @ Rho + Pi + vmov d26, d2 @ C[1] = A[0][1] + vshl.u64 d2, d3, #44 + vmov d27, d4 @ C[2] = A[0][2] + vshl.u64 d4, d14, #43 + vmov d28, d6 @ C[3] = A[0][3] + vshl.u64 d6, d17, #21 + vmov d29, d8 @ C[4] = A[0][4] + vshl.u64 d8, d24, #14 + vsri.u64 d2, d3, #64-44 @ A[0][1] = ROL64(A[1][1], rhotates[1][1]) + vsri.u64 d4, d14, #64-43 @ A[0][2] = ROL64(A[2][2], rhotates[2][2]) + vsri.u64 d6, d17, #64-21 @ A[0][3] = ROL64(A[3][3], rhotates[3][3]) + vsri.u64 d8, d24, #64-14 @ A[0][4] = ROL64(A[4][4], rhotates[4][4]) + + vshl.u64 d3, d9, #20 + vshl.u64 d14, d16, #25 + vshl.u64 d17, d15, #15 + vshl.u64 d24, d21, #2 + vsri.u64 d3, d9, #64-20 @ A[1][1] = ROL64(A[1][4], rhotates[1][4]) + vsri.u64 d14, d16, #64-25 @ A[2][2] = ROL64(A[2][3], rhotates[2][3]) + vsri.u64 d17, d15, #64-15 @ A[3][3] = ROL64(A[3][2], rhotates[3][2]) + vsri.u64 d24, d21, #64-2 @ A[4][4] = ROL64(A[4][1], rhotates[4][1]) + + vshl.u64 d9, d22, #61 + @ vshl.u64 d16, d19, #8 + vshl.u64 d15, d12, #10 + vshl.u64 d21, d7, #55 + vsri.u64 d9, d22, #64-61 @ A[1][4] = ROL64(A[4][2], rhotates[4][2]) + vext.8 d16, d19, d19, #8-1 @ A[2][3] = ROL64(A[3][4], rhotates[3][4]) + vsri.u64 d15, d12, #64-10 @ A[3][2] = ROL64(A[2][1], rhotates[2][1]) + vsri.u64 d21, d7, #64-55 @ A[4][1] = ROL64(A[1][3], rhotates[1][3]) + + vshl.u64 d22, d18, #39 + @ vshl.u64 d19, d23, #56 + vshl.u64 d12, d5, #6 + vshl.u64 d7, d13, #45 + vsri.u64 d22, d18, #64-39 @ A[4][2] = ROL64(A[2][4], rhotates[2][4]) + vext.8 d19, d23, d23, #8-7 @ A[3][4] = ROL64(A[4][3], rhotates[4][3]) + vsri.u64 d12, d5, #64-6 @ A[2][1] = ROL64(A[1][2], rhotates[1][2]) + vsri.u64 d7, d13, #64-45 @ A[1][3] = ROL64(A[3][1], rhotates[3][1]) + + vshl.u64 d18, d20, #18 + vshl.u64 d23, d11, #41 + vshl.u64 d5, d10, #3 + vshl.u64 d13, d1, #36 + vsri.u64 d18, d20, #64-18 @ A[2][4] = ROL64(A[4][0], rhotates[4][0]) + vsri.u64 d23, d11, #64-41 @ A[4][3] = ROL64(A[3][0], rhotates[3][0]) + vsri.u64 d5, d10, #64-3 @ A[1][2] = ROL64(A[2][0], rhotates[2][0]) + vsri.u64 d13, d1, #64-36 @ A[3][1] = ROL64(A[1][0], rhotates[1][0]) + + vshl.u64 d1, d28, #28 + vshl.u64 d10, d26, #1 + vshl.u64 d11, d29, #27 + vshl.u64 d20, d27, #62 + vsri.u64 d1, d28, #64-28 @ A[1][0] = ROL64(C[3], rhotates[0][3]) + vsri.u64 d10, d26, #64-1 @ A[2][0] = ROL64(C[1], rhotates[0][1]) + vsri.u64 d11, d29, #64-27 @ A[3][0] = ROL64(C[4], rhotates[0][4]) + vsri.u64 d20, d27, #64-62 @ A[4][0] = ROL64(C[2], rhotates[0][2]) + + @ Chi + Iota + vbic q13, q2, q1 + vbic q14, q3, q2 + vbic q15, q4, q3 + veor q13, q13, q0 @ A[0..1][0] ^ (~A[0..1][1] & A[0..1][2]) + veor q14, q14, q1 @ A[0..1][1] ^ (~A[0..1][2] & A[0..1][3]) + veor q2, q2, q15 @ A[0..1][2] ^= (~A[0..1][3] & A[0..1][4]) + vst1.64 {q13}, [r0:64] @ offload A[0..1][0] + vbic q13, q0, q4 + vbic q15, q1, q0 + vmov q1, q14 @ A[0..1][1] + veor q3, q3, q13 @ A[0..1][3] ^= (~A[0..1][4] & A[0..1][0]) + veor q4, q4, q15 @ A[0..1][4] ^= (~A[0..1][0] & A[0..1][1]) + + vbic q13, q7, q6 + vmov q0, q5 @ A[2..3][0] + vbic q14, q8, q7 + vmov q15, q6 @ A[2..3][1] + veor q5, q5, q13 @ A[2..3][0] ^= (~A[2..3][1] & A[2..3][2]) + vbic q13, q9, q8 + veor q6, q6, q14 @ A[2..3][1] ^= (~A[2..3][2] & A[2..3][3]) + vbic q14, q0, q9 + veor q7, q7, q13 @ A[2..3][2] ^= (~A[2..3][3] & A[2..3][4]) + vbic q13, q15, q0 + veor q8, q8, q14 @ A[2..3][3] ^= (~A[2..3][4] & A[2..3][0]) + vmov q14, q10 @ A[4][0..1] + veor q9, q9, q13 @ A[2..3][4] ^= (~A[2..3][0] & A[2..3][1]) + + vld1.64 d25, [r2:64]! @ Iota[i++] + vbic d26, d22, d21 + vbic d27, d23, d22 + vld1.64 {q0}, [r0:64] @ restore A[0..1][0] + veor d20, d20, d26 @ A[4][0] ^= (~A[4][1] & A[4][2]) + vbic d26, d24, d23 + veor d21, d21, d27 @ A[4][1] ^= (~A[4][2] & A[4][3]) + vbic d27, d28, d24 + veor d22, d22, d26 @ A[4][2] ^= (~A[4][3] & A[4][4]) + vbic d26, d29, d28 + veor d23, d23, d27 @ A[4][3] ^= (~A[4][4] & A[4][0]) + veor d0, d0, d25 @ A[0][0] ^= Iota[i] + veor d24, d24, d26 @ A[4][4] ^= (~A[4][0] & A[4][1]) + + subs r3, r3, #1 + bne .Loop_neon + + bx lr +.size KeccakF1600_neon,.-KeccakF1600_neon + +.global SHA3_absorb_neon +.type SHA3_absorb_neon, %function +.align 5 +SHA3_absorb_neon: + stmdb sp!, {r4-r6,lr} + vstmdb sp!, {d8-d15} + + mov r4, r1 @ inp + mov r5, r2 @ len + mov r6, r3 @ bsz + + vld1.32 {d0}, [r0:64]! @ A[0][0] + vld1.32 {d2}, [r0:64]! @ A[0][1] + vld1.32 {d4}, [r0:64]! @ A[0][2] + vld1.32 {d6}, [r0:64]! @ A[0][3] + vld1.32 {d8}, [r0:64]! @ A[0][4] + + vld1.32 {d1}, [r0:64]! @ A[1][0] + vld1.32 {d3}, [r0:64]! @ A[1][1] + vld1.32 {d5}, [r0:64]! @ A[1][2] + vld1.32 {d7}, [r0:64]! @ A[1][3] + vld1.32 {d9}, [r0:64]! @ A[1][4] + + vld1.32 {d10}, [r0:64]! @ A[2][0] + vld1.32 {d12}, [r0:64]! @ A[2][1] + vld1.32 {d14}, [r0:64]! @ A[2][2] + vld1.32 {d16}, [r0:64]! @ A[2][3] + vld1.32 {d18}, [r0:64]! @ A[2][4] + + vld1.32 {d11}, [r0:64]! @ A[3][0] + vld1.32 {d13}, [r0:64]! @ A[3][1] + vld1.32 {d15}, [r0:64]! @ A[3][2] + vld1.32 {d17}, [r0:64]! @ A[3][3] + vld1.32 {d19}, [r0:64]! @ A[3][4] + + vld1.32 {d20-d23}, [r0:64]! @ A[4][0..3] + vld1.32 {d24}, [r0:64] @ A[4][4] + sub r0, r0, #24*8 @ rewind + b .Loop_absorb_neon + +.align 4 +.Loop_absorb_neon: + subs r12, r5, r6 @ len - bsz + blo .Labsorbed_neon + mov r5, r12 + + vld1.8 {d31}, [r4]! @ endian-neutral loads... + cmp r6, #8*2 + veor d0, d0, d31 @ A[0][0] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d2, d2, d31 @ A[0][1] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31}, [r4]! + cmp r6, #8*4 + veor d4, d4, d31 @ A[0][2] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d6, d6, d31 @ A[0][3] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31},[r4]! + cmp r6, #8*6 + veor d8, d8, d31 @ A[0][4] ^= *inp++ + blo .Lprocess_neon + + vld1.8 {d31}, [r4]! + veor d1, d1, d31 @ A[1][0] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31}, [r4]! + cmp r6, #8*8 + veor d3, d3, d31 @ A[1][1] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d5, d5, d31 @ A[1][2] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31}, [r4]! + cmp r6, #8*10 + veor d7, d7, d31 @ A[1][3] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d9, d9, d31 @ A[1][4] ^= *inp++ + beq .Lprocess_neon + + vld1.8 {d31}, [r4]! + cmp r6, #8*12 + veor d10, d10, d31 @ A[2][0] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d12, d12, d31 @ A[2][1] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31}, [r4]! + cmp r6, #8*14 + veor d14, d14, d31 @ A[2][2] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d16, d16, d31 @ A[2][3] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31}, [r4]! + cmp r6, #8*16 + veor d18, d18, d31 @ A[2][4] ^= *inp++ + blo .Lprocess_neon + + vld1.8 {d31}, [r4]! + veor d11, d11, d31 @ A[3][0] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31}, [r4]! + cmp r6, #8*18 + veor d13, d13, d31 @ A[3][1] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d15, d15, d31 @ A[3][2] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31}, [r4]! + cmp r6, #8*20 + veor d17, d17, d31 @ A[3][3] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d19, d19, d31 @ A[3][4] ^= *inp++ + beq .Lprocess_neon + + vld1.8 {d31}, [r4]! + cmp r6, #8*22 + veor d20, d20, d31 @ A[4][0] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d21, d21, d31 @ A[4][1] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31}, [r4]! + cmp r6, #8*24 + veor d22, d22, d31 @ A[4][2] ^= *inp++ + blo .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d23, d23, d31 @ A[4][3] ^= *inp++ + beq .Lprocess_neon + vld1.8 {d31}, [r4]! + veor d24, d24, d31 @ A[4][4] ^= *inp++ + +.Lprocess_neon: + bl KeccakF1600_neon + b .Loop_absorb_neon + +.align 4 +.Labsorbed_neon: + vst1.32 {d0}, [r0:64]! @ A[0][0..4] + vst1.32 {d2}, [r0:64]! + vst1.32 {d4}, [r0:64]! + vst1.32 {d6}, [r0:64]! + vst1.32 {d8}, [r0:64]! + + vst1.32 {d1}, [r0:64]! @ A[1][0..4] + vst1.32 {d3}, [r0:64]! + vst1.32 {d5}, [r0:64]! + vst1.32 {d7}, [r0:64]! + vst1.32 {d9}, [r0:64]! + + vst1.32 {d10}, [r0:64]! @ A[2][0..4] + vst1.32 {d12}, [r0:64]! + vst1.32 {d14}, [r0:64]! + vst1.32 {d16}, [r0:64]! + vst1.32 {d18}, [r0:64]! + + vst1.32 {d11}, [r0:64]! @ A[3][0..4] + vst1.32 {d13}, [r0:64]! + vst1.32 {d15}, [r0:64]! + vst1.32 {d17}, [r0:64]! + vst1.32 {d19}, [r0:64]! + + vst1.32 {d20-d23}, [r0:64]! @ A[4][0..4] + vst1.32 {d24}, [r0:64] + + mov r0, r5 @ return value + vldmia sp!, {d8-d15} + ldmia sp!, {r4-r6,pc} +.size SHA3_absorb_neon,.-SHA3_absorb_neon + +.global SHA3_squeeze_neon +.type SHA3_squeeze_neon, %function +.align 5 +SHA3_squeeze_neon: + stmdb sp!, {r4-r6,lr} + + mov r4, r1 @ out + mov r5, r2 @ len + mov r6, r3 @ bsz + mov r12, r0 @ A_flat + mov r14, r3 @ bsz + b .Loop_squeeze_neon + +.align 4 +.Loop_squeeze_neon: + cmp r5, #8 + blo .Lsqueeze_neon_tail + vld1.32 {d0}, [r12]! + vst1.8 {d0}, [r4]! @ endian-neutral store + + subs r5, r5, #8 @ len -= 8 + beq .Lsqueeze_neon_done + + subs r14, r14, #8 @ bsz -= 8 + bhi .Loop_squeeze_neon + + vstmdb sp!, {d8-d15} + + vld1.32 {d0}, [r0:64]! @ A[0][0..4] + vld1.32 {d2}, [r0:64]! + vld1.32 {d4}, [r0:64]! + vld1.32 {d6}, [r0:64]! + vld1.32 {d8}, [r0:64]! + + vld1.32 {d1}, [r0:64]! @ A[1][0..4] + vld1.32 {d3}, [r0:64]! + vld1.32 {d5}, [r0:64]! + vld1.32 {d7}, [r0:64]! + vld1.32 {d9}, [r0:64]! + + vld1.32 {d10}, [r0:64]! @ A[2][0..4] + vld1.32 {d12}, [r0:64]! + vld1.32 {d14}, [r0:64]! + vld1.32 {d16}, [r0:64]! + vld1.32 {d18}, [r0:64]! + + vld1.32 {d11}, [r0:64]! @ A[3][0..4] + vld1.32 {d13}, [r0:64]! + vld1.32 {d15}, [r0:64]! + vld1.32 {d17}, [r0:64]! + vld1.32 {d19}, [r0:64]! + + vld1.32 {d20-d23}, [r0:64]! @ A[4][0..4] + vld1.32 {d24}, [r0:64] + sub r0, r0, #24*8 @ rewind + + bl KeccakF1600_neon + + mov r12, r0 @ A_flat + vst1.32 {d0}, [r0:64]! @ A[0][0..4] + vst1.32 {d2}, [r0:64]! + vst1.32 {d4}, [r0:64]! + vst1.32 {d6}, [r0:64]! + vst1.32 {d8}, [r0:64]! + + vst1.32 {d1}, [r0:64]! @ A[1][0..4] + vst1.32 {d3}, [r0:64]! + vst1.32 {d5}, [r0:64]! + vst1.32 {d7}, [r0:64]! + vst1.32 {d9}, [r0:64]! + + vst1.32 {d10}, [r0:64]! @ A[2][0..4] + vst1.32 {d12}, [r0:64]! + vst1.32 {d14}, [r0:64]! + vst1.32 {d16}, [r0:64]! + vst1.32 {d18}, [r0:64]! + + vst1.32 {d11}, [r0:64]! @ A[3][0..4] + vst1.32 {d13}, [r0:64]! + vst1.32 {d15}, [r0:64]! + vst1.32 {d17}, [r0:64]! + vst1.32 {d19}, [r0:64]! + + vst1.32 {d20-d23}, [r0:64]! @ A[4][0..4] + mov r14, r6 @ bsz + vst1.32 {d24}, [r0:64] + mov r0, r12 @ rewind + + vldmia sp!, {d8-d15} + b .Loop_squeeze_neon + +.align 4 +.Lsqueeze_neon_tail: + ldmia r12, {r2,r3} + cmp r5, #2 + strb r2, [r4],#1 @ endian-neutral store + lsr r2, r2, #8 + blo .Lsqueeze_neon_done + strb r2, [r4], #1 + lsr r2, r2, #8 + beq .Lsqueeze_neon_done + strb r2, [r4], #1 + lsr r2, r2, #8 + cmp r5, #4 + blo .Lsqueeze_neon_done + strb r2, [r4], #1 + beq .Lsqueeze_neon_done + + strb r3, [r4], #1 + lsr r3, r3, #8 + cmp r5, #6 + blo .Lsqueeze_neon_done + strb r3, [r4], #1 + lsr r3, r3, #8 + beq .Lsqueeze_neon_done + strb r3, [r4], #1 + +.Lsqueeze_neon_done: + ldmia sp!, {r4-r6,pc} +.size SHA3_squeeze_neon,.-SHA3_squeeze_neon +.asciz "Keccak-1600 absorb and squeeze for ARMv4/NEON, CRYPTOGAMS by " +.align 2 +___ + print $code; close STDOUT; # enforce flush -- 2.25.1