my ($sb9u,$sb9t,$sbdu,$sbdt,$sbbu,$sbbt,$sbeu,$sbet)=map("v$_.16b",(24..31));
$code.=<<___;
-##
-## _aes_preheat
-##
-## Fills register %r10 -> .aes_consts (so you can -fPIC)
-## and %xmm9-%xmm15 as specified below.
-##
+//
+// _aes_preheat
+//
+// Fills register %r10 -> .aes_consts (so you can -fPIC)
+// and %xmm9-%xmm15 as specified below.
+//
.type _vpaes_encrypt_preheat,%function
.align 4
_vpaes_encrypt_preheat:
ret
.size _vpaes_encrypt_preheat,.-_vpaes_encrypt_preheat
-##
-## _aes_encrypt_core
-##
-## AES-encrypt %xmm0.
-##
-## Inputs:
-## %xmm0 = input
-## %xmm9-%xmm15 as in _vpaes_preheat
-## (%rdx) = scheduled keys
-##
-## Output in %xmm0
-## Clobbers %xmm1-%xmm5, %r9, %r10, %r11, %rax
-## Preserves %xmm6 - %xmm8 so you get some local vectors
-##
-##
+//
+// _aes_encrypt_core
+//
+// AES-encrypt %xmm0.
+//
+// Inputs:
+// %xmm0 = input
+// %xmm9-%xmm15 as in _vpaes_preheat
+// (%rdx) = scheduled keys
+//
+// Output in %xmm0
+// Clobbers %xmm1-%xmm5, %r9, %r10, %r11, %rax
+// Preserves %xmm6 - %xmm8 so you get some local vectors
+//
+//
.type _vpaes_encrypt_core,%function
.align 4
_vpaes_encrypt_core:
ret
.size _vpaes_decrypt_preheat,.-_vpaes_decrypt_preheat
-##
-## Decryption core
-##
-## Same API as encryption core.
-##
+//
+// Decryption core
+//
+// Same API as encryption core.
+//
.type _vpaes_decrypt_core,%function
.align 4
_vpaes_decrypt_core:
my ($invlo,$invhi,$iptlo,$ipthi,$rcon) = map("v$_.16b",(18..21,8));
$code.=<<___;
-########################################################
-## ##
-## AES key schedule ##
-## ##
-########################################################
+////////////////////////////////////////////////////////
+// //
+// AES key schedule //
+// //
+////////////////////////////////////////////////////////
.type _vpaes_key_preheat,%function
.align 4
_vpaes_key_preheat:
b.eq .Lschedule_192
// 128: fall though
-##
-## .schedule_128
-##
-## 128-bit specific part of key schedule.
-##
-## This schedule is really simple, because all its parts
-## are accomplished by the subroutines.
-##
+//
+// .schedule_128
+//
+// 128-bit specific part of key schedule.
+//
+// This schedule is really simple, because all its parts
+// are accomplished by the subroutines.
+//
.Lschedule_128:
mov $inp, #10 // mov \$10, %esi
bl _vpaes_schedule_mangle // write output
b .Loop_schedule_128
-##
-## .aes_schedule_192
-##
-## 192-bit specific part of key schedule.
-##
-## The main body of this schedule is the same as the 128-bit
-## schedule, but with more smearing. The long, high side is
-## stored in %xmm7 as before, and the short, low side is in
-## the high bits of %xmm6.
-##
-## This schedule is somewhat nastier, however, because each
-## round produces 192 bits of key material, or 1.5 round keys.
-## Therefore, on each cycle we do 2 rounds and produce 3 round
-## keys.
-##
+//
+// .aes_schedule_192
+//
+// 192-bit specific part of key schedule.
+//
+// The main body of this schedule is the same as the 128-bit
+// schedule, but with more smearing. The long, high side is
+// stored in %xmm7 as before, and the short, low side is in
+// the high bits of %xmm6.
+//
+// This schedule is somewhat nastier, however, because each
+// round produces 192 bits of key material, or 1.5 round keys.
+// Therefore, on each cycle we do 2 rounds and produce 3 round
+// keys.
+//
.align 4
.Lschedule_192:
sub $inp, $inp, #8
bl _vpaes_schedule_192_smear
b .Loop_schedule_192
-##
-## .aes_schedule_256
-##
-## 256-bit specific part of key schedule.
-##
-## The structure here is very similar to the 128-bit
-## schedule, but with an additional "low side" in
-## %xmm6. The low side's rounds are the same as the
-## high side's, except no rcon and no rotation.
-##
+//
+// .aes_schedule_256
+//
+// 256-bit specific part of key schedule.
+//
+// The structure here is very similar to the 128-bit
+// schedule, but with an additional "low side" in
+// %xmm6. The low side's rounds are the same as the
+// high side's, except no rcon and no rotation.
+//
.align 4
.Lschedule_256:
ld1 {v0.16b}, [$inp] // vmovdqu 16(%rdi),%xmm0 # load key part 2 (unaligned)
b .Loop_schedule_256
-##
-## .aes_schedule_mangle_last
-##
-## Mangler for last round of key schedule
-## Mangles %xmm0
-## when encrypting, outputs out(%xmm0) ^ 63
-## when decrypting, outputs unskew(%xmm0)
-##
-## Always called right before return... jumps to cleanup and exits
-##
+//
+// .aes_schedule_mangle_last
+//
+// Mangler for last round of key schedule
+// Mangles %xmm0
+// when encrypting, outputs out(%xmm0) ^ 63
+// when decrypting, outputs unskew(%xmm0)
+//
+// Always called right before return... jumps to cleanup and exits
+//
.align 4
.Lschedule_mangle_last:
// schedule last round key from xmm0
ret
.size _vpaes_schedule_core,.-_vpaes_schedule_core
-##
-## .aes_schedule_192_smear
-##
-## Smear the short, low side in the 192-bit key schedule.
-##
-## Inputs:
-## %xmm7: high side, b a x y
-## %xmm6: low side, d c 0 0
-## %xmm13: 0
-##
-## Outputs:
-## %xmm6: b+c+d b+c 0 0
-## %xmm0: b+c+d b+c b a
-##
+//
+// .aes_schedule_192_smear
+//
+// Smear the short, low side in the 192-bit key schedule.
+//
+// Inputs:
+// %xmm7: high side, b a x y
+// %xmm6: low side, d c 0 0
+// %xmm13: 0
+//
+// Outputs:
+// %xmm6: b+c+d b+c 0 0
+// %xmm0: b+c+d b+c b a
+//
.type _vpaes_schedule_192_smear,%function
.align 4
_vpaes_schedule_192_smear:
ret
.size _vpaes_schedule_192_smear,.-_vpaes_schedule_192_smear
-##
-## .aes_schedule_round
-##
-## Runs one main round of the key schedule on %xmm0, %xmm7
-##
-## Specifically, runs subbytes on the high dword of %xmm0
-## then rotates it by one byte and xors into the low dword of
-## %xmm7.
-##
-## Adds rcon from low byte of %xmm8, then rotates %xmm8 for
-## next rcon.
-##
-## Smears the dwords of %xmm7 by xoring the low into the
-## second low, result into third, result into highest.
-##
-## Returns results in %xmm7 = %xmm0.
-## Clobbers %xmm1-%xmm4, %r11.
-##
+//
+// .aes_schedule_round
+//
+// Runs one main round of the key schedule on %xmm0, %xmm7
+//
+// Specifically, runs subbytes on the high dword of %xmm0
+// then rotates it by one byte and xors into the low dword of
+// %xmm7.
+//
+// Adds rcon from low byte of %xmm8, then rotates %xmm8 for
+// next rcon.
+//
+// Smears the dwords of %xmm7 by xoring the low into the
+// second low, result into third, result into highest.
+//
+// Returns results in %xmm7 = %xmm0.
+// Clobbers %xmm1-%xmm4, %r11.
+//
.type _vpaes_schedule_round,%function
.align 4
_vpaes_schedule_round:
ret
.size _vpaes_schedule_round,.-_vpaes_schedule_round
-##
-## .aes_schedule_transform
-##
-## Linear-transform %xmm0 according to tables at (%r11)
-##
-## Requires that %xmm9 = 0x0F0F... as in preheat
-## Output in %xmm0
-## Clobbers %xmm1, %xmm2
-##
+//
+// .aes_schedule_transform
+//
+// Linear-transform %xmm0 according to tables at (%r11)
+//
+// Requires that %xmm9 = 0x0F0F... as in preheat
+// Output in %xmm0
+// Clobbers %xmm1, %xmm2
+//
.type _vpaes_schedule_transform,%function
.align 4
_vpaes_schedule_transform:
ret
.size _vpaes_schedule_transform,.-_vpaes_schedule_transform
-##
-## .aes_schedule_mangle
-##
-## Mangle xmm0 from (basis-transformed) standard version
-## to our version.
-##
-## On encrypt,
-## xor with 0x63
-## multiply by circulant 0,1,1,1
-## apply shiftrows transform
-##
-## On decrypt,
-## xor with 0x63
-## multiply by "inverse mixcolumns" circulant E,B,D,9
-## deskew
-## apply shiftrows transform
-##
-##
-## Writes out to (%rdx), and increments or decrements it
-## Keeps track of round number mod 4 in %r8
-## Preserves xmm0
-## Clobbers xmm1-xmm5
-##
+//
+// .aes_schedule_mangle
+//
+// Mangle xmm0 from (basis-transformed) standard version
+// to our version.
+//
+// On encrypt,
+// xor with 0x63
+// multiply by circulant 0,1,1,1
+// apply shiftrows transform
+//
+// On decrypt,
+// xor with 0x63
+// multiply by "inverse mixcolumns" circulant E,B,D,9
+// deskew
+// apply shiftrows transform
+//
+//
+// Writes out to (%rdx), and increments or decrements it
+// Keeps track of round number mod 4 in %r8
+// Preserves xmm0
+// Clobbers xmm1-xmm5
+//
.type _vpaes_schedule_mangle,%function
.align 4
_vpaes_schedule_mangle:
csel x0,xzr,x0,eq
b.eq .Lno_key
-#ifdef __ILP32__
- ldrsw $t1,.LOPENSSL_armcap_P
-#else
- ldr $t1,.LOPENSSL_armcap_P
-#endif
- adr $t0,.LOPENSSL_armcap_P
+ adrp x17,OPENSSL_armcap_P
+ ldr w17,[x17,#:lo12:OPENSSL_armcap_P]
ldp $r0,$r1,[$inp] // load key
mov $s1,#0xfffffffc0fffffff
movk $s1,#0x0fff,lsl#48
- ldr w17,[$t0,$t1]
#ifdef __ARMEB__
rev $r0,$r0 // flip bytes
rev $r1,$r1
tst w17,#ARMV7_NEON
- adr $d0,poly1305_blocks
- adr $r0,poly1305_blocks_neon
- adr $d1,poly1305_emit
- adr $r1,poly1305_emit_neon
+ adr $d0,.Lpoly1305_blocks
+ adr $r0,.Lpoly1305_blocks_neon
+ adr $d1,.Lpoly1305_emit
+ adr $r1,.Lpoly1305_emit_neon
csel $d0,$d0,$r0,eq
csel $d1,$d1,$r1,eq
.type poly1305_blocks,%function
.align 5
poly1305_blocks:
+.Lpoly1305_blocks:
ands $len,$len,#-16
b.eq .Lno_data
.type poly1305_emit,%function
.align 5
poly1305_emit:
+.Lpoly1305_emit:
ldp $h0,$h1,[$ctx] // load hash base 2^64
ldr $h2,[$ctx,#16]
ldp $t0,$t1,[$nonce] // load nonce
.type poly1305_blocks_neon,%function
.align 5
poly1305_blocks_neon:
+.Lpoly1305_blocks_neon:
ldr $is_base2_26,[$ctx,#24]
cmp $len,#128
b.hs .Lblocks_neon
- cbz $is_base2_26,poly1305_blocks
+ cbz $is_base2_26,.Lpoly1305_blocks
.Lblocks_neon:
.inst 0xd503233f // paciasp
csel $in2,$zeros,$in2,lo
mov x4,#1
- str x4,[$ctx,#-24] // set is_base2_26
+ stur x4,[$ctx,#-24] // set is_base2_26
sub $ctx,$ctx,#48 // restore original $ctx
b .Ldo_neon
.type poly1305_emit_neon,%function
.align 5
poly1305_emit_neon:
+.Lpoly1305_emit_neon:
ldr $is_base2_26,[$ctx,#24]
cbz $is_base2_26,poly1305_emit
.align 5
.Lzeros:
.long 0,0,0,0,0,0,0,0
-.LOPENSSL_armcap_P:
-#ifdef __ILP32__
-.long OPENSSL_armcap_P-.
-#else
-.quad OPENSSL_armcap_P-.
-#endif
.asciz "Poly1305 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align 2
___
lsr @Xx[$i+1],@Xx[$i],#32
___
$code.=<<___ if ($i<14 && !($i&1));
- ldr @Xx[$i+2],[$inp,#`($i+2)*4-64`]
+ ldur @Xx[$i+2],[$inp,#`($i+2)*4-64`]
___
$code.=<<___ if ($i<14 && ($i&1));
-#ifdef __ARMEB__
+#ifdef __AARCH64EB__
ror @Xx[$i+1],@Xx[$i+1],#32
#else
rev32 @Xx[$i+1],@Xx[$i+1]
}
$code.=<<___;
-#include "arm_arch.h"
+#ifndef __KERNEL__
+# include "arm_arch.h"
+.extern OPENSSL_armcap_P
+#endif
.text
-.extern OPENSSL_armcap_P
.globl sha1_block_data_order
.type sha1_block_data_order,%function
.align 6
sha1_block_data_order:
-#ifdef __ILP32__
- ldrsw x16,.LOPENSSL_armcap_P
-#else
- ldr x16,.LOPENSSL_armcap_P
-#endif
- adr x17,.LOPENSSL_armcap_P
- add x16,x16,x17
- ldr w16,[x16]
+ adrp x16,OPENSSL_armcap_P
+ ldr w16,[x16,#:lo12:OPENSSL_armcap_P]
tst w16,#ARMV8_SHA1
b.ne .Lv8_entry
movz $K,#0x7999
sub $num,$num,#1
movk $K,#0x5a82,lsl#16
-#ifdef __ARMEB__
+#ifdef __AARCH64EB__
ror $Xx[0],@Xx[0],#32
#else
rev32 @Xx[0],@Xx[0]
.long 0x6ed9eba1,0x6ed9eba1,0x6ed9eba1,0x6ed9eba1 //K_20_39
.long 0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc,0x8f1bbcdc //K_40_59
.long 0xca62c1d6,0xca62c1d6,0xca62c1d6,0xca62c1d6 //K_60_79
-.LOPENSSL_armcap_P:
-#ifdef __ILP32__
-.long OPENSSL_armcap_P-.
-#else
-.quad OPENSSL_armcap_P-.
-#endif
.asciz "SHA1 block transform for ARMv8, CRYPTOGAMS by <appro\@openssl.org>"
.align 2
+#if !defined(__KERNELL__) && !defined(_WIN64)
.comm OPENSSL_armcap_P,4,4
+#endif
___
}}}
projects / oweals / openssl.git / commitdiff