2 * Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
10 #ifndef OSSL_AES_PLATFORM_H
11 # define OSSL_AES_PLATFORM_H
13 # include "openssl/aes.h"
16 int vpaes_set_encrypt_key(const unsigned char *userKey, int bits,
18 int vpaes_set_decrypt_key(const unsigned char *userKey, int bits,
20 void vpaes_encrypt(const unsigned char *in, unsigned char *out,
22 void vpaes_decrypt(const unsigned char *in, unsigned char *out,
24 void vpaes_cbc_encrypt(const unsigned char *in,
27 const AES_KEY *key, unsigned char *ivec, int enc);
28 # endif /* VPAES_ASM */
31 void bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out,
32 size_t length, const AES_KEY *key,
33 unsigned char ivec[16], int enc);
34 void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
35 size_t len, const AES_KEY *key,
36 const unsigned char ivec[16]);
37 void bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out,
38 size_t len, const AES_KEY *key1,
39 const AES_KEY *key2, const unsigned char iv[16]);
40 void bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out,
41 size_t len, const AES_KEY *key1,
42 const AES_KEY *key2, const unsigned char iv[16]);
43 # endif /* BSAES_ASM */
46 void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,
47 size_t blocks, const AES_KEY *key,
48 const unsigned char ivec[AES_BLOCK_SIZE]);
49 # endif /* AES_CTR_ASM */
52 void AES_xts_encrypt(const unsigned char *inp, unsigned char *out, size_t len,
53 const AES_KEY *key1, const AES_KEY *key2,
54 const unsigned char iv[16]);
55 void AES_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len,
56 const AES_KEY *key1, const AES_KEY *key2,
57 const unsigned char iv[16]);
58 # endif /* AES_XTS_ASM */
60 # if defined(OPENSSL_CPUID_OBJ)
61 # if (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
62 # include "ppc_arch.h"
64 # define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC)
66 # define HWAES_CAPABLE (OPENSSL_ppccap_P & PPC_CRYPTO207)
67 # define HWAES_set_encrypt_key aes_p8_set_encrypt_key
68 # define HWAES_set_decrypt_key aes_p8_set_decrypt_key
69 # define HWAES_encrypt aes_p8_encrypt
70 # define HWAES_decrypt aes_p8_decrypt
71 # define HWAES_cbc_encrypt aes_p8_cbc_encrypt
72 # define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
73 # define HWAES_xts_encrypt aes_p8_xts_encrypt
74 # define HWAES_xts_decrypt aes_p8_xts_decrypt
77 # if (defined(__arm__) || defined(__arm) || defined(__aarch64__))
78 # include "arm_arch.h"
79 # if __ARM_MAX_ARCH__>=7
80 # if defined(BSAES_ASM)
81 # define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
83 # if defined(VPAES_ASM)
84 # define VPAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
86 # define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
87 # define HWAES_set_encrypt_key aes_v8_set_encrypt_key
88 # define HWAES_set_decrypt_key aes_v8_set_decrypt_key
89 # define HWAES_encrypt aes_v8_encrypt
90 # define HWAES_decrypt aes_v8_decrypt
91 # define HWAES_cbc_encrypt aes_v8_cbc_encrypt
92 # define HWAES_ecb_encrypt aes_v8_ecb_encrypt
93 # define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
94 # define AES_PMULL_CAPABLE ((OPENSSL_armcap_P & ARMV8_PMULL) && (OPENSSL_armcap_P & ARMV8_AES))
95 # define AES_GCM_ENC_BYTES 512
96 # define AES_GCM_DEC_BYTES 512
97 # if __ARM_MAX_ARCH__>=8
98 # define AES_gcm_encrypt armv8_aes_gcm_encrypt
99 # define AES_gcm_decrypt armv8_aes_gcm_decrypt
100 # define AES_GCM_ASM(gctx) ((gctx)->ctr==aes_v8_ctr32_encrypt_blocks && \
101 (gctx)->gcm.ghash==gcm_ghash_v8)
102 size_t aes_gcm_enc_128_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext,
103 uint64_t *Xi, unsigned char ivec[16], const void *key);
104 size_t aes_gcm_enc_192_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext,
105 uint64_t *Xi, unsigned char ivec[16], const void *key);
106 size_t aes_gcm_enc_256_kernel(const uint8_t * plaintext, uint64_t plaintext_length, uint8_t * ciphertext,
107 uint64_t *Xi, unsigned char ivec[16], const void *key);
108 size_t aes_gcm_dec_128_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext,
109 uint64_t *Xi, unsigned char ivec[16], const void *key);
110 size_t aes_gcm_dec_192_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext,
111 uint64_t *Xi, unsigned char ivec[16], const void *key);
112 size_t aes_gcm_dec_256_kernel(const uint8_t * ciphertext, uint64_t plaintext_length, uint8_t * plaintext,
113 uint64_t *Xi, unsigned char ivec[16], const void *key);
114 size_t armv8_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len, const void *key,
115 unsigned char ivec[16], u64 *Xi);
116 size_t armv8_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len, const void *key,
117 unsigned char ivec[16], u64 *Xi);
118 void gcm_ghash_v8(u64 Xi[2],const u128 Htable[16],const u8 *inp, size_t len);
122 # endif /* OPENSSL_CPUID_OBJ */
124 # if defined(AES_ASM) && ( \
125 defined(__x86_64) || defined(__x86_64__) || \
126 defined(_M_AMD64) || defined(_M_X64) )
127 # define AES_CBC_HMAC_SHA_CAPABLE 1
128 # define AESNI_CBC_HMAC_SHA_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
131 # if defined(AES_ASM) && !defined(I386_ONLY) && ( \
132 ((defined(__i386) || defined(__i386__) || \
133 defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
134 defined(__x86_64) || defined(__x86_64__) || \
135 defined(_M_AMD64) || defined(_M_X64) )
139 # define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
141 # define VPAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
144 # define BSAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
147 # define AES_GCM_ENC_BYTES 32
148 # define AES_GCM_DEC_BYTES 16
150 int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
152 int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
155 void aesni_encrypt(const unsigned char *in, unsigned char *out,
157 void aesni_decrypt(const unsigned char *in, unsigned char *out,
160 void aesni_ecb_encrypt(const unsigned char *in,
162 size_t length, const AES_KEY *key, int enc);
163 void aesni_cbc_encrypt(const unsigned char *in,
166 const AES_KEY *key, unsigned char *ivec, int enc);
167 # ifndef OPENSSL_NO_OCB
168 void aesni_ocb_encrypt(const unsigned char *in, unsigned char *out,
169 size_t blocks, const void *key,
170 size_t start_block_num,
171 unsigned char offset_i[16],
172 const unsigned char L_[][16],
173 unsigned char checksum[16]);
174 void aesni_ocb_decrypt(const unsigned char *in, unsigned char *out,
175 size_t blocks, const void *key,
176 size_t start_block_num,
177 unsigned char offset_i[16],
178 const unsigned char L_[][16],
179 unsigned char checksum[16]);
180 # endif /* OPENSSL_NO_OCB */
182 void aesni_ctr32_encrypt_blocks(const unsigned char *in,
185 const void *key, const unsigned char *ivec);
187 void aesni_xts_encrypt(const unsigned char *in,
190 const AES_KEY *key1, const AES_KEY *key2,
191 const unsigned char iv[16]);
193 void aesni_xts_decrypt(const unsigned char *in,
196 const AES_KEY *key1, const AES_KEY *key2,
197 const unsigned char iv[16]);
199 void aesni_ccm64_encrypt_blocks(const unsigned char *in,
203 const unsigned char ivec[16],
204 unsigned char cmac[16]);
206 void aesni_ccm64_decrypt_blocks(const unsigned char *in,
210 const unsigned char ivec[16],
211 unsigned char cmac[16]);
213 # if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
214 size_t aesni_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
215 const void *key, unsigned char ivec[16], u64 *Xi);
216 size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
217 const void *key, unsigned char ivec[16], u64 *Xi);
218 void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *in, size_t len);
220 # define AES_gcm_encrypt aesni_gcm_encrypt
221 # define AES_gcm_decrypt aesni_gcm_decrypt
222 # define AES_GCM_ASM(ctx) (ctx->ctr == aesni_ctr32_encrypt_blocks && \
223 ctx->gcm.ghash == gcm_ghash_avx)
227 # elif defined(AES_ASM) && (defined(__sparc) || defined(__sparc__))
229 /* Fujitsu SPARC64 X support */
230 extern unsigned int OPENSSL_sparcv9cap_P[];
231 # include "sparc_arch.h"
233 # define SPARC_AES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_AES)
234 # define HWAES_CAPABLE (OPENSSL_sparcv9cap_P[0] & SPARCV9_FJAESX)
235 # define HWAES_set_encrypt_key aes_fx_set_encrypt_key
236 # define HWAES_set_decrypt_key aes_fx_set_decrypt_key
237 # define HWAES_encrypt aes_fx_encrypt
238 # define HWAES_decrypt aes_fx_decrypt
239 # define HWAES_cbc_encrypt aes_fx_cbc_encrypt
240 # define HWAES_ctr32_encrypt_blocks aes_fx_ctr32_encrypt_blocks
242 void aes_t4_set_encrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
243 void aes_t4_set_decrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
244 void aes_t4_encrypt(const unsigned char *in, unsigned char *out,
246 void aes_t4_decrypt(const unsigned char *in, unsigned char *out,
249 * Key-length specific subroutines were chosen for following reason.
250 * Each SPARC T4 core can execute up to 8 threads which share core's
251 * resources. Loading as much key material to registers allows to
252 * minimize references to shared memory interface, as well as amount
253 * of instructions in inner loops [much needed on T4]. But then having
254 * non-key-length specific routines would require conditional branches
255 * either in inner loops or on subroutines' entries. Former is hardly
256 * acceptable, while latter means code size increase to size occupied
257 * by multiple key-length specific subroutines, so why fight?
259 void aes128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
260 size_t len, const AES_KEY *key,
261 unsigned char *ivec);
262 void aes128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
263 size_t len, const AES_KEY *key,
264 unsigned char *ivec);
265 void aes192_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
266 size_t len, const AES_KEY *key,
267 unsigned char *ivec);
268 void aes192_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
269 size_t len, const AES_KEY *key,
270 unsigned char *ivec);
271 void aes256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
272 size_t len, const AES_KEY *key,
273 unsigned char *ivec);
274 void aes256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
275 size_t len, const AES_KEY *key,
276 unsigned char *ivec);
277 void aes128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
278 size_t blocks, const AES_KEY *key,
279 unsigned char *ivec);
280 void aes192_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
281 size_t blocks, const AES_KEY *key,
282 unsigned char *ivec);
283 void aes256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
284 size_t blocks, const AES_KEY *key,
285 unsigned char *ivec);
286 void aes128_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
287 size_t blocks, const AES_KEY *key1,
288 const AES_KEY *key2, const unsigned char *ivec);
289 void aes128_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
290 size_t blocks, const AES_KEY *key1,
291 const AES_KEY *key2, const unsigned char *ivec);
292 void aes256_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
293 size_t blocks, const AES_KEY *key1,
294 const AES_KEY *key2, const unsigned char *ivec);
295 void aes256_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
296 size_t blocks, const AES_KEY *key1,
297 const AES_KEY *key2, const unsigned char *ivec);
299 # elif defined(OPENSSL_CPUID_OBJ) && defined(__s390__)
300 /* IBM S390X support */
301 # include "s390x_arch.h"
304 /* Convert key size to function code: [16,24,32] -> [18,19,20]. */
305 # define S390X_AES_FC(keylen) (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
307 /* Most modes of operation need km for partial block processing. */
308 # define S390X_aes_128_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
309 S390X_CAPBIT(S390X_AES_128))
310 # define S390X_aes_192_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
311 S390X_CAPBIT(S390X_AES_192))
312 # define S390X_aes_256_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
313 S390X_CAPBIT(S390X_AES_256))
315 # define S390X_aes_128_cbc_CAPABLE 1 /* checked by callee */
316 # define S390X_aes_192_cbc_CAPABLE 1
317 # define S390X_aes_256_cbc_CAPABLE 1
319 # define S390X_aes_128_ecb_CAPABLE S390X_aes_128_CAPABLE
320 # define S390X_aes_192_ecb_CAPABLE S390X_aes_192_CAPABLE
321 # define S390X_aes_256_ecb_CAPABLE S390X_aes_256_CAPABLE
323 # define S390X_aes_128_ofb_CAPABLE (S390X_aes_128_CAPABLE && \
324 (OPENSSL_s390xcap_P.kmo[0] & \
325 S390X_CAPBIT(S390X_AES_128)))
326 # define S390X_aes_192_ofb_CAPABLE (S390X_aes_192_CAPABLE && \
327 (OPENSSL_s390xcap_P.kmo[0] & \
328 S390X_CAPBIT(S390X_AES_192)))
329 # define S390X_aes_256_ofb_CAPABLE (S390X_aes_256_CAPABLE && \
330 (OPENSSL_s390xcap_P.kmo[0] & \
331 S390X_CAPBIT(S390X_AES_256)))
333 # define S390X_aes_128_cfb_CAPABLE (S390X_aes_128_CAPABLE && \
334 (OPENSSL_s390xcap_P.kmf[0] & \
335 S390X_CAPBIT(S390X_AES_128)))
336 # define S390X_aes_192_cfb_CAPABLE (S390X_aes_192_CAPABLE && \
337 (OPENSSL_s390xcap_P.kmf[0] & \
338 S390X_CAPBIT(S390X_AES_192)))
339 # define S390X_aes_256_cfb_CAPABLE (S390X_aes_256_CAPABLE && \
340 (OPENSSL_s390xcap_P.kmf[0] & \
341 S390X_CAPBIT(S390X_AES_256)))
342 # define S390X_aes_128_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
343 S390X_CAPBIT(S390X_AES_128))
344 # define S390X_aes_192_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
345 S390X_CAPBIT(S390X_AES_192))
346 # define S390X_aes_256_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
347 S390X_CAPBIT(S390X_AES_256))
348 # define S390X_aes_128_cfb1_CAPABLE 0
349 # define S390X_aes_192_cfb1_CAPABLE 0
350 # define S390X_aes_256_cfb1_CAPABLE 0
352 # define S390X_aes_128_ctr_CAPABLE 1 /* checked by callee */
353 # define S390X_aes_192_ctr_CAPABLE 1
354 # define S390X_aes_256_ctr_CAPABLE 1
356 # define S390X_aes_128_xts_CAPABLE 1 /* checked by callee */
357 # define S390X_aes_256_xts_CAPABLE 1
359 # define S390X_aes_128_gcm_CAPABLE (S390X_aes_128_CAPABLE && \
360 (OPENSSL_s390xcap_P.kma[0] & \
361 S390X_CAPBIT(S390X_AES_128)))
362 # define S390X_aes_192_gcm_CAPABLE (S390X_aes_192_CAPABLE && \
363 (OPENSSL_s390xcap_P.kma[0] & \
364 S390X_CAPBIT(S390X_AES_192)))
365 # define S390X_aes_256_gcm_CAPABLE (S390X_aes_256_CAPABLE && \
366 (OPENSSL_s390xcap_P.kma[0] & \
367 S390X_CAPBIT(S390X_AES_256)))
369 # define S390X_aes_128_ccm_CAPABLE (S390X_aes_128_CAPABLE && \
370 (OPENSSL_s390xcap_P.kmac[0] & \
371 S390X_CAPBIT(S390X_AES_128)))
372 # define S390X_aes_192_ccm_CAPABLE (S390X_aes_192_CAPABLE && \
373 (OPENSSL_s390xcap_P.kmac[0] & \
374 S390X_CAPBIT(S390X_AES_192)))
375 # define S390X_aes_256_ccm_CAPABLE (S390X_aes_256_CAPABLE && \
376 (OPENSSL_s390xcap_P.kmac[0] & \
377 S390X_CAPBIT(S390X_AES_256)))
378 # define S390X_CCM_AAD_FLAG 0x40
380 # ifndef OPENSSL_NO_OCB
381 # define S390X_aes_128_ocb_CAPABLE 0
382 # define S390X_aes_192_ocb_CAPABLE 0
383 # define S390X_aes_256_ocb_CAPABLE 0
384 # endif /* OPENSSL_NO_OCB */
386 # ifndef OPENSSL_NO_SIV
387 # define S390X_aes_128_siv_CAPABLE 0
388 # define S390X_aes_192_siv_CAPABLE 0
389 # define S390X_aes_256_siv_CAPABLE 0
390 # endif /* OPENSSL_NO_SIV */
392 /* Convert key size to function code: [16,24,32] -> [18,19,20]. */
393 # define S390X_AES_FC(keylen) (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
396 # if defined(HWAES_CAPABLE)
397 int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits,
399 int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits,
401 void HWAES_encrypt(const unsigned char *in, unsigned char *out,
403 void HWAES_decrypt(const unsigned char *in, unsigned char *out,
405 void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out,
406 size_t length, const AES_KEY *key,
407 unsigned char *ivec, const int enc);
408 void HWAES_ecb_encrypt(const unsigned char *in, unsigned char *out,
409 size_t length, const AES_KEY *key,
411 void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
412 size_t len, const void *key,
413 const unsigned char ivec[16]);
414 void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out,
415 size_t len, const AES_KEY *key1,
416 const AES_KEY *key2, const unsigned char iv[16]);
417 void HWAES_xts_decrypt(const unsigned char *inp, unsigned char *out,
418 size_t len, const AES_KEY *key1,
419 const AES_KEY *key2, const unsigned char iv[16]);
420 # ifndef OPENSSL_NO_OCB
421 # ifdef HWAES_ocb_encrypt
422 void HWAES_ocb_encrypt(const unsigned char *in, unsigned char *out,
423 size_t blocks, const void *key,
424 size_t start_block_num,
425 unsigned char offset_i[16],
426 const unsigned char L_[][16],
427 unsigned char checksum[16]);
429 # define HWAES_ocb_encrypt ((ocb128_f)NULL)
431 # ifdef HWAES_ocb_decrypt
432 void HWAES_ocb_decrypt(const unsigned char *in, unsigned char *out,
433 size_t blocks, const void *key,
434 size_t start_block_num,
435 unsigned char offset_i[16],
436 const unsigned char L_[][16],
437 unsigned char checksum[16]);
439 # define HWAES_ocb_decrypt ((ocb128_f)NULL)
441 # endif /* OPENSSL_NO_OCB */
443 # endif /* HWAES_CAPABLE */
445 #endif /* OSSL_AES_PLATFORM_H */