2 * Copyright 2019 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
10 #ifndef HEADER_INTERNAL_CIPHERMODE_PLATFORM_H
11 # define HEADER_INTERNAL_CIPHERMODE_PLATFORM_H
13 # include "openssl/aes.h"
16 int vpaes_set_encrypt_key(const unsigned char *userKey, int bits,
18 int vpaes_set_decrypt_key(const unsigned char *userKey, int bits,
20 void vpaes_encrypt(const unsigned char *in, unsigned char *out,
22 void vpaes_decrypt(const unsigned char *in, unsigned char *out,
24 void vpaes_cbc_encrypt(const unsigned char *in,
27 const AES_KEY *key, unsigned char *ivec, int enc);
28 # endif /* VPAES_ASM */
31 void bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out,
32 size_t length, const AES_KEY *key,
33 unsigned char ivec[16], int enc);
34 void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
35 size_t len, const AES_KEY *key,
36 const unsigned char ivec[16]);
37 void bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out,
38 size_t len, const AES_KEY *key1,
39 const AES_KEY *key2, const unsigned char iv[16]);
40 void bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out,
41 size_t len, const AES_KEY *key1,
42 const AES_KEY *key2, const unsigned char iv[16]);
43 # endif /* BSAES_ASM */
46 void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,
47 size_t blocks, const AES_KEY *key,
48 const unsigned char ivec[AES_BLOCK_SIZE]);
49 # endif /* AES_CTR_ASM */
52 void AES_xts_encrypt(const unsigned char *inp, unsigned char *out, size_t len,
53 const AES_KEY *key1, const AES_KEY *key2,
54 const unsigned char iv[16]);
55 void AES_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len,
56 const AES_KEY *key1, const AES_KEY *key2,
57 const unsigned char iv[16]);
58 # endif /* AES_XTS_ASM */
60 # if defined(OPENSSL_CPUID_OBJ)
61 # if (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
62 # include "ppc_arch.h"
64 # define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC)
66 # define HWAES_CAPABLE (OPENSSL_ppccap_P & PPC_CRYPTO207)
67 # define HWAES_set_encrypt_key aes_p8_set_encrypt_key
68 # define HWAES_set_decrypt_key aes_p8_set_decrypt_key
69 # define HWAES_encrypt aes_p8_encrypt
70 # define HWAES_decrypt aes_p8_decrypt
71 # define HWAES_cbc_encrypt aes_p8_cbc_encrypt
72 # define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
73 # define HWAES_xts_encrypt aes_p8_xts_encrypt
74 # define HWAES_xts_decrypt aes_p8_xts_decrypt
77 # if (defined(__arm__) || defined(__arm) || defined(__aarch64__))
78 # include "arm_arch.h"
79 # if __ARM_MAX_ARCH__>=7
80 # if defined(BSAES_ASM)
81 # define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
83 # if defined(VPAES_ASM)
84 # define VPAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
86 # define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
87 # define HWAES_set_encrypt_key aes_v8_set_encrypt_key
88 # define HWAES_set_decrypt_key aes_v8_set_decrypt_key
89 # define HWAES_encrypt aes_v8_encrypt
90 # define HWAES_decrypt aes_v8_decrypt
91 # define HWAES_cbc_encrypt aes_v8_cbc_encrypt
92 # define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
95 # endif /* OPENSSL_CPUID_OBJ */
97 # if defined(AES_ASM) && !defined(I386_ONLY) && ( \
98 ((defined(__i386) || defined(__i386__) || \
99 defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
100 defined(__x86_64) || defined(__x86_64__) || \
101 defined(_M_AMD64) || defined(_M_X64) )
104 extern unsigned int OPENSSL_ia32cap_P[];
106 # define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
108 # define VPAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
111 # define BSAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
114 int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
116 int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
119 void aesni_encrypt(const unsigned char *in, unsigned char *out,
121 void aesni_decrypt(const unsigned char *in, unsigned char *out,
124 void aesni_ecb_encrypt(const unsigned char *in,
126 size_t length, const AES_KEY *key, int enc);
127 void aesni_cbc_encrypt(const unsigned char *in,
130 const AES_KEY *key, unsigned char *ivec, int enc);
131 # ifndef OPENSSL_NO_OCB
132 void aesni_ocb_encrypt(const unsigned char *in, unsigned char *out,
133 size_t blocks, const void *key,
134 size_t start_block_num,
135 unsigned char offset_i[16],
136 const unsigned char L_[][16],
137 unsigned char checksum[16]);
138 void aesni_ocb_decrypt(const unsigned char *in, unsigned char *out,
139 size_t blocks, const void *key,
140 size_t start_block_num,
141 unsigned char offset_i[16],
142 const unsigned char L_[][16],
143 unsigned char checksum[16]);
144 # endif /* OPENSSL_NO_OCB */
146 void aesni_ctr32_encrypt_blocks(const unsigned char *in,
149 const void *key, const unsigned char *ivec);
151 void aesni_xts_encrypt(const unsigned char *in,
154 const AES_KEY *key1, const AES_KEY *key2,
155 const unsigned char iv[16]);
157 void aesni_xts_decrypt(const unsigned char *in,
160 const AES_KEY *key1, const AES_KEY *key2,
161 const unsigned char iv[16]);
163 void aesni_ccm64_encrypt_blocks(const unsigned char *in,
167 const unsigned char ivec[16],
168 unsigned char cmac[16]);
170 void aesni_ccm64_decrypt_blocks(const unsigned char *in,
174 const unsigned char ivec[16],
175 unsigned char cmac[16]);
177 # if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
178 size_t aesni_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
179 const void *key, unsigned char ivec[16], u64 *Xi);
180 size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
181 const void *key, unsigned char ivec[16], u64 *Xi);
182 void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *in, size_t len);
184 # define AES_GCM_ASM(ctx) (ctx->ctr == aesni_ctr32_encrypt_blocks && \
185 ctx->gcm.ghash == gcm_ghash_avx)
189 # elif defined(AES_ASM) && (defined(__sparc) || defined(__sparc__))
191 /* Fujitsu SPARC64 X support */
192 extern unsigned int OPENSSL_sparcv9cap_P[];
193 # include "sparc_arch.h"
195 # ifndef OPENSSL_NO_CAMELLIA
196 # define SPARC_CMLL_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_CAMELLIA)
198 void cmll_t4_set_key(const unsigned char *key, int bits, CAMELLIA_KEY *ks);
199 void cmll_t4_encrypt(const unsigned char *in, unsigned char *out,
200 const CAMELLIA_KEY *key);
201 void cmll_t4_decrypt(const unsigned char *in, unsigned char *out,
202 const CAMELLIA_KEY *key);
204 void cmll128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
205 size_t len, const CAMELLIA_KEY *key,
206 unsigned char *ivec);
207 void cmll128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
208 size_t len, const CAMELLIA_KEY *key,
209 unsigned char *ivec);
210 void cmll256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
211 size_t len, const CAMELLIA_KEY *key,
212 unsigned char *ivec);
213 void cmll256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
214 size_t len, const CAMELLIA_KEY *key,
215 unsigned char *ivec);
216 void cmll128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
217 size_t blocks, const CAMELLIA_KEY *key,
218 unsigned char *ivec);
219 void cmll256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
220 size_t blocks, const CAMELLIA_KEY *key,
221 unsigned char *ivec);
222 # endif /* OPENSSL_NO_CAMELLIA */
225 # define SPARC_AES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_AES)
226 # define HWAES_CAPABLE (OPENSSL_sparcv9cap_P[0] & SPARCV9_FJAESX)
227 # define HWAES_set_encrypt_key aes_fx_set_encrypt_key
228 # define HWAES_set_decrypt_key aes_fx_set_decrypt_key
229 # define HWAES_encrypt aes_fx_encrypt
230 # define HWAES_decrypt aes_fx_decrypt
231 # define HWAES_cbc_encrypt aes_fx_cbc_encrypt
232 # define HWAES_ctr32_encrypt_blocks aes_fx_ctr32_encrypt_blocks
234 void aes_t4_set_encrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
235 void aes_t4_set_decrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
236 void aes_t4_encrypt(const unsigned char *in, unsigned char *out,
238 void aes_t4_decrypt(const unsigned char *in, unsigned char *out,
241 * Key-length specific subroutines were chosen for following reason.
242 * Each SPARC T4 core can execute up to 8 threads which share core's
243 * resources. Loading as much key material to registers allows to
244 * minimize references to shared memory interface, as well as amount
245 * of instructions in inner loops [much needed on T4]. But then having
246 * non-key-length specific routines would require conditional branches
247 * either in inner loops or on subroutines' entries. Former is hardly
248 * acceptable, while latter means code size increase to size occupied
249 * by multiple key-length specific subroutines, so why fight?
251 void aes128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
252 size_t len, const AES_KEY *key,
253 unsigned char *ivec);
254 void aes128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
255 size_t len, const AES_KEY *key,
256 unsigned char *ivec);
257 void aes192_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
258 size_t len, const AES_KEY *key,
259 unsigned char *ivec);
260 void aes192_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
261 size_t len, const AES_KEY *key,
262 unsigned char *ivec);
263 void aes256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
264 size_t len, const AES_KEY *key,
265 unsigned char *ivec);
266 void aes256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
267 size_t len, const AES_KEY *key,
268 unsigned char *ivec);
269 void aes128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
270 size_t blocks, const AES_KEY *key,
271 unsigned char *ivec);
272 void aes192_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
273 size_t blocks, const AES_KEY *key,
274 unsigned char *ivec);
275 void aes256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
276 size_t blocks, const AES_KEY *key,
277 unsigned char *ivec);
278 void aes128_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
279 size_t blocks, const AES_KEY *key1,
280 const AES_KEY *key2, const unsigned char *ivec);
281 void aes128_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
282 size_t blocks, const AES_KEY *key1,
283 const AES_KEY *key2, const unsigned char *ivec);
284 void aes256_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
285 size_t blocks, const AES_KEY *key1,
286 const AES_KEY *key2, const unsigned char *ivec);
287 void aes256_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
288 size_t blocks, const AES_KEY *key1,
289 const AES_KEY *key2, const unsigned char *ivec);
291 # elif defined(OPENSSL_CPUID_OBJ) && defined(__s390__)
292 /* IBM S390X support */
293 # include "s390x_arch.h"
296 /* Convert key size to function code: [16,24,32] -> [18,19,20]. */
297 # define S390X_AES_FC(keylen) (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
299 /* Most modes of operation need km for partial block processing. */
300 # define S390X_aes_128_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
301 S390X_CAPBIT(S390X_AES_128))
302 # define S390X_aes_192_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
303 S390X_CAPBIT(S390X_AES_192))
304 # define S390X_aes_256_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
305 S390X_CAPBIT(S390X_AES_256))
307 # define S390X_aes_128_cbc_CAPABLE 1 /* checked by callee */
308 # define S390X_aes_192_cbc_CAPABLE 1
309 # define S390X_aes_256_cbc_CAPABLE 1
311 # define S390X_aes_128_ecb_CAPABLE S390X_aes_128_CAPABLE
312 # define S390X_aes_192_ecb_CAPABLE S390X_aes_192_CAPABLE
313 # define S390X_aes_256_ecb_CAPABLE S390X_aes_256_CAPABLE
315 # define S390X_aes_128_ofb_CAPABLE (S390X_aes_128_CAPABLE && \
316 (OPENSSL_s390xcap_P.kmo[0] & \
317 S390X_CAPBIT(S390X_AES_128)))
318 # define S390X_aes_192_ofb_CAPABLE (S390X_aes_192_CAPABLE && \
319 (OPENSSL_s390xcap_P.kmo[0] & \
320 S390X_CAPBIT(S390X_AES_192)))
321 # define S390X_aes_256_ofb_CAPABLE (S390X_aes_256_CAPABLE && \
322 (OPENSSL_s390xcap_P.kmo[0] & \
323 S390X_CAPBIT(S390X_AES_256)))
325 # define S390X_aes_128_cfb_CAPABLE (S390X_aes_128_CAPABLE && \
326 (OPENSSL_s390xcap_P.kmf[0] & \
327 S390X_CAPBIT(S390X_AES_128)))
328 # define S390X_aes_192_cfb_CAPABLE (S390X_aes_192_CAPABLE && \
329 (OPENSSL_s390xcap_P.kmf[0] & \
330 S390X_CAPBIT(S390X_AES_192)))
331 # define S390X_aes_256_cfb_CAPABLE (S390X_aes_256_CAPABLE && \
332 (OPENSSL_s390xcap_P.kmf[0] & \
333 S390X_CAPBIT(S390X_AES_256)))
334 # define S390X_aes_128_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
335 S390X_CAPBIT(S390X_AES_128))
336 # define S390X_aes_192_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
337 S390X_CAPBIT(S390X_AES_192))
338 # define S390X_aes_256_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
339 S390X_CAPBIT(S390X_AES_256))
340 # define S390X_aes_128_cfb1_CAPABLE 0
341 # define S390X_aes_192_cfb1_CAPABLE 0
342 # define S390X_aes_256_cfb1_CAPABLE 0
344 # define S390X_aes_128_ctr_CAPABLE 1 /* checked by callee */
345 # define S390X_aes_192_ctr_CAPABLE 1
346 # define S390X_aes_256_ctr_CAPABLE 1
348 # define S390X_aes_128_xts_CAPABLE 1 /* checked by callee */
349 # define S390X_aes_256_xts_CAPABLE 1
351 # define S390X_aes_128_gcm_CAPABLE (S390X_aes_128_CAPABLE && \
352 (OPENSSL_s390xcap_P.kma[0] & \
353 S390X_CAPBIT(S390X_AES_128)))
354 # define S390X_aes_192_gcm_CAPABLE (S390X_aes_192_CAPABLE && \
355 (OPENSSL_s390xcap_P.kma[0] & \
356 S390X_CAPBIT(S390X_AES_192)))
357 # define S390X_aes_256_gcm_CAPABLE (S390X_aes_256_CAPABLE && \
358 (OPENSSL_s390xcap_P.kma[0] & \
359 S390X_CAPBIT(S390X_AES_256)))
361 # define S390X_aes_128_ccm_CAPABLE (S390X_aes_128_CAPABLE && \
362 (OPENSSL_s390xcap_P.kmac[0] & \
363 S390X_CAPBIT(S390X_AES_128)))
364 # define S390X_aes_192_ccm_CAPABLE (S390X_aes_192_CAPABLE && \
365 (OPENSSL_s390xcap_P.kmac[0] & \
366 S390X_CAPBIT(S390X_AES_192)))
367 # define S390X_aes_256_ccm_CAPABLE (S390X_aes_256_CAPABLE && \
368 (OPENSSL_s390xcap_P.kmac[0] & \
369 S390X_CAPBIT(S390X_AES_256)))
370 # define S390X_CCM_AAD_FLAG 0x40
372 # ifndef OPENSSL_NO_OCB
373 # define S390X_aes_128_ocb_CAPABLE 0
374 # define S390X_aes_192_ocb_CAPABLE 0
375 # define S390X_aes_256_ocb_CAPABLE 0
376 # endif /* OPENSSL_NO_OCB */
378 # ifndef OPENSSL_NO_SIV
379 # define S390X_aes_128_siv_CAPABLE 0
380 # define S390X_aes_192_siv_CAPABLE 0
381 # define S390X_aes_256_siv_CAPABLE 0
382 # endif /* OPENSSL_NO_SIV */
384 /* Convert key size to function code: [16,24,32] -> [18,19,20]. */
385 # define S390X_AES_FC(keylen) (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
388 # if defined(HWAES_CAPABLE)
389 int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits,
391 int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits,
393 void HWAES_encrypt(const unsigned char *in, unsigned char *out,
395 void HWAES_decrypt(const unsigned char *in, unsigned char *out,
397 void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out,
398 size_t length, const AES_KEY *key,
399 unsigned char *ivec, const int enc);
400 void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
401 size_t len, const AES_KEY *key,
402 const unsigned char ivec[16]);
403 void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out,
404 size_t len, const AES_KEY *key1,
405 const AES_KEY *key2, const unsigned char iv[16]);
406 void HWAES_xts_decrypt(const unsigned char *inp, unsigned char *out,
407 size_t len, const AES_KEY *key1,
408 const AES_KEY *key2, const unsigned char iv[16]);
409 # ifndef OPENSSL_NO_OCB
410 # ifdef HWAES_ocb_encrypt
411 void HWAES_ocb_encrypt(const unsigned char *in, unsigned char *out,
412 size_t blocks, const void *key,
413 size_t start_block_num,
414 unsigned char offset_i[16],
415 const unsigned char L_[][16],
416 unsigned char checksum[16]);
418 # define HWAES_ocb_encrypt ((ocb128_f)NULL)
420 # ifdef HWAES_ocb_decrypt
421 void HWAES_ocb_decrypt(const unsigned char *in, unsigned char *out,
422 size_t blocks, const void *key,
423 size_t start_block_num,
424 unsigned char offset_i[16],
425 const unsigned char L_[][16],
426 unsigned char checksum[16]);
428 # define HWAES_ocb_decrypt ((ocb128_f)NULL)
430 # endif /* OPENSSL_NO_OCB */
432 # endif /* HWAES_CAPABLE */
434 #endif /* HEADER_INTERNAL_CIPHERMODE_PLATFORM_H */