2 * Support for VIA PadLock Advanced Cryptography Engine (ACE)
3 * Written by Michal Ludvig <michal@logix.cz>
4 * http://www.logix.cz/michal
6 * Big thanks to Andy Polyakov for a help with optimization,
7 * assembler fixes, port to MS Windows and a lot of other
8 * valuable work on this engine!
11 /* ====================================================================
12 * Copyright (c) 1999-2001 The OpenSSL Project. All rights reserved.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in
23 * the documentation and/or other materials provided with the
26 * 3. All advertising materials mentioning features or use of this
27 * software must display the following acknowledgment:
28 * "This product includes software developed by the OpenSSL Project
29 * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
31 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
32 * endorse or promote products derived from this software without
33 * prior written permission. For written permission, please contact
34 * licensing@OpenSSL.org.
36 * 5. Products derived from this software may not be called "OpenSSL"
37 * nor may "OpenSSL" appear in their names without prior written
38 * permission of the OpenSSL Project.
40 * 6. Redistributions of any form whatsoever must retain the following
42 * "This product includes software developed by the OpenSSL Project
43 * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
45 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
46 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
48 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
49 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
51 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
52 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
54 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
55 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
56 * OF THE POSSIBILITY OF SUCH DAMAGE.
57 * ====================================================================
59 * This product includes cryptographic software written by Eric Young
60 * (eay@cryptsoft.com). This product includes software written by Tim
61 * Hudson (tjh@cryptsoft.com).
69 #include <openssl/opensslconf.h>
70 #include <openssl/crypto.h>
71 #include <openssl/dso.h>
72 #include <openssl/engine.h>
73 #include <openssl/evp.h>
74 #ifndef OPENSSL_NO_AES
75 #include <openssl/aes.h>
77 #include <openssl/rand.h>
78 #include <openssl/err.h>
81 #ifndef OPENSSL_NO_HW_PADLOCK
83 /* Attempt to have a single source for both 0.9.7 and 0.9.8 :-) */
84 #if (OPENSSL_VERSION_NUMBER >= 0x00908000L)
85 # ifndef OPENSSL_NO_DYNAMIC_ENGINE
86 # define DYNAMIC_ENGINE
88 #elif (OPENSSL_VERSION_NUMBER >= 0x00907000L)
89 # ifdef ENGINE_DYNAMIC_SUPPORT
90 # define DYNAMIC_ENGINE
93 # error "Only OpenSSL >= 0.9.7 is supported"
96 /* VIA PadLock AES is available *ONLY* on some x86 CPUs.
97 Not only that it doesn't exist elsewhere, but it
98 even can't be compiled on other platforms!
100 In addition, because of the heavy use of inline assembler,
101 compiler choice is limited to GCC and Microsoft C. */
102 #undef COMPILE_HW_PADLOCK
103 #if !defined(I386_ONLY) && !defined(OPENSSL_NO_INLINE_ASM)
104 # if (defined(__GNUC__) && (defined(__i386__) || defined(__i386))) || \
105 (defined(_MSC_VER) && defined(_M_IX86))
106 # define COMPILE_HW_PADLOCK
107 static ENGINE *ENGINE_padlock (void);
111 #ifdef OPENSSL_NO_DYNAMIC_ENGINE
113 void ENGINE_load_padlock (void)
115 /* On non-x86 CPUs it just returns. */
116 #ifdef COMPILE_HW_PADLOCK
117 ENGINE *toadd = ENGINE_padlock ();
127 #ifdef COMPILE_HW_PADLOCK
128 /* We do these includes here to avoid header problems on platforms that
129 do not have the VIA padlock anyway... */
134 # define alloca _alloca
136 #elif defined(__GNUC__)
138 # define alloca(s) __builtin_alloca(s)
142 /* Function for ENGINE detection and control */
143 static int padlock_available(void);
144 static int padlock_init(ENGINE *e);
147 static RAND_METHOD padlock_rand;
150 #ifndef OPENSSL_NO_AES
151 static int padlock_ciphers(ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid);
155 static const char *padlock_id = "padlock";
156 static char padlock_name[100];
158 /* Available features */
159 static int padlock_use_ace = 0; /* Advanced Cryptography Engine */
160 static int padlock_use_rng = 0; /* Random Number Generator */
161 #ifndef OPENSSL_NO_AES
162 static int padlock_aes_align_required = 1;
165 /* ===== Engine "management" functions ===== */
167 /* Prepare the ENGINE structure for registration */
169 padlock_bind_helper(ENGINE *e)
171 /* Check available features */
174 #if 1 /* disable RNG for now, see commentary in vicinity of RNG code */
178 /* Generate a nice engine name with available features */
179 BIO_snprintf(padlock_name, sizeof(padlock_name),
180 "VIA PadLock (%s, %s)",
181 padlock_use_rng ? "RNG" : "no-RNG",
182 padlock_use_ace ? "ACE" : "no-ACE");
184 /* Register everything or return with an error */
185 if (!ENGINE_set_id(e, padlock_id) ||
186 !ENGINE_set_name(e, padlock_name) ||
188 !ENGINE_set_init_function(e, padlock_init) ||
189 #ifndef OPENSSL_NO_AES
190 (padlock_use_ace && !ENGINE_set_ciphers (e, padlock_ciphers)) ||
192 (padlock_use_rng && !ENGINE_set_RAND (e, &padlock_rand))) {
196 /* Everything looks good */
204 ENGINE *eng = ENGINE_new();
210 if (!padlock_bind_helper(eng)) {
218 /* Check availability of the engine */
220 padlock_init(ENGINE *e)
222 return (padlock_use_rng || padlock_use_ace);
225 /* This stuff is needed if this ENGINE is being compiled into a self-contained
228 #ifdef DYNAMIC_ENGINE
230 padlock_bind_fn(ENGINE *e, const char *id)
232 if (id && (strcmp(id, padlock_id) != 0)) {
236 if (!padlock_bind_helper(e)) {
243 IMPLEMENT_DYNAMIC_CHECK_FN()
244 IMPLEMENT_DYNAMIC_BIND_FN (padlock_bind_fn)
245 #endif /* DYNAMIC_ENGINE */
247 /* ===== Here comes the "real" engine ===== */
249 #ifndef OPENSSL_NO_AES
250 /* Some AES-related constants */
251 #define AES_BLOCK_SIZE 16
252 #define AES_KEY_SIZE_128 16
253 #define AES_KEY_SIZE_192 24
254 #define AES_KEY_SIZE_256 32
256 /* Here we store the status information relevant to the
259 * Inline assembler in PADLOCK_XCRYPT_ASM()
260 * depends on the order of items in this structure.
261 * Don't blindly modify, reorder, etc!
263 struct padlock_cipher_data
265 unsigned char iv[AES_BLOCK_SIZE]; /* Initialization vector */
266 union { unsigned int pad[4];
269 int dgst:1; /* n/a in C3 */
270 int align:1; /* n/a in C3 */
271 int ciphr:1; /* n/a in C3 */
272 unsigned int keygen:1;
274 unsigned int encdec:1;
277 } cword; /* Control word */
278 AES_KEY ks; /* Encryption key */
282 * Essentially this variable belongs in thread local storage.
283 * Having this variable global on the other hand can only cause
284 * few bogus key reloads [if any at all on single-CPU system],
285 * so we accept the penatly...
287 static volatile struct padlock_cipher_data *padlock_saved_context;
291 * =======================================================
292 * Inline assembler section(s).
293 * =======================================================
294 * Order of arguments is chosen to facilitate Windows port
295 * using __fastcall calling convention. If you wish to add
296 * more routines, keep in mind that first __fastcall
297 * argument is passed in %ecx and second - in %edx.
298 * =======================================================
300 #if defined(__GNUC__) && __GNUC__>=2
302 * As for excessive "push %ebx"/"pop %ebx" found all over.
303 * When generating position-independent code GCC won't let
304 * us use "b" in assembler templates nor even respect "ebx"
305 * in "clobber description." Therefore the trouble...
308 /* Helper function - check if a CPUID instruction
309 is available on this CPU */
311 padlock_insn_cpuid_available(void)
315 /* We're checking if the bit #21 of EFLAGS
316 can be toggled. If yes = CPUID is available. */
320 "xorl $0x200000, %%eax\n"
321 "movl %%eax, %%ecx\n"
322 "andl $0x200000, %%ecx\n"
327 "andl $0x200000, %%eax\n"
328 "xorl %%eax, %%ecx\n"
330 : "=r" (result) : : "eax", "ecx");
332 return (result == 0);
335 /* Load supported features of the CPU to see if
336 the PadLock is available. */
338 padlock_available(void)
340 char vendor_string[16];
341 unsigned int eax, edx;
343 /* First check if the CPUID instruction is available at all... */
344 if (! padlock_insn_cpuid_available())
347 /* Are we running on the Centaur (VIA) CPU? */
349 vendor_string[12] = 0;
353 "movl %%ebx,(%%edi)\n"
354 "movl %%edx,4(%%edi)\n"
355 "movl %%ecx,8(%%edi)\n"
357 : "+a"(eax) : "D"(vendor_string) : "ecx", "edx");
358 if (strcmp(vendor_string, "CentaurHauls") != 0)
361 /* Check for Centaur Extended Feature Flags presence */
363 asm volatile ("pushl %%ebx; cpuid; popl %%ebx"
364 : "+a"(eax) : : "ecx", "edx");
365 if (eax < 0xC0000001)
368 /* Read the Centaur Extended Feature Flags */
370 asm volatile ("pushl %%ebx; cpuid; popl %%ebx"
371 : "+a"(eax), "=d"(edx) : : "ecx");
373 /* Fill up some flags */
374 padlock_use_ace = ((edx & (0x3<<6)) == (0x3<<6));
375 padlock_use_rng = ((edx & (0x3<<2)) == (0x3<<2));
377 return padlock_use_ace + padlock_use_rng;
380 #ifndef OPENSSL_NO_AES
381 /* Our own htonl()/ntohl() */
383 padlock_bswapl(AES_KEY *ks)
385 size_t i = sizeof(ks->rd_key)/sizeof(ks->rd_key[0]);
386 unsigned int *key = ks->rd_key;
389 asm volatile ("bswapl %0" : "+r"(*key));
395 /* Force key reload from memory to the CPU microcode.
396 Loading EFLAGS from the stack clears EFLAGS[30]
397 which does the trick. */
399 padlock_reload_key(void)
401 asm volatile ("pushfl; popfl");
404 #ifndef OPENSSL_NO_AES
406 * This is heuristic key context tracing. At first one
407 * believes that one should use atomic swap instructions,
408 * but it's not actually necessary. Point is that if
409 * padlock_saved_context was changed by another thread
410 * after we've read it and before we compare it with cdata,
411 * our key *shall* be reloaded upon thread context switch
412 * and we are therefore set in either case...
415 padlock_verify_context(struct padlock_cipher_data *cdata)
427 :"+m"(padlock_saved_context)
428 : "r"(padlock_saved_context), "r"(cdata) : "cc");
431 /* Template for padlock_xcrypt_* modes */
433 * The offsets used with 'leal' instructions
434 * describe items of the 'padlock_cipher_data'
437 #define PADLOCK_XCRYPT_ASM(name,rep_xcrypt) \
438 static inline void *name(size_t cnt, \
439 struct padlock_cipher_data *cdata, \
440 void *out, const void *inp) \
442 asm volatile ( "pushl %%ebx\n" \
443 " leal 16(%0),%%edx\n" \
444 " leal 32(%0),%%ebx\n" \
447 : "=a"(iv), "=c"(cnt), "=D"(out), "=S"(inp) \
448 : "0"(cdata), "1"(cnt), "2"(out), "3"(inp) \
449 : "edx", "cc", "memory"); \
453 /* Generate all functions with appropriate opcodes */
454 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb, ".byte 0xf3,0x0f,0xa7,0xc8") /* rep xcryptecb */
455 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc, ".byte 0xf3,0x0f,0xa7,0xd0") /* rep xcryptcbc */
456 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb, ".byte 0xf3,0x0f,0xa7,0xe0") /* rep xcryptcfb */
457 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb, ".byte 0xf3,0x0f,0xa7,0xe8") /* rep xcryptofb */
460 /* The RNG call itself */
461 static inline unsigned int
462 padlock_xstore(void *addr, unsigned int edx_in)
464 unsigned int eax_out;
466 asm volatile (".byte 0x0f,0xa7,0xc0" /* xstore */
467 : "=a"(eax_out),"=m"(*(unsigned *)addr)
468 : "D"(addr), "d" (edx_in)
474 /* Why not inline 'rep movsd'? I failed to find information on what
475 * value in Direction Flag one can expect and consequently have to
476 * apply "better-safe-than-sorry" approach and assume "undefined."
477 * I could explicitly clear it and restore the original value upon
478 * return from padlock_aes_cipher, but it's presumably too much
479 * trouble for too little gain...
481 * In case you wonder 'rep xcrypt*' instructions above are *not*
482 * affected by the Direction Flag and pointers advance toward
483 * larger addresses unconditionally.
485 static inline unsigned char *
486 padlock_memcpy(void *dst,const void *src,size_t n)
492 do { *d++ = *s++; } while (--n);
497 #elif defined(_MSC_VER)
499 * Unlike GCC these are real functions. In order to minimize impact
500 * on performance we adhere to __fastcall calling convention in
501 * order to get two first arguments passed through %ecx and %edx.
502 * Which kind of suits very well, as instructions in question use
503 * both %ecx and %edx as input:-)
505 #define REP_XCRYPT(code) \
507 _asm _emit 0x0f _asm _emit 0xa7 \
511 * The offsets used with 'lea' instructions
512 * describe items of the 'padlock_cipher_data'
515 #define PADLOCK_XCRYPT_ASM(name,code) \
516 static void * __fastcall \
517 name (size_t cnt, void *cdata, \
518 void *outp, const void *inp) \
520 _asm lea edx,[eax+16] \
521 _asm lea ebx,[eax+32] \
527 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb,0xc8)
528 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc,0xd0)
529 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb,0xe0)
530 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb,0xe8)
532 static int __fastcall
533 padlock_xstore(void *outp,unsigned int code)
535 _asm _emit 0x0f _asm _emit 0xa7 _asm _emit 0xc0
538 static void __fastcall
539 padlock_reload_key(void)
540 { _asm pushfd _asm popfd }
542 static void __fastcall
543 padlock_verify_context(void *cdata)
548 cmp ecx,padlock_saved_context
553 mov padlock_saved_context,ecx
558 padlock_available(void)
593 mov padlock_use_ace,1
599 mov padlock_use_rng,1
606 static void __fastcall
607 padlock_bswapl(void *key)
622 /* MS actually specifies status of Direction Flag and compiler even
623 * manages to compile following as 'rep movsd' all by itself...
625 #define padlock_memcpy(o,i,n) ((unsigned char *)memcpy((o),(i),(n)&~3U))
628 /* ===== AES encryption/decryption ===== */
629 #ifndef OPENSSL_NO_AES
631 #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb)
632 #define NID_aes_128_cfb NID_aes_128_cfb128
635 #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb)
636 #define NID_aes_128_ofb NID_aes_128_ofb128
639 #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb)
640 #define NID_aes_192_cfb NID_aes_192_cfb128
643 #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb)
644 #define NID_aes_192_ofb NID_aes_192_ofb128
647 #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb)
648 #define NID_aes_256_cfb NID_aes_256_cfb128
651 #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb)
652 #define NID_aes_256_ofb NID_aes_256_ofb128
655 /* List of supported ciphers. */
656 static int padlock_cipher_nids[] = {
672 static int padlock_cipher_nids_num = (sizeof(padlock_cipher_nids)/
673 sizeof(padlock_cipher_nids[0]));
675 /* Function prototypes ... */
676 static int padlock_aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
677 const unsigned char *iv, int enc);
678 static int padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
679 const unsigned char *in, size_t nbytes);
681 #define NEAREST_ALIGNED(ptr) ( (unsigned char *)(ptr) + \
682 ( (0x10 - ((size_t)(ptr) & 0x0F)) & 0x0F ) )
683 #define ALIGNED_CIPHER_DATA(ctx) ((struct padlock_cipher_data *)\
684 NEAREST_ALIGNED(ctx->cipher_data))
686 #define EVP_CIPHER_block_size_ECB AES_BLOCK_SIZE
687 #define EVP_CIPHER_block_size_CBC AES_BLOCK_SIZE
688 #define EVP_CIPHER_block_size_OFB 1
689 #define EVP_CIPHER_block_size_CFB 1
691 /* Declaring so many ciphers by hand would be a pain.
692 Instead introduce a bit of preprocessor magic :-) */
693 #define DECLARE_AES_EVP(ksize,lmode,umode) \
694 static const EVP_CIPHER padlock_aes_##ksize##_##lmode = { \
695 NID_aes_##ksize##_##lmode, \
696 EVP_CIPHER_block_size_##umode, \
697 AES_KEY_SIZE_##ksize, \
699 0 | EVP_CIPH_##umode##_MODE, \
700 padlock_aes_init_key, \
701 padlock_aes_cipher, \
703 sizeof(struct padlock_cipher_data) + 16, \
704 EVP_CIPHER_set_asn1_iv, \
705 EVP_CIPHER_get_asn1_iv, \
710 DECLARE_AES_EVP(128,ecb,ECB);
711 DECLARE_AES_EVP(128,cbc,CBC);
712 DECLARE_AES_EVP(128,cfb,CFB);
713 DECLARE_AES_EVP(128,ofb,OFB);
715 DECLARE_AES_EVP(192,ecb,ECB);
716 DECLARE_AES_EVP(192,cbc,CBC);
717 DECLARE_AES_EVP(192,cfb,CFB);
718 DECLARE_AES_EVP(192,ofb,OFB);
720 DECLARE_AES_EVP(256,ecb,ECB);
721 DECLARE_AES_EVP(256,cbc,CBC);
722 DECLARE_AES_EVP(256,cfb,CFB);
723 DECLARE_AES_EVP(256,ofb,OFB);
726 padlock_ciphers (ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid)
728 /* No specific cipher => return a list of supported nids ... */
730 *nids = padlock_cipher_nids;
731 return padlock_cipher_nids_num;
734 /* ... or the requested "cipher" otherwise */
736 case NID_aes_128_ecb:
737 *cipher = &padlock_aes_128_ecb;
739 case NID_aes_128_cbc:
740 *cipher = &padlock_aes_128_cbc;
742 case NID_aes_128_cfb:
743 *cipher = &padlock_aes_128_cfb;
745 case NID_aes_128_ofb:
746 *cipher = &padlock_aes_128_ofb;
749 case NID_aes_192_ecb:
750 *cipher = &padlock_aes_192_ecb;
752 case NID_aes_192_cbc:
753 *cipher = &padlock_aes_192_cbc;
755 case NID_aes_192_cfb:
756 *cipher = &padlock_aes_192_cfb;
758 case NID_aes_192_ofb:
759 *cipher = &padlock_aes_192_ofb;
762 case NID_aes_256_ecb:
763 *cipher = &padlock_aes_256_ecb;
765 case NID_aes_256_cbc:
766 *cipher = &padlock_aes_256_cbc;
768 case NID_aes_256_cfb:
769 *cipher = &padlock_aes_256_cfb;
771 case NID_aes_256_ofb:
772 *cipher = &padlock_aes_256_ofb;
776 /* Sorry, we don't support this NID */
784 /* Prepare the encryption key for PadLock usage */
786 padlock_aes_init_key (EVP_CIPHER_CTX *ctx, const unsigned char *key,
787 const unsigned char *iv, int enc)
789 struct padlock_cipher_data *cdata;
790 int key_len = EVP_CIPHER_CTX_key_length(ctx) * 8;
792 if (key==NULL) return 0; /* ERROR */
794 cdata = ALIGNED_CIPHER_DATA(ctx);
795 memset(cdata, 0, sizeof(struct padlock_cipher_data));
797 /* Prepare Control word. */
798 if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE)
799 cdata->cword.b.encdec = 0;
801 cdata->cword.b.encdec = (ctx->encrypt == 0);
802 cdata->cword.b.rounds = 10 + (key_len - 128) / 32;
803 cdata->cword.b.ksize = (key_len - 128) / 64;
807 /* PadLock can generate an extended key for
808 AES128 in hardware */
809 memcpy(cdata->ks.rd_key, key, AES_KEY_SIZE_128);
810 cdata->cword.b.keygen = 0;
815 /* Generate an extended AES key in software.
816 Needed for AES192/AES256 */
817 /* Well, the above applies to Stepping 8 CPUs
818 and is listed as hardware errata. They most
819 likely will fix it at some point and then
820 a check for stepping would be due here. */
821 if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_CFB_MODE ||
822 EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE ||
824 AES_set_encrypt_key(key, key_len, &cdata->ks);
826 AES_set_decrypt_key(key, key_len, &cdata->ks);
828 /* OpenSSL C functions use byte-swapped extended key. */
829 padlock_bswapl(&cdata->ks);
831 cdata->cword.b.keygen = 1;
840 * This is done to cover for cases when user reuses the
841 * context for new key. The catch is that if we don't do
842 * this, padlock_eas_cipher might proceed with old key...
844 padlock_reload_key ();
850 * Simplified version of padlock_aes_cipher() used when
851 * 1) both input and output buffers are at aligned addresses.
853 * 2) running on a newer CPU that doesn't require aligned buffers.
856 padlock_aes_cipher_omnivorous(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,
857 const unsigned char *in_arg, size_t nbytes)
859 struct padlock_cipher_data *cdata;
862 cdata = ALIGNED_CIPHER_DATA(ctx);
863 padlock_verify_context(cdata);
865 switch (EVP_CIPHER_CTX_mode(ctx)) {
866 case EVP_CIPH_ECB_MODE:
867 padlock_xcrypt_ecb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
870 case EVP_CIPH_CBC_MODE:
871 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
872 iv = padlock_xcrypt_cbc(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
873 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
876 case EVP_CIPH_CFB_MODE:
877 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
878 iv = padlock_xcrypt_cfb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
879 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
882 case EVP_CIPH_OFB_MODE:
883 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
884 padlock_xcrypt_ofb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
885 memcpy(ctx->iv, cdata->iv, AES_BLOCK_SIZE);
892 memset(cdata->iv, 0, AES_BLOCK_SIZE);
897 #ifndef PADLOCK_CHUNK
898 # define PADLOCK_CHUNK 512 /* Must be a power of 2 larger than 16 */
900 #if PADLOCK_CHUNK<16 || PADLOCK_CHUNK&(PADLOCK_CHUNK-1)
901 # error "insane PADLOCK_CHUNK..."
904 /* Re-align the arguments to 16-Bytes boundaries and run the
905 encryption function itself. This function is not AES-specific. */
907 padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,
908 const unsigned char *in_arg, size_t nbytes)
910 struct padlock_cipher_data *cdata;
914 int inp_misaligned, out_misaligned, realign_in_loop;
915 size_t chunk, allocated=0;
917 /* ctx->num is maintained in byte-oriented modes,
918 such as CFB and OFB... */
919 if ((chunk = ctx->num)) { /* borrow chunk variable */
920 unsigned char *ivp=ctx->iv;
922 switch (EVP_CIPHER_CTX_mode(ctx)) {
923 case EVP_CIPH_CFB_MODE:
924 if (chunk >= AES_BLOCK_SIZE)
925 return 0; /* bogus value */
928 while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
929 ivp[chunk] = *(out_arg++) = *(in_arg++) ^ ivp[chunk];
932 else while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
933 unsigned char c = *(in_arg++);
934 *(out_arg++) = c ^ ivp[chunk];
935 ivp[chunk++] = c, nbytes--;
938 ctx->num = chunk%AES_BLOCK_SIZE;
940 case EVP_CIPH_OFB_MODE:
941 if (chunk >= AES_BLOCK_SIZE)
942 return 0; /* bogus value */
944 while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
945 *(out_arg++) = *(in_arg++) ^ ivp[chunk];
949 ctx->num = chunk%AES_BLOCK_SIZE;
957 if (nbytes % AES_BLOCK_SIZE)
958 return 0; /* are we expected to do tail processing? */
960 /* nbytes is always multiple of AES_BLOCK_SIZE in ECB and CBC
961 modes and arbitrary value in byte-oriented modes, such as
965 /* VIA promises CPUs that won't require alignment in the future.
966 For now padlock_aes_align_required is initialized to 1 and
967 the condition is never met... */
968 /* C7 core is capable to manage unaligned input in non-ECB[!]
969 mode, but performance penalties appear to be approximately
970 same as for software alignment below or ~3x. They promise to
971 improve it in the future, but for now we can just as well
972 pretend that it can only handle aligned input... */
973 if (!padlock_aes_align_required && (nbytes%AES_BLOCK_SIZE)==0)
974 return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);
976 inp_misaligned = (((size_t)in_arg) & 0x0F);
977 out_misaligned = (((size_t)out_arg) & 0x0F);
979 /* Note that even if output is aligned and input not,
980 * I still prefer to loop instead of copy the whole
981 * input and then encrypt in one stroke. This is done
982 * in order to improve L1 cache utilization... */
983 realign_in_loop = out_misaligned|inp_misaligned;
985 if (!realign_in_loop && (nbytes%AES_BLOCK_SIZE)==0)
986 return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);
988 /* this takes one "if" out of the loops */
990 chunk %= PADLOCK_CHUNK;
991 if (chunk==0) chunk = PADLOCK_CHUNK;
993 if (out_misaligned) {
994 /* optmize for small input */
995 allocated = (chunk<nbytes?PADLOCK_CHUNK:nbytes);
996 out = alloca(0x10 + allocated);
997 out = NEAREST_ALIGNED(out);
1002 cdata = ALIGNED_CIPHER_DATA(ctx);
1003 padlock_verify_context(cdata);
1005 switch (EVP_CIPHER_CTX_mode(ctx)) {
1006 case EVP_CIPH_ECB_MODE:
1009 inp = padlock_memcpy(out, in_arg, chunk);
1014 padlock_xcrypt_ecb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1017 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1019 out = out_arg+=chunk;
1022 chunk = PADLOCK_CHUNK;
1026 case EVP_CIPH_CBC_MODE:
1027 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1030 if (iv != cdata->iv)
1031 memcpy(cdata->iv, iv, AES_BLOCK_SIZE);
1032 chunk = PADLOCK_CHUNK;
1033 cbc_shortcut: /* optimize for small input */
1035 inp = padlock_memcpy(out, in_arg, chunk);
1040 iv = padlock_xcrypt_cbc(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1043 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1045 out = out_arg+=chunk;
1047 } while (nbytes -= chunk);
1048 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
1051 case EVP_CIPH_CFB_MODE:
1052 memcpy (iv = cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1053 chunk &= ~(AES_BLOCK_SIZE-1);
1054 if (chunk) goto cfb_shortcut;
1055 else goto cfb_skiploop;
1057 if (iv != cdata->iv)
1058 memcpy(cdata->iv, iv, AES_BLOCK_SIZE);
1059 chunk = PADLOCK_CHUNK;
1060 cfb_shortcut: /* optimize for small input */
1062 inp = padlock_memcpy(out, in_arg, chunk);
1067 iv = padlock_xcrypt_cfb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1070 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1072 out = out_arg+=chunk;
1075 } while (nbytes >= AES_BLOCK_SIZE);
1079 unsigned char *ivp = cdata->iv;
1082 memcpy(ivp, iv, AES_BLOCK_SIZE);
1086 if (cdata->cword.b.encdec) {
1087 cdata->cword.b.encdec=0;
1088 padlock_reload_key();
1089 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1090 cdata->cword.b.encdec=1;
1091 padlock_reload_key();
1093 unsigned char c = *(in_arg++);
1094 *(out_arg++) = c ^ *ivp;
1095 *(ivp++) = c, nbytes--;
1098 else { padlock_reload_key();
1099 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1100 padlock_reload_key();
1102 *ivp = *(out_arg++) = *(in_arg++) ^ *ivp;
1108 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
1111 case EVP_CIPH_OFB_MODE:
1112 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1113 chunk &= ~(AES_BLOCK_SIZE-1);
1116 inp = padlock_memcpy(out, in_arg, chunk);
1121 padlock_xcrypt_ofb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1124 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1126 out = out_arg+=chunk;
1129 chunk = PADLOCK_CHUNK;
1130 } while (nbytes >= AES_BLOCK_SIZE);
1133 unsigned char *ivp = cdata->iv;
1136 padlock_reload_key(); /* empirically found */
1137 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1138 padlock_reload_key(); /* empirically found */
1140 *(out_arg++) = *(in_arg++) ^ *ivp;
1145 memcpy(ctx->iv, cdata->iv, AES_BLOCK_SIZE);
1152 /* Clean the realign buffer if it was used */
1153 if (out_misaligned) {
1154 volatile unsigned long *p=(void *)out;
1155 size_t n = allocated/sizeof(*p);
1159 memset(cdata->iv, 0, AES_BLOCK_SIZE);
1164 #endif /* OPENSSL_NO_AES */
1166 /* ===== Random Number Generator ===== */
1168 * This code is not engaged. The reason is that it does not comply
1169 * with recommendations for VIA RNG usage for secure applications
1170 * (posted at http://www.via.com.tw/en/viac3/c3.jsp) nor does it
1171 * provide meaningful error control...
1173 /* Wrapper that provides an interface between the API and
1174 the raw PadLock RNG */
1176 padlock_rand_bytes(unsigned char *output, int count)
1178 unsigned int eax, buf;
1180 while (count >= 8) {
1181 eax = padlock_xstore(output, 0);
1182 if (!(eax&(1<<6))) return 0; /* RNG disabled */
1183 /* this ---vv--- covers DC bias, Raw Bits and String Filter */
1184 if (eax&(0x1F<<10)) return 0;
1185 if ((eax&0x1F)==0) continue; /* no data, retry... */
1186 if ((eax&0x1F)!=8) return 0; /* fatal failure... */
1191 eax = padlock_xstore(&buf, 3);
1192 if (!(eax&(1<<6))) return 0; /* RNG disabled */
1193 /* this ---vv--- covers DC bias, Raw Bits and String Filter */
1194 if (eax&(0x1F<<10)) return 0;
1195 if ((eax&0x1F)==0) continue; /* no data, retry... */
1196 if ((eax&0x1F)!=1) return 0; /* fatal failure... */
1197 *output++ = (unsigned char)buf;
1200 *(volatile unsigned int *)&buf=0;
1205 /* Dummy but necessary function */
1207 padlock_rand_status(void)
1212 /* Prepare structure for registration */
1213 static RAND_METHOD padlock_rand = {
1215 padlock_rand_bytes, /* bytes */
1218 padlock_rand_bytes, /* pseudorand */
1219 padlock_rand_status, /* rand status */
1222 #else /* !COMPILE_HW_PADLOCK */
1223 #ifndef OPENSSL_NO_DYNAMIC_ENGINE
1225 int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns);
1227 int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns) { return 0; }
1228 IMPLEMENT_DYNAMIC_CHECK_FN()
1230 #endif /* COMPILE_HW_PADLOCK */
1232 #endif /* !OPENSSL_NO_HW_PADLOCK */
1233 #endif /* !OPENSSL_NO_HW */