2 * Support for VIA PadLock Advanced Cryptography Engine (ACE)
3 * Written by Michal Ludvig <michal@logix.cz>
4 * http://www.logix.cz/michal
6 * Big thanks to Andy Polyakov for a help with optimization,
7 * assembler fixes, port to MS Windows and a lot of other
8 * valuable work on this engine!
11 /* ====================================================================
12 * Copyright (c) 1999-2001 The OpenSSL Project. All rights reserved.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in
23 * the documentation and/or other materials provided with the
26 * 3. All advertising materials mentioning features or use of this
27 * software must display the following acknowledgment:
28 * "This product includes software developed by the OpenSSL Project
29 * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
31 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
32 * endorse or promote products derived from this software without
33 * prior written permission. For written permission, please contact
34 * licensing@OpenSSL.org.
36 * 5. Products derived from this software may not be called "OpenSSL"
37 * nor may "OpenSSL" appear in their names without prior written
38 * permission of the OpenSSL Project.
40 * 6. Redistributions of any form whatsoever must retain the following
42 * "This product includes software developed by the OpenSSL Project
43 * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
45 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
46 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
48 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
49 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
51 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
52 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
54 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
55 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
56 * OF THE POSSIBILITY OF SUCH DAMAGE.
57 * ====================================================================
59 * This product includes cryptographic software written by Eric Young
60 * (eay@cryptsoft.com). This product includes software written by Tim
61 * Hudson (tjh@cryptsoft.com).
69 #include <openssl/opensslconf.h>
70 #include <openssl/crypto.h>
71 #include <openssl/dso.h>
72 #include <openssl/engine.h>
73 #include <openssl/evp.h>
74 #ifndef OPENSSL_NO_AES
75 #include <openssl/aes.h>
77 #include <openssl/rand.h>
78 #include <openssl/err.h>
81 #ifndef OPENSSL_NO_HW_PADLOCK
83 /* Attempt to have a single source for both 0.9.7 and 0.9.8 :-) */
84 #if (OPENSSL_VERSION_NUMBER >= 0x00908000L)
85 # ifndef OPENSSL_NO_DYNAMIC_ENGINE
86 # define DYNAMIC_ENGINE
88 #elif (OPENSSL_VERSION_NUMBER >= 0x00907000L)
89 # ifdef ENGINE_DYNAMIC_SUPPORT
90 # define DYNAMIC_ENGINE
93 # error "Only OpenSSL >= 0.9.7 is supported"
96 /* VIA PadLock AES is available *ONLY* on some x86 CPUs.
97 Not only that it doesn't exist elsewhere, but it
98 even can't be compiled on other platforms!
100 In addition, because of the heavy use of inline assembler,
101 compiler choice is limited to GCC and Microsoft C. */
102 #undef COMPILE_HW_PADLOCK
103 #if !defined(I386_ONLY) && !defined(OPENSSL_NO_INLINE_ASM)
104 # if (defined(__GNUC__) && (defined(__i386__) || defined(__i386))) || \
105 (defined(_MSC_VER) && defined(_M_IX86))
106 # define COMPILE_HW_PADLOCK
107 static ENGINE *ENGINE_padlock (void);
111 void ENGINE_load_padlock (void)
113 /* On non-x86 CPUs it just returns. */
114 #ifdef COMPILE_HW_PADLOCK
115 ENGINE *toadd = ENGINE_padlock ();
123 #ifdef COMPILE_HW_PADLOCK
124 /* We do these includes here to avoid header problems on platforms that
125 do not have the VIA padlock anyway... */
130 # define alloca _alloca
132 #elif defined(__GNUC__)
134 # define alloca(s) __builtin_alloca(s)
138 /* Function for ENGINE detection and control */
139 static int padlock_available(void);
140 static int padlock_init(ENGINE *e);
143 static RAND_METHOD padlock_rand;
146 #ifndef OPENSSL_NO_AES
147 static int padlock_ciphers(ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid);
151 static const char *padlock_id = "padlock";
152 static char padlock_name[100];
154 /* Available features */
155 static int padlock_use_ace = 0; /* Advanced Cryptography Engine */
156 static int padlock_use_rng = 0; /* Random Number Generator */
157 #ifndef OPENSSL_NO_AES
158 static int padlock_aes_align_required = 1;
161 /* ===== Engine "management" functions ===== */
163 /* Prepare the ENGINE structure for registration */
165 padlock_bind_helper(ENGINE *e)
167 /* Check available features */
170 #if 1 /* disable RNG for now, see commentary in vicinity of RNG code */
174 /* Generate a nice engine name with available features */
175 BIO_snprintf(padlock_name, sizeof(padlock_name),
176 "VIA PadLock (%s, %s)",
177 padlock_use_rng ? "RNG" : "no-RNG",
178 padlock_use_ace ? "ACE" : "no-ACE");
180 /* Register everything or return with an error */
181 if (!ENGINE_set_id(e, padlock_id) ||
182 !ENGINE_set_name(e, padlock_name) ||
184 !ENGINE_set_init_function(e, padlock_init) ||
185 #ifndef OPENSSL_NO_AES
186 (padlock_use_ace && !ENGINE_set_ciphers (e, padlock_ciphers)) ||
188 (padlock_use_rng && !ENGINE_set_RAND (e, &padlock_rand))) {
192 /* Everything looks good */
200 ENGINE *eng = ENGINE_new();
206 if (!padlock_bind_helper(eng)) {
214 /* Check availability of the engine */
216 padlock_init(ENGINE *e)
218 return (padlock_use_rng || padlock_use_ace);
221 /* This stuff is needed if this ENGINE is being compiled into a self-contained
224 #ifdef DYNAMIC_ENGINE
226 padlock_bind_fn(ENGINE *e, const char *id)
228 if (id && (strcmp(id, padlock_id) != 0)) {
232 if (!padlock_bind_helper(e)) {
239 IMPLEMENT_DYNAMIC_CHECK_FN()
240 IMPLEMENT_DYNAMIC_BIND_FN (padlock_bind_fn)
241 #endif /* DYNAMIC_ENGINE */
243 /* ===== Here comes the "real" engine ===== */
245 #ifndef OPENSSL_NO_AES
246 /* Some AES-related constants */
247 #define AES_BLOCK_SIZE 16
248 #define AES_KEY_SIZE_128 16
249 #define AES_KEY_SIZE_192 24
250 #define AES_KEY_SIZE_256 32
252 /* Here we store the status information relevant to the
255 * Inline assembler in PADLOCK_XCRYPT_ASM()
256 * depends on the order of items in this structure.
257 * Don't blindly modify, reorder, etc!
259 struct padlock_cipher_data
261 unsigned char iv[AES_BLOCK_SIZE]; /* Initialization vector */
262 union { unsigned int pad[4];
265 int dgst:1; /* n/a in C3 */
266 int align:1; /* n/a in C3 */
267 int ciphr:1; /* n/a in C3 */
268 unsigned int keygen:1;
270 unsigned int encdec:1;
273 } cword; /* Control word */
274 AES_KEY ks; /* Encryption key */
278 * Essentially this variable belongs in thread local storage.
279 * Having this variable global on the other hand can only cause
280 * few bogus key reloads [if any at all on single-CPU system],
281 * so we accept the penatly...
283 static volatile struct padlock_cipher_data *padlock_saved_context;
287 * =======================================================
288 * Inline assembler section(s).
289 * =======================================================
290 * Order of arguments is chosen to facilitate Windows port
291 * using __fastcall calling convention. If you wish to add
292 * more routines, keep in mind that first __fastcall
293 * argument is passed in %ecx and second - in %edx.
294 * =======================================================
296 #if defined(__GNUC__) && __GNUC__>=2
298 * As for excessive "push %ebx"/"pop %ebx" found all over.
299 * When generating position-independent code GCC won't let
300 * us use "b" in assembler templates nor even respect "ebx"
301 * in "clobber description." Therefore the trouble...
304 /* Helper function - check if a CPUID instruction
305 is available on this CPU */
307 padlock_insn_cpuid_available(void)
311 /* We're checking if the bit #21 of EFLAGS
312 can be toggled. If yes = CPUID is available. */
316 "xorl $0x200000, %%eax\n"
317 "movl %%eax, %%ecx\n"
318 "andl $0x200000, %%ecx\n"
323 "andl $0x200000, %%eax\n"
324 "xorl %%eax, %%ecx\n"
326 : "=r" (result) : : "eax", "ecx");
328 return (result == 0);
331 /* Load supported features of the CPU to see if
332 the PadLock is available. */
334 padlock_available(void)
336 char vendor_string[16];
337 unsigned int eax, edx;
339 /* First check if the CPUID instruction is available at all... */
340 if (! padlock_insn_cpuid_available())
343 /* Are we running on the Centaur (VIA) CPU? */
345 vendor_string[12] = 0;
349 "movl %%ebx,(%%edi)\n"
350 "movl %%edx,4(%%edi)\n"
351 "movl %%ecx,8(%%edi)\n"
353 : "+a"(eax) : "D"(vendor_string) : "ecx", "edx");
354 if (strcmp(vendor_string, "CentaurHauls") != 0)
357 /* Check for Centaur Extended Feature Flags presence */
359 asm volatile ("pushl %%ebx; cpuid; popl %%ebx"
360 : "+a"(eax) : : "ecx", "edx");
361 if (eax < 0xC0000001)
364 /* Read the Centaur Extended Feature Flags */
366 asm volatile ("pushl %%ebx; cpuid; popl %%ebx"
367 : "+a"(eax), "=d"(edx) : : "ecx");
369 /* Fill up some flags */
370 padlock_use_ace = ((edx & (0x3<<6)) == (0x3<<6));
371 padlock_use_rng = ((edx & (0x3<<2)) == (0x3<<2));
373 return padlock_use_ace + padlock_use_rng;
376 #ifndef OPENSSL_NO_AES
377 /* Our own htonl()/ntohl() */
379 padlock_bswapl(AES_KEY *ks)
381 size_t i = sizeof(ks->rd_key)/sizeof(ks->rd_key[0]);
382 unsigned int *key = ks->rd_key;
385 asm volatile ("bswapl %0" : "+r"(*key));
391 /* Force key reload from memory to the CPU microcode.
392 Loading EFLAGS from the stack clears EFLAGS[30]
393 which does the trick. */
395 padlock_reload_key(void)
397 asm volatile ("pushfl; popfl");
400 #ifndef OPENSSL_NO_AES
402 * This is heuristic key context tracing. At first one
403 * believes that one should use atomic swap instructions,
404 * but it's not actually necessary. Point is that if
405 * padlock_saved_context was changed by another thread
406 * after we've read it and before we compare it with cdata,
407 * our key *shall* be reloaded upon thread context switch
408 * and we are therefore set in either case...
411 padlock_verify_context(struct padlock_cipher_data *cdata)
423 :"+m"(padlock_saved_context)
424 : "r"(padlock_saved_context), "r"(cdata) : "cc");
427 /* Template for padlock_xcrypt_* modes */
429 * The offsets used with 'leal' instructions
430 * describe items of the 'padlock_cipher_data'
433 #define PADLOCK_XCRYPT_ASM(name,rep_xcrypt) \
434 static inline void *name(size_t cnt, \
435 struct padlock_cipher_data *cdata, \
436 void *out, const void *inp) \
438 asm volatile ( "pushl %%ebx\n" \
439 " leal 16(%0),%%edx\n" \
440 " leal 32(%0),%%ebx\n" \
443 : "=a"(iv), "=c"(cnt), "=D"(out), "=S"(inp) \
444 : "0"(cdata), "1"(cnt), "2"(out), "3"(inp) \
445 : "edx", "cc", "memory"); \
449 /* Generate all functions with appropriate opcodes */
450 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb, ".byte 0xf3,0x0f,0xa7,0xc8") /* rep xcryptecb */
451 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc, ".byte 0xf3,0x0f,0xa7,0xd0") /* rep xcryptcbc */
452 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb, ".byte 0xf3,0x0f,0xa7,0xe0") /* rep xcryptcfb */
453 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb, ".byte 0xf3,0x0f,0xa7,0xe8") /* rep xcryptofb */
456 /* The RNG call itself */
457 static inline unsigned int
458 padlock_xstore(void *addr, unsigned int edx_in)
460 unsigned int eax_out;
462 asm volatile (".byte 0x0f,0xa7,0xc0" /* xstore */
463 : "=a"(eax_out),"=m"(*(unsigned *)addr)
464 : "D"(addr), "d" (edx_in)
470 /* Why not inline 'rep movsd'? I failed to find information on what
471 * value in Direction Flag one can expect and consequently have to
472 * apply "better-safe-than-sorry" approach and assume "undefined."
473 * I could explicitly clear it and restore the original value upon
474 * return from padlock_aes_cipher, but it's presumably too much
475 * trouble for too little gain...
477 * In case you wonder 'rep xcrypt*' instructions above are *not*
478 * affected by the Direction Flag and pointers advance toward
479 * larger addresses unconditionally.
481 static inline unsigned char *
482 padlock_memcpy(void *dst,const void *src,size_t n)
488 do { *d++ = *s++; } while (--n);
493 #elif defined(_MSC_VER)
495 * Unlike GCC these are real functions. In order to minimize impact
496 * on performance we adhere to __fastcall calling convention in
497 * order to get two first arguments passed through %ecx and %edx.
498 * Which kind of suits very well, as instructions in question use
499 * both %ecx and %edx as input:-)
501 #define REP_XCRYPT(code) \
503 _asm _emit 0x0f _asm _emit 0xa7 \
507 * The offsets used with 'lea' instructions
508 * describe items of the 'padlock_cipher_data'
511 #define PADLOCK_XCRYPT_ASM(name,code) \
512 static void * __fastcall \
513 name (size_t cnt, void *cdata, \
514 void *outp, const void *inp) \
516 _asm lea edx,[eax+16] \
517 _asm lea ebx,[eax+32] \
523 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb,0xc8)
524 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc,0xd0)
525 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb,0xe0)
526 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb,0xe8)
528 static int __fastcall
529 padlock_xstore(void *outp,unsigned int code)
531 _asm _emit 0x0f _asm _emit 0xa7 _asm _emit 0xc0
534 static void __fastcall
535 padlock_reload_key(void)
536 { _asm pushfd _asm popfd }
538 static void __fastcall
539 padlock_verify_context(void *cdata)
544 cmp ecx,padlock_saved_context
549 mov padlock_saved_context,ecx
554 padlock_available(void)
589 mov padlock_use_ace,1
595 mov padlock_use_rng,1
602 static void __fastcall
603 padlock_bswapl(void *key)
618 /* MS actually specifies status of Direction Flag and compiler even
619 * manages to compile following as 'rep movsd' all by itself...
621 #define padlock_memcpy(o,i,n) ((unsigned char *)memcpy((o),(i),(n)&~3U))
624 /* ===== AES encryption/decryption ===== */
625 #ifndef OPENSSL_NO_AES
627 #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb)
628 #define NID_aes_128_cfb NID_aes_128_cfb128
631 #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb)
632 #define NID_aes_128_ofb NID_aes_128_ofb128
635 #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb)
636 #define NID_aes_192_cfb NID_aes_192_cfb128
639 #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb)
640 #define NID_aes_192_ofb NID_aes_192_ofb128
643 #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb)
644 #define NID_aes_256_cfb NID_aes_256_cfb128
647 #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb)
648 #define NID_aes_256_ofb NID_aes_256_ofb128
651 /* List of supported ciphers. */
652 static int padlock_cipher_nids[] = {
668 static int padlock_cipher_nids_num = (sizeof(padlock_cipher_nids)/
669 sizeof(padlock_cipher_nids[0]));
671 /* Function prototypes ... */
672 static int padlock_aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
673 const unsigned char *iv, int enc);
674 static int padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
675 const unsigned char *in, size_t nbytes);
677 #define NEAREST_ALIGNED(ptr) ( (unsigned char *)(ptr) + \
678 ( (0x10 - ((size_t)(ptr) & 0x0F)) & 0x0F ) )
679 #define ALIGNED_CIPHER_DATA(ctx) ((struct padlock_cipher_data *)\
680 NEAREST_ALIGNED(ctx->cipher_data))
682 #define EVP_CIPHER_block_size_ECB AES_BLOCK_SIZE
683 #define EVP_CIPHER_block_size_CBC AES_BLOCK_SIZE
684 #define EVP_CIPHER_block_size_OFB 1
685 #define EVP_CIPHER_block_size_CFB 1
687 /* Declaring so many ciphers by hand would be a pain.
688 Instead introduce a bit of preprocessor magic :-) */
689 #define DECLARE_AES_EVP(ksize,lmode,umode) \
690 static const EVP_CIPHER padlock_aes_##ksize##_##lmode = { \
691 NID_aes_##ksize##_##lmode, \
692 EVP_CIPHER_block_size_##umode, \
693 AES_KEY_SIZE_##ksize, \
695 0 | EVP_CIPH_##umode##_MODE, \
696 padlock_aes_init_key, \
697 padlock_aes_cipher, \
699 sizeof(struct padlock_cipher_data) + 16, \
700 EVP_CIPHER_set_asn1_iv, \
701 EVP_CIPHER_get_asn1_iv, \
706 DECLARE_AES_EVP(128,ecb,ECB);
707 DECLARE_AES_EVP(128,cbc,CBC);
708 DECLARE_AES_EVP(128,cfb,CFB);
709 DECLARE_AES_EVP(128,ofb,OFB);
711 DECLARE_AES_EVP(192,ecb,ECB);
712 DECLARE_AES_EVP(192,cbc,CBC);
713 DECLARE_AES_EVP(192,cfb,CFB);
714 DECLARE_AES_EVP(192,ofb,OFB);
716 DECLARE_AES_EVP(256,ecb,ECB);
717 DECLARE_AES_EVP(256,cbc,CBC);
718 DECLARE_AES_EVP(256,cfb,CFB);
719 DECLARE_AES_EVP(256,ofb,OFB);
722 padlock_ciphers (ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid)
724 /* No specific cipher => return a list of supported nids ... */
726 *nids = padlock_cipher_nids;
727 return padlock_cipher_nids_num;
730 /* ... or the requested "cipher" otherwise */
732 case NID_aes_128_ecb:
733 *cipher = &padlock_aes_128_ecb;
735 case NID_aes_128_cbc:
736 *cipher = &padlock_aes_128_cbc;
738 case NID_aes_128_cfb:
739 *cipher = &padlock_aes_128_cfb;
741 case NID_aes_128_ofb:
742 *cipher = &padlock_aes_128_ofb;
745 case NID_aes_192_ecb:
746 *cipher = &padlock_aes_192_ecb;
748 case NID_aes_192_cbc:
749 *cipher = &padlock_aes_192_cbc;
751 case NID_aes_192_cfb:
752 *cipher = &padlock_aes_192_cfb;
754 case NID_aes_192_ofb:
755 *cipher = &padlock_aes_192_ofb;
758 case NID_aes_256_ecb:
759 *cipher = &padlock_aes_256_ecb;
761 case NID_aes_256_cbc:
762 *cipher = &padlock_aes_256_cbc;
764 case NID_aes_256_cfb:
765 *cipher = &padlock_aes_256_cfb;
767 case NID_aes_256_ofb:
768 *cipher = &padlock_aes_256_ofb;
772 /* Sorry, we don't support this NID */
780 /* Prepare the encryption key for PadLock usage */
782 padlock_aes_init_key (EVP_CIPHER_CTX *ctx, const unsigned char *key,
783 const unsigned char *iv, int enc)
785 struct padlock_cipher_data *cdata;
786 int key_len = EVP_CIPHER_CTX_key_length(ctx) * 8;
788 if (key==NULL) return 0; /* ERROR */
790 cdata = ALIGNED_CIPHER_DATA(ctx);
791 memset(cdata, 0, sizeof(struct padlock_cipher_data));
793 /* Prepare Control word. */
794 if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE)
795 cdata->cword.b.encdec = 0;
797 cdata->cword.b.encdec = (ctx->encrypt == 0);
798 cdata->cword.b.rounds = 10 + (key_len - 128) / 32;
799 cdata->cword.b.ksize = (key_len - 128) / 64;
803 /* PadLock can generate an extended key for
804 AES128 in hardware */
805 memcpy(cdata->ks.rd_key, key, AES_KEY_SIZE_128);
806 cdata->cword.b.keygen = 0;
811 /* Generate an extended AES key in software.
812 Needed for AES192/AES256 */
813 /* Well, the above applies to Stepping 8 CPUs
814 and is listed as hardware errata. They most
815 likely will fix it at some point and then
816 a check for stepping would be due here. */
817 if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_CFB_MODE ||
818 EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE ||
820 AES_set_encrypt_key(key, key_len, &cdata->ks);
822 AES_set_decrypt_key(key, key_len, &cdata->ks);
824 /* OpenSSL C functions use byte-swapped extended key. */
825 padlock_bswapl(&cdata->ks);
827 cdata->cword.b.keygen = 1;
836 * This is done to cover for cases when user reuses the
837 * context for new key. The catch is that if we don't do
838 * this, padlock_eas_cipher might proceed with old key...
840 padlock_reload_key ();
846 * Simplified version of padlock_aes_cipher() used when
847 * 1) both input and output buffers are at aligned addresses.
849 * 2) running on a newer CPU that doesn't require aligned buffers.
852 padlock_aes_cipher_omnivorous(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,
853 const unsigned char *in_arg, size_t nbytes)
855 struct padlock_cipher_data *cdata;
858 cdata = ALIGNED_CIPHER_DATA(ctx);
859 padlock_verify_context(cdata);
861 switch (EVP_CIPHER_CTX_mode(ctx)) {
862 case EVP_CIPH_ECB_MODE:
863 padlock_xcrypt_ecb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
866 case EVP_CIPH_CBC_MODE:
867 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
868 iv = padlock_xcrypt_cbc(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
869 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
872 case EVP_CIPH_CFB_MODE:
873 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
874 iv = padlock_xcrypt_cfb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
875 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
878 case EVP_CIPH_OFB_MODE:
879 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
880 padlock_xcrypt_ofb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
881 memcpy(ctx->iv, cdata->iv, AES_BLOCK_SIZE);
888 memset(cdata->iv, 0, AES_BLOCK_SIZE);
893 #ifndef PADLOCK_CHUNK
894 # define PADLOCK_CHUNK 512 /* Must be a power of 2 larger than 16 */
896 #if PADLOCK_CHUNK<16 || PADLOCK_CHUNK&(PADLOCK_CHUNK-1)
897 # error "insane PADLOCK_CHUNK..."
900 /* Re-align the arguments to 16-Bytes boundaries and run the
901 encryption function itself. This function is not AES-specific. */
903 padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,
904 const unsigned char *in_arg, size_t nbytes)
906 struct padlock_cipher_data *cdata;
910 int inp_misaligned, out_misaligned, realign_in_loop;
911 size_t chunk, allocated=0;
913 /* ctx->num is maintained in byte-oriented modes,
914 such as CFB and OFB... */
915 if ((chunk = ctx->num)) { /* borrow chunk variable */
916 unsigned char *ivp=ctx->iv;
918 switch (EVP_CIPHER_CTX_mode(ctx)) {
919 case EVP_CIPH_CFB_MODE:
920 if (chunk >= AES_BLOCK_SIZE)
921 return 0; /* bogus value */
924 while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
925 ivp[chunk] = *(out_arg++) = *(in_arg++) ^ ivp[chunk];
928 else while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
929 unsigned char c = *(in_arg++);
930 *(out_arg++) = c ^ ivp[chunk];
931 ivp[chunk++] = c, nbytes--;
934 ctx->num = chunk%AES_BLOCK_SIZE;
936 case EVP_CIPH_OFB_MODE:
937 if (chunk >= AES_BLOCK_SIZE)
938 return 0; /* bogus value */
940 while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
941 *(out_arg++) = *(in_arg++) ^ ivp[chunk];
945 ctx->num = chunk%AES_BLOCK_SIZE;
953 if (nbytes % AES_BLOCK_SIZE)
954 return 0; /* are we expected to do tail processing? */
956 /* nbytes is always multiple of AES_BLOCK_SIZE in ECB and CBC
957 modes and arbitrary value in byte-oriented modes, such as
961 /* VIA promises CPUs that won't require alignment in the future.
962 For now padlock_aes_align_required is initialized to 1 and
963 the condition is never met... */
964 /* C7 core is capable to manage unaligned input in non-ECB[!]
965 mode, but performance penalties appear to be approximately
966 same as for software alignment below or ~3x. They promise to
967 improve it in the future, but for now we can just as well
968 pretend that it can only handle aligned input... */
969 if (!padlock_aes_align_required && (nbytes%AES_BLOCK_SIZE)==0)
970 return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);
972 inp_misaligned = (((size_t)in_arg) & 0x0F);
973 out_misaligned = (((size_t)out_arg) & 0x0F);
975 /* Note that even if output is aligned and input not,
976 * I still prefer to loop instead of copy the whole
977 * input and then encrypt in one stroke. This is done
978 * in order to improve L1 cache utilization... */
979 realign_in_loop = out_misaligned|inp_misaligned;
981 if (!realign_in_loop && (nbytes%AES_BLOCK_SIZE)==0)
982 return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);
984 /* this takes one "if" out of the loops */
986 chunk %= PADLOCK_CHUNK;
987 if (chunk==0) chunk = PADLOCK_CHUNK;
989 if (out_misaligned) {
990 /* optmize for small input */
991 allocated = (chunk<nbytes?PADLOCK_CHUNK:nbytes);
992 out = alloca(0x10 + allocated);
993 out = NEAREST_ALIGNED(out);
998 cdata = ALIGNED_CIPHER_DATA(ctx);
999 padlock_verify_context(cdata);
1001 switch (EVP_CIPHER_CTX_mode(ctx)) {
1002 case EVP_CIPH_ECB_MODE:
1005 inp = padlock_memcpy(out, in_arg, chunk);
1010 padlock_xcrypt_ecb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1013 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1015 out = out_arg+=chunk;
1018 chunk = PADLOCK_CHUNK;
1022 case EVP_CIPH_CBC_MODE:
1023 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1026 if (iv != cdata->iv)
1027 memcpy(cdata->iv, iv, AES_BLOCK_SIZE);
1028 chunk = PADLOCK_CHUNK;
1029 cbc_shortcut: /* optimize for small input */
1031 inp = padlock_memcpy(out, in_arg, chunk);
1036 iv = padlock_xcrypt_cbc(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1039 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1041 out = out_arg+=chunk;
1043 } while (nbytes -= chunk);
1044 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
1047 case EVP_CIPH_CFB_MODE:
1048 memcpy (iv = cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1049 chunk &= ~(AES_BLOCK_SIZE-1);
1050 if (chunk) goto cfb_shortcut;
1051 else goto cfb_skiploop;
1053 if (iv != cdata->iv)
1054 memcpy(cdata->iv, iv, AES_BLOCK_SIZE);
1055 chunk = PADLOCK_CHUNK;
1056 cfb_shortcut: /* optimize for small input */
1058 inp = padlock_memcpy(out, in_arg, chunk);
1063 iv = padlock_xcrypt_cfb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1066 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1068 out = out_arg+=chunk;
1071 } while (nbytes >= AES_BLOCK_SIZE);
1075 unsigned char *ivp = cdata->iv;
1078 memcpy(ivp, iv, AES_BLOCK_SIZE);
1082 if (cdata->cword.b.encdec) {
1083 cdata->cword.b.encdec=0;
1084 padlock_reload_key();
1085 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1086 cdata->cword.b.encdec=1;
1087 padlock_reload_key();
1089 unsigned char c = *(in_arg++);
1090 *(out_arg++) = c ^ *ivp;
1091 *(ivp++) = c, nbytes--;
1094 else { padlock_reload_key();
1095 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1096 padlock_reload_key();
1098 *ivp = *(out_arg++) = *(in_arg++) ^ *ivp;
1104 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
1107 case EVP_CIPH_OFB_MODE:
1108 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1109 chunk &= ~(AES_BLOCK_SIZE-1);
1112 inp = padlock_memcpy(out, in_arg, chunk);
1117 padlock_xcrypt_ofb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1120 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1122 out = out_arg+=chunk;
1125 chunk = PADLOCK_CHUNK;
1126 } while (nbytes >= AES_BLOCK_SIZE);
1129 unsigned char *ivp = cdata->iv;
1132 padlock_reload_key(); /* empirically found */
1133 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1134 padlock_reload_key(); /* empirically found */
1136 *(out_arg++) = *(in_arg++) ^ *ivp;
1141 memcpy(ctx->iv, cdata->iv, AES_BLOCK_SIZE);
1148 /* Clean the realign buffer if it was used */
1149 if (out_misaligned) {
1150 volatile unsigned long *p=(void *)out;
1151 size_t n = allocated/sizeof(*p);
1155 memset(cdata->iv, 0, AES_BLOCK_SIZE);
1160 #endif /* OPENSSL_NO_AES */
1162 /* ===== Random Number Generator ===== */
1164 * This code is not engaged. The reason is that it does not comply
1165 * with recommendations for VIA RNG usage for secure applications
1166 * (posted at http://www.via.com.tw/en/viac3/c3.jsp) nor does it
1167 * provide meaningful error control...
1169 /* Wrapper that provides an interface between the API and
1170 the raw PadLock RNG */
1172 padlock_rand_bytes(unsigned char *output, int count)
1174 unsigned int eax, buf;
1176 while (count >= 8) {
1177 eax = padlock_xstore(output, 0);
1178 if (!(eax&(1<<6))) return 0; /* RNG disabled */
1179 /* this ---vv--- covers DC bias, Raw Bits and String Filter */
1180 if (eax&(0x1F<<10)) return 0;
1181 if ((eax&0x1F)==0) continue; /* no data, retry... */
1182 if ((eax&0x1F)!=8) return 0; /* fatal failure... */
1187 eax = padlock_xstore(&buf, 3);
1188 if (!(eax&(1<<6))) return 0; /* RNG disabled */
1189 /* this ---vv--- covers DC bias, Raw Bits and String Filter */
1190 if (eax&(0x1F<<10)) return 0;
1191 if ((eax&0x1F)==0) continue; /* no data, retry... */
1192 if ((eax&0x1F)!=1) return 0; /* fatal failure... */
1193 *output++ = (unsigned char)buf;
1196 *(volatile unsigned int *)&buf=0;
1201 /* Dummy but necessary function */
1203 padlock_rand_status(void)
1208 /* Prepare structure for registration */
1209 static RAND_METHOD padlock_rand = {
1211 padlock_rand_bytes, /* bytes */
1214 padlock_rand_bytes, /* pseudorand */
1215 padlock_rand_status, /* rand status */
1218 #else /* !COMPILE_HW_PADLOCK */
1219 #ifndef OPENSSL_NO_DYNAMIC_ENGINE
1221 int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns) { return 0; }
1222 IMPLEMENT_DYNAMIC_CHECK_FN()
1224 #endif /* COMPILE_HW_PADLOCK */
1226 #endif /* !OPENSSL_NO_HW_PADLOCK */
1227 #endif /* !OPENSSL_NO_HW */