2 * Support for VIA PadLock Advanced Cryptography Engine (ACE)
3 * Written by Michal Ludvig <michal@logix.cz>
4 * http://www.logix.cz/michal
6 * Big thanks to Andy Polyakov for a help with optimization,
7 * assembler fixes, port to MS Windows and a lot of other
8 * valuable work on this engine!
11 /* ====================================================================
12 * Copyright (c) 1999-2001 The OpenSSL Project. All rights reserved.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in
23 * the documentation and/or other materials provided with the
26 * 3. All advertising materials mentioning features or use of this
27 * software must display the following acknowledgment:
28 * "This product includes software developed by the OpenSSL Project
29 * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
31 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
32 * endorse or promote products derived from this software without
33 * prior written permission. For written permission, please contact
34 * licensing@OpenSSL.org.
36 * 5. Products derived from this software may not be called "OpenSSL"
37 * nor may "OpenSSL" appear in their names without prior written
38 * permission of the OpenSSL Project.
40 * 6. Redistributions of any form whatsoever must retain the following
42 * "This product includes software developed by the OpenSSL Project
43 * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
45 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
46 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
48 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
49 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
51 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
52 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
54 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
55 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
56 * OF THE POSSIBILITY OF SUCH DAMAGE.
57 * ====================================================================
59 * This product includes cryptographic software written by Eric Young
60 * (eay@cryptsoft.com). This product includes software written by Tim
61 * Hudson (tjh@cryptsoft.com).
69 #include <openssl/opensslconf.h>
70 #include <openssl/crypto.h>
71 #include <openssl/dso.h>
72 #include <openssl/engine.h>
73 #include <openssl/evp.h>
74 #ifndef OPENSSL_NO_AES
75 #include <openssl/aes.h>
77 #include <openssl/rand.h>
78 #include <openssl/err.h>
81 #ifndef OPENSSL_NO_HW_PADLOCK
83 /* Attempt to have a single source for both 0.9.7 and 0.9.8 :-) */
84 #if (OPENSSL_VERSION_NUMBER >= 0x00908000L)
85 # ifndef OPENSSL_NO_DYNAMIC_ENGINE
86 # define DYNAMIC_ENGINE
88 #elif (OPENSSL_VERSION_NUMBER >= 0x00907000L)
89 # ifdef ENGINE_DYNAMIC_SUPPORT
90 # define DYNAMIC_ENGINE
93 # error "Only OpenSSL >= 0.9.7 is supported"
96 /* VIA PadLock AES is available *ONLY* on some x86 CPUs.
97 Not only that it doesn't exist elsewhere, but it
98 even can't be compiled on other platforms!
100 In addition, because of the heavy use of inline assembler,
101 compiler choice is limited to GCC and Microsoft C. */
102 #undef COMPILE_HW_PADLOCK
103 #if !defined(I386_ONLY) && !defined(OPENSSL_NO_INLINE_ASM)
104 # if (defined(__GNUC__) && (defined(__i386__) || defined(__i386))) || \
105 (defined(_MSC_VER) && defined(_M_IX86))
106 # define COMPILE_HW_PADLOCK
110 #ifdef OPENSSL_NO_DYNAMIC_ENGINE
111 #ifdef COMPILE_HW_PADLOCK
112 static ENGINE *ENGINE_padlock (void);
115 void ENGINE_load_padlock (void)
117 /* On non-x86 CPUs it just returns. */
118 #ifdef COMPILE_HW_PADLOCK
119 ENGINE *toadd = ENGINE_padlock ();
129 #ifdef COMPILE_HW_PADLOCK
130 /* We do these includes here to avoid header problems on platforms that
131 do not have the VIA padlock anyway... */
136 # define alloca _alloca
138 #elif defined(__GNUC__)
140 # define alloca(s) __builtin_alloca(s)
144 /* Function for ENGINE detection and control */
145 static int padlock_available(void);
146 static int padlock_init(ENGINE *e);
149 static RAND_METHOD padlock_rand;
152 #ifndef OPENSSL_NO_AES
153 static int padlock_ciphers(ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid);
157 static const char *padlock_id = "padlock";
158 static char padlock_name[100];
160 /* Available features */
161 static int padlock_use_ace = 0; /* Advanced Cryptography Engine */
162 static int padlock_use_rng = 0; /* Random Number Generator */
163 #ifndef OPENSSL_NO_AES
164 static int padlock_aes_align_required = 1;
167 /* ===== Engine "management" functions ===== */
169 /* Prepare the ENGINE structure for registration */
171 padlock_bind_helper(ENGINE *e)
173 /* Check available features */
176 #if 1 /* disable RNG for now, see commentary in vicinity of RNG code */
180 /* Generate a nice engine name with available features */
181 BIO_snprintf(padlock_name, sizeof(padlock_name),
182 "VIA PadLock (%s, %s)",
183 padlock_use_rng ? "RNG" : "no-RNG",
184 padlock_use_ace ? "ACE" : "no-ACE");
186 /* Register everything or return with an error */
187 if (!ENGINE_set_id(e, padlock_id) ||
188 !ENGINE_set_name(e, padlock_name) ||
190 !ENGINE_set_init_function(e, padlock_init) ||
191 #ifndef OPENSSL_NO_AES
192 (padlock_use_ace && !ENGINE_set_ciphers (e, padlock_ciphers)) ||
194 (padlock_use_rng && !ENGINE_set_RAND (e, &padlock_rand))) {
198 /* Everything looks good */
202 #ifdef OPENSSL_NO_DYNAMIC_ENGINE
208 ENGINE *eng = ENGINE_new();
214 if (!padlock_bind_helper(eng)) {
224 /* Check availability of the engine */
226 padlock_init(ENGINE *e)
228 return (padlock_use_rng || padlock_use_ace);
231 /* This stuff is needed if this ENGINE is being compiled into a self-contained
234 #ifdef DYNAMIC_ENGINE
236 padlock_bind_fn(ENGINE *e, const char *id)
238 if (id && (strcmp(id, padlock_id) != 0)) {
242 if (!padlock_bind_helper(e)) {
249 IMPLEMENT_DYNAMIC_CHECK_FN()
250 IMPLEMENT_DYNAMIC_BIND_FN (padlock_bind_fn)
251 #endif /* DYNAMIC_ENGINE */
253 /* ===== Here comes the "real" engine ===== */
255 #ifndef OPENSSL_NO_AES
256 /* Some AES-related constants */
257 #define AES_BLOCK_SIZE 16
258 #define AES_KEY_SIZE_128 16
259 #define AES_KEY_SIZE_192 24
260 #define AES_KEY_SIZE_256 32
262 /* Here we store the status information relevant to the
265 * Inline assembler in PADLOCK_XCRYPT_ASM()
266 * depends on the order of items in this structure.
267 * Don't blindly modify, reorder, etc!
269 struct padlock_cipher_data
271 unsigned char iv[AES_BLOCK_SIZE]; /* Initialization vector */
272 union { unsigned int pad[4];
275 int dgst:1; /* n/a in C3 */
276 int align:1; /* n/a in C3 */
277 int ciphr:1; /* n/a in C3 */
278 unsigned int keygen:1;
280 unsigned int encdec:1;
283 } cword; /* Control word */
284 AES_KEY ks; /* Encryption key */
288 * Essentially this variable belongs in thread local storage.
289 * Having this variable global on the other hand can only cause
290 * few bogus key reloads [if any at all on single-CPU system],
291 * so we accept the penatly...
293 static volatile struct padlock_cipher_data *padlock_saved_context;
297 * =======================================================
298 * Inline assembler section(s).
299 * =======================================================
300 * Order of arguments is chosen to facilitate Windows port
301 * using __fastcall calling convention. If you wish to add
302 * more routines, keep in mind that first __fastcall
303 * argument is passed in %ecx and second - in %edx.
304 * =======================================================
306 #if defined(__GNUC__) && __GNUC__>=2
308 * As for excessive "push %ebx"/"pop %ebx" found all over.
309 * When generating position-independent code GCC won't let
310 * us use "b" in assembler templates nor even respect "ebx"
311 * in "clobber description." Therefore the trouble...
314 /* Helper function - check if a CPUID instruction
315 is available on this CPU */
317 padlock_insn_cpuid_available(void)
321 /* We're checking if the bit #21 of EFLAGS
322 can be toggled. If yes = CPUID is available. */
326 "xorl $0x200000, %%eax\n"
327 "movl %%eax, %%ecx\n"
328 "andl $0x200000, %%ecx\n"
333 "andl $0x200000, %%eax\n"
334 "xorl %%eax, %%ecx\n"
336 : "=r" (result) : : "eax", "ecx");
338 return (result == 0);
341 /* Load supported features of the CPU to see if
342 the PadLock is available. */
344 padlock_available(void)
346 char vendor_string[16];
347 unsigned int eax, edx;
349 /* First check if the CPUID instruction is available at all... */
350 if (! padlock_insn_cpuid_available())
353 /* Are we running on the Centaur (VIA) CPU? */
355 vendor_string[12] = 0;
359 "movl %%ebx,(%%edi)\n"
360 "movl %%edx,4(%%edi)\n"
361 "movl %%ecx,8(%%edi)\n"
363 : "+a"(eax) : "D"(vendor_string) : "ecx", "edx");
364 if (strcmp(vendor_string, "CentaurHauls") != 0)
367 /* Check for Centaur Extended Feature Flags presence */
369 asm volatile ("pushl %%ebx; cpuid; popl %%ebx"
370 : "+a"(eax) : : "ecx", "edx");
371 if (eax < 0xC0000001)
374 /* Read the Centaur Extended Feature Flags */
376 asm volatile ("pushl %%ebx; cpuid; popl %%ebx"
377 : "+a"(eax), "=d"(edx) : : "ecx");
379 /* Fill up some flags */
380 padlock_use_ace = ((edx & (0x3<<6)) == (0x3<<6));
381 padlock_use_rng = ((edx & (0x3<<2)) == (0x3<<2));
383 return padlock_use_ace + padlock_use_rng;
386 #ifndef OPENSSL_NO_AES
388 /* Our own htonl()/ntohl() */
390 padlock_bswapl(AES_KEY *ks)
392 size_t i = sizeof(ks->rd_key)/sizeof(ks->rd_key[0]);
393 unsigned int *key = ks->rd_key;
396 asm volatile ("bswapl %0" : "+r"(*key));
403 /* Force key reload from memory to the CPU microcode.
404 Loading EFLAGS from the stack clears EFLAGS[30]
405 which does the trick. */
407 padlock_reload_key(void)
409 asm volatile ("pushfl; popfl");
412 #ifndef OPENSSL_NO_AES
414 * This is heuristic key context tracing. At first one
415 * believes that one should use atomic swap instructions,
416 * but it's not actually necessary. Point is that if
417 * padlock_saved_context was changed by another thread
418 * after we've read it and before we compare it with cdata,
419 * our key *shall* be reloaded upon thread context switch
420 * and we are therefore set in either case...
423 padlock_verify_context(struct padlock_cipher_data *cdata)
435 :"+m"(padlock_saved_context)
436 : "r"(padlock_saved_context), "r"(cdata) : "cc");
439 /* Template for padlock_xcrypt_* modes */
441 * The offsets used with 'leal' instructions
442 * describe items of the 'padlock_cipher_data'
445 #define PADLOCK_XCRYPT_ASM(name,rep_xcrypt) \
446 static inline void *name(size_t cnt, \
447 struct padlock_cipher_data *cdata, \
448 void *out, const void *inp) \
450 asm volatile ( "pushl %%ebx\n" \
451 " leal 16(%0),%%edx\n" \
452 " leal 32(%0),%%ebx\n" \
455 : "=a"(iv), "=c"(cnt), "=D"(out), "=S"(inp) \
456 : "0"(cdata), "1"(cnt), "2"(out), "3"(inp) \
457 : "edx", "cc", "memory"); \
461 /* Generate all functions with appropriate opcodes */
462 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb, ".byte 0xf3,0x0f,0xa7,0xc8") /* rep xcryptecb */
463 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc, ".byte 0xf3,0x0f,0xa7,0xd0") /* rep xcryptcbc */
464 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb, ".byte 0xf3,0x0f,0xa7,0xe0") /* rep xcryptcfb */
465 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb, ".byte 0xf3,0x0f,0xa7,0xe8") /* rep xcryptofb */
468 /* The RNG call itself */
469 static inline unsigned int
470 padlock_xstore(void *addr, unsigned int edx_in)
472 unsigned int eax_out;
474 asm volatile (".byte 0x0f,0xa7,0xc0" /* xstore */
475 : "=a"(eax_out),"=m"(*(unsigned *)addr)
476 : "D"(addr), "d" (edx_in)
482 /* Why not inline 'rep movsd'? I failed to find information on what
483 * value in Direction Flag one can expect and consequently have to
484 * apply "better-safe-than-sorry" approach and assume "undefined."
485 * I could explicitly clear it and restore the original value upon
486 * return from padlock_aes_cipher, but it's presumably too much
487 * trouble for too little gain...
489 * In case you wonder 'rep xcrypt*' instructions above are *not*
490 * affected by the Direction Flag and pointers advance toward
491 * larger addresses unconditionally.
493 static inline unsigned char *
494 padlock_memcpy(void *dst,const void *src,size_t n)
500 do { *d++ = *s++; } while (--n);
505 #elif defined(_MSC_VER)
507 * Unlike GCC these are real functions. In order to minimize impact
508 * on performance we adhere to __fastcall calling convention in
509 * order to get two first arguments passed through %ecx and %edx.
510 * Which kind of suits very well, as instructions in question use
511 * both %ecx and %edx as input:-)
513 #define REP_XCRYPT(code) \
515 _asm _emit 0x0f _asm _emit 0xa7 \
519 * The offsets used with 'lea' instructions
520 * describe items of the 'padlock_cipher_data'
523 #define PADLOCK_XCRYPT_ASM(name,code) \
524 static void * __fastcall \
525 name (size_t cnt, void *cdata, \
526 void *outp, const void *inp) \
528 _asm lea edx,[eax+16] \
529 _asm lea ebx,[eax+32] \
535 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb,0xc8)
536 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc,0xd0)
537 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb,0xe0)
538 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb,0xe8)
540 static int __fastcall
541 padlock_xstore(void *outp,unsigned int code)
543 _asm _emit 0x0f _asm _emit 0xa7 _asm _emit 0xc0
546 static void __fastcall
547 padlock_reload_key(void)
548 { _asm pushfd _asm popfd }
550 static void __fastcall
551 padlock_verify_context(void *cdata)
556 cmp ecx,padlock_saved_context
561 mov padlock_saved_context,ecx
566 padlock_available(void)
601 mov padlock_use_ace,1
607 mov padlock_use_rng,1
614 static void __fastcall
615 padlock_bswapl(void *key)
630 /* MS actually specifies status of Direction Flag and compiler even
631 * manages to compile following as 'rep movsd' all by itself...
633 #define padlock_memcpy(o,i,n) ((unsigned char *)memcpy((o),(i),(n)&~3U))
636 /* ===== AES encryption/decryption ===== */
637 #ifndef OPENSSL_NO_AES
639 #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb)
640 #define NID_aes_128_cfb NID_aes_128_cfb128
643 #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb)
644 #define NID_aes_128_ofb NID_aes_128_ofb128
647 #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb)
648 #define NID_aes_192_cfb NID_aes_192_cfb128
651 #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb)
652 #define NID_aes_192_ofb NID_aes_192_ofb128
655 #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb)
656 #define NID_aes_256_cfb NID_aes_256_cfb128
659 #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb)
660 #define NID_aes_256_ofb NID_aes_256_ofb128
663 /* List of supported ciphers. */
664 static int padlock_cipher_nids[] = {
680 static int padlock_cipher_nids_num = (sizeof(padlock_cipher_nids)/
681 sizeof(padlock_cipher_nids[0]));
683 /* Function prototypes ... */
684 static int padlock_aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
685 const unsigned char *iv, int enc);
686 static int padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
687 const unsigned char *in, size_t nbytes);
689 #define NEAREST_ALIGNED(ptr) ( (unsigned char *)(ptr) + \
690 ( (0x10 - ((size_t)(ptr) & 0x0F)) & 0x0F ) )
691 #define ALIGNED_CIPHER_DATA(ctx) ((struct padlock_cipher_data *)\
692 NEAREST_ALIGNED(ctx->cipher_data))
694 #define EVP_CIPHER_block_size_ECB AES_BLOCK_SIZE
695 #define EVP_CIPHER_block_size_CBC AES_BLOCK_SIZE
696 #define EVP_CIPHER_block_size_OFB 1
697 #define EVP_CIPHER_block_size_CFB 1
699 /* Declaring so many ciphers by hand would be a pain.
700 Instead introduce a bit of preprocessor magic :-) */
701 #define DECLARE_AES_EVP(ksize,lmode,umode) \
702 static const EVP_CIPHER padlock_aes_##ksize##_##lmode = { \
703 NID_aes_##ksize##_##lmode, \
704 EVP_CIPHER_block_size_##umode, \
705 AES_KEY_SIZE_##ksize, \
707 0 | EVP_CIPH_##umode##_MODE, \
708 padlock_aes_init_key, \
709 padlock_aes_cipher, \
711 sizeof(struct padlock_cipher_data) + 16, \
712 EVP_CIPHER_set_asn1_iv, \
713 EVP_CIPHER_get_asn1_iv, \
718 DECLARE_AES_EVP(128,ecb,ECB);
719 DECLARE_AES_EVP(128,cbc,CBC);
720 DECLARE_AES_EVP(128,cfb,CFB);
721 DECLARE_AES_EVP(128,ofb,OFB);
723 DECLARE_AES_EVP(192,ecb,ECB);
724 DECLARE_AES_EVP(192,cbc,CBC);
725 DECLARE_AES_EVP(192,cfb,CFB);
726 DECLARE_AES_EVP(192,ofb,OFB);
728 DECLARE_AES_EVP(256,ecb,ECB);
729 DECLARE_AES_EVP(256,cbc,CBC);
730 DECLARE_AES_EVP(256,cfb,CFB);
731 DECLARE_AES_EVP(256,ofb,OFB);
734 padlock_ciphers (ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid)
736 /* No specific cipher => return a list of supported nids ... */
738 *nids = padlock_cipher_nids;
739 return padlock_cipher_nids_num;
742 /* ... or the requested "cipher" otherwise */
744 case NID_aes_128_ecb:
745 *cipher = &padlock_aes_128_ecb;
747 case NID_aes_128_cbc:
748 *cipher = &padlock_aes_128_cbc;
750 case NID_aes_128_cfb:
751 *cipher = &padlock_aes_128_cfb;
753 case NID_aes_128_ofb:
754 *cipher = &padlock_aes_128_ofb;
757 case NID_aes_192_ecb:
758 *cipher = &padlock_aes_192_ecb;
760 case NID_aes_192_cbc:
761 *cipher = &padlock_aes_192_cbc;
763 case NID_aes_192_cfb:
764 *cipher = &padlock_aes_192_cfb;
766 case NID_aes_192_ofb:
767 *cipher = &padlock_aes_192_ofb;
770 case NID_aes_256_ecb:
771 *cipher = &padlock_aes_256_ecb;
773 case NID_aes_256_cbc:
774 *cipher = &padlock_aes_256_cbc;
776 case NID_aes_256_cfb:
777 *cipher = &padlock_aes_256_cfb;
779 case NID_aes_256_ofb:
780 *cipher = &padlock_aes_256_ofb;
784 /* Sorry, we don't support this NID */
792 /* Prepare the encryption key for PadLock usage */
794 padlock_aes_init_key (EVP_CIPHER_CTX *ctx, const unsigned char *key,
795 const unsigned char *iv, int enc)
797 struct padlock_cipher_data *cdata;
798 int key_len = EVP_CIPHER_CTX_key_length(ctx) * 8;
800 if (key==NULL) return 0; /* ERROR */
802 cdata = ALIGNED_CIPHER_DATA(ctx);
803 memset(cdata, 0, sizeof(struct padlock_cipher_data));
805 /* Prepare Control word. */
806 if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE)
807 cdata->cword.b.encdec = 0;
809 cdata->cword.b.encdec = (ctx->encrypt == 0);
810 cdata->cword.b.rounds = 10 + (key_len - 128) / 32;
811 cdata->cword.b.ksize = (key_len - 128) / 64;
815 /* PadLock can generate an extended key for
816 AES128 in hardware */
817 memcpy(cdata->ks.rd_key, key, AES_KEY_SIZE_128);
818 cdata->cword.b.keygen = 0;
823 /* Generate an extended AES key in software.
824 Needed for AES192/AES256 */
825 /* Well, the above applies to Stepping 8 CPUs
826 and is listed as hardware errata. They most
827 likely will fix it at some point and then
828 a check for stepping would be due here. */
829 if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_CFB_MODE ||
830 EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE ||
832 AES_set_encrypt_key(key, key_len, &cdata->ks);
834 AES_set_decrypt_key(key, key_len, &cdata->ks);
836 /* OpenSSL C functions use byte-swapped extended key. */
837 padlock_bswapl(&cdata->ks);
839 cdata->cword.b.keygen = 1;
848 * This is done to cover for cases when user reuses the
849 * context for new key. The catch is that if we don't do
850 * this, padlock_eas_cipher might proceed with old key...
852 padlock_reload_key ();
858 * Simplified version of padlock_aes_cipher() used when
859 * 1) both input and output buffers are at aligned addresses.
861 * 2) running on a newer CPU that doesn't require aligned buffers.
864 padlock_aes_cipher_omnivorous(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,
865 const unsigned char *in_arg, size_t nbytes)
867 struct padlock_cipher_data *cdata;
870 cdata = ALIGNED_CIPHER_DATA(ctx);
871 padlock_verify_context(cdata);
873 switch (EVP_CIPHER_CTX_mode(ctx)) {
874 case EVP_CIPH_ECB_MODE:
875 padlock_xcrypt_ecb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
878 case EVP_CIPH_CBC_MODE:
879 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
880 iv = padlock_xcrypt_cbc(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
881 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
884 case EVP_CIPH_CFB_MODE:
885 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
886 iv = padlock_xcrypt_cfb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
887 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
890 case EVP_CIPH_OFB_MODE:
891 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
892 padlock_xcrypt_ofb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
893 memcpy(ctx->iv, cdata->iv, AES_BLOCK_SIZE);
900 memset(cdata->iv, 0, AES_BLOCK_SIZE);
905 #ifndef PADLOCK_CHUNK
906 # define PADLOCK_CHUNK 512 /* Must be a power of 2 larger than 16 */
908 #if PADLOCK_CHUNK<16 || PADLOCK_CHUNK&(PADLOCK_CHUNK-1)
909 # error "insane PADLOCK_CHUNK..."
912 /* Re-align the arguments to 16-Bytes boundaries and run the
913 encryption function itself. This function is not AES-specific. */
915 padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,
916 const unsigned char *in_arg, size_t nbytes)
918 struct padlock_cipher_data *cdata;
922 int inp_misaligned, out_misaligned, realign_in_loop;
923 size_t chunk, allocated=0;
925 /* ctx->num is maintained in byte-oriented modes,
926 such as CFB and OFB... */
927 if ((chunk = ctx->num)) { /* borrow chunk variable */
928 unsigned char *ivp=ctx->iv;
930 switch (EVP_CIPHER_CTX_mode(ctx)) {
931 case EVP_CIPH_CFB_MODE:
932 if (chunk >= AES_BLOCK_SIZE)
933 return 0; /* bogus value */
936 while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
937 ivp[chunk] = *(out_arg++) = *(in_arg++) ^ ivp[chunk];
940 else while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
941 unsigned char c = *(in_arg++);
942 *(out_arg++) = c ^ ivp[chunk];
943 ivp[chunk++] = c, nbytes--;
946 ctx->num = chunk%AES_BLOCK_SIZE;
948 case EVP_CIPH_OFB_MODE:
949 if (chunk >= AES_BLOCK_SIZE)
950 return 0; /* bogus value */
952 while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
953 *(out_arg++) = *(in_arg++) ^ ivp[chunk];
957 ctx->num = chunk%AES_BLOCK_SIZE;
965 if (nbytes % AES_BLOCK_SIZE)
966 return 0; /* are we expected to do tail processing? */
968 /* nbytes is always multiple of AES_BLOCK_SIZE in ECB and CBC
969 modes and arbitrary value in byte-oriented modes, such as
973 /* VIA promises CPUs that won't require alignment in the future.
974 For now padlock_aes_align_required is initialized to 1 and
975 the condition is never met... */
976 /* C7 core is capable to manage unaligned input in non-ECB[!]
977 mode, but performance penalties appear to be approximately
978 same as for software alignment below or ~3x. They promise to
979 improve it in the future, but for now we can just as well
980 pretend that it can only handle aligned input... */
981 if (!padlock_aes_align_required && (nbytes%AES_BLOCK_SIZE)==0)
982 return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);
984 inp_misaligned = (((size_t)in_arg) & 0x0F);
985 out_misaligned = (((size_t)out_arg) & 0x0F);
987 /* Note that even if output is aligned and input not,
988 * I still prefer to loop instead of copy the whole
989 * input and then encrypt in one stroke. This is done
990 * in order to improve L1 cache utilization... */
991 realign_in_loop = out_misaligned|inp_misaligned;
993 if (!realign_in_loop && (nbytes%AES_BLOCK_SIZE)==0)
994 return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);
996 /* this takes one "if" out of the loops */
998 chunk %= PADLOCK_CHUNK;
999 if (chunk==0) chunk = PADLOCK_CHUNK;
1001 if (out_misaligned) {
1002 /* optmize for small input */
1003 allocated = (chunk<nbytes?PADLOCK_CHUNK:nbytes);
1004 out = alloca(0x10 + allocated);
1005 out = NEAREST_ALIGNED(out);
1010 cdata = ALIGNED_CIPHER_DATA(ctx);
1011 padlock_verify_context(cdata);
1013 switch (EVP_CIPHER_CTX_mode(ctx)) {
1014 case EVP_CIPH_ECB_MODE:
1017 inp = padlock_memcpy(out, in_arg, chunk);
1022 padlock_xcrypt_ecb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1025 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1027 out = out_arg+=chunk;
1030 chunk = PADLOCK_CHUNK;
1034 case EVP_CIPH_CBC_MODE:
1035 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1038 if (iv != cdata->iv)
1039 memcpy(cdata->iv, iv, AES_BLOCK_SIZE);
1040 chunk = PADLOCK_CHUNK;
1041 cbc_shortcut: /* optimize for small input */
1043 inp = padlock_memcpy(out, in_arg, chunk);
1048 iv = padlock_xcrypt_cbc(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1051 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1053 out = out_arg+=chunk;
1055 } while (nbytes -= chunk);
1056 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
1059 case EVP_CIPH_CFB_MODE:
1060 memcpy (iv = cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1061 chunk &= ~(AES_BLOCK_SIZE-1);
1062 if (chunk) goto cfb_shortcut;
1063 else goto cfb_skiploop;
1065 if (iv != cdata->iv)
1066 memcpy(cdata->iv, iv, AES_BLOCK_SIZE);
1067 chunk = PADLOCK_CHUNK;
1068 cfb_shortcut: /* optimize for small input */
1070 inp = padlock_memcpy(out, in_arg, chunk);
1075 iv = padlock_xcrypt_cfb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1078 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1080 out = out_arg+=chunk;
1083 } while (nbytes >= AES_BLOCK_SIZE);
1087 unsigned char *ivp = cdata->iv;
1090 memcpy(ivp, iv, AES_BLOCK_SIZE);
1094 if (cdata->cword.b.encdec) {
1095 cdata->cword.b.encdec=0;
1096 padlock_reload_key();
1097 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1098 cdata->cword.b.encdec=1;
1099 padlock_reload_key();
1101 unsigned char c = *(in_arg++);
1102 *(out_arg++) = c ^ *ivp;
1103 *(ivp++) = c, nbytes--;
1106 else { padlock_reload_key();
1107 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1108 padlock_reload_key();
1110 *ivp = *(out_arg++) = *(in_arg++) ^ *ivp;
1116 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
1119 case EVP_CIPH_OFB_MODE:
1120 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1121 chunk &= ~(AES_BLOCK_SIZE-1);
1124 inp = padlock_memcpy(out, in_arg, chunk);
1129 padlock_xcrypt_ofb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1132 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1134 out = out_arg+=chunk;
1137 chunk = PADLOCK_CHUNK;
1138 } while (nbytes >= AES_BLOCK_SIZE);
1141 unsigned char *ivp = cdata->iv;
1144 padlock_reload_key(); /* empirically found */
1145 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1146 padlock_reload_key(); /* empirically found */
1148 *(out_arg++) = *(in_arg++) ^ *ivp;
1153 memcpy(ctx->iv, cdata->iv, AES_BLOCK_SIZE);
1160 /* Clean the realign buffer if it was used */
1161 if (out_misaligned) {
1162 volatile unsigned long *p=(void *)out;
1163 size_t n = allocated/sizeof(*p);
1167 memset(cdata->iv, 0, AES_BLOCK_SIZE);
1172 #endif /* OPENSSL_NO_AES */
1174 /* ===== Random Number Generator ===== */
1176 * This code is not engaged. The reason is that it does not comply
1177 * with recommendations for VIA RNG usage for secure applications
1178 * (posted at http://www.via.com.tw/en/viac3/c3.jsp) nor does it
1179 * provide meaningful error control...
1181 /* Wrapper that provides an interface between the API and
1182 the raw PadLock RNG */
1184 padlock_rand_bytes(unsigned char *output, int count)
1186 unsigned int eax, buf;
1188 while (count >= 8) {
1189 eax = padlock_xstore(output, 0);
1190 if (!(eax&(1<<6))) return 0; /* RNG disabled */
1191 /* this ---vv--- covers DC bias, Raw Bits and String Filter */
1192 if (eax&(0x1F<<10)) return 0;
1193 if ((eax&0x1F)==0) continue; /* no data, retry... */
1194 if ((eax&0x1F)!=8) return 0; /* fatal failure... */
1199 eax = padlock_xstore(&buf, 3);
1200 if (!(eax&(1<<6))) return 0; /* RNG disabled */
1201 /* this ---vv--- covers DC bias, Raw Bits and String Filter */
1202 if (eax&(0x1F<<10)) return 0;
1203 if ((eax&0x1F)==0) continue; /* no data, retry... */
1204 if ((eax&0x1F)!=1) return 0; /* fatal failure... */
1205 *output++ = (unsigned char)buf;
1208 *(volatile unsigned int *)&buf=0;
1213 /* Dummy but necessary function */
1215 padlock_rand_status(void)
1220 /* Prepare structure for registration */
1221 static RAND_METHOD padlock_rand = {
1223 padlock_rand_bytes, /* bytes */
1226 padlock_rand_bytes, /* pseudorand */
1227 padlock_rand_status, /* rand status */
1230 #else /* !COMPILE_HW_PADLOCK */
1231 #ifndef OPENSSL_NO_DYNAMIC_ENGINE
1233 int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns);
1235 int bind_engine(ENGINE *e, const char *id, const dynamic_fns *fns) { return 0; }
1236 IMPLEMENT_DYNAMIC_CHECK_FN()
1238 #endif /* COMPILE_HW_PADLOCK */
1240 #endif /* !OPENSSL_NO_HW_PADLOCK */
1241 #endif /* !OPENSSL_NO_HW */