2 * Support for VIA PadLock Advanced Cryptography Engine (ACE)
3 * Written by Michal Ludvig <michal@logix.cz>
4 * http://www.logix.cz/michal
6 * Big thanks to Andy Polyakov for a help with optimization,
7 * assembler fixes, port to MS Windows and a lot of other
8 * valuable work on this engine!
11 /* ====================================================================
12 * Copyright (c) 1999-2001 The OpenSSL Project. All rights reserved.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in
23 * the documentation and/or other materials provided with the
26 * 3. All advertising materials mentioning features or use of this
27 * software must display the following acknowledgment:
28 * "This product includes software developed by the OpenSSL Project
29 * for use in the OpenSSL Toolkit. (http://www.OpenSSL.org/)"
31 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
32 * endorse or promote products derived from this software without
33 * prior written permission. For written permission, please contact
34 * licensing@OpenSSL.org.
36 * 5. Products derived from this software may not be called "OpenSSL"
37 * nor may "OpenSSL" appear in their names without prior written
38 * permission of the OpenSSL Project.
40 * 6. Redistributions of any form whatsoever must retain the following
42 * "This product includes software developed by the OpenSSL Project
43 * for use in the OpenSSL Toolkit (http://www.OpenSSL.org/)"
45 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
46 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
48 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
49 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
51 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
52 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
54 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
55 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
56 * OF THE POSSIBILITY OF SUCH DAMAGE.
57 * ====================================================================
59 * This product includes cryptographic software written by Eric Young
60 * (eay@cryptsoft.com). This product includes software written by Tim
61 * Hudson (tjh@cryptsoft.com).
69 #include <openssl/opensslconf.h>
70 #include <openssl/crypto.h>
71 #include <openssl/dso.h>
72 #include <openssl/engine.h>
73 #include <openssl/evp.h>
74 #ifndef OPENSSL_NO_AES
75 #include <openssl/aes.h>
77 #include <openssl/rand.h>
78 #include <openssl/err.h>
81 #ifndef OPENSSL_NO_HW_PADLOCK
83 /* Attempt to have a single source for both 0.9.7 and 0.9.8 :-) */
84 #if (OPENSSL_VERSION_NUMBER >= 0x00908000L)
85 # ifndef OPENSSL_NO_DYNAMIC_ENGINE
86 # define DYNAMIC_ENGINE
88 #elif (OPENSSL_VERSION_NUMBER >= 0x00907000L)
89 # ifdef ENGINE_DYNAMIC_SUPPORT
90 # define DYNAMIC_ENGINE
93 # error "Only OpenSSL >= 0.9.7 is supported"
96 /* VIA PadLock AES is available *ONLY* on some x86 CPUs.
97 Not only that it doesn't exist elsewhere, but it
98 even can't be compiled on other platforms!
100 In addition, because of the heavy use of inline assembler,
101 compiler choice is limited to GCC and Microsoft C. */
102 #undef COMPILE_HW_PADLOCK
103 #if !defined(I386_ONLY) && !defined(OPENSSL_NO_INLINE_ASM)
104 # if (defined(__GNUC__) && (defined(__i386__) || defined(__i386))) || \
105 (defined(_MSC_VER) && defined(_M_IX86))
106 # define COMPILE_HW_PADLOCK
107 static ENGINE *ENGINE_padlock (void);
111 void ENGINE_load_padlock (void)
113 /* On non-x86 CPUs it just returns. */
114 #ifdef COMPILE_HW_PADLOCK
115 ENGINE *toadd = ENGINE_padlock ();
123 #ifdef COMPILE_HW_PADLOCK
124 /* We do these includes here to avoid header problems on platforms that
125 do not have the VIA padlock anyway... */
129 # define alloca _alloca
135 /* Function for ENGINE detection and control */
136 static int padlock_available(void);
137 static int padlock_init(ENGINE *e);
140 static RAND_METHOD padlock_rand;
143 #ifndef OPENSSL_NO_AES
144 static int padlock_ciphers(ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid);
148 static const char *padlock_id = "padlock";
149 static char padlock_name[100];
151 /* Available features */
152 static int padlock_use_ace = 0; /* Advanced Cryptography Engine */
153 static int padlock_use_rng = 0; /* Random Number Generator */
154 #ifndef OPENSSL_NO_AES
155 static int padlock_aes_align_required = 1;
158 /* ===== Engine "management" functions ===== */
160 /* Prepare the ENGINE structure for registration */
162 padlock_bind_helper(ENGINE *e)
164 /* Check available features */
167 #if 1 /* disable RNG for now, see commentary in vicinity of RNG code */
171 /* Generate a nice engine name with available features */
172 BIO_snprintf(padlock_name, sizeof(padlock_name),
173 "VIA PadLock (%s, %s)",
174 padlock_use_rng ? "RNG" : "no-RNG",
175 padlock_use_ace ? "ACE" : "no-ACE");
177 /* Register everything or return with an error */
178 if (!ENGINE_set_id(e, padlock_id) ||
179 !ENGINE_set_name(e, padlock_name) ||
181 !ENGINE_set_init_function(e, padlock_init) ||
182 #ifndef OPENSSL_NO_AES
183 (padlock_use_ace && !ENGINE_set_ciphers (e, padlock_ciphers)) ||
185 (padlock_use_rng && !ENGINE_set_RAND (e, &padlock_rand))) {
189 /* Everything looks good */
197 ENGINE *eng = ENGINE_new();
203 if (!padlock_bind_helper(eng)) {
211 /* Check availability of the engine */
213 padlock_init(ENGINE *e)
215 return (padlock_use_rng || padlock_use_ace);
218 /* This stuff is needed if this ENGINE is being compiled into a self-contained
221 #ifdef DYNAMIC_ENGINE
223 padlock_bind_fn(ENGINE *e, const char *id)
225 if (id && (strcmp(id, padlock_id) != 0)) {
229 if (!padlock_bind_helper(e)) {
236 IMPLEMENT_DYNAMIC_CHECK_FN ();
237 IMPLEMENT_DYNAMIC_BIND_FN (padlock_bind_fn);
238 #endif /* DYNAMIC_ENGINE */
240 /* ===== Here comes the "real" engine ===== */
242 #ifndef OPENSSL_NO_AES
243 /* Some AES-related constants */
244 #define AES_BLOCK_SIZE 16
245 #define AES_KEY_SIZE_128 16
246 #define AES_KEY_SIZE_192 24
247 #define AES_KEY_SIZE_256 32
249 /* Here we store the status information relevant to the
252 * Inline assembler in PADLOCK_XCRYPT_ASM()
253 * depends on the order of items in this structure.
254 * Don't blindly modify, reorder, etc!
256 struct padlock_cipher_data
258 unsigned char iv[AES_BLOCK_SIZE]; /* Initialization vector */
259 union { unsigned int pad[4];
262 int dgst:1; /* n/a in C3 */
263 int align:1; /* n/a in C3 */
264 int ciphr:1; /* n/a in C3 */
265 unsigned int keygen:1;
267 unsigned int encdec:1;
270 } cword; /* Control word */
271 AES_KEY ks; /* Encryption key */
275 * Essentially this variable belongs in thread local storage.
276 * Having this variable global on the other hand can only cause
277 * few bogus key reloads [if any at all on single-CPU system],
278 * so we accept the penatly...
280 static volatile struct padlock_cipher_data *padlock_saved_context;
284 * =======================================================
285 * Inline assembler section(s).
286 * =======================================================
287 * Order of arguments is chosen to facilitate Windows port
288 * using __fastcall calling convention. If you wish to add
289 * more routines, keep in mind that first __fastcall
290 * argument is passed in %ecx and second - in %edx.
291 * =======================================================
293 #if defined(__GNUC__) && __GNUC__>=2
295 * As for excessive "push %ebx"/"pop %ebx" found all over.
296 * When generating position-independent code GCC won't let
297 * us use "b" in assembler templates nor even respect "ebx"
298 * in "clobber description." Therefore the trouble...
301 /* Helper function - check if a CPUID instruction
302 is available on this CPU */
304 padlock_insn_cpuid_available(void)
308 /* We're checking if the bit #21 of EFLAGS
309 can be toggled. If yes = CPUID is available. */
313 "xorl $0x200000, %%eax\n"
314 "movl %%eax, %%ecx\n"
315 "andl $0x200000, %%ecx\n"
320 "andl $0x200000, %%eax\n"
321 "xorl %%eax, %%ecx\n"
323 : "=r" (result) : : "eax", "ecx");
325 return (result == 0);
328 /* Load supported features of the CPU to see if
329 the PadLock is available. */
331 padlock_available(void)
333 char vendor_string[16];
334 unsigned int eax, edx;
336 /* First check if the CPUID instruction is available at all... */
337 if (! padlock_insn_cpuid_available())
340 /* Are we running on the Centaur (VIA) CPU? */
342 vendor_string[12] = 0;
346 "movl %%ebx,(%%edi)\n"
347 "movl %%edx,4(%%edi)\n"
348 "movl %%ecx,8(%%edi)\n"
350 : "+a"(eax) : "D"(vendor_string) : "ecx", "edx");
351 if (strcmp(vendor_string, "CentaurHauls") != 0)
354 /* Check for Centaur Extended Feature Flags presence */
356 asm volatile ("pushl %%ebx; cpuid; popl %%ebx"
357 : "+a"(eax) : : "ecx", "edx");
358 if (eax < 0xC0000001)
361 /* Read the Centaur Extended Feature Flags */
363 asm volatile ("pushl %%ebx; cpuid; popl %%ebx"
364 : "+a"(eax), "=d"(edx) : : "ecx");
366 /* Fill up some flags */
367 padlock_use_ace = ((edx & (0x3<<6)) == (0x3<<6));
368 padlock_use_rng = ((edx & (0x3<<2)) == (0x3<<2));
370 return padlock_use_ace + padlock_use_rng;
373 #ifndef OPENSSL_NO_AES
374 /* Our own htonl()/ntohl() */
376 padlock_bswapl(AES_KEY *ks)
378 size_t i = sizeof(ks->rd_key)/sizeof(ks->rd_key[0]);
379 unsigned int *key = ks->rd_key;
382 asm volatile ("bswapl %0" : "+r"(*key));
388 /* Force key reload from memory to the CPU microcode.
389 Loading EFLAGS from the stack clears EFLAGS[30]
390 which does the trick. */
392 padlock_reload_key(void)
394 asm volatile ("pushfl; popfl");
397 #ifndef OPENSSL_NO_AES
399 * This is heuristic key context tracing. At first one
400 * believes that one should use atomic swap instructions,
401 * but it's not actually necessary. Point is that if
402 * padlock_saved_context was changed by another thread
403 * after we've read it and before we compare it with cdata,
404 * our key *shall* be reloaded upon thread context switch
405 * and we are therefore set in either case...
408 padlock_verify_context(struct padlock_cipher_data *cdata)
420 :"+m"(padlock_saved_context)
421 : "r"(padlock_saved_context), "r"(cdata) : "cc");
424 /* Template for padlock_xcrypt_* modes */
426 * The offsets used with 'leal' instructions
427 * describe items of the 'padlock_cipher_data'
430 #define PADLOCK_XCRYPT_ASM(name,rep_xcrypt) \
431 static inline void *name(size_t cnt, \
432 struct padlock_cipher_data *cdata, \
433 void *out, const void *inp) \
435 asm volatile ( "pushl %%ebx\n" \
436 " leal 16(%0),%%edx\n" \
437 " leal 32(%0),%%ebx\n" \
440 : "=a"(iv), "=c"(cnt), "=D"(out), "=S"(inp) \
441 : "0"(cdata), "1"(cnt), "2"(out), "3"(inp), "m"(*cdata) \
442 : "edx", "cc", "memory"); \
446 /* Generate all functions with appropriate opcodes */
447 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb, ".byte 0xf3,0x0f,0xa7,0xc8") /* rep xcryptecb */
448 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc, ".byte 0xf3,0x0f,0xa7,0xd0") /* rep xcryptcbc */
449 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb, ".byte 0xf3,0x0f,0xa7,0xe0") /* rep xcryptcfb */
450 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb, ".byte 0xf3,0x0f,0xa7,0xe8") /* rep xcryptofb */
453 /* The RNG call itself */
454 static inline unsigned int
455 padlock_xstore(void *addr, unsigned int edx_in)
457 unsigned int eax_out;
459 asm volatile (".byte 0x0f,0xa7,0xc0" /* xstore */
460 : "=a"(eax_out),"=m"(*(unsigned *)addr)
461 : "D"(addr), "d" (edx_in)
467 /* Why not inline 'rep movsd'? I failed to find information on what
468 * value in Direction Flag one can expect and consequently have to
469 * apply "better-safe-than-sorry" approach and assume "undefined."
470 * I could explicitly clear it and restore the original value upon
471 * return from padlock_aes_cipher, but it's presumably too much
472 * trouble for too little gain...
474 * In case you wonder 'rep xcrypt*' instructions above are *not*
475 * affected by the Direction Flag and pointers advance toward
476 * larger addresses unconditionally.
478 static inline unsigned char *
479 padlock_memcpy(void *dst,const void *src,size_t n)
485 do { *d++ = *s++; } while (--n);
490 #elif defined(_MSC_VER)
492 * Unlike GCC these are real functions. In order to minimize impact
493 * on performance we adhere to __fastcall calling convention in
494 * order to get two first arguments passed through %ecx and %edx.
495 * Which kind of suits very well, as instructions in question use
496 * both %ecx and %edx as input:-)
498 #define REP_XCRYPT(code) \
500 _asm _emit 0x0f _asm _emit 0xa7 \
504 * The offsets used with 'lea' instructions
505 * describe items of the 'padlock_cipher_data'
508 #define PADLOCK_XCRYPT_ASM(name,code) \
509 static void * __fastcall \
510 name (size_t cnt, void *cdata, \
511 void *outp, const void *inp) \
513 _asm lea edx,[eax+16] \
514 _asm lea ebx,[eax+32] \
520 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ecb,0xc8)
521 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cbc,0xd0)
522 PADLOCK_XCRYPT_ASM(padlock_xcrypt_cfb,0xe0)
523 PADLOCK_XCRYPT_ASM(padlock_xcrypt_ofb,0xe8)
525 static int __fastcall
526 padlock_xstore(void *outp,unsigned int code)
528 _asm _emit 0x0f _asm _emit 0xa7 _asm _emit 0xc0
531 static void __fastcall
532 padlock_reload_key(void)
533 { _asm pushfd _asm popfd }
535 static void __fastcall
536 padlock_verify_context(void *cdata)
541 cmp ecx,padlock_saved_context
546 mov padlock_saved_context,ecx
551 padlock_available(void)
586 mov padlock_use_ace,1
592 mov padlock_use_rng,1
599 static void __fastcall
600 padlock_bswapl(void *key)
615 /* MS actually specifies status of Direction Flag and compiler even
616 * manages to compile following as 'rep movsd' all by itself...
618 #define padlock_memcpy(o,i,n) ((unsigned char *)memcpy((o),(i),(n)&~3U))
621 /* ===== AES encryption/decryption ===== */
622 #ifndef OPENSSL_NO_AES
624 #if defined(NID_aes_128_cfb128) && ! defined (NID_aes_128_cfb)
625 #define NID_aes_128_cfb NID_aes_128_cfb128
628 #if defined(NID_aes_128_ofb128) && ! defined (NID_aes_128_ofb)
629 #define NID_aes_128_ofb NID_aes_128_ofb128
632 #if defined(NID_aes_192_cfb128) && ! defined (NID_aes_192_cfb)
633 #define NID_aes_192_cfb NID_aes_192_cfb128
636 #if defined(NID_aes_192_ofb128) && ! defined (NID_aes_192_ofb)
637 #define NID_aes_192_ofb NID_aes_192_ofb128
640 #if defined(NID_aes_256_cfb128) && ! defined (NID_aes_256_cfb)
641 #define NID_aes_256_cfb NID_aes_256_cfb128
644 #if defined(NID_aes_256_ofb128) && ! defined (NID_aes_256_ofb)
645 #define NID_aes_256_ofb NID_aes_256_ofb128
648 /* List of supported ciphers. */
649 static int padlock_cipher_nids[] = {
665 static int padlock_cipher_nids_num = (sizeof(padlock_cipher_nids)/
666 sizeof(padlock_cipher_nids[0]));
668 /* Function prototypes ... */
669 static int padlock_aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
670 const unsigned char *iv, int enc);
671 static int padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
672 const unsigned char *in, size_t nbytes);
674 #define NEAREST_ALIGNED(ptr) ( (unsigned char *)(ptr) + \
675 ( (0x10 - ((size_t)(ptr) & 0x0F)) & 0x0F ) )
676 #define ALIGNED_CIPHER_DATA(ctx) ((struct padlock_cipher_data *)\
677 NEAREST_ALIGNED(ctx->cipher_data))
679 #define EVP_CIPHER_block_size_ECB AES_BLOCK_SIZE
680 #define EVP_CIPHER_block_size_CBC AES_BLOCK_SIZE
681 #define EVP_CIPHER_block_size_OFB 1
682 #define EVP_CIPHER_block_size_CFB 1
684 /* Declaring so many ciphers by hand would be a pain.
685 Instead introduce a bit of preprocessor magic :-) */
686 #define DECLARE_AES_EVP(ksize,lmode,umode) \
687 static const EVP_CIPHER padlock_aes_##ksize##_##lmode = { \
688 NID_aes_##ksize##_##lmode, \
689 EVP_CIPHER_block_size_##umode, \
690 AES_KEY_SIZE_##ksize, \
692 0 | EVP_CIPH_##umode##_MODE, \
693 padlock_aes_init_key, \
694 padlock_aes_cipher, \
696 sizeof(struct padlock_cipher_data) + 16, \
697 EVP_CIPHER_set_asn1_iv, \
698 EVP_CIPHER_get_asn1_iv, \
703 DECLARE_AES_EVP(128,ecb,ECB);
704 DECLARE_AES_EVP(128,cbc,CBC);
705 DECLARE_AES_EVP(128,cfb,CFB);
706 DECLARE_AES_EVP(128,ofb,OFB);
708 DECLARE_AES_EVP(192,ecb,ECB);
709 DECLARE_AES_EVP(192,cbc,CBC);
710 DECLARE_AES_EVP(192,cfb,CFB);
711 DECLARE_AES_EVP(192,ofb,OFB);
713 DECLARE_AES_EVP(256,ecb,ECB);
714 DECLARE_AES_EVP(256,cbc,CBC);
715 DECLARE_AES_EVP(256,cfb,CFB);
716 DECLARE_AES_EVP(256,ofb,OFB);
719 padlock_ciphers (ENGINE *e, const EVP_CIPHER **cipher, const int **nids, int nid)
721 /* No specific cipher => return a list of supported nids ... */
723 *nids = padlock_cipher_nids;
724 return padlock_cipher_nids_num;
727 /* ... or the requested "cipher" otherwise */
729 case NID_aes_128_ecb:
730 *cipher = &padlock_aes_128_ecb;
732 case NID_aes_128_cbc:
733 *cipher = &padlock_aes_128_cbc;
735 case NID_aes_128_cfb:
736 *cipher = &padlock_aes_128_cfb;
738 case NID_aes_128_ofb:
739 *cipher = &padlock_aes_128_ofb;
742 case NID_aes_192_ecb:
743 *cipher = &padlock_aes_192_ecb;
745 case NID_aes_192_cbc:
746 *cipher = &padlock_aes_192_cbc;
748 case NID_aes_192_cfb:
749 *cipher = &padlock_aes_192_cfb;
751 case NID_aes_192_ofb:
752 *cipher = &padlock_aes_192_ofb;
755 case NID_aes_256_ecb:
756 *cipher = &padlock_aes_256_ecb;
758 case NID_aes_256_cbc:
759 *cipher = &padlock_aes_256_cbc;
761 case NID_aes_256_cfb:
762 *cipher = &padlock_aes_256_cfb;
764 case NID_aes_256_ofb:
765 *cipher = &padlock_aes_256_ofb;
769 /* Sorry, we don't support this NID */
777 /* Prepare the encryption key for PadLock usage */
779 padlock_aes_init_key (EVP_CIPHER_CTX *ctx, const unsigned char *key,
780 const unsigned char *iv, int enc)
782 struct padlock_cipher_data *cdata;
783 int key_len = EVP_CIPHER_CTX_key_length(ctx) * 8;
785 if (key==NULL) return 0; /* ERROR */
787 cdata = ALIGNED_CIPHER_DATA(ctx);
788 memset(cdata, 0, sizeof(struct padlock_cipher_data));
790 /* Prepare Control word. */
791 if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE)
792 cdata->cword.b.encdec = 0;
794 cdata->cword.b.encdec = (ctx->encrypt == 0);
795 cdata->cword.b.rounds = 10 + (key_len - 128) / 32;
796 cdata->cword.b.ksize = (key_len - 128) / 64;
800 /* PadLock can generate an extended key for
801 AES128 in hardware */
802 memcpy(cdata->ks.rd_key, key, AES_KEY_SIZE_128);
803 cdata->cword.b.keygen = 0;
808 /* Generate an extended AES key in software.
809 Needed for AES192/AES256 */
810 /* Well, the above applies to Stepping 8 CPUs
811 and is listed as hardware errata. They most
812 likely will fix it at some point and then
813 a check for stepping would be due here. */
814 if (EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_CFB_MODE ||
815 EVP_CIPHER_CTX_mode(ctx) == EVP_CIPH_OFB_MODE ||
817 AES_set_encrypt_key(key, key_len, &cdata->ks);
819 AES_set_decrypt_key(key, key_len, &cdata->ks);
821 /* OpenSSL C functions use byte-swapped extended key. */
822 padlock_bswapl(&cdata->ks);
824 cdata->cword.b.keygen = 1;
833 * This is done to cover for cases when user reuses the
834 * context for new key. The catch is that if we don't do
835 * this, padlock_eas_cipher might proceed with old key...
837 padlock_reload_key ();
843 * Simplified version of padlock_aes_cipher() used when
844 * 1) both input and output buffers are at aligned addresses.
846 * 2) running on a newer CPU that doesn't require aligned buffers.
849 padlock_aes_cipher_omnivorous(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,
850 const unsigned char *in_arg, size_t nbytes)
852 struct padlock_cipher_data *cdata;
855 cdata = ALIGNED_CIPHER_DATA(ctx);
856 padlock_verify_context(cdata);
858 switch (EVP_CIPHER_CTX_mode(ctx)) {
859 case EVP_CIPH_ECB_MODE:
860 padlock_xcrypt_ecb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
863 case EVP_CIPH_CBC_MODE:
864 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
865 iv = padlock_xcrypt_cbc(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
866 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
869 case EVP_CIPH_CFB_MODE:
870 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
871 iv = padlock_xcrypt_cfb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
872 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
875 case EVP_CIPH_OFB_MODE:
876 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
877 padlock_xcrypt_ofb(nbytes/AES_BLOCK_SIZE, cdata, out_arg, in_arg);
878 memcpy(ctx->iv, cdata->iv, AES_BLOCK_SIZE);
885 memset(cdata->iv, 0, AES_BLOCK_SIZE);
890 #ifndef PADLOCK_CHUNK
891 # define PADLOCK_CHUNK 512 /* Must be a power of 2 larger than 16 */
893 #if PADLOCK_CHUNK<16 || PADLOCK_CHUNK&(PADLOCK_CHUNK-1)
894 # error "insane PADLOCK_CHUNK..."
897 /* Re-align the arguments to 16-Bytes boundaries and run the
898 encryption function itself. This function is not AES-specific. */
900 padlock_aes_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out_arg,
901 const unsigned char *in_arg, size_t nbytes)
903 struct padlock_cipher_data *cdata;
907 int inp_misaligned, out_misaligned, realign_in_loop;
908 size_t chunk, allocated=0;
910 /* ctx->num is maintained in byte-oriented modes,
911 such as CFB and OFB... */
912 if ((chunk = ctx->num)) { /* borrow chunk variable */
913 unsigned char *ivp=ctx->iv;
915 switch (EVP_CIPHER_CTX_mode(ctx)) {
916 case EVP_CIPH_CFB_MODE:
917 if (chunk >= AES_BLOCK_SIZE)
918 return 0; /* bogus value */
921 while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
922 ivp[chunk] = *(out_arg++) = *(in_arg++) ^ ivp[chunk];
925 else while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
926 unsigned char c = *(in_arg++);
927 *(out_arg++) = c ^ ivp[chunk];
928 ivp[chunk++] = c, nbytes--;
931 ctx->num = chunk%AES_BLOCK_SIZE;
933 case EVP_CIPH_OFB_MODE:
934 if (chunk >= AES_BLOCK_SIZE)
935 return 0; /* bogus value */
937 while (chunk<AES_BLOCK_SIZE && nbytes!=0) {
938 *(out_arg++) = *(in_arg++) ^ ivp[chunk];
942 ctx->num = chunk%AES_BLOCK_SIZE;
950 if (nbytes % AES_BLOCK_SIZE)
951 return 0; /* are we expected to do tail processing? */
953 /* nbytes is always multiple of AES_BLOCK_SIZE in ECB and CBC
954 modes and arbitrary value in byte-oriented modes, such as
958 /* VIA promises CPUs that won't require alignment in the future.
959 For now padlock_aes_align_required is initialized to 1 and
960 the condition is never met... */
961 /* C7 core is capable to manage unaligned input in non-ECB[!]
962 mode, but performance penalties appear to be approximately
963 same as for software alignment below or ~3x. They promise to
964 improve it in the future, but for now we can just as well
965 pretend that it can only handle aligned input... */
966 if (!padlock_aes_align_required && (nbytes%AES_BLOCK_SIZE)==0)
967 return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);
969 inp_misaligned = (((size_t)in_arg) & 0x0F);
970 out_misaligned = (((size_t)out_arg) & 0x0F);
972 /* Note that even if output is aligned and input not,
973 * I still prefer to loop instead of copy the whole
974 * input and then encrypt in one stroke. This is done
975 * in order to improve L1 cache utilization... */
976 realign_in_loop = out_misaligned|inp_misaligned;
978 if (!realign_in_loop && (nbytes%AES_BLOCK_SIZE)==0)
979 return padlock_aes_cipher_omnivorous(ctx, out_arg, in_arg, nbytes);
981 /* this takes one "if" out of the loops */
983 chunk %= PADLOCK_CHUNK;
984 if (chunk==0) chunk = PADLOCK_CHUNK;
986 if (out_misaligned) {
987 /* optmize for small input */
988 allocated = (chunk<nbytes?PADLOCK_CHUNK:nbytes);
989 out = alloca(0x10 + allocated);
990 out = NEAREST_ALIGNED(out);
995 cdata = ALIGNED_CIPHER_DATA(ctx);
996 padlock_verify_context(cdata);
998 switch (EVP_CIPHER_CTX_mode(ctx)) {
999 case EVP_CIPH_ECB_MODE:
1002 inp = padlock_memcpy(out, in_arg, chunk);
1007 padlock_xcrypt_ecb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1010 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1012 out = out_arg+=chunk;
1015 chunk = PADLOCK_CHUNK;
1019 case EVP_CIPH_CBC_MODE:
1020 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1023 if (iv != cdata->iv)
1024 memcpy(cdata->iv, iv, AES_BLOCK_SIZE);
1025 chunk = PADLOCK_CHUNK;
1026 cbc_shortcut: /* optimize for small input */
1028 inp = padlock_memcpy(out, in_arg, chunk);
1033 iv = padlock_xcrypt_cbc(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1036 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1038 out = out_arg+=chunk;
1040 } while (nbytes -= chunk);
1041 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
1044 case EVP_CIPH_CFB_MODE:
1045 memcpy (iv = cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1046 chunk &= ~(AES_BLOCK_SIZE-1);
1047 if (chunk) goto cfb_shortcut;
1048 else goto cfb_skiploop;
1050 if (iv != cdata->iv)
1051 memcpy(cdata->iv, iv, AES_BLOCK_SIZE);
1052 chunk = PADLOCK_CHUNK;
1053 cfb_shortcut: /* optimize for small input */
1055 inp = padlock_memcpy(out, in_arg, chunk);
1060 iv = padlock_xcrypt_cfb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1063 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1065 out = out_arg+=chunk;
1068 } while (nbytes >= AES_BLOCK_SIZE);
1072 unsigned char *ivp = cdata->iv;
1075 memcpy(ivp, iv, AES_BLOCK_SIZE);
1079 if (cdata->cword.b.encdec) {
1080 cdata->cword.b.encdec=0;
1081 padlock_reload_key();
1082 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1083 cdata->cword.b.encdec=1;
1084 padlock_reload_key();
1086 unsigned char c = *(in_arg++);
1087 *(out_arg++) = c ^ *ivp;
1088 *(ivp++) = c, nbytes--;
1091 else { padlock_reload_key();
1092 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1093 padlock_reload_key();
1095 *ivp = *(out_arg++) = *(in_arg++) ^ *ivp;
1101 memcpy(ctx->iv, iv, AES_BLOCK_SIZE);
1104 case EVP_CIPH_OFB_MODE:
1105 memcpy(cdata->iv, ctx->iv, AES_BLOCK_SIZE);
1106 chunk &= ~(AES_BLOCK_SIZE-1);
1109 inp = padlock_memcpy(out, in_arg, chunk);
1114 padlock_xcrypt_ofb(chunk/AES_BLOCK_SIZE, cdata, out, inp);
1117 out_arg = padlock_memcpy(out_arg, out, chunk) + chunk;
1119 out = out_arg+=chunk;
1122 chunk = PADLOCK_CHUNK;
1123 } while (nbytes >= AES_BLOCK_SIZE);
1126 unsigned char *ivp = cdata->iv;
1129 padlock_reload_key(); /* empirically found */
1130 padlock_xcrypt_ecb(1,cdata,ivp,ivp);
1131 padlock_reload_key(); /* empirically found */
1133 *(out_arg++) = *(in_arg++) ^ *ivp;
1138 memcpy(ctx->iv, cdata->iv, AES_BLOCK_SIZE);
1145 /* Clean the realign buffer if it was used */
1146 if (out_misaligned) {
1147 volatile unsigned long *p=(void *)out;
1148 size_t n = allocated/sizeof(*p);
1152 memset(cdata->iv, 0, AES_BLOCK_SIZE);
1157 #endif /* OPENSSL_NO_AES */
1159 /* ===== Random Number Generator ===== */
1161 * This code is not engaged. The reason is that it does not comply
1162 * with recommendations for VIA RNG usage for secure applications
1163 * (posted at http://www.via.com.tw/en/viac3/c3.jsp) nor does it
1164 * provide meaningful error control...
1166 /* Wrapper that provides an interface between the API and
1167 the raw PadLock RNG */
1169 padlock_rand_bytes(unsigned char *output, int count)
1171 unsigned int eax, buf;
1173 while (count >= 8) {
1174 eax = padlock_xstore(output, 0);
1175 if (!(eax&(1<<6))) return 0; /* RNG disabled */
1176 /* this ---vv--- covers DC bias, Raw Bits and String Filter */
1177 if (eax&(0x1F<<10)) return 0;
1178 if ((eax&0x1F)==0) continue; /* no data, retry... */
1179 if ((eax&0x1F)!=8) return 0; /* fatal failure... */
1184 eax = padlock_xstore(&buf, 3);
1185 if (!(eax&(1<<6))) return 0; /* RNG disabled */
1186 /* this ---vv--- covers DC bias, Raw Bits and String Filter */
1187 if (eax&(0x1F<<10)) return 0;
1188 if ((eax&0x1F)==0) continue; /* no data, retry... */
1189 if ((eax&0x1F)!=1) return 0; /* fatal failure... */
1190 *output++ = (unsigned char)buf;
1193 *(volatile unsigned int *)&buf=0;
1198 /* Dummy but necessary function */
1200 padlock_rand_status(void)
1205 /* Prepare structure for registration */
1206 static RAND_METHOD padlock_rand = {
1208 padlock_rand_bytes, /* bytes */
1211 padlock_rand_bytes, /* pseudorand */
1212 padlock_rand_status, /* rand status */
1215 #endif /* COMPILE_HW_PADLOCK */
1217 #endif /* !OPENSSL_NO_HW_PADLOCK */
1218 #endif /* !OPENSSL_NO_HW */