const u8 *xi = (const u8 *)Xi+15;
size_t rem, n = *xi;
const union { long one; char little; } is_endian = {1};
+ __fips_constseg
static const size_t rem_8bit[256] = {
PACK(0x0000), PACK(0x01C2), PACK(0x0384), PACK(0x0246),
PACK(0x0708), PACK(0x06CA), PACK(0x048C), PACK(0x054E),
}
#ifndef GHASH_ASM
+__fips_constseg
static const size_t rem_4bit[16] = {
PACK(0x0000), PACK(0x1C20), PACK(0x3840), PACK(0x2460),
PACK(0x7080), PACK(0x6CA0), PACK(0x48C0), PACK(0x54E0),
*/
u128 Hshr4[16]; /* Htable shifted right by 4 bits */
u8 Hshl4[16]; /* Htable shifted left by 4 bits */
+ __fips_constseg
static const unsigned short rem_8bit[256] = {
0x0000, 0x01C2, 0x0384, 0x0246, 0x0708, 0x06CA, 0x048C, 0x054E,
0x0E10, 0x0FD2, 0x0D94, 0x0C56, 0x0918, 0x08DA, 0x0A9C, 0x0B5E,
# if __ARM_ARCH__>=7
# define GHASH_ASM_ARM
# define GCM_FUNCREF_4BIT
-extern unsigned int OPENSSL_armcap;
-
void gcm_gmult_neon(u64 Xi[2],const u128 Htable[16]);
void gcm_ghash_neon(u64 Xi[2],const u128 Htable[16],const u8 *inp,size_t len);
# endif
# endif
#endif
+#ifdef GCM_FUNCREF_4BIT
+# undef GCM_MUL
+# define GCM_MUL(ctx,Xi) (*gcm_gmult_p)(ctx->Xi.u,ctx->Htable)
+# ifdef GHASH
+# undef GHASH
+# define GHASH(ctx,in,len) (*gcm_ghash_p)(ctx->Xi.u,ctx->Htable,in,len)
+# endif
+#endif
+
void CRYPTO_gcm128_init(GCM128_CONTEXT *ctx,void *key,block128_f block)
{
const union { long one; char little; } is_endian = {1};
#elif TABLE_BITS==4
# if defined(GHASH_ASM_X86_OR_64)
# if !defined(GHASH_ASM_X86) || defined(OPENSSL_IA32_SSE2)
- if (OPENSSL_ia32cap_P[1]&(1<<1)) {
+ if (OPENSSL_ia32cap_P[0]&(1<<24) && /* check FXSR bit */
+ OPENSSL_ia32cap_P[1]&(1<<1) ) { /* check PCLMULQDQ bit */
gcm_init_clmul(ctx->Htable,ctx->H.u);
ctx->gmult = gcm_gmult_clmul;
ctx->ghash = gcm_ghash_clmul;
# endif
gcm_init_4bit(ctx->Htable,ctx->H.u);
# if defined(GHASH_ASM_X86) /* x86 only */
- if (OPENSSL_ia32cap_P[0]&(1<<23)) {
+# if defined(OPENSSL_IA32_SSE2)
+ if (OPENSSL_ia32cap_P[0]&(1<<25)) { /* check SSE bit */
+# else
+ if (OPENSSL_ia32cap_P[0]&(1<<23)) { /* check MMX bit */
+# endif
ctx->gmult = gcm_gmult_4bit_mmx;
ctx->ghash = gcm_ghash_4bit_mmx;
} else {
ctx->ghash = gcm_ghash_4bit;
# endif
# elif defined(GHASH_ASM_ARM)
- if (OPENSSL_armcap & 1) {
+ if (OPENSSL_armcap_P & ARMV7_NEON) {
ctx->gmult = gcm_gmult_neon;
ctx->ghash = gcm_ghash_neon;
} else {
const union { long one; char little; } is_endian = {1};
unsigned int ctr;
#ifdef GCM_FUNCREF_4BIT
- void (*gcm_gmult_4bit)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
+ void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
#endif
ctx->Yi.u[0] = 0;
unsigned int n;
u64 alen = ctx->len.u[0];
#ifdef GCM_FUNCREF_4BIT
- void (*gcm_gmult_4bit)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
+ void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
# ifdef GHASH
- void (*gcm_ghash_4bit)(u64 Xi[2],const u128 Htable[16],
- const u8 *inp,size_t len) = ctx->ghash;
+ void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+ const u8 *inp,size_t len) = ctx->ghash;
# endif
#endif
const union { long one; char little; } is_endian = {1};
unsigned int n, ctr;
size_t i;
- u64 mlen = ctx->len.u[1];
+ u64 mlen = ctx->len.u[1];
+ block128_f block = ctx->block;
+ void *key = ctx->key;
#ifdef GCM_FUNCREF_4BIT
- void (*gcm_gmult_4bit)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
+ void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
# ifdef GHASH
- void (*gcm_ghash_4bit)(u64 Xi[2],const u128 Htable[16],
- const u8 *inp,size_t len) = ctx->ghash;
+ void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+ const u8 *inp,size_t len) = ctx->ghash;
# endif
#endif
size_t j=GHASH_CHUNK;
while (j) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
size_t j=i;
while (len>=16) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
}
#else
while (len>=16) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
}
#endif
if (len) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
#endif
for (i=0;i<len;++i) {
if (n==0) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
const union { long one; char little; } is_endian = {1};
unsigned int n, ctr;
size_t i;
- u64 mlen = ctx->len.u[1];
+ u64 mlen = ctx->len.u[1];
+ block128_f block = ctx->block;
+ void *key = ctx->key;
#ifdef GCM_FUNCREF_4BIT
- void (*gcm_gmult_4bit)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
+ void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
# ifdef GHASH
- void (*gcm_ghash_4bit)(u64 Xi[2],const u128 Htable[16],
- const u8 *inp,size_t len) = ctx->ghash;
+ void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+ const u8 *inp,size_t len) = ctx->ghash;
# endif
#endif
GHASH(ctx,in,GHASH_CHUNK);
while (j) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
if ((i = (len&(size_t)-16))) {
GHASH(ctx,in,i);
while (len>=16) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
}
#else
while (len>=16) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
}
#endif
if (len) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
for (i=0;i<len;++i) {
u8 c;
if (n==0) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
const union { long one; char little; } is_endian = {1};
unsigned int n, ctr;
size_t i;
- u64 mlen = ctx->len.u[1];
+ u64 mlen = ctx->len.u[1];
+ void *key = ctx->key;
#ifdef GCM_FUNCREF_4BIT
- void (*gcm_gmult_4bit)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
+ void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
# ifdef GHASH
- void (*gcm_ghash_4bit)(u64 Xi[2],const u128 Htable[16],
- const u8 *inp,size_t len) = ctx->ghash;
+ void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+ const u8 *inp,size_t len) = ctx->ghash;
# endif
#endif
}
#if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
while (len>=GHASH_CHUNK) {
- (*stream)(in,out,GHASH_CHUNK/16,ctx->key,ctx->Yi.c);
+ (*stream)(in,out,GHASH_CHUNK/16,key,ctx->Yi.c);
ctr += GHASH_CHUNK/16;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
if ((i = (len&(size_t)-16))) {
size_t j=i/16;
- (*stream)(in,out,j,ctx->key,ctx->Yi.c);
+ (*stream)(in,out,j,key,ctx->Yi.c);
ctr += (unsigned int)j;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
#endif
}
if (len) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*ctx->block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
const union { long one; char little; } is_endian = {1};
unsigned int n, ctr;
size_t i;
- u64 mlen = ctx->len.u[1];
+ u64 mlen = ctx->len.u[1];
+ void *key = ctx->key;
#ifdef GCM_FUNCREF_4BIT
- void (*gcm_gmult_4bit)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
+ void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
# ifdef GHASH
- void (*gcm_ghash_4bit)(u64 Xi[2],const u128 Htable[16],
- const u8 *inp,size_t len) = ctx->ghash;
+ void (*gcm_ghash_p)(u64 Xi[2],const u128 Htable[16],
+ const u8 *inp,size_t len) = ctx->ghash;
# endif
#endif
#if defined(GHASH) && !defined(OPENSSL_SMALL_FOOTPRINT)
while (len>=GHASH_CHUNK) {
GHASH(ctx,in,GHASH_CHUNK);
- (*stream)(in,out,GHASH_CHUNK/16,ctx->key,ctx->Yi.c);
+ (*stream)(in,out,GHASH_CHUNK/16,key,ctx->Yi.c);
ctr += GHASH_CHUNK/16;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
j = i/16;
in -= i;
#endif
- (*stream)(in,out,j,ctx->key,ctx->Yi.c);
+ (*stream)(in,out,j,key,ctx->Yi.c);
ctr += (unsigned int)j;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
len -= i;
}
if (len) {
- (*ctx->block)(ctx->Yi.c,ctx->EKi.c,ctx->key);
+ (*ctx->block)(ctx->Yi.c,ctx->EKi.c,key);
++ctr;
if (is_endian.little)
PUTU32(ctx->Yi.c+12,ctr);
u64 alen = ctx->len.u[0]<<3;
u64 clen = ctx->len.u[1]<<3;
#ifdef GCM_FUNCREF_4BIT
- void (*gcm_gmult_4bit)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
+ void (*gcm_gmult_p)(u64 Xi[2],const u128 Htable[16]) = ctx->gmult;
#endif
if (ctx->mres)