From: Emilia Kasper Date: Sat, 14 Mar 2015 04:10:13 +0000 (-0700) Subject: Fix undefined behaviour in shifts. X-Git-Tag: OpenSSL_1_1_0-pre1~1543 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=8b37e5c14f0eddb10c7f91ef91004622d90ef361;p=oweals%2Fopenssl.git Fix undefined behaviour in shifts. Td4 and Te4 are arrays of u8. A u8 << int promotes the u8 to an int first then shifts. If the mathematical result of a shift (as modelled by lhs * 2^{rhs}) is not representable in an integer, behaviour is undefined. In other words, you can't shift into the sign bit of a signed integer. Fix this by casting to u32 whenever we're shifting left by 24. (For consistency, cast other shifts, too.) Caught by -fsanitize=shift Submitted by Nick Lewycky (Google) Reviewed-by: Andy Polyakov --- diff --git a/crypto/aes/aes_core.c b/crypto/aes/aes_core.c index e8ea36912a..1e4455a3e1 100644 --- a/crypto/aes/aes_core.c +++ b/crypto/aes/aes_core.c @@ -1131,31 +1131,31 @@ void AES_decrypt(const unsigned char *in, unsigned char *out, * map cipher state to byte array block: */ s0 = - (Td4[(t0 >> 24) ] << 24) ^ - (Td4[(t3 >> 16) & 0xff] << 16) ^ - (Td4[(t2 >> 8) & 0xff] << 8) ^ - (Td4[(t1 ) & 0xff]) ^ + ((u32)Td4[(t0 >> 24) ] << 24) ^ + ((u32)Td4[(t3 >> 16) & 0xff] << 16) ^ + ((u32)Td4[(t2 >> 8) & 0xff] << 8) ^ + ((u32)Td4[(t1 ) & 0xff]) ^ rk[0]; PUTU32(out , s0); s1 = - (Td4[(t1 >> 24) ] << 24) ^ - (Td4[(t0 >> 16) & 0xff] << 16) ^ - (Td4[(t3 >> 8) & 0xff] << 8) ^ - (Td4[(t2 ) & 0xff]) ^ + ((u32)Td4[(t1 >> 24) ] << 24) ^ + ((u32)Td4[(t0 >> 16) & 0xff] << 16) ^ + ((u32)Td4[(t3 >> 8) & 0xff] << 8) ^ + ((u32)Td4[(t2 ) & 0xff]) ^ rk[1]; PUTU32(out + 4, s1); s2 = - (Td4[(t2 >> 24) ] << 24) ^ - (Td4[(t1 >> 16) & 0xff] << 16) ^ - (Td4[(t0 >> 8) & 0xff] << 8) ^ - (Td4[(t3 ) & 0xff]) ^ + ((u32)Td4[(t2 >> 24) ] << 24) ^ + ((u32)Td4[(t1 >> 16) & 0xff] << 16) ^ + ((u32)Td4[(t0 >> 8) & 0xff] << 8) ^ + ((u32)Td4[(t3 ) & 0xff]) ^ rk[2]; PUTU32(out + 8, s2); s3 = - (Td4[(t3 >> 24) ] << 24) ^ - (Td4[(t2 >> 16) & 0xff] << 16) ^ - (Td4[(t1 >> 8) & 0xff] << 8) ^ - (Td4[(t0 ) & 0xff]) ^ + ((u32)Td4[(t3 >> 24) ] << 24) ^ + ((u32)Td4[(t2 >> 16) & 0xff] << 16) ^ + ((u32)Td4[(t1 >> 8) & 0xff] << 8) ^ + ((u32)Td4[(t0 ) & 0xff]) ^ rk[3]; PUTU32(out + 12, s3); } @@ -1234,10 +1234,10 @@ int AES_set_encrypt_key(const unsigned char *userKey, const int bits, while (1) { temp = rk[3]; rk[4] = rk[0] ^ - (Te4[(temp >> 16) & 0xff] << 24) ^ - (Te4[(temp >> 8) & 0xff] << 16) ^ - (Te4[(temp ) & 0xff] << 8) ^ - (Te4[(temp >> 24) ]) ^ + ((u32)Te4[(temp >> 16) & 0xff] << 24) ^ + ((u32)Te4[(temp >> 8) & 0xff] << 16) ^ + ((u32)Te4[(temp ) & 0xff] << 8) ^ + ((u32)Te4[(temp >> 24) ]) ^ rcon[i]; rk[5] = rk[1] ^ rk[4]; rk[6] = rk[2] ^ rk[5]; @@ -1254,10 +1254,10 @@ int AES_set_encrypt_key(const unsigned char *userKey, const int bits, while (1) { temp = rk[ 5]; rk[ 6] = rk[ 0] ^ - (Te4[(temp >> 16) & 0xff] << 24) ^ - (Te4[(temp >> 8) & 0xff] << 16) ^ - (Te4[(temp ) & 0xff] << 8) ^ - (Te4[(temp >> 24) ]) ^ + ((u32)Te4[(temp >> 16) & 0xff] << 24) ^ + ((u32)Te4[(temp >> 8) & 0xff] << 16) ^ + ((u32)Te4[(temp ) & 0xff] << 8) ^ + ((u32)Te4[(temp >> 24) ]) ^ rcon[i]; rk[ 7] = rk[ 1] ^ rk[ 6]; rk[ 8] = rk[ 2] ^ rk[ 7]; @@ -1276,10 +1276,10 @@ int AES_set_encrypt_key(const unsigned char *userKey, const int bits, while (1) { temp = rk[ 7]; rk[ 8] = rk[ 0] ^ - (Te4[(temp >> 16) & 0xff] << 24) ^ - (Te4[(temp >> 8) & 0xff] << 16) ^ - (Te4[(temp ) & 0xff] << 8) ^ - (Te4[(temp >> 24) ]) ^ + ((u32)Te4[(temp >> 16) & 0xff] << 24) ^ + ((u32)Te4[(temp >> 8) & 0xff] << 16) ^ + ((u32)Te4[(temp ) & 0xff] << 8) ^ + ((u32)Te4[(temp >> 24) ]) ^ rcon[i]; rk[ 9] = rk[ 1] ^ rk[ 8]; rk[10] = rk[ 2] ^ rk[ 9]; @@ -1289,10 +1289,10 @@ int AES_set_encrypt_key(const unsigned char *userKey, const int bits, } temp = rk[11]; rk[12] = rk[ 4] ^ - (Te4[(temp >> 24) ] << 24) ^ - (Te4[(temp >> 16) & 0xff] << 16) ^ - (Te4[(temp >> 8) & 0xff] << 8) ^ - (Te4[(temp ) & 0xff]); + ((u32)Te4[(temp >> 24) ] << 24) ^ + ((u32)Te4[(temp >> 16) & 0xff] << 16) ^ + ((u32)Te4[(temp >> 8) & 0xff] << 8) ^ + ((u32)Te4[(temp ) & 0xff]); rk[13] = rk[ 5] ^ rk[12]; rk[14] = rk[ 6] ^ rk[13]; rk[15] = rk[ 7] ^ rk[14]; diff --git a/crypto/aes/aes_x86core.c b/crypto/aes/aes_x86core.c index 132b09a212..c869ed7198 100644 --- a/crypto/aes/aes_x86core.c +++ b/crypto/aes/aes_x86core.c @@ -499,10 +499,10 @@ int AES_set_encrypt_key(const unsigned char *userKey, const int bits, while (1) { temp = rk[3]; rk[4] = rk[0] ^ - (Te4[(temp >> 8) & 0xff] ) ^ - (Te4[(temp >> 16) & 0xff] << 8) ^ - (Te4[(temp >> 24) ] << 16) ^ - (Te4[(temp ) & 0xff] << 24) ^ + ((u32)Te4[(temp >> 8) & 0xff] ) ^ + ((u32)Te4[(temp >> 16) & 0xff] << 8) ^ + ((u32)Te4[(temp >> 24) ] << 16) ^ + ((u32)Te4[(temp ) & 0xff] << 24) ^ rcon[i]; rk[5] = rk[1] ^ rk[4]; rk[6] = rk[2] ^ rk[5]; @@ -519,10 +519,10 @@ int AES_set_encrypt_key(const unsigned char *userKey, const int bits, while (1) { temp = rk[ 5]; rk[ 6] = rk[ 0] ^ - (Te4[(temp >> 8) & 0xff] ) ^ - (Te4[(temp >> 16) & 0xff] << 8) ^ - (Te4[(temp >> 24) ] << 16) ^ - (Te4[(temp ) & 0xff] << 24) ^ + ((u32)Te4[(temp >> 8) & 0xff] ) ^ + ((u32)Te4[(temp >> 16) & 0xff] << 8) ^ + ((u32)Te4[(temp >> 24) ] << 16) ^ + ((u32)Te4[(temp ) & 0xff] << 24) ^ rcon[i]; rk[ 7] = rk[ 1] ^ rk[ 6]; rk[ 8] = rk[ 2] ^ rk[ 7]; @@ -541,10 +541,10 @@ int AES_set_encrypt_key(const unsigned char *userKey, const int bits, while (1) { temp = rk[ 7]; rk[ 8] = rk[ 0] ^ - (Te4[(temp >> 8) & 0xff] ) ^ - (Te4[(temp >> 16) & 0xff] << 8) ^ - (Te4[(temp >> 24) ] << 16) ^ - (Te4[(temp ) & 0xff] << 24) ^ + ((u32)Te4[(temp >> 8) & 0xff] ) ^ + ((u32)Te4[(temp >> 16) & 0xff] << 8) ^ + ((u32)Te4[(temp >> 24) ] << 16) ^ + ((u32)Te4[(temp ) & 0xff] << 24) ^ rcon[i]; rk[ 9] = rk[ 1] ^ rk[ 8]; rk[10] = rk[ 2] ^ rk[ 9]; @@ -554,10 +554,10 @@ int AES_set_encrypt_key(const unsigned char *userKey, const int bits, } temp = rk[11]; rk[12] = rk[ 4] ^ - (Te4[(temp ) & 0xff] ) ^ - (Te4[(temp >> 8) & 0xff] << 8) ^ - (Te4[(temp >> 16) & 0xff] << 16) ^ - (Te4[(temp >> 24) ] << 24); + ((u32)Te4[(temp ) & 0xff] ) ^ + ((u32)Te4[(temp >> 8) & 0xff] << 8) ^ + ((u32)Te4[(temp >> 16) & 0xff] << 16) ^ + ((u32)Te4[(temp >> 24) ] << 24); rk[13] = rk[ 5] ^ rk[12]; rk[14] = rk[ 6] ^ rk[13]; rk[15] = rk[ 7] ^ rk[14]; @@ -676,22 +676,22 @@ void AES_encrypt(const unsigned char *in, unsigned char *out, #if defined(AES_COMPACT_IN_OUTER_ROUNDS) prefetch256(Te4); - t[0] = Te4[(s0 ) & 0xff] ^ - Te4[(s1 >> 8) & 0xff] << 8 ^ - Te4[(s2 >> 16) & 0xff] << 16 ^ - Te4[(s3 >> 24) ] << 24; - t[1] = Te4[(s1 ) & 0xff] ^ - Te4[(s2 >> 8) & 0xff] << 8 ^ - Te4[(s3 >> 16) & 0xff] << 16 ^ - Te4[(s0 >> 24) ] << 24; - t[2] = Te4[(s2 ) & 0xff] ^ - Te4[(s3 >> 8) & 0xff] << 8 ^ - Te4[(s0 >> 16) & 0xff] << 16 ^ - Te4[(s1 >> 24) ] << 24; - t[3] = Te4[(s3 ) & 0xff] ^ - Te4[(s0 >> 8) & 0xff] << 8 ^ - Te4[(s1 >> 16) & 0xff] << 16 ^ - Te4[(s2 >> 24) ] << 24; + t[0] = (u32)Te4[(s0 ) & 0xff] ^ + (u32)Te4[(s1 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s2 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s3 >> 24) ] << 24; + t[1] = (u32)Te4[(s1 ) & 0xff] ^ + (u32)Te4[(s2 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s3 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s0 >> 24) ] << 24; + t[2] = (u32)Te4[(s2 ) & 0xff] ^ + (u32)Te4[(s3 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s0 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s1 >> 24) ] << 24; + t[3] = (u32)Te4[(s3 ) & 0xff] ^ + (u32)Te4[(s0 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s1 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s2 >> 24) ] << 24; /* now do the linear transform using words */ { int i; @@ -742,22 +742,22 @@ void AES_encrypt(const unsigned char *in, unsigned char *out, */ for (rk+=8,r=key->rounds-2; r>0; rk+=4,r--) { #if defined(AES_COMPACT_IN_INNER_ROUNDS) - t[0] = Te4[(s0 ) & 0xff] ^ - Te4[(s1 >> 8) & 0xff] << 8 ^ - Te4[(s2 >> 16) & 0xff] << 16 ^ - Te4[(s3 >> 24) ] << 24; - t[1] = Te4[(s1 ) & 0xff] ^ - Te4[(s2 >> 8) & 0xff] << 8 ^ - Te4[(s3 >> 16) & 0xff] << 16 ^ - Te4[(s0 >> 24) ] << 24; - t[2] = Te4[(s2 ) & 0xff] ^ - Te4[(s3 >> 8) & 0xff] << 8 ^ - Te4[(s0 >> 16) & 0xff] << 16 ^ - Te4[(s1 >> 24) ] << 24; - t[3] = Te4[(s3 ) & 0xff] ^ - Te4[(s0 >> 8) & 0xff] << 8 ^ - Te4[(s1 >> 16) & 0xff] << 16 ^ - Te4[(s2 >> 24) ] << 24; + t[0] = (u32)Te4[(s0 ) & 0xff] ^ + (u32)Te4[(s1 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s2 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s3 >> 24) ] << 24; + t[1] = (u32)Te4[(s1 ) & 0xff] ^ + (u32)Te4[(s2 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s3 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s0 >> 24) ] << 24; + t[2] = (u32)Te4[(s2 ) & 0xff] ^ + (u32)Te4[(s3 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s0 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s1 >> 24) ] << 24; + t[3] = (u32)Te4[(s3 ) & 0xff] ^ + (u32)Te4[(s0 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s1 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s2 >> 24) ] << 24; /* now do the linear transform using words */ { @@ -812,28 +812,28 @@ void AES_encrypt(const unsigned char *in, unsigned char *out, prefetch256(Te4); *(u32*)(out+0) = - Te4[(s0 ) & 0xff] ^ - Te4[(s1 >> 8) & 0xff] << 8 ^ - Te4[(s2 >> 16) & 0xff] << 16 ^ - Te4[(s3 >> 24) ] << 24 ^ + (u32)Te4[(s0 ) & 0xff] ^ + (u32)Te4[(s1 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s2 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s3 >> 24) ] << 24 ^ rk[0]; *(u32*)(out+4) = - Te4[(s1 ) & 0xff] ^ - Te4[(s2 >> 8) & 0xff] << 8 ^ - Te4[(s3 >> 16) & 0xff] << 16 ^ - Te4[(s0 >> 24) ] << 24 ^ + (u32)Te4[(s1 ) & 0xff] ^ + (u32)Te4[(s2 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s3 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s0 >> 24) ] << 24 ^ rk[1]; *(u32*)(out+8) = - Te4[(s2 ) & 0xff] ^ - Te4[(s3 >> 8) & 0xff] << 8 ^ - Te4[(s0 >> 16) & 0xff] << 16 ^ - Te4[(s1 >> 24) ] << 24 ^ + (u32)Te4[(s2 ) & 0xff] ^ + (u32)Te4[(s3 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s0 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s1 >> 24) ] << 24 ^ rk[2]; *(u32*)(out+12) = - Te4[(s3 ) & 0xff] ^ - Te4[(s0 >> 8) & 0xff] << 8 ^ - Te4[(s1 >> 16) & 0xff] << 16 ^ - Te4[(s2 >> 24) ] << 24 ^ + (u32)Te4[(s3 ) & 0xff] ^ + (u32)Te4[(s0 >> 8) & 0xff] << 8 ^ + (u32)Te4[(s1 >> 16) & 0xff] << 16 ^ + (u32)Te4[(s2 >> 24) ] << 24 ^ rk[3]; #else *(u32*)(out+0) = @@ -890,22 +890,22 @@ void AES_decrypt(const unsigned char *in, unsigned char *out, #if defined(AES_COMPACT_IN_OUTER_ROUNDS) prefetch256(Td4); - t[0] = Td4[(s0 ) & 0xff] ^ - Td4[(s3 >> 8) & 0xff] << 8 ^ - Td4[(s2 >> 16) & 0xff] << 16 ^ - Td4[(s1 >> 24) ] << 24; - t[1] = Td4[(s1 ) & 0xff] ^ - Td4[(s0 >> 8) & 0xff] << 8 ^ - Td4[(s3 >> 16) & 0xff] << 16 ^ - Td4[(s2 >> 24) ] << 24; - t[2] = Td4[(s2 ) & 0xff] ^ - Td4[(s1 >> 8) & 0xff] << 8 ^ - Td4[(s0 >> 16) & 0xff] << 16 ^ - Td4[(s3 >> 24) ] << 24; - t[3] = Td4[(s3 ) & 0xff] ^ - Td4[(s2 >> 8) & 0xff] << 8 ^ - Td4[(s1 >> 16) & 0xff] << 16 ^ - Td4[(s0 >> 24) ] << 24; + t[0] = (u32)Td4[(s0 ) & 0xff] ^ + (u32)Td4[(s3 >> 8) & 0xff] << 8 ^ + (u32)Td4[(s2 >> 16) & 0xff] << 16 ^ + (u32)Td4[(s1 >> 24) ] << 24; + t[1] = (u32)Td4[(s1 ) & 0xff] ^ + (u32)Td4[(s0 >> 8) & 0xff] << 8 ^ + (u32)Td4[(s3 >> 16) & 0xff] << 16 ^ + (u32)Td4[(s2 >> 24) ] << 24; + t[2] = (u32)Td4[(s2 ) & 0xff] ^ + (u32)Td4[(s1 >> 8) & 0xff] << 8 ^ + (u32)Td4[(s0 >> 16) & 0xff] << 16 ^ + (u32)Td4[(s3 >> 24) ] << 24; + t[3] = (u32)Td4[(s3 ) & 0xff] ^ + (u32)Td4[(s2 >> 8) & 0xff] << 8 ^ + (u32)Td4[(s1 >> 16) & 0xff] << 16 ^ + (u32)Td4[(s0 >> 24) ] << 24; /* now do the linear transform using words */ { @@ -967,22 +967,22 @@ void AES_decrypt(const unsigned char *in, unsigned char *out, */ for (rk+=8,r=key->rounds-2; r>0; rk+=4,r--) { #if defined(AES_COMPACT_IN_INNER_ROUNDS) - t[0] = Td4[(s0 ) & 0xff] ^ - Td4[(s3 >> 8) & 0xff] << 8 ^ - Td4[(s2 >> 16) & 0xff] << 16 ^ - Td4[(s1 >> 24) ] << 24; - t[1] = Td4[(s1 ) & 0xff] ^ - Td4[(s0 >> 8) & 0xff] << 8 ^ - Td4[(s3 >> 16) & 0xff] << 16 ^ - Td4[(s2 >> 24) ] << 24; - t[2] = Td4[(s2 ) & 0xff] ^ - Td4[(s1 >> 8) & 0xff] << 8 ^ - Td4[(s0 >> 16) & 0xff] << 16 ^ - Td4[(s3 >> 24) ] << 24; - t[3] = Td4[(s3 ) & 0xff] ^ - Td4[(s2 >> 8) & 0xff] << 8 ^ - Td4[(s1 >> 16) & 0xff] << 16 ^ - Td4[(s0 >> 24) ] << 24; + t[0] = (u32)Td4[(s0 ) & 0xff] ^ + (u32)Td4[(s3 >> 8) & 0xff] << 8 ^ + (u32)Td4[(s2 >> 16) & 0xff] << 16 ^ + (u32)Td4[(s1 >> 24) ] << 24; + t[1] = (u32)Td4[(s1 ) & 0xff] ^ + (u32)Td4[(s0 >> 8) & 0xff] << 8 ^ + (u32)Td4[(s3 >> 16) & 0xff] << 16 ^ + (u32)Td4[(s2 >> 24) ] << 24; + t[2] = (u32)Td4[(s2 ) & 0xff] ^ + (u32)Td4[(s1 >> 8) & 0xff] << 8 ^ + (u32)Td4[(s0 >> 16) & 0xff] << 16 ^ + (u32)Td4[(s3 >> 24) ] << 24; + t[3] = (u32)Td4[(s3 ) & 0xff] ^ + (u32)Td4[(s2 >> 8) & 0xff] << 8 ^ + (u32)Td4[(s1 >> 16) & 0xff] << 16 ^ + (u32)Td4[(s0 >> 24) ] << 24; /* now do the linear transform using words */ { @@ -1046,27 +1046,27 @@ void AES_decrypt(const unsigned char *in, unsigned char *out, prefetch256(Td4); *(u32*)(out+0) = - (Td4[(s0 ) & 0xff]) ^ - (Td4[(s3 >> 8) & 0xff] << 8) ^ - (Td4[(s2 >> 16) & 0xff] << 16) ^ - (Td4[(s1 >> 24) ] << 24) ^ + ((u32)Td4[(s0 ) & 0xff]) ^ + ((u32)Td4[(s3 >> 8) & 0xff] << 8) ^ + ((u32)Td4[(s2 >> 16) & 0xff] << 16) ^ + ((u32)Td4[(s1 >> 24) ] << 24) ^ rk[0]; *(u32*)(out+4) = - (Td4[(s1 ) & 0xff]) ^ - (Td4[(s0 >> 8) & 0xff] << 8) ^ - (Td4[(s3 >> 16) & 0xff] << 16) ^ - (Td4[(s2 >> 24) ] << 24) ^ + ((u32)Td4[(s1 ) & 0xff]) ^ + ((u32)Td4[(s0 >> 8) & 0xff] << 8) ^ + ((u32)Td4[(s3 >> 16) & 0xff] << 16) ^ + ((u32)Td4[(s2 >> 24) ] << 24) ^ rk[1]; *(u32*)(out+8) = - (Td4[(s2 ) & 0xff]) ^ - (Td4[(s1 >> 8) & 0xff] << 8) ^ - (Td4[(s0 >> 16) & 0xff] << 16) ^ - (Td4[(s3 >> 24) ] << 24) ^ + ((u32)Td4[(s2 ) & 0xff]) ^ + ((u32)Td4[(s1 >> 8) & 0xff] << 8) ^ + ((u32)Td4[(s0 >> 16) & 0xff] << 16) ^ + ((u32)Td4[(s3 >> 24) ] << 24) ^ rk[2]; *(u32*)(out+12) = - (Td4[(s3 ) & 0xff]) ^ - (Td4[(s2 >> 8) & 0xff] << 8) ^ - (Td4[(s1 >> 16) & 0xff] << 16) ^ - (Td4[(s0 >> 24) ] << 24) ^ + ((u32)Td4[(s3 ) & 0xff]) ^ + ((u32)Td4[(s2 >> 8) & 0xff] << 8) ^ + ((u32)Td4[(s1 >> 16) & 0xff] << 16) ^ + ((u32)Td4[(s0 >> 24) ] << 24) ^ rk[3]; }