Configure, x86gas.pl: fix linker warnings in 32-bit Darwin build.

[oweals/openssl.git] / crypto / rsa / rsa_oaep.c

diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index 3652677a99822b9cd47b9354bc0ba6366e3a2fc7..eaae71223682099f737e5c8377894662de1a15da 100644 (file)
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -18,6 +18,8 @@
  * an equivalent notion.
  */
 
+#define OPENSSL_FIPSAPI
+
 
 #if !defined(OPENSSL_NO_SHA) && !defined(OPENSSL_NO_SHA1)
 #include <stdio.h>
@@ -28,7 +30,7 @@
 #include <openssl/rand.h>
 #include <openssl/sha.h>
 
-int MGF1(unsigned char *mask, long len,
+static int MGF1(unsigned char *mask, long len,
        const unsigned char *seed, long seedlen);
 
 int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
@@ -52,18 +54,12 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
                return 0;
                }
 
-       dbmask = OPENSSL_malloc(emlen - SHA_DIGEST_LENGTH);
-       if (dbmask == NULL)
-               {
-               RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
-               return 0;
-               }
-
        to[0] = 0;
        seed = to + 1;
        db = to + SHA_DIGEST_LENGTH + 1;
 
-       EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL);
+       if (!EVP_Digest((void *)param, plen, db, NULL, EVP_sha1(), NULL))
+               return 0;
        memset(db + SHA_DIGEST_LENGTH, 0,
                emlen - flen - 2 * SHA_DIGEST_LENGTH - 1);
        db[emlen - flen - SHA_DIGEST_LENGTH - 1] = 0x01;
@@ -76,11 +72,20 @@ int RSA_padding_add_PKCS1_OAEP(unsigned char *to, int tlen,
           20);
 #endif
 
-       MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH);
+       dbmask = OPENSSL_malloc(emlen - SHA_DIGEST_LENGTH);
+       if (dbmask == NULL)
+               {
+               RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_OAEP, ERR_R_MALLOC_FAILURE);
+               return 0;
+               }
+
+       if (MGF1(dbmask, emlen - SHA_DIGEST_LENGTH, seed, SHA_DIGEST_LENGTH) < 0)
+               return 0;
        for (i = 0; i < emlen - SHA_DIGEST_LENGTH; i++)
                db[i] ^= dbmask[i];
 
-       MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH);
+       if (MGF1(seedmask, SHA_DIGEST_LENGTH, db, emlen - SHA_DIGEST_LENGTH) < 0)
+               return 0;
        for (i = 0; i < SHA_DIGEST_LENGTH; i++)
                seed[i] ^= seedmask[i];
 
@@ -133,15 +138,18 @@ int RSA_padding_check_PKCS1_OAEP(unsigned char *to, int tlen,
 
        maskeddb = padded_from + SHA_DIGEST_LENGTH;
 
-       MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen);
+       if (MGF1(seed, SHA_DIGEST_LENGTH, maskeddb, dblen))
+               return -1;
        for (i = 0; i < SHA_DIGEST_LENGTH; i++)
                seed[i] ^= padded_from[i];
   
-       MGF1(db, dblen, seed, SHA_DIGEST_LENGTH);
+       if (MGF1(db, dblen, seed, SHA_DIGEST_LENGTH))
+               return -1;
        for (i = 0; i < dblen; i++)
                db[i] ^= maskeddb[i];
 
-       EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL);
+       if (!EVP_Digest((void *)param, plen, phash, NULL, EVP_sha1(), NULL))
+               return -1;
 
        if (memcmp(db, phash, SHA_DIGEST_LENGTH) != 0 || bad)
                goto decoding_err;
@@ -185,35 +193,44 @@ int PKCS1_MGF1(unsigned char *mask, long len,
        EVP_MD_CTX c;
        unsigned char md[EVP_MAX_MD_SIZE];
        int mdlen;
+       int rv = -1;
 
        EVP_MD_CTX_init(&c);
-       mdlen = EVP_MD_size(dgst);
+       mdlen = M_EVP_MD_size(dgst);
+       if (mdlen < 0)
+               goto err;
        for (i = 0; outlen < len; i++)
                {
                cnt[0] = (unsigned char)((i >> 24) & 255);
                cnt[1] = (unsigned char)((i >> 16) & 255);
                cnt[2] = (unsigned char)((i >> 8)) & 255;
                cnt[3] = (unsigned char)(i & 255);
-               EVP_DigestInit_ex(&c,dgst, NULL);
-               EVP_DigestUpdate(&c, seed, seedlen);
-               EVP_DigestUpdate(&c, cnt, 4);
+               if (!EVP_DigestInit_ex(&c,dgst, NULL)
+                       || !EVP_DigestUpdate(&c, seed, seedlen)
+                       || !EVP_DigestUpdate(&c, cnt, 4))
+                       goto err;
                if (outlen + mdlen <= len)
                        {
-                       EVP_DigestFinal_ex(&c, mask + outlen, NULL);
+                       if (!EVP_DigestFinal_ex(&c, mask + outlen, NULL))
+                               goto err;
                        outlen += mdlen;
                        }
                else
                        {
-                       EVP_DigestFinal_ex(&c, md, NULL);
+                       if (!EVP_DigestFinal_ex(&c, md, NULL))
+                               goto err;
                        memcpy(mask + outlen, md, len - outlen);
                        outlen = len;
                        }
                }
+       rv = 0;
+       err:
        EVP_MD_CTX_cleanup(&c);
-       return 0;
+       return rv;
        }
 
-int MGF1(unsigned char *mask, long len, const unsigned char *seed, long seedlen)
+static int MGF1(unsigned char *mask, long len, const unsigned char *seed,
+                long seedlen)
        {
        return PKCS1_MGF1(mask, len, seed, seedlen, EVP_sha1());
        }