Improve error detection when streaming S/MIME.

Only use streaming when appropriate for detached data in smime utility.

diff --git a/apps/smime.c b/apps/smime.c
index a76e88d14172a25bd078a75db627925dcc7bf27d..af2960685f6389d71041c0931166981d8aec9e7e 100644 (file)
--- a/apps/smime.c
+++ b/apps/smime.c
@@ -688,15 +688,22 @@ int MAIN(int argc, char **argv)
        else if (operation & SMIME_SIGNERS)
                {
                int i;
-               /* If detached data and SMIME output enable partial
-                * signing.
+               /* If detached data content we only enable streaming if
+                * S/MIME output format.
                 */
                if (operation == SMIME_SIGN)
                        {
-                       if (indef || (flags & PKCS7_DETACHED))
+                       if (flags & PKCS7_DETACHED)
+                               {
+                               if (outformat == FORMAT_SMIME)
+                                       flags |= PKCS7_STREAM;
+                               }
+                       else if (indef)
                                flags |= PKCS7_STREAM;
                        flags |= PKCS7_PARTIAL;
                        p7 = PKCS7_sign(NULL, NULL, other, in, flags);
+                       if (!p7)
+                               goto end;
                        }
                else
                        flags |= PKCS7_REUSE_DIGEST;

diff --git a/crypto/pkcs7/bio_pk7.c b/crypto/pkcs7/bio_pk7.c
index 533e596a1c394ee5ed6df73ce37c02000e37a4bf..581ba09f7aab831826d8cfe986216f4495535fbd 100644 (file)
--- a/crypto/pkcs7/bio_pk7.c
+++ b/crypto/pkcs7/bio_pk7.c
@@ -114,13 +114,17 @@ BIO *BIO_new_PKCS7(BIO *out, PKCS7 *p7)
 
        out = BIO_push(asn_bio, out);
 
+       if (!p7aux || !asn_bio || !out)
+               goto err;
+
        BIO_asn1_set_prefix(asn_bio, pkcs7_prefix, pkcs7_prefix_free);
        BIO_asn1_set_suffix(asn_bio, pkcs7_suffix, pkcs7_suffix_free);
 
        /* Now initialize BIO for PKCS#7 output */
 
        p7bio = PKCS7_dataInit(p7, out);
-       PKCS7_stream(&boundary, p7);
+       if (!p7bio || !PKCS7_stream(&boundary, p7))
+               goto err;
 
        p7aux->p7 = p7;
        p7aux->p7bio = p7bio;
@@ -131,6 +135,14 @@ BIO *BIO_new_PKCS7(BIO *out, PKCS7 *p7)
 
        return p7bio;
 
+       err:
+       if (p7bio)
+               BIO_free(p7bio);
+       if (asn_bio)
+               BIO_free(asn_bio);
+       if (p7aux)
+               OPENSSL_free(p7aux);
+       return NULL;
        }
 
 static int pkcs7_prefix(BIO *b, unsigned char **pbuf, int *plen, void *parg)

diff --git a/crypto/pkcs7/pk7_lib.c b/crypto/pkcs7/pk7_lib.c
index 9ad1448bf7862fbf99a4c50f28167aca2498b6fc..3ca0952792a3d4f3bb1bfabfc9102e04842719fb 100644 (file)
--- a/crypto/pkcs7/pk7_lib.c
+++ b/crypto/pkcs7/pk7_lib.c
@@ -665,8 +665,3 @@ int PKCS7_stream(unsigned char ***boundary, PKCS7 *p7)
 
        return 1;
        }
-
-
-
-
-