Fix error handling in i2d* functions.

CLA: trivial

Reviewed-by: Kurt Roeckx <kurt@roeckx.be>
Reviewed-by: Bernd Edlinger <bernd.edlinger@hotmail.de>
(Merged from https://github.com/openssl/openssl/pull/4600)

diff --git a/crypto/x509/x_pubkey.c b/crypto/x509/x_pubkey.c
index 76635b8952365dffb8f248176a206a8b73bf629b..73977cd8c0f15f894b24abb92fd60761f43d6b1c 100644 (file)
--- a/crypto/x509/x_pubkey.c
+++ b/crypto/x509/x_pubkey.c
@@ -206,7 +206,7 @@ int i2d_PUBKEY(EVP_PKEY *a, unsigned char **pp)
     if (!a)
         return 0;
     if (!X509_PUBKEY_set(&xpk, a))
-        return 0;
+        return -1;
     ret = i2d_X509_PUBKEY(xpk, pp);
     X509_PUBKEY_free(xpk);
     return ret;
@@ -246,7 +246,7 @@ int i2d_RSA_PUBKEY(RSA *a, unsigned char **pp)
     pktmp = EVP_PKEY_new();
     if (pktmp == NULL) {
         ASN1err(ASN1_F_I2D_RSA_PUBKEY, ERR_R_MALLOC_FAILURE);
-        return 0;
+        return -1;
     }
     EVP_PKEY_set1_RSA(pktmp, a);
     ret = i2d_PUBKEY(pktmp, pp);
@@ -286,7 +286,7 @@ int i2d_DSA_PUBKEY(DSA *a, unsigned char **pp)
     pktmp = EVP_PKEY_new();
     if (pktmp == NULL) {
         ASN1err(ASN1_F_I2D_DSA_PUBKEY, ERR_R_MALLOC_FAILURE);
-        return 0;
+        return -1;
     }
     EVP_PKEY_set1_DSA(pktmp, a);
     ret = i2d_PUBKEY(pktmp, pp);
@@ -325,7 +325,7 @@ int i2d_EC_PUBKEY(EC_KEY *a, unsigned char **pp)
         return 0;
     if ((pktmp = EVP_PKEY_new()) == NULL) {
         ASN1err(ASN1_F_I2D_EC_PUBKEY, ERR_R_MALLOC_FAILURE);
-        return 0;
+        return -1;
     }
     EVP_PKEY_set1_EC_KEY(pktmp, a);
     ret = i2d_PUBKEY(pktmp, pp);