Ensure ECDSA_size always returns >= 0

Fixes #10484

Signed-off-by: William Brawner <me@wbrawner.com>
Reviewed-by: Nicola Tuveri <nic.tuv@gmail.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/11472)

diff --git a/crypto/ec/ec_asn1.c b/crypto/ec/ec_asn1.c
index 336afc989d30160d1cdadb8ccdf4cc7daace2db5..831b74ce6c23a5d208bd280d79bf25a212b85e04 100644 (file)
--- a/crypto/ec/ec_asn1.c
+++ b/crypto/ec/ec_asn1.c
@@ -1297,5 +1297,7 @@ int ECDSA_size(const EC_KEY *r)
     i = i2d_ASN1_INTEGER(&bs, NULL);
     i += i;                     /* r and s */
     ret = ASN1_object_size(1, i, V_ASN1_SEQUENCE);
+    if (ret < 0)
+        return 0;
     return ret;
 }