Convert asn1_dsa.c to use the WPACKET API instead

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/9111)

diff --git a/crypto/asn1_dsa.c b/crypto/asn1_dsa.c
index 5eddc2dad5eee57eb3f2740c16d06d28df351121..63979d767ce45ec037930249d3cfdf2916e90663 100644 (file)
--- a/crypto/asn1_dsa.c
+++ b/crypto/asn1_dsa.c
@@ -31,75 +31,44 @@
 
 /*
  * Outputs the encoding of the length octets for a DER value with a content
- * length of cont_len bytes to *ppout and, if successful, increments *ppout
- * past the data just written.
+ * length of cont_len bytes to pkt. The maximum supported content length is
+ * 65535 (0xffff) bytes.
  *
- * The maximum supported content length is 65535 (0xffff) bytes.
- * The maximum returned length in bytes of the encoded output is 3.
- *
- * If ppout is NULL then the output size is calculated and returned but no
- * output is produced.
- * If ppout is not NULL then *ppout must not be NULL.
- *
- * An attempt to produce more than len bytes results in an error.
- * Returns the number of bytes of output produced (or that would be produced)
- * or 0 if an error occurs.
+ * Returns 1 on success or 0 on error.
  */
-size_t encode_der_length(size_t cont_len, unsigned char **ppout, size_t len)
+int encode_der_length(WPACKET *pkt, size_t cont_len)
 {
-    size_t encoded_len;
+    if (cont_len > 0xffff)
+        return 0; /* Too large for supported length encodings */
 
-    if (cont_len <= 0x7f) {
-        encoded_len = 1;
-    } else if (cont_len <= 0xff) {
-        encoded_len = 2;
-    } else if (cont_len <= 0xffff) {
-        encoded_len = 3;
+    if (cont_len > 0xff) {
+        if (!WPACKET_put_bytes_u8(pkt, 0x82)
+                || !WPACKET_put_bytes_u16(pkt, cont_len))
+            return 0;
     } else {
-        /* Too large for supported length encodings */
-        return 0;
-    }
-    if (encoded_len > len)
-        return 0;
-    if (ppout != NULL) {
-        unsigned char *out = *ppout;
-        switch (encoded_len) {
-        case 2:
-            *out++ = 0x81;
-            break;
-        case 3:
-            *out++ = 0x82;
-            *out++ = (unsigned char)(cont_len >> 8);
-            break;
-        }
-        *out++ = (unsigned char)cont_len;
-        *ppout = out;
+        if (cont_len > 0x7f
+                && !WPACKET_put_bytes_u8(pkt, 0x81))
+            return 0;
+        if (!WPACKET_put_bytes_u8(pkt, cont_len))
+            return 0;
     }
-    return encoded_len;
+
+    return 1;
 }
 
 /*
- * Outputs the DER encoding of a positive ASN.1 INTEGER to *ppout and, if
- * successful, increments *ppout past the data just written.
+ * Outputs the DER encoding of a positive ASN.1 INTEGER to pkt.
  *
- * If n is negative then an error results.
- * If ppout is NULL then the output size is calculated and returned but no
- * output is produced.
- * If ppout is not NULL then *ppout must not be NULL.
+ * Results in an error if n is negative or too large.
  *
- * An attempt to produce more than len bytes results in an error.
- * Returns the number of bytes of output produced (or that would be produced)
- * or 0 if an error occurs.
+ * Returns 1 on success or 0 on error.
  */
-size_t encode_der_integer(const BIGNUM *n, unsigned char **ppout, size_t len)
+int encode_der_integer(WPACKET *pkt, const BIGNUM *n)
 {
-    unsigned char *out = NULL;
-    unsigned char **pp = NULL;
-    size_t produced;
-    size_t c;
+    unsigned char *bnbytes;
     size_t cont_len;
 
-    if (len < 1 || BN_is_negative(n))
+    if (BN_is_negative(n))
         return 0;
 
     /*
@@ -113,75 +82,68 @@ size_t encode_der_integer(const BIGNUM *n, unsigned char **ppout, size_t len)
      */
     cont_len = BN_num_bits(n) / 8 + 1;
 
-    if (ppout != NULL) {
-        out = *ppout;
-        pp = &out;
-        *out++ = ID_INTEGER;
-    }
-    produced = 1;
-    if ((c = encode_der_length(cont_len, pp, len - produced)) == 0)
+    if (!WPACKET_start_sub_packet(pkt)
+            || !WPACKET_put_bytes_u8(pkt, ID_INTEGER)
+            || !encode_der_length(pkt, cont_len)
+            || !WPACKET_allocate_bytes(pkt, cont_len, &bnbytes)
+            || !WPACKET_close(pkt))
         return 0;
-    produced += c;
-    if (cont_len > len - produced)
+
+    if (bnbytes != NULL
+            && BN_bn2binpad(n, bnbytes, (int)cont_len) != (int)cont_len)
         return 0;
-    if (pp != NULL) {
-        if (BN_bn2binpad(n, out, (int)cont_len) != (int)cont_len)
-            return 0;
-        out += cont_len;
-        *ppout = out;
-    }
-    produced += cont_len;
-    return produced;
+
+    return 1;
 }
 
 /*
- * Outputs the DER encoding of a DSA-Sig-Value or ECDSA-Sig-Value to *ppout
- * and increments *ppout past the data just written.
- *
- * If ppout is NULL then the output size is calculated and returned but no
- * output is produced.
- * If ppout is not NULL then *ppout must not be NULL.
+ * Outputs the DER encoding of a DSA-Sig-Value or ECDSA-Sig-Value to pkt. pkt
+ * may be initialised with a NULL buffer which enables pkt to be used to
+ * calulate how many bytes would be needed.
  *
- * An attempt to produce more than len bytes results in an error.
- * Returns the number of bytes of output produced (or that would be produced)
- * or 0 if an error occurs.
+ * Returns 1 on success or 0 on error.
  */
-size_t encode_der_dsa_sig(const BIGNUM *r, const BIGNUM *s,
-                          unsigned char **ppout, size_t len)
+int encode_der_dsa_sig(WPACKET *pkt, const BIGNUM *r, const BIGNUM *s)
 {
-    unsigned char *out = NULL;
-    unsigned char **pp = NULL;
-    size_t produced;
-    size_t c;
-    size_t r_der_len;
-    size_t s_der_len;
+    WPACKET tmppkt, *dummypkt;
     size_t cont_len;
+    int isnull = WPACKET_is_null_buf(pkt);
 
-    if (len < 1
-            || (r_der_len = encode_der_integer(r, NULL, SIZE_MAX)) == 0
-            || (s_der_len = encode_der_integer(s, NULL, SIZE_MAX)) == 0)
+    if (!WPACKET_start_sub_packet(pkt))
         return 0;
 
-    cont_len = r_der_len + s_der_len;
-
-    if (ppout != NULL) {
-        out = *ppout;
-        pp = &out;
-        *out++ = ID_SEQUENCE;
+    if (!isnull) {
+        if (!WPACKET_init_null(&tmppkt, 0))
+            return 0;
+        dummypkt = &tmppkt;
+    } else {
+        /* If the input packet has a NULL buffer, we don't need a dummy packet */
+        dummypkt = pkt;
     }
-    produced = 1;
-    if ((c = encode_der_length(cont_len, pp, len - produced)) == 0)
-        return 0;
-    produced += c;
-    if ((c = encode_der_integer(r, pp, len - produced)) == 0)
+
+    /* Calculate the content length */
+    if (!encode_der_integer(dummypkt, r)
+            || !encode_der_integer(dummypkt, s)
+            || !WPACKET_get_length(dummypkt, &cont_len)
+            || (!isnull && !WPACKET_finish(dummypkt))) {
+        if (!isnull)
+            WPACKET_cleanup(dummypkt);
         return 0;
-    produced += c;
-    if ((c = encode_der_integer(s, pp, len - produced)) == 0)
+    }
+
+    /* Add the tag and length bytes */
+    if (!WPACKET_put_bytes_u8(pkt, ID_SEQUENCE)
+            || !encode_der_length(pkt, cont_len)
+               /*
+                * Really encode the integers. We already wrote to the main pkt
+                * if it had a NULL buffer, so don't do it again
+                */
+            || (!isnull && !encode_der_integer(pkt, r))
+            || (!isnull && !encode_der_integer(pkt, s))
+            || !WPACKET_close(pkt))
         return 0;
-    produced += c;
-    if (pp != NULL)
-        *ppout = out;
-    return produced;
+
+    return 1;
 }
 
 /*

diff --git a/crypto/dsa/dsa_asn1.c b/crypto/dsa/dsa_asn1.c
index 1c7e8c86bdd464dbb3a3d638db576de5164992fc..eddcc11819a442f8aa65e1eddd5046e78580ddf3 100644 (file)
--- a/crypto/dsa/dsa_asn1.c
+++ b/crypto/dsa/dsa_asn1.c
@@ -61,30 +61,42 @@ DSA_SIG *d2i_DSA_SIG(DSA_SIG **psig, const unsigned char **ppin, long len)
 
 int i2d_DSA_SIG(const DSA_SIG *sig, unsigned char **ppout)
 {
-    unsigned char *buf = NULL;
-    unsigned char *tmp;
-    unsigned char **pp = NULL;
-    size_t len;
+    BUF_MEM *buf = NULL;
     size_t encoded_len;
+    WPACKET pkt;
 
-    if (ppout != NULL && *ppout == NULL) {
-        if ((len = encode_der_dsa_sig(sig->r, sig->s, NULL, SIZE_MAX)) == 0)
+    if (ppout == NULL) {
+        if (!WPACKET_init_null(&pkt, 0))
             return -1;
-        buf = OPENSSL_malloc(len);
-        if (buf == NULL)
+    } else if (*ppout == NULL) {
+        if ((buf = BUF_MEM_new()) == NULL
+                || !WPACKET_init_len(&pkt, buf, 0)) {
+            BUF_MEM_free(buf);
             return -1;
-        tmp = buf;
-        pp = &tmp;
+        }
     } else {
-        len = SIZE_MAX;
-        pp = ppout;
+        if (!WPACKET_init_static_len(&pkt, *ppout, SIZE_MAX, 0))
+            return -1;
     }
-    if ((encoded_len = encode_der_dsa_sig(sig->r, sig->s, pp, len)) == 0) {
-        OPENSSL_free(buf);
+
+    if (!encode_der_dsa_sig(&pkt, sig->r, sig->s)
+            || !WPACKET_get_total_written(&pkt, &encoded_len)
+            || !WPACKET_finish(&pkt)) {
+        BUF_MEM_free(buf);
+        WPACKET_cleanup(&pkt);
         return -1;
     }
-    if (buf != NULL)
-        *ppout = buf;
+
+    if (ppout != NULL) {
+        if (*ppout == NULL) {
+            *ppout = (unsigned char *)buf->data;
+            buf->data = NULL;
+            BUF_MEM_free(buf);
+        } else {
+            *ppout += encoded_len;
+        }
+    }
+
     return (int)encoded_len;
 }
 

diff --git a/crypto/ec/ec_asn1.c b/crypto/ec/ec_asn1.c
index 57de90d3674787be1c1a1138e68ec6529114e37d..c2f9679c0cc6ecf5998c2314c31b962149ede44f 100644 (file)
--- a/crypto/ec/ec_asn1.c
+++ b/crypto/ec/ec_asn1.c
@@ -1187,30 +1187,42 @@ ECDSA_SIG *d2i_ECDSA_SIG(ECDSA_SIG **psig, const unsigned char **ppin, long len)
 
 int i2d_ECDSA_SIG(const ECDSA_SIG *sig, unsigned char **ppout)
 {
-    unsigned char *buf = NULL;
-    unsigned char *tmp;
-    unsigned char **pp = NULL;
-    size_t len;
+    BUF_MEM *buf = NULL;
     size_t encoded_len;
+    WPACKET pkt;
 
-    if (ppout != NULL && *ppout == NULL) {
-        if ((len = encode_der_dsa_sig(sig->r, sig->s, NULL, SIZE_MAX)) == 0)
+    if (ppout == NULL) {
+        if (!WPACKET_init_null(&pkt, 0))
             return -1;
-        buf = OPENSSL_malloc(len);
-        if (buf == NULL)
+    } else if (*ppout == NULL) {
+        if ((buf = BUF_MEM_new()) == NULL
+                || !WPACKET_init_len(&pkt, buf, 0)) {
+            BUF_MEM_free(buf);
             return -1;
-        tmp = buf;
-        pp = &tmp;
+        }
     } else {
-        len = SIZE_MAX;
-        pp = ppout;
+        if (!WPACKET_init_static_len(&pkt, *ppout, SIZE_MAX, 0))
+            return -1;
     }
-    if ((encoded_len = encode_der_dsa_sig(sig->r, sig->s, pp, len)) == 0) {
-        OPENSSL_free(buf);
+
+    if (!encode_der_dsa_sig(&pkt, sig->r, sig->s)
+            || !WPACKET_get_total_written(&pkt, &encoded_len)
+            || !WPACKET_finish(&pkt)) {
+        BUF_MEM_free(buf);
+        WPACKET_cleanup(&pkt);
         return -1;
     }
-    if (buf != NULL)
-        *ppout = buf;
+
+    if (ppout != NULL) {
+        if (*ppout == NULL) {
+            *ppout = (unsigned char *)buf->data;
+            buf->data = NULL;
+            BUF_MEM_free(buf);
+        } else {
+            *ppout += encoded_len;
+        }
+    }
+
     return (int)encoded_len;
 }
 

diff --git a/crypto/include/internal/asn1_dsa.h b/crypto/include/internal/asn1_dsa.h
index da01690df1df553113952627dd077585db715ea9..d2570516a107f6c20ee24b4e3c7fc489aeefacf2 100644 (file)
--- a/crypto/include/internal/asn1_dsa.h
+++ b/crypto/include/internal/asn1_dsa.h
@@ -12,10 +12,9 @@
 
 #include "internal/packet.h"
 
-size_t encode_der_length(size_t cont_len, unsigned char **ppout, size_t len);
-size_t encode_der_integer(const BIGNUM *n, unsigned char **ppout, size_t len);
-size_t encode_der_dsa_sig(const BIGNUM *r, const BIGNUM *s,
-                          unsigned char **ppout, size_t len);
+int encode_der_length(WPACKET *pkt, size_t cont_len);
+int encode_der_integer(WPACKET *pkt, const BIGNUM *n);
+int encode_der_dsa_sig(WPACKET *pkt, const BIGNUM *r, const BIGNUM *s);
 int decode_der_length(PACKET *pkt, PACKET *subpkt);
 int decode_der_integer(PACKET *pkt, BIGNUM *n);
 size_t decode_der_dsa_sig(BIGNUM *r, BIGNUM *s, const unsigned char **ppin,