Fix decoding of ASN.1 LONG and ZLONG items

LONG and ZLONG items (which are OpenSSL private special cases of
ASN1_INTEGER) are encoded into DER with padding if the leading octet
has the high bit set, where the padding can be 0x00 (for positive
numbers) or 0xff (for negative ones).

When decoding DER to LONG or ZLONG, the padding wasn't taken in
account at all, which means that if the encoded size with padding
is one byte more than the size of long, decoding fails.  This change
fixes that issue.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3000)
(cherry picked from commit ca2045dc545ba4afe8abbe29d0316ee3d36da7df)

diff --git a/crypto/asn1/x_long.c b/crypto/asn1/x_long.c
index 2fd3a903d180bb093528e19949eb71735e9b48d3..aecb95069de723bb71ec36dccbf47b35d436a3ff 100644 (file)
--- a/crypto/asn1/x_long.c
+++ b/crypto/asn1/x_long.c
@@ -155,19 +155,41 @@ static int long_i2c(ASN1_VALUE **pval, unsigned char *cont, int *putype,
 static int long_c2i(ASN1_VALUE **pval, const unsigned char *cont, int len,
                     int utype, char *free_cont, const ASN1_ITEM *it)
 {
-    int neg, i;
+    int neg = -1, i;
     long ltmp;
     unsigned long utmp = 0;
     char *cp = (char *)pval;
+
+    if (len) {
+        /*
+         * Check possible pad byte.  Worst case, we're skipping past actual
+         * content, but since that's only with 0x00 and 0xff and we set neg
+         * accordingly, the result will be correct in the end anyway.
+         */
+        switch (cont[0]) {
+        case 0xff:
+            cont++;
+            len--;
+            neg = 1;
+            break;
+        case 0:
+            cont++;
+            len--;
+            neg = 0;
+            break;
+        }
+    }
     if (len > (int)sizeof(long)) {
         ASN1err(ASN1_F_LONG_C2I, ASN1_R_INTEGER_TOO_LARGE_FOR_LONG);
         return 0;
     }
-    /* Is it negative? */
-    if (len && (cont[0] & 0x80))
-        neg = 1;
-    else
-        neg = 0;
+    if (neg == -1) {
+        /* Is it negative? */
+        if (len && (cont[0] & 0x80))
+            neg = 1;
+        else
+            neg = 0;
+    }
     utmp = 0;
     for (i = 0; i < len; i++) {
         utmp <<= 8;