Ensure dhparams can handle X9.42 params in DER

dhparams correctly handles X9.42 params in PEM format. However it failed
to correctly processes them when reading/writing DER format.

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3112)

diff --git a/apps/dhparam.c b/apps/dhparam.c
index 1210adb104d9f39fa57e7276acb5f186b5456b77..bd91234abd664d6e2cbeb46c5862e12aae4ff17e 100644 (file)
--- a/apps/dhparam.c
+++ b/apps/dhparam.c
@@ -381,10 +381,19 @@ int MAIN(int argc, char **argv)
         } else
 # endif
         {
-            if (informat == FORMAT_ASN1)
+            if (informat == FORMAT_ASN1) {
+                /*
+                 * We have no PEM header to determine what type of DH params it
+                 * is. We'll just try both.
+                 */
                 dh = d2i_DHparams_bio(in, NULL);
-            else                /* informat == FORMAT_PEM */
+                /* BIO_reset() returns 0 for success for file BIOs only!!! */
+                if (dh == NULL && BIO_reset(in) == 0)
+                    dh = d2i_DHxparams_bio(in, NULL);
+            } else {
+                /* informat == FORMAT_PEM */
                 dh = PEM_read_bio_DHparams(in, NULL, NULL, NULL);
+            }
 
             if (dh == NULL) {
                 BIO_printf(bio_err, "unable to load DH parameters\n");
@@ -484,10 +493,13 @@ int MAIN(int argc, char **argv)
     }
 
     if (!noout) {
-        if (outformat == FORMAT_ASN1)
-            i = i2d_DHparams_bio(out, dh);
-        else if (outformat == FORMAT_PEM) {
-            if (dh->q)
+        if (outformat == FORMAT_ASN1) {
+            if (dh->q != NULL)
+                i = i2d_DHxparams_bio(out, dh);
+            else
+                i = i2d_DHparams_bio(out, dh);
+        } else if (outformat == FORMAT_PEM) {
+            if (dh->q != NULL)
                 i = PEM_write_bio_DHxparams(out, dh);
             else
                 i = PEM_write_bio_DHparams(out, dh);