Make the handling of output and input formats consistent

Most of all, we needed to sort out which ones are binary and which
ones are text, and make sure they are treated accordingly and
consistently so

Reviewed-by: Tim Hudson <tjh@openssl.org>

diff --git a/apps/apps.h b/apps/apps.h
index a134b8da98995f8c9f2eab30d37626af34bed839..cd709483136225431d956fecfb2780e824240301 100644 (file)
--- a/apps/apps.h
+++ b/apps/apps.h
@@ -163,8 +163,9 @@ int app_load_modules(const CONF *config);
 void unbuffer(FILE *fp);
 
 /* Often used in calls to bio_open_default. */
-# define RB(xformat)  ((xformat) == FORMAT_ASN1 ? "rb" : "r")
-# define WB(xformat)  ((xformat) == FORMAT_ASN1 ? "wb" : "w")
+# define RB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "rb" : "r")
+# define WB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "wb" : "w")
+# define AB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "ab" : "a")
 
 /*
  * Common verification options.
@@ -535,19 +536,20 @@ void print_cert_checks(BIO *bio, X509 *x,
 void store_setup_crl_download(X509_STORE *st);
 
 /* See OPT_FMT_xxx, above. */
+# define B_FORMAT_TEXT   0x8000
 # define FORMAT_UNDEF    0
 # define FORMAT_ASN1     1
-# define FORMAT_TEXT     2
-# define FORMAT_PEM      3
+# define FORMAT_TEXT    (2 | B_FORMAT_TEXT)
+# define FORMAT_PEM     (3 | B_FORMAT_TEXT)
 # define FORMAT_PKCS12   5
-# define FORMAT_SMIME    6
+# define FORMAT_SMIME   (6 | B_FORMAT_TEXT)
 # define FORMAT_ENGINE   7
-# define FORMAT_PEMRSA   9      /* PEM RSAPubicKey format */
-# define FORMAT_ASN1RSA  10     /* DER RSAPubicKey format */
-# define FORMAT_MSBLOB   11     /* MS Key blob format */
-# define FORMAT_PVK      12     /* MS PVK file format */
-# define FORMAT_HTTP     13     /* Download using HTTP */
-# define FORMAT_NSS      14     /* NSS keylog format */
+# define FORMAT_PEMRSA  (9 | B_FORMAT_TEXT)     /* PEM RSAPubicKey format */
+# define FORMAT_ASN1RSA  10                     /* DER RSAPubicKey format */
+# define FORMAT_MSBLOB   11                     /* MS Key blob format */
+# define FORMAT_PVK      12                     /* MS PVK file format */
+# define FORMAT_HTTP     13                     /* Download using HTTP */
+# define FORMAT_NSS      14                     /* NSS keylog format */
 
 # define EXT_COPY_NONE   0
 # define EXT_COPY_ADD    1

diff --git a/apps/asn1pars.c b/apps/asn1pars.c
index 24b55681f0123dacabd405a51a3ddcd48e9e418c..8881ad4e33789f77521610a1840bbf545f9d74be 100644 (file)
--- a/apps/asn1pars.c
+++ b/apps/asn1pars.c
@@ -197,7 +197,7 @@ int asn1parse_main(int argc, char **argv)
         BIO_free(in);
     }
 
-    if ((in = bio_open_default(infile, "r")) == NULL)
+    if ((in = bio_open_default(infile, RB(informat))) == NULL)
         goto end;
 
     if (derfile && (derout = bio_open_default(derfile, "wb")) == NULL)

diff --git a/apps/ca.c b/apps/ca.c
index 5cd80020679bbfa1e50f2a7498fef8db290819e8..ce09155f2894ecf3ffc50ba26b67449e4faa09e2 100644 (file)
--- a/apps/ca.c
+++ b/apps/ca.c
@@ -793,7 +793,7 @@ end_of_options:
             extensions = "default";
     }
 
-        /*****************************************************************/
+    /*****************************************************************/
     if (req || gencrl) {
         Sout = bio_open_default(outfile, "w");
         if (Sout == NULL)

diff --git a/apps/cms.c b/apps/cms.c
index e40686b5d4b38a67b280502a7b8a53ca0498a785..599f21774e75c84198680043afe96420ddc9c9ef 100644 (file)
--- a/apps/cms.c
+++ b/apps/cms.c
@@ -690,16 +690,14 @@ int cms_main(int argc, char **argv)
         flags &= ~CMS_DETACHED;
 
     if (operation & SMIME_OP) {
-        if (outformat == FORMAT_ASN1)
-            outmode = "wb";
+        outmode = WB(outformat);
     } else {
         if (flags & CMS_BINARY)
             outmode = "wb";
     }
 
     if (operation & SMIME_IP) {
-        if (informat == FORMAT_ASN1)
-            inmode = "rb";
+        inmode = RB(informat);
     } else {
         if (flags & CMS_BINARY)
             inmode = "rb";

diff --git a/apps/crl.c b/apps/crl.c
index c0bf8749d2655574ea007061a336d8f9eaa6dbab..1ea0c319c85ed81906aa017024c10cc3b30062e3 100644 (file)
--- a/apps/crl.c
+++ b/apps/crl.c
@@ -346,7 +346,7 @@ int crl_main(int argc, char **argv)
             }
         }
     }
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, WB(outformat));
     if (out == NULL)
         goto end;
 

diff --git a/apps/dhparam.c b/apps/dhparam.c
index 2e5ce2c6d34be483c14d4eaf9023ff731b354eb9..0640cf808b565c463f750124293ec487c4e94b9c 100644 (file)
--- a/apps/dhparam.c
+++ b/apps/dhparam.c
@@ -352,7 +352,7 @@ int dhparam_main(int argc, char **argv)
         /* dh != NULL */
     }
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, WB(outformat));
     if (out == NULL)
         goto end;
 

diff --git a/apps/dsa.c b/apps/dsa.c
index 9998bfe30a606545415a0432e376e8e71eb88030..4fca8526388afd0ab4cc6ef16fbfeb892beddd9f 100644 (file)
--- a/apps/dsa.c
+++ b/apps/dsa.c
@@ -225,7 +225,7 @@ int dsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/dsaparam.c b/apps/dsaparam.c
index 8d48313d2bd324859d9ee4f6fc2d1ad704057c35..d61bb70a703ff102644d48bc59eb270b0b8703d2 100644 (file)
--- a/apps/dsaparam.c
+++ b/apps/dsaparam.c
@@ -195,10 +195,10 @@ int dsaparam_main(int argc, char **argv)
     }
     private = genkey ? 1 : 0;
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, RB(informat));
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/enc.c b/apps/enc.c
index 18fcb9505dc4add9e285bc1c64f27fb8eefa1107..3b8d7eb26512d606bccfaf58ef94e5910747512c 100644 (file)
--- a/apps/enc.c
+++ b/apps/enc.c
@@ -330,7 +330,7 @@ int enc_main(int argc, char **argv)
         unbuffer(stdin);
         in = dup_bio_in();
     } else
-        in = bio_open_default(infile, "r");
+        in = bio_open_default(infile, base64 ? "r" : "rb");
     if (in == NULL)
         goto end;
 
@@ -366,7 +366,7 @@ int enc_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, base64 ? "w" : "wb");
     if (out == NULL)
         goto end;
 

diff --git a/apps/genpkey.c b/apps/genpkey.c
index dbbedaa35e45e615dc8d4156b36010c70abdd1da..c29b1947a674ffa7b4eaa3e6c368250eb9fb27a2 100644 (file)
--- a/apps/genpkey.c
+++ b/apps/genpkey.c
@@ -184,7 +184,7 @@ int genpkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, "wb", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/pkcs8.c b/apps/pkcs8.c
index b120b93aa9c3da7adb8b4e9571e6e37f13fd78d2..e3cb7750e1ae11ab1b0eac12673d153701c784c4 100644 (file)
--- a/apps/pkcs8.c
+++ b/apps/pkcs8.c
@@ -239,10 +239,10 @@ int pkcs8_main(int argc, char **argv)
     if ((pbe_nid == -1) && !cipher)
         pbe_nid = NID_pbeWithMD5AndDES_CBC;
 
-    in = bio_open_default(infile, "rb");
+    in = bio_open_default(infile, RB(informat));
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, "wb", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/pkey.c b/apps/pkey.c
index 80c2e154dd31d9a10dc21077cae69fce444313b8..9ef228f3a0934def409426a094aa3f2e664fcc9c 100644 (file)
--- a/apps/pkey.c
+++ b/apps/pkey.c
@@ -172,7 +172,7 @@ int pkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, "wb", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/rand.c b/apps/rand.c
index 432e784496d0d0d82586699366a5b9935122751a..a5aa5d94103256ce069f9a0266491eba15eb7941 100644 (file)
--- a/apps/rand.c
+++ b/apps/rand.c
@@ -134,7 +134,7 @@ int rand_main(int argc, char **argv)
         BIO_printf(bio_err, "%ld semi-random bytes loaded\n",
                    app_RAND_load_files(inrand));
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, base64 ? "w" : "wb");
     if (out == NULL)
         goto end;
 

diff --git a/apps/req.c b/apps/req.c
index 59cc6b466407b040fef140e20231c82fb8f0293a..bae3eeca0c990a2cce0da679709c55a2adc63bfb 100644 (file)
--- a/apps/req.c
+++ b/apps/req.c
@@ -764,7 +764,8 @@ int req_main(int argc, char **argv)
 
     out = bio_open_default(outfile,
                            keyout != NULL && outfile != NULL &&
-                           strcmp(keyout, outfile) == 0 ? "a" : "w");
+                           strcmp(keyout, outfile) == 0
+                           ? AB(outformat) : WB(outformat));
     if (out == NULL)
         goto end;
 

diff --git a/apps/rsa.c b/apps/rsa.c
index c7ad44b75d96097b028d4622f16f1dc80d0c3e91..01cc6ea18da6476017eb3806555c74f0ef9203af 100644 (file)
--- a/apps/rsa.c
+++ b/apps/rsa.c
@@ -292,7 +292,7 @@ int rsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, WB(outformat), private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/sess_id.c b/apps/sess_id.c
index 681654335ab1dde43dcec996965f4a272c35563b..e743791d23f5eb8eb438707031648bfbe1adc75a 100644 (file)
--- a/apps/sess_id.c
+++ b/apps/sess_id.c
@@ -160,10 +160,7 @@ int sess_id_main(int argc, char **argv)
     }
 
     if (!noout || text) {
-        const char* modeflag = "w";
-        if (outformat == FORMAT_ASN1 || outformat == FORMAT_NSS)
-            modeflag = "wb";
-        out = bio_open_default(outfile, modeflag);
+        out = bio_open_default(outfile, WB(outformat));
         if (out == NULL)
             goto end;
     }

diff --git a/apps/smime.c b/apps/smime.c
index 45898de4c14ddc39571c8f552c26d5ea340136e8..d597ebf53479669b200004ddc3f042cfb837b5c0 100644 (file)
--- a/apps/smime.c
+++ b/apps/smime.c
@@ -427,16 +427,14 @@ int smime_main(int argc, char **argv)
         flags &= ~PKCS7_DETACHED;
 
     if (operation & SMIME_OP) {
-        if (outformat == FORMAT_ASN1)
-            outmode = "wb";
+        outmode = WB(outformat);
     } else {
         if (flags & PKCS7_BINARY)
             outmode = "wb";
     }
 
     if (operation & SMIME_IP) {
-        if (informat == FORMAT_ASN1)
-            inmode = "rb";
+        inmode = RB(informat);
     } else {
         if (flags & PKCS7_BINARY)
             inmode = "rb";

diff --git a/apps/x509.c b/apps/x509.c
index acce9e9ddd2629699a56deeaaca28ba4afa7987c..8020e8a086a2ec98a3cb1a3a8428b733494b32ae 100644 (file)
--- a/apps/x509.c
+++ b/apps/x509.c
@@ -496,7 +496,7 @@ int x509_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, WB(outformat));
     if (out == NULL)
         goto end;
 
@@ -556,7 +556,7 @@ int x509_main(int argc, char **argv)
             BIO_printf(bio_err, "We need a private key to sign with\n");
             goto end;
         }
-        in = bio_open_default(infile, "r");
+        in = bio_open_default(infile, RB(informat));
         if (in == NULL)
             goto end;
         req = PEM_read_bio_X509_REQ(in, NULL, NULL, NULL);