From: Richard Levitte <levitte@openssl.org>
Date: Fri, 4 Sep 2015 10:49:06 +0000 (+0200)
Subject: Change the way apps open their input and output files
X-Git-Tag: OpenSSL_1_1_0-pre1~702
X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=bdd58d98467e9f0f6635c1628e1eae304383afb1;p=oweals%2Fopenssl.git

Change the way apps open their input and output files

The different apps had the liberty to decide whether they would open their
input and output files in binary mode or not, which could be confusing if
two different apps were handling the same type of file in different ways.

The solution is to centralise the decision of low level file organisation,
and that the apps would use a selection of formats to state the intent of
the file.

Reviewed-by: Tim Hudson <tjh@openssl.org>
---

diff --git a/apps/apps.c b/apps/apps.c
index 80e777774f..f3b2d48f3a 100644
--- a/apps/apps.c
+++ b/apps/apps.c
@@ -514,7 +514,7 @@ CONF *app_load_config(const char *filename)
     BIO *in;
     CONF *conf;
 
-    in = bio_open_default(filename, "r");
+    in = bio_open_default(filename, 'r', FORMAT_TEXT);
     if (in == NULL)
         return NULL;
 
@@ -527,7 +527,7 @@ CONF *app_load_config_quiet(const char *filename)
     BIO *in;
     CONF *conf;
 
-    in = bio_open_default_quiet(filename, "r");
+    in = bio_open_default_quiet(filename, 'r', FORMAT_TEXT);
     if (in == NULL)
         return NULL;
 
@@ -683,7 +683,7 @@ X509 *load_cert(const char *file, int format,
         unbuffer(stdin);
         cert = dup_bio_in();
     } else
-        cert = bio_open_default(file, RB(format));
+        cert = bio_open_default(file, 'r', format);
     if (cert == NULL)
         goto end;
 
@@ -718,7 +718,7 @@ X509_CRL *load_crl(const char *infile, int format)
         return x;
     }
 
-    in = bio_open_default(infile, RB(format));
+    in = bio_open_default(infile, 'r', format);
     if (in == NULL)
         goto end;
     if (format == FORMAT_ASN1)
@@ -772,7 +772,7 @@ EVP_PKEY *load_key(const char *file, int format, int maybe_stdin,
         unbuffer(stdin);
         key = dup_bio_in();
     } else
-        key = bio_open_default(file, RB(format));
+        key = bio_open_default(file, 'r', format);
     if (key == NULL)
         goto end;
     if (format == FORMAT_ASN1) {
@@ -808,13 +808,6 @@ EVP_PKEY *load_key(const char *file, int format, int maybe_stdin,
     return (pkey);
 }
 
-static const char *key_file_format(int format)
-{
-    if (format == FORMAT_PEM || format == FORMAT_PEMRSA)
-        return "r";
-    return "rb";
-}
-
 EVP_PKEY *load_pubkey(const char *file, int format, int maybe_stdin,
                       const char *pass, ENGINE *e, const char *key_descrip)
 {
@@ -842,7 +835,7 @@ EVP_PKEY *load_pubkey(const char *file, int format, int maybe_stdin,
         unbuffer(stdin);
         key = dup_bio_in();
     } else
-        key = bio_open_default(file, key_file_format(format));
+        key = bio_open_default(file, 'r', format);
     if (key == NULL)
         goto end;
     if (format == FORMAT_ASN1) {
@@ -909,7 +902,7 @@ static int load_certs_crls(const char *file, int format,
         return 0;
     }
 
-    bio = bio_open_default(file, "r");
+    bio = bio_open_default(file, 'r', FORMAT_PEM);
     if (bio == NULL)
         return 0;
 
diff --git a/apps/apps.h b/apps/apps.h
index cd70948313..c34d22ed1d 100644
--- a/apps/apps.h
+++ b/apps/apps.h
@@ -154,19 +154,14 @@ extern BIO *bio_out;
 extern BIO *bio_err;
 BIO *dup_bio_in(void);
 BIO *dup_bio_out(void);
-BIO *bio_open_owner(const char *filename, const char *mode, int private);
-BIO *bio_open_default(const char *filename, const char *mode);
-BIO *bio_open_default_quiet(const char *filename, const char *mode);
+BIO *bio_open_owner(const char *filename, int format, int private);
+BIO *bio_open_default(const char *filename, char mode, int format);
+BIO *bio_open_default_quiet(const char *filename, char mode, int format);
 CONF *app_load_config(const char *filename);
 CONF *app_load_config_quiet(const char *filename);
 int app_load_modules(const CONF *config);
 void unbuffer(FILE *fp);
 
-/* Often used in calls to bio_open_default. */
-# define RB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "rb" : "r")
-# define WB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "wb" : "w")
-# define AB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "ab" : "a")
-
 /*
  * Common verification options.
  */
@@ -536,14 +531,21 @@ void print_cert_checks(BIO *bio, X509 *x,
 void store_setup_crl_download(X509_STORE *st);
 
 /* See OPT_FMT_xxx, above. */
+/* On some platforms, it's important to distinguish between text and binary
+ * files.  On some, there might even be specific file formats for different
+ * contents.  The FORMAT_xxx macros are meant to express an intent with the
+ * file being read or created.
+ */
 # define B_FORMAT_TEXT   0x8000
 # define FORMAT_UNDEF    0
-# define FORMAT_ASN1     1
-# define FORMAT_TEXT    (2 | B_FORMAT_TEXT)
-# define FORMAT_PEM     (3 | B_FORMAT_TEXT)
-# define FORMAT_PKCS12   5
-# define FORMAT_SMIME   (6 | B_FORMAT_TEXT)
-# define FORMAT_ENGINE   7
+# define FORMAT_TEXT    (1 | B_FORMAT_TEXT)     /* Generic text */
+# define FORMAT_BINARY   2                      /* Generic binary */
+# define FORMAT_BASE64  (3 | B_FORMAT_TEXT)     /* Base64 */
+# define FORMAT_ASN1     4                      /* ASN.1/DER */
+# define FORMAT_PEM     (5 | B_FORMAT_TEXT)
+# define FORMAT_PKCS12   6
+# define FORMAT_SMIME   (7 | B_FORMAT_TEXT)
+# define FORMAT_ENGINE   8                      /* Not really a file format */
 # define FORMAT_PEMRSA  (9 | B_FORMAT_TEXT)     /* PEM RSAPubicKey format */
 # define FORMAT_ASN1RSA  10                     /* DER RSAPubicKey format */
 # define FORMAT_MSBLOB   11                     /* MS Key blob format */
diff --git a/apps/asn1pars.c b/apps/asn1pars.c
index 8881ad4e33..89afd5b15b 100644
--- a/apps/asn1pars.c
+++ b/apps/asn1pars.c
@@ -190,17 +190,17 @@ int asn1parse_main(int argc, char **argv)
         goto end;
 
     if (oidfile != NULL) {
-      in = bio_open_default(oidfile, "r");
+        in = bio_open_default(oidfile, 'r', FORMAT_TEXT);
         if (in == NULL)
             goto end;
         OBJ_create_objects(in);
         BIO_free(in);
     }
 
-    if ((in = bio_open_default(infile, RB(informat))) == NULL)
+    if ((in = bio_open_default(infile, 'r', informat)) == NULL)
         goto end;
 
-    if (derfile && (derout = bio_open_default(derfile, "wb")) == NULL)
+    if (derfile && (derout = bio_open_default(derfile, 'w', FORMAT_ASN1)) == NULL)
         goto end;
 
     if (strictpem) {
diff --git a/apps/ca.c b/apps/ca.c
index ce09155f28..defbf007f6 100644
--- a/apps/ca.c
+++ b/apps/ca.c
@@ -795,7 +795,8 @@ end_of_options:
 
     /*****************************************************************/
     if (req || gencrl) {
-        Sout = bio_open_default(outfile, "w");
+        /* FIXME: Is it really always text? */
+        Sout = bio_open_default(outfile, 'w', FORMAT_TEXT);
         if (Sout == NULL)
             goto end;
     }
diff --git a/apps/cms.c b/apps/cms.c
index 599f21774e..2331ea2e28 100644
--- a/apps/cms.c
+++ b/apps/cms.c
@@ -247,7 +247,6 @@ int cms_main(int argc, char **argv)
         NULL;
     char *to = NULL, *from = NULL, *subject = NULL, *prog;
     cms_key_param *key_first = NULL, *key_param = NULL;
-    const char *inmode = "r", *outmode = "w";
     int flags = CMS_DETACHED, noout = 0, print = 0, keyidx = -1, vpmtouched =
         0;
     int informat = FORMAT_SMIME, outformat = FORMAT_SMIME;
@@ -689,18 +688,14 @@ int cms_main(int argc, char **argv)
     if (!(operation & SMIME_SIGNERS))
         flags &= ~CMS_DETACHED;
 
-    if (operation & SMIME_OP) {
-        outmode = WB(outformat);
-    } else {
+    if (!(operation & SMIME_OP)) {
         if (flags & CMS_BINARY)
-            outmode = "wb";
+            outformat = FORMAT_BINARY;
     }
 
-    if (operation & SMIME_IP) {
-        inmode = RB(informat);
-    } else {
+    if (!(operation & SMIME_IP)) {
         if (flags & CMS_BINARY)
-            inmode = "rb";
+            informat = FORMAT_BINARY;
     }
 
     if (operation == SMIME_ENCRYPT) {
@@ -770,7 +765,7 @@ int cms_main(int argc, char **argv)
             goto end;
     }
 
-    in = bio_open_default(infile, inmode);
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -834,7 +829,7 @@ int cms_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, outmode);
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 
diff --git a/apps/crl.c b/apps/crl.c
index 1ea0c319c8..735c8c014f 100644
--- a/apps/crl.c
+++ b/apps/crl.c
@@ -346,7 +346,7 @@ int crl_main(int argc, char **argv)
             }
         }
     }
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 
diff --git a/apps/crl2p7.c b/apps/crl2p7.c
index e4e39cfe38..8cc1b62efe 100644
--- a/apps/crl2p7.c
+++ b/apps/crl2p7.c
@@ -152,7 +152,7 @@ int crl2pkcs7_main(int argc, char **argv)
         goto end;
 
     if (!nocrl) {
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
 
@@ -201,7 +201,7 @@ int crl2pkcs7_main(int argc, char **argv)
 
     sk_OPENSSL_STRING_free(certflst);
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 
diff --git a/apps/dgst.c b/apps/dgst.c
index e6142caaff..99568f42a6 100644
--- a/apps/dgst.c
+++ b/apps/dgst.c
@@ -275,7 +275,7 @@ int dgst_main(int argc, char **argv)
     if (randfile)
         app_RAND_load_file(randfile, 0);
 
-    out = bio_open_default(outfile, out_bin ? "wb" : "w");
+    out = bio_open_default(outfile, 'w', out_bin ? FORMAT_BINARY : FORMAT_TEXT);
     if (out == NULL)
         goto end;
 
diff --git a/apps/dhparam.c b/apps/dhparam.c
index 0640cf808b..334a129b1b 100644
--- a/apps/dhparam.c
+++ b/apps/dhparam.c
@@ -309,7 +309,7 @@ int dhparam_main(int argc, char **argv)
         app_RAND_write_file(NULL);
     } else {
 
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
 
@@ -352,7 +352,7 @@ int dhparam_main(int argc, char **argv)
         /* dh != NULL */
     }
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 
diff --git a/apps/dsa.c b/apps/dsa.c
index 4fca852638..d829f980fa 100644
--- a/apps/dsa.c
+++ b/apps/dsa.c
@@ -225,7 +225,7 @@ int dsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/dsaparam.c b/apps/dsaparam.c
index d61bb70a70..1ba93e603f 100644
--- a/apps/dsaparam.c
+++ b/apps/dsaparam.c
@@ -195,10 +195,10 @@ int dsaparam_main(int argc, char **argv)
     }
     private = genkey ? 1 : 0;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/ec.c b/apps/ec.c
index e4f2db39e6..a30d3f0a40 100644
--- a/apps/ec.c
+++ b/apps/ec.c
@@ -205,7 +205,7 @@ int ec_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -227,7 +227,7 @@ int ec_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/ecparam.c b/apps/ecparam.c
index 8464c882ca..145f55c0e6 100644
--- a/apps/ecparam.c
+++ b/apps/ecparam.c
@@ -223,10 +223,10 @@ int ecparam_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/enc.c b/apps/enc.c
index 3b8d7eb265..0bdba38161 100644
--- a/apps/enc.c
+++ b/apps/enc.c
@@ -138,7 +138,7 @@ int enc_main(int argc, char **argv)
     char mbuf[sizeof magic - 1];
     OPTION_CHOICE o;
     int bsize = BSIZE, verbose = 0, debug = 0, olb64 = 0, nosalt = 0;
-    int enc = 1, printkey = 0, i, k, base64 = 0;
+    int enc = 1, printkey = 0, i, k, format = FORMAT_BINARY;
     int ret = 1, inl, nopad = 0, non_fips_allow = 0;
     unsigned char key[EVP_MAX_KEY_LENGTH], iv[EVP_MAX_IV_LENGTH];
     unsigned char *buff = NULL, salt[PKCS5_SALT_LEN];
@@ -151,7 +151,7 @@ int enc_main(int argc, char **argv)
     /* first check the program name */
     prog = opt_progname(argv[0]);
     if (strcmp(prog, "base64") == 0)
-        base64 = 1;
+        format = FORMAT_BASE64;
 #ifdef ZLIB
     else if (strcmp(prog, "zlib") == 0)
         do_zlib = 1;
@@ -223,7 +223,7 @@ int enc_main(int argc, char **argv)
             olb64 = 1;
             break;
         case OPT_A:
-            base64 = 1;
+            format = FORMAT_BASE64;
             break;
         case OPT_Z:
 #ifdef ZLIB
@@ -246,7 +246,7 @@ int enc_main(int argc, char **argv)
             str = opt_arg();
             break;
         case OPT_KFILE:
-            in = bio_open_default(opt_arg(), "r");
+            in = bio_open_default(opt_arg(), 'r', FORMAT_TEXT);
             if (in == NULL)
                 goto opthelp;
             i = BIO_gets(in, buf, sizeof buf);
@@ -311,7 +311,7 @@ int enc_main(int argc, char **argv)
         dgst = EVP_md5();
 
     /* It must be large enough for a base64 encoded line */
-    if (base64 && bsize < 80)
+    if (format == FORMAT_BASE64 && bsize < 80)
         bsize = 80;
     if (verbose)
         BIO_printf(bio_err, "bufsize=%d\n", bsize);
@@ -330,7 +330,7 @@ int enc_main(int argc, char **argv)
         unbuffer(stdin);
         in = dup_bio_in();
     } else
-        in = bio_open_default(infile, base64 ? "r" : "rb");
+        in = bio_open_default(infile, 'r', format);
     if (in == NULL)
         goto end;
 
@@ -366,7 +366,7 @@ int enc_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, base64 ? "w" : "wb");
+    out = bio_open_default(outfile, 'w', format);
     if (out == NULL)
         goto end;
 
@@ -384,7 +384,7 @@ int enc_main(int argc, char **argv)
     }
 #endif
 
-    if (base64) {
+    if (format == FORMAT_BASE64) {
         if ((b64 = BIO_new(BIO_f_base64())) == NULL)
             goto end;
         if (debug) {
diff --git a/apps/gendsa.c b/apps/gendsa.c
index 087a44a1e8..f1e1f54b8a 100644
--- a/apps/gendsa.c
+++ b/apps/gendsa.c
@@ -147,7 +147,7 @@ int gendsa_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(dsaparams, "r");
+    in = bio_open_default(dsaparams, 'r', FORMAT_PEM);
     if (in == NULL)
         goto end2;
 
@@ -158,7 +158,7 @@ int gendsa_main(int argc, char **argv)
     BIO_free(in);
     in = NULL;
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, FORMAT_PEM, private);
     if (out == NULL)
         goto end2;
 
diff --git a/apps/genpkey.c b/apps/genpkey.c
index c29b1947a6..d80983350a 100644
--- a/apps/genpkey.c
+++ b/apps/genpkey.c
@@ -184,7 +184,7 @@ int genpkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/genrsa.c b/apps/genrsa.c
index 1fea351fb1..54484b5273 100644
--- a/apps/genrsa.c
+++ b/apps/genrsa.c
@@ -172,7 +172,7 @@ int genrsa_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, FORMAT_PEM, private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/nseq.c b/apps/nseq.c
index 5c8ed172c2..06893c82ce 100644
--- a/apps/nseq.c
+++ b/apps/nseq.c
@@ -112,10 +112,10 @@ int nseq_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, 'r', FORMAT_PEM);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_PEM);
     if (out == NULL)
         goto end;
 
diff --git a/apps/ocsp.c b/apps/ocsp.c
index 7193dae20b..e97d06e7c1 100644
--- a/apps/ocsp.c
+++ b/apps/ocsp.c
@@ -486,7 +486,7 @@ int ocsp_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_TEXT);
     if (out == NULL)
         goto end;
 
@@ -494,7 +494,7 @@ int ocsp_main(int argc, char **argv)
         add_nonce = 0;
 
     if (!req && reqin) {
-        derbio = bio_open_default(reqin, "rb");
+        derbio = bio_open_default(reqin, 'r', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         req = d2i_OCSP_REQUEST_bio(derbio, NULL);
@@ -589,7 +589,7 @@ int ocsp_main(int argc, char **argv)
         OCSP_REQUEST_print(out, req, 0);
 
     if (reqout) {
-        derbio = bio_open_default(reqout, "wb");
+        derbio = bio_open_default(reqout, 'w', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         i2d_OCSP_REQUEST_bio(derbio, req);
@@ -627,7 +627,7 @@ int ocsp_main(int argc, char **argv)
         goto end;
 # endif
     } else if (respin) {
-        derbio = bio_open_default(respin, "rb");
+        derbio = bio_open_default(respin, 'r', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         resp = d2i_OCSP_RESPONSE_bio(derbio, NULL);
@@ -644,7 +644,7 @@ int ocsp_main(int argc, char **argv)
  done_resp:
 
     if (respout) {
-        derbio = bio_open_default(respout, "wb");
+        derbio = bio_open_default(respout, 'w', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         i2d_OCSP_RESPONSE_bio(derbio, resp);
diff --git a/apps/openssl.c b/apps/openssl.c
index 7c202cf8d1..7e340be21a 100644
--- a/apps/openssl.c
+++ b/apps/openssl.c
@@ -299,17 +299,46 @@ void unbuffer(FILE *fp)
     setbuf(fp, NULL);
 }
 
+static const char *modestr(char mode, int format)
+{
+    OPENSSL_assert(mode == 'a' || mode == 'r' || mode == 'w');
+
+    switch (mode) {
+    case 'a':
+        return (format) & B_FORMAT_TEXT ? "ab" : "a";
+    case 'r':
+        return (format) & B_FORMAT_TEXT ? "rb" : "r";
+    case 'w':
+        return (format) & B_FORMAT_TEXT ? "wb" : "w";
+    }
+    /* The assert above should make sure we never reach this point */
+    return NULL;
+}
+
+static const char *modeverb(char mode)
+{
+    switch (mode) {
+    case 'a':
+        return "appending";
+    case 'r':
+        return "reading";
+    case 'w':
+        return "writing";
+    }
+    return "(doing something)";
+}
+
 /*
  * Open a file for writing, owner-read-only.
  */
-BIO *bio_open_owner(const char *filename, const char *modestr, int private)
+BIO *bio_open_owner(const char *filename, int format, int private)
 {
     FILE *fp = NULL;
     BIO *b = NULL;
     int fd = -1, bflags, mode, binmode;
 
     if (!private || filename == NULL || strcmp(filename, "-") == 0)
-        return bio_open_default(filename, modestr);
+        return bio_open_default(filename, 'w', format);
 
     mode = O_WRONLY;
 #ifdef O_CREAT
@@ -318,7 +347,7 @@ BIO *bio_open_owner(const char *filename, const char *modestr, int private)
 #ifdef O_TRUNC
     mode |= O_TRUNC;
 #endif
-    binmode = strchr(modestr, 'b') != NULL;
+    binmode = !(format & B_FORMAT_TEXT);
     if (binmode) {
 #ifdef O_BINARY
         mode |= O_BINARY;
@@ -330,7 +359,7 @@ BIO *bio_open_owner(const char *filename, const char *modestr, int private)
     fd = open(filename, mode, 0600);
     if (fd < 0)
         goto err;
-    fp = fdopen(fd, modestr);
+    fp = fdopen(fd, modestr('w', format));
     if (fp == NULL)
         goto err;
     bflags = BIO_CLOSE;
@@ -352,12 +381,13 @@ BIO *bio_open_owner(const char *filename, const char *modestr, int private)
     return NULL;
 }
 
-static BIO *bio_open_default_(const char *filename, const char *mode, int quiet)
+static BIO *bio_open_default_(const char *filename, char mode, int format,
+                              int quiet)
 {
     BIO *ret;
 
     if (filename == NULL || strcmp(filename, "-") == 0) {
-        ret = *mode == 'r' ? dup_bio_in() : dup_bio_out();
+        ret = mode == 'r' ? dup_bio_in() : dup_bio_out();
         if (quiet) {
             ERR_clear_error();
             return ret;
@@ -366,9 +396,9 @@ static BIO *bio_open_default_(const char *filename, const char *mode, int quiet)
             return ret;
         BIO_printf(bio_err,
                    "Can't open %s, %s\n",
-                   *mode == 'r' ? "stdin" : "stdout", strerror(errno));
+                   mode == 'r' ? "stdin" : "stdout", strerror(errno));
     } else {
-        ret = BIO_new_file(filename, mode);
+        ret = BIO_new_file(filename, modestr(mode, format));
         if (quiet) {
             ERR_clear_error();
             return ret;
@@ -377,21 +407,20 @@ static BIO *bio_open_default_(const char *filename, const char *mode, int quiet)
             return ret;
         BIO_printf(bio_err,
                    "Can't open %s for %s, %s\n",
-                   filename,
-                   *mode == 'r' ? "reading" : "writing", strerror(errno));
+                   filename, modeverb(mode), strerror(errno));
     }
     ERR_print_errors(bio_err);
     return NULL;
 }
 
-BIO *bio_open_default(const char *filename, const char *mode)
+BIO *bio_open_default(const char *filename, char mode, int format)
 {
-    return bio_open_default_(filename, mode, 0);
+    return bio_open_default_(filename, mode, format, 0);
 }
 
-BIO *bio_open_default_quiet(const char *filename, const char *mode)
+BIO *bio_open_default_quiet(const char *filename, char mode, int format)
 {
-    return bio_open_default_(filename, mode, 1);
+    return bio_open_default_(filename, mode, format, 1);
 }
 
 #if defined( OPENSSL_SYS_VMS)
diff --git a/apps/passwd.c b/apps/passwd.c
index dbae620645..8988313375 100644
--- a/apps/passwd.c
+++ b/apps/passwd.c
@@ -209,7 +209,7 @@ int passwd_main(int argc, char **argv)
         goto end;
     }
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, 'r', FORMAT_TEXT);
     if (in == NULL)
         goto end;
 
diff --git a/apps/pkcs12.c b/apps/pkcs12.c
index 5b14dd5320..2e74cd4bae 100644
--- a/apps/pkcs12.c
+++ b/apps/pkcs12.c
@@ -353,13 +353,6 @@ int pkcs12_main(int argc, char **argv)
                        app_RAND_load_files(inrand));
     }
 
-    in = bio_open_default(infile, "rb");
-    if (in == NULL)
-        goto end;
-    out = bio_open_owner(outfile, "wb", private);
-    if (out == NULL)
-        goto end;
-
     if (twopass) {
         if (EVP_read_pw_string
             (macpass, sizeof macpass, "Enter MAC Password:", export_cert)) {
@@ -501,6 +494,11 @@ int pkcs12_main(int argc, char **argv)
             PKCS12_set_mac(p12, mpass, -1, NULL, 0, maciter, macmd);
 
         assert(private);
+
+        out = bio_open_owner(outfile, FORMAT_PKCS12, private);
+        if (out == NULL)
+            goto end;
+
         i2d_PKCS12_bio(out, p12);
 
         ret = 0;
@@ -515,6 +513,13 @@ int pkcs12_main(int argc, char **argv)
 
     }
 
+    in = bio_open_default(infile, 'r', FORMAT_PKCS12);
+    if (in == NULL)
+        goto end;
+    out = bio_open_owner(outfile, FORMAT_PEM, private);
+    if (out == NULL)
+        goto end;
+
     if ((p12 = d2i_PKCS12_bio(in, NULL)) == NULL) {
         ERR_print_errors(bio_err);
         goto end;
diff --git a/apps/pkcs7.c b/apps/pkcs7.c
index 248e0d6db2..fff14dc959 100644
--- a/apps/pkcs7.c
+++ b/apps/pkcs7.c
@@ -196,7 +196,7 @@ int pkcs7_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -210,7 +210,7 @@ int pkcs7_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 
diff --git a/apps/pkcs8.c b/apps/pkcs8.c
index e3cb7750e1..765744ffbe 100644
--- a/apps/pkcs8.c
+++ b/apps/pkcs8.c
@@ -239,10 +239,10 @@ int pkcs8_main(int argc, char **argv)
     if ((pbe_nid == -1) && !cipher)
         pbe_nid = NID_pbeWithMD5AndDES_CBC;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/pkey.c b/apps/pkey.c
index 9ef228f3a0..d2a66eb4a0 100644
--- a/apps/pkey.c
+++ b/apps/pkey.c
@@ -172,7 +172,7 @@ int pkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/pkeyparam.c b/apps/pkeyparam.c
index 6039dad9d1..215611eb34 100644
--- a/apps/pkeyparam.c
+++ b/apps/pkeyparam.c
@@ -121,10 +121,10 @@ int pkeyparam_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, 'r', FORMAT_PEM);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_PEM);
     if (out == NULL)
         goto end;
     pkey = PEM_read_bio_Parameters(in, NULL);
diff --git a/apps/pkeyutl.c b/apps/pkeyutl.c
index 741dd642b1..c3e18895a7 100644
--- a/apps/pkeyutl.c
+++ b/apps/pkeyutl.c
@@ -249,11 +249,11 @@ int pkeyutl_main(int argc, char **argv)
     app_RAND_load_file(NULL, 0);
 
     if (pkey_op != EVP_PKEY_OP_DERIVE) {
-        in = bio_open_default(infile, "rb");
+        in = bio_open_default(infile, 'r', FORMAT_BINARY);
         if (in == NULL)
             goto end;
     }
-    out = bio_open_default(outfile, "wb");
+    out = bio_open_default(outfile, 'w', FORMAT_BINARY);
     if (out == NULL)
         goto end;
 
diff --git a/apps/rand.c b/apps/rand.c
index a5aa5d9410..315e6be02f 100644
--- a/apps/rand.c
+++ b/apps/rand.c
@@ -87,7 +87,7 @@ int rand_main(int argc, char **argv)
     BIO *out = NULL;
     char *inrand = NULL, *outfile = NULL, *prog;
     OPTION_CHOICE o;
-    int base64 = 0, hex = 0, i, num = -1, r, ret = 1;
+    int format = FORMAT_BINARY, i, num = -1, r, ret = 1;
 
     prog = opt_init(argc, argv, rand_options);
     while ((o = opt_next()) != OPT_EOF) {
@@ -111,17 +111,17 @@ int rand_main(int argc, char **argv)
             inrand = opt_arg();
             break;
         case OPT_BASE64:
-            base64 = 1;
+            format = FORMAT_BASE64;
             break;
         case OPT_HEX:
-            hex = 1;
+            format = FORMAT_TEXT;
             break;
         }
     }
     argc = opt_num_rest();
     argv = opt_rest();
 
-    if (argc != 1 || (hex && base64))
+    if (argc != 1)
         goto opthelp;
     if (sscanf(argv[0], "%d", &num) != 1 || num < 0)
         goto opthelp;
@@ -134,11 +134,11 @@ int rand_main(int argc, char **argv)
         BIO_printf(bio_err, "%ld semi-random bytes loaded\n",
                    app_RAND_load_files(inrand));
 
-    out = bio_open_default(outfile, base64 ? "w" : "wb");
+    out = bio_open_default(outfile, 'w', format);
     if (out == NULL)
         goto end;
 
-    if (base64) {
+    if (format == FORMAT_BASE64) {
         BIO *b64 = BIO_new(BIO_f_base64());
         if (b64 == NULL)
             goto end;
@@ -155,7 +155,7 @@ int rand_main(int argc, char **argv)
         r = RAND_bytes(buf, chunk);
         if (r <= 0)
             goto end;
-        if (!hex)
+        if (format != FORMAT_TEXT) /* hex */
             BIO_write(out, buf, chunk);
         else {
             for (i = 0; i < chunk; i++)
@@ -163,7 +163,7 @@ int rand_main(int argc, char **argv)
         }
         num -= chunk;
     }
-    if (hex)
+    if (format == FORMAT_TEXT)
         BIO_puts(out, "\n");
     (void)BIO_flush(out);
 
diff --git a/apps/req.c b/apps/req.c
index bae3eeca0c..ce0fcbcb3b 100644
--- a/apps/req.c
+++ b/apps/req.c
@@ -566,7 +566,7 @@ int req_main(int argc, char **argv)
             BIO_printf(bio_err, "writing new private key to stdout\n");
         else
             BIO_printf(bio_err, "writing new private key to '%s'\n", keyout);
-        out = bio_open_owner(keyout, "w", private);
+        out = bio_open_owner(keyout, outformat, private);
         if (out == NULL)
             goto end;
 
@@ -601,7 +601,7 @@ int req_main(int argc, char **argv)
     }
 
     if (!newreq) {
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
 
@@ -764,8 +764,8 @@ int req_main(int argc, char **argv)
 
     out = bio_open_default(outfile,
                            keyout != NULL && outfile != NULL &&
-                           strcmp(keyout, outfile) == 0
-                           ? AB(outformat) : WB(outformat));
+                           strcmp(keyout, outfile) == 0 ? 'a' : 'w',
+                           outformat);
     if (out == NULL)
         goto end;
 
diff --git a/apps/rsa.c b/apps/rsa.c
index 01cc6ea18d..f8a0deceb6 100644
--- a/apps/rsa.c
+++ b/apps/rsa.c
@@ -292,7 +292,7 @@ int rsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 
diff --git a/apps/rsautl.c b/apps/rsautl.c
index 8ba838b43f..84a1de1ae9 100644
--- a/apps/rsautl.c
+++ b/apps/rsautl.c
@@ -251,10 +251,10 @@ int rsautl_main(int argc, char **argv)
         goto end;
     }
 
-    in = bio_open_default(infile, "rb");
+    in = bio_open_default(infile, 'r', FORMAT_BINARY);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "wb");
+    out = bio_open_default(outfile, 'w', FORMAT_BINARY);
     if (out == NULL)
         goto end;
 
diff --git a/apps/sess_id.c b/apps/sess_id.c
index e743791d23..39711e2150 100644
--- a/apps/sess_id.c
+++ b/apps/sess_id.c
@@ -160,7 +160,7 @@ int sess_id_main(int argc, char **argv)
     }
 
     if (!noout || text) {
-        out = bio_open_default(outfile, WB(outformat));
+        out = bio_open_default(outfile, 'w', outformat);
         if (out == NULL)
             goto end;
     }
@@ -217,7 +217,7 @@ static SSL_SESSION *load_sess_id(char *infile, int format)
     SSL_SESSION *x = NULL;
     BIO *in = NULL;
 
-    in = bio_open_default(infile, RB(format));
+    in = bio_open_default(infile, 'r', format);
     if (in == NULL)
         goto end;
     if (format == FORMAT_ASN1)
diff --git a/apps/smime.c b/apps/smime.c
index d597ebf534..4da56cdf08 100644
--- a/apps/smime.c
+++ b/apps/smime.c
@@ -170,7 +170,6 @@ int smime_main(int argc, char **argv)
         NULL;
     char *passinarg = NULL, *passin = NULL, *to = NULL, *from =
         NULL, *subject = NULL;
-    const char *inmode = "r", *outmode = "w";
     OPTION_CHOICE o;
     int flags = PKCS7_DETACHED, operation = 0, ret = 0, need_rand = 0, indef =
         0;
@@ -426,18 +425,14 @@ int smime_main(int argc, char **argv)
     if (!(operation & SMIME_SIGNERS))
         flags &= ~PKCS7_DETACHED;
 
-    if (operation & SMIME_OP) {
-        outmode = WB(outformat);
-    } else {
+    if (!(operation & SMIME_OP)) {
         if (flags & PKCS7_BINARY)
-            outmode = "wb";
+            outformat = FORMAT_BINARY;
     }
 
-    if (operation & SMIME_IP) {
-        inmode = RB(informat);
-    } else {
+    if (!(operation & SMIME_IP)) {
         if (flags & PKCS7_BINARY)
-            inmode = "rb";
+            informat = FORMAT_BINARY;
     }
 
     if (operation == SMIME_ENCRYPT) {
@@ -494,7 +489,7 @@ int smime_main(int argc, char **argv)
             goto end;
     }
 
-    in = bio_open_default(infile, inmode);
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -523,7 +518,7 @@ int smime_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, outmode);
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 
diff --git a/apps/spkac.c b/apps/spkac.c
index d41331caba..180f80fcb7 100644
--- a/apps/spkac.c
+++ b/apps/spkac.c
@@ -175,7 +175,7 @@ int spkac_main(int argc, char **argv)
         NETSCAPE_SPKI_sign(spki, pkey, EVP_md5());
         spkstr = NETSCAPE_SPKI_b64_encode(spki);
 
-        out = bio_open_default(outfile, "w");
+        out = bio_open_default(outfile, 'w', FORMAT_TEXT);
         if (out == NULL)
             goto end;
         BIO_printf(out, "SPKAC=%s\n", spkstr);
@@ -205,7 +205,7 @@ int spkac_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_TEXT);
     if (out == NULL)
         goto end;
 
diff --git a/apps/ts.c b/apps/ts.c
index 6e6b834401..70729c5629 100644
--- a/apps/ts.c
+++ b/apps/ts.c
@@ -429,13 +429,13 @@ static int query_command(const char *data, char *digest, const EVP_MD *md,
 
     /* Build query object either from file or from scratch. */
     if (in != NULL) {
-        if ((in_bio = BIO_new_file(in, "rb")) == NULL)
+        if ((in_bio = bio_open_default(in, 'r', FORMAT_ASN1)) == NULL)
             goto end;
         query = d2i_TS_REQ_bio(in_bio, NULL);
     } else {
         /* Open the file if no explicit digest bytes were specified. */
         if (digest == NULL
-            && (data_bio = bio_open_default(data, "rb")) == NULL)
+            && (data_bio = bio_open_default(data, 'r', FORMAT_ASN1)) == NULL)
             goto end;
         query = create_query(data_bio, digest, md, policy, no_nonce, cert);
     }
@@ -443,14 +443,16 @@ static int query_command(const char *data, char *digest, const EVP_MD *md,
         goto end;
 
     /* Write query either in ASN.1 or in text format. */
-    if ((out_bio = bio_open_default(out, "wb")) == NULL)
-        goto end;
     if (text) {
         /* Text output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_TEXT)) == NULL)
+            goto end;
         if (!TS_REQ_print_bio(out_bio, query))
             goto end;
     } else {
         /* ASN.1 output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_ASN1)) == NULL)
+            goto end;
         if (!i2d_TS_REQ_bio(out_bio, query))
             goto end;
     }
@@ -662,10 +664,10 @@ static int reply_command(CONF *conf, char *section, char *engine,
         goto end;
 
     /* Write response either in ASN.1 or text format. */
-    if ((out_bio = bio_open_default(out, "wb")) == NULL)
-        goto end;
     if (text) {
         /* Text output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_TEXT)) == NULL)
+        goto end;
         if (token_out) {
             TS_TST_INFO *tst_info = TS_RESP_get_tst_info(response);
             if (!TS_TST_INFO_print_bio(out_bio, tst_info))
@@ -676,6 +678,8 @@ static int reply_command(CONF *conf, char *section, char *engine,
         }
     } else {
         /* ASN.1 DER output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_ASN1)) == NULL)
+            goto end;
         if (token_out) {
             PKCS7 *token = TS_RESP_get_token(response);
             if (!i2d_PKCS7_bio(out_bio, token))
diff --git a/apps/x509.c b/apps/x509.c
index 8020e8a086..9472b68da5 100644
--- a/apps/x509.c
+++ b/apps/x509.c
@@ -496,7 +496,7 @@ int x509_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 
@@ -556,7 +556,7 @@ int x509_main(int argc, char **argv)
             BIO_printf(bio_err, "We need a private key to sign with\n");
             goto end;
         }
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
         req = PEM_read_bio_X509_REQ(in, NULL, NULL, NULL);