Change the way apps open their input and output files

The different apps had the liberty to decide whether they would open their
input and output files in binary mode or not, which could be confusing if
two different apps were handling the same type of file in different ways.

The solution is to centralise the decision of low level file organisation,
and that the apps would use a selection of formats to state the intent of
the file.

Reviewed-by: Tim Hudson <tjh@openssl.org>

diff --git a/apps/apps.c b/apps/apps.c
index 80e777774fe677759ded90fabc2208ac0765248a..f3b2d48f3a7b21572259b2c3c7fd8970efce536d 100644 (file)
--- a/apps/apps.c
+++ b/apps/apps.c
@@ -514,7 +514,7 @@ CONF *app_load_config(const char *filename)
     BIO *in;
     CONF *conf;
 
-    in = bio_open_default(filename, "r");
+    in = bio_open_default(filename, 'r', FORMAT_TEXT);
     if (in == NULL)
         return NULL;
 
@@ -527,7 +527,7 @@ CONF *app_load_config_quiet(const char *filename)
     BIO *in;
     CONF *conf;
 
-    in = bio_open_default_quiet(filename, "r");
+    in = bio_open_default_quiet(filename, 'r', FORMAT_TEXT);
     if (in == NULL)
         return NULL;
 
@@ -683,7 +683,7 @@ X509 *load_cert(const char *file, int format,
         unbuffer(stdin);
         cert = dup_bio_in();
     } else
-        cert = bio_open_default(file, RB(format));
+        cert = bio_open_default(file, 'r', format);
     if (cert == NULL)
         goto end;
 
@@ -718,7 +718,7 @@ X509_CRL *load_crl(const char *infile, int format)
         return x;
     }
 
-    in = bio_open_default(infile, RB(format));
+    in = bio_open_default(infile, 'r', format);
     if (in == NULL)
         goto end;
     if (format == FORMAT_ASN1)
@@ -772,7 +772,7 @@ EVP_PKEY *load_key(const char *file, int format, int maybe_stdin,
         unbuffer(stdin);
         key = dup_bio_in();
     } else
-        key = bio_open_default(file, RB(format));
+        key = bio_open_default(file, 'r', format);
     if (key == NULL)
         goto end;
     if (format == FORMAT_ASN1) {
@@ -808,13 +808,6 @@ EVP_PKEY *load_key(const char *file, int format, int maybe_stdin,
     return (pkey);
 }
 
-static const char *key_file_format(int format)
-{
-    if (format == FORMAT_PEM || format == FORMAT_PEMRSA)
-        return "r";
-    return "rb";
-}
-
 EVP_PKEY *load_pubkey(const char *file, int format, int maybe_stdin,
                       const char *pass, ENGINE *e, const char *key_descrip)
 {
@@ -842,7 +835,7 @@ EVP_PKEY *load_pubkey(const char *file, int format, int maybe_stdin,
         unbuffer(stdin);
         key = dup_bio_in();
     } else
-        key = bio_open_default(file, key_file_format(format));
+        key = bio_open_default(file, 'r', format);
     if (key == NULL)
         goto end;
     if (format == FORMAT_ASN1) {
@@ -909,7 +902,7 @@ static int load_certs_crls(const char *file, int format,
         return 0;
     }
 
-    bio = bio_open_default(file, "r");
+    bio = bio_open_default(file, 'r', FORMAT_PEM);
     if (bio == NULL)
         return 0;
 

diff --git a/apps/apps.h b/apps/apps.h
index cd709483136225431d956fecfb2780e824240301..c34d22ed1dbb10801acc1fc71810aa45d95cd1d5 100644 (file)
--- a/apps/apps.h
+++ b/apps/apps.h
@@ -154,19 +154,14 @@ extern BIO *bio_out;
 extern BIO *bio_err;
 BIO *dup_bio_in(void);
 BIO *dup_bio_out(void);
-BIO *bio_open_owner(const char *filename, const char *mode, int private);
-BIO *bio_open_default(const char *filename, const char *mode);
-BIO *bio_open_default_quiet(const char *filename, const char *mode);
+BIO *bio_open_owner(const char *filename, int format, int private);
+BIO *bio_open_default(const char *filename, char mode, int format);
+BIO *bio_open_default_quiet(const char *filename, char mode, int format);
 CONF *app_load_config(const char *filename);
 CONF *app_load_config_quiet(const char *filename);
 int app_load_modules(const CONF *config);
 void unbuffer(FILE *fp);
 
-/* Often used in calls to bio_open_default. */
-# define RB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "rb" : "r")
-# define WB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "wb" : "w")
-# define AB(xformat)  (((xformat) & B_FORMAT_TEXT) ? "ab" : "a")
-
 /*
  * Common verification options.
  */
@@ -536,14 +531,21 @@ void print_cert_checks(BIO *bio, X509 *x,
 void store_setup_crl_download(X509_STORE *st);
 
 /* See OPT_FMT_xxx, above. */
+/* On some platforms, it's important to distinguish between text and binary
+ * files.  On some, there might even be specific file formats for different
+ * contents.  The FORMAT_xxx macros are meant to express an intent with the
+ * file being read or created.
+ */
 # define B_FORMAT_TEXT   0x8000
 # define FORMAT_UNDEF    0
-# define FORMAT_ASN1     1
-# define FORMAT_TEXT    (2 | B_FORMAT_TEXT)
-# define FORMAT_PEM     (3 | B_FORMAT_TEXT)
-# define FORMAT_PKCS12   5
-# define FORMAT_SMIME   (6 | B_FORMAT_TEXT)
-# define FORMAT_ENGINE   7
+# define FORMAT_TEXT    (1 | B_FORMAT_TEXT)     /* Generic text */
+# define FORMAT_BINARY   2                      /* Generic binary */
+# define FORMAT_BASE64  (3 | B_FORMAT_TEXT)     /* Base64 */
+# define FORMAT_ASN1     4                      /* ASN.1/DER */
+# define FORMAT_PEM     (5 | B_FORMAT_TEXT)
+# define FORMAT_PKCS12   6
+# define FORMAT_SMIME   (7 | B_FORMAT_TEXT)
+# define FORMAT_ENGINE   8                      /* Not really a file format */
 # define FORMAT_PEMRSA  (9 | B_FORMAT_TEXT)     /* PEM RSAPubicKey format */
 # define FORMAT_ASN1RSA  10                     /* DER RSAPubicKey format */
 # define FORMAT_MSBLOB   11                     /* MS Key blob format */

diff --git a/apps/asn1pars.c b/apps/asn1pars.c
index 8881ad4e33789f77521610a1840bbf545f9d74be..89afd5b15b379037fff3e3043dc22ac0ec995f9c 100644 (file)
--- a/apps/asn1pars.c
+++ b/apps/asn1pars.c
@@ -190,17 +190,17 @@ int asn1parse_main(int argc, char **argv)
         goto end;
 
     if (oidfile != NULL) {
-      in = bio_open_default(oidfile, "r");
+        in = bio_open_default(oidfile, 'r', FORMAT_TEXT);
         if (in == NULL)
             goto end;
         OBJ_create_objects(in);
         BIO_free(in);
     }
 
-    if ((in = bio_open_default(infile, RB(informat))) == NULL)
+    if ((in = bio_open_default(infile, 'r', informat)) == NULL)
         goto end;
 
-    if (derfile && (derout = bio_open_default(derfile, "wb")) == NULL)
+    if (derfile && (derout = bio_open_default(derfile, 'w', FORMAT_ASN1)) == NULL)
         goto end;
 
     if (strictpem) {

diff --git a/apps/ca.c b/apps/ca.c
index ce09155f2894ecf3ffc50ba26b67449e4faa09e2..defbf007f6732ed1537feb92b2673fd6f190b427 100644 (file)
--- a/apps/ca.c
+++ b/apps/ca.c
@@ -795,7 +795,8 @@ end_of_options:
 
     /*****************************************************************/
     if (req || gencrl) {
-        Sout = bio_open_default(outfile, "w");
+        /* FIXME: Is it really always text? */
+        Sout = bio_open_default(outfile, 'w', FORMAT_TEXT);
         if (Sout == NULL)
             goto end;
     }

diff --git a/apps/cms.c b/apps/cms.c
index 599f21774e75c84198680043afe96420ddc9c9ef..2331ea2e28c25a07b21beeb37cbc5ddb313b7dfb 100644 (file)
--- a/apps/cms.c
+++ b/apps/cms.c
@@ -247,7 +247,6 @@ int cms_main(int argc, char **argv)
         NULL;
     char *to = NULL, *from = NULL, *subject = NULL, *prog;
     cms_key_param *key_first = NULL, *key_param = NULL;
-    const char *inmode = "r", *outmode = "w";
     int flags = CMS_DETACHED, noout = 0, print = 0, keyidx = -1, vpmtouched =
         0;
     int informat = FORMAT_SMIME, outformat = FORMAT_SMIME;
@@ -689,18 +688,14 @@ int cms_main(int argc, char **argv)
     if (!(operation & SMIME_SIGNERS))
         flags &= ~CMS_DETACHED;
 
-    if (operation & SMIME_OP) {
-        outmode = WB(outformat);
-    } else {
+    if (!(operation & SMIME_OP)) {
         if (flags & CMS_BINARY)
-            outmode = "wb";
+            outformat = FORMAT_BINARY;
     }
 
-    if (operation & SMIME_IP) {
-        inmode = RB(informat);
-    } else {
+    if (!(operation & SMIME_IP)) {
         if (flags & CMS_BINARY)
-            inmode = "rb";
+            informat = FORMAT_BINARY;
     }
 
     if (operation == SMIME_ENCRYPT) {
@@ -770,7 +765,7 @@ int cms_main(int argc, char **argv)
             goto end;
     }
 
-    in = bio_open_default(infile, inmode);
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -834,7 +829,7 @@ int cms_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, outmode);
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

diff --git a/apps/crl.c b/apps/crl.c
index 1ea0c319c85ed81906aa017024c10cc3b30062e3..735c8c014fcf095d3cea476cf5570a33d95331ab 100644 (file)
--- a/apps/crl.c
+++ b/apps/crl.c
@@ -346,7 +346,7 @@ int crl_main(int argc, char **argv)
             }
         }
     }
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

diff --git a/apps/crl2p7.c b/apps/crl2p7.c
index e4e39cfe38672cb5d0bdda3617b9ea9ef99b9d6e..8cc1b62efe0485553ede7928b2a448e24b8261a8 100644 (file)
--- a/apps/crl2p7.c
+++ b/apps/crl2p7.c
@@ -152,7 +152,7 @@ int crl2pkcs7_main(int argc, char **argv)
         goto end;
 
     if (!nocrl) {
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
 
@@ -201,7 +201,7 @@ int crl2pkcs7_main(int argc, char **argv)
 
     sk_OPENSSL_STRING_free(certflst);
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

diff --git a/apps/dgst.c b/apps/dgst.c
index e6142caaffca21e791949a5f6e489a159705ff0e..99568f42a6279d29e393158f82fa504a93aa5352 100644 (file)
--- a/apps/dgst.c
+++ b/apps/dgst.c
@@ -275,7 +275,7 @@ int dgst_main(int argc, char **argv)
     if (randfile)
         app_RAND_load_file(randfile, 0);
 
-    out = bio_open_default(outfile, out_bin ? "wb" : "w");
+    out = bio_open_default(outfile, 'w', out_bin ? FORMAT_BINARY : FORMAT_TEXT);
     if (out == NULL)
         goto end;
 

diff --git a/apps/dhparam.c b/apps/dhparam.c
index 0640cf808b565c463f750124293ec487c4e94b9c..334a129b1bf80c98d1ae00de92c2597f5e0b375e 100644 (file)
--- a/apps/dhparam.c
+++ b/apps/dhparam.c
@@ -309,7 +309,7 @@ int dhparam_main(int argc, char **argv)
         app_RAND_write_file(NULL);
     } else {
 
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
 
@@ -352,7 +352,7 @@ int dhparam_main(int argc, char **argv)
         /* dh != NULL */
     }
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

diff --git a/apps/dsa.c b/apps/dsa.c
index 4fca8526388afd0ab4cc6ef16fbfeb892beddd9f..d829f980fa4b55b5eaced7539486157ab1ce8ebe 100644 (file)
--- a/apps/dsa.c
+++ b/apps/dsa.c
@@ -225,7 +225,7 @@ int dsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/dsaparam.c b/apps/dsaparam.c
index d61bb70a703ff102644d48bc59eb270b0b8703d2..1ba93e603f7a2bc39fc6c1cb0d664e974c78e970 100644 (file)
--- a/apps/dsaparam.c
+++ b/apps/dsaparam.c
@@ -195,10 +195,10 @@ int dsaparam_main(int argc, char **argv)
     }
     private = genkey ? 1 : 0;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/ec.c b/apps/ec.c
index e4f2db39e60b53be42dd547410c7ccaf467b7f41..a30d3f0a40db4a31d518245e79992a93036cc844 100644 (file)
--- a/apps/ec.c
+++ b/apps/ec.c
@@ -205,7 +205,7 @@ int ec_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -227,7 +227,7 @@ int ec_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/ecparam.c b/apps/ecparam.c
index 8464c882ca0f0206f5cb0c0f1bc731080910c86c..145f55c0e69bbb7323ecb06693cc0b54608e54f8 100644 (file)
--- a/apps/ecparam.c
+++ b/apps/ecparam.c
@@ -223,10 +223,10 @@ int ecparam_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/enc.c b/apps/enc.c
index 3b8d7eb26512d606bccfaf58ef94e5910747512c..0bdba381617f5c564c51e1bba7bd8e93aa71e145 100644 (file)
--- a/apps/enc.c
+++ b/apps/enc.c
@@ -138,7 +138,7 @@ int enc_main(int argc, char **argv)
     char mbuf[sizeof magic - 1];
     OPTION_CHOICE o;
     int bsize = BSIZE, verbose = 0, debug = 0, olb64 = 0, nosalt = 0;
-    int enc = 1, printkey = 0, i, k, base64 = 0;
+    int enc = 1, printkey = 0, i, k, format = FORMAT_BINARY;
     int ret = 1, inl, nopad = 0, non_fips_allow = 0;
     unsigned char key[EVP_MAX_KEY_LENGTH], iv[EVP_MAX_IV_LENGTH];
     unsigned char *buff = NULL, salt[PKCS5_SALT_LEN];
@@ -151,7 +151,7 @@ int enc_main(int argc, char **argv)
     /* first check the program name */
     prog = opt_progname(argv[0]);
     if (strcmp(prog, "base64") == 0)
-        base64 = 1;
+        format = FORMAT_BASE64;
 #ifdef ZLIB
     else if (strcmp(prog, "zlib") == 0)
         do_zlib = 1;
@@ -223,7 +223,7 @@ int enc_main(int argc, char **argv)
             olb64 = 1;
             break;
         case OPT_A:
-            base64 = 1;
+            format = FORMAT_BASE64;
             break;
         case OPT_Z:
 #ifdef ZLIB
@@ -246,7 +246,7 @@ int enc_main(int argc, char **argv)
             str = opt_arg();
             break;
         case OPT_KFILE:
-            in = bio_open_default(opt_arg(), "r");
+            in = bio_open_default(opt_arg(), 'r', FORMAT_TEXT);
             if (in == NULL)
                 goto opthelp;
             i = BIO_gets(in, buf, sizeof buf);
@@ -311,7 +311,7 @@ int enc_main(int argc, char **argv)
         dgst = EVP_md5();
 
     /* It must be large enough for a base64 encoded line */
-    if (base64 && bsize < 80)
+    if (format == FORMAT_BASE64 && bsize < 80)
         bsize = 80;
     if (verbose)
         BIO_printf(bio_err, "bufsize=%d\n", bsize);
@@ -330,7 +330,7 @@ int enc_main(int argc, char **argv)
         unbuffer(stdin);
         in = dup_bio_in();
     } else
-        in = bio_open_default(infile, base64 ? "r" : "rb");
+        in = bio_open_default(infile, 'r', format);
     if (in == NULL)
         goto end;
 
@@ -366,7 +366,7 @@ int enc_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, base64 ? "w" : "wb");
+    out = bio_open_default(outfile, 'w', format);
     if (out == NULL)
         goto end;
 
@@ -384,7 +384,7 @@ int enc_main(int argc, char **argv)
     }
 #endif
 
-    if (base64) {
+    if (format == FORMAT_BASE64) {
         if ((b64 = BIO_new(BIO_f_base64())) == NULL)
             goto end;
         if (debug) {

diff --git a/apps/gendsa.c b/apps/gendsa.c
index 087a44a1e899b2bb69c92e1f5300dec2c916f6b8..f1e1f54b8a463f3f8d845b262d5041129151b586 100644 (file)
--- a/apps/gendsa.c
+++ b/apps/gendsa.c
@@ -147,7 +147,7 @@ int gendsa_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(dsaparams, "r");
+    in = bio_open_default(dsaparams, 'r', FORMAT_PEM);
     if (in == NULL)
         goto end2;
 
@@ -158,7 +158,7 @@ int gendsa_main(int argc, char **argv)
     BIO_free(in);
     in = NULL;
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, FORMAT_PEM, private);
     if (out == NULL)
         goto end2;
 

diff --git a/apps/genpkey.c b/apps/genpkey.c
index c29b1947a674ffa7b4eaa3e6c368250eb9fb27a2..d80983350a86b5e3ee86d08a1747a2cc2049f3bd 100644 (file)
--- a/apps/genpkey.c
+++ b/apps/genpkey.c
@@ -184,7 +184,7 @@ int genpkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/genrsa.c b/apps/genrsa.c
index 1fea351fb148f755f0460280ee647b2a82847f51..54484b5273c0d8af02d478716eabe2b4edfe8656 100644 (file)
--- a/apps/genrsa.c
+++ b/apps/genrsa.c
@@ -172,7 +172,7 @@ int genrsa_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, "w", private);
+    out = bio_open_owner(outfile, FORMAT_PEM, private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/nseq.c b/apps/nseq.c
index 5c8ed172c2cafccee21cfff3380d11306bbaf214..06893c82ce6647a41e9c220593d9954cdbbe8f3e 100644 (file)
--- a/apps/nseq.c
+++ b/apps/nseq.c
@@ -112,10 +112,10 @@ int nseq_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, 'r', FORMAT_PEM);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_PEM);
     if (out == NULL)
         goto end;
 

diff --git a/apps/ocsp.c b/apps/ocsp.c
index 7193dae20b38d7a03728391a4b229aeec4c8028b..e97d06e7c10bd8f21769696b9a5b1c597f57b092 100644 (file)
--- a/apps/ocsp.c
+++ b/apps/ocsp.c
@@ -486,7 +486,7 @@ int ocsp_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_TEXT);
     if (out == NULL)
         goto end;
 
@@ -494,7 +494,7 @@ int ocsp_main(int argc, char **argv)
         add_nonce = 0;
 
     if (!req && reqin) {
-        derbio = bio_open_default(reqin, "rb");
+        derbio = bio_open_default(reqin, 'r', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         req = d2i_OCSP_REQUEST_bio(derbio, NULL);
@@ -589,7 +589,7 @@ int ocsp_main(int argc, char **argv)
         OCSP_REQUEST_print(out, req, 0);
 
     if (reqout) {
-        derbio = bio_open_default(reqout, "wb");
+        derbio = bio_open_default(reqout, 'w', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         i2d_OCSP_REQUEST_bio(derbio, req);
@@ -627,7 +627,7 @@ int ocsp_main(int argc, char **argv)
         goto end;
 # endif
     } else if (respin) {
-        derbio = bio_open_default(respin, "rb");
+        derbio = bio_open_default(respin, 'r', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         resp = d2i_OCSP_RESPONSE_bio(derbio, NULL);
@@ -644,7 +644,7 @@ int ocsp_main(int argc, char **argv)
  done_resp:
 
     if (respout) {
-        derbio = bio_open_default(respout, "wb");
+        derbio = bio_open_default(respout, 'w', FORMAT_ASN1);
         if (derbio == NULL)
             goto end;
         i2d_OCSP_RESPONSE_bio(derbio, resp);

diff --git a/apps/openssl.c b/apps/openssl.c
index 7c202cf8d1dbc716d5a6b7bcc5c142114987c38d..7e340be21ab987c14de398a8cd0f585c7e51e9ea 100644 (file)
--- a/apps/openssl.c
+++ b/apps/openssl.c
@@ -299,17 +299,46 @@ void unbuffer(FILE *fp)
     setbuf(fp, NULL);
 }
 
+static const char *modestr(char mode, int format)
+{
+    OPENSSL_assert(mode == 'a' || mode == 'r' || mode == 'w');
+
+    switch (mode) {
+    case 'a':
+        return (format) & B_FORMAT_TEXT ? "ab" : "a";
+    case 'r':
+        return (format) & B_FORMAT_TEXT ? "rb" : "r";
+    case 'w':
+        return (format) & B_FORMAT_TEXT ? "wb" : "w";
+    }
+    /* The assert above should make sure we never reach this point */
+    return NULL;
+}
+
+static const char *modeverb(char mode)
+{
+    switch (mode) {
+    case 'a':
+        return "appending";
+    case 'r':
+        return "reading";
+    case 'w':
+        return "writing";
+    }
+    return "(doing something)";
+}
+
 /*
  * Open a file for writing, owner-read-only.
  */
-BIO *bio_open_owner(const char *filename, const char *modestr, int private)
+BIO *bio_open_owner(const char *filename, int format, int private)
 {
     FILE *fp = NULL;
     BIO *b = NULL;
     int fd = -1, bflags, mode, binmode;
 
     if (!private || filename == NULL || strcmp(filename, "-") == 0)
-        return bio_open_default(filename, modestr);
+        return bio_open_default(filename, 'w', format);
 
     mode = O_WRONLY;
 #ifdef O_CREAT
@@ -318,7 +347,7 @@ BIO *bio_open_owner(const char *filename, const char *modestr, int private)
 #ifdef O_TRUNC
     mode |= O_TRUNC;
 #endif
-    binmode = strchr(modestr, 'b') != NULL;
+    binmode = !(format & B_FORMAT_TEXT);
     if (binmode) {
 #ifdef O_BINARY
         mode |= O_BINARY;
@@ -330,7 +359,7 @@ BIO *bio_open_owner(const char *filename, const char *modestr, int private)
     fd = open(filename, mode, 0600);
     if (fd < 0)
         goto err;
-    fp = fdopen(fd, modestr);
+    fp = fdopen(fd, modestr('w', format));
     if (fp == NULL)
         goto err;
     bflags = BIO_CLOSE;
@@ -352,12 +381,13 @@ BIO *bio_open_owner(const char *filename, const char *modestr, int private)
     return NULL;
 }
 
-static BIO *bio_open_default_(const char *filename, const char *mode, int quiet)
+static BIO *bio_open_default_(const char *filename, char mode, int format,
+                              int quiet)
 {
     BIO *ret;
 
     if (filename == NULL || strcmp(filename, "-") == 0) {
-        ret = *mode == 'r' ? dup_bio_in() : dup_bio_out();
+        ret = mode == 'r' ? dup_bio_in() : dup_bio_out();
         if (quiet) {
             ERR_clear_error();
             return ret;
@@ -366,9 +396,9 @@ static BIO *bio_open_default_(const char *filename, const char *mode, int quiet)
             return ret;
         BIO_printf(bio_err,
                    "Can't open %s, %s\n",
-                   *mode == 'r' ? "stdin" : "stdout", strerror(errno));
+                   mode == 'r' ? "stdin" : "stdout", strerror(errno));
     } else {
-        ret = BIO_new_file(filename, mode);
+        ret = BIO_new_file(filename, modestr(mode, format));
         if (quiet) {
             ERR_clear_error();
             return ret;
@@ -377,21 +407,20 @@ static BIO *bio_open_default_(const char *filename, const char *mode, int quiet)
             return ret;
         BIO_printf(bio_err,
                    "Can't open %s for %s, %s\n",
-                   filename,
-                   *mode == 'r' ? "reading" : "writing", strerror(errno));
+                   filename, modeverb(mode), strerror(errno));
     }
     ERR_print_errors(bio_err);
     return NULL;
 }
 
-BIO *bio_open_default(const char *filename, const char *mode)
+BIO *bio_open_default(const char *filename, char mode, int format)
 {
-    return bio_open_default_(filename, mode, 0);
+    return bio_open_default_(filename, mode, format, 0);
 }
 
-BIO *bio_open_default_quiet(const char *filename, const char *mode)
+BIO *bio_open_default_quiet(const char *filename, char mode, int format)
 {
-    return bio_open_default_(filename, mode, 1);
+    return bio_open_default_(filename, mode, format, 1);
 }
 
 #if defined( OPENSSL_SYS_VMS)

diff --git a/apps/passwd.c b/apps/passwd.c
index dbae620645acb440fbd22bde237110f316ade835..898831337563488b4e40d2f1f0e867f9ba2a7437 100644 (file)
--- a/apps/passwd.c
+++ b/apps/passwd.c
@@ -209,7 +209,7 @@ int passwd_main(int argc, char **argv)
         goto end;
     }
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, 'r', FORMAT_TEXT);
     if (in == NULL)
         goto end;
 

diff --git a/apps/pkcs12.c b/apps/pkcs12.c
index 5b14dd532003576d8c0e73deb72170d6cd74a4d8..2e74cd4baeba904436812fb5ba206874dd466bb9 100644 (file)
--- a/apps/pkcs12.c
+++ b/apps/pkcs12.c
@@ -353,13 +353,6 @@ int pkcs12_main(int argc, char **argv)
                        app_RAND_load_files(inrand));
     }
 
-    in = bio_open_default(infile, "rb");
-    if (in == NULL)
-        goto end;
-    out = bio_open_owner(outfile, "wb", private);
-    if (out == NULL)
-        goto end;
-
     if (twopass) {
         if (EVP_read_pw_string
             (macpass, sizeof macpass, "Enter MAC Password:", export_cert)) {
@@ -501,6 +494,11 @@ int pkcs12_main(int argc, char **argv)
             PKCS12_set_mac(p12, mpass, -1, NULL, 0, maciter, macmd);
 
         assert(private);
+
+        out = bio_open_owner(outfile, FORMAT_PKCS12, private);
+        if (out == NULL)
+            goto end;
+
         i2d_PKCS12_bio(out, p12);
 
         ret = 0;
@@ -515,6 +513,13 @@ int pkcs12_main(int argc, char **argv)
 
     }
 
+    in = bio_open_default(infile, 'r', FORMAT_PKCS12);
+    if (in == NULL)
+        goto end;
+    out = bio_open_owner(outfile, FORMAT_PEM, private);
+    if (out == NULL)
+        goto end;
+
     if ((p12 = d2i_PKCS12_bio(in, NULL)) == NULL) {
         ERR_print_errors(bio_err);
         goto end;

diff --git a/apps/pkcs7.c b/apps/pkcs7.c
index 248e0d6db26cd477cc04624517a44dcc4313f7ea..fff14dc95958ac9e62055d7f5a6b0c37ad6c3159 100644 (file)
--- a/apps/pkcs7.c
+++ b/apps/pkcs7.c
@@ -196,7 +196,7 @@ int pkcs7_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -210,7 +210,7 @@ int pkcs7_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

diff --git a/apps/pkcs8.c b/apps/pkcs8.c
index e3cb7750e1ae11ab1b0eac12673d153701c784c4..765744ffbe23047a33e57ea4e4a017e0913a447b 100644 (file)
--- a/apps/pkcs8.c
+++ b/apps/pkcs8.c
@@ -239,10 +239,10 @@ int pkcs8_main(int argc, char **argv)
     if ((pbe_nid == -1) && !cipher)
         pbe_nid = NID_pbeWithMD5AndDES_CBC;
 
-    in = bio_open_default(infile, RB(informat));
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/pkey.c b/apps/pkey.c
index 9ef228f3a0934def409426a094aa3f2e664fcc9c..d2a66eb4a0e22c719cbc5f77d35ed3841f50e186 100644 (file)
--- a/apps/pkey.c
+++ b/apps/pkey.c
@@ -172,7 +172,7 @@ int pkey_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/pkeyparam.c b/apps/pkeyparam.c
index 6039dad9d1905429cac32de01a6573842d090c0c..215611eb347d2a4243d7fd389cd208bc4070a65c 100644 (file)
--- a/apps/pkeyparam.c
+++ b/apps/pkeyparam.c
@@ -121,10 +121,10 @@ int pkeyparam_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    in = bio_open_default(infile, "r");
+    in = bio_open_default(infile, 'r', FORMAT_PEM);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_PEM);
     if (out == NULL)
         goto end;
     pkey = PEM_read_bio_Parameters(in, NULL);

diff --git a/apps/pkeyutl.c b/apps/pkeyutl.c
index 741dd642b1430b75975578b4207d28404ce4667a..c3e18895a78db01a96ae4af53abc5d71256cf63d 100644 (file)
--- a/apps/pkeyutl.c
+++ b/apps/pkeyutl.c
@@ -249,11 +249,11 @@ int pkeyutl_main(int argc, char **argv)
     app_RAND_load_file(NULL, 0);
 
     if (pkey_op != EVP_PKEY_OP_DERIVE) {
-        in = bio_open_default(infile, "rb");
+        in = bio_open_default(infile, 'r', FORMAT_BINARY);
         if (in == NULL)
             goto end;
     }
-    out = bio_open_default(outfile, "wb");
+    out = bio_open_default(outfile, 'w', FORMAT_BINARY);
     if (out == NULL)
         goto end;
 

diff --git a/apps/rand.c b/apps/rand.c
index a5aa5d94103256ce069f9a0266491eba15eb7941..315e6be02f6819e47c85c18b4c33d2f3e21ab48c 100644 (file)
--- a/apps/rand.c
+++ b/apps/rand.c
@@ -87,7 +87,7 @@ int rand_main(int argc, char **argv)
     BIO *out = NULL;
     char *inrand = NULL, *outfile = NULL, *prog;
     OPTION_CHOICE o;
-    int base64 = 0, hex = 0, i, num = -1, r, ret = 1;
+    int format = FORMAT_BINARY, i, num = -1, r, ret = 1;
 
     prog = opt_init(argc, argv, rand_options);
     while ((o = opt_next()) != OPT_EOF) {
@@ -111,17 +111,17 @@ int rand_main(int argc, char **argv)
             inrand = opt_arg();
             break;
         case OPT_BASE64:
-            base64 = 1;
+            format = FORMAT_BASE64;
             break;
         case OPT_HEX:
-            hex = 1;
+            format = FORMAT_TEXT;
             break;
         }
     }
     argc = opt_num_rest();
     argv = opt_rest();
 
-    if (argc != 1 || (hex && base64))
+    if (argc != 1)
         goto opthelp;
     if (sscanf(argv[0], "%d", &num) != 1 || num < 0)
         goto opthelp;
@@ -134,11 +134,11 @@ int rand_main(int argc, char **argv)
         BIO_printf(bio_err, "%ld semi-random bytes loaded\n",
                    app_RAND_load_files(inrand));
 
-    out = bio_open_default(outfile, base64 ? "w" : "wb");
+    out = bio_open_default(outfile, 'w', format);
     if (out == NULL)
         goto end;
 
-    if (base64) {
+    if (format == FORMAT_BASE64) {
         BIO *b64 = BIO_new(BIO_f_base64());
         if (b64 == NULL)
             goto end;
@@ -155,7 +155,7 @@ int rand_main(int argc, char **argv)
         r = RAND_bytes(buf, chunk);
         if (r <= 0)
             goto end;
-        if (!hex)
+        if (format != FORMAT_TEXT) /* hex */
             BIO_write(out, buf, chunk);
         else {
             for (i = 0; i < chunk; i++)
@@ -163,7 +163,7 @@ int rand_main(int argc, char **argv)
         }
         num -= chunk;
     }
-    if (hex)
+    if (format == FORMAT_TEXT)
         BIO_puts(out, "\n");
     (void)BIO_flush(out);
 

diff --git a/apps/req.c b/apps/req.c
index bae3eeca0c990a2cce0da679709c55a2adc63bfb..ce0fcbcb3b3c3d13ecb02b3db0e72f3053702f1f 100644 (file)
--- a/apps/req.c
+++ b/apps/req.c
@@ -566,7 +566,7 @@ int req_main(int argc, char **argv)
             BIO_printf(bio_err, "writing new private key to stdout\n");
         else
             BIO_printf(bio_err, "writing new private key to '%s'\n", keyout);
-        out = bio_open_owner(keyout, "w", private);
+        out = bio_open_owner(keyout, outformat, private);
         if (out == NULL)
             goto end;
 
@@ -601,7 +601,7 @@ int req_main(int argc, char **argv)
     }
 
     if (!newreq) {
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
 
@@ -764,8 +764,8 @@ int req_main(int argc, char **argv)
 
     out = bio_open_default(outfile,
                            keyout != NULL && outfile != NULL &&
-                           strcmp(keyout, outfile) == 0
-                           ? AB(outformat) : WB(outformat));
+                           strcmp(keyout, outfile) == 0 ? 'a' : 'w',
+                           outformat);
     if (out == NULL)
         goto end;
 

diff --git a/apps/rsa.c b/apps/rsa.c
index 01cc6ea18da6476017eb3806555c74f0ef9203af..f8a0deceb6511a949a218cc44ac1d45552c20f38 100644 (file)
--- a/apps/rsa.c
+++ b/apps/rsa.c
@@ -292,7 +292,7 @@ int rsa_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_owner(outfile, WB(outformat), private);
+    out = bio_open_owner(outfile, outformat, private);
     if (out == NULL)
         goto end;
 

diff --git a/apps/rsautl.c b/apps/rsautl.c
index 8ba838b43fa2df1e288186da1f01316cc7db7b42..84a1de1ae96a71557efa715a5f71609f503f40f1 100644 (file)
--- a/apps/rsautl.c
+++ b/apps/rsautl.c
@@ -251,10 +251,10 @@ int rsautl_main(int argc, char **argv)
         goto end;
     }
 
-    in = bio_open_default(infile, "rb");
+    in = bio_open_default(infile, 'r', FORMAT_BINARY);
     if (in == NULL)
         goto end;
-    out = bio_open_default(outfile, "wb");
+    out = bio_open_default(outfile, 'w', FORMAT_BINARY);
     if (out == NULL)
         goto end;
 

diff --git a/apps/sess_id.c b/apps/sess_id.c
index e743791d23f5eb8eb438707031648bfbe1adc75a..39711e2150e821dcf440e6c0cac6fb8f8a496562 100644 (file)
--- a/apps/sess_id.c
+++ b/apps/sess_id.c
@@ -160,7 +160,7 @@ int sess_id_main(int argc, char **argv)
     }
 
     if (!noout || text) {
-        out = bio_open_default(outfile, WB(outformat));
+        out = bio_open_default(outfile, 'w', outformat);
         if (out == NULL)
             goto end;
     }
@@ -217,7 +217,7 @@ static SSL_SESSION *load_sess_id(char *infile, int format)
     SSL_SESSION *x = NULL;
     BIO *in = NULL;
 
-    in = bio_open_default(infile, RB(format));
+    in = bio_open_default(infile, 'r', format);
     if (in == NULL)
         goto end;
     if (format == FORMAT_ASN1)

diff --git a/apps/smime.c b/apps/smime.c
index d597ebf53479669b200004ddc3f042cfb837b5c0..4da56cdf08cccddb1e940231b1c26d788eefe5eb 100644 (file)
--- a/apps/smime.c
+++ b/apps/smime.c
@@ -170,7 +170,6 @@ int smime_main(int argc, char **argv)
         NULL;
     char *passinarg = NULL, *passin = NULL, *to = NULL, *from =
         NULL, *subject = NULL;
-    const char *inmode = "r", *outmode = "w";
     OPTION_CHOICE o;
     int flags = PKCS7_DETACHED, operation = 0, ret = 0, need_rand = 0, indef =
         0;
@@ -426,18 +425,14 @@ int smime_main(int argc, char **argv)
     if (!(operation & SMIME_SIGNERS))
         flags &= ~PKCS7_DETACHED;
 
-    if (operation & SMIME_OP) {
-        outmode = WB(outformat);
-    } else {
+    if (!(operation & SMIME_OP)) {
         if (flags & PKCS7_BINARY)
-            outmode = "wb";
+            outformat = FORMAT_BINARY;
     }
 
-    if (operation & SMIME_IP) {
-        inmode = RB(informat);
-    } else {
+    if (!(operation & SMIME_IP)) {
         if (flags & PKCS7_BINARY)
-            inmode = "rb";
+            informat = FORMAT_BINARY;
     }
 
     if (operation == SMIME_ENCRYPT) {
@@ -494,7 +489,7 @@ int smime_main(int argc, char **argv)
             goto end;
     }
 
-    in = bio_open_default(infile, inmode);
+    in = bio_open_default(infile, 'r', informat);
     if (in == NULL)
         goto end;
 
@@ -523,7 +518,7 @@ int smime_main(int argc, char **argv)
         }
     }
 
-    out = bio_open_default(outfile, outmode);
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 

diff --git a/apps/spkac.c b/apps/spkac.c
index d41331caba4ce6f193c0ea565d22b07056da2c9d..180f80fcb7b88084c6268ad39188002f40057c99 100644 (file)
--- a/apps/spkac.c
+++ b/apps/spkac.c
@@ -175,7 +175,7 @@ int spkac_main(int argc, char **argv)
         NETSCAPE_SPKI_sign(spki, pkey, EVP_md5());
         spkstr = NETSCAPE_SPKI_b64_encode(spki);
 
-        out = bio_open_default(outfile, "w");
+        out = bio_open_default(outfile, 'w', FORMAT_TEXT);
         if (out == NULL)
             goto end;
         BIO_printf(out, "SPKAC=%s\n", spkstr);
@@ -205,7 +205,7 @@ int spkac_main(int argc, char **argv)
         goto end;
     }
 
-    out = bio_open_default(outfile, "w");
+    out = bio_open_default(outfile, 'w', FORMAT_TEXT);
     if (out == NULL)
         goto end;
 

diff --git a/apps/ts.c b/apps/ts.c
index 6e6b834401324d34b3134eccf3bb88d720f14fc8..70729c5629660f5a638cdfa22e046049e4ba2942 100644 (file)
--- a/apps/ts.c
+++ b/apps/ts.c
@@ -429,13 +429,13 @@ static int query_command(const char *data, char *digest, const EVP_MD *md,
 
     /* Build query object either from file or from scratch. */
     if (in != NULL) {
-        if ((in_bio = BIO_new_file(in, "rb")) == NULL)
+        if ((in_bio = bio_open_default(in, 'r', FORMAT_ASN1)) == NULL)
             goto end;
         query = d2i_TS_REQ_bio(in_bio, NULL);
     } else {
         /* Open the file if no explicit digest bytes were specified. */
         if (digest == NULL
-            && (data_bio = bio_open_default(data, "rb")) == NULL)
+            && (data_bio = bio_open_default(data, 'r', FORMAT_ASN1)) == NULL)
             goto end;
         query = create_query(data_bio, digest, md, policy, no_nonce, cert);
     }
@@ -443,14 +443,16 @@ static int query_command(const char *data, char *digest, const EVP_MD *md,
         goto end;
 
     /* Write query either in ASN.1 or in text format. */
-    if ((out_bio = bio_open_default(out, "wb")) == NULL)
-        goto end;
     if (text) {
         /* Text output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_TEXT)) == NULL)
+            goto end;
         if (!TS_REQ_print_bio(out_bio, query))
             goto end;
     } else {
         /* ASN.1 output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_ASN1)) == NULL)
+            goto end;
         if (!i2d_TS_REQ_bio(out_bio, query))
             goto end;
     }
@@ -662,10 +664,10 @@ static int reply_command(CONF *conf, char *section, char *engine,
         goto end;
 
     /* Write response either in ASN.1 or text format. */
-    if ((out_bio = bio_open_default(out, "wb")) == NULL)
-        goto end;
     if (text) {
         /* Text output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_TEXT)) == NULL)
+        goto end;
         if (token_out) {
             TS_TST_INFO *tst_info = TS_RESP_get_tst_info(response);
             if (!TS_TST_INFO_print_bio(out_bio, tst_info))
@@ -676,6 +678,8 @@ static int reply_command(CONF *conf, char *section, char *engine,
         }
     } else {
         /* ASN.1 DER output. */
+        if ((out_bio = bio_open_default(out, 'w', FORMAT_ASN1)) == NULL)
+            goto end;
         if (token_out) {
             PKCS7 *token = TS_RESP_get_token(response);
             if (!i2d_PKCS7_bio(out_bio, token))

diff --git a/apps/x509.c b/apps/x509.c
index 8020e8a086a2ec98a3cb1a3a8428b733494b32ae..9472b68da5226594616e8c4ba14a5e8fd49c40d4 100644 (file)
--- a/apps/x509.c
+++ b/apps/x509.c
@@ -496,7 +496,7 @@ int x509_main(int argc, char **argv)
     if (!app_load_modules(NULL))
         goto end;
 
-    out = bio_open_default(outfile, WB(outformat));
+    out = bio_open_default(outfile, 'w', outformat);
     if (out == NULL)
         goto end;
 
@@ -556,7 +556,7 @@ int x509_main(int argc, char **argv)
             BIO_printf(bio_err, "We need a private key to sign with\n");
             goto end;
         }
-        in = bio_open_default(infile, RB(informat));
+        in = bio_open_default(infile, 'r', informat);
         if (in == NULL)
             goto end;
         req = PEM_read_bio_X509_REQ(in, NULL, NULL, NULL);