Fix some RAND bugs

RT2630 -- segfault for int overlow
RT2877 -- check return values in apps/rand
Update CHANGES file for previous "windows rand" changes.

Reviewed-by: Richard Levitte <levitte@openssl.org>

diff --git a/CHANGES b/CHANGES
index 541efc390ce31bba5adade0a69c6327d8c35be2e..c64d677582e36c7e216a0d1825f5665d4fd52e5d 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -4,6 +4,11 @@
 
  Changes between 1.0.2h and 1.1.0  [xx XXX 2016]
 
+  *) Windows RAND implementation was simplified to only get entropy by
+     calling CryptGenRandom(). Various other RAND-related tickets
+     were also closed.
+     [Joseph Wylie Yandle, Rich Salz]
+
   *) The stack and lhash API's were renamed to start with OPENSSL_SK_
      and OPENSSL_LH_, respectively.  The old names are available
      with API compatibility.  They new names are now completely documented.

diff --git a/apps/rand.c b/apps/rand.c
index 89a23a293a93c5ee7f0d746af8f73a7a6cba25c7..d60f1ecf8658f5b47d1cb1570a007cab5d372c1b 100644 (file)
--- a/apps/rand.c
+++ b/apps/rand.c
@@ -105,22 +105,26 @@ int rand_main(int argc, char **argv)
         r = RAND_bytes(buf, chunk);
         if (r <= 0)
             goto end;
-        if (format != FORMAT_TEXT) /* hex */
-            BIO_write(out, buf, chunk);
-        else {
+        if (format != FORMAT_TEXT) {
+            if (BIO_write(out, buf, chunk) != chunk)
+                goto end;
+        } else {
             for (i = 0; i < chunk; i++)
-                BIO_printf(out, "%02x", buf[i]);
+                if (BIO_printf(out, "%02x", buf[i]) != 2)
+                    goto end;
         }
         num -= chunk;
     }
     if (format == FORMAT_TEXT)
         BIO_puts(out, "\n");
-    (void)BIO_flush(out);
+    if (BIO_flush(out) <= 0 || !app_RAND_write_file(NULL))
+        goto end;
 
-    app_RAND_write_file(NULL);
     ret = 0;
 
  end:
+    if (ret != 0)
+        ERR_print_errors(bio_err);
     BIO_free_all(out);
     return (ret);
 }

diff --git a/crypto/rand/md_rand.c b/crypto/rand/md_rand.c
index 4b874e3bedf4a9459e1ee8487769076983938e02..137851f596d8b670b5b3091dc9fcb6801f781631 100644 (file)
--- a/crypto/rand/md_rand.c
+++ b/crypto/rand/md_rand.c
@@ -38,7 +38,7 @@
 /* #define PREDICT      1 */
 
 #define STATE_SIZE      1023
-static int state_num = 0, state_index = 0;
+static size_t state_num = 0, state_index = 0;
 static unsigned char state[STATE_SIZE + MD_DIGEST_LENGTH];
 static unsigned char md[MD_DIGEST_LENGTH];
 static long md_count[2] = { 0, 0 };
@@ -268,8 +268,8 @@ static int rand_seed(const void *buf, int num)
 static int rand_bytes(unsigned char *buf, int num, int pseudo)
 {
     static volatile int stirred_pool = 0;
-    int i, j, k, st_num, st_idx;
-    int num_ceil;
+    int i, j, k;
+    size_t num_ceil, st_idx, st_num;
     int ok;
     long md_c[2];
     unsigned char local_md[MD_DIGEST_LENGTH];