sha3: fix to conform to final SHA3 padding standard, add -a BITS option
authorDenys Vlasenko <vda.linux@googlemail.com>
Mon, 29 Aug 2016 12:05:25 +0000 (14:05 +0200)
committerDenys Vlasenko <vda.linux@googlemail.com>
Mon, 29 Aug 2016 12:05:25 +0000 (14:05 +0200)
function                                             old     new   delta
hash_file                                            331     396     +65
md5_sha1_sum_main                                    485     538     +53
packed_usage                                       30423   30464     +41
sha3_begin                                            17      31     +14
sha3_hash                                            101     110      +9
sha3_end                                              41      49      +8

Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>
coreutils/Config.src
coreutils/Kbuild.src
coreutils/md5_sha1_sum.c
include/applets.src.h
include/libbb.h
libbb/hash_md5_sha.c
testsuite/sha3sum.tests

index 619c2efe83fea4656121534e61f48d6b99373b23..b9dde1209f1490851bd768c6fa2f931125c3eb16 100644 (file)
@@ -328,12 +328,6 @@ config FEATURE_LS_COLOR_IS_DEFAULT
          configurable, and the output may not be legible on
          many output screens.
 
-config MD5SUM
-       bool "md5sum"
-       default y
-       help
-         md5sum is used to print or check MD5 checksums.
-
 config MKDIR
        bool "mkdir"
        default y
@@ -458,30 +452,6 @@ config SEQ
        help
          print a sequence of numbers
 
-config SHA1SUM
-       bool "sha1sum"
-       default y
-       help
-         Compute and check SHA1 message digest
-
-config SHA256SUM
-       bool "sha256sum"
-       default y
-       help
-         Compute and check SHA256 message digest
-
-config SHA512SUM
-       bool "sha512sum"
-       default y
-       help
-         Compute and check SHA512 message digest
-
-config SHA3SUM
-       bool "sha3sum"
-       default y
-       help
-         Compute and check SHA3 (512-bit) message digest
-
 config SLEEP
        bool "sleep"
        default y
@@ -731,17 +701,4 @@ config FEATURE_HUMAN_READABLE
        help
          Allow df, du, and ls to have human readable output.
 
-comment "Common options for md5sum, sha1sum, sha256sum, sha512sum, sha3sum"
-       depends on MD5SUM || SHA1SUM || SHA256SUM || SHA512SUM || SHA3SUM
-
-config FEATURE_MD5_SHA1_SUM_CHECK
-       bool "Enable -c, -s and -w options"
-       default y
-       depends on MD5SUM || SHA1SUM || SHA256SUM || SHA512SUM || SHA3SUM
-       help
-         Enabling the -c options allows files to be checked
-         against pre-calculated hash values.
-
-         -s and -w are useful options when verifying checksums.
-
 endmenu
index 4ec075ac67284ae4033fde8c43c1adeb647222ca..5a64fee35bdbde6a181897788ebfa2605fd0438a 100644 (file)
@@ -41,7 +41,6 @@ lib-$(CONFIG_LN)        += ln.o
 lib-$(CONFIG_LOGNAME)   += logname.o
 lib-$(CONFIG_LS)        += ls.o
 lib-$(CONFIG_FTPD)      += ls.o
-lib-$(CONFIG_MD5SUM)    += md5_sha1_sum.o
 lib-$(CONFIG_MKDIR)     += mkdir.o
 lib-$(CONFIG_MKFIFO)    += mkfifo.o
 lib-$(CONFIG_MKNOD)     += mknod.o
@@ -58,10 +57,6 @@ lib-$(CONFIG_REALPATH)  += realpath.o
 lib-$(CONFIG_RM)        += rm.o
 lib-$(CONFIG_RMDIR)     += rmdir.o
 lib-$(CONFIG_SEQ)       += seq.o
-lib-$(CONFIG_SHA1SUM)   += md5_sha1_sum.o
-lib-$(CONFIG_SHA256SUM) += md5_sha1_sum.o
-lib-$(CONFIG_SHA512SUM) += md5_sha1_sum.o
-lib-$(CONFIG_SHA3SUM)   += md5_sha1_sum.o
 lib-$(CONFIG_SLEEP)     += sleep.o
 lib-$(CONFIG_SPLIT)     += split.o
 lib-$(CONFIG_SORT)      += sort.o
index c0e816ba66776f046376b9323e690c6487004625..5e017b191721031316edf701cdc7b352d9056e5e 100644 (file)
@@ -5,6 +5,60 @@
  *
  * Licensed under GPLv2 or later, see file LICENSE in this source tree.
  */
+//config:config MD5SUM
+//config:      bool "md5sum"
+//config:      default y
+//config:      help
+//config:        md5sum is used to print or check MD5 checksums.
+//config:
+//config:config SHA1SUM
+//config:      bool "sha1sum"
+//config:      default y
+//config:      help
+//config:        Compute and check SHA1 message digest
+//config:
+//config:config SHA256SUM
+//config:      bool "sha256sum"
+//config:      default y
+//config:      help
+//config:        Compute and check SHA256 message digest
+//config:
+//config:config SHA512SUM
+//config:      bool "sha512sum"
+//config:      default y
+//config:      help
+//config:        Compute and check SHA512 message digest
+//config:
+//config:config SHA3SUM
+//config:      bool "sha3sum"
+//config:      default y
+//config:      help
+//config:        Compute and check SHA3 (512-bit) message digest
+//config:
+//config:comment "Common options for md5sum, sha1sum, sha256sum, sha512sum, sha3sum"
+//config:      depends on MD5SUM || SHA1SUM || SHA256SUM || SHA512SUM || SHA3SUM
+//config:
+//config:config FEATURE_MD5_SHA1_SUM_CHECK
+//config:      bool "Enable -c, -s and -w options"
+//config:      default y
+//config:      depends on MD5SUM || SHA1SUM || SHA256SUM || SHA512SUM || SHA3SUM
+//config:      help
+//config:        Enabling the -c options allows files to be checked
+//config:        against pre-calculated hash values.
+//config:
+//config:        -s and -w are useful options when verifying checksums.
+
+//applet:IF_MD5SUM(APPLET_NOEXEC(md5sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, md5sum))
+//applet:IF_SHA1SUM(APPLET_NOEXEC(sha1sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha1sum))
+//applet:IF_SHA3SUM(APPLET_NOEXEC(sha3sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha3sum))
+//applet:IF_SHA256SUM(APPLET_NOEXEC(sha256sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha256sum))
+//applet:IF_SHA512SUM(APPLET_NOEXEC(sha512sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha512sum))
+
+//kbuild:lib-$(CONFIG_MD5SUM)    += md5_sha1_sum.o
+//kbuild:lib-$(CONFIG_SHA1SUM)   += md5_sha1_sum.o
+//kbuild:lib-$(CONFIG_SHA256SUM) += md5_sha1_sum.o
+//kbuild:lib-$(CONFIG_SHA512SUM) += md5_sha1_sum.o
+//kbuild:lib-$(CONFIG_SHA3SUM)   += md5_sha1_sum.o
 
 //usage:#define md5sum_trivial_usage
 //usage:       IF_FEATURE_MD5_SHA1_SUM_CHECK("[-c[sw]] ")"[FILE]..."
 //usage:       )
 //usage:
 //usage:#define sha3sum_trivial_usage
-//usage:       IF_FEATURE_MD5_SHA1_SUM_CHECK("[-c[sw]] ")"[FILE]..."
+//usage:       IF_FEATURE_MD5_SHA1_SUM_CHECK("[-c[sw]] ")"[-a BITS] [FILE]..."
 //usage:#define sha3sum_full_usage "\n\n"
-//usage:       "Print" IF_FEATURE_MD5_SHA1_SUM_CHECK(" or check") " SHA3-512 checksums"
+//usage:       "Print" IF_FEATURE_MD5_SHA1_SUM_CHECK(" or check") " SHA3 checksums"
 //usage:       IF_FEATURE_MD5_SHA1_SUM_CHECK( "\n"
 //usage:     "\n       -c      Check sums against list in FILEs"
 //usage:     "\n       -s      Don't output anything, status code shows success"
 //usage:     "\n       -w      Warn about improperly formatted checksum lines"
+//usage:     "\n       -a BITS 224 (default), 256, 384, 512"
 //usage:       )
 
 //FIXME: GNU coreutils 8.25 has no -s option, it has only these two long opts:
@@ -97,7 +152,10 @@ static unsigned char *hash_bin_to_hex(unsigned char *hash_value,
        return (unsigned char *)hex_value;
 }
 
-static uint8_t *hash_file(const char *filename)
+#if !ENABLE_SHA3SUM
+# define hash_file(f,w) hash_file(f)
+#endif
+static uint8_t *hash_file(const char *filename, unsigned sha3_width)
 {
        int src_fd, hash_len, count;
        union _ctx_ {
@@ -125,27 +183,47 @@ static uint8_t *hash_file(const char *filename)
                update = (void*)md5_hash;
                final = (void*)md5_end;
                hash_len = 16;
-       } else if (ENABLE_SHA1SUM && hash_algo == HASH_SHA1) {
+       }
+       else if (ENABLE_SHA1SUM && hash_algo == HASH_SHA1) {
                sha1_begin(&context.sha1);
                update = (void*)sha1_hash;
                final = (void*)sha1_end;
                hash_len = 20;
-       } else if (ENABLE_SHA256SUM && hash_algo == HASH_SHA256) {
+       }
+       else if (ENABLE_SHA256SUM && hash_algo == HASH_SHA256) {
                sha256_begin(&context.sha256);
                update = (void*)sha256_hash;
                final = (void*)sha256_end;
                hash_len = 32;
-       } else if (ENABLE_SHA512SUM && hash_algo == HASH_SHA512) {
+       }
+       else if (ENABLE_SHA512SUM && hash_algo == HASH_SHA512) {
                sha512_begin(&context.sha512);
                update = (void*)sha512_hash;
                final = (void*)sha512_end;
                hash_len = 64;
-       } else if (ENABLE_SHA3SUM && hash_algo == HASH_SHA3) {
+       }
+#if ENABLE_SHA3SUM
+       else if (ENABLE_SHA3SUM && hash_algo == HASH_SHA3) {
                sha3_begin(&context.sha3);
                update = (void*)sha3_hash;
                final = (void*)sha3_end;
-               hash_len = 64;
-       } else {
+               /*
+                * Should support 224, 256, 384, 512.
+                * We allow any value which does not blow the algorithm up.
+                */
+               if (sha3_width >= 1600/2 /* input block can't be <= 0 */
+                || sha3_width == 0      /* hash len can't be 0 */
+                || (sha3_width & 0x1f)  /* should be multiple of 32 */
+               /* (because input uses up to 8 byte wide word XORs. 32/4=8) */
+               ) {
+                       bb_error_msg_and_die("bad -a%u", sha3_width);
+               }
+               sha3_width /= 4;
+               context.sha3.input_block_bytes = 1600/8 - sha3_width;
+               hash_len = sha3_width/2;
+       }
+#endif
+       else {
                xfunc_die(); /* can't reach this */
        }
 
@@ -176,18 +254,30 @@ int md5_sha1_sum_main(int argc UNUSED_PARAM, char **argv)
 {
        int return_value = EXIT_SUCCESS;
        unsigned flags;
+#if ENABLE_SHA3SUM
+       unsigned sha3_width = 224;
+#endif
 
        if (ENABLE_FEATURE_MD5_SHA1_SUM_CHECK) {
                /* -s and -w require -c */
                opt_complementary = "s?c:w?c";
                /* -b "binary", -t "text" are ignored (shaNNNsum compat) */
-               flags = getopt32(argv, "scwbt");
-               argv += optind;
-               //argc -= optind;
+#if ENABLE_SHA3SUM
+               if (applet_name[3] == HASH_SHA3)
+                       flags = getopt32(argv, "scwbta:+", &sha3_width);
+               else
+#endif
+                       flags = getopt32(argv, "scwbt");
        } else {
-               argv += 1;
-               //argc -= 1;
+#if ENABLE_SHA3SUM
+               if (applet_name[3] == HASH_SHA3)
+                       getopt32(argv, "a:+", &sha3_width);
+               else
+#endif
+                       getopt32(argv, "");
        }
+       argv += optind;
+       //argc -= optind;
        if (!*argv)
                *--argv = (char*)"-";
 
@@ -222,7 +312,7 @@ int md5_sha1_sum_main(int argc UNUSED_PARAM, char **argv)
                                *filename_ptr = '\0';
                                filename_ptr += 2;
 
-                               hash_value = hash_file(filename_ptr);
+                               hash_value = hash_file(filename_ptr, sha3_width);
 
                                if (hash_value && (strcmp((char*)hash_value, line) == 0)) {
                                        if (!(flags & FLAG_SILENT))
@@ -251,7 +341,7 @@ int md5_sha1_sum_main(int argc UNUSED_PARAM, char **argv)
                        }
                        fclose_if_not_stdin(pre_computed_stream);
                } else {
-                       uint8_t *hash_value = hash_file(*argv);
+                       uint8_t *hash_value = hash_file(*argv, sha3_width);
                        if (hash_value == NULL) {
                                return_value = EXIT_FAILURE;
                        } else {
index 6e1b02fc399c6e9deb868fe2a6dd76ed91a8306d..248d539c48d0f1b427ba214be02baeba3637443f 100644 (file)
@@ -205,7 +205,6 @@ IF_MAKEDEVS(APPLET(makedevs, BB_DIR_SBIN, BB_SUID_DROP))
 IF_MAKEMIME(APPLET(makemime, BB_DIR_BIN, BB_SUID_DROP))
 IF_MAN(APPLET(man, BB_DIR_USR_BIN, BB_SUID_DROP))
 IF_MATCHPATHCON(APPLET(matchpathcon, BB_DIR_USR_SBIN, BB_SUID_DROP))
-IF_MD5SUM(APPLET_NOEXEC(md5sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, md5sum))
 IF_MICROCOM(APPLET(microcom, BB_DIR_USR_BIN, BB_SUID_DROP))
 IF_MKDIR(APPLET_NOFORK(mkdir, mkdir, BB_DIR_BIN, BB_SUID_DROP, mkdir))
 IF_MKFS_VFAT(APPLET_ODDNAME(mkdosfs, mkfs_vfat, BB_DIR_SBIN, BB_SUID_DROP, mkfs_vfat))
@@ -280,10 +279,6 @@ IF_SETKEYCODES(APPLET(setkeycodes, BB_DIR_USR_BIN, BB_SUID_DROP))
 IF_SETLOGCONS(APPLET(setlogcons, BB_DIR_USR_SBIN, BB_SUID_DROP))
 IF_SETSEBOOL(APPLET(setsebool, BB_DIR_USR_SBIN, BB_SUID_DROP))
 IF_SETSID(APPLET(setsid, BB_DIR_USR_BIN, BB_SUID_DROP))
-IF_SHA1SUM(APPLET_NOEXEC(sha1sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha1sum))
-IF_SHA3SUM(APPLET_NOEXEC(sha3sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha3sum))
-IF_SHA256SUM(APPLET_NOEXEC(sha256sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha256sum))
-IF_SHA512SUM(APPLET_NOEXEC(sha512sum, md5_sha1_sum, BB_DIR_USR_BIN, BB_SUID_DROP, sha512sum))
 IF_SHOWKEY(APPLET(showkey, BB_DIR_USR_BIN, BB_SUID_DROP))
 IF_SLATTACH(APPLET(slattach, BB_DIR_SBIN, BB_SUID_DROP))
 /* Do not make this applet NOFORK. It breaks ^C-ing of pauses in shells: */
index b2e4299dd52fd5f69ce5fbc5488f777c3894438d..bf356d727367b42d8165bbd416f55803d0754c40 100644 (file)
@@ -1737,6 +1737,7 @@ typedef struct sha512_ctx_t {
 typedef struct sha3_ctx_t {
        uint64_t state[25];
        unsigned bytes_queued;
+       unsigned input_block_bytes;
 } sha3_ctx_t;
 void md5_begin(md5_ctx_t *ctx) FAST_FUNC;
 void md5_hash(md5_ctx_t *ctx, const void *buffer, size_t len) FAST_FUNC;
index d08c6b2f74d26983481d9aed6dc68ef2c97b3e04..7e7d8da2f9922c53d4067a1b711783b989e4be0c 100644 (file)
@@ -941,10 +941,6 @@ void FAST_FUNC sha512_end(sha512_ctx_t *ctx, void *resbuf)
 # define OPTIMIZE_SHA3_FOR_32 1
 #endif
 
-enum {
-       SHA3_IBLK_BYTES = 72, /* 576 bits / 8 */
-};
-
 #if OPTIMIZE_SHA3_FOR_32
 /* This splits every 64-bit word into a pair of 32-bit words,
  * even bits go into first word, odd bits go to second one.
@@ -1352,6 +1348,8 @@ static void sha3_process_block72(uint64_t *state)
 void FAST_FUNC sha3_begin(sha3_ctx_t *ctx)
 {
        memset(ctx, 0, sizeof(*ctx));
+       /* SHA3-512, user can override */
+       ctx->input_block_bytes = (1600 - 512*2) / 8; /* 72 bytes */
 }
 
 void FAST_FUNC sha3_hash(sha3_ctx_t *ctx, const void *buffer, size_t len)
@@ -1361,7 +1359,7 @@ void FAST_FUNC sha3_hash(sha3_ctx_t *ctx, const void *buffer, size_t len)
        unsigned bufpos = ctx->bytes_queued;
 
        while (1) {
-               unsigned remaining = SHA3_IBLK_BYTES - bufpos;
+               unsigned remaining = ctx->input_block_bytes - bufpos;
                if (remaining > len)
                        remaining = len;
                len -= remaining;
@@ -1373,38 +1371,41 @@ void FAST_FUNC sha3_hash(sha3_ctx_t *ctx, const void *buffer, size_t len)
                        remaining--;
                }
                /* Clever way to do "if (bufpos != N) break; ... ; bufpos = 0;" */
-               bufpos -= SHA3_IBLK_BYTES;
+               bufpos -= ctx->input_block_bytes;
                if (bufpos != 0)
                        break;
                /* Buffer is filled up, process it */
                sha3_process_block72(ctx->state);
                /*bufpos = 0; - already is */
        }
-       ctx->bytes_queued = bufpos + SHA3_IBLK_BYTES;
+       ctx->bytes_queued = bufpos + ctx->input_block_bytes;
 #else
        /* +50 bytes code size, but a bit faster because of long-sized XORs */
        const uint8_t *data = buffer;
        unsigned bufpos = ctx->bytes_queued;
+       unsigned iblk_bytes = ctx->input_block_bytes;
 
        /* If already data in queue, continue queuing first */
-       while (len != 0 && bufpos != 0) {
-               uint8_t *buf = (uint8_t*)ctx->state;
-               buf[bufpos] ^= *data++;
-               len--;
-               bufpos++;
-               if (bufpos == SHA3_IBLK_BYTES) {
-                       bufpos = 0;
-                       goto do_block;
+       if (bufpos != 0) {
+               while (len != 0) {
+                       uint8_t *buf = (uint8_t*)ctx->state;
+                       buf[bufpos] ^= *data++;
+                       len--;
+                       bufpos++;
+                       if (bufpos == iblk_bytes) {
+                               bufpos = 0;
+                               goto do_block;
+                       }
                }
        }
 
        /* Absorb complete blocks */
-       while (len >= SHA3_IBLK_BYTES) {
+       while (len >= iblk_bytes) {
                /* XOR data onto beginning of state[].
                 * We try to be efficient - operate one word at a time, not byte.
                 * Careful wrt unaligned access: can't just use "*(long*)data"!
                 */
-               unsigned count = SHA3_IBLK_BYTES / sizeof(long);
+               unsigned count = iblk_bytes / sizeof(long);
                long *buf = (long*)ctx->state;
                do {
                        long v;
@@ -1412,7 +1413,7 @@ void FAST_FUNC sha3_hash(sha3_ctx_t *ctx, const void *buffer, size_t len)
                        *buf++ ^= v;
                        data += sizeof(long);
                } while (--count);
-               len -= SHA3_IBLK_BYTES;
+               len -= iblk_bytes;
  do_block:
                sha3_process_block72(ctx->state);
        }
@@ -1433,8 +1434,22 @@ void FAST_FUNC sha3_end(sha3_ctx_t *ctx, void *resbuf)
 {
        /* Padding */
        uint8_t *buf = (uint8_t*)ctx->state;
-       buf[ctx->bytes_queued]   ^= 1;
-       buf[SHA3_IBLK_BYTES - 1] ^= 0x80;
+       /*
+        * Keccak block padding is: add 1 bit after last bit of input,
+        * then add zero bits until the end of block, and add the last 1 bit
+        * (the last bit in the block) - the "10*1" pattern.
+        * SHA3 standard appends additional two bits, 01,  before that padding:
+        *
+        * SHA3-224(M) = KECCAK[448](M||01, 224)
+        * SHA3-256(M) = KECCAK[512](M||01, 256)
+        * SHA3-384(M) = KECCAK[768](M||01, 384)
+        * SHA3-512(M) = KECCAK[1024](M||01, 512)
+        * (M is the input, || is bit concatenation)
+        *
+        * The 6 below contains 01 "SHA3" bits and the first 1 "Keccak" bit:
+        */
+       buf[ctx->bytes_queued]          ^= 6; /* bit pattern 00000110 */
+       buf[ctx->input_block_bytes - 1] ^= 0x80;
 
        sha3_process_block72(ctx->state);
 
index 82fada633f11c05497286781cd64b9810ec4082a..2cd8e3bf21051c21eb0babf2d97d83dad248cdd6 100755 (executable)
@@ -1,3 +1,3 @@
 #!/bin/sh
 
-. ./md5sum.tests sha3sum c29d77bc548fa2b20a04c861400a5360879c52156e2a54a3415b99a9a3123e1d5f36714a24eca8c1f05a8e2d8ba859c930d41141f64a255c6794436fc99c486a
+. ./md5sum.tests sha3sum 11659f09370139f8ef384f4a6260947fafa6e4fcd87a1ef3f35410e9