From 84bd74057f0d2543ddbc7de2351b0a310df329c6 Mon Sep 17 00:00:00 2001 From: Felix Fietkau Date: Sun, 25 Dec 2016 16:40:05 +0100 Subject: [PATCH] build: use mkhash to replace various quirky md5sum/openssl calls Signed-off-by: Felix Fietkau --- include/depends.mk | 2 +- include/download.mk | 2 +- include/image.mk | 2 +- include/kernel-defaults.mk | 2 +- include/prereq-build.mk | 5 ----- include/quilt.mk | 3 ++- include/scan.mk | 2 +- include/shell.sh | 7 ------- package/base-files/Makefile | 2 +- package/kernel/linux/Makefile | 2 +- package/network/services/dropbear/Makefile | 2 +- package/network/utils/iptables/Makefile | 2 +- package/system/procd/Makefile | 2 +- package/utils/busybox/Makefile | 2 +- rules.mk | 2 +- scripts/combined-ext-image.sh | 2 +- scripts/combined-image.sh | 2 +- scripts/download.pl | 8 ++------ scripts/ipkg-make-index.sh | 2 +- scripts/om-fwupgradecfg-gen.sh | 10 +++++----- target/linux/ramips/image/mt7620.mk | 6 ++---- target/linux/ramips/image/mt7621.mk | 4 ++-- 22 files changed, 28 insertions(+), 45 deletions(-) diff --git a/include/depends.mk b/include/depends.mk index 7135e52c0f..293409d8b7 100644 --- a/include/depends.mk +++ b/include/depends.mk @@ -13,7 +13,7 @@ DEP_FINDPARAMS := -x "*/.svn*" -x ".*" -x "*:*" -x "*\!*" -x "* *" -x "*\\\#*" -x "*/.*_check" -x "*/.*.swp" -find_md5=$(SH_FUNC) find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) | md5s +find_md5=find $(wildcard $(1)) -type f $(patsubst -x,-and -not -path,$(DEP_FINDPARAMS) $(2)) | mkhash md5 define rdep .PRECIOUS: $(2) diff --git a/include/download.mk b/include/download.mk index 09b929bd77..b87686dd07 100644 --- a/include/download.mk +++ b/include/download.mk @@ -61,7 +61,7 @@ else check_warn = $(if $(filter-out undefined,$(origin F_$(1))),$(filter ,$(shell $(call F_$(1),$(2),$(3),$(4)) >&2)),$(check_warn_nofix)) endif -gen_sha256sum = $(shell openssl dgst -sha256 $(DL_DIR)/$(1) | awk '{print $$2}') +gen_sha256sum = $(shell mkhash sha256 $(DL_DIR)/$(1)) ifdef FIXUP F_hash_deprecated = $(SCRIPT_DIR)/fixup-makefile.pl $(CURDIR)/Makefile fix-hash $(3) $(call gen_sha256sum,$(1)) $(2) diff --git a/include/image.mk b/include/image.mk index 93c3bbd246..65f9b0e96d 100644 --- a/include/image.mk +++ b/include/image.mk @@ -33,7 +33,7 @@ param_get_default = $(firstword $(call param_get,$(1),$(2)) $(3)) param_mangle = $(subst $(space),_,$(strip $(1))) param_unmangle = $(subst _,$(space),$(1)) -mkfs_packages_id = $(shell echo $(sort $(1)) | md5sum | head -c 8) +mkfs_packages_id = $(shell echo $(sort $(1)) | mkhash md5 | head -c 8) mkfs_target_dir = $(if $(call param_get,pkg,$(1)),$(KDIR)/target-dir-$(call param_get,pkg,$(1)),$(TARGET_DIR)) KDIR=$(KERNEL_BUILD_DIR) diff --git a/include/kernel-defaults.mk b/include/kernel-defaults.mk index a17b489c98..10bd67a571 100644 --- a/include/kernel-defaults.mk +++ b/include/kernel-defaults.mk @@ -125,7 +125,7 @@ define Kernel/Configure/Default cp $(LINUX_DIR)/.config.set $(LINUX_DIR)/.config.prev; \ } $(_SINGLE) [ -d $(LINUX_DIR)/user_headers ] || $(MAKE) $(KERNEL_MAKEOPTS) INSTALL_HDR_PATH=$(LINUX_DIR)/user_headers headers_install - $(SH_FUNC) grep '=[ym]' $(LINUX_DIR)/.config.set | LC_ALL=C sort | md5s > $(LINUX_DIR)/.vermagic + grep '=[ym]' $(LINUX_DIR)/.config.set | LC_ALL=C sort | mkhash md5 > $(LINUX_DIR)/.vermagic endef define Kernel/Configure/Initramfs diff --git a/include/prereq-build.mk b/include/prereq-build.mk index 496fc04f1b..af4e8b5e14 100644 --- a/include/prereq-build.mk +++ b/include/prereq-build.mk @@ -127,11 +127,6 @@ $(eval $(call SetupHostCommand,stat,Cannot find a file stat utility, \ gstat -c%s $(TMP_DIR)/.host.mk, \ stat -c%s $(TMP_DIR)/.host.mk)) -$(eval $(call SetupHostCommand,md5sum,, \ - gmd5sum /dev/null | grep d41d8cd98f00b204e9800998ecf8427e, \ - md5sum /dev/null | grep d41d8cd98f00b204e9800998ecf8427e, \ - $(SCRIPT_DIR)/md5sum /dev/null | grep d41d8cd98f00b204e9800998ecf8427e)) - $(eval $(call SetupHostCommand,unzip,Please install 'unzip', \ unzip 2>&1 | grep zipfile, \ unzip)) diff --git a/include/quilt.mk b/include/quilt.mk index cd392a7c5d..796e7160c4 100644 --- a/include/quilt.mk +++ b/include/quilt.mk @@ -149,7 +149,8 @@ define Quilt/Template echo "The source directory contains no quilt patches."; \ false; \ } - @[ -n "$$$$(ls $(1)/patches/series)" -o "$$$$(cat $(1)/patches/series | md5sum)" = "$$(sort $(1)/patches/series | md5sum)" ] || { \ + @[ -n "$$$$(ls $(1)/patches/series)" -o \ + "$$$$(cat $(1)/patches/series | mkhash md5)" = "$$(sort $(1)/patches/series | mkhash md5)" ] || { \ echo "The patches are not sorted in the right order. Please fix."; \ false; \ } diff --git a/include/scan.mk b/include/scan.mk index e6bc0688b7..5b1f53bdca 100644 --- a/include/scan.mk +++ b/include/scan.mk @@ -85,7 +85,7 @@ $(TMP_DIR)/info/.files-$(SCAN_TARGET).mk: $(FILELIST) $(TARGET_STAMP):: +( \ $(NO_TRACE_MAKE) $(FILELIST); \ - MD5SUM=$$(cat $(FILELIST) $(OVERRIDELIST) | (md5sum || md5) 2>/dev/null | awk '{print $$1}'); \ + MD5SUM=$$(cat $(FILELIST) $(OVERRIDELIST) | mkhash md5 | awk '{print $$1}'); \ [ -f "$@.$$MD5SUM" ] || { \ rm -f $@.*; \ touch $@.$$MD5SUM; \ diff --git a/include/shell.sh b/include/shell.sh index 8f6f6f04bb..6ee0cf6030 100644 --- a/include/shell.sh +++ b/include/shell.sh @@ -13,10 +13,3 @@ isset() { eval "var=\"\${$1}\"" [ -n "$var" ] } - -md5s() { - cat "$@" | ( - md5sum 2>/dev/null || - md5 - ) | awk '{print $1}' -} diff --git a/package/base-files/Makefile b/package/base-files/Makefile index 85406a035d..4586f8189d 100644 --- a/package/base-files/Makefile +++ b/package/base-files/Makefile @@ -23,7 +23,7 @@ PKG_CONFIG_DEPENDS := CONFIG_SIGNED_PACKAGES CONFIG_TARGET_INIT_PATH CONFIG_TARG include $(INCLUDE_DIR)/package.mk ifneq ($(DUMP),1) - STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell $(SH_FUNC) echo $(CONFIG_TARGET_INIT_PATH) | md5s) + STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell echo $(CONFIG_TARGET_INIT_PATH) | mkhash md5) TARGET:=-$(BOARD) endif diff --git a/package/kernel/linux/Makefile b/package/kernel/linux/Makefile index 7f38f3e6e2..c2d10ec188 100644 --- a/package/kernel/linux/Makefile +++ b/package/kernel/linux/Makefile @@ -24,7 +24,7 @@ export SHELL:=/bin/sh include $(INCLUDE_DIR)/package.mk ifeq ($(DUMP),) - STAMP_BUILT:=$(STAMP_BUILT)_$(firstword $(shell $(SCRIPT_DIR)/kconfig.pl $(LINUX_DIR)/.config | md5sum)) + STAMP_BUILT:=$(STAMP_BUILT)_$(shell $(SCRIPT_DIR)/kconfig.pl $(LINUX_DIR)/.config | mkhash md5) -include $(LINUX_DIR)/.config endif diff --git a/package/network/services/dropbear/Makefile b/package/network/services/dropbear/Makefile index c8d680acc1..e6bae9f21f 100644 --- a/package/network/services/dropbear/Makefile +++ b/package/network/services/dropbear/Makefile @@ -28,7 +28,7 @@ PKG_CONFIG_DEPENDS:=CONFIG_TARGET_INIT_PATH CONFIG_DROPBEAR_ECC CONFIG_DROPBEAR_ include $(INCLUDE_DIR)/package.mk ifneq ($(DUMP),1) - STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell $(SH_FUNC) echo $(CONFIG_TARGET_INIT_PATH) | md5s) + STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell echo $(CONFIG_TARGET_INIT_PATH) | mkhash md5) endif define Package/dropbear/Default diff --git a/package/network/utils/iptables/Makefile b/package/network/utils/iptables/Makefile index ee7ec476cf..ac869f0385 100644 --- a/package/network/utils/iptables/Makefile +++ b/package/network/utils/iptables/Makefile @@ -28,7 +28,7 @@ include $(INCLUDE_DIR)/package.mk ifeq ($(DUMP),) -include $(LINUX_DIR)/.config include $(INCLUDE_DIR)/netfilter.mk - STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell $(SH_FUNC) grep 'NETFILTER' $(LINUX_DIR)/.config | md5s) + STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell grep 'NETFILTER' $(LINUX_DIR)/.config | mkhash md5) endif diff --git a/package/system/procd/Makefile b/package/system/procd/Makefile index 03f9422f5d..c4b1366efa 100644 --- a/package/system/procd/Makefile +++ b/package/system/procd/Makefile @@ -33,7 +33,7 @@ include $(INCLUDE_DIR)/package.mk include $(INCLUDE_DIR)/cmake.mk ifeq ($(DUMP),) - STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell $(SH_FUNC) echo $(CONFIG_TARGET_INIT_PATH) | md5s) + STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell echo $(CONFIG_TARGET_INIT_PATH) | mkhash md5) endif CMAKE_OPTIONS += -DEARLY_PATH="$(TARGET_INIT_PATH)" diff --git a/package/utils/busybox/Makefile b/package/utils/busybox/Makefile index c27f1fce58..e4f159ef83 100644 --- a/package/utils/busybox/Makefile +++ b/package/utils/busybox/Makefile @@ -27,7 +27,7 @@ PKG_LICENSE_FILES:=LICENSE archival/libarchive/bz/LICENSE include $(INCLUDE_DIR)/package.mk ifeq ($(DUMP),) - STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell $(SH_FUNC) grep '^CONFIG_BUSYBOX_' $(TOPDIR)/.config | md5s) + STAMP_CONFIGURED:=$(strip $(STAMP_CONFIGURED))_$(shell grep '^CONFIG_BUSYBOX_' $(TOPDIR)/.config | mkhash md5) endif ifneq ($(findstring c,$(OPENWRT_VERBOSE)),) diff --git a/rules.mk b/rules.mk index b1d9926270..04fd936660 100644 --- a/rules.mk +++ b/rules.mk @@ -383,7 +383,7 @@ endef # $(1) => Input directory define sha256sums (cd $(1); find . -maxdepth 1 -type f -not -name 'sha256sums' -printf "%P\n" | sort | \ - xargs openssl dgst -sha256 | sed -ne 's!^SHA256(\(.*\))= \(.*\)$$!\2 *\1!p' > sha256sums) + xargs -r $(STAGING_DIR_HOST)/bin/mkhash -n sha256 | sed -ne 's!^\(.*\) \(.*\)$$!\1 *\2!p' > sha256sums) endef # file extension diff --git a/scripts/combined-ext-image.sh b/scripts/combined-ext-image.sh index 374fe6e344..0c7b1d993d 100755 --- a/scripts/combined-ext-image.sh +++ b/scripts/combined-ext-image.sh @@ -47,7 +47,7 @@ while [ "$#" -gt 1 ] [ ! -f "$file" ] && echo "$ME: Not a valid file: $file" && usage FILES="$FILES $file" - md5=$(cat "$file" | md5sum -) + md5=$(mkhash md5 "$file") printf "%-32s%08x%32s" "$filename" $(stat -c "%s" "$file") "${md5%% *}" >> $IMG_OUT shift 2 done diff --git a/scripts/combined-image.sh b/scripts/combined-image.sh index 5472b2cfd2..4e00cb89a2 100644 --- a/scripts/combined-image.sh +++ b/scripts/combined-image.sh @@ -16,7 +16,7 @@ dd if="$1" of="$kern" bs=$BLKSZ conv=sync 2>/dev/null dd if="$2" of="$root" bs=$BLKSZ conv=sync 2>/dev/null # Calculate md5sum over combined kernel and rootfs image. -md5=$(cat "$kern" "$root" | md5sum -) +md5=$(cat "$kern" "$root" | mkhash md5) # Write image header followed by kernel and rootfs image. # The header is padded to 64k, format is: diff --git a/scripts/download.pl b/scripts/download.pl index ad8b837288..0398c564ef 100755 --- a/scripts/download.pl +++ b/scripts/download.pl @@ -65,12 +65,8 @@ sub hash_cmd() { my $len = length($file_hash); my $cmd; - $len == 64 and return "openssl dgst -sha256 | sed -e 's,.*= ,,'"; - $len == 32 and do { - my $cmd = which("md5sum") || which("md5") || die 'no md5 checksum program found, please install md5 or md5sum'; - chomp $cmd; - return $cmd; - }; + $len == 64 and return "mkhash sha256"; + $len == 32 and return "mkhash md5"; return undef; } diff --git a/scripts/ipkg-make-index.sh b/scripts/ipkg-make-index.sh index c99c5fe8a3..dcd11ca191 100755 --- a/scripts/ipkg-make-index.sh +++ b/scripts/ipkg-make-index.sh @@ -18,7 +18,7 @@ for pkg in `find $pkg_dir -name '*.ipk' | sort`; do [[ "$name" = "libc" ]] && continue echo "Generating index for package $pkg" >&2 file_size=$(ls -l $pkg | awk '{print $5}') - sha256sum=$(openssl dgst -sha256 $pkg | awk '{print $2}') + sha256sum=$(mkhash sha256 $pkg) # Take pains to make variable value sed-safe sed_safe_pkg=`echo $pkg | sed -e 's/^\.\///g' -e 's/\\//\\\\\\//g'` tar -xzOf $pkg ./control.tar.gz | tar xzOf - ./control | sed -e "s/^Description:/Filename: $sed_safe_pkg\\ diff --git a/scripts/om-fwupgradecfg-gen.sh b/scripts/om-fwupgradecfg-gen.sh index 6c3b74cc1c..bb30252aa0 100644 --- a/scripts/om-fwupgradecfg-gen.sh +++ b/scripts/om-fwupgradecfg-gen.sh @@ -41,16 +41,16 @@ esac CHECK_BS=65536 KERNEL_SIZE=$(stat -c%s "$KERNEL_PATH") -KERNEL_MD5=$(md5=$(md5sum $KERNEL_PATH); echo ${md5%% *}) -KERNEL_SHA256=$(openssl dgst -sha256 $KERNEL_PATH | awk '{print $2}') +KERNEL_MD5=$(mkhash md5 $KERNEL_PATH) +KERNEL_SHA256=$(mkhash sha256 $KERNEL_PATH) KERNEL_PART_SIZE=$(size=$(($KERNEL_SIZE / $FLASH_BS)); [ $(($size * $FLASH_BS)) -lt $KERNEL_SIZE ] && size=$(($size + 1)); echo $(($size * $FLASH_BS / 1024))) ROOTFS_FLASH_ADDR=$(addr=$(($KERNEL_FLASH_ADDR + ($KERNEL_PART_SIZE * 1024))); printf "0x%x" $addr) ROOTFS_SIZE=$(stat -c%s "$ROOTFS_PATH") ROOTFS_CHECK_BLOCKS=$((($ROOTFS_SIZE / $CHECK_BS) - $MD5_SKIP_BLOCKS)) -ROOTFS_MD5=$(md5=$(dd if=$ROOTFS_PATH bs=$CHECK_BS count=$ROOTFS_CHECK_BLOCKS 2>&- | md5sum); echo ${md5%% *}) -ROOTFS_MD5_FULL=$(md5=$(md5sum $ROOTFS_PATH); echo ${md5%% *}) -ROOTFS_SHA256_FULL=$(openssl dgst -sha256 $ROOTFS_PATH | awk '{print $2}') +ROOTFS_MD5=$(dd if=$ROOTFS_PATH bs=$CHECK_BS count=$ROOTFS_CHECK_BLOCKS 2>&- | mkhash md5) +ROOTFS_MD5_FULL=$(mkhash md5 $ROOTFS_PATH) +ROOTFS_SHA256_FULL=$(mkhash sha256 $ROOTFS_PATH) ROOTFS_CHECK_SIZE=$(printf '0x%x' $(($ROOTFS_CHECK_BLOCKS * $CHECK_BS))) ROOTFS_PART_SIZE=$(($MAX_PART_SIZE - $KERNEL_PART_SIZE)) diff --git a/target/linux/ramips/image/mt7620.mk b/target/linux/ramips/image/mt7620.mk index 95542ae34a..1889bcf708 100644 --- a/target/linux/ramips/image/mt7620.mk +++ b/target/linux/ramips/image/mt7620.mk @@ -22,11 +22,9 @@ endef define Build/elecom-header cp $@ $(KDIR)/v_0.0.0.bin ( \ - $(STAGING_DIR_HOST)/bin/md5sum $(KDIR)/v_0.0.0.bin | \ - sed 's/ .*//' && \ + mkhash md5 $(KDIR)/v_0.0.0.bin && \ echo 458 \ - ) | $(STAGING_DIR_HOST)/bin/md5sum | \ - sed 's/ .*//' > $(KDIR)/v_0.0.0.md5 + ) | mkhash md5 > $(KDIR)/v_0.0.0.md5 $(STAGING_DIR_HOST)/bin/tar -cf $@ -C $(KDIR) v_0.0.0.bin v_0.0.0.md5 endef diff --git a/target/linux/ramips/image/mt7621.mk b/target/linux/ramips/image/mt7621.mk index 6d8551104c..6d9a7273d4 100644 --- a/target/linux/ramips/image/mt7621.mk +++ b/target/linux/ramips/image/mt7621.mk @@ -8,13 +8,13 @@ define Build/ubnt-erx-factory-image $(TAR) -cf $(1) --transform='s/^.*/compat/' $(1).compat; \ \ $(TAR) -rf $(1) --transform='s/^.*/vmlinux.tmp/' $(KDIR)/tmp/$(KERNEL_INITRAMFS_IMAGE); \ - md5sum --binary $(KDIR)/tmp/$(KERNEL_INITRAMFS_IMAGE) | awk '{print $$1}'> $(1).md5; \ + mkhash md5 $(KDIR)/tmp/$(KERNEL_INITRAMFS_IMAGE) > $(1).md5; \ $(TAR) -rf $(1) --transform='s/^.*/vmlinux.tmp.md5/' $(1).md5; \ \ echo "dummy" > $(1).rootfs; \ $(TAR) -rf $(1) --transform='s/^.*/squashfs.tmp/' $(1).rootfs; \ \ - md5sum --binary $(1).rootfs | awk '{print $$1}'> $(1).md5; \ + mkhash md5 $(1).rootfs > $(1).md5; \ $(TAR) -rf $(1) --transform='s/^.*/squashfs.tmp.md5/' $(1).md5; \ \ echo '$(BOARD) $(VERSION_CODE) $(VERSION_NUMBER)' > $(1).version; \ -- 2.25.1