From b4c11c139717729b8257ee9382d4e5ed713d4dde Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 7 Dec 2014 00:44:00 +0100 Subject: [PATCH] libarchive: add capability to unpack to mem.buffer The performance and number of processes for a "depmod -a" with gzipped modules was abysmal. This patch adds a code path without fork, benefiting all users of xmalloc_open_zipped_read_close. "modinfo radeon.ko.gz", a single-file reader, got 30% faster. "depmod -a", which used to fork over 800 times, got 20% faster. Heavily based on a patch by Lauri Kasanen function old new delta setup_transformer_on_fd - 159 +159 transformer_write - 122 +122 fork_transformer - 112 +112 xmalloc_open_zipped_read_close 63 118 +55 read_bunzip 1866 1896 +30 xtransformer_write - 19 +19 unzip_main 2449 2462 +13 bbunpack 755 766 +11 unpack_lzma_stream 2717 2723 +6 unpack_xz_stream 2393 2397 +4 unpack_Z_stream 1173 1175 +2 inflate_unzip 111 105 -6 check_signature16 70 63 -7 unpack_bz2_stream 359 349 -10 unpack_unxz 12 - -12 unpack_unlzma 12 - -12 unpack_uncompress 12 - -12 unpack_gunzip 12 - -12 unpack_bunzip2 12 - -12 open_transformer 106 92 -14 inflate_unzip_internal 1945 1916 -29 unpack_gz_stream 693 655 -38 open_zipped 89 47 -42 setup_unzip_on_fd 142 53 -89 ------------------------------------------------------------------------------ (add/remove: 4/5 grow/shrink: 7/8 up/down: 533/-295) Total: 238 bytes Signed-off-by: Denys Vlasenko --- archival/bbunzip.c | 37 +--- archival/libarchive/decompress_bunzip2.c | 11 +- archival/libarchive/decompress_gunzip.c | 30 ++- archival/libarchive/decompress_uncompress.c | 12 +- archival/libarchive/decompress_unlzma.c | 12 +- archival/libarchive/decompress_unxz.c | 6 +- archival/libarchive/get_header_tar_bz2.c | 2 +- archival/libarchive/get_header_tar_gz.c | 2 +- archival/libarchive/get_header_tar_lzma.c | 2 +- archival/libarchive/open_transformer.c | 199 ++++++++++++++++---- archival/tar.c | 4 +- archival/unzip.c | 4 +- include/bb_archive.h | 42 +++-- include/libbb.h | 3 +- 14 files changed, 241 insertions(+), 125 deletions(-) diff --git a/archival/bbunzip.c b/archival/bbunzip.c index 84f58f138..90aac1427 100644 --- a/archival/bbunzip.c +++ b/archival/bbunzip.c @@ -122,6 +122,8 @@ int FAST_FUNC bbunpack(char **argv, if (!(option_mask32 & SEAMLESS_MAGIC)) { init_transformer_state(&xstate); xstate.check_signature = 1; + /*xstate.src_fd = STDIN_FILENO; - already is */ + xstate.dst_fd = STDOUT_FILENO; status = unpacker(&xstate); if (status < 0) exitcode = 1; @@ -226,18 +228,13 @@ char* FAST_FUNC make_new_name_generic(char *filename, const char *expected_ext) //applet:IF_UNCOMPRESS(APPLET(uncompress, BB_DIR_BIN, BB_SUID_DROP)) //kbuild:lib-$(CONFIG_UNCOMPRESS) += bbunzip.o #if ENABLE_UNCOMPRESS -static -IF_DESKTOP(long long) int FAST_FUNC unpack_uncompress(transformer_state_t *xstate) -{ - return unpack_Z_stream(xstate, STDIN_FILENO, STDOUT_FILENO); -} int uncompress_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int uncompress_main(int argc UNUSED_PARAM, char **argv) { getopt32(argv, "cf"); argv += optind; - return bbunpack(argv, unpack_uncompress, make_new_name_generic, "Z"); + return bbunpack(argv, unpack_Z_stream, make_new_name_generic, "Z"); } #endif @@ -324,11 +321,6 @@ char* FAST_FUNC make_new_name_gunzip(char *filename, const char *expected_ext UN } return filename; } -static -IF_DESKTOP(long long) int FAST_FUNC unpack_gunzip(transformer_state_t *xstate) -{ - return unpack_gz_stream(xstate, STDIN_FILENO, STDOUT_FILENO); -} /* * Linux kernel build uses gzip -d -n. We accept and ignore it. * Man page says: @@ -355,7 +347,7 @@ int gunzip_main(int argc UNUSED_PARAM, char **argv) if (applet_name[1] == 'c') option_mask32 |= OPT_STDOUT | SEAMLESS_MAGIC; - return bbunpack(argv, unpack_gunzip, make_new_name_gunzip, /*unused:*/ NULL); + return bbunpack(argv, unpack_gz_stream, make_new_name_gunzip, /*unused:*/ NULL); } #endif @@ -395,11 +387,6 @@ int gunzip_main(int argc UNUSED_PARAM, char **argv) //kbuild:lib-$(CONFIG_BZIP2) += bbunzip.o //kbuild:lib-$(CONFIG_BUNZIP2) += bbunzip.o #if ENABLE_BUNZIP2 -static -IF_DESKTOP(long long) int FAST_FUNC unpack_bunzip2(transformer_state_t *xstate) -{ - return unpack_bz2_stream(xstate, STDIN_FILENO, STDOUT_FILENO); -} int bunzip2_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int bunzip2_main(int argc UNUSED_PARAM, char **argv) { @@ -408,7 +395,7 @@ int bunzip2_main(int argc UNUSED_PARAM, char **argv) if (applet_name[2] == 'c') /* bzcat */ option_mask32 |= OPT_STDOUT; - return bbunpack(argv, unpack_bunzip2, make_new_name_generic, "bz2"); + return bbunpack(argv, unpack_bz2_stream, make_new_name_generic, "bz2"); } #endif @@ -494,11 +481,6 @@ int bunzip2_main(int argc UNUSED_PARAM, char **argv) //applet:IF_LZMA(APPLET_ODDNAME(lzma, unlzma, BB_DIR_USR_BIN, BB_SUID_DROP, lzma)) //kbuild:lib-$(CONFIG_UNLZMA) += bbunzip.o #if ENABLE_UNLZMA -static -IF_DESKTOP(long long) int FAST_FUNC unpack_unlzma(transformer_state_t *xstate) -{ - return unpack_lzma_stream(xstate, STDIN_FILENO, STDOUT_FILENO); -} int unlzma_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int unlzma_main(int argc UNUSED_PARAM, char **argv) { @@ -513,7 +495,7 @@ int unlzma_main(int argc UNUSED_PARAM, char **argv) option_mask32 |= OPT_STDOUT; argv += optind; - return bbunpack(argv, unpack_unlzma, make_new_name_generic, "lzma"); + return bbunpack(argv, unpack_lzma_stream, make_new_name_generic, "lzma"); } #endif @@ -537,11 +519,6 @@ int unlzma_main(int argc UNUSED_PARAM, char **argv) //applet:IF_XZ(APPLET_ODDNAME(xz, unxz, BB_DIR_USR_BIN, BB_SUID_DROP, xz)) //kbuild:lib-$(CONFIG_UNXZ) += bbunzip.o #if ENABLE_UNXZ -static -IF_DESKTOP(long long) int FAST_FUNC unpack_unxz(transformer_state_t *xstate) -{ - return unpack_xz_stream(xstate, STDIN_FILENO, STDOUT_FILENO); -} int unxz_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE; int unxz_main(int argc UNUSED_PARAM, char **argv) { @@ -556,6 +533,6 @@ int unxz_main(int argc UNUSED_PARAM, char **argv) option_mask32 |= OPT_STDOUT; argv += optind; - return bbunpack(argv, unpack_unxz, make_new_name_generic, "xz"); + return bbunpack(argv, unpack_xz_stream, make_new_name_generic, "xz"); } #endif diff --git a/archival/libarchive/decompress_bunzip2.c b/archival/libarchive/decompress_bunzip2.c index 36237e221..fe5953da2 100644 --- a/archival/libarchive/decompress_bunzip2.c +++ b/archival/libarchive/decompress_bunzip2.c @@ -731,7 +731,7 @@ void FAST_FUNC dealloc_bunzip(bunzip_data *bd) /* Decompress src_fd to dst_fd. Stops at end of bzip data, not end of file. */ IF_DESKTOP(long long) int FAST_FUNC -unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd) +unpack_bz2_stream(transformer_state_t *xstate) { IF_DESKTOP(long long total_written = 0;) bunzip_data *bd; @@ -739,14 +739,14 @@ unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd) int i; unsigned len; - if (check_signature16(xstate, src_fd, BZIP2_MAGIC)) + if (check_signature16(xstate, BZIP2_MAGIC)) return -1; outbuf = xmalloc(IOBUF_SIZE); len = 0; while (1) { /* "Process one BZ... stream" loop */ - i = start_bunzip(&bd, src_fd, outbuf + 2, len); + i = start_bunzip(&bd, xstate->src_fd, outbuf + 2, len); if (i == 0) { while (1) { /* "Produce some output bytes" loop */ @@ -756,8 +756,7 @@ unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd) i = IOBUF_SIZE - i; /* number of bytes produced */ if (i == 0) /* EOF? */ break; - if (i != full_write(dst_fd, outbuf, i)) { - bb_error_msg("short write"); + if (i != transformer_write(xstate, outbuf, i)) { i = RETVAL_SHORT_WRITE; goto release_mem; } @@ -790,7 +789,7 @@ unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd) len = bd->inbufCount - bd->inbufPos; memcpy(outbuf, &bd->inbuf[bd->inbufPos], len); if (len < 2) { - if (safe_read(src_fd, outbuf + len, 2 - len) != 2 - len) + if (safe_read(xstate->src_fd, outbuf + len, 2 - len) != 2 - len) break; len = 2; } diff --git a/archival/libarchive/decompress_gunzip.c b/archival/libarchive/decompress_gunzip.c index 62a3d78b6..1360abef7 100644 --- a/archival/libarchive/decompress_gunzip.c +++ b/archival/libarchive/decompress_gunzip.c @@ -971,7 +971,7 @@ static int inflate_get_next_window(STATE_PARAM_ONLY) /* Called from unpack_gz_stream() and inflate_unzip() */ static IF_DESKTOP(long long) int -inflate_unzip_internal(STATE_PARAM int in, int out) +inflate_unzip_internal(STATE_PARAM transformer_state_t *xstate) { IF_DESKTOP(long long) int n = 0; ssize_t nwrote; @@ -980,7 +980,7 @@ inflate_unzip_internal(STATE_PARAM int in, int out) gunzip_window = xmalloc(GUNZIP_WSIZE); gunzip_outbuf_count = 0; gunzip_bytes_out = 0; - gunzip_src_fd = in; + gunzip_src_fd = xstate->src_fd; /* (re) initialize state */ method = -1; @@ -1002,9 +1002,8 @@ inflate_unzip_internal(STATE_PARAM int in, int out) while (1) { int r = inflate_get_next_window(PASS_STATE_ONLY); - nwrote = full_write(out, gunzip_window, gunzip_outbuf_count); - if (nwrote != (ssize_t)gunzip_outbuf_count) { - bb_perror_msg("write"); + nwrote = transformer_write(xstate, gunzip_window, gunzip_outbuf_count); + if (nwrote == (ssize_t)-1) { n = -1; goto ret; } @@ -1034,7 +1033,7 @@ inflate_unzip_internal(STATE_PARAM int in, int out) /* For unzip */ IF_DESKTOP(long long) int FAST_FUNC -inflate_unzip(transformer_state_t *xstate, int in, int out) +inflate_unzip(transformer_state_t *xstate) { IF_DESKTOP(long long) int n; DECLARE_STATE; @@ -1045,7 +1044,7 @@ inflate_unzip(transformer_state_t *xstate, int in, int out) // bytebuffer_max = 0x8000; bytebuffer_offset = 4; bytebuffer = xmalloc(bytebuffer_max); - n = inflate_unzip_internal(PASS_STATE in, out); + n = inflate_unzip_internal(PASS_STATE xstate); free(bytebuffer); xstate->crc32 = gunzip_crc; @@ -1169,8 +1168,7 @@ static int check_header_gzip(STATE_PARAM transformer_state_t *xstate) } } - if (xstate) - xstate->mtime = SWAP_LE32(header.formatted.mtime); + xstate->mtime = SWAP_LE32(header.formatted.mtime); /* Read the header checksum */ if (header.formatted.flags & 0x02) { @@ -1182,27 +1180,27 @@ static int check_header_gzip(STATE_PARAM transformer_state_t *xstate) } IF_DESKTOP(long long) int FAST_FUNC -unpack_gz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) +unpack_gz_stream(transformer_state_t *xstate) { uint32_t v32; IF_DESKTOP(long long) int total, n; DECLARE_STATE; #if !ENABLE_FEATURE_SEAMLESS_Z - if (check_signature16(xstate, src_fd, GZIP_MAGIC)) + if (check_signature16(xstate, GZIP_MAGIC)) return -1; #else - if (xstate && xstate->check_signature) { + if (xstate->check_signature) { uint16_t magic2; - if (full_read(src_fd, &magic2, 2) != 2) { + if (full_read(xstate->src_fd, &magic2, 2) != 2) { bad_magic: bb_error_msg("invalid magic"); return -1; } if (magic2 == COMPRESS_MAGIC) { xstate->check_signature = 0; - return unpack_Z_stream(xstate, src_fd, dst_fd); + return unpack_Z_stream(xstate); } if (magic2 != GZIP_MAGIC) goto bad_magic; @@ -1215,7 +1213,7 @@ unpack_gz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) to_read = -1; // bytebuffer_max = 0x8000; bytebuffer = xmalloc(bytebuffer_max); - gunzip_src_fd = src_fd; + gunzip_src_fd = xstate->src_fd; again: if (!check_header_gzip(PASS_STATE xstate)) { @@ -1224,7 +1222,7 @@ unpack_gz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) goto ret; } - n = inflate_unzip_internal(PASS_STATE src_fd, dst_fd); + n = inflate_unzip_internal(PASS_STATE xstate); if (n < 0) { total = -1; goto ret; diff --git a/archival/libarchive/decompress_uncompress.c b/archival/libarchive/decompress_uncompress.c index cb3d55a88..496d864a7 100644 --- a/archival/libarchive/decompress_uncompress.c +++ b/archival/libarchive/decompress_uncompress.c @@ -73,7 +73,7 @@ */ IF_DESKTOP(long long) int FAST_FUNC -unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd) +unpack_Z_stream(transformer_state_t *xstate) { IF_DESKTOP(long long total_written = 0;) IF_DESKTOP(long long) int retval = -1; @@ -102,7 +102,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd) /* block compress mode -C compatible with 2.0 */ int block_mode; /* = BLOCK_MODE; */ - if (check_signature16(xstate, src_fd, COMPRESS_MAGIC)) + if (check_signature16(xstate, COMPRESS_MAGIC)) return -1; inbuf = xzalloc(IBUFSIZ + 64); @@ -114,7 +114,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd) /* xread isn't good here, we have to return - caller may want * to do some cleanup (e.g. delete incomplete unpacked file etc) */ - if (full_read(src_fd, inbuf, 1) != 1) { + if (full_read(xstate->src_fd, inbuf, 1) != 1) { bb_error_msg("short read"); goto err; } @@ -166,7 +166,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd) } if (insize < (int) (IBUFSIZ + 64) - IBUFSIZ) { - rsize = safe_read(src_fd, inbuf + insize, IBUFSIZ); + rsize = safe_read(xstate->src_fd, inbuf + insize, IBUFSIZ); if (rsize < 0) bb_error_msg_and_die(bb_msg_read_error); insize += rsize; @@ -274,7 +274,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd) } if (outpos >= OBUFSIZ) { - xwrite(dst_fd, outbuf, outpos); + xtransformer_write(xstate, outbuf, outpos); IF_DESKTOP(total_written += outpos;) outpos = 0; } @@ -301,7 +301,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd) } while (rsize > 0); if (outpos > 0) { - xwrite(dst_fd, outbuf, outpos); + xtransformer_write(xstate, outbuf, outpos); IF_DESKTOP(total_written += outpos;) } diff --git a/archival/libarchive/decompress_unlzma.c b/archival/libarchive/decompress_unlzma.c index ccedac49d..c8622f97b 100644 --- a/archival/libarchive/decompress_unlzma.c +++ b/archival/libarchive/decompress_unlzma.c @@ -206,7 +206,7 @@ enum { IF_DESKTOP(long long) int FAST_FUNC -unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst_fd) +unpack_lzma_stream(transformer_state_t *xstate) { IF_DESKTOP(long long total_written = 0;) lzma_header_t header; @@ -223,7 +223,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst int state = 0; uint32_t rep0 = 1, rep1 = 1, rep2 = 1, rep3 = 1; - if (full_read(src_fd, &header, sizeof(header)) != sizeof(header) + if (full_read(xstate->src_fd, &header, sizeof(header)) != sizeof(header) || header.pos >= (9 * 5 * 5) ) { bb_error_msg("bad lzma header"); @@ -258,7 +258,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst p[i] = (1 << RC_MODEL_TOTAL_BITS) >> 1; } - rc = rc_init(src_fd); /*, RC_BUFFER_SIZE); */ + rc = rc_init(xstate->src_fd); /*, RC_BUFFER_SIZE); */ while (global_pos + buffer_pos < header.dst_size) { int pos_state = (buffer_pos + global_pos) & pos_state_mask; @@ -306,7 +306,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst if (buffer_pos == header.dict_size) { buffer_pos = 0; global_pos += header.dict_size; - if (full_write(dst_fd, buffer, header.dict_size) != (ssize_t)header.dict_size) + if (transformer_write(xstate, buffer, header.dict_size) != (ssize_t)header.dict_size) goto bad; IF_DESKTOP(total_written += header.dict_size;) } @@ -440,7 +440,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst if (buffer_pos == header.dict_size) { buffer_pos = 0; global_pos += header.dict_size; - if (full_write(dst_fd, buffer, header.dict_size) != (ssize_t)header.dict_size) + if (transformer_write(xstate, buffer, header.dict_size) != (ssize_t)header.dict_size) goto bad; IF_DESKTOP(total_written += header.dict_size;) } @@ -455,7 +455,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst { IF_NOT_DESKTOP(int total_written = 0; /* success */) IF_DESKTOP(total_written += buffer_pos;) - if (full_write(dst_fd, buffer, buffer_pos) != (ssize_t)buffer_pos) { + if (transformer_write(xstate, buffer, buffer_pos) != (ssize_t)buffer_pos) { bad: total_written = -1; /* failure */ } diff --git a/archival/libarchive/decompress_unxz.c b/archival/libarchive/decompress_unxz.c index 6df54e131..1f408abfd 100644 --- a/archival/libarchive/decompress_unxz.c +++ b/archival/libarchive/decompress_unxz.c @@ -38,7 +38,7 @@ static uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc) #include "unxz/xz_dec_stream.c" IF_DESKTOP(long long) int FAST_FUNC -unpack_xz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) +unpack_xz_stream(transformer_state_t *xstate) { enum xz_ret xz_result; struct xz_buf iobuf; @@ -67,7 +67,7 @@ unpack_xz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) xz_result = X_OK; while (1) { if (iobuf.in_pos == iobuf.in_size) { - int rd = safe_read(src_fd, membuf, BUFSIZ); + int rd = safe_read(xstate->src_fd, membuf, BUFSIZ); if (rd < 0) { bb_error_msg(bb_msg_read_error); total = -1; @@ -104,7 +104,7 @@ unpack_xz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) // bb_error_msg("seek = seek_by_read; - open_transformer_with_sig(archive_handle->src_fd, unpack_bz2_stream, "bunzip2"); + fork_transformer_with_sig(archive_handle->src_fd, unpack_bz2_stream, "bunzip2"); archive_handle->offset = 0; while (get_header_tar(archive_handle) == EXIT_SUCCESS) continue; diff --git a/archival/libarchive/get_header_tar_gz.c b/archival/libarchive/get_header_tar_gz.c index 03284342b..b11f503dc 100644 --- a/archival/libarchive/get_header_tar_gz.c +++ b/archival/libarchive/get_header_tar_gz.c @@ -11,7 +11,7 @@ char FAST_FUNC get_header_tar_gz(archive_handle_t *archive_handle) /* Can't lseek over pipes */ archive_handle->seek = seek_by_read; - open_transformer_with_sig(archive_handle->src_fd, unpack_gz_stream, "gunzip"); + fork_transformer_with_sig(archive_handle->src_fd, unpack_gz_stream, "gunzip"); archive_handle->offset = 0; while (get_header_tar(archive_handle) == EXIT_SUCCESS) continue; diff --git a/archival/libarchive/get_header_tar_lzma.c b/archival/libarchive/get_header_tar_lzma.c index d565a217d..d228cbc13 100644 --- a/archival/libarchive/get_header_tar_lzma.c +++ b/archival/libarchive/get_header_tar_lzma.c @@ -14,7 +14,7 @@ char FAST_FUNC get_header_tar_lzma(archive_handle_t *archive_handle) /* Can't lseek over pipes */ archive_handle->seek = seek_by_read; - open_transformer_with_sig(archive_handle->src_fd, unpack_lzma_stream, "unlzma"); + fork_transformer_with_sig(archive_handle->src_fd, unpack_lzma_stream, "unlzma"); archive_handle->offset = 0; while (get_header_tar(archive_handle) == EXIT_SUCCESS) continue; diff --git a/archival/libarchive/open_transformer.c b/archival/libarchive/open_transformer.c index 584b15de4..9d762a859 100644 --- a/archival/libarchive/open_transformer.c +++ b/archival/libarchive/open_transformer.c @@ -11,11 +11,11 @@ void FAST_FUNC init_transformer_state(transformer_state_t *xstate) memset(xstate, 0, sizeof(*xstate)); } -int FAST_FUNC check_signature16(transformer_state_t *xstate, int src_fd, unsigned magic16) +int FAST_FUNC check_signature16(transformer_state_t *xstate, unsigned magic16) { - if (xstate && xstate->check_signature) { + if (xstate->check_signature) { uint16_t magic2; - if (full_read(src_fd, &magic2, 2) != 2 || magic2 != magic16) { + if (full_read(xstate->src_fd, &magic2, 2) != 2 || magic2 != magic16) { bb_error_msg("invalid magic"); #if 0 /* possible future extension */ if (xstate->check_signature > 1) @@ -27,6 +27,46 @@ int FAST_FUNC check_signature16(transformer_state_t *xstate, int src_fd, unsigne return 0; } +ssize_t FAST_FUNC transformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize) +{ + ssize_t nwrote; + + if (xstate->mem_output_size_max != 0) { + size_t pos = xstate->mem_output_size; + size_t size; + + size = (xstate->mem_output_size += bufsize); + if (size > xstate->mem_output_size_max) { + free(xstate->mem_output_buf); + xstate->mem_output_buf = NULL; + bb_perror_msg("buffer %u too small", (unsigned)xstate->mem_output_size_max); + nwrote = -1; + goto ret; + } + xstate->mem_output_buf = xrealloc(xstate->mem_output_buf, size); + memcpy(xstate->mem_output_buf + pos, buf, bufsize); + nwrote = bufsize; + } else { + nwrote = full_write(xstate->dst_fd, buf, bufsize); + if (nwrote != (ssize_t)bufsize) { + bb_perror_msg("write"); + nwrote = -1; + goto ret; + } + } + ret: + return nwrote; +} + +ssize_t FAST_FUNC xtransformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize) +{ + ssize_t nwrote = transformer_write(xstate, buf, bufsize); + if (nwrote != (ssize_t)bufsize) { + xfunc_die(); + } + return nwrote; +} + void check_errors_in_children(int signo) { int status; @@ -60,12 +100,12 @@ void check_errors_in_children(int signo) /* transformer(), more than meets the eye */ #if BB_MMU -void FAST_FUNC open_transformer(int fd, +void FAST_FUNC fork_transformer(int fd, int check_signature, - IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate, int src_fd, int dst_fd) + IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate) ) #else -void FAST_FUNC open_transformer(int fd, const char *transform_prog) +void FAST_FUNC fork_transformer(int fd, const char *transform_prog) #endif { struct fd_pair fd_pipe; @@ -83,7 +123,9 @@ void FAST_FUNC open_transformer(int fd, const char *transform_prog) transformer_state_t xstate; init_transformer_state(&xstate); xstate.check_signature = check_signature; - r = transformer(&xstate, fd, fd_pipe.wr); + xstate.src_fd = fd; + xstate.dst_fd = fd_pipe.wr; + r = transformer(&xstate); if (ENABLE_FEATURE_CLEAN_UP) { close(fd_pipe.wr); /* send EOF */ close(fd); @@ -118,16 +160,19 @@ void FAST_FUNC open_transformer(int fd, const char *transform_prog) /* Used by e.g. rpm which gives us a fd without filename, * thus we can't guess the format from filename's extension. */ -int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed) +static transformer_state_t *setup_transformer_on_fd(int fd, int fail_if_not_compressed) { union { uint8_t b[4]; uint16_t b16[2]; uint32_t b32[1]; } magic; - int offset = -2; - USE_FOR_MMU(IF_DESKTOP(long long) int FAST_FUNC (*xformer)(transformer_state_t *xstate, int src_fd, int dst_fd);) - USE_FOR_NOMMU(const char *xformer_prog;) + int offset; + transformer_state_t *xstate; + + offset = -2; + xstate = xzalloc(sizeof(*xstate)); + xstate->src_fd = fd; /* .gz and .bz2 both have 2-byte signature, and their * unpack_XXX_stream wants this header skipped. */ @@ -135,15 +180,15 @@ int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed) if (ENABLE_FEATURE_SEAMLESS_GZ && magic.b16[0] == GZIP_MAGIC ) { - USE_FOR_MMU(xformer = unpack_gz_stream;) - USE_FOR_NOMMU(xformer_prog = "gunzip";) + xstate->xformer = unpack_gz_stream; + USE_FOR_NOMMU(xstate->xformer_prog = "gunzip";) goto found_magic; } if (ENABLE_FEATURE_SEAMLESS_BZ2 && magic.b16[0] == BZIP2_MAGIC ) { - USE_FOR_MMU(xformer = unpack_bz2_stream;) - USE_FOR_NOMMU(xformer_prog = "bunzip2";) + xstate->xformer = unpack_bz2_stream; + USE_FOR_NOMMU(xstate->xformer_prog = "bunzip2";) goto found_magic; } if (ENABLE_FEATURE_SEAMLESS_XZ @@ -152,8 +197,8 @@ int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed) offset = -6; xread(fd, magic.b32, sizeof(magic.b32[0])); if (magic.b32[0] == XZ_MAGIC2) { - USE_FOR_MMU(xformer = unpack_xz_stream;) - USE_FOR_NOMMU(xformer_prog = "unxz";) + xstate->xformer = unpack_xz_stream; + USE_FOR_NOMMU(xstate->xformer_prog = "unxz";) goto found_magic; } } @@ -164,52 +209,130 @@ int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed) IF_FEATURE_SEAMLESS_BZ2("/bzip2") IF_FEATURE_SEAMLESS_XZ("/xz") " magic"); + + /* Some callers expect this function to "consume" fd + * even if data is not compressed. In this case, + * we return a state with trivial transformer. + */ +// USE_FOR_MMU(xstate->xformer = copy_stream;) +// USE_FOR_NOMMU(xstate->xformer_prog = "cat";) + /* fall through to seeking bck over bytes we read earlier */ + + USE_FOR_NOMMU(found_magic:) + /* NOMMU version of fork_transformer execs + * an external unzipper that wants + * file position at the start of the file. + */ xlseek(fd, offset, SEEK_CUR); - return 1; - found_magic: + USE_FOR_MMU(found_magic:) + /* In MMU case, if magic was found, seeking back is not necessary */ + + return xstate; +} + +/* Used by e.g. rpm which gives us a fd without filename, + * thus we can't guess the format from filename's extension. + */ +int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed) +{ + transformer_state_t *xstate = setup_transformer_on_fd(fd, fail_if_not_compressed); + + if (!xstate || !xstate->xformer) { + free(xstate); + return 1; + } + # if BB_MMU - open_transformer_with_no_sig(fd, xformer); + fork_transformer_with_no_sig(xstate->src_fd, xstate->xformer); # else - /* NOMMU version of open_transformer execs - * an external unzipper that wants - * file position at the start of the file */ - xlseek(fd, offset, SEEK_CUR); - open_transformer_with_sig(fd, xformer, xformer_prog); + fork_transformer_with_sig(xstate->src_fd, xstate->xformer, xstate->xformer_prog); # endif + free(xstate); return 0; } -int FAST_FUNC open_zipped(const char *fname, int fail_if_not_compressed) +static transformer_state_t *open_transformer(const char *fname, int fail_if_not_compressed) { + transformer_state_t *xstate; int fd; fd = open(fname, O_RDONLY); if (fd < 0) - return fd; + return NULL; if (ENABLE_FEATURE_SEAMLESS_LZMA) { /* .lzma has no header/signature, can only detect it by extension */ char *sfx = strrchr(fname, '.'); if (sfx && strcmp(sfx+1, "lzma") == 0) { - open_transformer_with_sig(fd, unpack_lzma_stream, "unlzma"); - return fd; + xstate = xzalloc(sizeof(*xstate)); + xstate->src_fd = fd; + xstate->xformer = unpack_lzma_stream; + USE_FOR_NOMMU(xstate->xformer_prog = "unlzma";) + return xstate; } } - if ((ENABLE_FEATURE_SEAMLESS_GZ) - || (ENABLE_FEATURE_SEAMLESS_BZ2) - || (ENABLE_FEATURE_SEAMLESS_XZ) - ) { - setup_unzip_on_fd(fd, fail_if_not_compressed); + + xstate = setup_transformer_on_fd(fd, fail_if_not_compressed); + + return xstate; +} + +int FAST_FUNC open_zipped(const char *fname, int fail_if_not_compressed) +{ + int fd; + transformer_state_t *xstate; + + xstate = open_transformer(fname, fail_if_not_compressed); + if (!xstate) + return -1; + + fd = xstate->src_fd; + if (xstate->xformer) { +# if BB_MMU + fork_transformer_with_no_sig(xstate->src_fd, xstate->xformer); +# else + fork_transformer_with_sig(xstate->src_fd, xstate->xformer, xstate->xformer_prog); +# endif } + /* else: the file is not compressed */ + free(xstate); return fd; } -#endif /* SEAMLESS_COMPRESSION */ - void* FAST_FUNC xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_p) { +# if 1 + transformer_state_t *xstate; + char *image; + + xstate = open_transformer(fname, /*fail_if_not_compressed:*/ 0); + if (!xstate) /* file open error */ + return NULL; + + image = NULL; + if (xstate->xformer) { + /* In-memory decompression */ + xstate->mem_output_size_max = maxsz_p ? *maxsz_p : (size_t)(INT_MAX - 4095); + xstate->xformer(xstate); + if (xstate->mem_output_buf) { + image = xstate->mem_output_buf; + if (maxsz_p) + *maxsz_p = xstate->mem_output_size; + } + } else { + /* File is not compressed */ + image = xmalloc_read(xstate->src_fd, maxsz_p); + } + + if (!image) + bb_perror_msg("read error from '%s'", fname); + close(xstate->src_fd); + free(xstate); + return image; +# else + /* This version forks a subprocess - much more expensive */ int fd; char *image; @@ -221,6 +344,8 @@ void* FAST_FUNC xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_ if (!image) bb_perror_msg("read error from '%s'", fname); close(fd); - return image; +# endif } + +#endif /* SEAMLESS_COMPRESSION */ diff --git a/archival/tar.c b/archival/tar.c index 7897a4af6..5bd473aac 100644 --- a/archival/tar.c +++ b/archival/tar.c @@ -1171,7 +1171,7 @@ int tar_main(int argc UNUSED_PARAM, char **argv) } if (opt & OPT_ANY_COMPRESS) { - USE_FOR_MMU(IF_DESKTOP(long long) int FAST_FUNC (*xformer)(transformer_state_t *xstate, int src_fd, int dst_fd);) + USE_FOR_MMU(IF_DESKTOP(long long) int FAST_FUNC (*xformer)(transformer_state_t *xstate);) USE_FOR_NOMMU(const char *xformer_prog;) if (opt & OPT_COMPRESS) @@ -1190,7 +1190,7 @@ int tar_main(int argc UNUSED_PARAM, char **argv) USE_FOR_MMU(xformer = unpack_xz_stream;) USE_FOR_NOMMU(xformer_prog = "unxz";) - open_transformer_with_sig(tar_handle->src_fd, xformer, xformer_prog); + fork_transformer_with_sig(tar_handle->src_fd, xformer, xformer_prog); /* Can't lseek over pipes */ tar_handle->seek = seek_by_read; /*tar_handle->offset = 0; - already is */ diff --git a/archival/unzip.c b/archival/unzip.c index c622bde57..38a07e212 100644 --- a/archival/unzip.c +++ b/archival/unzip.c @@ -283,7 +283,9 @@ static void unzip_extract(zip_header_t *zip_header, int dst_fd) transformer_state_t xstate; init_transformer_state(&xstate); xstate.bytes_in = zip_header->formatted.cmpsize; - if (inflate_unzip(&xstate, zip_fd, dst_fd) < 0) + xstate.src_fd = zip_fd; + xstate.dst_fd = dst_fd; + if (inflate_unzip(&xstate) < 0) bb_error_msg_and_die("inflate error"); /* Validate decompression - crc */ if (zip_header->formatted.crc32 != (xstate.crc32 ^ 0xffffffffL)) { diff --git a/include/bb_archive.h b/include/bb_archive.h index 8a2dd2735..a6b166fe3 100644 --- a/include/bb_archive.h +++ b/include/bb_archive.h @@ -205,6 +205,18 @@ void dealloc_bunzip(bunzip_data *bd) FAST_FUNC; /* Meaning and direction (input/output) of the fields are transformer-specific */ typedef struct transformer_state_t { smallint check_signature; /* most often referenced member */ + + IF_DESKTOP(long long) int FAST_FUNC (*xformer)(struct transformer_state_t *xstate); + USE_FOR_NOMMU(const char *xformer_prog;) + + /* Source */ + int src_fd; + /* Output */ + int dst_fd; + size_t mem_output_size_max; /* if non-zero, decompress to RAM instead of fd */ + size_t mem_output_size; + char *mem_output_buf; + off_t bytes_out; off_t bytes_in; /* used in unzip code only: needs to know packed size */ uint32_t crc32; @@ -212,14 +224,16 @@ typedef struct transformer_state_t { } transformer_state_t; void init_transformer_state(transformer_state_t *xstate) FAST_FUNC; -int FAST_FUNC check_signature16(transformer_state_t *xstate, int src_fd, unsigned magic16) FAST_FUNC; +ssize_t transformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize) FAST_FUNC; +ssize_t xtransformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize) FAST_FUNC; +int check_signature16(transformer_state_t *xstate, unsigned magic16) FAST_FUNC; -IF_DESKTOP(long long) int inflate_unzip(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC; -IF_DESKTOP(long long) int unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC; -IF_DESKTOP(long long) int unpack_gz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC; -IF_DESKTOP(long long) int unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC; -IF_DESKTOP(long long) int unpack_lzma_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC; -IF_DESKTOP(long long) int unpack_xz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC; +IF_DESKTOP(long long) int inflate_unzip(transformer_state_t *xstate) FAST_FUNC; +IF_DESKTOP(long long) int unpack_Z_stream(transformer_state_t *xstate) FAST_FUNC; +IF_DESKTOP(long long) int unpack_gz_stream(transformer_state_t *xstate) FAST_FUNC; +IF_DESKTOP(long long) int unpack_bz2_stream(transformer_state_t *xstate) FAST_FUNC; +IF_DESKTOP(long long) int unpack_lzma_stream(transformer_state_t *xstate) FAST_FUNC; +IF_DESKTOP(long long) int unpack_xz_stream(transformer_state_t *xstate) FAST_FUNC; char* append_ext(char *filename, const char *expected_ext) FAST_FUNC; int bbunpack(char **argv, @@ -230,16 +244,16 @@ int bbunpack(char **argv, void check_errors_in_children(int signo); #if BB_MMU -void open_transformer(int fd, +void fork_transformer(int fd, int check_signature, - IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate, int src_fd, int dst_fd) + IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate) ) FAST_FUNC; -#define open_transformer_with_sig(fd, transformer, transform_prog) open_transformer((fd), 1, (transformer)) -#define open_transformer_with_no_sig(fd, transformer) open_transformer((fd), 0, (transformer)) +#define fork_transformer_with_sig(fd, transformer, transform_prog) fork_transformer((fd), 1, (transformer)) +#define fork_transformer_with_no_sig(fd, transformer) fork_transformer((fd), 0, (transformer)) #else -void open_transformer(int fd, const char *transform_prog) FAST_FUNC; -#define open_transformer_with_sig(fd, transformer, transform_prog) open_transformer((fd), (transform_prog)) -/* open_transformer_with_no_sig() does not exist on NOMMU */ +void fork_transformer(int fd, const char *transform_prog) FAST_FUNC; +#define fork_transformer_with_sig(fd, transformer, transform_prog) fork_transformer((fd), (transform_prog)) +/* fork_transformer_with_no_sig() does not exist on NOMMU */ #endif diff --git a/include/libbb.h b/include/libbb.h index 17a0089d8..8e8b9ca0e 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -755,11 +755,12 @@ unsigned bb_clk_tck(void) FAST_FUNC; extern int setup_unzip_on_fd(int fd, int fail_if_not_compressed) FAST_FUNC; /* Autodetects .gz etc */ extern int open_zipped(const char *fname, int fail_if_not_compressed) FAST_FUNC; +extern void *xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_p) FAST_FUNC RETURNS_MALLOC; #else # define setup_unzip_on_fd(...) (0) # define open_zipped(fname, fail_if_not_compressed) open((fname), O_RDONLY); +# define xmalloc_open_zipped_read_close(fname, maxsz_p) xmalloc_open_read_close((fname), (maxsz_p)) #endif -extern void *xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_p) FAST_FUNC RETURNS_MALLOC; extern ssize_t safe_write(int fd, const void *buf, size_t count) FAST_FUNC; // NB: will return short write on error, not -1, -- 2.25.1