libarchive: add capability to unpack to mem.buffer
authorDenys Vlasenko <vda.linux@googlemail.com>
Sat, 6 Dec 2014 23:44:00 +0000 (00:44 +0100)
committerDenys Vlasenko <vda.linux@googlemail.com>
Sat, 6 Dec 2014 23:44:00 +0000 (00:44 +0100)
The performance and number of processes for a "depmod -a" with gzipped
modules was abysmal. This patch adds a code path without fork,
benefiting all users of xmalloc_open_zipped_read_close.

"modinfo radeon.ko.gz", a single-file reader, got 30% faster.
"depmod -a", which used to fork over 800 times, got 20% faster.

Heavily based on a patch by Lauri Kasanen <curaga@operamail.com>

function                                             old     new   delta
setup_transformer_on_fd                                -     159    +159
transformer_write                                      -     122    +122
fork_transformer                                       -     112    +112
xmalloc_open_zipped_read_close                        63     118     +55
read_bunzip                                         1866    1896     +30
xtransformer_write                                     -      19     +19
unzip_main                                          2449    2462     +13
bbunpack                                             755     766     +11
unpack_lzma_stream                                  2717    2723      +6
unpack_xz_stream                                    2393    2397      +4
unpack_Z_stream                                     1173    1175      +2
inflate_unzip                                        111     105      -6
check_signature16                                     70      63      -7
unpack_bz2_stream                                    359     349     -10
unpack_unxz                                           12       -     -12
unpack_unlzma                                         12       -     -12
unpack_uncompress                                     12       -     -12
unpack_gunzip                                         12       -     -12
unpack_bunzip2                                        12       -     -12
open_transformer                                     106      92     -14
inflate_unzip_internal                              1945    1916     -29
unpack_gz_stream                                     693     655     -38
open_zipped                                           89      47     -42
setup_unzip_on_fd                                    142      53     -89
------------------------------------------------------------------------------
(add/remove: 4/5 grow/shrink: 7/8 up/down: 533/-295)          Total: 238 bytes

Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>
14 files changed:
archival/bbunzip.c
archival/libarchive/decompress_bunzip2.c
archival/libarchive/decompress_gunzip.c
archival/libarchive/decompress_uncompress.c
archival/libarchive/decompress_unlzma.c
archival/libarchive/decompress_unxz.c
archival/libarchive/get_header_tar_bz2.c
archival/libarchive/get_header_tar_gz.c
archival/libarchive/get_header_tar_lzma.c
archival/libarchive/open_transformer.c
archival/tar.c
archival/unzip.c
include/bb_archive.h
include/libbb.h

index 84f58f138ad4715f480a241d1b90fd68f603cebd..90aac142703a558d5c038714e5092bfd2e1002d5 100644 (file)
@@ -122,6 +122,8 @@ int FAST_FUNC bbunpack(char **argv,
                if (!(option_mask32 & SEAMLESS_MAGIC)) {
                        init_transformer_state(&xstate);
                        xstate.check_signature = 1;
+                       /*xstate.src_fd = STDIN_FILENO; - already is */
+                       xstate.dst_fd = STDOUT_FILENO;
                        status = unpacker(&xstate);
                        if (status < 0)
                                exitcode = 1;
@@ -226,18 +228,13 @@ char* FAST_FUNC make_new_name_generic(char *filename, const char *expected_ext)
 //applet:IF_UNCOMPRESS(APPLET(uncompress, BB_DIR_BIN, BB_SUID_DROP))
 //kbuild:lib-$(CONFIG_UNCOMPRESS) += bbunzip.o
 #if ENABLE_UNCOMPRESS
-static
-IF_DESKTOP(long long) int FAST_FUNC unpack_uncompress(transformer_state_t *xstate)
-{
-       return unpack_Z_stream(xstate, STDIN_FILENO, STDOUT_FILENO);
-}
 int uncompress_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
 int uncompress_main(int argc UNUSED_PARAM, char **argv)
 {
        getopt32(argv, "cf");
        argv += optind;
 
-       return bbunpack(argv, unpack_uncompress, make_new_name_generic, "Z");
+       return bbunpack(argv, unpack_Z_stream, make_new_name_generic, "Z");
 }
 #endif
 
@@ -324,11 +321,6 @@ char* FAST_FUNC make_new_name_gunzip(char *filename, const char *expected_ext UN
        }
        return filename;
 }
-static
-IF_DESKTOP(long long) int FAST_FUNC unpack_gunzip(transformer_state_t *xstate)
-{
-       return unpack_gz_stream(xstate, STDIN_FILENO, STDOUT_FILENO);
-}
 /*
  * Linux kernel build uses gzip -d -n. We accept and ignore it.
  * Man page says:
@@ -355,7 +347,7 @@ int gunzip_main(int argc UNUSED_PARAM, char **argv)
        if (applet_name[1] == 'c')
                option_mask32 |= OPT_STDOUT | SEAMLESS_MAGIC;
 
-       return bbunpack(argv, unpack_gunzip, make_new_name_gunzip, /*unused:*/ NULL);
+       return bbunpack(argv, unpack_gz_stream, make_new_name_gunzip, /*unused:*/ NULL);
 }
 #endif
 
@@ -395,11 +387,6 @@ int gunzip_main(int argc UNUSED_PARAM, char **argv)
 //kbuild:lib-$(CONFIG_BZIP2) += bbunzip.o
 //kbuild:lib-$(CONFIG_BUNZIP2) += bbunzip.o
 #if ENABLE_BUNZIP2
-static
-IF_DESKTOP(long long) int FAST_FUNC unpack_bunzip2(transformer_state_t *xstate)
-{
-       return unpack_bz2_stream(xstate, STDIN_FILENO, STDOUT_FILENO);
-}
 int bunzip2_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
 int bunzip2_main(int argc UNUSED_PARAM, char **argv)
 {
@@ -408,7 +395,7 @@ int bunzip2_main(int argc UNUSED_PARAM, char **argv)
        if (applet_name[2] == 'c') /* bzcat */
                option_mask32 |= OPT_STDOUT;
 
-       return bbunpack(argv, unpack_bunzip2, make_new_name_generic, "bz2");
+       return bbunpack(argv, unpack_bz2_stream, make_new_name_generic, "bz2");
 }
 #endif
 
@@ -494,11 +481,6 @@ int bunzip2_main(int argc UNUSED_PARAM, char **argv)
 //applet:IF_LZMA(APPLET_ODDNAME(lzma, unlzma, BB_DIR_USR_BIN, BB_SUID_DROP, lzma))
 //kbuild:lib-$(CONFIG_UNLZMA) += bbunzip.o
 #if ENABLE_UNLZMA
-static
-IF_DESKTOP(long long) int FAST_FUNC unpack_unlzma(transformer_state_t *xstate)
-{
-       return unpack_lzma_stream(xstate, STDIN_FILENO, STDOUT_FILENO);
-}
 int unlzma_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
 int unlzma_main(int argc UNUSED_PARAM, char **argv)
 {
@@ -513,7 +495,7 @@ int unlzma_main(int argc UNUSED_PARAM, char **argv)
                option_mask32 |= OPT_STDOUT;
 
        argv += optind;
-       return bbunpack(argv, unpack_unlzma, make_new_name_generic, "lzma");
+       return bbunpack(argv, unpack_lzma_stream, make_new_name_generic, "lzma");
 }
 #endif
 
@@ -537,11 +519,6 @@ int unlzma_main(int argc UNUSED_PARAM, char **argv)
 //applet:IF_XZ(APPLET_ODDNAME(xz, unxz, BB_DIR_USR_BIN, BB_SUID_DROP, xz))
 //kbuild:lib-$(CONFIG_UNXZ) += bbunzip.o
 #if ENABLE_UNXZ
-static
-IF_DESKTOP(long long) int FAST_FUNC unpack_unxz(transformer_state_t *xstate)
-{
-       return unpack_xz_stream(xstate, STDIN_FILENO, STDOUT_FILENO);
-}
 int unxz_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
 int unxz_main(int argc UNUSED_PARAM, char **argv)
 {
@@ -556,6 +533,6 @@ int unxz_main(int argc UNUSED_PARAM, char **argv)
                option_mask32 |= OPT_STDOUT;
 
        argv += optind;
-       return bbunpack(argv, unpack_unxz, make_new_name_generic, "xz");
+       return bbunpack(argv, unpack_xz_stream, make_new_name_generic, "xz");
 }
 #endif
index 36237e2215beb33cf6f8a289fc3adee5418604cf..fe5953da2412df5c428e281c3c50a86d735db735 100644 (file)
@@ -731,7 +731,7 @@ void FAST_FUNC dealloc_bunzip(bunzip_data *bd)
 
 /* Decompress src_fd to dst_fd.  Stops at end of bzip data, not end of file. */
 IF_DESKTOP(long long) int FAST_FUNC
-unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
+unpack_bz2_stream(transformer_state_t *xstate)
 {
        IF_DESKTOP(long long total_written = 0;)
        bunzip_data *bd;
@@ -739,14 +739,14 @@ unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
        int i;
        unsigned len;
 
-       if (check_signature16(xstate, src_fd, BZIP2_MAGIC))
+       if (check_signature16(xstate, BZIP2_MAGIC))
                return -1;
 
        outbuf = xmalloc(IOBUF_SIZE);
        len = 0;
        while (1) { /* "Process one BZ... stream" loop */
 
-               i = start_bunzip(&bd, src_fd, outbuf + 2, len);
+               i = start_bunzip(&bd, xstate->src_fd, outbuf + 2, len);
 
                if (i == 0) {
                        while (1) { /* "Produce some output bytes" loop */
@@ -756,8 +756,7 @@ unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
                                i = IOBUF_SIZE - i; /* number of bytes produced */
                                if (i == 0) /* EOF? */
                                        break;
-                               if (i != full_write(dst_fd, outbuf, i)) {
-                                       bb_error_msg("short write");
+                               if (i != transformer_write(xstate, outbuf, i)) {
                                        i = RETVAL_SHORT_WRITE;
                                        goto release_mem;
                                }
@@ -790,7 +789,7 @@ unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
                len = bd->inbufCount - bd->inbufPos;
                memcpy(outbuf, &bd->inbuf[bd->inbufPos], len);
                if (len < 2) {
-                       if (safe_read(src_fd, outbuf + len, 2 - len) != 2 - len)
+                       if (safe_read(xstate->src_fd, outbuf + len, 2 - len) != 2 - len)
                                break;
                        len = 2;
                }
index 62a3d78b6c6977d115bc39dd352cf78eacea6fe0..1360abef7d202ef2bc6e354a379dbae68a3887ff 100644 (file)
@@ -971,7 +971,7 @@ static int inflate_get_next_window(STATE_PARAM_ONLY)
 
 /* Called from unpack_gz_stream() and inflate_unzip() */
 static IF_DESKTOP(long long) int
-inflate_unzip_internal(STATE_PARAM int in, int out)
+inflate_unzip_internal(STATE_PARAM transformer_state_t *xstate)
 {
        IF_DESKTOP(long long) int n = 0;
        ssize_t nwrote;
@@ -980,7 +980,7 @@ inflate_unzip_internal(STATE_PARAM int in, int out)
        gunzip_window = xmalloc(GUNZIP_WSIZE);
        gunzip_outbuf_count = 0;
        gunzip_bytes_out = 0;
-       gunzip_src_fd = in;
+       gunzip_src_fd = xstate->src_fd;
 
        /* (re) initialize state */
        method = -1;
@@ -1002,9 +1002,8 @@ inflate_unzip_internal(STATE_PARAM int in, int out)
 
        while (1) {
                int r = inflate_get_next_window(PASS_STATE_ONLY);
-               nwrote = full_write(out, gunzip_window, gunzip_outbuf_count);
-               if (nwrote != (ssize_t)gunzip_outbuf_count) {
-                       bb_perror_msg("write");
+               nwrote = transformer_write(xstate, gunzip_window, gunzip_outbuf_count);
+               if (nwrote == (ssize_t)-1) {
                        n = -1;
                        goto ret;
                }
@@ -1034,7 +1033,7 @@ inflate_unzip_internal(STATE_PARAM int in, int out)
 /* For unzip */
 
 IF_DESKTOP(long long) int FAST_FUNC
-inflate_unzip(transformer_state_t *xstate, int in, int out)
+inflate_unzip(transformer_state_t *xstate)
 {
        IF_DESKTOP(long long) int n;
        DECLARE_STATE;
@@ -1045,7 +1044,7 @@ inflate_unzip(transformer_state_t *xstate, int in, int out)
 //     bytebuffer_max = 0x8000;
        bytebuffer_offset = 4;
        bytebuffer = xmalloc(bytebuffer_max);
-       n = inflate_unzip_internal(PASS_STATE in, out);
+       n = inflate_unzip_internal(PASS_STATE xstate);
        free(bytebuffer);
 
        xstate->crc32 = gunzip_crc;
@@ -1169,8 +1168,7 @@ static int check_header_gzip(STATE_PARAM transformer_state_t *xstate)
                }
        }
 
-       if (xstate)
-               xstate->mtime = SWAP_LE32(header.formatted.mtime);
+       xstate->mtime = SWAP_LE32(header.formatted.mtime);
 
        /* Read the header checksum */
        if (header.formatted.flags & 0x02) {
@@ -1182,27 +1180,27 @@ static int check_header_gzip(STATE_PARAM transformer_state_t *xstate)
 }
 
 IF_DESKTOP(long long) int FAST_FUNC
-unpack_gz_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
+unpack_gz_stream(transformer_state_t *xstate)
 {
        uint32_t v32;
        IF_DESKTOP(long long) int total, n;
        DECLARE_STATE;
 
 #if !ENABLE_FEATURE_SEAMLESS_Z
-       if (check_signature16(xstate, src_fd, GZIP_MAGIC))
+       if (check_signature16(xstate, GZIP_MAGIC))
                return -1;
 #else
-       if (xstate && xstate->check_signature) {
+       if (xstate->check_signature) {
                uint16_t magic2;
 
-               if (full_read(src_fd, &magic2, 2) != 2) {
+               if (full_read(xstate->src_fd, &magic2, 2) != 2) {
  bad_magic:
                        bb_error_msg("invalid magic");
                        return -1;
                }
                if (magic2 == COMPRESS_MAGIC) {
                        xstate->check_signature = 0;
-                       return unpack_Z_stream(xstate, src_fd, dst_fd);
+                       return unpack_Z_stream(xstate);
                }
                if (magic2 != GZIP_MAGIC)
                        goto bad_magic;
@@ -1215,7 +1213,7 @@ unpack_gz_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
        to_read = -1;
 //     bytebuffer_max = 0x8000;
        bytebuffer = xmalloc(bytebuffer_max);
-       gunzip_src_fd = src_fd;
+       gunzip_src_fd = xstate->src_fd;
 
  again:
        if (!check_header_gzip(PASS_STATE xstate)) {
@@ -1224,7 +1222,7 @@ unpack_gz_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
                goto ret;
        }
 
-       n = inflate_unzip_internal(PASS_STATE src_fd, dst_fd);
+       n = inflate_unzip_internal(PASS_STATE xstate);
        if (n < 0) {
                total = -1;
                goto ret;
index cb3d55a88e9f756b98afa7032248c1070d9d6850..496d864a7b932938df3350fd77a655e48f3666db 100644 (file)
@@ -73,7 +73,7 @@
  */
 
 IF_DESKTOP(long long) int FAST_FUNC
-unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
+unpack_Z_stream(transformer_state_t *xstate)
 {
        IF_DESKTOP(long long total_written = 0;)
        IF_DESKTOP(long long) int retval = -1;
@@ -102,7 +102,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
        /* block compress mode -C compatible with 2.0 */
        int block_mode; /* = BLOCK_MODE; */
 
-       if (check_signature16(xstate, src_fd, COMPRESS_MAGIC))
+       if (check_signature16(xstate, COMPRESS_MAGIC))
                return -1;
 
        inbuf = xzalloc(IBUFSIZ + 64);
@@ -114,7 +114,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
 
        /* xread isn't good here, we have to return - caller may want
         * to do some cleanup (e.g. delete incomplete unpacked file etc) */
-       if (full_read(src_fd, inbuf, 1) != 1) {
+       if (full_read(xstate->src_fd, inbuf, 1) != 1) {
                bb_error_msg("short read");
                goto err;
        }
@@ -166,7 +166,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
                }
 
                if (insize < (int) (IBUFSIZ + 64) - IBUFSIZ) {
-                       rsize = safe_read(src_fd, inbuf + insize, IBUFSIZ);
+                       rsize = safe_read(xstate->src_fd, inbuf + insize, IBUFSIZ);
                        if (rsize < 0)
                                bb_error_msg_and_die(bb_msg_read_error);
                        insize += rsize;
@@ -274,7 +274,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
                                                }
 
                                                if (outpos >= OBUFSIZ) {
-                                                       xwrite(dst_fd, outbuf, outpos);
+                                                       xtransformer_write(xstate, outbuf, outpos);
                                                        IF_DESKTOP(total_written += outpos;)
                                                        outpos = 0;
                                                }
@@ -301,7 +301,7 @@ unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
        } while (rsize > 0);
 
        if (outpos > 0) {
-               xwrite(dst_fd, outbuf, outpos);
+               xtransformer_write(xstate, outbuf, outpos);
                IF_DESKTOP(total_written += outpos;)
        }
 
index ccedac49df9f5b4a3f3a51835194a326168bd1e0..c8622f97b3b02fbfcd659508e63d73ce370307d6 100644 (file)
@@ -206,7 +206,7 @@ enum {
 
 
 IF_DESKTOP(long long) int FAST_FUNC
-unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst_fd)
+unpack_lzma_stream(transformer_state_t *xstate)
 {
        IF_DESKTOP(long long total_written = 0;)
        lzma_header_t header;
@@ -223,7 +223,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst
        int state = 0;
        uint32_t rep0 = 1, rep1 = 1, rep2 = 1, rep3 = 1;
 
-       if (full_read(src_fd, &header, sizeof(header)) != sizeof(header)
+       if (full_read(xstate->src_fd, &header, sizeof(header)) != sizeof(header)
         || header.pos >= (9 * 5 * 5)
        ) {
                bb_error_msg("bad lzma header");
@@ -258,7 +258,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst
                        p[i] = (1 << RC_MODEL_TOTAL_BITS) >> 1;
        }
 
-       rc = rc_init(src_fd); /*, RC_BUFFER_SIZE); */
+       rc = rc_init(xstate->src_fd); /*, RC_BUFFER_SIZE); */
 
        while (global_pos + buffer_pos < header.dst_size) {
                int pos_state = (buffer_pos + global_pos) & pos_state_mask;
@@ -306,7 +306,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst
                        if (buffer_pos == header.dict_size) {
                                buffer_pos = 0;
                                global_pos += header.dict_size;
-                               if (full_write(dst_fd, buffer, header.dict_size) != (ssize_t)header.dict_size)
+                               if (transformer_write(xstate, buffer, header.dict_size) != (ssize_t)header.dict_size)
                                        goto bad;
                                IF_DESKTOP(total_written += header.dict_size;)
                        }
@@ -440,7 +440,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst
                                if (buffer_pos == header.dict_size) {
                                        buffer_pos = 0;
                                        global_pos += header.dict_size;
-                                       if (full_write(dst_fd, buffer, header.dict_size) != (ssize_t)header.dict_size)
+                                       if (transformer_write(xstate, buffer, header.dict_size) != (ssize_t)header.dict_size)
                                                goto bad;
                                        IF_DESKTOP(total_written += header.dict_size;)
                                }
@@ -455,7 +455,7 @@ unpack_lzma_stream(transformer_state_t *xstate UNUSED_PARAM, int src_fd, int dst
        {
                IF_NOT_DESKTOP(int total_written = 0; /* success */)
                IF_DESKTOP(total_written += buffer_pos;)
-               if (full_write(dst_fd, buffer, buffer_pos) != (ssize_t)buffer_pos) {
+               if (transformer_write(xstate, buffer, buffer_pos) != (ssize_t)buffer_pos) {
  bad:
                        total_written = -1; /* failure */
                }
index 6df54e1319a10d94afec8463148b1a45cc35294f..1f408abfd4a2e85b0dbab4ea0d55bcf80cafefaf 100644 (file)
@@ -38,7 +38,7 @@ static uint32_t xz_crc32(const uint8_t *buf, size_t size, uint32_t crc)
 #include "unxz/xz_dec_stream.c"
 
 IF_DESKTOP(long long) int FAST_FUNC
-unpack_xz_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
+unpack_xz_stream(transformer_state_t *xstate)
 {
        enum xz_ret xz_result;
        struct xz_buf iobuf;
@@ -67,7 +67,7 @@ unpack_xz_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
        xz_result = X_OK;
        while (1) {
                if (iobuf.in_pos == iobuf.in_size) {
-                       int rd = safe_read(src_fd, membuf, BUFSIZ);
+                       int rd = safe_read(xstate->src_fd, membuf, BUFSIZ);
                        if (rd < 0) {
                                bb_error_msg(bb_msg_read_error);
                                total = -1;
@@ -104,7 +104,7 @@ unpack_xz_stream(transformer_state_t *xstate, int src_fd, int dst_fd)
 //             bb_error_msg("<in pos:%d size:%d out pos:%d size:%d r:%d",
 //                             iobuf.in_pos, iobuf.in_size, iobuf.out_pos, iobuf.out_size, xz_result);
                if (iobuf.out_pos) {
-                       xwrite(dst_fd, iobuf.out, iobuf.out_pos);
+                       xtransformer_write(xstate, iobuf.out, iobuf.out_pos);
                        IF_DESKTOP(total += iobuf.out_pos;)
                        iobuf.out_pos = 0;
                }
index 0ee00df53d4ceece1c5f0b4d68e3523baa6c6c50..78f78a858ce69d3b66b5e0dcdc8ebdca1a2a59fb 100644 (file)
@@ -11,7 +11,7 @@ char FAST_FUNC get_header_tar_bz2(archive_handle_t *archive_handle)
        /* Can't lseek over pipes */
        archive_handle->seek = seek_by_read;
 
-       open_transformer_with_sig(archive_handle->src_fd, unpack_bz2_stream, "bunzip2");
+       fork_transformer_with_sig(archive_handle->src_fd, unpack_bz2_stream, "bunzip2");
        archive_handle->offset = 0;
        while (get_header_tar(archive_handle) == EXIT_SUCCESS)
                continue;
index 03284342b1cf683b7c5fea3bba09e490d2508c7a..b11f503dc6d75518c60f04e269296f9960d38b46 100644 (file)
@@ -11,7 +11,7 @@ char FAST_FUNC get_header_tar_gz(archive_handle_t *archive_handle)
        /* Can't lseek over pipes */
        archive_handle->seek = seek_by_read;
 
-       open_transformer_with_sig(archive_handle->src_fd, unpack_gz_stream, "gunzip");
+       fork_transformer_with_sig(archive_handle->src_fd, unpack_gz_stream, "gunzip");
        archive_handle->offset = 0;
        while (get_header_tar(archive_handle) == EXIT_SUCCESS)
                continue;
index d565a217d92d015e27216abca817e88b909a5152..d228cbc138ae26cb878619e53f071dedc19f11fa 100644 (file)
@@ -14,7 +14,7 @@ char FAST_FUNC get_header_tar_lzma(archive_handle_t *archive_handle)
        /* Can't lseek over pipes */
        archive_handle->seek = seek_by_read;
 
-       open_transformer_with_sig(archive_handle->src_fd, unpack_lzma_stream, "unlzma");
+       fork_transformer_with_sig(archive_handle->src_fd, unpack_lzma_stream, "unlzma");
        archive_handle->offset = 0;
        while (get_header_tar(archive_handle) == EXIT_SUCCESS)
                continue;
index 584b15de4fce5f8ad0e28a9491ecaaffe56cca35..9d762a8599489b3748f3300939df3ba370749e78 100644 (file)
@@ -11,11 +11,11 @@ void FAST_FUNC init_transformer_state(transformer_state_t *xstate)
        memset(xstate, 0, sizeof(*xstate));
 }
 
-int FAST_FUNC check_signature16(transformer_state_t *xstate, int src_fd, unsigned magic16)
+int FAST_FUNC check_signature16(transformer_state_t *xstate, unsigned magic16)
 {
-       if (xstate && xstate->check_signature) {
+       if (xstate->check_signature) {
                uint16_t magic2;
-               if (full_read(src_fd, &magic2, 2) != 2 || magic2 != magic16) {
+               if (full_read(xstate->src_fd, &magic2, 2) != 2 || magic2 != magic16) {
                        bb_error_msg("invalid magic");
 #if 0 /* possible future extension */
                        if (xstate->check_signature > 1)
@@ -27,6 +27,46 @@ int FAST_FUNC check_signature16(transformer_state_t *xstate, int src_fd, unsigne
        return 0;
 }
 
+ssize_t FAST_FUNC transformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize)
+{
+       ssize_t nwrote;
+
+       if (xstate->mem_output_size_max != 0) {
+               size_t pos = xstate->mem_output_size;
+               size_t size;
+
+               size = (xstate->mem_output_size += bufsize);
+               if (size > xstate->mem_output_size_max) {
+                       free(xstate->mem_output_buf);
+                       xstate->mem_output_buf = NULL;
+                       bb_perror_msg("buffer %u too small", (unsigned)xstate->mem_output_size_max);
+                       nwrote = -1;
+                       goto ret;
+               }
+               xstate->mem_output_buf = xrealloc(xstate->mem_output_buf, size);
+               memcpy(xstate->mem_output_buf + pos, buf, bufsize);
+               nwrote = bufsize;
+       } else {
+               nwrote = full_write(xstate->dst_fd, buf, bufsize);
+               if (nwrote != (ssize_t)bufsize) {
+                       bb_perror_msg("write");
+                       nwrote = -1;
+                       goto ret;
+               }
+       }
+ ret:
+       return nwrote;
+}
+
+ssize_t FAST_FUNC xtransformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize)
+{
+       ssize_t nwrote = transformer_write(xstate, buf, bufsize);
+       if (nwrote != (ssize_t)bufsize) {
+               xfunc_die();
+       }
+       return nwrote;
+}
+
 void check_errors_in_children(int signo)
 {
        int status;
@@ -60,12 +100,12 @@ void check_errors_in_children(int signo)
 
 /* transformer(), more than meets the eye */
 #if BB_MMU
-void FAST_FUNC open_transformer(int fd,
+void FAST_FUNC fork_transformer(int fd,
        int check_signature,
-       IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate, int src_fd, int dst_fd)
+       IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate)
 )
 #else
-void FAST_FUNC open_transformer(int fd, const char *transform_prog)
+void FAST_FUNC fork_transformer(int fd, const char *transform_prog)
 #endif
 {
        struct fd_pair fd_pipe;
@@ -83,7 +123,9 @@ void FAST_FUNC open_transformer(int fd, const char *transform_prog)
                        transformer_state_t xstate;
                        init_transformer_state(&xstate);
                        xstate.check_signature = check_signature;
-                       r = transformer(&xstate, fd, fd_pipe.wr);
+                       xstate.src_fd = fd;
+                       xstate.dst_fd = fd_pipe.wr;
+                       r = transformer(&xstate);
                        if (ENABLE_FEATURE_CLEAN_UP) {
                                close(fd_pipe.wr); /* send EOF */
                                close(fd);
@@ -118,16 +160,19 @@ void FAST_FUNC open_transformer(int fd, const char *transform_prog)
 /* Used by e.g. rpm which gives us a fd without filename,
  * thus we can't guess the format from filename's extension.
  */
-int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed)
+static transformer_state_t *setup_transformer_on_fd(int fd, int fail_if_not_compressed)
 {
        union {
                uint8_t b[4];
                uint16_t b16[2];
                uint32_t b32[1];
        } magic;
-       int offset = -2;
-       USE_FOR_MMU(IF_DESKTOP(long long) int FAST_FUNC (*xformer)(transformer_state_t *xstate, int src_fd, int dst_fd);)
-       USE_FOR_NOMMU(const char *xformer_prog;)
+       int offset;
+       transformer_state_t *xstate;
+
+       offset = -2;
+       xstate = xzalloc(sizeof(*xstate));
+       xstate->src_fd = fd;
 
        /* .gz and .bz2 both have 2-byte signature, and their
         * unpack_XXX_stream wants this header skipped. */
@@ -135,15 +180,15 @@ int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed)
        if (ENABLE_FEATURE_SEAMLESS_GZ
         && magic.b16[0] == GZIP_MAGIC
        ) {
-               USE_FOR_MMU(xformer = unpack_gz_stream;)
-               USE_FOR_NOMMU(xformer_prog = "gunzip";)
+               xstate->xformer = unpack_gz_stream;
+               USE_FOR_NOMMU(xstate->xformer_prog = "gunzip";)
                goto found_magic;
        }
        if (ENABLE_FEATURE_SEAMLESS_BZ2
         && magic.b16[0] == BZIP2_MAGIC
        ) {
-               USE_FOR_MMU(xformer = unpack_bz2_stream;)
-               USE_FOR_NOMMU(xformer_prog = "bunzip2";)
+               xstate->xformer = unpack_bz2_stream;
+               USE_FOR_NOMMU(xstate->xformer_prog = "bunzip2";)
                goto found_magic;
        }
        if (ENABLE_FEATURE_SEAMLESS_XZ
@@ -152,8 +197,8 @@ int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed)
                offset = -6;
                xread(fd, magic.b32, sizeof(magic.b32[0]));
                if (magic.b32[0] == XZ_MAGIC2) {
-                       USE_FOR_MMU(xformer = unpack_xz_stream;)
-                       USE_FOR_NOMMU(xformer_prog = "unxz";)
+                       xstate->xformer = unpack_xz_stream;
+                       USE_FOR_NOMMU(xstate->xformer_prog = "unxz";)
                        goto found_magic;
                }
        }
@@ -164,52 +209,130 @@ int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed)
                        IF_FEATURE_SEAMLESS_BZ2("/bzip2")
                        IF_FEATURE_SEAMLESS_XZ("/xz")
                        " magic");
+
+       /* Some callers expect this function to "consume" fd
+        * even if data is not compressed. In this case,
+        * we return a state with trivial transformer.
+        */
+//     USE_FOR_MMU(xstate->xformer = copy_stream;)
+//     USE_FOR_NOMMU(xstate->xformer_prog = "cat";)
+       /* fall through to seeking bck over bytes we read earlier */
+
+ USE_FOR_NOMMU(found_magic:)
+       /* NOMMU version of fork_transformer execs
+        * an external unzipper that wants
+        * file position at the start of the file.
+        */
        xlseek(fd, offset, SEEK_CUR);
-       return 1;
 
- found_magic:
+ USE_FOR_MMU(found_magic:)
+       /* In MMU case, if magic was found, seeking back is not necessary */
+
+       return xstate;
+}
+
+/* Used by e.g. rpm which gives us a fd without filename,
+ * thus we can't guess the format from filename's extension.
+ */
+int FAST_FUNC setup_unzip_on_fd(int fd, int fail_if_not_compressed)
+{
+       transformer_state_t *xstate = setup_transformer_on_fd(fd, fail_if_not_compressed);
+
+       if (!xstate || !xstate->xformer) {
+               free(xstate);
+               return 1;
+       }
+
 # if BB_MMU
-       open_transformer_with_no_sig(fd, xformer);
+       fork_transformer_with_no_sig(xstate->src_fd, xstate->xformer);
 # else
-       /* NOMMU version of open_transformer execs
-        * an external unzipper that wants
-        * file position at the start of the file */
-       xlseek(fd, offset, SEEK_CUR);
-       open_transformer_with_sig(fd, xformer, xformer_prog);
+       fork_transformer_with_sig(xstate->src_fd, xstate->xformer, xstate->xformer_prog);
 # endif
+       free(xstate);
        return 0;
 }
 
-int FAST_FUNC open_zipped(const char *fname, int fail_if_not_compressed)
+static transformer_state_t *open_transformer(const char *fname, int fail_if_not_compressed)
 {
+       transformer_state_t *xstate;
        int fd;
 
        fd = open(fname, O_RDONLY);
        if (fd < 0)
-               return fd;
+               return NULL;
 
        if (ENABLE_FEATURE_SEAMLESS_LZMA) {
                /* .lzma has no header/signature, can only detect it by extension */
                char *sfx = strrchr(fname, '.');
                if (sfx && strcmp(sfx+1, "lzma") == 0) {
-                       open_transformer_with_sig(fd, unpack_lzma_stream, "unlzma");
-                       return fd;
+                       xstate = xzalloc(sizeof(*xstate));
+                       xstate->src_fd = fd;
+                       xstate->xformer = unpack_lzma_stream;
+                       USE_FOR_NOMMU(xstate->xformer_prog = "unlzma";)
+                       return xstate;
                }
        }
-       if ((ENABLE_FEATURE_SEAMLESS_GZ)
-        || (ENABLE_FEATURE_SEAMLESS_BZ2)
-        || (ENABLE_FEATURE_SEAMLESS_XZ)
-       ) {
-               setup_unzip_on_fd(fd, fail_if_not_compressed);
+
+       xstate = setup_transformer_on_fd(fd, fail_if_not_compressed);
+
+       return xstate;
+}
+
+int FAST_FUNC open_zipped(const char *fname, int fail_if_not_compressed)
+{
+       int fd;
+       transformer_state_t *xstate;
+
+       xstate = open_transformer(fname, fail_if_not_compressed);
+       if (!xstate)
+               return -1;
+
+       fd = xstate->src_fd;
+       if (xstate->xformer) {
+# if BB_MMU
+               fork_transformer_with_no_sig(xstate->src_fd, xstate->xformer);
+# else
+               fork_transformer_with_sig(xstate->src_fd, xstate->xformer, xstate->xformer_prog);
+# endif
        }
+       /* else: the file is not compressed */
 
+       free(xstate);
        return fd;
 }
 
-#endif /* SEAMLESS_COMPRESSION */
-
 void* FAST_FUNC xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_p)
 {
+# if 1
+       transformer_state_t *xstate;
+       char *image;
+
+       xstate = open_transformer(fname, /*fail_if_not_compressed:*/ 0);
+       if (!xstate) /* file open error */
+               return NULL;
+
+       image = NULL;
+       if (xstate->xformer) {
+               /* In-memory decompression */
+               xstate->mem_output_size_max = maxsz_p ? *maxsz_p : (size_t)(INT_MAX - 4095);
+               xstate->xformer(xstate);
+               if (xstate->mem_output_buf) {
+                       image = xstate->mem_output_buf;
+                       if (maxsz_p)
+                               *maxsz_p = xstate->mem_output_size;
+               }
+       } else {
+               /* File is not compressed */
+               image = xmalloc_read(xstate->src_fd, maxsz_p);
+       }
+
+       if (!image)
+               bb_perror_msg("read error from '%s'", fname);
+       close(xstate->src_fd);
+       free(xstate);
+       return image;
+# else
+       /* This version forks a subprocess - much more expensive */
        int fd;
        char *image;
 
@@ -221,6 +344,8 @@ void* FAST_FUNC xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_
        if (!image)
                bb_perror_msg("read error from '%s'", fname);
        close(fd);
-
        return image;
+# endif
 }
+
+#endif /* SEAMLESS_COMPRESSION */
index 7897a4af68f4345aa572ca48860063304b101ad3..5bd473aac60f9ed8423912079d353f1a7bd2696e 100644 (file)
@@ -1171,7 +1171,7 @@ int tar_main(int argc UNUSED_PARAM, char **argv)
        }
 
        if (opt & OPT_ANY_COMPRESS) {
-               USE_FOR_MMU(IF_DESKTOP(long long) int FAST_FUNC (*xformer)(transformer_state_t *xstate, int src_fd, int dst_fd);)
+               USE_FOR_MMU(IF_DESKTOP(long long) int FAST_FUNC (*xformer)(transformer_state_t *xstate);)
                USE_FOR_NOMMU(const char *xformer_prog;)
 
                if (opt & OPT_COMPRESS)
@@ -1190,7 +1190,7 @@ int tar_main(int argc UNUSED_PARAM, char **argv)
                        USE_FOR_MMU(xformer = unpack_xz_stream;)
                        USE_FOR_NOMMU(xformer_prog = "unxz";)
 
-               open_transformer_with_sig(tar_handle->src_fd, xformer, xformer_prog);
+               fork_transformer_with_sig(tar_handle->src_fd, xformer, xformer_prog);
                /* Can't lseek over pipes */
                tar_handle->seek = seek_by_read;
                /*tar_handle->offset = 0; - already is */
index c622bde57573d16c5c703ea1e293fbfb3d93b045..38a07e2120112d673953ffa8a3d23e2de2602e2a 100644 (file)
@@ -283,7 +283,9 @@ static void unzip_extract(zip_header_t *zip_header, int dst_fd)
                transformer_state_t xstate;
                init_transformer_state(&xstate);
                xstate.bytes_in = zip_header->formatted.cmpsize;
-               if (inflate_unzip(&xstate, zip_fd, dst_fd) < 0)
+               xstate.src_fd = zip_fd;
+               xstate.dst_fd = dst_fd;
+               if (inflate_unzip(&xstate) < 0)
                        bb_error_msg_and_die("inflate error");
                /* Validate decompression - crc */
                if (zip_header->formatted.crc32 != (xstate.crc32 ^ 0xffffffffL)) {
index 8a2dd2735878d929f01d6e019429e02675ae591e..a6b166fe319d492d5e48f6ce00104ccb32b65171 100644 (file)
@@ -205,6 +205,18 @@ void dealloc_bunzip(bunzip_data *bd) FAST_FUNC;
 /* Meaning and direction (input/output) of the fields are transformer-specific */
 typedef struct transformer_state_t {
        smallint check_signature; /* most often referenced member */
+
+       IF_DESKTOP(long long) int FAST_FUNC (*xformer)(struct transformer_state_t *xstate);
+       USE_FOR_NOMMU(const char *xformer_prog;)
+
+       /* Source */
+       int      src_fd;
+       /* Output */
+       int      dst_fd;
+       size_t   mem_output_size_max; /* if non-zero, decompress to RAM instead of fd */
+       size_t   mem_output_size;
+       char     *mem_output_buf;
+
        off_t    bytes_out;
        off_t    bytes_in;  /* used in unzip code only: needs to know packed size */
        uint32_t crc32;
@@ -212,14 +224,16 @@ typedef struct transformer_state_t {
 } transformer_state_t;
 
 void init_transformer_state(transformer_state_t *xstate) FAST_FUNC;
-int FAST_FUNC check_signature16(transformer_state_t *xstate, int src_fd, unsigned magic16) FAST_FUNC;
+ssize_t transformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize) FAST_FUNC;
+ssize_t xtransformer_write(transformer_state_t *xstate, const void *buf, size_t bufsize) FAST_FUNC;
+int check_signature16(transformer_state_t *xstate, unsigned magic16) FAST_FUNC;
 
-IF_DESKTOP(long long) int inflate_unzip(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_Z_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_gz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_bz2_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_lzma_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
-IF_DESKTOP(long long) int unpack_xz_stream(transformer_state_t *xstate, int src_fd, int dst_fd) FAST_FUNC;
+IF_DESKTOP(long long) int inflate_unzip(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_Z_stream(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_gz_stream(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_bz2_stream(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_lzma_stream(transformer_state_t *xstate) FAST_FUNC;
+IF_DESKTOP(long long) int unpack_xz_stream(transformer_state_t *xstate) FAST_FUNC;
 
 char* append_ext(char *filename, const char *expected_ext) FAST_FUNC;
 int bbunpack(char **argv,
@@ -230,16 +244,16 @@ int bbunpack(char **argv,
 
 void check_errors_in_children(int signo);
 #if BB_MMU
-void open_transformer(int fd,
+void fork_transformer(int fd,
        int check_signature,
-       IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate, int src_fd, int dst_fd)
+       IF_DESKTOP(long long) int FAST_FUNC (*transformer)(transformer_state_t *xstate)
 ) FAST_FUNC;
-#define open_transformer_with_sig(fd, transformer, transform_prog) open_transformer((fd), 1, (transformer))
-#define open_transformer_with_no_sig(fd, transformer)              open_transformer((fd), 0, (transformer))
+#define fork_transformer_with_sig(fd, transformer, transform_prog) fork_transformer((fd), 1, (transformer))
+#define fork_transformer_with_no_sig(fd, transformer)              fork_transformer((fd), 0, (transformer))
 #else
-void open_transformer(int fd, const char *transform_prog) FAST_FUNC;
-#define open_transformer_with_sig(fd, transformer, transform_prog) open_transformer((fd), (transform_prog))
-/* open_transformer_with_no_sig() does not exist on NOMMU */
+void fork_transformer(int fd, const char *transform_prog) FAST_FUNC;
+#define fork_transformer_with_sig(fd, transformer, transform_prog) fork_transformer((fd), (transform_prog))
+/* fork_transformer_with_no_sig() does not exist on NOMMU */
 #endif
 
 
index 17a0089d8afbc914507a844281b0d1c637fb779c..8e8b9ca0e99667f6c8c9aae3356a5f951efd4473 100644 (file)
@@ -755,11 +755,12 @@ unsigned bb_clk_tck(void) FAST_FUNC;
 extern int setup_unzip_on_fd(int fd, int fail_if_not_compressed) FAST_FUNC;
 /* Autodetects .gz etc */
 extern int open_zipped(const char *fname, int fail_if_not_compressed) FAST_FUNC;
+extern void *xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_p) FAST_FUNC RETURNS_MALLOC;
 #else
 # define setup_unzip_on_fd(...) (0)
 # define open_zipped(fname, fail_if_not_compressed)  open((fname), O_RDONLY);
+# define xmalloc_open_zipped_read_close(fname, maxsz_p) xmalloc_open_read_close((fname), (maxsz_p))
 #endif
-extern void *xmalloc_open_zipped_read_close(const char *fname, size_t *maxsz_p) FAST_FUNC RETURNS_MALLOC;
 
 extern ssize_t safe_write(int fd, const void *buf, size_t count) FAST_FUNC;
 // NB: will return short write on error, not -1,