From: Felix Fietkau Date: Sun, 17 Jan 2016 12:37:59 +0000 (+0100) Subject: uclient-fetch: add progress bar support X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=8e6590cb48c611f480a5817d0cd5c7d7e6c5794d;p=oweals%2Fuclient.git uclient-fetch: add progress bar support Signed-off-by: Felix Fietkau --- diff --git a/CMakeLists.txt b/CMakeLists.txt index ee7d91d..1f5efce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ ENDIF() ADD_LIBRARY(uclient SHARED uclient.c uclient-http.c uclient-utils.c) TARGET_LINK_LIBRARIES(uclient ubox dl) -ADD_EXECUTABLE(uclient-fetch uclient-fetch.c) +ADD_EXECUTABLE(uclient-fetch uclient-fetch.c progress.c) TARGET_LINK_LIBRARIES(uclient-fetch uclient) INSTALL(FILES uclient.h uclient-utils.h diff --git a/progress.c b/progress.c new file mode 100644 index 0000000..f62bc9c --- /dev/null +++ b/progress.c @@ -0,0 +1,229 @@ +/* vi: set sw=4 ts=4: */ +/* + * Progress bar code. + */ +/* Original copyright notice which applies to the CONFIG_FEATURE_WGET_STATUSBAR stuff, + * much of which was blatantly stolen from openssh. + */ +/*- + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. BSD Advertising Clause omitted per the July 22, 1999 licensing change + * ftp://ftp.cs.berkeley.edu/pub/4bsd/README.Impt.License.Change + * + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include "progress.h" + +enum { + /* Seconds when xfer considered "stalled" */ + STALLTIME = 5 +}; + +static int +wh_helper(int value, int def_val, const char *env_name) +{ + if (value == 0) { + char *s = getenv(env_name); + if (s) + value = atoi(s); + } + if (value <= 1 || value >= 30000) + value = def_val; + return value; +} + +static unsigned int +get_tty2_width(void) +{ +#ifndef TIOCGWINSZ + return wh_helper(0, 80, "COLUMNS"); +#else + struct winsize win = {0}; + ioctl(2, TIOCGWINSZ, &win); + return wh_helper(win.ws_col, 80, "COLUMNS"); +#endif +} + +static unsigned int +monotonic_sec(void) +{ + struct timespec tv; + clock_gettime(CLOCK_MONOTONIC, &tv); + return tv.tv_sec; +} + +void +progress_init(struct progress *p, const char *curfile) +{ + p->curfile = strdup(curfile); + p->start_sec = monotonic_sec(); + p->last_update_sec = p->start_sec; + p->last_change_sec = p->start_sec; + p->last_size = 0; +} + +/* File already had beg_size bytes. + * Then we started downloading. + * We downloaded "transferred" bytes so far. + * Download is expected to stop when total size (beg_size + transferred) + * will be "totalsize" bytes. + * If totalsize == 0, then it is unknown. + */ +void +progress_update(struct progress *p, off_t beg_size, + off_t transferred, off_t totalsize) +{ + off_t beg_and_transferred; + unsigned since_last_update, elapsed; + int barlength; + int kiloscale; + + //transferred = 1234; /* use for stall detection testing */ + //totalsize = 0; /* use for unknown size download testing */ + + elapsed = monotonic_sec(); + since_last_update = elapsed - p->last_update_sec; + p->last_update_sec = elapsed; + + if (totalsize != 0 && transferred >= totalsize - beg_size) { + /* Last call. Do not skip this update */ + transferred = totalsize - beg_size; /* sanitize just in case */ + } + else if (since_last_update == 0) { + /* + * Do not update on every call + * (we can be called on every network read!) + */ + return; + } + + kiloscale = 0; + /* + * Scale sizes down if they are close to overflowing. + * This allows calculations like (100 * transferred / totalsize) + * without risking overflow: we guarantee 10 highest bits to be 0. + * Introduced error is less than 1 / 2^12 ~= 0.025% + */ + if (ULONG_MAX > 0xffffffff || sizeof(off_t) == 4 || sizeof(off_t) != 8) { + /* + * 64-bit CPU || small off_t: in either case, + * >> is cheap, single-word operation. + * ... || strange off_t: also use this code + * (it is safe, just suboptimal wrt code size), + * because 32/64 optimized one works only for 64-bit off_t. + */ + if (totalsize >= (1 << 22)) { + totalsize >>= 10; + beg_size >>= 10; + transferred >>= 10; + kiloscale = 1; + } + } else { + /* 32-bit CPU and 64-bit off_t. + * Use a 40-bit shift, it is easier to do on 32-bit CPU. + */ +/* ONE suppresses "warning: shift count >= width of type" */ +#define ONE (sizeof(off_t) > 4) + if (totalsize >= (off_t)(1ULL << 54*ONE)) { + totalsize = (uint32_t)(totalsize >> 32*ONE) >> 8; + beg_size = (uint32_t)(beg_size >> 32*ONE) >> 8; + transferred = (uint32_t)(transferred >> 32*ONE) >> 8; + kiloscale = 4; + } + } + + fprintf(stderr, "\r%-20.20s", p->curfile); + + beg_and_transferred = beg_size + transferred; + + if (totalsize != 0) { + unsigned ratio = 100 * beg_and_transferred / totalsize; + fprintf(stderr, "%4u%%", ratio); + + barlength = get_tty2_width() - 49; + if (barlength > 0) { + /* god bless gcc for variable arrays :) */ + char buf[barlength + 1]; + unsigned stars = (unsigned)barlength * beg_and_transferred / totalsize; + memset(buf, ' ', barlength); + buf[barlength] = '\0'; + memset(buf, '*', stars); + fprintf(stderr, " |%s|", buf); + } + } + + while (beg_and_transferred >= 100000) { + beg_and_transferred >>= 10; + kiloscale++; + } + /* see http://en.wikipedia.org/wiki/Tera */ + fprintf(stderr, "%6u%c", (unsigned)beg_and_transferred, " kMGTPEZY"[kiloscale]); + + since_last_update = elapsed - p->last_change_sec; + if ((unsigned)transferred != p->last_size) { + p->last_change_sec = elapsed; + p->last_size = (unsigned)transferred; + if (since_last_update >= STALLTIME) { + /* We "cut out" these seconds from elapsed time + * by adjusting start time */ + p->start_sec += since_last_update; + } + since_last_update = 0; /* we are un-stalled now */ + } + + elapsed -= p->start_sec; /* now it's "elapsed since start" */ + + if (since_last_update >= STALLTIME) { + fprintf(stderr, " - stalled -"); + } else if (!totalsize || !transferred || (int)elapsed < 0) { + fprintf(stderr, " --:--:-- ETA"); + } else { + unsigned eta, secs, hours; + + totalsize -= beg_size; /* now it's "total to upload" */ + + /* Estimated remaining time = + * estimated_sec_to_dl_totalsize_bytes - elapsed_sec = + * totalsize / average_bytes_sec_so_far - elapsed = + * totalsize / (transferred/elapsed) - elapsed = + * totalsize * elapsed / transferred - elapsed + */ + eta = totalsize * elapsed / transferred - elapsed; + if (eta >= 1000*60*60) + eta = 1000*60*60 - 1; + secs = eta % 3600; + hours = eta / 3600; + fprintf(stderr, "%3u:%02u:%02u ETA", hours, secs / 60, secs % 60); + } +} diff --git a/progress.h b/progress.h new file mode 100644 index 0000000..720060d --- /dev/null +++ b/progress.h @@ -0,0 +1,25 @@ +#ifndef __PROGRESS_H +#define __PROGRESS_H + +#include + +struct progress { + unsigned int last_size; + unsigned int last_update_sec; + unsigned int last_change_sec; + unsigned int start_sec; + char *curfile; +}; + + +void progress_init(struct progress *p, const char *curfile); +void progress_update(struct progress *p, off_t beg_size, + off_t transferred, off_t totalsize); + +static inline void +progress_free(struct progress *p) +{ + free(p->curfile); +} + +#endif diff --git a/uclient-fetch.c b/uclient-fetch.c index 5ab2ea0..3f875c9 100644 --- a/uclient-fetch.c +++ b/uclient-fetch.c @@ -30,6 +30,7 @@ #include +#include "progress.h" #include "uclient.h" #include "uclient-utils.h" @@ -51,16 +52,27 @@ static bool no_output; static const char *output_file; static int output_fd = -1; static int error_ret; -static int out_bytes; +static off_t out_offset; +static off_t out_bytes; +static off_t out_len; static char *auth_str; static char **urls; static int n_urls; static int timeout; static bool resume, cur_resume; +static struct progress pmt; +static struct uloop_timeout pmt_timer; + static int init_request(struct uclient *cl); static void request_done(struct uclient *cl); +static void pmt_update(struct uloop_timeout *t) +{ + progress_update(&pmt, out_offset, out_bytes, out_len); + uloop_timeout_set(t, 1000); +} + static const char * get_proxy_url(char *url) { @@ -110,31 +122,43 @@ static int open_output_file(const char *path, uint64_t resume_offset) if (!quiet) fprintf(stderr, "Writing to '%s'\n", output_file); ret = open(output_file, flags, 0644); - free(filename); - if (ret < 0) - return ret; + goto free; if (resume_offset && lseek(ret, resume_offset, SEEK_SET) < 0) { if (!quiet) fprintf(stderr, "Failed to seek %"PRIu64" bytes in output file\n", resume_offset); close(ret); - return -1; + ret = -1; + goto free; } + out_offset = resume_offset; out_bytes += resume_offset; + if (!quiet) { + progress_init(&pmt, output_file); + pmt_timer.cb = pmt_update; + pmt_timer.cb(&pmt_timer); + } +free: + free(filename); return ret; } static void header_done_cb(struct uclient *cl) { - static const struct blobmsg_policy policy = { - .name = "content-range", - .type = BLOBMSG_TYPE_STRING + enum { + H_RANGE, + H_LEN, + __H_MAX + }; + static const struct blobmsg_policy policy[__H_MAX] = { + [H_RANGE] = { .name = "content-range", .type = BLOBMSG_TYPE_STRING }, + [H_LEN] = { .name = "content-length", .type = BLOBMSG_TYPE_STRING }, }; - struct blob_attr *attr; + struct blob_attr *tb[__H_MAX]; uint64_t resume_offset = 0, resume_end, resume_size; static int retries; @@ -153,6 +177,8 @@ static void header_done_cb(struct uclient *cl) return; } + blobmsg_parse(policy, __H_MAX, tb, blob_data(cl->meta), blob_len(cl->meta)); + switch (cl->status_code) { case 416: if (!quiet) @@ -168,15 +194,15 @@ static void header_done_cb(struct uclient *cl) break; } - blobmsg_parse(&policy, 1, &attr, blob_data(cl->meta), blob_len(cl->meta)); - if (!attr) { + if (!tb[H_RANGE]) { if (!quiet) fprintf(stderr, "Content-Range header is missing\n"); error_ret = 8; break; } - if (sscanf(blobmsg_get_string(attr), "bytes %"PRIu64"-%"PRIu64"/%"PRIu64, + if (sscanf(blobmsg_get_string(tb[H_RANGE]), + "bytes %"PRIu64"-%"PRIu64"/%"PRIu64, &resume_offset, &resume_end, &resume_size) != 3) { if (!quiet) fprintf(stderr, "Content-Range header is invalid\n"); @@ -187,6 +213,10 @@ static void header_done_cb(struct uclient *cl) case 200: if (no_output) break; + + if (tb[H_LEN]) + out_len = strtoul(blobmsg_get_string(tb[H_LEN]), NULL, 10); + output_fd = open_output_file(cl->url->location, resume_offset); if (output_fd < 0) { if (!quiet) @@ -263,7 +293,9 @@ static int init_request(struct uclient *cl) { int rc; + out_offset = 0; out_bytes = 0; + out_len = 0; uclient_http_set_ssl_ctx(cl, ssl_ops, ssl_ctx, verify); if (timeout) @@ -326,12 +358,17 @@ static void request_done(struct uclient *cl) static void eof_cb(struct uclient *cl) { + if (!quiet) { + pmt_update(&pmt_timer); + uloop_timeout_cancel(&pmt_timer); + } + if (!cl->data_eof) { if (!quiet) fprintf(stderr, "Connection reset prematurely\n"); error_ret = 4; } else if (!quiet) { - fprintf(stderr, "Download completed (%d bytes)\n", out_bytes); + fprintf(stderr, "Download completed (%"PRIu64" bytes)\n", (uint64_t) out_bytes); } request_done(cl); }