1 /* vi: set sw=4 ts=4: */
3 * wget - retrieve a file using HTTP or FTP
5 * Chip Rosenthal Covad Communications <chip@laserlink.net>
6 * Licensed under GPLv2, see file LICENSE in this source tree.
8 * Copyright (C) 2010 Bradley M. Kuhn <bkuhn@ebb.org>
9 * Kuhn's copyrights are licensed GPLv2-or-later. File as a whole remains GPLv2.
16 //config: wget is a utility for non-interactive download of files from HTTP
17 //config: and FTP servers.
19 //config:config FEATURE_WGET_LONG_OPTIONS
20 //config: bool "Enable long options"
22 //config: depends on WGET && LONG_OPTS
24 //config:config FEATURE_WGET_STATUSBAR
25 //config: bool "Enable progress bar (+2k)"
27 //config: depends on WGET
29 //config:config FEATURE_WGET_AUTHENTICATION
30 //config: bool "Enable HTTP authentication"
32 //config: depends on WGET
34 //config: Support authenticated HTTP transfers.
36 //config:config FEATURE_WGET_TIMEOUT
37 //config: bool "Enable timeout option -T SEC"
39 //config: depends on WGET
41 //config: Supports network read and connect timeouts for wget,
42 //config: so that wget will give up and timeout, through the -T
43 //config: command line option.
45 //config: Currently only connect and network data read timeout are
46 //config: supported (i.e., timeout is not applied to the DNS query). When
47 //config: FEATURE_WGET_LONG_OPTIONS is also enabled, the --timeout option
48 //config: will work in addition to -T.
50 //config:config FEATURE_WGET_HTTPS
51 //config: bool "Support HTTPS using internal TLS code"
53 //config: depends on WGET
56 //config: wget will use internal TLS code to connect to https:// URLs.
58 //config: On NOMMU machines, ssl_helper applet should be available
59 //config: in the $PATH for this to work. Make sure to select that applet.
61 //config: Note: currently, TLS code only makes TLS I/O work, it
62 //config: does *not* check that the peer is who it claims to be, etc.
63 //config: IOW: it uses peer-supplied public keys to establish encryption
64 //config: and signing keys, then encrypts and signs outgoing data and
65 //config: decrypts incoming data.
66 //config: It does not check signature hashes on the incoming data:
67 //config: this means that attackers manipulating TCP packets can
68 //config: send altered data and we unknowingly receive garbage.
69 //config: (This check might be relatively easy to add).
70 //config: It does not check public key's certificate:
71 //config: this means that the peer may be an attacker impersonating
72 //config: the server we think we are talking to.
74 //config: If you think this is unacceptable, consider this. As more and more
75 //config: servers switch to HTTPS-only operation, without such "crippled"
76 //config: TLS code it is *impossible* to simply download a kernel source
77 //config: from kernel.org. Which can in real world translate into
78 //config: "my small automatic tooling to build cross-compilers from sources
79 //config: no longer works, I need to additionally keep a local copy
80 //config: of ~4 megabyte source tarball of a SSL library and ~2 megabyte
81 //config: source of wget, need to compile and built both before I can
82 //config: download anything. All this despite the fact that the build
83 //config: is done in a QEMU sandbox on a machine with absolutely nothing
84 //config: worth stealing, so I don't care if someone would go to a lot
85 //config: of trouble to intercept my HTTPS download to send me an altered
86 //config: kernel tarball".
88 //config: If you still think this is unacceptable, send patches.
90 //config: If you still think this is unacceptable, do not want to send
91 //config: patches, but do want to waste bandwidth expaining how wrong
92 //config: it is, you will be ignored.
94 //config:config FEATURE_WGET_OPENSSL
95 //config: bool "Try to connect to HTTPS using openssl"
97 //config: depends on WGET
99 //config: Try to use openssl to handle HTTPS.
101 //config: OpenSSL has a simple SSL client for debug purposes.
102 //config: If you select this option, wget will effectively run:
103 //config: "openssl s_client -quiet -connect hostname:443
104 //config: -servername hostname 2>/dev/null" and pipe its data
105 //config: through it. -servername is not used if hostname is numeric.
106 //config: Note inconvenient API: host resolution is done twice,
107 //config: and there is no guarantee openssl's idea of IPv6 address
108 //config: format is the same as ours.
109 //config: Another problem is that s_client prints debug information
110 //config: to stderr, and it needs to be suppressed. This means
111 //config: all error messages get suppressed too.
112 //config: openssl is also a big binary, often dynamically linked
113 //config: against ~15 libraries.
115 //config: If openssl can't be executed, internal TLS code will be used
116 //config: (if you enabled it); if openssl can be executed but fails later,
117 //config: wget can't detect this, and download will fail.
119 //applet:IF_WGET(APPLET(wget, BB_DIR_USR_BIN, BB_SUID_DROP))
121 //kbuild:lib-$(CONFIG_WGET) += wget.o
123 //usage:#define wget_trivial_usage
124 //usage: IF_FEATURE_WGET_LONG_OPTIONS(
125 //usage: "[-c|--continue] [--spider] [-q|--quiet] [-O|--output-document FILE]\n"
126 //usage: " [--header 'header: value'] [-Y|--proxy on/off] [-P DIR]\n"
127 /* Since we ignore these opts, we don't show them in --help */
128 /* //usage: " [--no-check-certificate] [--no-cache] [--passive-ftp] [-t TRIES]" */
129 /* //usage: " [-nv] [-nc] [-nH] [-np]" */
130 //usage: " [-S|--server-response] [-U|--user-agent AGENT]" IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
132 //usage: IF_NOT_FEATURE_WGET_LONG_OPTIONS(
133 //usage: "[-cq] [-O FILE] [-Y on/off] [-P DIR] [-S] [-U AGENT]"
134 //usage: IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
136 //usage:#define wget_full_usage "\n\n"
137 //usage: "Retrieve files via HTTP or FTP\n"
138 //usage: IF_FEATURE_WGET_LONG_OPTIONS(
139 //usage: "\n --spider Only check URL existence: $? is 0 if exists"
141 //usage: "\n -c Continue retrieval of aborted transfer"
142 //usage: "\n -q Quiet"
143 //usage: "\n -P DIR Save to DIR (default .)"
144 //usage: "\n -S Show server response"
145 //usage: IF_FEATURE_WGET_TIMEOUT(
146 //usage: "\n -T SEC Network read timeout is SEC seconds"
148 //usage: "\n -O FILE Save to FILE ('-' for stdout)"
149 //usage: "\n -U STR Use STR for User-Agent header"
150 //usage: "\n -Y on/off Use proxy"
155 # define log_io(...) bb_error_msg(__VA_ARGS__)
156 # define SENDFMT(fp, fmt, ...) \
158 log_io("> " fmt, ##__VA_ARGS__); \
159 fprintf(fp, fmt, ##__VA_ARGS__); \
162 # define log_io(...) ((void)0)
163 # define SENDFMT(fp, fmt, ...) fprintf(fp, fmt, ##__VA_ARGS__)
167 #define SSL_SUPPORTED (ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_HTTPS)
173 const char *protocol;
177 static const char P_FTP[] ALIGN1 = "ftp";
178 static const char P_HTTP[] ALIGN1 = "http";
180 static const char P_HTTPS[] ALIGN1 = "https";
183 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
184 /* User-specified headers prevent using our corresponding built-in headers. */
187 HDR_USER_AGENT = (1<<1),
189 HDR_AUTH = (1<<3) * ENABLE_FEATURE_WGET_AUTHENTICATION,
190 HDR_PROXY_AUTH = (1<<4) * ENABLE_FEATURE_WGET_AUTHENTICATION,
192 static const char wget_user_headers[] ALIGN1 =
196 # if ENABLE_FEATURE_WGET_AUTHENTICATION
198 "Proxy-Authorization:\0"
201 # define USR_HEADER_HOST (G.user_headers & HDR_HOST)
202 # define USR_HEADER_USER_AGENT (G.user_headers & HDR_USER_AGENT)
203 # define USR_HEADER_RANGE (G.user_headers & HDR_RANGE)
204 # define USR_HEADER_AUTH (G.user_headers & HDR_AUTH)
205 # define USR_HEADER_PROXY_AUTH (G.user_headers & HDR_PROXY_AUTH)
206 #else /* No long options, no user-headers :( */
207 # define USR_HEADER_HOST 0
208 # define USR_HEADER_USER_AGENT 0
209 # define USR_HEADER_RANGE 0
210 # define USR_HEADER_AUTH 0
211 # define USR_HEADER_PROXY_AUTH 0
216 off_t content_len; /* Content-length of the file */
217 off_t beg_range; /* Range at which continue begins */
218 #if ENABLE_FEATURE_WGET_STATUSBAR
219 off_t transferred; /* Number of bytes transferred so far */
220 const char *curfile; /* Name of current file being transferred */
224 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
227 unsigned char user_headers; /* Headers mentioned by the user */
229 char *fname_out; /* where to direct output (-O) */
230 const char *proxy_flag; /* Use proxies if env vars are set */
231 const char *user_agent; /* "User-Agent" header field */
232 #if ENABLE_FEATURE_WGET_TIMEOUT
233 unsigned timeout_seconds;
234 bool die_if_timed_out;
238 smallint chunked; /* chunked transfer encoding */
239 smallint got_clen; /* got content-length: from server */
240 /* Local downloads do benefit from big buffer.
241 * With 512 byte buffer, it was measured to be
242 * an order of magnitude slower than with big one.
244 uint64_t just_to_align_next_member;
245 char wget_buf[CONFIG_FEATURE_COPYBUF_KB*1024];
247 #define G (*ptr_to_globals)
248 #define INIT_G() do { \
249 SET_PTR_TO_GLOBALS(xzalloc(sizeof(G))); \
251 #define FINI_G() do { \
252 FREE_PTR_TO_GLOBALS(); \
256 /* Must match option string! */
258 WGET_OPT_CONTINUE = (1 << 0),
259 WGET_OPT_QUIET = (1 << 1),
260 WGET_OPT_SERVER_RESPONSE = (1 << 2),
261 WGET_OPT_OUTNAME = (1 << 3),
262 WGET_OPT_PREFIX = (1 << 4),
263 WGET_OPT_PROXY = (1 << 5),
264 WGET_OPT_USER_AGENT = (1 << 6),
265 WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 7),
266 WGET_OPT_RETRIES = (1 << 8),
267 WGET_OPT_nsomething = (1 << 9),
268 WGET_OPT_HEADER = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
269 WGET_OPT_POST_DATA = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
270 WGET_OPT_SPIDER = (1 << 12) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
278 #if ENABLE_FEATURE_WGET_STATUSBAR
279 static void progress_meter(int flag)
281 if (option_mask32 & WGET_OPT_QUIET)
284 if (flag == PROGRESS_START)
285 bb_progress_init(&G.pmt, G.curfile);
287 bb_progress_update(&G.pmt,
290 (G.chunked || !G.got_clen) ? 0 : G.beg_range + G.transferred + G.content_len
293 if (flag == PROGRESS_END) {
294 bb_progress_free(&G.pmt);
295 bb_putchar_stderr('\n');
300 static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) { }
304 /* IPv6 knows scoped address types i.e. link and site local addresses. Link
305 * local addresses can have a scope identifier to specify the
306 * interface/link an address is valid on (e.g. fe80::1%eth0). This scope
307 * identifier is only valid on a single node.
309 * RFC 4007 says that the scope identifier MUST NOT be sent across the wire,
310 * unless all nodes agree on the semantic. Apache e.g. regards zone identifiers
311 * in the Host header as invalid requests, see
312 * https://issues.apache.org/bugzilla/show_bug.cgi?id=35122
314 static void strip_ipv6_scope_id(char *host)
318 /* bbox wget actually handles IPv6 addresses without [], like
319 * wget "http://::1/xxx", but this is not standard.
320 * To save code, _here_ we do not support it. */
323 return; /* not IPv6 */
325 scope = strchr(host, '%');
329 /* Remove the IPv6 zone identifier from the host address */
330 cp = strchr(host, ']');
331 if (!cp || (cp[1] != ':' && cp[1] != '\0')) {
332 /* malformed address (not "[xx]:nn" or "[xx]") */
336 /* cp points to "]...", scope points to "%eth0]..." */
337 overlapping_strcpy(scope, cp);
340 #if ENABLE_FEATURE_WGET_AUTHENTICATION
341 /* Base64-encode character string. */
342 static char *base64enc(const char *str)
344 unsigned len = strlen(str);
345 if (len > sizeof(G.wget_buf)/4*3 - 10) /* paranoia */
346 len = sizeof(G.wget_buf)/4*3 - 10;
347 bb_uuencode(G.wget_buf, str, len, bb_uuenc_tbl_base64);
352 static char* sanitize_string(char *s)
354 unsigned char *p = (void *) s;
361 #if ENABLE_FEATURE_WGET_TIMEOUT
362 static void alarm_handler(int sig UNUSED_PARAM)
364 /* This is theoretically unsafe (uses stdio and malloc in signal handler) */
365 if (G.die_if_timed_out)
366 bb_error_msg_and_die("download timed out");
368 static void set_alarm(void)
370 if (G.timeout_seconds) {
371 alarm(G.timeout_seconds);
372 G.die_if_timed_out = 1;
375 # define clear_alarm() ((void)(G.die_if_timed_out = 0))
377 # define set_alarm() ((void)0)
378 # define clear_alarm() ((void)0)
381 #if ENABLE_FEATURE_WGET_OPENSSL
383 * is_ip_address() attempts to verify whether or not a string
384 * contains an IPv4 or IPv6 address (vs. an FQDN). The result
385 * of inet_pton() can be used to determine this.
387 * TODO add proper error checking when inet_pton() returns -1
388 * (some form of system error has occurred, and errno is set)
390 static int is_ip_address(const char *string)
392 struct sockaddr_in sa;
394 int result = inet_pton(AF_INET, string, &(sa.sin_addr));
395 # if ENABLE_FEATURE_IPV6
397 struct sockaddr_in6 sa6;
398 result = inet_pton(AF_INET6, string, &(sa6.sin6_addr));
401 return (result == 1);
405 static FILE *open_socket(len_and_sockaddr *lsa)
411 fd = xconnect_stream(lsa);
414 /* glibc 2.4 seems to try seeking on it - ??! */
415 /* hopefully it understands what ESPIPE means... */
416 fp = fdopen(fd, "r+");
418 bb_perror_msg_and_die(bb_msg_memory_exhausted);
423 /* Returns '\n' if it was seen, else '\0'. Trims at first '\r' or '\n' */
424 static char fgets_and_trim(FILE *fp, const char *fmt)
430 if (fgets(G.wget_buf, sizeof(G.wget_buf) - 1, fp) == NULL)
431 bb_perror_msg_and_die("error getting response");
434 buf_ptr = strchrnul(G.wget_buf, '\n');
437 buf_ptr = strchrnul(G.wget_buf, '\r');
440 log_io("< %s", G.wget_buf);
442 if (fmt && (option_mask32 & WGET_OPT_SERVER_RESPONSE))
443 fprintf(stderr, fmt, G.wget_buf);
448 static int ftpcmd(const char *s1, const char *s2, FILE *fp)
454 fprintf(fp, "%s%s\r\n", s1, s2);
455 /* With --server-response, wget also shows its ftp commands */
456 if (option_mask32 & WGET_OPT_SERVER_RESPONSE)
457 fprintf(stderr, "--> %s%s\n\n", s1, s2);
459 log_io("> %s%s", s1, s2);
463 fgets_and_trim(fp, "%s\n");
464 } while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' ');
466 G.wget_buf[3] = '\0';
467 result = xatoi_positive(G.wget_buf);
472 static void parse_url(const char *src_url, struct host_info *h)
477 h->allocated = url = xstrdup(src_url);
480 p = strstr(url, "://");
484 if (strcmp(url, P_FTP) == 0) {
485 h->port = bb_lookup_port(P_FTP, "tcp", 21);
488 if (strcmp(url, P_HTTPS) == 0) {
489 h->port = bb_lookup_port(P_HTTPS, "tcp", 443);
490 h->protocol = P_HTTPS;
493 if (strcmp(url, P_HTTP) == 0) {
495 h->port = bb_lookup_port(P_HTTP, "tcp", 80);
496 h->protocol = P_HTTP;
499 bb_error_msg_and_die("not an http or ftp url: %s", sanitize_string(url));
502 // GNU wget is user-friendly and falls back to http://
508 // "Real" wget 'http://busybox.net?var=a/b' sends this request:
509 // 'GET /?var=a/b HTTP/1.0'
510 // and saves 'index.html?var=a%2Fb' (we save 'b')
511 // wget 'http://busybox.net?login=john@doe':
512 // request: 'GET /?login=john@doe HTTP/1.0'
513 // saves: 'index.html?login=john@doe' (we save '?login=john@doe')
514 // wget 'http://busybox.net#test/test':
515 // request: 'GET / HTTP/1.0'
516 // saves: 'index.html' (we save 'test')
518 // We also don't add unique .N suffix if file exists...
519 sp = strchr(h->host, '/');
520 p = strchr(h->host, '?'); if (!sp || (p && sp > p)) sp = p;
521 p = strchr(h->host, '#'); if (!sp || (p && sp > p)) sp = p;
524 } else if (*sp == '/') {
527 } else { // '#' or '?'
528 // http://busybox.net?login=john@doe is a valid URL
529 // memmove converts to:
530 // http:/busybox.nett?login=john@doe...
531 memmove(h->host - 1, h->host, sp - h->host);
537 sp = strrchr(h->host, '@');
539 // URL-decode "user:password" string before base64-encoding:
540 // wget http://test:my%20pass@example.com should send
541 // Authorization: Basic dGVzdDpteSBwYXNz
542 // which decodes to "test:my pass".
543 // Standard wget and curl do this too.
546 h->user = xstrdup(percent_decode_in_place(h->host, /*strict:*/ 0));
549 /* else: h->user remains NULL, or as set by original request
550 * before redirect (if we are here after a redirect).
554 static char *gethdr(FILE *fp)
559 /* retrieve header line */
560 c = fgets_and_trim(fp, " %s\n");
562 /* end of the headers? */
563 if (G.wget_buf[0] == '\0')
566 /* convert the header name to lower case */
567 for (s = G.wget_buf; isalnum(*s) || *s == '-' || *s == '.' || *s == '_'; ++s) {
569 * No-op for 20-3f and 60-7f. "0-9a-z-." are in these ranges.
570 * 40-5f range ("@A-Z[\]^_") maps to 60-7f.
571 * "A-Z" maps to "a-z".
572 * "@[\]" can't occur in header names.
573 * "^_" maps to "~,DEL" (which is wrong).
574 * "^" was never seen yet, "_" was seen from web.archive.org
575 * (x-archive-orig-x_commoncrawl_Signature: HEXSTRING).
580 /* verify we are at the end of the header name */
582 bb_error_msg_and_die("bad header line: %s", sanitize_string(G.wget_buf));
584 /* locate the start of the header value */
586 hdrval = skip_whitespace(s);
589 /* Rats! The buffer isn't big enough to hold the entire header value */
590 while (c = getc(fp), c != EOF && c != '\n')
597 static void reset_beg_range_to_zero(void)
599 bb_error_msg("restart failed");
601 xlseek(G.output_fd, 0, SEEK_SET);
602 /* Done at the end instead: */
603 /* ftruncate(G.output_fd, 0); */
606 static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa)
613 target->user = xstrdup("anonymous:busybox@");
615 sfp = open_socket(lsa);
616 if (ftpcmd(NULL, NULL, sfp) != 220)
617 bb_error_msg_and_die("%s", sanitize_string(G.wget_buf + 4));
620 * Splitting username:password pair,
623 str = strchr(target->user, ':');
626 switch (ftpcmd("USER ", target->user, sfp)) {
630 if (ftpcmd("PASS ", str, sfp) == 230)
632 /* fall through (failed login) */
634 bb_error_msg_and_die("ftp login: %s", sanitize_string(G.wget_buf + 4));
637 ftpcmd("TYPE I", NULL, sfp);
642 if (ftpcmd("SIZE ", target->path, sfp) == 213) {
643 G.content_len = BB_STRTOOFF(G.wget_buf + 4, NULL, 10);
644 if (G.content_len < 0 || errno) {
645 bb_error_msg_and_die("SIZE value is garbage");
651 * Entering passive mode
653 if (ftpcmd("PASV", NULL, sfp) != 227) {
655 bb_error_msg_and_die("bad response to %s: %s", "PASV", sanitize_string(G.wget_buf));
657 // Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage]
658 // Server's IP is N1.N2.N3.N4 (we ignore it)
659 // Server's port for data connection is P1*256+P2
660 str = strrchr(G.wget_buf, ')');
661 if (str) str[0] = '\0';
662 str = strrchr(G.wget_buf, ',');
663 if (!str) goto pasv_error;
664 port = xatou_range(str+1, 0, 255);
666 str = strrchr(G.wget_buf, ',');
667 if (!str) goto pasv_error;
668 port += xatou_range(str+1, 0, 255) * 256;
669 set_nport(&lsa->u.sa, htons(port));
671 *dfpp = open_socket(lsa);
673 if (G.beg_range != 0) {
674 sprintf(G.wget_buf, "REST %"OFF_FMT"u", G.beg_range);
675 if (ftpcmd(G.wget_buf, NULL, sfp) == 350)
676 G.content_len -= G.beg_range;
678 reset_beg_range_to_zero();
681 if (ftpcmd("RETR ", target->path, sfp) > 150)
682 bb_error_msg_and_die("bad response to %s: %s", "RETR", sanitize_string(G.wget_buf));
687 #if ENABLE_FEATURE_WGET_OPENSSL
688 static int spawn_https_helper_openssl(const char *host, unsigned port)
690 char *allocated = NULL;
694 IF_FEATURE_WGET_HTTPS(volatile int child_failed = 0;)
696 if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
697 /* Kernel can have AF_UNIX support disabled */
698 bb_perror_msg_and_die("socketpair");
700 if (!strchr(host, ':'))
701 host = allocated = xasprintf("%s:%u", host, port);
702 servername = xstrdup(host);
703 strrchr(servername, ':')[0] = '\0';
715 * openssl s_client -quiet -connect www.kernel.org:443 2>/dev/null
716 * It prints some debug stuff on stderr, don't know how to suppress it.
717 * Work around by dev-nulling stderr. We lose all error messages :(
720 xopen("/dev/null", O_RDWR);
721 memset(&argv, 0, sizeof(argv));
722 argv[0] = (char*)"openssl";
723 argv[1] = (char*)"s_client";
724 argv[2] = (char*)"-quiet";
725 argv[3] = (char*)"-connect";
726 argv[4] = (char*)host;
728 * Per RFC 6066 Section 3, the only permitted values in the
729 * TLS server_name (SNI) field are FQDNs (DNS hostnames).
730 * IPv4 and IPv6 addresses, port numbers are not allowed.
732 if (!is_ip_address(servername)) {
733 argv[5] = (char*)"-servername";
734 argv[6] = (char*)servername;
737 BB_EXECVP(argv[0], argv);
739 # if ENABLE_FEATURE_WGET_HTTPS
743 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
752 # if ENABLE_FEATURE_WGET_HTTPS
762 #if ENABLE_FEATURE_WGET_HTTPS
763 static void spawn_ssl_client(const char *host, int network_fd)
767 char *servername, *p;
769 servername = xstrdup(host);
770 p = strrchr(servername, ':');
773 if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
774 /* Kernel can have AF_UNIX support disabled */
775 bb_perror_msg_and_die("socketpair");
778 pid = BB_MMU ? xfork() : xvfork();
785 tls_state_t *tls = new_tls_state();
786 tls->ifd = tls->ofd = network_fd;
787 tls_handshake(tls, servername);
788 tls_run_copy_loop(tls);
792 xmove_fd(network_fd, 3);
793 argv[0] = (char*)"ssl_client";
794 argv[1] = (char*)"-s3";
795 //TODO: if (!is_ip_address(servername))...
796 argv[2] = (char*)"-n";
797 argv[3] = servername;
799 BB_EXECVP(argv[0], argv);
800 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
808 xmove_fd(sp[0], network_fd);
812 static void NOINLINE retrieve_file_data(FILE *dfp)
814 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
815 # if ENABLE_FEATURE_WGET_TIMEOUT
816 unsigned second_cnt = G.timeout_seconds;
818 struct pollfd polldata;
820 polldata.fd = fileno(dfp);
821 polldata.events = POLLIN | POLLPRI;
823 progress_meter(PROGRESS_START);
828 /* Loops only if chunked */
831 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
832 /* Must use nonblocking I/O, otherwise fread will loop
833 * and *block* until it reads full buffer,
834 * which messes up progress bar and/or timeout logic.
835 * Because of nonblocking I/O, we need to dance
836 * very carefully around EAGAIN. See explanation at
839 ndelay_on(polldata.fd);
845 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
846 /* fread internally uses read loop, which in our case
847 * is usually exited when we get EAGAIN.
848 * In this case, libc sets error marker on the stream.
849 * Need to clear it before next fread to avoid possible
850 * rare false positive ferror below. Rare because usually
851 * fread gets more than zero bytes, and we don't fall
852 * into if (n <= 0) ...
857 rdsz = sizeof(G.wget_buf);
859 if (G.content_len < (off_t)sizeof(G.wget_buf)) {
860 if ((int)G.content_len <= 0)
862 rdsz = (unsigned)G.content_len;
865 n = fread(G.wget_buf, 1, rdsz, dfp);
868 xwrite(G.output_fd, G.wget_buf, n);
869 #if ENABLE_FEATURE_WGET_STATUSBAR
874 if (G.content_len == 0)
877 #if ENABLE_FEATURE_WGET_TIMEOUT
878 second_cnt = G.timeout_seconds;
885 * If error occurs, or EOF is reached, the return value
886 * is a short item count (or zero).
887 * fread does not distinguish between EOF and error.
889 if (errno != EAGAIN) {
891 progress_meter(PROGRESS_END);
892 bb_perror_msg_and_die(bb_msg_read_error);
894 break; /* EOF, not error */
897 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
898 /* It was EAGAIN. There is no data. Wait up to one second
899 * then abort if timed out, or update the bar and try reading again.
901 if (safe_poll(&polldata, 1, 1000) == 0) {
902 # if ENABLE_FEATURE_WGET_TIMEOUT
903 if (second_cnt != 0 && --second_cnt == 0) {
904 progress_meter(PROGRESS_END);
905 bb_error_msg_and_die("download timed out");
908 /* We used to loop back to poll here,
909 * but there is no great harm in letting fread
910 * to try reading anyway.
915 /* Need to do it _every_ second for "stalled" indicator
916 * to be shown properly.
918 progress_meter(PROGRESS_BUMP);
919 } /* while (reading data) */
921 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
923 ndelay_off(polldata.fd); /* else fgets can get very unhappy */
928 fgets_and_trim(dfp, NULL); /* Eat empty line */
930 fgets_and_trim(dfp, NULL);
931 G.content_len = STRTOOFF(G.wget_buf, NULL, 16);
932 /* FIXME: error check? */
933 if (G.content_len == 0)
934 break; /* all done! */
937 * Note that fgets may result in some data being buffered in dfp.
938 * We loop back to fread, which will retrieve this data.
939 * Also note that code has to be arranged so that fread
940 * is done _before_ one-second poll wait - poll doesn't know
941 * about stdio buffering and can result in spurious one second waits!
945 /* If -c failed, we restart from the beginning,
946 * but we do not truncate file then, we do it only now, at the end.
947 * This lets user to ^C if his 99% complete 10 GB file download
948 * failed to restart *without* losing the almost complete file.
951 off_t pos = lseek(G.output_fd, 0, SEEK_CUR);
952 if (pos != (off_t)-1)
953 ftruncate(G.output_fd, pos);
956 /* Draw full bar and free its resources */
957 G.chunked = 0; /* makes it show 100% even for chunked download */
958 G.got_clen = 1; /* makes it show 100% even for download of (formerly) unknown size */
959 progress_meter(PROGRESS_END);
962 static void download_one_url(const char *url)
964 bool use_proxy; /* Use proxies if env vars are set */
966 len_and_sockaddr *lsa;
967 FILE *sfp; /* socket to web/ftp server */
968 FILE *dfp; /* socket to ftp server (data) */
970 char *fname_out_alloc;
971 char *redirected_path = NULL;
972 struct host_info server;
973 struct host_info target;
975 server.allocated = NULL;
976 target.allocated = NULL;
980 parse_url(url, &target);
982 /* Use the proxy if necessary */
983 use_proxy = (strcmp(G.proxy_flag, "off") != 0);
985 proxy = getenv(target.protocol == P_FTP ? "ftp_proxy" : "http_proxy");
986 //FIXME: what if protocol is https? Ok to use http_proxy?
987 use_proxy = (proxy && proxy[0]);
989 parse_url(proxy, &server);
992 server.port = target.port;
993 if (ENABLE_FEATURE_IPV6) {
994 //free(server.allocated); - can't be non-NULL
995 server.host = server.allocated = xstrdup(target.host);
997 server.host = target.host;
1001 if (ENABLE_FEATURE_IPV6)
1002 strip_ipv6_scope_id(target.host);
1004 /* If there was no -O FILE, guess output filename */
1005 fname_out_alloc = NULL;
1006 if (!(option_mask32 & WGET_OPT_OUTNAME)) {
1007 G.fname_out = bb_get_last_path_component_nostrip(target.path);
1008 /* handle "wget http://kernel.org//" */
1009 if (G.fname_out[0] == '/' || !G.fname_out[0])
1010 G.fname_out = (char*)"index.html";
1011 /* -P DIR is considered only if there was no -O FILE */
1013 G.fname_out = fname_out_alloc = concat_path_file(G.dir_prefix, G.fname_out);
1015 /* redirects may free target.path later, need to make a copy */
1016 G.fname_out = fname_out_alloc = xstrdup(G.fname_out);
1019 #if ENABLE_FEATURE_WGET_STATUSBAR
1020 G.curfile = bb_get_last_path_component_nostrip(G.fname_out);
1023 /* Determine where to start transfer */
1025 if (option_mask32 & WGET_OPT_CONTINUE) {
1026 G.output_fd = open(G.fname_out, O_WRONLY);
1027 if (G.output_fd >= 0) {
1028 G.beg_range = xlseek(G.output_fd, 0, SEEK_END);
1030 /* File doesn't exist. We do not create file here yet.
1031 * We are not sure it exists on remote side */
1036 lsa = xhost2sockaddr(server.host, server.port);
1037 if (!(option_mask32 & WGET_OPT_QUIET)) {
1038 char *s = xmalloc_sockaddr2dotted(&lsa->u.sa);
1039 fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);
1043 /*G.content_len = 0; - redundant, got_clen = 0 is enough */
1046 if (use_proxy || target.protocol != P_FTP) {
1053 /* Open socket to http(s) server */
1054 #if ENABLE_FEATURE_WGET_OPENSSL
1055 /* openssl (and maybe internal TLS) support is configured */
1056 if (target.protocol == P_HTTPS) {
1057 /* openssl-based helper
1058 * Inconvenient API since we can't give it an open fd
1060 int fd = spawn_https_helper_openssl(server.host, server.port);
1061 # if ENABLE_FEATURE_WGET_HTTPS
1062 if (fd < 0) { /* no openssl? try internal */
1063 sfp = open_socket(lsa);
1064 spawn_ssl_client(server.host, fileno(sfp));
1068 /* We don't check for exec("openssl") failure in this case */
1070 sfp = fdopen(fd, "r+");
1072 bb_perror_msg_and_die(bb_msg_memory_exhausted);
1075 sfp = open_socket(lsa);
1077 #elif ENABLE_FEATURE_WGET_HTTPS
1078 /* Only internal TLS support is configured */
1079 sfp = open_socket(lsa);
1080 if (target.protocol == P_HTTPS)
1081 spawn_ssl_client(server.host, fileno(sfp));
1083 /* ssl (https) support is not configured */
1084 sfp = open_socket(lsa);
1086 /* Send HTTP request */
1088 SENDFMT(sfp, "GET %s://%s/%s HTTP/1.1\r\n",
1089 target.protocol, target.host,
1092 SENDFMT(sfp, "%s /%s HTTP/1.1\r\n",
1093 (option_mask32 & WGET_OPT_POST_DATA) ? "POST" : "GET",
1096 if (!USR_HEADER_HOST)
1097 SENDFMT(sfp, "Host: %s\r\n", target.host);
1098 if (!USR_HEADER_USER_AGENT)
1099 SENDFMT(sfp, "User-Agent: %s\r\n", G.user_agent);
1101 /* Ask server to close the connection as soon as we are done
1102 * (IOW: we do not intend to send more requests)
1104 SENDFMT(sfp, "Connection: close\r\n");
1106 #if ENABLE_FEATURE_WGET_AUTHENTICATION
1107 if (target.user && !USR_HEADER_AUTH) {
1108 SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n"+6,
1109 base64enc(target.user));
1111 if (use_proxy && server.user && !USR_HEADER_PROXY_AUTH) {
1112 SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n",
1113 base64enc(server.user));
1117 if (G.beg_range != 0 && !USR_HEADER_RANGE)
1118 SENDFMT(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range);
1120 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1121 if (G.extra_headers) {
1122 log_io(G.extra_headers);
1123 fputs(G.extra_headers, sfp);
1126 if (option_mask32 & WGET_OPT_POST_DATA) {
1128 "Content-Type: application/x-www-form-urlencoded\r\n"
1129 "Content-Length: %u\r\n"
1132 (int) strlen(G.post_data), G.post_data
1137 SENDFMT(sfp, "\r\n");
1142 /* Tried doing this unconditionally.
1143 * Cloudflare and nginx/1.11.5 are shocked to see SHUT_WR on non-HTTPS.
1146 if (target.protocol == P_HTTPS) {
1147 /* If we use SSL helper, keeping our end of the socket open for writing
1148 * makes our end (i.e. the same fd!) readable (EAGAIN instead of EOF)
1149 * even after child closes its copy of the fd.
1152 shutdown(fileno(sfp), SHUT_WR);
1157 * Retrieve HTTP response line and check for "200" status code.
1160 fgets_and_trim(sfp, " %s\n");
1163 str = skip_non_whitespace(str);
1164 str = skip_whitespace(str);
1165 // FIXME: no error check
1166 // xatou wouldn't work: "200 OK"
1171 while (gethdr(sfp) != NULL)
1172 /* eat all remaining headers */;
1175 /* Success responses */
1178 case 201: /* 201 Created */
1179 /* "The request has been fulfilled and resulted in a new resource being created" */
1180 /* Standard wget is reported to treat this as success */
1182 case 202: /* 202 Accepted */
1183 /* "The request has been accepted for processing, but the processing has not been completed" */
1184 /* Treat as success: fall through */
1185 case 203: /* 203 Non-Authoritative Information */
1186 /* "Use of this response code is not required and is only appropriate when the response would otherwise be 200 (OK)" */
1188 case 204: /* 204 No Content */
1190 Response 204 doesn't say "null file", it says "metadata
1191 has changed but data didn't":
1193 "10.2.5 204 No Content
1194 The server has fulfilled the request but does not need to return
1195 an entity-body, and might want to return updated metainformation.
1196 The response MAY include new or updated metainformation in the form
1197 of entity-headers, which if present SHOULD be associated with
1198 the requested variant.
1200 If the client is a user agent, it SHOULD NOT change its document
1201 view from that which caused the request to be sent. This response
1202 is primarily intended to allow input for actions to take place
1203 without causing a change to the user agent's active document view,
1204 although any new or updated metainformation SHOULD be applied
1205 to the document currently in the user agent's active view.
1207 The 204 response MUST NOT include a message-body, and thus
1208 is always terminated by the first empty line after the header fields."
1210 However, in real world it was observed that some web servers
1211 (e.g. Boa/0.94.14rc21) simply use code 204 when file size is zero.
1213 if (G.beg_range != 0) {
1214 /* "Range:..." was not honored by the server.
1215 * Restart download from the beginning.
1217 reset_beg_range_to_zero();
1220 /* 205 Reset Content ?? what to do on this ?? */
1222 case 300: /* redirection */
1228 case 206: /* Partial Content */
1229 if (G.beg_range != 0)
1230 /* "Range:..." worked. Good. */
1232 /* Partial Content even though we did not ask for it??? */
1235 bb_error_msg_and_die("server returned error: %s", sanitize_string(G.wget_buf));
1239 * Retrieve HTTP headers.
1241 while ((str = gethdr(sfp)) != NULL) {
1242 static const char keywords[] ALIGN1 =
1243 "content-length\0""transfer-encoding\0""location\0";
1245 KEY_content_length = 1, KEY_transfer_encoding, KEY_location
1249 /* gethdr converted "FOO:" string to lowercase */
1251 /* strip trailing whitespace */
1252 char *s = strchrnul(str, '\0') - 1;
1253 while (s >= str && (*s == ' ' || *s == '\t')) {
1257 key = index_in_strings(keywords, G.wget_buf) + 1;
1258 if (key == KEY_content_length) {
1259 G.content_len = BB_STRTOOFF(str, NULL, 10);
1260 if (G.content_len < 0 || errno) {
1261 bb_error_msg_and_die("content-length %s is garbage", sanitize_string(str));
1266 if (key == KEY_transfer_encoding) {
1267 if (strcmp(str_tolower(str), "chunked") != 0)
1268 bb_error_msg_and_die("transfer encoding '%s' is not supported", sanitize_string(str));
1271 if (key == KEY_location && status >= 300) {
1272 if (--redir_limit == 0)
1273 bb_error_msg_and_die("too many redirections");
1275 if (str[0] == '/') {
1276 free(redirected_path);
1277 target.path = redirected_path = xstrdup(str+1);
1278 /* lsa stays the same: it's on the same server */
1280 parse_url(str, &target);
1282 /* server.user remains untouched */
1283 free(server.allocated);
1284 server.allocated = NULL;
1285 server.host = target.host;
1286 /* strip_ipv6_scope_id(target.host); - no! */
1287 /* we assume remote never gives us IPv6 addr with scope id */
1288 server.port = target.port;
1291 } /* else: lsa stays the same: we use proxy */
1293 goto establish_session;
1296 // if (status >= 300)
1297 // bb_error_msg_and_die("bad redirection (no Location: header from server)");
1299 /* For HTTP, data is pumped over the same connection */
1305 sfp = prepare_ftp_session(&dfp, &target, lsa);
1310 if (!(option_mask32 & WGET_OPT_SPIDER)) {
1311 if (G.output_fd < 0)
1312 G.output_fd = xopen(G.fname_out, G.o_flags);
1313 retrieve_file_data(dfp);
1314 if (!(option_mask32 & WGET_OPT_OUTNAME)) {
1315 xclose(G.output_fd);
1321 /* It's ftp. Close data connection properly */
1323 if (ftpcmd(NULL, NULL, sfp) != 226)
1324 bb_error_msg_and_die("ftp error: %s", sanitize_string(G.wget_buf + 4));
1325 /* ftpcmd("QUIT", NULL, sfp); - why bother? */
1329 free(server.allocated);
1330 free(target.allocated);
1333 free(fname_out_alloc);
1334 free(redirected_path);
1337 int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
1338 int wget_main(int argc UNUSED_PARAM, char **argv)
1340 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1341 static const char wget_longopts[] ALIGN1 =
1342 /* name, has_arg, val */
1343 "continue\0" No_argument "c"
1344 "quiet\0" No_argument "q"
1345 "server-response\0" No_argument "S"
1346 "output-document\0" Required_argument "O"
1347 "directory-prefix\0" Required_argument "P"
1348 "proxy\0" Required_argument "Y"
1349 "user-agent\0" Required_argument "U"
1350 IF_FEATURE_WGET_TIMEOUT(
1351 "timeout\0" Required_argument "T")
1353 IF_DESKTOP( "tries\0" Required_argument "t")
1354 "header\0" Required_argument "\xff"
1355 "post-data\0" Required_argument "\xfe"
1356 "spider\0" No_argument "\xfd"
1357 /* Ignored (we always use PASV): */
1358 IF_DESKTOP( "passive-ftp\0" No_argument "\xf0")
1359 /* Ignored (we don't do ssl) */
1360 IF_DESKTOP( "no-check-certificate\0" No_argument "\xf0")
1361 /* Ignored (we don't support caching) */
1362 IF_DESKTOP( "no-cache\0" No_argument "\xf0")
1363 IF_DESKTOP( "no-verbose\0" No_argument "\xf0")
1364 IF_DESKTOP( "no-clobber\0" No_argument "\xf0")
1365 IF_DESKTOP( "no-host-directories\0" No_argument "\xf0")
1366 IF_DESKTOP( "no-parent\0" No_argument "\xf0")
1370 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1371 llist_t *headers_llist = NULL;
1376 #if ENABLE_FEATURE_WGET_TIMEOUT
1377 G.timeout_seconds = 900;
1378 signal(SIGALRM, alarm_handler);
1380 G.proxy_flag = "on"; /* use proxies if env vars are set */
1381 G.user_agent = "Wget"; /* "User-Agent" header field */
1383 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1384 applet_long_options = wget_longopts;
1386 opt_complementary = "-1" /* at least one URL */
1387 IF_FEATURE_WGET_LONG_OPTIONS(":\xff::"); /* --header is a list */
1388 getopt32(argv, "cqSO:P:Y:U:T:+"
1391 /* wget has exactly four -n<letter> opts, all of which we can ignore:
1392 * -nv --no-verbose: be moderately quiet (-q is full quiet)
1393 * -nc --no-clobber: abort if exists, neither download to FILE.n nor overwrite FILE
1394 * -nH --no-host-directories: wget -r http://host/ won't create host/
1396 * "n::" above says that we accept -n[ARG].
1397 * Specifying "n:" would be a bug: "-n ARG" would eat ARG!
1399 , &G.fname_out, &G.dir_prefix,
1400 &G.proxy_flag, &G.user_agent,
1401 IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL),
1402 NULL, /* -t RETRIES */
1404 IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
1405 IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data)
1407 #if 0 /* option bits debug */
1408 if (option_mask32 & WGET_OPT_RETRIES) bb_error_msg("-t NUM");
1409 if (option_mask32 & WGET_OPT_nsomething) bb_error_msg("-nsomething");
1410 if (option_mask32 & WGET_OPT_HEADER) bb_error_msg("--header");
1411 if (option_mask32 & WGET_OPT_POST_DATA) bb_error_msg("--post-data");
1412 if (option_mask32 & WGET_OPT_SPIDER) bb_error_msg("--spider");
1417 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1418 if (headers_llist) {
1421 llist_t *ll = headers_llist;
1423 size += strlen(ll->data) + 2;
1426 G.extra_headers = hdr = xmalloc(size + 1);
1427 while (headers_llist) {
1431 size = sprintf(hdr, "%s\r\n",
1432 (char*)llist_pop(&headers_llist));
1433 /* a bit like index_in_substrings but don't match full key */
1435 words = wget_user_headers;
1437 if (strstr(hdr, words) == hdr) {
1438 G.user_headers |= bit;
1442 words += strlen(words) + 1;
1450 G.o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL;
1451 if (G.fname_out) { /* -O FILE ? */
1452 if (LONE_DASH(G.fname_out)) { /* -O - ? */
1454 option_mask32 &= ~WGET_OPT_CONTINUE;
1456 /* compat with wget: -O FILE can overwrite */
1457 G.o_flags = O_WRONLY | O_CREAT | O_TRUNC;
1461 download_one_url(*argv++);
1463 if (G.output_fd >= 0)
1464 xclose(G.output_fd);
1466 #if ENABLE_FEATURE_CLEAN_UP && ENABLE_FEATURE_WGET_LONG_OPTIONS
1467 free(G.extra_headers);
1471 return EXIT_SUCCESS;