1 /* vi: set sw=4 ts=4: */
3 * wget - retrieve a file using HTTP or FTP
5 * Chip Rosenthal Covad Communications <chip@laserlink.net>
7 * Licensed under GPLv2, see file LICENSE in this tarball for details.
12 // May be used if we ever will want to free() all xstrdup()s...
13 /* char *allocated; */
22 /* Globals (can be accessed from signal handlers) */
24 off_t content_len; /* Content-length of the file */
25 off_t beg_range; /* Range at which continue begins */
26 #if ENABLE_FEATURE_WGET_STATUSBAR
29 off_t transferred; /* Number of bytes transferred so far */
30 const char *curfile; /* Name of current file being transferred */
31 unsigned lastupdate_sec;
34 smallint chunked; /* chunked transfer encoding */
35 smallint got_clen; /* got content-length: from server */
37 #define G (*(struct globals*)&bb_common_bufsiz1)
38 struct BUG_G_too_big {
39 char BUG_G_too_big[sizeof(G) <= COMMON_BUFSIZE ? 1 : -1];
41 #define content_len (G.content_len )
42 #define beg_range (G.beg_range )
43 #define lastsize (G.lastsize )
44 #define totalsize (G.totalsize )
45 #define transferred (G.transferred )
46 #define curfile (G.curfile )
47 #define lastupdate_sec (G.lastupdate_sec )
48 #define start_sec (G.start_sec )
49 #define INIT_G() do { } while (0)
52 #if ENABLE_FEATURE_WGET_STATUSBAR
54 STALLTIME = 5 /* Seconds when xfer considered "stalled" */
57 static unsigned int get_tty2_width(void)
60 get_terminal_width_height(2, &width, NULL);
64 static void progress_meter(int flag)
66 /* We can be called from signal handler */
67 int save_errno = errno;
69 unsigned since_last_update, elapsed;
73 if (flag == -1) { /* first call to progress_meter */
74 start_sec = monotonic_sec();
75 lastupdate_sec = start_sec;
77 totalsize = content_len + beg_range; /* as content_len changes.. */
81 if (totalsize != 0 && !G.chunked) {
82 /* long long helps to have it working even if !LFS */
83 ratio = (unsigned) (100ULL * (transferred+beg_range) / totalsize);
84 if (ratio > 100) ratio = 100;
87 fprintf(stderr, "\r%-20.20s%4d%% ", curfile, ratio);
89 barlength = get_tty2_width() - 49;
91 /* god bless gcc for variable arrays :) */
92 i = barlength * ratio / 100;
97 fprintf(stderr, "|%s%*s|", buf, barlength - i, "");
101 abbrevsize = transferred + beg_range;
102 while (abbrevsize >= 100000) {
106 /* see http://en.wikipedia.org/wiki/Tera */
107 fprintf(stderr, "%6d%c ", (int)abbrevsize, " kMGTPEZY"[i]);
109 // Nuts! Ain't it easier to update progress meter ONLY when we transferred++?
111 elapsed = monotonic_sec();
112 since_last_update = elapsed - lastupdate_sec;
113 if (transferred > lastsize) {
114 lastupdate_sec = elapsed;
115 lastsize = transferred;
116 if (since_last_update >= STALLTIME) {
117 /* We "cut off" these seconds from elapsed time
118 * by adjusting start time */
119 start_sec += since_last_update;
121 since_last_update = 0; /* we are un-stalled now */
123 elapsed -= start_sec; /* now it's "elapsed since start" */
125 if (since_last_update >= STALLTIME) {
126 fprintf(stderr, " - stalled -");
128 off_t to_download = totalsize - beg_range;
129 if (transferred <= 0 || (int)elapsed <= 0 || transferred > to_download || G.chunked) {
130 fprintf(stderr, "--:--:-- ETA");
132 /* to_download / (transferred/elapsed) - elapsed: */
133 int eta = (int) ((unsigned long long)to_download*elapsed/transferred - elapsed);
134 /* (long long helps to have working ETA even if !LFS) */
136 fprintf(stderr, "%02d:%02d:%02d ETA", eta / 3600, i / 60, i % 60);
141 /* last call to progress_meter */
146 if (flag == -1) { /* first call to progress_meter */
147 signal_SA_RESTART_empty_mask(SIGALRM, progress_meter);
154 /* Original copyright notice which applies to the CONFIG_FEATURE_WGET_STATUSBAR stuff,
155 * much of which was blatantly stolen from openssh. */
157 * Copyright (c) 1992, 1993
158 * The Regents of the University of California. All rights reserved.
160 * Redistribution and use in source and binary forms, with or without
161 * modification, are permitted provided that the following conditions
163 * 1. Redistributions of source code must retain the above copyright
164 * notice, this list of conditions and the following disclaimer.
165 * 2. Redistributions in binary form must reproduce the above copyright
166 * notice, this list of conditions and the following disclaimer in the
167 * documentation and/or other materials provided with the distribution.
169 * 3. <BSD Advertising Clause omitted per the July 22, 1999 licensing change
170 * ftp://ftp.cs.berkeley.edu/pub/4bsd/README.Impt.License.Change>
172 * 4. Neither the name of the University nor the names of its contributors
173 * may be used to endorse or promote products derived from this software
174 * without specific prior written permission.
176 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
177 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
178 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
179 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
180 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
181 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
182 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
183 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
184 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
185 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
189 #else /* FEATURE_WGET_STATUSBAR */
191 static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) { }
196 /* IPv6 knows scoped address types i.e. link and site local addresses. Link
197 * local addresses can have a scope identifier to specify the
198 * interface/link an address is valid on (e.g. fe80::1%eth0). This scope
199 * identifier is only valid on a single node.
201 * RFC 4007 says that the scope identifier MUST NOT be sent across the wire,
202 * unless all nodes agree on the semantic. Apache e.g. regards zone identifiers
203 * in the Host header as invalid requests, see
204 * https://issues.apache.org/bugzilla/show_bug.cgi?id=35122
206 static void strip_ipv6_scope_id(char *host)
210 /* bbox wget actually handles IPv6 addresses without [], like
211 * wget "http://::1/xxx", but this is not standard.
212 * To save code, _here_ we do not support it. */
215 return; /* not IPv6 */
217 scope = strchr(host, '%');
221 /* Remove the IPv6 zone identifier from the host address */
222 cp = strchr(host, ']');
223 if (!cp || (cp[1] != ':' && cp[1] != '\0')) {
224 /* malformed address (not "[xx]:nn" or "[xx]") */
228 /* cp points to "]...", scope points to "%eth0]..." */
229 overlapping_strcpy(scope, cp);
232 /* Read NMEMB bytes into PTR from STREAM. Returns the number of bytes read,
233 * and a short count if an eof or non-interrupt error is encountered. */
234 static size_t safe_fread(void *ptr, size_t nmemb, FILE *stream)
237 char *p = (char*)ptr;
242 ret = fread(p, 1, nmemb, stream);
245 } while (nmemb && ferror(stream) && errno == EINTR);
247 return p - (char*)ptr;
250 /* Read a line or SIZE-1 bytes into S, whichever is less, from STREAM.
251 * Returns S, or NULL if an eof or non-interrupt error is encountered. */
252 static char *safe_fgets(char *s, int size, FILE *stream)
259 ret = fgets(s, size, stream);
260 } while (ret == NULL && ferror(stream) && errno == EINTR);
265 #if ENABLE_FEATURE_WGET_AUTHENTICATION
266 /* Base64-encode character string. buf is assumed to be char buf[512]. */
267 static char *base64enc_512(char buf[512], const char *str)
269 unsigned len = strlen(str);
270 if (len > 512/4*3 - 10) /* paranoia */
272 bb_uuencode(buf, str, len, bb_uuenc_tbl_base64);
277 static char* sanitize_string(char *s)
279 unsigned char *p = (void *) s;
286 static FILE *open_socket(len_and_sockaddr *lsa)
290 /* glibc 2.4 seems to try seeking on it - ??! */
291 /* hopefully it understands what ESPIPE means... */
292 fp = fdopen(xconnect_stream(lsa), "r+");
294 bb_perror_msg_and_die("fdopen");
299 static int ftpcmd(const char *s1, const char *s2, FILE *fp, char *buf)
304 fprintf(fp, "%s%s\r\n", s1, s2);
311 if (fgets(buf, 510, fp) == NULL) {
312 bb_perror_msg_and_die("error getting response");
314 buf_ptr = strstr(buf, "\r\n");
318 } while (!isdigit(buf[0]) || buf[3] != ' ');
321 result = xatoi_u(buf);
326 static void parse_url(char *src_url, struct host_info *h)
330 /* h->allocated = */ url = xstrdup(src_url);
332 if (strncmp(url, "http://", 7) == 0) {
333 h->port = bb_lookup_port("http", "tcp", 80);
336 } else if (strncmp(url, "ftp://", 6) == 0) {
337 h->port = bb_lookup_port("ftp", "tcp", 21);
341 bb_error_msg_and_die("not an http or ftp url: %s", sanitize_string(url));
344 // "Real" wget 'http://busybox.net?var=a/b' sends this request:
345 // 'GET /?var=a/b HTTP 1.0'
346 // and saves 'index.html?var=a%2Fb' (we save 'b')
347 // wget 'http://busybox.net?login=john@doe':
348 // request: 'GET /?login=john@doe HTTP/1.0'
349 // saves: 'index.html?login=john@doe' (we save '?login=john@doe')
350 // wget 'http://busybox.net#test/test':
351 // request: 'GET / HTTP/1.0'
352 // saves: 'index.html' (we save 'test')
354 // We also don't add unique .N suffix if file exists...
355 sp = strchr(h->host, '/');
356 p = strchr(h->host, '?'); if (!sp || (p && sp > p)) sp = p;
357 p = strchr(h->host, '#'); if (!sp || (p && sp > p)) sp = p;
360 } else if (*sp == '/') {
363 } else { // '#' or '?'
364 // http://busybox.net?login=john@doe is a valid URL
365 // memmove converts to:
366 // http:/busybox.nett?login=john@doe...
367 memmove(h->host - 1, h->host, sp - h->host);
373 // We used to set h->user to NULL here, but this interferes
374 // with handling of code 302 ("object was moved")
376 sp = strrchr(h->host, '@');
386 static char *gethdr(char *buf, size_t bufsiz, FILE *fp /*, int *istrunc*/)
393 /* retrieve header line */
394 if (fgets(buf, bufsiz, fp) == NULL)
397 /* see if we are at the end of the headers */
398 for (s = buf; *s == '\r'; ++s)
403 /* convert the header name to lower case */
404 for (s = buf; isalnum(*s) || *s == '-' || *s == '.'; ++s)
407 /* verify we are at the end of the header name */
409 bb_error_msg_and_die("bad header line: %s", sanitize_string(buf));
411 /* locate the start of the header value */
413 hdrval = skip_whitespace(s);
415 /* locate the end of header */
416 while (*s && *s != '\r' && *s != '\n')
419 /* end of header found */
425 /* Rats! The buffer isn't big enough to hold the entire header value */
426 while (c = getc(fp), c != EOF && c != '\n')
432 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
433 static char *URL_escape(const char *str)
435 /* URL encode, see RFC 2396 */
437 char *res = dst = xmalloc(strlen(str) * 3 + 1);
443 /* || strchr("!&'()*-.=_~", c) - more code */
455 || (c >= '0' && c <= '9')
456 || ((c|0x20) >= 'a' && (c|0x20) <= 'z')
463 *dst++ = bb_hexdigits_upcase[c >> 4];
464 *dst++ = bb_hexdigits_upcase[c & 0xf];
470 static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa)
478 target->user = xstrdup("anonymous:busybox@");
480 sfp = open_socket(lsa);
481 if (ftpcmd(NULL, NULL, sfp, buf) != 220)
482 bb_error_msg_and_die("%s", sanitize_string(buf+4));
485 * Splitting username:password pair,
488 str = strchr(target->user, ':');
491 switch (ftpcmd("USER ", target->user, sfp, buf)) {
495 if (ftpcmd("PASS ", str, sfp, buf) == 230)
497 /* fall through (failed login) */
499 bb_error_msg_and_die("ftp login: %s", sanitize_string(buf+4));
502 ftpcmd("TYPE I", NULL, sfp, buf);
507 if (ftpcmd("SIZE ", target->path, sfp, buf) == 213) {
508 content_len = BB_STRTOOFF(buf+4, NULL, 10);
509 if (errno || content_len < 0) {
510 bb_error_msg_and_die("SIZE value is garbage");
516 * Entering passive mode
518 if (ftpcmd("PASV", NULL, sfp, buf) != 227) {
520 bb_error_msg_and_die("bad response to %s: %s", "PASV", sanitize_string(buf));
522 // Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage]
523 // Server's IP is N1.N2.N3.N4 (we ignore it)
524 // Server's port for data connection is P1*256+P2
525 str = strrchr(buf, ')');
526 if (str) str[0] = '\0';
527 str = strrchr(buf, ',');
528 if (!str) goto pasv_error;
529 port = xatou_range(str+1, 0, 255);
531 str = strrchr(buf, ',');
532 if (!str) goto pasv_error;
533 port += xatou_range(str+1, 0, 255) * 256;
534 set_nport(lsa, htons(port));
536 *dfpp = open_socket(lsa);
539 sprintf(buf, "REST %"OFF_FMT"d", beg_range);
540 if (ftpcmd(buf, NULL, sfp, buf) == 350)
541 content_len -= beg_range;
544 if (ftpcmd("RETR ", target->path, sfp, buf) > 150)
545 bb_error_msg_and_die("bad response to %s: %s", "RETR", sanitize_string(buf));
550 /* Must match option string! */
552 WGET_OPT_CONTINUE = (1 << 0),
553 WGET_OPT_SPIDER = (1 << 1),
554 WGET_OPT_QUIET = (1 << 2),
555 WGET_OPT_OUTNAME = (1 << 3),
556 WGET_OPT_PREFIX = (1 << 4),
557 WGET_OPT_PROXY = (1 << 5),
558 WGET_OPT_USER_AGENT = (1 << 6),
559 WGET_OPT_RETRIES = (1 << 7),
560 WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 8),
561 WGET_OPT_PASSIVE = (1 << 9),
562 WGET_OPT_HEADER = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
563 WGET_OPT_POST_DATA = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
566 static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd)
570 if (!(option_mask32 & WGET_OPT_QUIET))
576 /* Loops only if chunked */
578 while (content_len > 0 || !G.got_clen) {
580 unsigned rdsz = sizeof(buf);
582 if (content_len < sizeof(buf) && (G.chunked || G.got_clen))
583 rdsz = (unsigned)content_len;
584 n = safe_fread(buf, rdsz, dfp);
587 /* perror will not work: ferror doesn't set errno */
588 bb_error_msg_and_die(bb_msg_read_error);
592 xwrite(output_fd, buf, n);
593 #if ENABLE_FEATURE_WGET_STATUSBAR
603 safe_fgets(buf, sizeof(buf), dfp); /* This is a newline */
605 safe_fgets(buf, sizeof(buf), dfp);
606 content_len = STRTOOFF(buf, NULL, 16);
607 /* FIXME: error check? */
608 if (content_len == 0)
609 break; /* all done! */
612 if (!(option_mask32 & WGET_OPT_QUIET))
616 int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
617 int wget_main(int argc UNUSED_PARAM, char **argv)
620 struct host_info server, target;
621 len_and_sockaddr *lsa;
625 char *dir_prefix = NULL;
626 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
628 char *extra_headers = NULL;
629 llist_t *headers_llist = NULL;
631 FILE *sfp; /* socket to web/ftp server */
632 FILE *dfp; /* socket to ftp server (data) */
633 char *fname_out; /* where to direct output (-O) */
635 bool use_proxy; /* Use proxies if env vars are set */
636 const char *proxy_flag = "on"; /* Use proxies if env vars are set */
637 const char *user_agent = "Wget";/* "User-Agent" header field */
639 static const char keywords[] ALIGN1 =
640 "content-length\0""transfer-encoding\0""chunked\0""location\0";
642 KEY_content_length = 1, KEY_transfer_encoding, KEY_chunked, KEY_location
644 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
645 static const char wget_longopts[] ALIGN1 =
646 /* name, has_arg, val */
647 "continue\0" No_argument "c"
648 "spider\0" No_argument "s"
649 "quiet\0" No_argument "q"
650 "output-document\0" Required_argument "O"
651 "directory-prefix\0" Required_argument "P"
652 "proxy\0" Required_argument "Y"
653 "user-agent\0" Required_argument "U"
655 // "tries\0" Required_argument "t"
656 // "timeout\0" Required_argument "T"
657 /* Ignored (we always use PASV): */
658 "passive-ftp\0" No_argument "\xff"
659 "header\0" Required_argument "\xfe"
660 "post-data\0" Required_argument "\xfd"
666 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
667 applet_long_options = wget_longopts;
669 /* server.allocated = target.allocated = NULL; */
670 opt_complementary = "-1" IF_FEATURE_WGET_LONG_OPTIONS(":\xfe::");
671 opt = getopt32(argv, "csqO:P:Y:U:" /*ignored:*/ "t:T:",
672 &fname_out, &dir_prefix,
673 &proxy_flag, &user_agent,
674 NULL, /* -t RETRIES */
675 NULL /* -T NETWORK_READ_TIMEOUT */
676 IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
677 IF_FEATURE_WGET_LONG_OPTIONS(, &post_data)
679 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
683 llist_t *ll = headers_llist;
685 size += strlen(ll->data) + 2;
688 extra_headers = cp = xmalloc(size);
689 while (headers_llist) {
690 cp += sprintf(cp, "%s\r\n", (char*)llist_pop(&headers_llist));
695 /* TODO: compat issue: should handle "wget URL1 URL2..." */
698 parse_url(argv[optind], &target);
700 /* Use the proxy if necessary */
701 use_proxy = (strcmp(proxy_flag, "off") != 0);
703 proxy = getenv(target.is_ftp ? "ftp_proxy" : "http_proxy");
704 if (proxy && proxy[0]) {
705 parse_url(proxy, &server);
711 server.port = target.port;
712 if (ENABLE_FEATURE_IPV6) {
713 server.host = xstrdup(target.host);
715 server.host = target.host;
719 if (ENABLE_FEATURE_IPV6)
720 strip_ipv6_scope_id(target.host);
722 /* Guess an output filename, if there was no -O FILE */
723 if (!(opt & WGET_OPT_OUTNAME)) {
724 fname_out = bb_get_last_path_component_nostrip(target.path);
725 /* handle "wget http://kernel.org//" */
726 if (fname_out[0] == '/' || !fname_out[0])
727 fname_out = (char*)"index.html";
728 /* -P DIR is considered only if there was no -O FILE */
730 fname_out = concat_path_file(dir_prefix, fname_out);
732 if (LONE_DASH(fname_out)) {
735 opt &= ~WGET_OPT_CONTINUE;
738 #if ENABLE_FEATURE_WGET_STATUSBAR
739 curfile = bb_get_last_path_component_nostrip(fname_out);
743 if ((opt & WGET_OPT_CONTINUE) && !fname_out)
744 bb_error_msg_and_die("cannot specify continue (-c) without a filename (-O)");
747 /* Determine where to start transfer */
748 if (opt & WGET_OPT_CONTINUE) {
749 output_fd = open(fname_out, O_WRONLY);
750 if (output_fd >= 0) {
751 beg_range = xlseek(output_fd, 0, SEEK_END);
753 /* File doesn't exist. We do not create file here yet.
754 * We are not sure it exists on remove side */
759 lsa = xhost2sockaddr(server.host, server.port);
760 if (!(opt & WGET_OPT_QUIET)) {
761 char *s = xmalloc_sockaddr2dotted(&lsa->u.sa);
762 fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);
766 if (use_proxy || !target.is_ftp) {
773 /* Open socket to http server */
774 sfp = open_socket(lsa);
776 /* Send HTTP request */
778 fprintf(sfp, "GET %stp://%s/%s HTTP/1.1\r\n",
779 target.is_ftp ? "f" : "ht", target.host,
782 if (opt & WGET_OPT_POST_DATA)
783 fprintf(sfp, "POST /%s HTTP/1.1\r\n", target.path);
785 fprintf(sfp, "GET /%s HTTP/1.1\r\n", target.path);
788 fprintf(sfp, "Host: %s\r\nUser-Agent: %s\r\n",
789 target.host, user_agent);
791 #if ENABLE_FEATURE_WGET_AUTHENTICATION
793 fprintf(sfp, "Proxy-Authorization: Basic %s\r\n"+6,
794 base64enc_512(buf, target.user));
796 if (use_proxy && server.user) {
797 fprintf(sfp, "Proxy-Authorization: Basic %s\r\n",
798 base64enc_512(buf, server.user));
803 fprintf(sfp, "Range: bytes=%"OFF_FMT"d-\r\n", beg_range);
804 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
806 fputs(extra_headers, sfp);
808 if (opt & WGET_OPT_POST_DATA) {
809 char *estr = URL_escape(post_data);
810 fprintf(sfp, "Content-Type: application/x-www-form-urlencoded\r\n");
811 fprintf(sfp, "Content-Length: %u\r\n" "\r\n" "%s",
812 (int) strlen(estr), estr);
813 /*fprintf(sfp, "Connection: Keep-Alive\r\n\r\n");*/
814 /*fprintf(sfp, "%s\r\n", estr);*/
818 { /* If "Connection:" is needed, document why */
819 fprintf(sfp, /* "Connection: close\r\n" */ "\r\n");
823 * Retrieve HTTP response line and check for "200" status code.
826 if (fgets(buf, sizeof(buf), sfp) == NULL)
827 bb_error_msg_and_die("no response from server");
830 str = skip_non_whitespace(str);
831 str = skip_whitespace(str);
832 // FIXME: no error check
833 // xatou wouldn't work: "200 OK"
838 while (gethdr(buf, sizeof(buf), sfp /*, &n*/) != NULL)
839 /* eat all remaining headers */;
843 Response 204 doesn't say "null file", it says "metadata
844 has changed but data didn't":
846 "10.2.5 204 No Content
847 The server has fulfilled the request but does not need to return
848 an entity-body, and might want to return updated metainformation.
849 The response MAY include new or updated metainformation in the form
850 of entity-headers, which if present SHOULD be associated with
851 the requested variant.
853 If the client is a user agent, it SHOULD NOT change its document
854 view from that which caused the request to be sent. This response
855 is primarily intended to allow input for actions to take place
856 without causing a change to the user agent's active document view,
857 although any new or updated metainformation SHOULD be applied
858 to the document currently in the user agent's active view.
860 The 204 response MUST NOT include a message-body, and thus
861 is always terminated by the first empty line after the header fields."
863 However, in real world it was observed that some web servers
864 (e.g. Boa/0.94.14rc21) simply use code 204 when file size is zero.
868 case 300: /* redirection */
878 bb_error_msg_and_die("server returned error: %s", sanitize_string(buf));
882 * Retrieve HTTP headers.
884 while ((str = gethdr(buf, sizeof(buf), sfp /*, &n*/)) != NULL) {
885 /* gethdr converted "FOO:" string to lowercase */
887 /* strip trailing whitespace */
888 char *s = strchrnul(str, '\0') - 1;
889 while (s >= str && (*s == ' ' || *s == '\t')) {
893 key = index_in_strings(keywords, buf) + 1;
894 if (key == KEY_content_length) {
895 content_len = BB_STRTOOFF(str, NULL, 10);
896 if (errno || content_len < 0) {
897 bb_error_msg_and_die("content-length %s is garbage", sanitize_string(str));
902 if (key == KEY_transfer_encoding) {
903 if (index_in_strings(keywords, str_tolower(str)) + 1 != KEY_chunked)
904 bb_error_msg_and_die("transfer encoding '%s' is not supported", sanitize_string(str));
905 G.chunked = G.got_clen = 1;
907 if (key == KEY_location && status >= 300) {
908 if (--redir_limit == 0)
909 bb_error_msg_and_die("too many redirections");
914 /* free(target.allocated); */
915 target.path = /* target.allocated = */ xstrdup(str+1);
916 /* lsa stays the same: it's on the same server */
918 parse_url(str, &target);
920 server.host = target.host;
921 /* strip_ipv6_scope_id(target.host); - no! */
922 /* we assume remote never gives us IPv6 addr with scope id */
923 server.port = target.port;
926 } /* else: lsa stays the same: we use proxy */
928 goto establish_session;
931 // if (status >= 300)
932 // bb_error_msg_and_die("bad redirection (no Location: header from server)");
934 /* For HTTP, data is pumped over the same connection */
941 sfp = prepare_ftp_session(&dfp, &target, lsa);
944 if (opt & WGET_OPT_SPIDER) {
945 if (ENABLE_FEATURE_CLEAN_UP)
951 int o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL;
952 /* compat with wget: -O FILE can overwrite */
953 if (opt & WGET_OPT_OUTNAME)
954 o_flags = O_WRONLY | O_CREAT | O_TRUNC;
955 output_fd = xopen(fname_out, o_flags);
958 retrieve_file_data(dfp, output_fd);
961 /* It's ftp. Close it properly */
963 if (ftpcmd(NULL, NULL, sfp, buf) != 226)
964 bb_error_msg_and_die("ftp error: %s", sanitize_string(buf+4));
965 ftpcmd("QUIT", NULL, sfp, buf);