networking/wget.c

   1 /* vi: set sw=4 ts=4: */
   2 /*
   3  * wget - retrieve a file using HTTP or FTP
   4  *
   5  * Chip Rosenthal Covad Communications <chip@laserlink.net>
   6  * Licensed under GPLv2, see file LICENSE in this source tree.
   7  *
   8  * Copyright (C) 2010 Bradley M. Kuhn <bkuhn@ebb.org>
   9  * Kuhn's copyrights are licensed GPLv2-or-later.  File as a whole remains GPLv2.
  10  */
  11
  12 //config:config WGET
  13 //config:       bool "wget"
  14 //config:       default y
  15 //config:       help
  16 //config:         wget is a utility for non-interactive download of files from HTTP
  17 //config:         and FTP servers.
  18 //config:
  19 //config:config FEATURE_WGET_STATUSBAR
  20 //config:       bool "Enable a nifty process meter (+2k)"
  21 //config:       default y
  22 //config:       depends on WGET
  23 //config:       help
  24 //config:         Enable the transfer progress bar for wget transfers.
  25 //config:
  26 //config:config FEATURE_WGET_AUTHENTICATION
  27 //config:       bool "Enable HTTP authentication"
  28 //config:       default y
  29 //config:       depends on WGET
  30 //config:       help
  31 //config:         Support authenticated HTTP transfers.
  32 //config:
  33 //config:config FEATURE_WGET_LONG_OPTIONS
  34 //config:       bool "Enable long options"
  35 //config:       default y
  36 //config:       depends on WGET && LONG_OPTS
  37 //config:       help
  38 //config:         Support long options for the wget applet.
  39 //config:
  40 //config:config FEATURE_WGET_TIMEOUT
  41 //config:       bool "Enable timeout option -T SEC"
  42 //config:       default y
  43 //config:       depends on WGET
  44 //config:       help
  45 //config:         Supports network read and connect timeouts for wget,
  46 //config:         so that wget will give up and timeout, through the -T
  47 //config:         command line option.
  48 //config:
  49 //config:         Currently only connect and network data read timeout are
  50 //config:         supported (i.e., timeout is not applied to the DNS query). When
  51 //config:         FEATURE_WGET_LONG_OPTIONS is also enabled, the --timeout option
  52 //config:         will work in addition to -T.
  53 //config:
  54 //config:config FEATURE_WGET_OPENSSL
  55 //config:       bool "Try to connect to HTTPS using openssl"
  56 //config:       default y
  57 //config:       depends on WGET
  58 //config:       help
  59 //config:         Choose how wget establishes SSL connection for https:// URLs.
  60 //config:
  61 //config:         Busybox itself contains no SSL code. wget will spawn
  62 //config:         a helper program to talk over HTTPS.
  63 //config:
  64 //config:         OpenSSL has a simple SSL client for debug purposes.
  65 //config:         If you select "openssl" helper, wget will effectively call
  66 //config:         "openssl s_client -quiet -connect IP:443 2>/dev/null"
  67 //config:         and pipe its data through it.
  68 //config:         Note inconvenient API: host resolution is done twice,
  69 //config:         and there is no guarantee openssl's idea of IPv6 address
  70 //config:         format is the same as ours.
  71 //config:         Another problem is that s_client prints debug information
  72 //config:         to stderr, and it needs to be suppressed. This means
  73 //config:         all error messages get suppressed too.
  74 //config:         openssl is also a big binary, often dynamically linked
  75 //config:         against ~15 libraries.
  76 //config:
  77 //config:config FEATURE_WGET_SSL_HELPER
  78 //config:       bool "Try to connect to HTTPS using ssl_helper"
  79 //config:       default y
  80 //config:       depends on WGET
  81 //config:       help
  82 //config:         Choose how wget establishes SSL connection for https:// URLs.
  83 //config:
  84 //config:         Busybox itself contains no SSL code. wget will spawn
  85 //config:         a helper program to talk over HTTPS.
  86 //config:
  87 //config:         ssl_helper is a tool which can be built statically
  88 //config:         from busybox sources against a small embedded SSL library.
  89 //config:         Please see networking/ssl_helper/README.
  90 //config:         It does not require double host resolution and emits
  91 //config:         error messages to stderr.
  92 //config:
  93 //config:         Precompiled static binary may be available at
  94 //config:         http://busybox.net/downloads/binaries/
  95
  96 //applet:IF_WGET(APPLET(wget, BB_DIR_USR_BIN, BB_SUID_DROP))
  97
  98 //kbuild:lib-$(CONFIG_WGET) += wget.o
  99
 100 //usage:#define wget_trivial_usage
 101 //usage:        IF_FEATURE_WGET_LONG_OPTIONS(
 102 //usage:       "[-c|--continue] [-s|--spider] [-q|--quiet] [-O|--output-document FILE]\n"
 103 //usage:       "        [--header 'header: value'] [-Y|--proxy on/off] [-P DIR]\n"
 104 /* Since we ignore these opts, we don't show them in --help */
 105 /* //usage:    "        [--no-check-certificate] [--no-cache] [--passive-ftp] [-t TRIES]" */
 106 /* //usage:    "        [-nv] [-nc] [-nH] [-np]" */
 107 //usage:       "        [-U|--user-agent AGENT]" IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
 108 //usage:        )
 109 //usage:        IF_NOT_FEATURE_WGET_LONG_OPTIONS(
 110 //usage:       "[-csq] [-O FILE] [-Y on/off] [-P DIR] [-U AGENT]"
 111 //usage:                        IF_FEATURE_WGET_TIMEOUT(" [-T SEC]") " URL..."
 112 //usage:        )
 113 //usage:#define wget_full_usage "\n\n"
 114 //usage:       "Retrieve files via HTTP or FTP\n"
 115 //usage:     "\n        -s      Spider mode - only check file existence"
 116 //usage:     "\n        -c      Continue retrieval of aborted transfer"
 117 //usage:     "\n        -q      Quiet"
 118 //usage:     "\n        -P DIR  Save to DIR (default .)"
 119 //usage:        IF_FEATURE_WGET_TIMEOUT(
 120 //usage:     "\n        -T SEC  Network read timeout is SEC seconds"
 121 //usage:        )
 122 //usage:     "\n        -O FILE Save to FILE ('-' for stdout)"
 123 //usage:     "\n        -U STR  Use STR for User-Agent header"
 124 //usage:     "\n        -Y      Use proxy ('on' or 'off')"
 125
 126 #include "libbb.h"
 127
 128 #if 0
 129 # define log_io(...) bb_error_msg(__VA_ARGS__)
 130 # define SENDFMT(fp, fmt, ...) \
 131         do { \
 132                 log_io("> " fmt, ##__VA_ARGS__); \
 133                 fprintf(fp, fmt, ##__VA_ARGS__); \
 134         } while (0);
 135 #else
 136 # define log_io(...) ((void)0)
 137 # define SENDFMT(fp, fmt, ...) fprintf(fp, fmt, ##__VA_ARGS__)
 138 #endif
 139
 140
 141 struct host_info {
 142         char *allocated;
 143         const char *path;
 144         char       *user;
 145         const char *protocol;
 146         char       *host;
 147         int         port;
 148 };
 149 static const char P_FTP[] = "ftp";
 150 static const char P_HTTP[] = "http";
 151 #if ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_SSL_HELPER
 152 static const char P_HTTPS[] = "https";
 153 #endif
 154
 155 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
 156 /* User-specified headers prevent using our corresponding built-in headers.  */
 157 enum {
 158         HDR_HOST          = (1<<0),
 159         HDR_USER_AGENT    = (1<<1),
 160         HDR_RANGE         = (1<<2),
 161         HDR_AUTH          = (1<<3) * ENABLE_FEATURE_WGET_AUTHENTICATION,
 162         HDR_PROXY_AUTH    = (1<<4) * ENABLE_FEATURE_WGET_AUTHENTICATION,
 163 };
 164 static const char wget_user_headers[] ALIGN1 =
 165         "Host:\0"
 166         "User-Agent:\0"
 167         "Range:\0"
 168 # if ENABLE_FEATURE_WGET_AUTHENTICATION
 169         "Authorization:\0"
 170         "Proxy-Authorization:\0"
 171 # endif
 172         ;
 173 # define USR_HEADER_HOST       (G.user_headers & HDR_HOST)
 174 # define USR_HEADER_USER_AGENT (G.user_headers & HDR_USER_AGENT)
 175 # define USR_HEADER_RANGE      (G.user_headers & HDR_RANGE)
 176 # define USR_HEADER_AUTH       (G.user_headers & HDR_AUTH)
 177 # define USR_HEADER_PROXY_AUTH (G.user_headers & HDR_PROXY_AUTH)
 178 #else /* No long options, no user-headers :( */
 179 # define USR_HEADER_HOST       0
 180 # define USR_HEADER_USER_AGENT 0
 181 # define USR_HEADER_RANGE      0
 182 # define USR_HEADER_AUTH       0
 183 # define USR_HEADER_PROXY_AUTH 0
 184 #endif
 185
 186 /* Globals */
 187 struct globals {
 188         off_t content_len;        /* Content-length of the file */
 189         off_t beg_range;          /* Range at which continue begins */
 190 #if ENABLE_FEATURE_WGET_STATUSBAR
 191         off_t transferred;        /* Number of bytes transferred so far */
 192         const char *curfile;      /* Name of current file being transferred */
 193         bb_progress_t pmt;
 194 #endif
 195         char *dir_prefix;
 196 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
 197         char *post_data;
 198         char *extra_headers;
 199         unsigned char user_headers; /* Headers mentioned by the user */
 200 #endif
 201         char *fname_out;        /* where to direct output (-O) */
 202         const char *proxy_flag; /* Use proxies if env vars are set */
 203         const char *user_agent; /* "User-Agent" header field */
 204 #if ENABLE_FEATURE_WGET_TIMEOUT
 205         unsigned timeout_seconds;
 206         bool die_if_timed_out;
 207 #endif
 208         int output_fd;
 209         int o_flags;
 210         smallint chunked;         /* chunked transfer encoding */
 211         smallint got_clen;        /* got content-length: from server  */
 212         /* Local downloads do benefit from big buffer.
 213          * With 512 byte buffer, it was measured to be
 214          * an order of magnitude slower than with big one.
 215          */
 216         uint64_t just_to_align_next_member;
 217         char wget_buf[CONFIG_FEATURE_COPYBUF_KB*1024];
 218 } FIX_ALIASING;
 219 #define G (*ptr_to_globals)
 220 #define INIT_G() do { \
 221         SET_PTR_TO_GLOBALS(xzalloc(sizeof(G))); \
 222 } while (0)
 223 #define FINI_G() do { \
 224         FREE_PTR_TO_GLOBALS(); \
 225 } while (0)
 226
 227
 228 /* Must match option string! */
 229 enum {
 230         WGET_OPT_CONTINUE   = (1 << 0),
 231         WGET_OPT_SPIDER     = (1 << 1),
 232         WGET_OPT_QUIET      = (1 << 2),
 233         WGET_OPT_OUTNAME    = (1 << 3),
 234         WGET_OPT_PREFIX     = (1 << 4),
 235         WGET_OPT_PROXY      = (1 << 5),
 236         WGET_OPT_USER_AGENT = (1 << 6),
 237         WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 7),
 238         WGET_OPT_RETRIES    = (1 << 8),
 239         WGET_OPT_PASSIVE    = (1 << 9),
 240         WGET_OPT_HEADER     = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
 241         WGET_OPT_POST_DATA  = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
 242 };
 243
 244 enum {
 245         PROGRESS_START = -1,
 246         PROGRESS_END   = 0,
 247         PROGRESS_BUMP  = 1,
 248 };
 249 #if ENABLE_FEATURE_WGET_STATUSBAR
 250 static void progress_meter(int flag)
 251 {
 252         if (option_mask32 & WGET_OPT_QUIET)
 253                 return;
 254
 255         if (flag == PROGRESS_START)
 256                 bb_progress_init(&G.pmt, G.curfile);
 257
 258         bb_progress_update(&G.pmt,
 259                         G.beg_range,
 260                         G.transferred,
 261                         (G.chunked || !G.got_clen) ? 0 : G.beg_range + G.transferred + G.content_len
 262         );
 263
 264         if (flag == PROGRESS_END) {
 265                 bb_progress_free(&G.pmt);
 266                 bb_putchar_stderr('\n');
 267                 G.transferred = 0;
 268         }
 269 }
 270 #else
 271 static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) { }
 272 #endif
 273
 274
 275 /* IPv6 knows scoped address types i.e. link and site local addresses. Link
 276  * local addresses can have a scope identifier to specify the
 277  * interface/link an address is valid on (e.g. fe80::1%eth0). This scope
 278  * identifier is only valid on a single node.
 279  *
 280  * RFC 4007 says that the scope identifier MUST NOT be sent across the wire,
 281  * unless all nodes agree on the semantic. Apache e.g. regards zone identifiers
 282  * in the Host header as invalid requests, see
 283  * https://issues.apache.org/bugzilla/show_bug.cgi?id=35122
 284  */
 285 static void strip_ipv6_scope_id(char *host)
 286 {
 287         char *scope, *cp;
 288
 289         /* bbox wget actually handles IPv6 addresses without [], like
 290          * wget "http://::1/xxx", but this is not standard.
 291          * To save code, _here_ we do not support it. */
 292
 293         if (host[0] != '[')
 294                 return; /* not IPv6 */
 295
 296         scope = strchr(host, '%');
 297         if (!scope)
 298                 return;
 299
 300         /* Remove the IPv6 zone identifier from the host address */
 301         cp = strchr(host, ']');
 302         if (!cp || (cp[1] != ':' && cp[1] != '\0')) {
 303                 /* malformed address (not "[xx]:nn" or "[xx]") */
 304                 return;
 305         }
 306
 307         /* cp points to "]...", scope points to "%eth0]..." */
 308         overlapping_strcpy(scope, cp);
 309 }
 310
 311 #if ENABLE_FEATURE_WGET_AUTHENTICATION
 312 /* Base64-encode character string. */
 313 static char *base64enc(const char *str)
 314 {
 315         unsigned len = strlen(str);
 316         if (len > sizeof(G.wget_buf)/4*3 - 10) /* paranoia */
 317                 len = sizeof(G.wget_buf)/4*3 - 10;
 318         bb_uuencode(G.wget_buf, str, len, bb_uuenc_tbl_base64);
 319         return G.wget_buf;
 320 }
 321 #endif
 322
 323 static char* sanitize_string(char *s)
 324 {
 325         unsigned char *p = (void *) s;
 326         while (*p >= ' ')
 327                 p++;
 328         *p = '\0';
 329         return s;
 330 }
 331
 332 #if ENABLE_FEATURE_WGET_TIMEOUT
 333 static void alarm_handler(int sig UNUSED_PARAM)
 334 {
 335         /* This is theoretically unsafe (uses stdio and malloc in signal handler) */
 336         if (G.die_if_timed_out)
 337                 bb_error_msg_and_die("download timed out");
 338 }
 339 static void set_alarm(void)
 340 {
 341         if (G.timeout_seconds) {
 342                 alarm(G.timeout_seconds);
 343                 G.die_if_timed_out = 1;
 344         }
 345 }
 346 # define clear_alarm() ((void)(G.die_if_timed_out = 0))
 347 #else
 348 # define set_alarm()   ((void)0)
 349 # define clear_alarm() ((void)0)
 350 #endif
 351
 352 static FILE *open_socket(len_and_sockaddr *lsa)
 353 {
 354         int fd;
 355         FILE *fp;
 356
 357         set_alarm();
 358         fd = xconnect_stream(lsa);
 359         clear_alarm();
 360
 361         /* glibc 2.4 seems to try seeking on it - ??! */
 362         /* hopefully it understands what ESPIPE means... */
 363         fp = fdopen(fd, "r+");
 364         if (!fp)
 365                 bb_perror_msg_and_die(bb_msg_memory_exhausted);
 366
 367         return fp;
 368 }
 369
 370 /* Returns '\n' if it was seen, else '\0'. Trims at first '\r' or '\n' */
 371 static char fgets_and_trim(FILE *fp)
 372 {
 373         char c;
 374         char *buf_ptr;
 375
 376         set_alarm();
 377         if (fgets(G.wget_buf, sizeof(G.wget_buf) - 1, fp) == NULL)
 378                 bb_perror_msg_and_die("error getting response");
 379         clear_alarm();
 380
 381         buf_ptr = strchrnul(G.wget_buf, '\n');
 382         c = *buf_ptr;
 383         *buf_ptr = '\0';
 384         buf_ptr = strchrnul(G.wget_buf, '\r');
 385         *buf_ptr = '\0';
 386
 387         log_io("< %s", G.wget_buf);
 388
 389         return c;
 390 }
 391
 392 static int ftpcmd(const char *s1, const char *s2, FILE *fp)
 393 {
 394         int result;
 395         if (s1) {
 396                 if (!s2)
 397                         s2 = "";
 398                 fprintf(fp, "%s%s\r\n", s1, s2);
 399                 fflush(fp);
 400                 log_io("> %s%s", s1, s2);
 401         }
 402
 403         do {
 404                 fgets_and_trim(fp);
 405         } while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' ');
 406
 407         G.wget_buf[3] = '\0';
 408         result = xatoi_positive(G.wget_buf);
 409         G.wget_buf[3] = ' ';
 410         return result;
 411 }
 412
 413 static void parse_url(const char *src_url, struct host_info *h)
 414 {
 415         char *url, *p, *sp;
 416
 417         free(h->allocated);
 418         h->allocated = url = xstrdup(src_url);
 419
 420         h->protocol = P_FTP;
 421         p = strstr(url, "://");
 422         if (p) {
 423                 *p = '\0';
 424                 h->host = p + 3;
 425                 if (strcmp(url, P_FTP) == 0) {
 426                         h->port = bb_lookup_port(P_FTP, "tcp", 21);
 427                 } else
 428 #if ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_SSL_HELPER
 429                 if (strcmp(url, P_HTTPS) == 0) {
 430                         h->port = bb_lookup_port(P_HTTPS, "tcp", 443);
 431                         h->protocol = P_HTTPS;
 432                 } else
 433 #endif
 434                 if (strcmp(url, P_HTTP) == 0) {
 435  http:
 436                         h->port = bb_lookup_port(P_HTTP, "tcp", 80);
 437                         h->protocol = P_HTTP;
 438                 } else {
 439                         *p = ':';
 440                         bb_error_msg_and_die("not an http or ftp url: %s", sanitize_string(url));
 441                 }
 442         } else {
 443                 // GNU wget is user-friendly and falls back to http://
 444                 h->host = url;
 445                 goto http;
 446         }
 447
 448         // FYI:
 449         // "Real" wget 'http://busybox.net?var=a/b' sends this request:
 450         //   'GET /?var=a/b HTTP 1.0'
 451         //   and saves 'index.html?var=a%2Fb' (we save 'b')
 452         // wget 'http://busybox.net?login=john@doe':
 453         //   request: 'GET /?login=john@doe HTTP/1.0'
 454         //   saves: 'index.html?login=john@doe' (we save '?login=john@doe')
 455         // wget 'http://busybox.net#test/test':
 456         //   request: 'GET / HTTP/1.0'
 457         //   saves: 'index.html' (we save 'test')
 458         //
 459         // We also don't add unique .N suffix if file exists...
 460         sp = strchr(h->host, '/');
 461         p = strchr(h->host, '?'); if (!sp || (p && sp > p)) sp = p;
 462         p = strchr(h->host, '#'); if (!sp || (p && sp > p)) sp = p;
 463         if (!sp) {
 464                 h->path = "";
 465         } else if (*sp == '/') {
 466                 *sp = '\0';
 467                 h->path = sp + 1;
 468         } else { // '#' or '?'
 469                 // http://busybox.net?login=john@doe is a valid URL
 470                 // memmove converts to:
 471                 // http:/busybox.nett?login=john@doe...
 472                 memmove(h->host - 1, h->host, sp - h->host);
 473                 h->host--;
 474                 sp[-1] = '\0';
 475                 h->path = sp;
 476         }
 477
 478         sp = strrchr(h->host, '@');
 479         if (sp != NULL) {
 480                 // URL-decode "user:password" string before base64-encoding:
 481                 // wget http://test:my%20pass@example.com should send
 482                 // Authorization: Basic dGVzdDpteSBwYXNz
 483                 // which decodes to "test:my pass".
 484                 // Standard wget and curl do this too.
 485                 *sp = '\0';
 486                 free(h->user);
 487                 h->user = xstrdup(percent_decode_in_place(h->host, /*strict:*/ 0));
 488                 h->host = sp + 1;
 489         }
 490         /* else: h->user remains NULL, or as set by original request
 491          * before redirect (if we are here after a redirect).
 492          */
 493 }
 494
 495 static char *gethdr(FILE *fp)
 496 {
 497         char *s, *hdrval;
 498         int c;
 499
 500         /* retrieve header line */
 501         c = fgets_and_trim(fp);
 502
 503         /* end of the headers? */
 504         if (G.wget_buf[0] == '\0')
 505                 return NULL;
 506
 507         /* convert the header name to lower case */
 508         for (s = G.wget_buf; isalnum(*s) || *s == '-' || *s == '.' || *s == '_'; ++s) {
 509                 /*
 510                  * No-op for 20-3f and 60-7f. "0-9a-z-." are in these ranges.
 511                  * 40-5f range ("@A-Z[\]^_") maps to 60-7f.
 512                  * "A-Z" maps to "a-z".
 513                  * "@[\]" can't occur in header names.
 514                  * "^_" maps to "~,DEL" (which is wrong).
 515                  * "^" was never seen yet, "_" was seen from web.archive.org
 516                  * (x-archive-orig-x_commoncrawl_Signature: HEXSTRING).
 517                  */
 518                 *s |= 0x20;
 519         }
 520
 521         /* verify we are at the end of the header name */
 522         if (*s != ':')
 523                 bb_error_msg_and_die("bad header line: %s", sanitize_string(G.wget_buf));
 524
 525         /* locate the start of the header value */
 526         *s++ = '\0';
 527         hdrval = skip_whitespace(s);
 528
 529         if (c != '\n') {
 530                 /* Rats! The buffer isn't big enough to hold the entire header value */
 531                 while (c = getc(fp), c != EOF && c != '\n')
 532                         continue;
 533         }
 534
 535         return hdrval;
 536 }
 537
 538 static void reset_beg_range_to_zero(void)
 539 {
 540         bb_error_msg("restart failed");
 541         G.beg_range = 0;
 542         xlseek(G.output_fd, 0, SEEK_SET);
 543         /* Done at the end instead: */
 544         /* ftruncate(G.output_fd, 0); */
 545 }
 546
 547 static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa)
 548 {
 549         FILE *sfp;
 550         char *str;
 551         int port;
 552
 553         if (!target->user)
 554                 target->user = xstrdup("anonymous:busybox@");
 555
 556         sfp = open_socket(lsa);
 557         if (ftpcmd(NULL, NULL, sfp) != 220)
 558                 bb_error_msg_and_die("%s", sanitize_string(G.wget_buf + 4));
 559
 560         /*
 561          * Splitting username:password pair,
 562          * trying to log in
 563          */
 564         str = strchr(target->user, ':');
 565         if (str)
 566                 *str++ = '\0';
 567         switch (ftpcmd("USER ", target->user, sfp)) {
 568         case 230:
 569                 break;
 570         case 331:
 571                 if (ftpcmd("PASS ", str, sfp) == 230)
 572                         break;
 573                 /* fall through (failed login) */
 574         default:
 575                 bb_error_msg_and_die("ftp login: %s", sanitize_string(G.wget_buf + 4));
 576         }
 577
 578         ftpcmd("TYPE I", NULL, sfp);
 579
 580         /*
 581          * Querying file size
 582          */
 583         if (ftpcmd("SIZE ", target->path, sfp) == 213) {
 584                 G.content_len = BB_STRTOOFF(G.wget_buf + 4, NULL, 10);
 585                 if (G.content_len < 0 || errno) {
 586                         bb_error_msg_and_die("SIZE value is garbage");
 587                 }
 588                 G.got_clen = 1;
 589         }
 590
 591         /*
 592          * Entering passive mode
 593          */
 594         if (ftpcmd("PASV", NULL, sfp) != 227) {
 595  pasv_error:
 596                 bb_error_msg_and_die("bad response to %s: %s", "PASV", sanitize_string(G.wget_buf));
 597         }
 598         // Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage]
 599         // Server's IP is N1.N2.N3.N4 (we ignore it)
 600         // Server's port for data connection is P1*256+P2
 601         str = strrchr(G.wget_buf, ')');
 602         if (str) str[0] = '\0';
 603         str = strrchr(G.wget_buf, ',');
 604         if (!str) goto pasv_error;
 605         port = xatou_range(str+1, 0, 255);
 606         *str = '\0';
 607         str = strrchr(G.wget_buf, ',');
 608         if (!str) goto pasv_error;
 609         port += xatou_range(str+1, 0, 255) * 256;
 610         set_nport(&lsa->u.sa, htons(port));
 611
 612         *dfpp = open_socket(lsa);
 613
 614         if (G.beg_range != 0) {
 615                 sprintf(G.wget_buf, "REST %"OFF_FMT"u", G.beg_range);
 616                 if (ftpcmd(G.wget_buf, NULL, sfp) == 350)
 617                         G.content_len -= G.beg_range;
 618                 else
 619                         reset_beg_range_to_zero();
 620         }
 621
 622         if (ftpcmd("RETR ", target->path, sfp) > 150)
 623                 bb_error_msg_and_die("bad response to %s: %s", "RETR", sanitize_string(G.wget_buf));
 624
 625         return sfp;
 626 }
 627
 628 #if ENABLE_FEATURE_WGET_OPENSSL
 629 static int spawn_https_helper_openssl(const char *host, unsigned port)
 630 {
 631         char *allocated = NULL;
 632         int sp[2];
 633         int pid;
 634         IF_FEATURE_WGET_SSL_HELPER(volatile int child_failed = 0;)
 635
 636         if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
 637                 /* Kernel can have AF_UNIX support disabled */
 638                 bb_perror_msg_and_die("socketpair");
 639
 640         if (!strchr(host, ':'))
 641                 host = allocated = xasprintf("%s:%u", host, port);
 642
 643         fflush_all();
 644         pid = xvfork();
 645         if (pid == 0) {
 646                 /* Child */
 647                 char *argv[6];
 648
 649                 close(sp[0]);
 650                 xmove_fd(sp[1], 0);
 651                 xdup2(0, 1);
 652                 /*
 653                  * openssl s_client -quiet -connect www.kernel.org:443 2>/dev/null
 654                  * It prints some debug stuff on stderr, don't know how to suppress it.
 655                  * Work around by dev-nulling stderr. We lose all error messages :(
 656                  */
 657                 xmove_fd(2, 3);
 658                 xopen("/dev/null", O_RDWR);
 659                 argv[0] = (char*)"openssl";
 660                 argv[1] = (char*)"s_client";
 661                 argv[2] = (char*)"-quiet";
 662                 argv[3] = (char*)"-connect";
 663                 argv[4] = (char*)host;
 664                 argv[5] = NULL;
 665                 BB_EXECVP(argv[0], argv);
 666                 xmove_fd(3, 2);
 667 # if ENABLE_FEATURE_WGET_SSL_HELPER
 668                 child_failed = 1;
 669                 xfunc_die();
 670 # else
 671                 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
 672 # endif
 673                 /* notreached */
 674         }
 675
 676         /* Parent */
 677         free(allocated);
 678         close(sp[1]);
 679 # if ENABLE_FEATURE_WGET_SSL_HELPER
 680         if (child_failed) {
 681                 close(sp[0]);
 682                 return -1;
 683         }
 684 # endif
 685         return sp[0];
 686 }
 687 #endif
 688
 689 /* See networking/ssl_helper/README how to build one */
 690 #if ENABLE_FEATURE_WGET_SSL_HELPER
 691 static void spawn_https_helper_small(int network_fd)
 692 {
 693         int sp[2];
 694         int pid;
 695
 696         if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
 697                 /* Kernel can have AF_UNIX support disabled */
 698                 bb_perror_msg_and_die("socketpair");
 699
 700         pid = BB_MMU ? xfork() : xvfork();
 701         if (pid == 0) {
 702                 /* Child */
 703                 char *argv[3];
 704
 705                 close(sp[0]);
 706                 xmove_fd(sp[1], 0);
 707                 xdup2(0, 1);
 708                 xmove_fd(network_fd, 3);
 709                 /*
 710                  * A simple ssl/tls helper
 711                  */
 712                 argv[0] = (char*)"ssl_helper";
 713                 argv[1] = (char*)"-d3";
 714                 argv[2] = NULL;
 715                 BB_EXECVP(argv[0], argv);
 716                 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
 717                 /* notreached */
 718         }
 719
 720         /* Parent */
 721         close(sp[1]);
 722         xmove_fd(sp[0], network_fd);
 723 }
 724 #endif
 725
 726 static void NOINLINE retrieve_file_data(FILE *dfp)
 727 {
 728 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
 729 # if ENABLE_FEATURE_WGET_TIMEOUT
 730         unsigned second_cnt = G.timeout_seconds;
 731 # endif
 732         struct pollfd polldata;
 733
 734         polldata.fd = fileno(dfp);
 735         polldata.events = POLLIN | POLLPRI;
 736 #endif
 737         progress_meter(PROGRESS_START);
 738
 739         if (G.chunked)
 740                 goto get_clen;
 741
 742         /* Loops only if chunked */
 743         while (1) {
 744
 745 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
 746                 /* Must use nonblocking I/O, otherwise fread will loop
 747                  * and *block* until it reads full buffer,
 748                  * which messes up progress bar and/or timeout logic.
 749                  * Because of nonblocking I/O, we need to dance
 750                  * very carefully around EAGAIN. See explanation at
 751                  * clearerr() calls.
 752                  */
 753                 ndelay_on(polldata.fd);
 754 #endif
 755                 while (1) {
 756                         int n;
 757                         unsigned rdsz;
 758
 759 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
 760                         /* fread internally uses read loop, which in our case
 761                          * is usually exited when we get EAGAIN.
 762                          * In this case, libc sets error marker on the stream.
 763                          * Need to clear it before next fread to avoid possible
 764                          * rare false positive ferror below. Rare because usually
 765                          * fread gets more than zero bytes, and we don't fall
 766                          * into if (n <= 0) ...
 767                          */
 768                         clearerr(dfp);
 769 #endif
 770                         errno = 0;
 771                         rdsz = sizeof(G.wget_buf);
 772                         if (G.got_clen) {
 773                                 if (G.content_len < (off_t)sizeof(G.wget_buf)) {
 774                                         if ((int)G.content_len <= 0)
 775                                                 break;
 776                                         rdsz = (unsigned)G.content_len;
 777                                 }
 778                         }
 779                         n = fread(G.wget_buf, 1, rdsz, dfp);
 780
 781                         if (n > 0) {
 782                                 xwrite(G.output_fd, G.wget_buf, n);
 783 #if ENABLE_FEATURE_WGET_STATUSBAR
 784                                 G.transferred += n;
 785 #endif
 786                                 if (G.got_clen) {
 787                                         G.content_len -= n;
 788                                         if (G.content_len == 0)
 789                                                 break;
 790                                 }
 791 #if ENABLE_FEATURE_WGET_TIMEOUT
 792                                 second_cnt = G.timeout_seconds;
 793 #endif
 794                                 goto bump;
 795                         }
 796
 797                         /* n <= 0.
 798                          * man fread:
 799                          * If error occurs, or EOF is reached, the return value
 800                          * is a short item count (or zero).
 801                          * fread does not distinguish between EOF and error.
 802                          */
 803                         if (errno != EAGAIN) {
 804                                 if (ferror(dfp)) {
 805                                         progress_meter(PROGRESS_END);
 806                                         bb_perror_msg_and_die(bb_msg_read_error);
 807                                 }
 808                                 break; /* EOF, not error */
 809                         }
 810
 811 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
 812                         /* It was EAGAIN. There is no data. Wait up to one second
 813                          * then abort if timed out, or update the bar and try reading again.
 814                          */
 815                         if (safe_poll(&polldata, 1, 1000) == 0) {
 816 # if ENABLE_FEATURE_WGET_TIMEOUT
 817                                 if (second_cnt != 0 && --second_cnt == 0) {
 818                                         progress_meter(PROGRESS_END);
 819                                         bb_error_msg_and_die("download timed out");
 820                                 }
 821 # endif
 822                                 /* We used to loop back to poll here,
 823                                  * but there is no great harm in letting fread
 824                                  * to try reading anyway.
 825                                  */
 826                         }
 827 #endif
 828  bump:
 829                         /* Need to do it _every_ second for "stalled" indicator
 830                          * to be shown properly.
 831                          */
 832                         progress_meter(PROGRESS_BUMP);
 833                 } /* while (reading data) */
 834
 835 #if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
 836                 clearerr(dfp);
 837                 ndelay_off(polldata.fd); /* else fgets can get very unhappy */
 838 #endif
 839                 if (!G.chunked)
 840                         break;
 841
 842                 fgets_and_trim(dfp); /* Eat empty line */
 843  get_clen:
 844                 fgets_and_trim(dfp);
 845                 G.content_len = STRTOOFF(G.wget_buf, NULL, 16);
 846                 /* FIXME: error check? */
 847                 if (G.content_len == 0)
 848                         break; /* all done! */
 849                 G.got_clen = 1;
 850                 /*
 851                  * Note that fgets may result in some data being buffered in dfp.
 852                  * We loop back to fread, which will retrieve this data.
 853                  * Also note that code has to be arranged so that fread
 854                  * is done _before_ one-second poll wait - poll doesn't know
 855                  * about stdio buffering and can result in spurious one second waits!
 856                  */
 857         }
 858
 859         /* If -c failed, we restart from the beginning,
 860          * but we do not truncate file then, we do it only now, at the end.
 861          * This lets user to ^C if his 99% complete 10 GB file download
 862          * failed to restart *without* losing the almost complete file.
 863          */
 864         {
 865                 off_t pos = lseek(G.output_fd, 0, SEEK_CUR);
 866                 if (pos != (off_t)-1)
 867                         ftruncate(G.output_fd, pos);
 868         }
 869
 870         /* Draw full bar and free its resources */
 871         G.chunked = 0;  /* makes it show 100% even for chunked download */
 872         G.got_clen = 1; /* makes it show 100% even for download of (formerly) unknown size */
 873         progress_meter(PROGRESS_END);
 874 }
 875
 876 static void download_one_url(const char *url)
 877 {
 878         bool use_proxy;                 /* Use proxies if env vars are set  */
 879         int redir_limit;
 880         len_and_sockaddr *lsa;
 881         FILE *sfp;                      /* socket to web/ftp server         */
 882         FILE *dfp;                      /* socket to ftp server (data)      */
 883         char *proxy = NULL;
 884         char *fname_out_alloc;
 885         char *redirected_path = NULL;
 886         struct host_info server;
 887         struct host_info target;
 888
 889         server.allocated = NULL;
 890         target.allocated = NULL;
 891         server.user = NULL;
 892         target.user = NULL;
 893
 894         parse_url(url, &target);
 895
 896         /* Use the proxy if necessary */
 897         use_proxy = (strcmp(G.proxy_flag, "off") != 0);
 898         if (use_proxy) {
 899                 proxy = getenv(target.protocol == P_FTP ? "ftp_proxy" : "http_proxy");
 900 //FIXME: what if protocol is https? Ok to use http_proxy?
 901                 use_proxy = (proxy && proxy[0]);
 902                 if (use_proxy)
 903                         parse_url(proxy, &server);
 904         }
 905         if (!use_proxy) {
 906                 server.port = target.port;
 907                 if (ENABLE_FEATURE_IPV6) {
 908                         //free(server.allocated); - can't be non-NULL
 909                         server.host = server.allocated = xstrdup(target.host);
 910                 } else {
 911                         server.host = target.host;
 912                 }
 913         }
 914
 915         if (ENABLE_FEATURE_IPV6)
 916                 strip_ipv6_scope_id(target.host);
 917
 918         /* If there was no -O FILE, guess output filename */
 919         fname_out_alloc = NULL;
 920         if (!(option_mask32 & WGET_OPT_OUTNAME)) {
 921                 G.fname_out = bb_get_last_path_component_nostrip(target.path);
 922                 /* handle "wget http://kernel.org//" */
 923                 if (G.fname_out[0] == '/' || !G.fname_out[0])
 924                         G.fname_out = (char*)"index.html";
 925                 /* -P DIR is considered only if there was no -O FILE */
 926                 if (G.dir_prefix)
 927                         G.fname_out = fname_out_alloc = concat_path_file(G.dir_prefix, G.fname_out);
 928                 else {
 929                         /* redirects may free target.path later, need to make a copy */
 930                         G.fname_out = fname_out_alloc = xstrdup(G.fname_out);
 931                 }
 932         }
 933 #if ENABLE_FEATURE_WGET_STATUSBAR
 934         G.curfile = bb_get_last_path_component_nostrip(G.fname_out);
 935 #endif
 936
 937         /* Determine where to start transfer */
 938         G.beg_range = 0;
 939         if (option_mask32 & WGET_OPT_CONTINUE) {
 940                 G.output_fd = open(G.fname_out, O_WRONLY);
 941                 if (G.output_fd >= 0) {
 942                         G.beg_range = xlseek(G.output_fd, 0, SEEK_END);
 943                 }
 944                 /* File doesn't exist. We do not create file here yet.
 945                  * We are not sure it exists on remote side */
 946         }
 947
 948         redir_limit = 5;
 949  resolve_lsa:
 950         lsa = xhost2sockaddr(server.host, server.port);
 951         if (!(option_mask32 & WGET_OPT_QUIET)) {
 952                 char *s = xmalloc_sockaddr2dotted(&lsa->u.sa);
 953                 fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);
 954                 free(s);
 955         }
 956  establish_session:
 957         /*G.content_len = 0; - redundant, got_clen = 0 is enough */
 958         G.got_clen = 0;
 959         G.chunked = 0;
 960         if (use_proxy || target.protocol != P_FTP) {
 961                 /*
 962                  *  HTTP session
 963                  */
 964                 char *str;
 965                 int status;
 966
 967                 /* Open socket to http(s) server */
 968 #if ENABLE_FEATURE_WGET_OPENSSL
 969                 /* openssl (and maybe ssl_helper) support is configured */
 970                 if (target.protocol == P_HTTPS) {
 971                         /* openssl-based helper
 972                          * Inconvenient API since we can't give it an open fd
 973                          */
 974                         int fd = spawn_https_helper_openssl(server.host, server.port);
 975 # if ENABLE_FEATURE_WGET_SSL_HELPER
 976                         if (fd < 0) { /* no openssl? try ssl_helper */
 977                                 sfp = open_socket(lsa);
 978                                 spawn_https_helper_small(fileno(sfp));
 979                                 goto socket_opened;
 980                         }
 981 # else
 982                         /* We don't check for exec("openssl") failure in this case */
 983 # endif
 984                         sfp = fdopen(fd, "r+");
 985                         if (!sfp)
 986                                 bb_perror_msg_and_die(bb_msg_memory_exhausted);
 987                         goto socket_opened;
 988                 }
 989                 sfp = open_socket(lsa);
 990  socket_opened:
 991 #elif ENABLE_FEATURE_WGET_SSL_HELPER
 992                 /* Only ssl_helper support is configured */
 993                 sfp = open_socket(lsa);
 994                 if (target.protocol == P_HTTPS)
 995                         spawn_https_helper_small(fileno(sfp));
 996 #else
 997                 /* ssl (https) support is not configured */
 998                 sfp = open_socket(lsa);
 999 #endif
1000                 /* Send HTTP request */
1001                 if (use_proxy) {
1002                         SENDFMT(sfp, "GET %s://%s/%s HTTP/1.1\r\n",
1003                                 target.protocol, target.host,
1004                                 target.path);
1005                 } else {
1006                         SENDFMT(sfp, "%s /%s HTTP/1.1\r\n",
1007                                 (option_mask32 & WGET_OPT_POST_DATA) ? "POST" : "GET",
1008                                 target.path);
1009                 }
1010                 if (!USR_HEADER_HOST)
1011                         SENDFMT(sfp, "Host: %s\r\n", target.host);
1012                 if (!USR_HEADER_USER_AGENT)
1013                         SENDFMT(sfp, "User-Agent: %s\r\n", G.user_agent);
1014
1015                 /* Ask server to close the connection as soon as we are done
1016                  * (IOW: we do not intend to send more requests)
1017                  */
1018                 SENDFMT(sfp, "Connection: close\r\n");
1019
1020 #if ENABLE_FEATURE_WGET_AUTHENTICATION
1021                 if (target.user && !USR_HEADER_AUTH) {
1022                         SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n"+6,
1023                                 base64enc(target.user));
1024                 }
1025                 if (use_proxy && server.user && !USR_HEADER_PROXY_AUTH) {
1026                         SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n",
1027                                 base64enc(server.user));
1028                 }
1029 #endif
1030
1031                 if (G.beg_range != 0 && !USR_HEADER_RANGE)
1032                         SENDFMT(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range);
1033
1034 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1035                 if (G.extra_headers) {
1036                         log_io(G.extra_headers);
1037                         fputs(G.extra_headers, sfp);
1038                 }
1039
1040                 if (option_mask32 & WGET_OPT_POST_DATA) {
1041                         SENDFMT(sfp,
1042                                 "Content-Type: application/x-www-form-urlencoded\r\n"
1043                                 "Content-Length: %u\r\n"
1044                                 "\r\n"
1045                                 "%s",
1046                                 (int) strlen(G.post_data), G.post_data
1047                         );
1048                 } else
1049 #endif
1050                 {
1051                         SENDFMT(sfp, "\r\n");
1052                 }
1053
1054                 fflush(sfp);
1055
1056                 /*
1057                  * Retrieve HTTP response line and check for "200" status code.
1058                  */
1059  read_response:
1060                 fgets_and_trim(sfp);
1061
1062                 str = G.wget_buf;
1063                 str = skip_non_whitespace(str);
1064                 str = skip_whitespace(str);
1065                 // FIXME: no error check
1066                 // xatou wouldn't work: "200 OK"
1067                 status = atoi(str);
1068                 switch (status) {
1069                 case 0:
1070                 case 100:
1071                         while (gethdr(sfp) != NULL)
1072                                 /* eat all remaining headers */;
1073                         goto read_response;
1074                 case 200:
1075 /*
1076 Response 204 doesn't say "null file", it says "metadata
1077 has changed but data didn't":
1078
1079 "10.2.5 204 No Content
1080 The server has fulfilled the request but does not need to return
1081 an entity-body, and might want to return updated metainformation.
1082 The response MAY include new or updated metainformation in the form
1083 of entity-headers, which if present SHOULD be associated with
1084 the requested variant.
1085
1086 If the client is a user agent, it SHOULD NOT change its document
1087 view from that which caused the request to be sent. This response
1088 is primarily intended to allow input for actions to take place
1089 without causing a change to the user agent's active document view,
1090 although any new or updated metainformation SHOULD be applied
1091 to the document currently in the user agent's active view.
1092
1093 The 204 response MUST NOT include a message-body, and thus
1094 is always terminated by the first empty line after the header fields."
1095
1096 However, in real world it was observed that some web servers
1097 (e.g. Boa/0.94.14rc21) simply use code 204 when file size is zero.
1098 */
1099                 case 204:
1100                         if (G.beg_range != 0) {
1101                                 /* "Range:..." was not honored by the server.
1102                                  * Restart download from the beginning.
1103                                  */
1104                                 reset_beg_range_to_zero();
1105                         }
1106                         break;
1107                 case 300:  /* redirection */
1108                 case 301:
1109                 case 302:
1110                 case 303:
1111                         break;
1112                 case 206: /* Partial Content */
1113                         if (G.beg_range != 0)
1114                                 /* "Range:..." worked. Good. */
1115                                 break;
1116                         /* Partial Content even though we did not ask for it??? */
1117                         /* fall through */
1118                 default:
1119                         bb_error_msg_and_die("server returned error: %s", sanitize_string(G.wget_buf));
1120                 }
1121
1122                 /*
1123                  * Retrieve HTTP headers.
1124                  */
1125                 while ((str = gethdr(sfp)) != NULL) {
1126                         static const char keywords[] ALIGN1 =
1127                                 "content-length\0""transfer-encoding\0""location\0";
1128                         enum {
1129                                 KEY_content_length = 1, KEY_transfer_encoding, KEY_location
1130                         };
1131                         smalluint key;
1132
1133                         /* gethdr converted "FOO:" string to lowercase */
1134
1135                         /* strip trailing whitespace */
1136                         char *s = strchrnul(str, '\0') - 1;
1137                         while (s >= str && (*s == ' ' || *s == '\t')) {
1138                                 *s = '\0';
1139                                 s--;
1140                         }
1141                         key = index_in_strings(keywords, G.wget_buf) + 1;
1142                         if (key == KEY_content_length) {
1143                                 G.content_len = BB_STRTOOFF(str, NULL, 10);
1144                                 if (G.content_len < 0 || errno) {
1145                                         bb_error_msg_and_die("content-length %s is garbage", sanitize_string(str));
1146                                 }
1147                                 G.got_clen = 1;
1148                                 continue;
1149                         }
1150                         if (key == KEY_transfer_encoding) {
1151                                 if (strcmp(str_tolower(str), "chunked") != 0)
1152                                         bb_error_msg_and_die("transfer encoding '%s' is not supported", sanitize_string(str));
1153                                 G.chunked = 1;
1154                         }
1155                         if (key == KEY_location && status >= 300) {
1156                                 if (--redir_limit == 0)
1157                                         bb_error_msg_and_die("too many redirections");
1158                                 fclose(sfp);
1159                                 if (str[0] == '/') {
1160                                         free(redirected_path);
1161                                         target.path = redirected_path = xstrdup(str+1);
1162                                         /* lsa stays the same: it's on the same server */
1163                                 } else {
1164                                         parse_url(str, &target);
1165                                         if (!use_proxy) {
1166                                                 /* server.user remains untouched */
1167                                                 free(server.allocated);
1168                                                 server.allocated = NULL;
1169                                                 server.host = target.host;
1170                                                 /* strip_ipv6_scope_id(target.host); - no! */
1171                                                 /* we assume remote never gives us IPv6 addr with scope id */
1172                                                 server.port = target.port;
1173                                                 free(lsa);
1174                                                 goto resolve_lsa;
1175                                         } /* else: lsa stays the same: we use proxy */
1176                                 }
1177                                 goto establish_session;
1178                         }
1179                 }
1180 //              if (status >= 300)
1181 //                      bb_error_msg_and_die("bad redirection (no Location: header from server)");
1182
1183                 /* For HTTP, data is pumped over the same connection */
1184                 dfp = sfp;
1185         } else {
1186                 /*
1187                  *  FTP session
1188                  */
1189                 sfp = prepare_ftp_session(&dfp, &target, lsa);
1190         }
1191
1192         free(lsa);
1193
1194         if (!(option_mask32 & WGET_OPT_SPIDER)) {
1195                 if (G.output_fd < 0)
1196                         G.output_fd = xopen(G.fname_out, G.o_flags);
1197                 retrieve_file_data(dfp);
1198                 if (!(option_mask32 & WGET_OPT_OUTNAME)) {
1199                         xclose(G.output_fd);
1200                         G.output_fd = -1;
1201                 }
1202         }
1203
1204         if (dfp != sfp) {
1205                 /* It's ftp. Close data connection properly */
1206                 fclose(dfp);
1207                 if (ftpcmd(NULL, NULL, sfp) != 226)
1208                         bb_error_msg_and_die("ftp error: %s", sanitize_string(G.wget_buf + 4));
1209                 /* ftpcmd("QUIT", NULL, sfp); - why bother? */
1210         }
1211         fclose(sfp);
1212
1213         free(server.allocated);
1214         free(target.allocated);
1215         free(server.user);
1216         free(target.user);
1217         free(fname_out_alloc);
1218         free(redirected_path);
1219 }
1220
1221 int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
1222 int wget_main(int argc UNUSED_PARAM, char **argv)
1223 {
1224 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1225         static const char wget_longopts[] ALIGN1 =
1226                 /* name, has_arg, val */
1227                 "continue\0"         No_argument       "c"
1228 //FIXME: -s isn't --spider, it's --save-headers!
1229                 "spider\0"           No_argument       "s"
1230                 "quiet\0"            No_argument       "q"
1231                 "output-document\0"  Required_argument "O"
1232                 "directory-prefix\0" Required_argument "P"
1233                 "proxy\0"            Required_argument "Y"
1234                 "user-agent\0"       Required_argument "U"
1235 IF_FEATURE_WGET_TIMEOUT(
1236                 "timeout\0"          Required_argument "T")
1237                 /* Ignored: */
1238 IF_DESKTOP(     "tries\0"            Required_argument "t")
1239                 "header\0"           Required_argument "\xff"
1240                 "post-data\0"        Required_argument "\xfe"
1241                 /* Ignored (we always use PASV): */
1242 IF_DESKTOP(     "passive-ftp\0"      No_argument       "\xf0")
1243                 /* Ignored (we don't do ssl) */
1244 IF_DESKTOP(     "no-check-certificate\0" No_argument   "\xf0")
1245                 /* Ignored (we don't support caching) */
1246 IF_DESKTOP(     "no-cache\0"         No_argument       "\xf0")
1247 IF_DESKTOP(     "no-verbose\0"       No_argument       "\xf0")
1248 IF_DESKTOP(     "no-clobber\0"       No_argument       "\xf0")
1249 IF_DESKTOP(     "no-host-directories\0" No_argument    "\xf0")
1250 IF_DESKTOP(     "no-parent\0"        No_argument       "\xf0")
1251                 ;
1252 #endif
1253
1254 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1255         llist_t *headers_llist = NULL;
1256 #endif
1257
1258         INIT_G();
1259
1260 #if ENABLE_FEATURE_WGET_TIMEOUT
1261         G.timeout_seconds = 900;
1262         signal(SIGALRM, alarm_handler);
1263 #endif
1264         G.proxy_flag = "on";   /* use proxies if env vars are set */
1265         G.user_agent = "Wget"; /* "User-Agent" header field */
1266
1267 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1268         applet_long_options = wget_longopts;
1269 #endif
1270         opt_complementary = "-1" /* at least one URL */
1271                 IF_FEATURE_WGET_TIMEOUT(":T+") /* -T NUM */
1272                 IF_FEATURE_WGET_LONG_OPTIONS(":\xff::"); /* --header is a list */
1273         getopt32(argv, "csqO:P:Y:U:T:"
1274                 /*ignored:*/ "t:"
1275                 /*ignored:*/ "n::"
1276                 /* wget has exactly four -n<letter> opts, all of which we can ignore:
1277                  * -nv --no-verbose: be moderately quiet (-q is full quiet)
1278                  * -nc --no-clobber: abort if exists, neither download to FILE.n nor overwrite FILE
1279                  * -nH --no-host-directories: wget -r http://host/ won't create host/
1280                  * -np --no-parent
1281                  * "n::" above says that we accept -n[ARG].
1282                  * Specifying "n:" would be a bug: "-n ARG" would eat ARG!
1283                  */
1284                 , &G.fname_out, &G.dir_prefix,
1285                 &G.proxy_flag, &G.user_agent,
1286                 IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL),
1287                 NULL, /* -t RETRIES */
1288                 NULL  /* -n[ARG] */
1289                 IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
1290                 IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data)
1291         );
1292         argv += optind;
1293
1294 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
1295         if (headers_llist) {
1296                 int size = 0;
1297                 char *hdr;
1298                 llist_t *ll = headers_llist;
1299                 while (ll) {
1300                         size += strlen(ll->data) + 2;
1301                         ll = ll->link;
1302                 }
1303                 G.extra_headers = hdr = xmalloc(size + 1);
1304                 while (headers_llist) {
1305                         int bit;
1306                         const char *words;
1307
1308                         size = sprintf(hdr, "%s\r\n",
1309                                         (char*)llist_pop(&headers_llist));
1310                         /* a bit like index_in_substrings but don't match full key */
1311                         bit = 1;
1312                         words = wget_user_headers;
1313                         while (*words) {
1314                                 if (strstr(hdr, words) == hdr) {
1315                                         G.user_headers |= bit;
1316                                         break;
1317                                 }
1318                                 bit <<= 1;
1319                                 words += strlen(words) + 1;
1320                         }
1321                         hdr += size;
1322                 }
1323         }
1324 #endif
1325
1326         G.output_fd = -1;
1327         G.o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL;
1328         if (G.fname_out) { /* -O FILE ? */
1329                 if (LONE_DASH(G.fname_out)) { /* -O - ? */
1330                         G.output_fd = 1;
1331                         option_mask32 &= ~WGET_OPT_CONTINUE;
1332                 }
1333                 /* compat with wget: -O FILE can overwrite */
1334                 G.o_flags = O_WRONLY | O_CREAT | O_TRUNC;
1335         }
1336
1337         while (*argv)
1338                 download_one_url(*argv++);
1339
1340         if (G.output_fd >= 0)
1341                 xclose(G.output_fd);
1342
1343 #if ENABLE_FEATURE_CLEAN_UP && ENABLE_FEATURE_WGET_LONG_OPTIONS
1344         free(G.extra_headers);
1345 #endif
1346         FINI_G();
1347
1348         return EXIT_SUCCESS;
1349 }