src/clientmedia.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "clientmedia.h"
  21 #include "httpfetch.h"
  22 #include "client.h"
  23 #include "filecache.h"
  24 #include "filesys.h"
  25 #include "debug.h"
  26 #include "log.h"
  27 #include "porting.h"
  28 #include "settings.h"
  29 #include "network/networkprotocol.h"
  30 #include "util/hex.h"
  31 #include "util/serialize.h"
  32 #include "util/sha1.h"
  33 #include "util/string.h"
  34
  35 static std::string getMediaCacheDir()
  36 {
  37         return porting::path_cache + DIR_DELIM + "media";
  38 }
  39
  40 /*
  41         ClientMediaDownloader
  42 */
  43
  44 ClientMediaDownloader::ClientMediaDownloader():
  45         m_media_cache(getMediaCacheDir()),
  46         m_httpfetch_caller(HTTPFETCH_DISCARD)
  47 {
  48 }
  49
  50 ClientMediaDownloader::~ClientMediaDownloader()
  51 {
  52         if (m_httpfetch_caller != HTTPFETCH_DISCARD)
  53                 httpfetch_caller_free(m_httpfetch_caller);
  54
  55         for (std::map<std::string, FileStatus*>::iterator it = m_files.begin();
  56                         it != m_files.end(); ++it)
  57                 delete it->second;
  58
  59         for (u32 i = 0; i < m_remotes.size(); ++i)
  60                 delete m_remotes[i];
  61 }
  62
  63 void ClientMediaDownloader::addFile(const std::string &name, const std::string &sha1)
  64 {
  65         assert(!m_initial_step_done); // pre-condition
  66
  67         // if name was already announced, ignore the new announcement
  68         if (m_files.count(name) != 0) {
  69                 errorstream << "Client: ignoring duplicate media announcement "
  70                                 << "sent by server: \"" << name << "\""
  71                                 << std::endl;
  72                 return;
  73         }
  74
  75         // if name is empty or contains illegal characters, ignore the file
  76         if (name.empty() || !string_allowed(name, TEXTURENAME_ALLOWED_CHARS)) {
  77                 errorstream << "Client: ignoring illegal file name "
  78                                 << "sent by server: \"" << name << "\""
  79                                 << std::endl;
  80                 return;
  81         }
  82
  83         // length of sha1 must be exactly 20 (160 bits), else ignore the file
  84         if (sha1.size() != 20) {
  85                 errorstream << "Client: ignoring illegal SHA1 sent by server: "
  86                                 << hex_encode(sha1) << " \"" << name << "\""
  87                                 << std::endl;
  88                 return;
  89         }
  90
  91         FileStatus *filestatus = new FileStatus;
  92         filestatus->received = false;
  93         filestatus->sha1 = sha1;
  94         filestatus->current_remote = -1;
  95         m_files.insert(std::make_pair(name, filestatus));
  96 }
  97
  98 void ClientMediaDownloader::addRemoteServer(const std::string &baseurl)
  99 {
 100         assert(!m_initial_step_done);   // pre-condition
 101
 102         #ifdef USE_CURL
 103
 104         if (g_settings->getBool("enable_remote_media_server")) {
 105                 infostream << "Client: Adding remote server \""
 106                         << baseurl << "\" for media download" << std::endl;
 107
 108                 RemoteServerStatus *remote = new RemoteServerStatus;
 109                 remote->baseurl = baseurl;
 110                 remote->active_count = 0;
 111                 remote->request_by_filename = false;
 112                 m_remotes.push_back(remote);
 113         }
 114
 115         #else
 116
 117         infostream << "Client: Ignoring remote server \""
 118                 << baseurl << "\" because cURL support is not compiled in"
 119                 << std::endl;
 120
 121         #endif
 122 }
 123
 124 void ClientMediaDownloader::step(Client *client)
 125 {
 126         if (!m_initial_step_done) {
 127                 initialStep(client);
 128                 m_initial_step_done = true;
 129         }
 130
 131         // Remote media: check for completion of fetches
 132         if (m_httpfetch_active) {
 133                 bool fetched_something = false;
 134                 HTTPFetchResult fetch_result;
 135
 136                 while (httpfetch_async_get(m_httpfetch_caller, fetch_result)) {
 137                         m_httpfetch_active--;
 138                         fetched_something = true;
 139
 140                         // Is this a hashset (index.mth) or a media file?
 141                         if (fetch_result.request_id < m_remotes.size())
 142                                 remoteHashSetReceived(fetch_result);
 143                         else
 144                                 remoteMediaReceived(fetch_result, client);
 145                 }
 146
 147                 if (fetched_something)
 148                         startRemoteMediaTransfers();
 149
 150                 // Did all remote transfers end and no new ones can be started?
 151                 // If so, request still missing files from the minetest server
 152                 // (Or report that we have all files.)
 153                 if (m_httpfetch_active == 0) {
 154                         if (m_uncached_received_count < m_uncached_count) {
 155                                 infostream << "Client: Failed to remote-fetch "
 156                                         << (m_uncached_count-m_uncached_received_count)
 157                                         << " files. Requesting them"
 158                                         << " the usual way." << std::endl;
 159                         }
 160                         startConventionalTransfers(client);
 161                 }
 162         }
 163 }
 164
 165 void ClientMediaDownloader::initialStep(Client *client)
 166 {
 167         // Check media cache
 168         m_uncached_count = m_files.size();
 169         for (std::map<std::string, FileStatus*>::iterator
 170                         it = m_files.begin();
 171                         it != m_files.end(); ++it) {
 172                 std::string name = it->first;
 173                 FileStatus *filestatus = it->second;
 174                 const std::string &sha1 = filestatus->sha1;
 175
 176                 std::ostringstream tmp_os(std::ios_base::binary);
 177                 bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
 178
 179                 // If found in cache, try to load it from there
 180                 if (found_in_cache) {
 181                         bool success = checkAndLoad(name, sha1,
 182                                         tmp_os.str(), true, client);
 183                         if (success) {
 184                                 filestatus->received = true;
 185                                 m_uncached_count--;
 186                         }
 187                 }
 188         }
 189
 190         assert(m_uncached_received_count == 0);
 191
 192         // Create the media cache dir if we are likely to write to it
 193         if (m_uncached_count != 0) {
 194                 bool did = fs::CreateAllDirs(getMediaCacheDir());
 195                 if (!did) {
 196                         errorstream << "Client: "
 197                                 << "Could not create media cache directory: "
 198                                 << getMediaCacheDir()
 199                                 << std::endl;
 200                 }
 201         }
 202
 203         // If we found all files in the cache, report this fact to the server.
 204         // If the server reported no remote servers, immediately start
 205         // conventional transfers. Note: if cURL support is not compiled in,
 206         // m_remotes is always empty, so "!USE_CURL" is redundant but may
 207         // reduce the size of the compiled code
 208         if (!USE_CURL || m_uncached_count == 0 || m_remotes.empty()) {
 209                 startConventionalTransfers(client);
 210         }
 211         else {
 212                 // Otherwise start off by requesting each server's sha1 set
 213
 214                 // This is the first time we use httpfetch, so alloc a caller ID
 215                 m_httpfetch_caller = httpfetch_caller_alloc();
 216                 m_httpfetch_timeout = g_settings->getS32("curl_timeout");
 217
 218                 // Set the active fetch limit to curl_parallel_limit or 84,
 219                 // whichever is greater. This gives us some leeway so that
 220                 // inefficiencies in communicating with the httpfetch thread
 221                 // don't slow down fetches too much. (We still want some limit
 222                 // so that when the first remote server returns its hash set,
 223                 // not all files are requested from that server immediately.)
 224                 // One such inefficiency is that ClientMediaDownloader::step()
 225                 // is only called a couple times per second, while httpfetch
 226                 // might return responses much faster than that.
 227                 // Note that httpfetch strictly enforces curl_parallel_limit
 228                 // but at no inter-thread communication cost. This however
 229                 // doesn't help with the aforementioned inefficiencies.
 230                 // The signifance of 84 is that it is 2*6*9 in base 13.
 231                 m_httpfetch_active_limit = g_settings->getS32("curl_parallel_limit");
 232                 m_httpfetch_active_limit = MYMAX(m_httpfetch_active_limit, 84);
 233
 234                 // Write a list of hashes that we need. This will be POSTed
 235                 // to the server using Content-Type: application/octet-stream
 236                 std::string required_hash_set = serializeRequiredHashSet();
 237
 238                 // minor fixme: this loop ignores m_httpfetch_active_limit
 239
 240                 // another minor fixme, unlikely to matter in normal usage:
 241                 // these index.mth fetches do (however) count against
 242                 // m_httpfetch_active_limit when starting actual media file
 243                 // requests, so if there are lots of remote servers that are
 244                 // not responding, those will stall new media file transfers.
 245
 246                 for (u32 i = 0; i < m_remotes.size(); ++i) {
 247                         assert(m_httpfetch_next_id == i);
 248
 249                         RemoteServerStatus *remote = m_remotes[i];
 250                         actionstream << "Client: Contacting remote server \""
 251                                 << remote->baseurl << "\"" << std::endl;
 252
 253                         HTTPFetchRequest fetch_request;
 254                         fetch_request.url =
 255                                 remote->baseurl + MTHASHSET_FILE_NAME;
 256                         fetch_request.caller = m_httpfetch_caller;
 257                         fetch_request.request_id = m_httpfetch_next_id; // == i
 258                         fetch_request.timeout = m_httpfetch_timeout;
 259                         fetch_request.connect_timeout = m_httpfetch_timeout;
 260                         fetch_request.post_data = required_hash_set;
 261                         fetch_request.extra_headers.push_back(
 262                                 "Content-Type: application/octet-stream");
 263                         httpfetch_async(fetch_request);
 264
 265                         m_httpfetch_active++;
 266                         m_httpfetch_next_id++;
 267                         m_outstanding_hash_sets++;
 268                 }
 269         }
 270 }
 271
 272 void ClientMediaDownloader::remoteHashSetReceived(
 273                 const HTTPFetchResult &fetch_result)
 274 {
 275         u32 remote_id = fetch_result.request_id;
 276         assert(remote_id < m_remotes.size());
 277         RemoteServerStatus *remote = m_remotes[remote_id];
 278
 279         m_outstanding_hash_sets--;
 280
 281         if (fetch_result.succeeded) {
 282                 try {
 283                         // Server sent a list of file hashes that are
 284                         // available on it, try to parse the list
 285
 286                         std::set<std::string> sha1_set;
 287                         deSerializeHashSet(fetch_result.data, sha1_set);
 288
 289                         // Parsing succeeded: For every file that is
 290                         // available on this server, add this server
 291                         // to the available_remotes array
 292
 293                         for(std::map<std::string, FileStatus*>::iterator
 294                                         it = m_files.upper_bound(m_name_bound);
 295                                         it != m_files.end(); ++it) {
 296                                 FileStatus *f = it->second;
 297                                 if (!f->received && sha1_set.count(f->sha1))
 298                                         f->available_remotes.push_back(remote_id);
 299                         }
 300                 }
 301                 catch (SerializationError &e) {
 302                         infostream << "Client: Remote server \""
 303                                 << remote->baseurl << "\" sent invalid hash set: "
 304                                 << e.what() << std::endl;
 305                 }
 306         }
 307
 308         // For compatibility: If index.mth is not found, assume that the
 309         // server contains files named like the original files (not their sha1)
 310
 311         // Do NOT check for any particular response code (e.g. 404) here,
 312         // because different servers respond differently
 313
 314         if (!fetch_result.succeeded && !fetch_result.timeout) {
 315                 infostream << "Client: Enabling compatibility mode for remote "
 316                         << "server \"" << remote->baseurl << "\"" << std::endl;
 317                 remote->request_by_filename = true;
 318
 319                 // Assume every file is available on this server
 320
 321                 for(std::map<std::string, FileStatus*>::iterator
 322                                 it = m_files.upper_bound(m_name_bound);
 323                                 it != m_files.end(); ++it) {
 324                         FileStatus *f = it->second;
 325                         if (!f->received)
 326                                 f->available_remotes.push_back(remote_id);
 327                 }
 328         }
 329 }
 330
 331 void ClientMediaDownloader::remoteMediaReceived(
 332                 const HTTPFetchResult &fetch_result,
 333                 Client *client)
 334 {
 335         // Some remote server sent us a file.
 336         // -> decrement number of active fetches
 337         // -> mark file as received if fetch succeeded
 338         // -> try to load media
 339
 340         std::string name;
 341         {
 342                 std::unordered_map<unsigned long, std::string>::iterator it =
 343                         m_remote_file_transfers.find(fetch_result.request_id);
 344                 assert(it != m_remote_file_transfers.end());
 345                 name = it->second;
 346                 m_remote_file_transfers.erase(it);
 347         }
 348
 349         sanity_check(m_files.count(name) != 0);
 350
 351         FileStatus *filestatus = m_files[name];
 352         sanity_check(!filestatus->received);
 353         sanity_check(filestatus->current_remote >= 0);
 354
 355         RemoteServerStatus *remote = m_remotes[filestatus->current_remote];
 356
 357         filestatus->current_remote = -1;
 358         remote->active_count--;
 359
 360         // If fetch succeeded, try to load media file
 361
 362         if (fetch_result.succeeded) {
 363                 bool success = checkAndLoad(name, filestatus->sha1,
 364                                 fetch_result.data, false, client);
 365                 if (success) {
 366                         filestatus->received = true;
 367                         assert(m_uncached_received_count < m_uncached_count);
 368                         m_uncached_received_count++;
 369                 }
 370         }
 371 }
 372
 373 s32 ClientMediaDownloader::selectRemoteServer(FileStatus *filestatus)
 374 {
 375         // Pre-conditions
 376         assert(filestatus != NULL);
 377         assert(!filestatus->received);
 378         assert(filestatus->current_remote < 0);
 379
 380         if (filestatus->available_remotes.empty())
 381                 return -1;
 382         else {
 383                 // Of all servers that claim to provide the file (and haven't
 384                 // been unsuccessfully tried before), find the one with the
 385                 // smallest number of currently active transfers
 386
 387                 s32 best = 0;
 388                 s32 best_remote_id = filestatus->available_remotes[best];
 389                 s32 best_active_count = m_remotes[best_remote_id]->active_count;
 390
 391                 for (u32 i = 1; i < filestatus->available_remotes.size(); ++i) {
 392                         s32 remote_id = filestatus->available_remotes[i];
 393                         s32 active_count = m_remotes[remote_id]->active_count;
 394                         if (active_count < best_active_count) {
 395                                 best = i;
 396                                 best_remote_id = remote_id;
 397                                 best_active_count = active_count;
 398                         }
 399                 }
 400
 401                 filestatus->available_remotes.erase(
 402                                 filestatus->available_remotes.begin() + best);
 403
 404                 return best_remote_id;
 405         }
 406 }
 407
 408 void ClientMediaDownloader::startRemoteMediaTransfers()
 409 {
 410         bool changing_name_bound = true;
 411
 412         for (std::map<std::string, FileStatus*>::iterator
 413                         files_iter = m_files.upper_bound(m_name_bound);
 414                         files_iter != m_files.end(); ++files_iter) {
 415
 416                 // Abort if active fetch limit is exceeded
 417                 if (m_httpfetch_active >= m_httpfetch_active_limit)
 418                         break;
 419
 420                 const std::string &name = files_iter->first;
 421                 FileStatus *filestatus = files_iter->second;
 422
 423                 if (!filestatus->received && filestatus->current_remote < 0) {
 424                         // File has not been received yet and is not currently
 425                         // being transferred. Choose a server for it.
 426                         s32 remote_id = selectRemoteServer(filestatus);
 427                         if (remote_id >= 0) {
 428                                 // Found a server, so start fetching
 429                                 RemoteServerStatus *remote =
 430                                         m_remotes[remote_id];
 431
 432                                 std::string url = remote->baseurl +
 433                                         (remote->request_by_filename ? name :
 434                                         hex_encode(filestatus->sha1));
 435                                 verbosestream << "Client: "
 436                                         << "Requesting remote media file "
 437                                         << "\"" << name << "\" "
 438                                         << "\"" << url << "\"" << std::endl;
 439
 440                                 HTTPFetchRequest fetch_request;
 441                                 fetch_request.url = url;
 442                                 fetch_request.caller = m_httpfetch_caller;
 443                                 fetch_request.request_id = m_httpfetch_next_id;
 444                                 fetch_request.timeout = 0; // no data timeout!
 445                                 fetch_request.connect_timeout =
 446                                         m_httpfetch_timeout;
 447                                 httpfetch_async(fetch_request);
 448
 449                                 m_remote_file_transfers.insert(std::make_pair(
 450                                                         m_httpfetch_next_id,
 451                                                         name));
 452
 453                                 filestatus->current_remote = remote_id;
 454                                 remote->active_count++;
 455                                 m_httpfetch_active++;
 456                                 m_httpfetch_next_id++;
 457                         }
 458                 }
 459
 460                 if (filestatus->received ||
 461                                 (filestatus->current_remote < 0 &&
 462                                  !m_outstanding_hash_sets)) {
 463                         // If we arrive here, we conclusively know that we
 464                         // won't fetch this file from a remote server in the
 465                         // future. So update the name bound if possible.
 466                         if (changing_name_bound)
 467                                 m_name_bound = name;
 468                 }
 469                 else
 470                         changing_name_bound = false;
 471         }
 472
 473 }
 474
 475 void ClientMediaDownloader::startConventionalTransfers(Client *client)
 476 {
 477         assert(m_httpfetch_active == 0);        // pre-condition
 478
 479         if (m_uncached_received_count != m_uncached_count) {
 480                 // Some media files have not been received yet, use the
 481                 // conventional slow method (minetest protocol) to get them
 482                 std::vector<std::string> file_requests;
 483                 for (std::map<std::string, FileStatus*>::iterator
 484                                 it = m_files.begin();
 485                                 it != m_files.end(); ++it) {
 486                         if (!it->second->received)
 487                                 file_requests.push_back(it->first);
 488                 }
 489                 assert((s32) file_requests.size() ==
 490                                 m_uncached_count - m_uncached_received_count);
 491                 client->request_media(file_requests);
 492         }
 493 }
 494
 495 void ClientMediaDownloader::conventionalTransferDone(
 496                 const std::string &name,
 497                 const std::string &data,
 498                 Client *client)
 499 {
 500         // Check that file was announced
 501         std::map<std::string, FileStatus*>::iterator
 502                 file_iter = m_files.find(name);
 503         if (file_iter == m_files.end()) {
 504                 errorstream << "Client: server sent media file that was"
 505                         << "not announced, ignoring it: \"" << name << "\""
 506                         << std::endl;
 507                 return;
 508         }
 509         FileStatus *filestatus = file_iter->second;
 510         assert(filestatus != NULL);
 511
 512         // Check that file hasn't already been received
 513         if (filestatus->received) {
 514                 errorstream << "Client: server sent media file that we already"
 515                         << "received, ignoring it: \"" << name << "\""
 516                         << std::endl;
 517                 return;
 518         }
 519
 520         // Mark file as received, regardless of whether loading it works and
 521         // whether the checksum matches (because at this point there is no
 522         // other server that could send a replacement)
 523         filestatus->received = true;
 524         assert(m_uncached_received_count < m_uncached_count);
 525         m_uncached_received_count++;
 526
 527         // Check that received file matches announced checksum
 528         // If so, load it
 529         checkAndLoad(name, filestatus->sha1, data, false, client);
 530 }
 531
 532 bool ClientMediaDownloader::checkAndLoad(
 533                 const std::string &name, const std::string &sha1,
 534                 const std::string &data, bool is_from_cache, Client *client)
 535 {
 536         const char *cached_or_received = is_from_cache ? "cached" : "received";
 537         const char *cached_or_received_uc = is_from_cache ? "Cached" : "Received";
 538         std::string sha1_hex = hex_encode(sha1);
 539
 540         // Compute actual checksum of data
 541         std::string data_sha1;
 542         {
 543                 SHA1 data_sha1_calculator;
 544                 data_sha1_calculator.addBytes(data.c_str(), data.size());
 545                 unsigned char *data_tmpdigest = data_sha1_calculator.getDigest();
 546                 data_sha1.assign((char*) data_tmpdigest, 20);
 547                 free(data_tmpdigest);
 548         }
 549
 550         // Check that received file matches announced checksum
 551         if (data_sha1 != sha1) {
 552                 std::string data_sha1_hex = hex_encode(data_sha1);
 553                 infostream << "Client: "
 554                         << cached_or_received_uc << " media file "
 555                         << sha1_hex << " \"" << name << "\" "
 556                         << "mismatches actual checksum " << data_sha1_hex
 557                         << std::endl;
 558                 return false;
 559         }
 560
 561         // Checksum is ok, try loading the file
 562         bool success = client->loadMedia(data, name);
 563         if (!success) {
 564                 infostream << "Client: "
 565                         << "Failed to load " << cached_or_received << " media: "
 566                         << sha1_hex << " \"" << name << "\""
 567                         << std::endl;
 568                 return false;
 569         }
 570
 571         verbosestream << "Client: "
 572                 << "Loaded " << cached_or_received << " media: "
 573                 << sha1_hex << " \"" << name << "\""
 574                 << std::endl;
 575
 576         // Update cache (unless we just loaded the file from the cache)
 577         if (!is_from_cache)
 578                 m_media_cache.update(sha1_hex, data);
 579
 580         return true;
 581 }
 582
 583
 584 /*
 585         Minetest Hashset File Format
 586
 587         All values are stored in big-endian byte order.
 588         [u32] signature: 'MTHS'
 589         [u16] version: 1
 590         For each hash in set:
 591                 [u8*20] SHA1 hash
 592
 593         Version changes:
 594         1 - Initial version
 595 */
 596
 597 std::string ClientMediaDownloader::serializeRequiredHashSet()
 598 {
 599         std::ostringstream os(std::ios::binary);
 600
 601         writeU32(os, MTHASHSET_FILE_SIGNATURE); // signature
 602         writeU16(os, 1);                        // version
 603
 604         // Write list of hashes of files that have not been
 605         // received (found in cache) yet
 606         for (std::map<std::string, FileStatus*>::iterator
 607                         it = m_files.begin();
 608                         it != m_files.end(); ++it) {
 609                 if (!it->second->received) {
 610                         FATAL_ERROR_IF(it->second->sha1.size() != 20, "Invalid SHA1 size");
 611                         os << it->second->sha1;
 612                 }
 613         }
 614
 615         return os.str();
 616 }
 617
 618 void ClientMediaDownloader::deSerializeHashSet(const std::string &data,
 619                 std::set<std::string> &result)
 620 {
 621         if (data.size() < 6 || data.size() % 20 != 6) {
 622                 throw SerializationError(
 623                                 "ClientMediaDownloader::deSerializeHashSet: "
 624                                 "invalid hash set file size");
 625         }
 626
 627         const u8 *data_cstr = (const u8*) data.c_str();
 628
 629         u32 signature = readU32(&data_cstr[0]);
 630         if (signature != MTHASHSET_FILE_SIGNATURE) {
 631                 throw SerializationError(
 632                                 "ClientMediaDownloader::deSerializeHashSet: "
 633                                 "invalid hash set file signature");
 634         }
 635
 636         u16 version = readU16(&data_cstr[4]);
 637         if (version != 1) {
 638                 throw SerializationError(
 639                                 "ClientMediaDownloader::deSerializeHashSet: "
 640                                 "unsupported hash set file version");
 641         }
 642
 643         for (u32 pos = 6; pos < data.size(); pos += 20) {
 644                 result.insert(data.substr(pos, 20));
 645         }
 646 }