src/clientmedia.cpp

   1 /*
   2 Minetest
   3 Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
   4
   5 This program is free software; you can redistribute it and/or modify
   6 it under the terms of the GNU Lesser General Public License as published by
   7 the Free Software Foundation; either version 2.1 of the License, or
   8 (at your option) any later version.
   9
  10 This program is distributed in the hope that it will be useful,
  11 but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 GNU Lesser General Public License for more details.
  14
  15 You should have received a copy of the GNU Lesser General Public License along
  16 with this program; if not, write to the Free Software Foundation, Inc.,
  17 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18 */
  19
  20 #include "clientmedia.h"
  21 #include "util/serialize.h"
  22 #include "util/string.h"
  23 #include "httpfetch.h"
  24 #include "client.h"
  25 #include "clientserver.h"
  26 #include "filecache.h"
  27 #include "filesys.h"
  28 #include "hex.h"
  29 #include "sha1.h"
  30 #include "debug.h"
  31 #include "log.h"
  32 #include "porting.h"
  33 #include "settings.h"
  34 #include "main.h"
  35
  36 static std::string getMediaCacheDir()
  37 {
  38         return porting::path_user + DIR_DELIM + "cache" + DIR_DELIM + "media";
  39 }
  40
  41 /*
  42         ClientMediaDownloader
  43 */
  44
  45 ClientMediaDownloader::ClientMediaDownloader():
  46         m_media_cache(getMediaCacheDir())
  47 {
  48         m_initial_step_done = false;
  49         m_name_bound = "";  // works because "" is an invalid file name
  50         m_uncached_count = 0;
  51         m_uncached_received_count = 0;
  52         m_httpfetch_caller = HTTPFETCH_DISCARD;
  53         m_httpfetch_active = 0;
  54         m_httpfetch_active_limit = 0;
  55         m_httpfetch_next_id = 0;
  56         m_httpfetch_timeout = 0;
  57         m_outstanding_hash_sets = 0;
  58 }
  59
  60 ClientMediaDownloader::~ClientMediaDownloader()
  61 {
  62         if (m_httpfetch_caller != HTTPFETCH_DISCARD)
  63                 httpfetch_caller_free(m_httpfetch_caller);
  64
  65         for (std::map<std::string, FileStatus*>::iterator it = m_files.begin();
  66                         it != m_files.end(); ++it)
  67                 delete it->second;
  68
  69         for (u32 i = 0; i < m_remotes.size(); ++i)
  70                 delete m_remotes[i];
  71 }
  72
  73 void ClientMediaDownloader::addFile(std::string name, std::string sha1)
  74 {
  75         assert(!m_initial_step_done);
  76
  77         // if name was already announced, ignore the new announcement
  78         if (m_files.count(name) != 0) {
  79                 errorstream << "Client: ignoring duplicate media announcement "
  80                                 << "sent by server: \"" << name << "\""
  81                                 << std::endl;
  82                 return;
  83         }
  84
  85         // if name is empty or contains illegal characters, ignore the file
  86         if (name.empty() || !string_allowed(name, TEXTURENAME_ALLOWED_CHARS)) {
  87                 errorstream << "Client: ignoring illegal file name "
  88                                 << "sent by server: \"" << name << "\""
  89                                 << std::endl;
  90                 return;
  91         }
  92
  93         // length of sha1 must be exactly 20 (160 bits), else ignore the file
  94         if (sha1.size() != 20) {
  95                 errorstream << "Client: ignoring illegal SHA1 sent by server: "
  96                                 << hex_encode(sha1) << " \"" << name << "\""
  97                                 << std::endl;
  98                 return;
  99         }
 100
 101         FileStatus *filestatus = new FileStatus;
 102         filestatus->received = false;
 103         filestatus->sha1 = sha1;
 104         filestatus->current_remote = -1;
 105         m_files.insert(std::make_pair(name, filestatus));
 106 }
 107
 108 void ClientMediaDownloader::addRemoteServer(std::string baseurl)
 109 {
 110         assert(!m_initial_step_done);
 111
 112         #ifdef USE_CURL
 113
 114         if (g_settings->getBool("enable_remote_media_server")) {
 115                 infostream << "Client: Adding remote server \""
 116                         << baseurl << "\" for media download" << std::endl;
 117
 118                 RemoteServerStatus *remote = new RemoteServerStatus;
 119                 remote->baseurl = baseurl;
 120                 remote->active_count = 0;
 121                 remote->request_by_filename = false;
 122                 m_remotes.push_back(remote);
 123         }
 124
 125         #else
 126
 127         infostream << "Client: Ignoring remote server \""
 128                 << baseurl << "\" because cURL support is not compiled in"
 129                 << std::endl;
 130
 131         #endif
 132 }
 133
 134 void ClientMediaDownloader::step(Client *client)
 135 {
 136         if (!m_initial_step_done) {
 137                 initialStep(client);
 138                 m_initial_step_done = true;
 139         }
 140
 141         // Remote media: check for completion of fetches
 142         if (m_httpfetch_active) {
 143                 bool fetched_something = false;
 144                 HTTPFetchResult fetch_result;
 145
 146                 while (httpfetch_async_get(m_httpfetch_caller, fetch_result)) {
 147                         m_httpfetch_active--;
 148                         fetched_something = true;
 149
 150                         // Is this a hashset (index.mth) or a media file?
 151                         if (fetch_result.request_id < m_remotes.size())
 152                                 remoteHashSetReceived(fetch_result);
 153                         else
 154                                 remoteMediaReceived(fetch_result, client);
 155                 }
 156
 157                 if (fetched_something)
 158                         startRemoteMediaTransfers();
 159
 160                 // Did all remote transfers end and no new ones can be started?
 161                 // If so, request still missing files from the minetest server
 162                 // (Or report that we have all files.)
 163                 if (m_httpfetch_active == 0) {
 164                         if (m_uncached_received_count < m_uncached_count) {
 165                                 infostream << "Client: Failed to remote-fetch "
 166                                         << (m_uncached_count-m_uncached_received_count)
 167                                         << " files. Requesting them"
 168                                         << " the usual way." << std::endl;
 169                         }
 170                         startConventionalTransfers(client);
 171                 }
 172         }
 173 }
 174
 175 void ClientMediaDownloader::initialStep(Client *client)
 176 {
 177         // Check media cache
 178         m_uncached_count = m_files.size();
 179         for (std::map<std::string, FileStatus*>::iterator
 180                         it = m_files.begin();
 181                         it != m_files.end(); ++it) {
 182                 std::string name = it->first;
 183                 FileStatus *filestatus = it->second;
 184                 const std::string &sha1 = filestatus->sha1;
 185
 186                 std::ostringstream tmp_os(std::ios_base::binary);
 187                 bool found_in_cache = m_media_cache.load(hex_encode(sha1), tmp_os);
 188
 189                 // If found in cache, try to load it from there
 190                 if (found_in_cache) {
 191                         bool success = checkAndLoad(name, sha1,
 192                                         tmp_os.str(), true, client);
 193                         if (success) {
 194                                 filestatus->received = true;
 195                                 m_uncached_count--;
 196                         }
 197                 }
 198         }
 199
 200         assert(m_uncached_received_count == 0);
 201
 202         // Create the media cache dir if we are likely to write to it
 203         if (m_uncached_count != 0) {
 204                 bool did = fs::CreateAllDirs(getMediaCacheDir());
 205                 if (!did) {
 206                         errorstream << "Client: "
 207                                 << "Could not create media cache directory: "
 208                                 << getMediaCacheDir()
 209                                 << std::endl;
 210                 }
 211         }
 212
 213         // If we found all files in the cache, report this fact to the server.
 214         // If the server reported no remote servers, immediately start
 215         // conventional transfers. Note: if cURL support is not compiled in,
 216         // m_remotes is always empty, so "!USE_CURL" is redundant but may
 217         // reduce the size of the compiled code
 218         if (!USE_CURL || m_uncached_count == 0 || m_remotes.empty()) {
 219                 startConventionalTransfers(client);
 220         }
 221         else {
 222                 // Otherwise start off by requesting each server's sha1 set
 223
 224                 // This is the first time we use httpfetch, so alloc a caller ID
 225                 m_httpfetch_caller = httpfetch_caller_alloc();
 226                 m_httpfetch_timeout = g_settings->getS32("curl_timeout");
 227
 228                 // Set the active fetch limit to curl_parallel_limit or 84,
 229                 // whichever is greater. This gives us some leeway so that
 230                 // inefficiencies in communicating with the httpfetch thread
 231                 // don't slow down fetches too much. (We still want some limit
 232                 // so that when the first remote server returns its hash set,
 233                 // not all files are requested from that server immediately.)
 234                 // One such inefficiency is that ClientMediaDownloader::step()
 235                 // is only called a couple times per second, while httpfetch
 236                 // might return responses much faster than that.
 237                 // Note that httpfetch strictly enforces curl_parallel_limit
 238                 // but at no inter-thread communication cost. This however
 239                 // doesn't help with the aforementioned inefficiencies.
 240                 // The signifance of 84 is that it is 2*6*9 in base 13.
 241                 m_httpfetch_active_limit = g_settings->getS32("curl_parallel_limit");
 242                 m_httpfetch_active_limit = MYMAX(m_httpfetch_active_limit, 84);
 243
 244                 // Write a list of hashes that we need. This will be POSTed
 245                 // to the server using Content-Type: application/octet-stream
 246                 std::string required_hash_set = serializeRequiredHashSet();
 247
 248                 // minor fixme: this loop ignores m_httpfetch_active_limit
 249
 250                 // another minor fixme, unlikely to matter in normal usage:
 251                 // these index.mth fetches do (however) count against
 252                 // m_httpfetch_active_limit when starting actual media file
 253                 // requests, so if there are lots of remote servers that are
 254                 // not responding, those will stall new media file transfers.
 255
 256                 for (u32 i = 0; i < m_remotes.size(); ++i) {
 257                         assert(m_httpfetch_next_id == i);
 258
 259                         RemoteServerStatus *remote = m_remotes[i];
 260                         actionstream << "Client: Contacting remote server \""
 261                                 << remote->baseurl << "\"" << std::endl;
 262
 263                         HTTPFetchRequest fetch_request;
 264                         fetch_request.url =
 265                                 remote->baseurl + MTHASHSET_FILE_NAME;
 266                         fetch_request.caller = m_httpfetch_caller;
 267                         fetch_request.request_id = m_httpfetch_next_id; // == i
 268                         fetch_request.timeout = m_httpfetch_timeout;
 269                         fetch_request.connect_timeout = m_httpfetch_timeout;
 270                         fetch_request.post_data = required_hash_set;
 271                         fetch_request.extra_headers.push_back(
 272                                 "Content-Type: application/octet-stream");
 273                         httpfetch_async(fetch_request);
 274
 275                         m_httpfetch_active++;
 276                         m_httpfetch_next_id++;
 277                         m_outstanding_hash_sets++;
 278                 }
 279         }
 280 }
 281
 282 void ClientMediaDownloader::remoteHashSetReceived(
 283                 const HTTPFetchResult &fetch_result)
 284 {
 285         u32 remote_id = fetch_result.request_id;
 286         assert(remote_id < m_remotes.size());
 287         RemoteServerStatus *remote = m_remotes[remote_id];
 288
 289         m_outstanding_hash_sets--;
 290
 291         if (fetch_result.succeeded) {
 292                 try {
 293                         // Server sent a list of file hashes that are
 294                         // available on it, try to parse the list
 295
 296                         std::set<std::string> sha1_set;
 297                         deSerializeHashSet(fetch_result.data, sha1_set);
 298
 299                         // Parsing succeeded: For every file that is
 300                         // available on this server, add this server
 301                         // to the available_remotes array
 302
 303                         for(std::map<std::string, FileStatus*>::iterator
 304                                         it = m_files.upper_bound(m_name_bound);
 305                                         it != m_files.end(); ++it) {
 306                                 FileStatus *f = it->second;
 307                                 if (!f->received && sha1_set.count(f->sha1))
 308                                         f->available_remotes.push_back(remote_id);
 309                         }
 310                 }
 311                 catch (SerializationError &e) {
 312                         infostream << "Client: Remote server \""
 313                                 << remote->baseurl << "\" sent invalid hash set: "
 314                                 << e.what() << std::endl;
 315                 }
 316         }
 317
 318         // For compatibility: If index.mth is not found, assume that the
 319         // server contains files named like the original files (not their sha1)
 320
 321         // Do NOT check for any particular response code (e.g. 404) here,
 322         // because different servers respond differently
 323
 324         if (!fetch_result.succeeded && !fetch_result.timeout) {
 325                 infostream << "Client: Enabling compatibility mode for remote "
 326                         << "server \"" << remote->baseurl << "\"" << std::endl;
 327                 remote->request_by_filename = true;
 328
 329                 // Assume every file is available on this server
 330
 331                 for(std::map<std::string, FileStatus*>::iterator
 332                                 it = m_files.upper_bound(m_name_bound);
 333                                 it != m_files.end(); ++it) {
 334                         FileStatus *f = it->second;
 335                         if (!f->received)
 336                                 f->available_remotes.push_back(remote_id);
 337                 }
 338         }
 339 }
 340
 341 void ClientMediaDownloader::remoteMediaReceived(
 342                 const HTTPFetchResult &fetch_result,
 343                 Client *client)
 344 {
 345         // Some remote server sent us a file.
 346         // -> decrement number of active fetches
 347         // -> mark file as received if fetch succeeded
 348         // -> try to load media
 349
 350         std::string name;
 351         {
 352                 std::map<unsigned long, std::string>::iterator it =
 353                         m_remote_file_transfers.find(fetch_result.request_id);
 354                 assert(it != m_remote_file_transfers.end());
 355                 name = it->second;
 356                 m_remote_file_transfers.erase(it);
 357         }
 358
 359         assert(m_files.count(name) != 0);
 360
 361         FileStatus *filestatus = m_files[name];
 362         assert(!filestatus->received);
 363         assert(filestatus->current_remote >= 0);
 364
 365         RemoteServerStatus *remote = m_remotes[filestatus->current_remote];
 366
 367         filestatus->current_remote = -1;
 368         remote->active_count--;
 369
 370         // If fetch succeeded, try to load media file
 371
 372         if (fetch_result.succeeded) {
 373                 bool success = checkAndLoad(name, filestatus->sha1,
 374                                 fetch_result.data, false, client);
 375                 if (success) {
 376                         filestatus->received = true;
 377                         assert(m_uncached_received_count < m_uncached_count);
 378                         m_uncached_received_count++;
 379                 }
 380         }
 381 }
 382
 383 s32 ClientMediaDownloader::selectRemoteServer(FileStatus *filestatus)
 384 {
 385         assert(filestatus != NULL);
 386         assert(!filestatus->received);
 387         assert(filestatus->current_remote < 0);
 388
 389         if (filestatus->available_remotes.empty())
 390                 return -1;
 391         else {
 392                 // Of all servers that claim to provide the file (and haven't
 393                 // been unsuccessfully tried before), find the one with the
 394                 // smallest number of currently active transfers
 395
 396                 s32 best = 0;
 397                 s32 best_remote_id = filestatus->available_remotes[best];
 398                 s32 best_active_count = m_remotes[best_remote_id]->active_count;
 399
 400                 for (u32 i = 1; i < filestatus->available_remotes.size(); ++i) {
 401                         s32 remote_id = filestatus->available_remotes[i];
 402                         s32 active_count = m_remotes[remote_id]->active_count;
 403                         if (active_count < best_active_count) {
 404                                 best = i;
 405                                 best_remote_id = remote_id;
 406                                 best_active_count = active_count;
 407                         }
 408                 }
 409
 410                 filestatus->available_remotes.erase(
 411                                 filestatus->available_remotes.begin() + best);
 412
 413                 return best_remote_id;
 414         }
 415 }
 416
 417 void ClientMediaDownloader::startRemoteMediaTransfers()
 418 {
 419         bool changing_name_bound = true;
 420
 421         for (std::map<std::string, FileStatus*>::iterator
 422                         files_iter = m_files.upper_bound(m_name_bound);
 423                         files_iter != m_files.end(); ++files_iter) {
 424
 425                 // Abort if active fetch limit is exceeded
 426                 if (m_httpfetch_active >= m_httpfetch_active_limit)
 427                         break;
 428
 429                 const std::string &name = files_iter->first;
 430                 FileStatus *filestatus = files_iter->second;
 431
 432                 if (!filestatus->received && filestatus->current_remote < 0) {
 433                         // File has not been received yet and is not currently
 434                         // being transferred. Choose a server for it.
 435                         s32 remote_id = selectRemoteServer(filestatus);
 436                         if (remote_id >= 0) {
 437                                 // Found a server, so start fetching
 438                                 RemoteServerStatus *remote =
 439                                         m_remotes[remote_id];
 440
 441                                 std::string url = remote->baseurl +
 442                                         (remote->request_by_filename ? name :
 443                                         hex_encode(filestatus->sha1));
 444                                 verbosestream << "Client: "
 445                                         << "Requesting remote media file "
 446                                         << "\"" << name << "\" "
 447                                         << "\"" << url << "\"" << std::endl;
 448
 449                                 HTTPFetchRequest fetch_request;
 450                                 fetch_request.url = url;
 451                                 fetch_request.caller = m_httpfetch_caller;
 452                                 fetch_request.request_id = m_httpfetch_next_id;
 453                                 fetch_request.timeout = 0; // no data timeout!
 454                                 fetch_request.connect_timeout =
 455                                         m_httpfetch_timeout;
 456                                 httpfetch_async(fetch_request);
 457
 458                                 m_remote_file_transfers.insert(std::make_pair(
 459                                                         m_httpfetch_next_id,
 460                                                         name));
 461
 462                                 filestatus->current_remote = remote_id;
 463                                 remote->active_count++;
 464                                 m_httpfetch_active++;
 465                                 m_httpfetch_next_id++;
 466                         }
 467                 }
 468
 469                 if (filestatus->received ||
 470                                 (filestatus->current_remote < 0 &&
 471                                  !m_outstanding_hash_sets)) {
 472                         // If we arrive here, we conclusively know that we
 473                         // won't fetch this file from a remote server in the
 474                         // future. So update the name bound if possible.
 475                         if (changing_name_bound)
 476                                 m_name_bound = name;
 477                 }
 478                 else
 479                         changing_name_bound = false;
 480         }
 481
 482 }
 483
 484 void ClientMediaDownloader::startConventionalTransfers(Client *client)
 485 {
 486         assert(m_httpfetch_active == 0);
 487
 488         if (m_uncached_received_count != m_uncached_count) {
 489                 // Some media files have not been received yet, use the
 490                 // conventional slow method (minetest protocol) to get them
 491                 std::list<std::string> file_requests;
 492                 for (std::map<std::string, FileStatus*>::iterator
 493                                 it = m_files.begin();
 494                                 it != m_files.end(); ++it) {
 495                         if (!it->second->received)
 496                                 file_requests.push_back(it->first);
 497                 }
 498                 assert((s32) file_requests.size() ==
 499                                 m_uncached_count - m_uncached_received_count);
 500                 client->request_media(file_requests);
 501         }
 502 }
 503
 504 void ClientMediaDownloader::conventionalTransferDone(
 505                 const std::string &name,
 506                 const std::string &data,
 507                 Client *client)
 508 {
 509         // Check that file was announced
 510         std::map<std::string, FileStatus*>::iterator
 511                 file_iter = m_files.find(name);
 512         if (file_iter == m_files.end()) {
 513                 errorstream << "Client: server sent media file that was"
 514                         << "not announced, ignoring it: \"" << name << "\""
 515                         << std::endl;
 516                 return;
 517         }
 518         FileStatus *filestatus = file_iter->second;
 519         assert(filestatus != NULL);
 520
 521         // Check that file hasn't already been received
 522         if (filestatus->received) {
 523                 errorstream << "Client: server sent media file that we already"
 524                         << "received, ignoring it: \"" << name << "\""
 525                         << std::endl;
 526                 return;
 527         }
 528
 529         // Mark file as received, regardless of whether loading it works and
 530         // whether the checksum matches (because at this point there is no
 531         // other server that could send a replacement)
 532         filestatus->received = true;
 533         assert(m_uncached_received_count < m_uncached_count);
 534         m_uncached_received_count++;
 535
 536         // Check that received file matches announced checksum
 537         // If so, load it
 538         checkAndLoad(name, filestatus->sha1, data, false, client);
 539 }
 540
 541 bool ClientMediaDownloader::checkAndLoad(
 542                 const std::string &name, const std::string &sha1,
 543                 const std::string &data, bool is_from_cache, Client *client)
 544 {
 545         const char *cached_or_received = is_from_cache ? "cached" : "received";
 546         const char *cached_or_received_uc = is_from_cache ? "Cached" : "Received";
 547         std::string sha1_hex = hex_encode(sha1);
 548
 549         // Compute actual checksum of data
 550         std::string data_sha1;
 551         {
 552                 SHA1 data_sha1_calculator;
 553                 data_sha1_calculator.addBytes(data.c_str(), data.size());
 554                 unsigned char *data_tmpdigest = data_sha1_calculator.getDigest();
 555                 data_sha1.assign((char*) data_tmpdigest, 20);
 556                 free(data_tmpdigest);
 557         }
 558
 559         // Check that received file matches announced checksum
 560         if (data_sha1 != sha1) {
 561                 std::string data_sha1_hex = hex_encode(data_sha1);
 562                 infostream << "Client: "
 563                         << cached_or_received_uc << " media file "
 564                         << sha1_hex << " \"" << name << "\" "
 565                         << "mismatches actual checksum " << data_sha1_hex
 566                         << std::endl;
 567                 return false;
 568         }
 569
 570         // Checksum is ok, try loading the file
 571         bool success = client->loadMedia(data, name);
 572         if (!success) {
 573                 infostream << "Client: "
 574                         << "Failed to load " << cached_or_received << " media: "
 575                         << sha1_hex << " \"" << name << "\""
 576                         << std::endl;
 577                 return false;
 578         }
 579
 580         verbosestream << "Client: "
 581                 << "Loaded " << cached_or_received << " media: "
 582                 << sha1_hex << " \"" << name << "\""
 583                 << std::endl;
 584
 585         // Update cache (unless we just loaded the file from the cache)
 586         if (!is_from_cache)
 587                 m_media_cache.update(sha1_hex, data);
 588
 589         return true;
 590 }
 591
 592
 593 /*
 594         Minetest Hashset File Format
 595
 596         All values are stored in big-endian byte order.
 597         [u32] signature: 'MTHS'
 598         [u16] version: 1
 599         For each hash in set:
 600                 [u8*20] SHA1 hash
 601
 602         Version changes:
 603         1 - Initial version
 604 */
 605
 606 std::string ClientMediaDownloader::serializeRequiredHashSet()
 607 {
 608         std::ostringstream os(std::ios::binary);
 609
 610         writeU32(os, MTHASHSET_FILE_SIGNATURE); // signature
 611         writeU16(os, 1);                        // version
 612
 613         // Write list of hashes of files that have not been
 614         // received (found in cache) yet
 615         for (std::map<std::string, FileStatus*>::iterator
 616                         it = m_files.begin();
 617                         it != m_files.end(); ++it) {
 618                 if (!it->second->received) {
 619                         assert(it->second->sha1.size() == 20);
 620                         os << it->second->sha1;
 621                 }
 622         }
 623
 624         return os.str();
 625 }
 626
 627 void ClientMediaDownloader::deSerializeHashSet(const std::string &data,
 628                 std::set<std::string> &result)
 629 {
 630         if (data.size() < 6 || data.size() % 20 != 6) {
 631                 throw SerializationError(
 632                                 "ClientMediaDownloader::deSerializeHashSet: "
 633                                 "invalid hash set file size");
 634         }
 635
 636         const u8 *data_cstr = (const u8*) data.c_str();
 637
 638         u32 signature = readU32(&data_cstr[0]);
 639         if (signature != MTHASHSET_FILE_SIGNATURE) {
 640                 throw SerializationError(
 641                                 "ClientMediaDownloader::deSerializeHashSet: "
 642                                 "invalid hash set file signature");
 643         }
 644
 645         u16 version = readU16(&data_cstr[4]);
 646         if (version != 1) {
 647                 throw SerializationError(
 648                                 "ClientMediaDownloader::deSerializeHashSet: "
 649                                 "unsupported hash set file version");
 650         }
 651
 652         for (u32 pos = 6; pos < data.size(); pos += 20) {
 653                 result.insert(data.substr(pos, 20));
 654         }
 655 }