From: sfan5 Date: Sat, 13 Jun 2020 17:03:26 +0000 (+0200) Subject: Server pushing media at runtime (#9961) X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=2424dfe007e451bb02f87884c2b272cf307d6e7c;p=oweals%2Fminetest.git Server pushing media at runtime (#9961) --- diff --git a/doc/lua_api.txt b/doc/lua_api.txt index ed060c4ad..cb968958f 100644 --- a/doc/lua_api.txt +++ b/doc/lua_api.txt @@ -5217,6 +5217,20 @@ Server * Returns a code (0: successful, 1: no such player, 2: player is connected) * `minetest.remove_player_auth(name)`: remove player authentication data * Returns boolean indicating success (false if player nonexistant) +* `minetest.dynamic_add_media(filepath)` + * Adds the file at the given path to the media sent to clients by the server + on startup and also pushes this file to already connected clients. + The file must be a supported image, sound or model format. It must not be + modified, deleted, moved or renamed after calling this function. + The list of dynamically added media is not persisted. + * Returns boolean indicating success (duplicate files count as error) + * The media will be ready to use (in e.g. entity textures, sound_play) + immediately after calling this function. + Old clients that lack support for this feature will not see the media + unless they reconnect to the server. + * Since media transferred this way does not use client caching or HTTP + transfers, dynamic media should not be used with big files or performance + will suffer. Bans ---- diff --git a/src/client/client.cpp b/src/client/client.cpp index c03c062c6..34f97a9de 100644 --- a/src/client/client.cpp +++ b/src/client/client.cpp @@ -670,11 +670,9 @@ void Client::step(float dtime) } } -bool Client::loadMedia(const std::string &data, const std::string &filename) +bool Client::loadMedia(const std::string &data, const std::string &filename, + bool from_media_push) { - // Silly irrlicht's const-incorrectness - Buffer data_rw(data.c_str(), data.size()); - std::string name; const char *image_ext[] = { @@ -690,6 +688,9 @@ bool Client::loadMedia(const std::string &data, const std::string &filename) io::IFileSystem *irrfs = RenderingEngine::get_filesystem(); video::IVideoDriver *vdrv = RenderingEngine::get_video_driver(); + // Silly irrlicht's const-incorrectness + Buffer data_rw(data.c_str(), data.size()); + // Create an irrlicht memory file io::IReadFile *rfile = irrfs->createMemoryReadFile( *data_rw, data_rw.getSize(), "_tempreadfile"); @@ -727,7 +728,6 @@ bool Client::loadMedia(const std::string &data, const std::string &filename) ".x", ".b3d", ".md2", ".obj", NULL }; - name = removeStringEnd(filename, model_ext); if (!name.empty()) { verbosestream<<"Client: Storing model into memory: " @@ -744,6 +744,8 @@ bool Client::loadMedia(const std::string &data, const std::string &filename) }; name = removeStringEnd(filename, translate_ext); if (!name.empty()) { + if (from_media_push) + return false; TRACESTREAM(<< "Client: Loading translation: " << "\"" << filename << "\"" << std::endl); g_client_translations->loadTranslation(data); diff --git a/src/client/client.h b/src/client/client.h index 3b1095ac2..733634db1 100644 --- a/src/client/client.h +++ b/src/client/client.h @@ -222,6 +222,7 @@ public: void handleCommand_FormspecPrepend(NetworkPacket *pkt); void handleCommand_CSMRestrictionFlags(NetworkPacket *pkt); void handleCommand_PlayerSpeed(NetworkPacket *pkt); + void handleCommand_MediaPush(NetworkPacket *pkt); void ProcessData(NetworkPacket *pkt); @@ -376,7 +377,8 @@ public: // The following set of functions is used by ClientMediaDownloader // Insert a media file appropriately into the appropriate manager - bool loadMedia(const std::string &data, const std::string &filename); + bool loadMedia(const std::string &data, const std::string &filename, + bool from_media_push = false); // Send a request for conventional media transfer void request_media(const std::vector &file_requests); @@ -488,6 +490,7 @@ private: Camera *m_camera = nullptr; Minimap *m_minimap = nullptr; bool m_minimap_disabled_by_server = false; + // Server serialization version u8 m_server_ser_ver; @@ -529,7 +532,6 @@ private: AuthMechanism m_chosen_auth_mech; void *m_auth_data = nullptr; - bool m_access_denied = false; bool m_access_denied_reconnect = false; std::string m_access_denied_reason = ""; @@ -538,7 +540,10 @@ private: bool m_nodedef_received = false; bool m_activeobjects_received = false; bool m_mods_loaded = false; + ClientMediaDownloader *m_media_downloader; + // Set of media filenames pushed by server at runtime + std::unordered_set m_media_pushed_files; // time_of_day speed approximation for old protocol bool m_time_of_day_set = false; diff --git a/src/client/clientmedia.cpp b/src/client/clientmedia.cpp index 6da99bbbf..8cd3b6bcc 100644 --- a/src/client/clientmedia.cpp +++ b/src/client/clientmedia.cpp @@ -35,6 +35,15 @@ static std::string getMediaCacheDir() return porting::path_cache + DIR_DELIM + "media"; } +bool clientMediaUpdateCache(const std::string &raw_hash, const std::string &filedata) +{ + FileCache media_cache(getMediaCacheDir()); + std::string sha1_hex = hex_encode(raw_hash); + if (!media_cache.exists(sha1_hex)) + return media_cache.update(sha1_hex, filedata); + return true; +} + /* ClientMediaDownloader */ @@ -559,7 +568,6 @@ bool ClientMediaDownloader::checkAndLoad( return true; } - /* Minetest Hashset File Format diff --git a/src/client/clientmedia.h b/src/client/clientmedia.h index 92831082c..5a918535b 100644 --- a/src/client/clientmedia.h +++ b/src/client/clientmedia.h @@ -33,6 +33,11 @@ struct HTTPFetchResult; #define MTHASHSET_FILE_SIGNATURE 0x4d544853 // 'MTHS' #define MTHASHSET_FILE_NAME "index.mth" +// Store file into media cache (unless it exists already) +// Validating the hash is responsibility of the caller +bool clientMediaUpdateCache(const std::string &raw_hash, + const std::string &filedata); + class ClientMediaDownloader { public: diff --git a/src/client/filecache.cpp b/src/client/filecache.cpp index 3d1b302a8..46bbe4059 100644 --- a/src/client/filecache.cpp +++ b/src/client/filecache.cpp @@ -82,8 +82,16 @@ bool FileCache::update(const std::string &name, const std::string &data) std::string path = m_dir + DIR_DELIM + name; return updateByPath(path, data); } + bool FileCache::load(const std::string &name, std::ostream &os) { std::string path = m_dir + DIR_DELIM + name; return loadByPath(path, os); } + +bool FileCache::exists(const std::string &name) +{ + std::string path = m_dir + DIR_DELIM + name; + std::ifstream fis(path.c_str(), std::ios_base::binary); + return fis.good(); +} diff --git a/src/client/filecache.h b/src/client/filecache.h index 96e4c8ba1..ea6afc4b2 100644 --- a/src/client/filecache.h +++ b/src/client/filecache.h @@ -33,6 +33,7 @@ public: bool update(const std::string &name, const std::string &data); bool load(const std::string &name, std::ostream &os); + bool exists(const std::string &name); private: std::string m_dir; diff --git a/src/filesys.cpp b/src/filesys.cpp index f61b39b94..0bc351669 100644 --- a/src/filesys.cpp +++ b/src/filesys.cpp @@ -691,6 +691,12 @@ std::string AbsolutePath(const std::string &path) const char *GetFilenameFromPath(const char *path) { const char *filename = strrchr(path, DIR_DELIM_CHAR); + // Consistent with IsDirDelimiter this function handles '/' too + if (DIR_DELIM_CHAR != '/') { + const char *tmp = strrchr(path, '/'); + if (tmp && tmp > filename) + filename = tmp; + } return filename ? filename + 1 : path; } diff --git a/src/network/clientopcodes.cpp b/src/network/clientopcodes.cpp index 0f20047c0..f812a08a1 100644 --- a/src/network/clientopcodes.cpp +++ b/src/network/clientopcodes.cpp @@ -68,7 +68,7 @@ const ToClientCommandHandler toClientCommandTable[TOCLIENT_NUM_MSG_TYPES] = { "TOCLIENT_TIME_OF_DAY", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_TimeOfDay }, // 0x29 { "TOCLIENT_CSM_RESTRICTION_FLAGS", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_CSMRestrictionFlags }, // 0x2A { "TOCLIENT_PLAYER_SPEED", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_PlayerSpeed }, // 0x2B - null_command_handler, + { "TOCLIENT_MEDIA_PUSH", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_MediaPush }, // 0x2C null_command_handler, null_command_handler, { "TOCLIENT_CHAT_MESSAGE", TOCLIENT_STATE_CONNECTED, &Client::handleCommand_ChatMessage }, // 0x2F diff --git a/src/network/clientpackethandler.cpp b/src/network/clientpackethandler.cpp index e000acc92..5934eaf8c 100644 --- a/src/network/clientpackethandler.cpp +++ b/src/network/clientpackethandler.cpp @@ -39,6 +39,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "script/scripting_client.h" #include "util/serialize.h" #include "util/srp.h" +#include "util/sha1.h" #include "tileanimation.h" #include "gettext.h" #include "skyparams.h" @@ -1471,6 +1472,51 @@ void Client::handleCommand_PlayerSpeed(NetworkPacket *pkt) player->addVelocity(added_vel); } +void Client::handleCommand_MediaPush(NetworkPacket *pkt) +{ + std::string raw_hash, filename, filedata; + bool cached; + + *pkt >> raw_hash >> filename >> cached; + filedata = pkt->readLongString(); + + if (raw_hash.size() != 20 || filedata.empty() || filename.empty() || + !string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) { + throw PacketError("Illegal filename, data or hash"); + } + + verbosestream << "Server pushes media file \"" << filename << "\" with " + << filedata.size() << " bytes of data (cached=" << cached + << ")" << std::endl; + + if (m_media_pushed_files.count(filename) != 0) { + // Silently ignore for synchronization purposes + return; + } + + // Compute and check checksum of data + std::string computed_hash; + { + SHA1 ctx; + ctx.addBytes(filedata.c_str(), filedata.size()); + unsigned char *buf = ctx.getDigest(); + computed_hash.assign((char*) buf, 20); + free(buf); + } + if (raw_hash != computed_hash) { + verbosestream << "Hash of file data mismatches, ignoring." << std::endl; + return; + } + + // Actually load media + loadMedia(filedata, filename, true); + m_media_pushed_files.insert(filename); + + // Cache file for the next time when this client joins the same server + if (cached) + clientMediaUpdateCache(raw_hash, filedata); +} + /* * Mod channels */ diff --git a/src/network/networkprotocol.h b/src/network/networkprotocol.h index ab924f1db..fd683eac9 100644 --- a/src/network/networkprotocol.h +++ b/src/network/networkprotocol.h @@ -323,6 +323,15 @@ enum ToClientCommand v3f added_vel */ + TOCLIENT_MEDIA_PUSH = 0x2C, + /* + std::string raw_hash + std::string filename + bool should_be_cached + u32 len + char filedata[len] + */ + // (oops, there is some gap here) TOCLIENT_CHAT_MESSAGE = 0x2F, diff --git a/src/network/serveropcodes.cpp b/src/network/serveropcodes.cpp index 6ee4ff256..2fc3197c2 100644 --- a/src/network/serveropcodes.cpp +++ b/src/network/serveropcodes.cpp @@ -167,7 +167,7 @@ const ClientCommandFactory clientCommandFactoryTable[TOCLIENT_NUM_MSG_TYPES] = { "TOCLIENT_TIME_OF_DAY", 0, true }, // 0x29 { "TOCLIENT_CSM_RESTRICTION_FLAGS", 0, true }, // 0x2A { "TOCLIENT_PLAYER_SPEED", 0, true }, // 0x2B - null_command_factory, // 0x2C + { "TOCLIENT_MEDIA_PUSH", 0, true }, // 0x2C (sent over channel 1 too) null_command_factory, // 0x2D null_command_factory, // 0x2E { "TOCLIENT_CHAT_MESSAGE", 0, true }, // 0x2F diff --git a/src/script/lua_api/l_server.cpp b/src/script/lua_api/l_server.cpp index b6754938e..6f934bb9d 100644 --- a/src/script/lua_api/l_server.cpp +++ b/src/script/lua_api/l_server.cpp @@ -22,6 +22,7 @@ with this program; if not, write to the Free Software Foundation, Inc., #include "common/c_converter.h" #include "common/c_content.h" #include "cpp_api/s_base.h" +#include "cpp_api/s_security.h" #include "server.h" #include "environment.h" #include "remoteplayer.h" @@ -412,9 +413,6 @@ int ModApiServer::l_get_modnames(lua_State *L) std::vector modlist; getServer(L)->getModNames(modlist); - // Take unsorted items from mods_unsorted and sort them into - // mods_sorted; not great performance but the number of mods on a - // server will likely be small. std::sort(modlist.begin(), modlist.end()); // Package them up for Lua @@ -474,6 +472,23 @@ int ModApiServer::l_sound_fade(lua_State *L) return 0; } +// dynamic_add_media(filepath) +int ModApiServer::l_dynamic_add_media(lua_State *L) +{ + NO_MAP_LOCK_REQUIRED; + + // Reject adding media before the server has started up + if (!getEnv(L)) + throw LuaError("Dynamic media cannot be added before server has started up"); + + std::string filepath = readParam(L, 1); + CHECK_SECURE_PATH(L, filepath.c_str(), false); + + bool ok = getServer(L)->dynamicAddMedia(filepath); + lua_pushboolean(L, ok); + return 1; +} + // is_singleplayer() int ModApiServer::l_is_singleplayer(lua_State *L) { @@ -538,6 +553,7 @@ void ModApiServer::Initialize(lua_State *L, int top) API_FCT(sound_play); API_FCT(sound_stop); API_FCT(sound_fade); + API_FCT(dynamic_add_media); API_FCT(get_player_information); API_FCT(get_player_privs); diff --git a/src/script/lua_api/l_server.h b/src/script/lua_api/l_server.h index 3aa1785a2..938bfa8ef 100644 --- a/src/script/lua_api/l_server.h +++ b/src/script/lua_api/l_server.h @@ -70,6 +70,9 @@ private: // sound_fade(handle, step, gain) static int l_sound_fade(lua_State *L); + // dynamic_add_media(filepath) + static int l_dynamic_add_media(lua_State *L); + // get_player_privs(name, text) static int l_get_player_privs(lua_State *L); diff --git a/src/server.cpp b/src/server.cpp index 6ecbd7097..fe2bb3840 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -2405,9 +2405,87 @@ bool Server::SendBlock(session_t peer_id, const v3s16 &blockpos) return true; } +bool Server::addMediaFile(const std::string &filename, + const std::string &filepath, std::string *filedata_to, + std::string *digest_to) +{ + // If name contains illegal characters, ignore the file + if (!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) { + infostream << "Server: ignoring illegal file name: \"" + << filename << "\"" << std::endl; + return false; + } + // If name is not in a supported format, ignore it + const char *supported_ext[] = { + ".png", ".jpg", ".bmp", ".tga", + ".pcx", ".ppm", ".psd", ".wal", ".rgb", + ".ogg", + ".x", ".b3d", ".md2", ".obj", + // Custom translation file format + ".tr", + NULL + }; + if (removeStringEnd(filename, supported_ext).empty()) { + infostream << "Server: ignoring unsupported file extension: \"" + << filename << "\"" << std::endl; + return false; + } + // Ok, attempt to load the file and add to cache + + // Read data + std::ifstream fis(filepath.c_str(), std::ios_base::binary); + if (!fis.good()) { + errorstream << "Server::addMediaFile(): Could not open \"" + << filename << "\" for reading" << std::endl; + return false; + } + std::string filedata; + bool bad = false; + for (;;) { + char buf[1024]; + fis.read(buf, sizeof(buf)); + std::streamsize len = fis.gcount(); + filedata.append(buf, len); + if (fis.eof()) + break; + if (!fis.good()) { + bad = true; + break; + } + } + if (bad) { + errorstream << "Server::addMediaFile(): Failed to read \"" + << filename << "\"" << std::endl; + return false; + } else if (filedata.empty()) { + errorstream << "Server::addMediaFile(): Empty file \"" + << filepath << "\"" << std::endl; + return false; + } + + SHA1 sha1; + sha1.addBytes(filedata.c_str(), filedata.length()); + + unsigned char *digest = sha1.getDigest(); + std::string sha1_base64 = base64_encode(digest, 20); + std::string sha1_hex = hex_encode((char*) digest, 20); + if (digest_to) + *digest_to = std::string((char*) digest, 20); + free(digest); + + // Put in list + m_media[filename] = MediaInfo(filepath, sha1_base64); + verbosestream << "Server: " << sha1_hex << " is " << filename + << std::endl; + + if (filedata_to) + *filedata_to = std::move(filedata); + return true; +} + void Server::fillMediaCache() { - infostream<<"Server: Calculating media file checksums"< paths; @@ -2419,80 +2497,15 @@ void Server::fillMediaCache() for (const std::string &mediapath : paths) { std::vector dirlist = fs::GetDirListing(mediapath); for (const fs::DirListNode &dln : dirlist) { - if (dln.dir) // Ignode dirs - continue; - std::string filename = dln.name; - // If name contains illegal characters, ignore the file - if (!string_allowed(filename, TEXTURENAME_ALLOWED_CHARS)) { - infostream<<"Server: ignoring illegal file name: \"" - << filename << "\"" << std::endl; - continue; - } - // If name is not in a supported format, ignore it - const char *supported_ext[] = { - ".png", ".jpg", ".bmp", ".tga", - ".pcx", ".ppm", ".psd", ".wal", ".rgb", - ".ogg", - ".x", ".b3d", ".md2", ".obj", - // Custom translation file format - ".tr", - NULL - }; - if (removeStringEnd(filename, supported_ext).empty()){ - infostream << "Server: ignoring unsupported file extension: \"" - << filename << "\"" << std::endl; + if (dln.dir) // Ignore dirs continue; - } - // Ok, attempt to load the file and add to cache - std::string filepath; - filepath.append(mediapath).append(DIR_DELIM).append(filename); - - // Read data - std::ifstream fis(filepath.c_str(), std::ios_base::binary); - if (!fis.good()) { - errorstream << "Server::fillMediaCache(): Could not open \"" - << filename << "\" for reading" << std::endl; - continue; - } - std::ostringstream tmp_os(std::ios_base::binary); - bool bad = false; - for(;;) { - char buf[1024]; - fis.read(buf, 1024); - std::streamsize len = fis.gcount(); - tmp_os.write(buf, len); - if (fis.eof()) - break; - if (!fis.good()) { - bad = true; - break; - } - } - if(bad) { - errorstream<<"Server::fillMediaCache(): Failed to read \"" - << filename << "\"" << std::endl; - continue; - } - if(tmp_os.str().length() == 0) { - errorstream << "Server::fillMediaCache(): Empty file \"" - << filepath << "\"" << std::endl; - continue; - } - - SHA1 sha1; - sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length()); - - unsigned char *digest = sha1.getDigest(); - std::string sha1_base64 = base64_encode(digest, 20); - std::string sha1_hex = hex_encode((char*)digest, 20); - free(digest); - - // Put in list - m_media[filename] = MediaInfo(filepath, sha1_base64); - verbosestream << "Server: " << sha1_hex << " is " << filename - << std::endl; + std::string filepath = mediapath; + filepath.append(DIR_DELIM).append(dln.name); + addMediaFile(dln.name, filepath); } } + + infostream << "Server: " << m_media.size() << " media files collected" << std::endl; } void Server::sendMediaAnnouncement(session_t peer_id, const std::string &lang_code) @@ -3428,6 +3441,44 @@ void Server::deleteParticleSpawner(const std::string &playername, u32 id) SendDeleteParticleSpawner(peer_id, id); } +bool Server::dynamicAddMedia(const std::string &filepath) +{ + std::string filename = fs::GetFilenameFromPath(filepath.c_str()); + if (m_media.find(filename) != m_media.end()) { + errorstream << "Server::dynamicAddMedia(): file \"" << filename + << "\" already exists in media cache" << std::endl; + return false; + } + + // Load the file and add it to our media cache + std::string filedata, raw_hash; + bool ok = addMediaFile(filename, filepath, &filedata, &raw_hash); + if (!ok) + return false; + + // Push file to existing clients + NetworkPacket pkt(TOCLIENT_MEDIA_PUSH, 0); + pkt << raw_hash << filename << (bool) true; + pkt.putLongString(filedata); + + auto client_ids = m_clients.getClientIDs(CS_DefinitionsSent); + for (session_t client_id : client_ids) { + /* + The network layer only guarantees ordered delivery inside a channel. + Since the very next packet could be one that uses the media, we have + to push the media over ALL channels to ensure it is processed before + it is used. + In practice this means we have to send it twice: + - channel 1 (HUD) + - channel 0 (everything else: e.g. play_sound, object messages) + */ + m_clients.send(client_id, 1, &pkt, true); + m_clients.send(client_id, 0, &pkt, true); + } + + return true; +} + // actions: time-reversed list // Return value: success/failure bool Server::rollbackRevertActions(const std::list &actions, diff --git a/src/server.h b/src/server.h index 27943cc29..f44716531 100644 --- a/src/server.h +++ b/src/server.h @@ -236,6 +236,8 @@ public: void deleteParticleSpawner(const std::string &playername, u32 id); + bool dynamicAddMedia(const std::string &filepath); + ServerInventoryManager *getInventoryMgr() const { return m_inventory_mgr.get(); } void sendDetachedInventory(Inventory *inventory, const std::string &name, session_t peer_id); @@ -435,6 +437,8 @@ private: // Sends blocks to clients (locks env and con on its own) void SendBlocks(float dtime); + bool addMediaFile(const std::string &filename, const std::string &filepath, + std::string *filedata = nullptr, std::string *digest = nullptr); void fillMediaCache(); void sendMediaAnnouncement(session_t peer_id, const std::string &lang_code); void sendRequestedMedia(session_t peer_id,