Cache textures by checksum
authorJonathan Neuschäfer <j.neuschaefer@gmx.net>
Wed, 8 Feb 2012 10:49:24 +0000 (11:49 +0100)
committerPerttu Ahola <celeron55@gmail.com>
Sun, 25 Mar 2012 08:51:00 +0000 (11:51 +0300)
src/CMakeLists.txt
src/client.cpp
src/client.h
src/filecache.cpp [new file with mode: 0644]
src/filecache.h [new file with mode: 0644]
src/hex.h [new file with mode: 0644]

index fd06503834c20e172d13c4a96d47e2629e06a0a4..5fea0546d24335a8a777095df2ab2d988560b6d3 100644 (file)
@@ -238,6 +238,7 @@ set(minetest_SRCS
        guiCreateWorld.cpp
        guiConfirmMenu.cpp
        client.cpp
+       filecache.cpp
        tile.cpp
        game.cpp
        main.cpp
index 89070d66bf481f03fd8b3a9bc33ebbea0edc1bf3..3a08b25c2b1b5416c4d49f0bddfe4b9603eb7835 100644 (file)
@@ -38,6 +38,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "sha1.h"
 #include "base64.h"
 #include "clientmap.h"
+#include "filecache.h"
 #include "sound.h"
 
 static std::string getTextureCacheDir()
@@ -255,6 +256,7 @@ Client::Client(
        m_map_seed(0),
        m_password(password),
        m_access_denied(false),
+       m_texture_cache(getTextureCacheDir()),
        m_texture_receive_progress(0),
        m_textures_received(false),
        m_itemdef_received(false),
@@ -1412,7 +1414,7 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
                        //read texture from cache
                        std::string name = deSerializeString(is);
                        std::string sha1_texture = deSerializeString(is);
-                       
+
                        // if name contains illegal characters, ignore the texture
                        if(!string_allowed(name, TEXTURENAME_ALLOWED_CHARS)){
                                errorstream<<"Client: ignoring illegal texture name "
@@ -1420,74 +1422,50 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
                                continue;
                        }
 
-                       std::string tpath = getTextureCacheDir() + DIR_DELIM + name;
-                       // Read data
-                       std::ifstream fis(tpath.c_str(), std::ios_base::binary);
-
-
-                       if(fis.good() == false){
-                               infostream<<"Client::Texture not found in cache: "
-                                               <<name << " expected it at: "<<tpath<<std::endl;
-                       }
-                       else
-                       {
-                               std::ostringstream tmp_os(std::ios_base::binary);
-                               bool bad = false;
-                               for(;;){
-                                       char buf[1024];
-                                       fis.read(buf, 1024);
-                                       std::streamsize len = fis.gcount();
-                                       tmp_os.write(buf, len);
-                                       if(fis.eof())
-                                               break;
-                                       if(!fis.good()){
-                                               bad = true;
-                                               break;
-                                       }
-                               }
-                               if(bad){
-                                       infostream<<"Client: Failed to read texture from cache\""
-                                                       <<name<<"\""<<std::endl;
-                               }
-                               else {
+                       std::string sha1_decoded = base64_decode(sha1_texture);
+                       std::ostringstream tmp_os(std::ios_base::binary);
+                       bool tex_in_cache = m_texture_cache.loadByChecksum(name,
+                                       tmp_os, sha1_decoded);
+                       m_texture_name_sha1_map.set(name, sha1_decoded);
 
-                                       SHA1 sha1;
-                                       sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
+                       if(tex_in_cache) {
 
-                                       unsigned char *digest = sha1.getDigest();
+                               SHA1 sha1;
+                               sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
 
-                                       std::string digest_string = base64_encode(digest, 20);
+                               unsigned char *digest = sha1.getDigest();
 
-                                       if (digest_string == sha1_texture) {
-                                               // Silly irrlicht's const-incorrectness
-                                               Buffer<char> data_rw(tmp_os.str().c_str(), tmp_os.str().size());
+                               std::string digest_string = base64_encode(digest, 20);
 
-                                               // Create an irrlicht memory file
-                                               io::IReadFile *rfile = irrfs->createMemoryReadFile(
-                                                               *data_rw,  tmp_os.str().size(), "_tempreadfile");
-                                               assert(rfile);
-                                               // Read image
-                                               video::IImage *img = vdrv->createImageFromFile(rfile);
-                                               if(!img){
-                                                       infostream<<"Client: Cannot create image from data of "
-                                                                       <<"received texture \""<<name<<"\""<<std::endl;
-                                                       rfile->drop();
-                                               }
-                                               else {
-                                                       m_tsrc->insertSourceImage(name, img);
-                                                       img->drop();
-                                                       rfile->drop();
+                               if (digest_string == sha1_texture) {
+                                       // Silly irrlicht's const-incorrectness
+                                       Buffer<char> data_rw(tmp_os.str().c_str(), tmp_os.str().size());
 
-                                                       texture_found = true;
-                                               }
+                                       // Create an irrlicht memory file
+                                       io::IReadFile *rfile = irrfs->createMemoryReadFile(
+                                                       *data_rw,  tmp_os.str().size(), "_tempreadfile");
+                                       assert(rfile);
+                                       // Read image
+                                       video::IImage *img = vdrv->createImageFromFile(rfile);
+                                       if(!img){
+                                               infostream<<"Client: Cannot create image from data of "
+                                                               <<"received texture \""<<name<<"\""<<std::endl;
+                                               rfile->drop();
                                        }
                                        else {
-                                               infostream<<"Client::Texture cached sha1 hash not matching server hash: "
-                                                               <<name << ": server ->"<<sha1_texture <<" client -> "<<digest_string<<std::endl;
-                                       }
+                                               m_tsrc->insertSourceImage(name, img);
+                                               img->drop();
+                                               rfile->drop();
 
-                                       free(digest);
+                                               texture_found = true;
+                                       }
                                }
+                               else {
+                                       infostream<<"Client::Texture cached sha1 hash not matching server hash: "
+                                                       <<name << ": server ->"<<sha1_texture <<" client -> "<<digest_string<<std::endl;
+                               }
+
+                               free(digest);
                        }
 
                        //add texture request
@@ -1598,15 +1576,15 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
 
                        fs::CreateAllDirs(getTextureCacheDir());
 
-                       std::string filename = getTextureCacheDir() + DIR_DELIM + name;
-                       std::ofstream outfile(filename.c_str(), std::ios_base::binary | std::ios_base::trunc);
-
-                       if (outfile.good()) {
-                               outfile.write(data.c_str(),data.length());
-                               outfile.close();
-                       }
-                       else {
-                               errorstream<<"Client: Unable to open cached texture file "<< filename <<std::endl;
+                       {
+                               core::map<std::string, std::string>::Node *n;
+                               n = m_texture_name_sha1_map.find(name);
+                               if(n == NULL)
+                                       errorstream<<"The server sent a texture that has not been announced."
+                                               <<std::endl;
+                               else
+                                       m_texture_cache.updateByChecksum(name,
+                                                       data, n->getValue());
                        }
 
                        m_tsrc->insertSourceImage(name, img);
@@ -2382,6 +2360,10 @@ void Client::afterContentReceived()
        assert(m_nodedef_received);
        assert(m_textures_received);
 
+       // remove the information about which checksum each texture
+       // ought to have
+       m_texture_name_sha1_map.clear();
+
        // Rebuild inherited images and recreate textures
        m_tsrc->rebuildImagesAndTextures();
 
index 3a47a08f6810c792776b5461381932c44a7c7c1c..8d7597e807846a2884d7e0ed4973ff3919862b56 100644 (file)
@@ -34,6 +34,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #include "gamedef.h"
 #include "inventorymanager.h"
 #include "filesys.h"
+#include "filecache.h"
 
 struct MeshMakeData;
 class MapBlockMesh;
@@ -366,6 +367,10 @@ private:
        bool m_access_denied;
        std::wstring m_access_denied_reason;
        Queue<ClientEvent> m_client_event_queue;
+       FileCache m_texture_cache;
+       // a map of the name and SHA1 checksum of each texture;
+       // cleared after content has been recieved
+       core::map<std::string, std::string> m_texture_name_sha1_map;
        float m_texture_receive_progress;
        bool m_textures_received;
        bool m_itemdef_received;
diff --git a/src/filecache.cpp b/src/filecache.cpp
new file mode 100644 (file)
index 0000000..28d6bbc
--- /dev/null
@@ -0,0 +1,117 @@
+/*
+Minetest-c55
+Copyright (C) 2010 celeron55, Perttu Ahola <celeron55@gmail.com>
+Copyright (C) 2012 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "filecache.h"
+#include "clientserver.h"
+#include "log.h"
+#include "filesys.h"
+#include "utility.h"
+#include "hex.h"
+
+#include <string>
+#include <iostream>
+
+bool FileCache::loadByPath(const std::string &name, std::ostream &os,
+               const std::string &path)
+{
+       std::ifstream fis(path.c_str(), std::ios_base::binary);
+
+       if(!fis.good()){
+               infostream<<"FileCache: File not found in cache: "
+                       <<name << " expected it at: "<<path<<std::endl;
+               return false;
+       }
+
+       bool bad = false;
+       for(;;){
+               char buf[1024];
+               fis.read(buf, 1024);
+               std::streamsize len = fis.gcount();
+               os.write(buf, len);
+               if(fis.eof())
+                       break;
+               if(!fis.good()){
+                       bad = true;
+                       break;
+               }
+       }
+       if(bad){
+               infostream<<"FileCache: Failed to read file from cache: \""
+                       <<path<<"\""<<std::endl;
+       }
+
+       return !bad;
+}
+
+bool FileCache::updateByPath(const std::string &name, const std::string &data,
+               const std::string &path)
+{
+       std::ofstream file(path.c_str(), std::ios_base::binary |
+                       std::ios_base::trunc);
+
+       if(!file.good())
+       {
+               errorstream<<"FileCache: Can't write to file at "
+                       <<path<<std::endl;
+               return false;
+       }
+
+       file.write(data.c_str(), data.length());
+       file.close();
+
+       return !file.fail();
+}
+
+bool FileCache::loadByName(const std::string &name, std::ostream &os)
+{
+       std::string path = m_dir + DIR_DELIM + name;
+       return loadByPath(name, os, path);
+}
+
+
+bool FileCache::updateByName(const std::string &name, const std::string &data)
+{
+       std::string path = m_dir + DIR_DELIM + name;
+       return updateByPath(name, data, path);
+}
+
+std::string FileCache::getPathFromChecksum(const std::string &name,
+               const std::string &checksum)
+{
+       std::string checksum_hex = hex_encode(checksum.c_str(), checksum.length());
+       size_t dot = name.find_last_of('.');;
+       std::string ext = (dot == std::string::npos)? "" :
+               name.substr(dot, std::string::npos);
+       return m_dir + DIR_DELIM + checksum_hex + ext;
+}
+
+bool FileCache::loadByChecksum(const std::string &name, std::ostream &os,
+               const std::string &checksum)
+{
+       std::string path = getPathFromChecksum(name, checksum);
+       return loadByPath(name, os, path);
+}
+
+bool FileCache::updateByChecksum(const std::string &name,
+               const std::string &data, const std::string &checksum)
+{
+       std::string path = getPathFromChecksum(name, checksum);
+       return updateByPath(name, data, path);
+}
diff --git a/src/filecache.h b/src/filecache.h
new file mode 100644 (file)
index 0000000..35ae9aa
--- /dev/null
@@ -0,0 +1,79 @@
+/*
+Minetest-c55
+Copyright (C) 2010 celeron55, Perttu Ahola <celeron55@gmail.com>
+Copyright (C) 2012 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef FILECACHE_HEADER
+#define FILECACHE_HEADER
+
+#include <string>
+#include <iostream>
+
+class FileCache
+{
+public:
+       /*
+               'dir' is the file cache directory to use.
+       */
+       FileCache(std::string dir):
+               m_dir(dir)
+       {
+       }
+
+       /*
+               Searches the cache for a file with a given name.
+               If the file is found, lookup copies it into 'os' and
+               returns true. Otherwise false is returned.
+       */
+       bool loadByName(const std::string &name, std::ostream &os);
+
+       /*
+               Stores a file in the cache based on its name.
+               Returns true on success, false otherwise.
+       */
+       bool updateByName(const std::string &name, const std::string &data);
+
+       /*
+               Loads a file based on a check sum, which may be any kind of
+               rather unique byte sequence. Returns true, if the file could
+               be written into os, false otherwise.
+               A file name is required to give the disk file a name that
+               has the right file name extension (e.g. ".png").
+       */
+       bool loadByChecksum(const std::string &name, std::ostream &os,
+                       const std::string &checksum);
+
+       /*
+               Stores a file in the cache based on its checksum.
+               Returns true on success, false otherwise.
+       */
+       bool updateByChecksum(const std::string &name, const std::string &data,
+                       const std::string &checksum);
+
+private:
+       std::string m_dir;
+
+       bool loadByPath(const std::string &name, std::ostream &os,
+                       const std::string &path);
+       bool updateByPath(const std::string &name, const std::string &data,
+                       const std::string &path);
+       std::string getPathFromChecksum(const std::string &name,
+                       const std::string &checksum);
+};
+
+#endif
diff --git a/src/hex.h b/src/hex.h
new file mode 100644 (file)
index 0000000..1afb875
--- /dev/null
+++ b/src/hex.h
@@ -0,0 +1,44 @@
+/*
+Minetest-c55
+Copyright (C) 2012 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef HEX_HEADER
+#define HEX_HEADER
+
+#include <string>
+
+static const char hex_chars[] = "0123456789abcdef";
+
+static std::string hex_encode(const char *data, unsigned int data_size)
+{
+       std::string ret;
+       char buf2[3];
+       buf2[2] = '\0';
+
+       for(unsigned int i = 0; i < data_size; i++)
+       {
+               unsigned char c = (unsigned char) data[i];
+               buf2[0] = hex_chars[(c & 0xf0) >> 4];
+               buf2[1] = hex_chars[c & 0x0f];
+               ret.append(buf2);
+       }
+
+       return ret;
+}
+
+#endif