New map directory structure that avoids map size being limited by filesystem

diff --git a/src/filesys.cpp b/src/filesys.cpp
index 287090e8a94ec068a4ba92e755330481f69887fb..8248a13d46f3703c48061faa68f3b2e150289f8e 100644 (file)
--- a/src/filesys.cpp
+++ b/src/filesys.cpp
@@ -290,5 +290,24 @@ bool RecursiveDeleteContent(std::string path)
        return true;
 }
 
+bool CreateAllDirs(std::string path)
+{
+
+       size_t pos;
+       std::vector<std::string> tocreate;
+       std::string basepath = path;
+       while(!PathExists(basepath))
+       {
+               tocreate.push_back(basepath);
+               pos = basepath.rfind('/');
+               if(pos == std::string::npos)
+                       return false;
+               basepath = basepath.substr(0,pos);
+       }
+       for(int i=tocreate.size()-1;i>=0;i--)
+               CreateDir(tocreate[i]);
+       return true;
+}
+
 } // namespace fs
 

diff --git a/src/filesys.h b/src/filesys.h
index 4dd90b84e29618a5126d0f9396768d361034ed1b..b74b34f3da6cc41f272959ca06fa44552b65f29c 100644 (file)
--- a/src/filesys.h
+++ b/src/filesys.h
@@ -38,6 +38,9 @@ std::vector<DirListNode> GetDirListing(std::string path);
 // Returns true if already exists
 bool CreateDir(std::string path);
 
+// Create all directories on the given path that don't already exist.
+bool CreateAllDirs(std::string path);
+
 bool PathExists(std::string path);
 
 // Only pass full paths to this one. True on success.

diff --git a/src/map.cpp b/src/map.cpp
index 63f01ddee0de773018e2d02481467b4eb0874a97..cd2ba9154eadac0202782697b47cf2ff5052ce92 100644 (file)
--- a/src/map.cpp
+++ b/src/map.cpp
@@ -4875,9 +4875,9 @@ plan_b:
        return (s16)level;
 }
 
-void ServerMap::createDir(std::string path)
+void ServerMap::createDirs(std::string path)
 {
-       if(fs::CreateDir(path) == false)
+       if(fs::CreateAllDirs(path) == false)
        {
                m_dout<<DTIME<<"ServerMap: Failed to create directory "
                                <<"\""<<path<<"\""<<std::endl;
@@ -4885,29 +4885,52 @@ void ServerMap::createDir(std::string path)
        }
 }
 
-std::string ServerMap::getSectorSubDir(v2s16 pos)
+std::string ServerMap::getSectorDir(v2s16 pos, int layout)
 {
        char cc[9];
-       snprintf(cc, 9, "%.4x%.4x",
-                       (unsigned int)pos.X&0xffff,
-                       (unsigned int)pos.Y&0xffff);
+       switch(layout)
+       {
+               case 1:
+                       snprintf(cc, 9, "%.4x%.4x",
+                               (unsigned int)pos.X&0xffff,
+                               (unsigned int)pos.Y&0xffff);
 
-       return std::string(cc);
-}
+                       return m_savedir + "/sectors/" + cc;
+               case 2:
+                       snprintf(cc, 9, "%.3x/%.3x",
+                               (unsigned int)pos.X&0xfff,
+                               (unsigned int)pos.Y&0xfff);
 
-std::string ServerMap::getSectorDir(v2s16 pos)
-{
-       return m_savedir + "/sectors/" + getSectorSubDir(pos);
+                       return m_savedir + "/sectors2/" + cc;
+               default:
+                       assert(false);
+       }
 }
 
 v2s16 ServerMap::getSectorPos(std::string dirname)
 {
-       if(dirname.size() != 8)
-               throw InvalidFilenameException("Invalid sector directory name");
        unsigned int x, y;
-       int r = sscanf(dirname.c_str(), "%4x%4x", &x, &y);
-       if(r != 2)
-               throw InvalidFilenameException("Invalid sector directory name");
+       int r;
+       size_t spos = dirname.rfind('/') + 1;
+       assert(spos != std::string::npos);
+       if(dirname.size() - spos == 8)
+       {
+               // Old layout
+               r = sscanf(dirname.substr(spos).c_str(), "%4x%4x", &x, &y);
+       }
+       else if(dirname.size() - spos == 3)
+       {
+               // New layout
+               r = sscanf(dirname.substr(spos-4).c_str(), "%3x/%3x", &x, &y);
+               // Sign-extend the 12 bit values up to 16 bits...
+               if(x&0x800) x|=0xF000;
+               if(y&0x800) y|=0xF000;
+       }
+       else
+       {
+               assert(false);
+       }
+       assert(r == 2);
        v2s16 pos((s16)x, (s16)y);
        return pos;
 }
@@ -5106,7 +5129,7 @@ void ServerMap::saveMapMeta()
                        <<"seed="<<m_seed<<", chunksize="<<m_chunksize
                        <<std::endl;
 
-       createDir(m_savedir);
+       createDirs(m_savedir);
        
        std::string fullpath = m_savedir + "/map_meta.txt";
        std::ofstream os(fullpath.c_str(), std::ios_base::binary);
@@ -5179,7 +5202,7 @@ void ServerMap::saveChunkMeta()
        dstream<<"INFO: ServerMap::saveChunkMeta(): Saving metadata of "
                        <<count<<" chunks"<<std::endl;
 
-       createDir(m_savedir);
+       createDirs(m_savedir);
        
        std::string fullpath = m_savedir + "/chunk_meta";
        std::ofstream os(fullpath.c_str(), std::ios_base::binary);
@@ -5267,10 +5290,8 @@ void ServerMap::saveSectorMeta(ServerMapSector *sector)
        u8 version = SER_FMT_VER_HIGHEST;
        // Get destination
        v2s16 pos = sector->getPos();
-       createDir(m_savedir);
-       createDir(m_savedir+"/sectors");
        std::string dir = getSectorDir(pos);
-       createDir(dir);
+       createDirs(dir);
        
        std::string fullpath = dir + "/meta";
        std::ofstream o(fullpath.c_str(), std::ios_base::binary);
@@ -5282,22 +5303,21 @@ void ServerMap::saveSectorMeta(ServerMapSector *sector)
        sector->differs_from_disk = false;
 }
 
-MapSector* ServerMap::loadSectorMeta(std::string dirname)
+MapSector* ServerMap::loadSectorMeta(std::string sectordir, bool save_after_load)
 {
        DSTACK(__FUNCTION_NAME);
        // Get destination
-       v2s16 p2d = getSectorPos(dirname);
-       std::string dir = m_savedir + "/sectors/" + dirname;
+       v2s16 p2d = getSectorPos(sectordir);
 
        ServerMapSector *sector = NULL;
-       
-       std::string fullpath = dir + "/meta";
+
+       std::string fullpath = sectordir + "/meta";
        std::ifstream is(fullpath.c_str(), std::ios_base::binary);
        if(is.good() == false)
        {
                // If the directory exists anyway, it probably is in some old
                // format. Just go ahead and create the sector.
-               if(fs::PathExists(dir))
+               if(fs::PathExists(sectordir))
                {
                        dstream<<"ServerMap::loadSectorMeta(): Sector metafile "
                                        <<fullpath<<" doesn't exist but directory does."
@@ -5307,12 +5327,16 @@ MapSector* ServerMap::loadSectorMeta(std::string dirname)
                        m_sectors.insert(p2d, sector);
                }
                else
+               {
                        throw FileNotGoodException("Cannot open sector metafile");
+               }
        }
        else
        {
                sector = ServerMapSector::deSerialize
                                (is, this, p2d, m_sectors);
+               if(save_after_load)
+                       saveSectorMeta(sector);
        }
        
        sector->differs_from_disk = false;
@@ -5323,14 +5347,31 @@ MapSector* ServerMap::loadSectorMeta(std::string dirname)
 bool ServerMap::loadSectorFull(v2s16 p2d)
 {
        DSTACK(__FUNCTION_NAME);
-       std::string sectorsubdir = getSectorSubDir(p2d);
 
        MapSector *sector = NULL;
 
+       // The directory layout we're going to load from.
+       //  1 - original sectors/xxxxzzzz/
+       //  2 - new sectors2/xxx/zzz/
+       //  If we load from anything but the latest structure, we will
+       //  immediately save to the new one, and remove the old.
+       int loadlayout = 1;
+       std::string sectordir1 = getSectorDir(p2d, 1);
+       std::string sectordir;
+       if(fs::PathExists(sectordir1))
+       {
+               sectordir = sectordir1;
+       }
+       else
+       {
+               loadlayout = 2;
+               sectordir = getSectorDir(p2d, 2);
+       }
+
        //JMutexAutoLock lock(m_sector_mutex); // Bulk comment-out
 
        try{
-               sector = loadSectorMeta(sectorsubdir);
+               sector = loadSectorMeta(sectordir, loadlayout != 2);
        }
        catch(InvalidFilenameException &e)
        {
@@ -5349,7 +5390,7 @@ bool ServerMap::loadSectorFull(v2s16 p2d)
                Load blocks
        */
        std::vector<fs::DirListNode> list2 = fs::GetDirListing
-                       (m_savedir+"/sectors/"+sectorsubdir);
+                       (sectordir);
        std::vector<fs::DirListNode>::iterator i2;
        for(i2=list2.begin(); i2!=list2.end(); i2++)
        {
@@ -5357,16 +5398,25 @@ bool ServerMap::loadSectorFull(v2s16 p2d)
                if(i2->dir)
                        continue;
                try{
-                       loadBlock(sectorsubdir, i2->name, sector);
+                       loadBlock(sectordir, i2->name, sector, loadlayout != 2);
                }
                catch(InvalidFilenameException &e)
                {
                        // This catches unknown crap in directory
                }
        }
+
+       if(loadlayout != 2)
+       {
+               dstream<<"Sector converted to new layout - deleting "<<
+                       sectordir1<<std::endl;
+               fs::RecursiveDelete(sectordir1);
+       }
+
        return true;
 }
 
+
 void ServerMap::saveBlock(MapBlock *block)
 {
        DSTACK(__FUNCTION_NAME);
@@ -5386,12 +5436,9 @@ void ServerMap::saveBlock(MapBlock *block)
        // Get destination
        v3s16 p3d = block->getPos();
        v2s16 p2d(p3d.X, p3d.Z);
-       createDir(m_savedir);
-       createDir(m_savedir+"/sectors");
        std::string dir = getSectorDir(p2d);
-       createDir(dir);
+       createDirs(dir);
        
-       // Block file is map/sectors/xxxxxxxx/xxxx
        char cc[5];
        snprintf(cc, 5, "%.4x", (unsigned int)p3d.Y&0xffff);
        std::string fullpath = dir + "/" + cc;
@@ -5427,12 +5474,11 @@ void ServerMap::saveBlock(MapBlock *block)
        block->resetChangedFlag();
 }
 
-void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSector *sector)
+void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSector *sector, bool save_after_load)
 {
        DSTACK(__FUNCTION_NAME);
 
-       // Block file is map/sectors/xxxxxxxx/xxxx
-       std::string fullpath = m_savedir+"/sectors/"+sectordir+"/"+blockfile;
+       std::string fullpath = sectordir+"/"+blockfile;
        try{
 
                std::ifstream is(fullpath.c_str(), std::ios_base::binary);
@@ -5496,7 +5542,7 @@ void ServerMap::loadBlock(std::string sectordir, std::string blockfile, MapSecto
                */
 
                // Save old format blocks in new format
-               if(version < SER_FMT_VER_HIGHEST)
+               if(version < SER_FMT_VER_HIGHEST || save_after_load)
                {
                        saveBlock(block);
                }

diff --git a/src/map.h b/src/map.h
index fa52dc26b33ca1de90ed7e3031e9a60410d9b003..09154547cf6240bf5bd44ea2aff0a9c39df8b2db 100644 (file)
--- a/src/map.h
+++ b/src/map.h
@@ -545,13 +545,9 @@ public:
                Misc. helper functions for fiddling with directory and file
                names when saving
        */
-       void createDir(std::string path);
-       void createSaveDir();
-       // returns something like "xxxxxxxx"
-       std::string getSectorSubDir(v2s16 pos);
+       void createDirs(std::string path);
        // returns something like "map/sectors/xxxxxxxx"
-       std::string getSectorDir(v2s16 pos);
-       std::string createSectorDir(v2s16 pos);
+       std::string getSectorDir(v2s16 pos, int layout = 2);
        // dirname: final directory name
        v2s16 getSectorPos(std::string dirname);
        v3s16 getBlockPos(std::string sectordir, std::string blockfile);
@@ -572,7 +568,7 @@ public:
        // (no MapBlocks)
        // DEPRECATED? Sectors have no metadata anymore.
        void saveSectorMeta(ServerMapSector *sector);
-       MapSector* loadSectorMeta(std::string dirname);
+       MapSector* loadSectorMeta(std::string dirname, bool save_after_load);
        
        // Full load of a sector including all blocks.
        // returns true on success, false on failure.
@@ -583,7 +579,7 @@ public:
        
        void saveBlock(MapBlock *block);
        // This will generate a sector with getSector if not found.
-       void loadBlock(std::string sectordir, std::string blockfile, MapSector *sector);
+       void loadBlock(std::string sectordir, std::string blockfile, MapSector *sector, bool save_after_load=false);
 
        // For debug printing
        virtual void PrintInfo(std::ostream &out);