Add redis database backend
authorSfan5 <sfan5@live.de>
Tue, 8 Apr 2014 19:39:21 +0000 (21:39 +0200)
committerSfan5 <sfan5@live.de>
Wed, 16 Apr 2014 20:05:06 +0000 (22:05 +0200)
src/CMakeLists.txt
src/cmake_config.h.in
src/config.h
src/database-redis.cpp [new file with mode: 0644]
src/database-redis.h [new file with mode: 0644]
src/main.cpp
src/map.cpp

index 24bbe7e1d688950ad22afce93a1a6c047f25278d..6836ad6eb19992092ab7f41c9205689cecd7f184 100644 (file)
@@ -291,6 +291,25 @@ if(ENABLE_LEVELDB)
        endif(LEVELDB_LIBRARY AND LEVELDB_INCLUDE_DIR)
 endif(ENABLE_LEVELDB)
 
+set(USE_REDIS 0)
+
+OPTION(ENABLE_REDIS "Enable redis backend" 1)
+
+if(ENABLE_REDIS)
+       find_library(REDIS_LIBRARY hiredis)
+       find_path(REDIS_INCLUDE_DIR hiredis.h PATH_SUFFIXES hiredis)
+       message(STATUS "redis library: ${REDIS_LIBRARY}")
+       message(STATUS "redis headers: ${REDIS_INCLUDE_DIR}")
+       if(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
+               set(USE_REDIS 1)
+               message(STATUS "redis backend enabled")
+               include_directories(${REDIS_INCLUDE_DIR})
+       else(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
+               set(USE_REDIS 0)
+               message(STATUS "redis not found!")
+       endif(REDIS_LIBRARY AND REDIS_INCLUDE_DIR)
+endif(ENABLE_REDIS)
+
 configure_file(
        "${PROJECT_SOURCE_DIR}/cmake_config.h.in"
        "${PROJECT_BINARY_DIR}/cmake_config.h"
@@ -368,6 +387,7 @@ set(common_SRCS
        database-dummy.cpp
        database-leveldb.cpp
        database-sqlite3.cpp
+       database-redis.cpp
        player.cpp
        test.cpp
        sha1.cpp
@@ -533,6 +553,9 @@ if(BUILD_CLIENT)
        if (USE_LEVELDB)
                target_link_libraries(${PROJECT_NAME} ${LEVELDB_LIBRARY})
        endif(USE_LEVELDB)
+       if (USE_REDIS)
+               target_link_libraries(${PROJECT_NAME} ${REDIS_LIBRARY})
+       endif(USE_REDIS)
 endif(BUILD_CLIENT)
 
 if(BUILD_SERVER)
@@ -550,6 +573,9 @@ if(BUILD_SERVER)
        if (USE_LEVELDB)
                target_link_libraries(${PROJECT_NAME}server ${LEVELDB_LIBRARY})
        endif(USE_LEVELDB)
+       if (USE_REDIS)
+               target_link_libraries(${PROJECT_NAME}server ${REDIS_LIBRARY})
+       endif(USE_REDIS)
        if(USE_CURL)
                target_link_libraries(
                        ${PROJECT_NAME}server
index 17d3c2249d51c84bae71a0575192b52dae19c4b2..906b3e47d163ac1a3c1e4496227ad13f58f89e2d 100644 (file)
@@ -14,6 +14,7 @@
 #define CMAKE_STATIC_SHAREDIR "@SHAREDIR@"
 #define CMAKE_USE_LEVELDB @USE_LEVELDB@
 #define CMAKE_USE_LUAJIT @USE_LUAJIT@
+#define CMAKE_USE_REDIS @USE_REDIS@
 #define CMAKE_VERSION_MAJOR @VERSION_MAJOR@
 #define CMAKE_VERSION_MINOR @VERSION_MINOR@
 #define CMAKE_VERSION_PATCH @VERSION_PATCH@
index 1558ebe09777072b7dc66dffbf473515a406063f..8a9d7d63df1b6d8586d0e3fa27bb822ccae4e7df 100644 (file)
        #define USE_LUAJIT 0
 #endif
 
+#ifndef USE_REDIS
+       #define USE_REDIS 0
+#endif
+
 #ifdef USE_CMAKE_CONFIG_H
        #include "cmake_config.h"
        #undef PROJECT_NAME
@@ -48,6 +52,8 @@
        #define USE_LEVELDB CMAKE_USE_LEVELDB
        #undef USE_LUAJIT
        #define USE_LUAJIT CMAKE_USE_LUAJIT
+       #undef USE_REDIS
+       #define USE_REDIS CMAKE_USE_REDIS
        #undef VERSION_MAJOR
        #define VERSION_MAJOR CMAKE_VERSION_MAJOR
        #undef VERSION_MINOR
diff --git a/src/database-redis.cpp b/src/database-redis.cpp
new file mode 100644 (file)
index 0000000..1d77608
--- /dev/null
@@ -0,0 +1,224 @@
+/*
+Minetest
+Copyright (C) 2014 celeron55, Perttu Ahola <celeron55@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "config.h"
+
+#if USE_REDIS
+/*
+       Redis databases
+*/
+
+
+#include "database-redis.h"
+#include <hiredis.h>
+
+#include "map.h"
+#include "mapsector.h"
+#include "mapblock.h"
+#include "serialization.h"
+#include "main.h"
+#include "settings.h"
+#include "log.h"
+
+Database_Redis::Database_Redis(ServerMap *map, std::string savedir)
+{
+       Settings conf;
+       conf.readConfigFile((std::string(savedir) + DIR_DELIM + "world.mt").c_str());
+       std::string tmp;
+       try {
+       tmp = conf.get("redis_address");
+       hash = conf.get("redis_hash");
+       } catch(SettingNotFoundException e) {
+               throw SettingNotFoundException("Set redis_address and redis_hash in world.mt to use the redis backend");
+       }
+       const char *addr = tmp.c_str();
+       int port = conf.exists("redis_port") ? conf.getU16("redis_port") : 6379;
+       ctx = redisConnect(addr, port);
+       if(!ctx)
+               throw FileNotGoodException("Cannot allocate redis context");
+       else if(ctx->err) {
+               std::string err = std::string("Connection error: ") + ctx->errstr;
+               redisFree(ctx);
+               throw FileNotGoodException(err);
+       }
+       srvmap = map;
+}
+
+int Database_Redis::Initialized(void)
+{
+       return 1;
+}
+
+void Database_Redis::beginSave() {
+       redisReply *reply;
+       reply = (redisReply*) redisCommand(ctx, "MULTI");
+       if(!reply)
+               throw FileNotGoodException(std::string("redis command 'MULTI' failed: ") + ctx->errstr);
+       freeReplyObject(reply);
+}
+
+void Database_Redis::endSave() {
+       redisReply *reply;
+       reply = (redisReply*) redisCommand(ctx, "EXEC");
+       if(!reply)
+               throw FileNotGoodException(std::string("redis command 'EXEC' failed: ") + ctx->errstr);
+       freeReplyObject(reply);
+}
+
+void Database_Redis::saveBlock(MapBlock *block)
+{
+       DSTACK(__FUNCTION_NAME);
+       /*
+               Dummy blocks are not written
+       */
+       if(block->isDummy())
+       {
+               return;
+       }
+
+       // Format used for writing
+       u8 version = SER_FMT_VER_HIGHEST_WRITE;
+       // Get destination
+       v3s16 p3d = block->getPos();
+
+       /*
+               [0] u8 serialization version
+               [1] data
+       */
+
+       std::ostringstream o(std::ios_base::binary);
+       o.write((char*)&version, 1);
+       // Write basic data
+       block->serialize(o, version, true);
+       // Write block to database
+       std::string tmp1 = o.str();
+       std::string tmp2 = i64tos(getBlockAsInteger(p3d));
+
+       redisReply *reply;
+       reply = (redisReply*) redisCommand(ctx, "HSET %s %s %b", hash.c_str(), tmp2.c_str(), tmp1.c_str(), tmp1.size());
+       if(!reply)
+               throw FileNotGoodException(std::string("redis command 'HSET %s %s %b' failed: ") + ctx->errstr);
+       if(reply->type == REDIS_REPLY_ERROR)
+               throw FileNotGoodException("Failed to store block in Database");
+
+       // We just wrote it to the disk so clear modified flag
+       block->resetModified();
+}
+
+MapBlock* Database_Redis::loadBlock(v3s16 blockpos)
+{
+       v2s16 p2d(blockpos.X, blockpos.Z);
+
+       std::string tmp = i64tos(getBlockAsInteger(blockpos));
+       redisReply *reply;
+       reply = (redisReply*) redisCommand(ctx, "HGET %s %s", hash.c_str(), tmp.c_str());
+       if(!reply)
+               throw FileNotGoodException(std::string("redis command 'HGET %s %s' failed: ") + ctx->errstr);
+
+       if (reply->type == REDIS_REPLY_STRING && reply->len == 0) {
+               freeReplyObject(reply);
+               errorstream << "Blank block data in database (reply->len == 0) ("
+                       << blockpos.X << "," << blockpos.Y << "," << blockpos.Z << ")" << std::endl;
+
+               if (g_settings->getBool("ignore_world_load_errors")) {
+                       errorstream << "Ignoring block load error. Duck and cover! "
+                               << "(ignore_world_load_errors)" << std::endl;
+               } else {
+                       throw SerializationError("Blank block data in database");
+               }
+               return NULL;
+       }
+
+       if (reply->type == REDIS_REPLY_STRING) {
+               /*
+                       Make sure sector is loaded
+               */
+               MapSector *sector = srvmap->createSector(p2d);
+
+               try {
+                       std::istringstream is(std::string(reply->str, reply->len), std::ios_base::binary);
+                       freeReplyObject(reply); // std::string copies the memory so we can already do this here
+                       u8 version = SER_FMT_VER_INVALID;
+                       is.read((char *)&version, 1);
+
+                       if (is.fail())
+                               throw SerializationError("ServerMap::loadBlock(): Failed"
+                                       " to read MapBlock version");
+
+                       MapBlock *block = NULL;
+                       bool created_new = false;
+                       block = sector->getBlockNoCreateNoEx(blockpos.Y);
+                       if (block == NULL)
+                       {
+                               block = sector->createBlankBlockNoInsert(blockpos.Y);
+                               created_new = true;
+                       }
+
+                       // Read basic data
+                       block->deSerialize(is, version, true);
+
+                       // If it's a new block, insert it to the map
+                       if (created_new)
+                               sector->insertBlock(block);
+
+                       // We just loaded it from, so it's up-to-date.
+                       block->resetModified();
+               }
+               catch (SerializationError &e)
+               {
+                       errorstream << "Invalid block data in database"
+                               << " (" << blockpos.X << "," << blockpos.Y << "," << blockpos.Z
+                               << ") (SerializationError): " << e.what() << std::endl;
+                       // TODO: Block should be marked as invalid in memory so that it is
+                       // not touched but the game can run
+
+                       if (g_settings->getBool("ignore_world_load_errors")) {
+                               errorstream << "Ignoring block load error. Duck and cover! "
+                                       << "(ignore_world_load_errors)" << std::endl;
+                       } else {
+                               throw SerializationError("Invalid block data in database");
+                       }
+               }
+
+               return srvmap->getBlockNoCreateNoEx(blockpos);  // should not be using this here
+       }
+       return NULL;
+}
+
+void Database_Redis::listAllLoadableBlocks(std::list<v3s16> &dst)
+{
+       redisReply *reply;
+       reply = (redisReply*) redisCommand(ctx, "HKEYS %s", hash.c_str());
+       if(!reply)
+               throw FileNotGoodException(std::string("redis command 'HKEYS %s' failed: ") + ctx->errstr);
+       if(reply->type != REDIS_REPLY_ARRAY)
+               throw FileNotGoodException("Failed to get keys from database");
+       for(size_t i = 0; i < reply->elements; i++)
+       {
+               assert(reply->element[i]->type == REDIS_REPLY_STRING);
+               dst.push_back(getIntegerAsBlock(stoi64(reply->element[i]->str)));
+       }
+       freeReplyObject(reply);
+}
+
+Database_Redis::~Database_Redis()
+{
+       redisFree(ctx);
+}
+#endif
diff --git a/src/database-redis.h b/src/database-redis.h
new file mode 100644 (file)
index 0000000..da76775
--- /dev/null
@@ -0,0 +1,50 @@
+/*
+Minetest
+Copyright (C) 2014 celeron55, Perttu Ahola <celeron55@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef DATABASE_REDIS_HEADER
+#define DATABASE_REDIS_HEADER
+
+#include "config.h"
+
+#if USE_REDIS
+
+#include "database.h"
+#include <hiredis.h>
+#include <string>
+
+class ServerMap;
+
+class Database_Redis : public Database
+{
+public:
+       Database_Redis(ServerMap *map, std::string savedir);
+       virtual void beginSave();
+       virtual void endSave();
+       virtual void saveBlock(MapBlock *block);
+       virtual MapBlock* loadBlock(v3s16 blockpos);
+       virtual void listAllLoadableBlocks(std::list<v3s16> &dst);
+       virtual int Initialized(void);
+       ~Database_Redis();
+private:
+       ServerMap *srvmap;
+       redisContext *ctx;
+       std::string hash;
+};
+#endif
+#endif
index c93af0d7704519ae3e10c2b761e02b0b700c56d8..bb0c3a273c9803b11ceababf8120395f8882672b 100644 (file)
@@ -84,6 +84,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #ifdef USE_LEVELDB
 #include "database-leveldb.h"
 #endif
+#if USE_REDIS
+#include "database-redis.h"
+#endif
 
 /*
        Settings.
@@ -1242,7 +1245,7 @@ int main(int argc, char *argv[])
                        }
                        if (!world_mt.exists("backend")) {
                                errorstream << "Please specify your current backend in world.mt file:"
-                                       << std::endl << "       backend = {sqlite3|leveldb|dummy}" << std::endl;
+                                       << std::endl << "       backend = {sqlite3|leveldb|redis|dummy}" << std::endl;
                                return 1;
                        }
                        std::string backend = world_mt.get("backend");
@@ -1257,6 +1260,10 @@ int main(int argc, char *argv[])
                        else if (migrate_to == "leveldb")
                                new_db = new Database_LevelDB(&(ServerMap&)server.getMap(), world_path);
                        #endif
+                       #if USE_REDIS
+                       else if (migrate_to == "redis")
+                               new_db = new Database_Redis(&(ServerMap&)server.getMap(), world_path);
+                       #endif
                        else {
                                errorstream << "Migration to " << migrate_to << " is not supported" << std::endl;
                                return 1;
index ec97fedf2da9a4464c4a13d4f7f82ee945d06f22..8f612aeacb153e5164c125a86acd223d9ff71e2f 100644 (file)
@@ -46,6 +46,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #if USE_LEVELDB
 #include "database-leveldb.h"
 #endif
+#if USE_REDIS
+#include "database-redis.h"
+#endif
 
 #define PP(x) "("<<(x).X<<","<<(x).Y<<","<<(x).Z<<")"
 
@@ -2434,6 +2437,10 @@ ServerMap::ServerMap(std::string savedir, IGameDef *gamedef, EmergeManager *emer
                else if (backend == "leveldb")
                        dbase = new Database_LevelDB(this, savedir);
                #endif
+               #if USE_REDIS
+               else if (backend == "redis")
+                       dbase = new Database_Redis(this, savedir);
+               #endif
                else
                        throw BaseException("Unknown map backend");
        }