Implement a PostgreSQL backend 4124/head
authorLoic Blot <loic.blot@unix-experience.fr>
Sat, 14 May 2016 09:00:42 +0000 (11:00 +0200)
committerLoic Blot <loic.blot@unix-experience.fr>
Sun, 22 May 2016 09:34:47 +0000 (11:34 +0200)
README.txt
src/CMakeLists.txt
src/cmake_config.h.in
src/database-postgresql.cpp [new file with mode: 0644]
src/database-postgresql.h [new file with mode: 0644]
src/map.cpp
util/travis/before_install.sh

index 9ca9b331e4ab1ee7549e5f476c7924c25e4e43e6..bd2dadebbce178780e65a308ee0b04f7dd4c3b93 100644 (file)
@@ -169,7 +169,8 @@ ENABLE_CURSES       - Build with (n)curses; Enables a server side terminal (comm
 ENABLE_FREETYPE     - Build with FreeType2; Allows using TTF fonts
 ENABLE_GETTEXT      - Build with Gettext; Allows using translations
 ENABLE_GLES         - Search for Open GLES headers & libraries and use them
-ENABLE_LEVELDB      - Build with LevelDB; Enables use of LevelDB map backend (faster than SQLite3)
+ENABLE_LEVELDB      - Build with LevelDB; Enables use of LevelDB map backend
+ENABLE_POSTGRESQL   - Build with libpq; Enables use of PostgreSQL map backend (PostgreSQL 9.5 or greater required)
 ENABLE_REDIS        - Build with libhiredis; Enables use of Redis map backend
 ENABLE_SPATIAL      - Build with LibSpatial; Speeds up AreaStores
 ENABLE_SOUND        - Build with OpenAL, libogg & libvorbis; in-game Sounds
@@ -203,6 +204,8 @@ IRRLICHT_LIBRARY                - Path to libIrrlicht.a/libIrrlicht.so/libIrrlic
 LEVELDB_INCLUDE_DIR             - Only when building with LevelDB; directory that contains db.h
 LEVELDB_LIBRARY                 - Only when building with LevelDB; path to libleveldb.a/libleveldb.so/libleveldb.dll.a
 LEVELDB_DLL                     - Only when building with LevelDB on Windows; path to libleveldb.dll
+POSTGRESQL_INCLUDE_DIR          - Only when building with PostgreSQL; directory that contains libpq-fe.h
+POSTGRESQL_LIBRARY              - Only when building with PostgreSQL; path to libpq.a/libpq.so
 REDIS_INCLUDE_DIR               - Only when building with Redis; directory that contains hiredis.h
 REDIS_LIBRARY                   - Only when building with Redis; path to libhiredis.a/libhiredis.so
 SPATIAL_INCLUDE_DIR             - Only when building with LibSpatial; directory that contains spatialindex/SpatialIndex.h
index feca199c1d89ca8c7cd23399bb032c58aadc7977..ea1564eebc2c39d4fcaba7f02a74b6fdd732a5ed 100644 (file)
@@ -189,6 +189,36 @@ if(ENABLE_CURSES)
        endif()
 endif(ENABLE_CURSES)
 
+option(ENABLE_POSTGRESQL "Enable PostgreSQL backend" TRUE)
+set(USE_POSTGRESQL FALSE)
+
+if(ENABLE_POSTGRESQL)
+       find_program(POSTGRESQL_CONFIG_EXECUTABLE pg_config DOC "pg_config")
+       find_library(POSTGRESQL_LIBRARY pq)
+       if(POSTGRESQL_CONFIG_EXECUTABLE)
+               execute_process(COMMAND ${POSTGRESQL_CONFIG_EXECUTABLE} --includedir-server
+                       OUTPUT_VARIABLE POSTGRESQL_SERVER_INCLUDE_DIRS
+                       OUTPUT_STRIP_TRAILING_WHITESPACE)
+               execute_process(COMMAND ${POSTGRESQL_CONFIG_EXECUTABLE}
+                       OUTPUT_VARIABLE POSTGRESQL_CLIENT_INCLUDE_DIRS
+                       OUTPUT_STRIP_TRAILING_WHITESPACE)
+               # This variable is case sensitive for the cmake PostgreSQL module
+               set(PostgreSQL_ADDITIONAL_SEARCH_PATHS ${POSTGRESQL_SERVER_INCLUDE_DIRS} ${POSTGRESQL_CLIENT_INCLUDE_DIRS})
+       endif()
+
+       find_package("PostgreSQL")
+
+       if(POSTGRESQL_FOUND)
+               set(USE_POSTGRESQL TRUE)
+               message(STATUS "PostgreSQL backend enabled")
+               # This variable is case sensitive, don't try to change it to POSTGRESQL_INCLUDE_DIR
+               message(STATUS "PostgreSQL includes: ${PostgreSQL_INCLUDE_DIR}")
+               include_directories(${PostgreSQL_INCLUDE_DIR})
+       else()
+               message(STATUS "PostgreSQL not found!")
+       endif()
+endif(ENABLE_POSTGRESQL)
+
 option(ENABLE_LEVELDB "Enable LevelDB backend" TRUE)
 set(USE_LEVELDB FALSE)
 
@@ -361,6 +391,7 @@ set(common_SRCS
        craftdef.cpp
        database-dummy.cpp
        database-leveldb.cpp
+       database-postgresql.cpp
        database-redis.cpp
        database-sqlite3.cpp
        database.cpp
@@ -592,6 +623,9 @@ if(BUILD_CLIENT)
        if (USE_CURSES)
                target_link_libraries(${PROJECT_NAME} ${CURSES_LIBRARIES})
        endif()
+       if (USE_POSTGRESQL)
+               target_link_libraries(${PROJECT_NAME} ${POSTGRESQL_LIBRARY})
+       endif()
        if (USE_LEVELDB)
                target_link_libraries(${PROJECT_NAME} ${LEVELDB_LIBRARY})
        endif()
@@ -622,6 +656,9 @@ if(BUILD_SERVER)
        if (USE_CURSES)
                target_link_libraries(${PROJECT_NAME}server ${CURSES_LIBRARIES})
        endif()
+       if (USE_POSTGRESQL)
+               target_link_libraries(${PROJECT_NAME}server ${POSTGRESQL_LIBRARY})
+       endif()
        if (USE_LEVELDB)
                target_link_libraries(${PROJECT_NAME}server ${LEVELDB_LIBRARY})
        endif()
index 018532d131927f5fb94b67e4438c365f1a2acb9a..50f34a0b8b02853aba47c58363d26e49b6d99a4c 100644 (file)
@@ -22,6 +22,7 @@
 #cmakedefine01 USE_CURSES
 #cmakedefine01 USE_LEVELDB
 #cmakedefine01 USE_LUAJIT
+#cmakedefine01 USE_POSTGRESQL
 #cmakedefine01 USE_SPATIAL
 #cmakedefine01 USE_SYSTEM_GMP
 #cmakedefine01 USE_REDIS
diff --git a/src/database-postgresql.cpp b/src/database-postgresql.cpp
new file mode 100644 (file)
index 0000000..3b6b42a
--- /dev/null
@@ -0,0 +1,286 @@
+/*
+Copyright (C) 2016 Loic Blot <loic.blot@unix-experience.fr>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "config.h"
+
+#if USE_POSTGRESQL
+
+#include "database-postgresql.h"
+
+#ifdef _WIN32
+        #ifndef WIN32_LEAN_AND_MEAN
+                #define WIN32_LEAN_AND_MEAN
+        #endif
+        // Without this some of the network functions are not found on mingw
+        #ifndef _WIN32_WINNT
+                #define _WIN32_WINNT 0x0501
+        #endif
+        #include <windows.h>
+        #include <winsock2.h>
+#else
+#include <netinet/in.h>
+#endif
+
+#include "log.h"
+#include "exceptions.h"
+#include "settings.h"
+
+Database_PostgreSQL::Database_PostgreSQL(const Settings &conf) :
+       m_connect_string(""),
+       m_conn(NULL),
+       m_pgversion(0)
+{
+       if (!conf.getNoEx("pgsql_connection", m_connect_string)) {
+               throw SettingNotFoundException(
+                       "Set pgsql_connection string in world.mt to "
+                       "use the postgresql backend\n"
+                       "Notes:\n"
+                       "pgsql_connection has the following form: \n"
+                       "\tpgsql_connection = host=127.0.0.1 port=5432 user=mt_user "
+                       "password=mt_password dbname=minetest_world\n"
+                       "mt_user should have CREATE TABLE, INSERT, SELECT, UPDATE and "
+                       "DELETE rights on the database.\n"
+                       "Don't create mt_user as a SUPERUSER!");
+       }
+
+       connectToDatabase();
+}
+
+Database_PostgreSQL::~Database_PostgreSQL()
+{
+       PQfinish(m_conn);
+}
+
+void Database_PostgreSQL::connectToDatabase()
+{
+       m_conn = PQconnectdb(m_connect_string.c_str());
+
+       if (PQstatus(m_conn) != CONNECTION_OK) {
+               throw DatabaseException(std::string(
+                       "PostgreSQL database error: ") +
+                       PQerrorMessage(m_conn));
+       }
+
+       m_pgversion = PQserverVersion(m_conn);
+
+       /*
+       * We are using UPSERT feature from PostgreSQL 9.5
+       * to have the better performance,
+       * set the minimum version to 90500
+       */
+       if (m_pgversion < 90500) {
+               throw DatabaseException("PostgreSQL database error: "
+                       "Server version 9.5 or greater required.");
+       }
+
+       infostream << "PostgreSQL Database: Version " << m_pgversion
+                       << " Connection made." << std::endl;
+
+       createDatabase();
+       initStatements();
+}
+
+void Database_PostgreSQL::verifyDatabase()
+{
+       if (PQstatus(m_conn) == CONNECTION_OK)
+               return;
+
+       PQreset(m_conn);
+       ping();
+}
+
+void Database_PostgreSQL::ping()
+{
+       if (PQping(m_connect_string.c_str()) != PQPING_OK) {
+               throw DatabaseException(std::string(
+                       "PostgreSQL database error: ") +
+                       PQerrorMessage(m_conn));
+       }
+}
+
+bool Database_PostgreSQL::initialized() const
+{
+       return (PQstatus(m_conn) == CONNECTION_OK);
+}
+
+void Database_PostgreSQL::initStatements()
+{
+       prepareStatement("read_block",
+                       "SELECT data FROM blocks "
+                       "WHERE posX = $1::int4 AND posY = $2::int4 AND "
+                       "posZ = $3::int4");
+
+       prepareStatement("write_block",
+                       "INSERT INTO blocks (posX, posY, posZ, data) VALUES "
+                       "($1::int4, $2::int4, $3::int4, $4::bytea) "
+                       "ON CONFLICT ON CONSTRAINT blocks_pkey DO "
+                       "UPDATE SET data = $4::bytea");
+
+       prepareStatement("delete_block", "DELETE FROM blocks WHERE "
+                       "posX = $1::int4 AND posY = $2::int4 AND posZ = $3::int4");
+
+       prepareStatement("list_all_loadable_blocks",
+                       "SELECT posX, posY, posZ FROM blocks");
+}
+
+PGresult *Database_PostgreSQL::checkResults(PGresult *result, bool clear)
+{
+       ExecStatusType statusType = PQresultStatus(result);
+
+       switch (statusType) {
+       case PGRES_COMMAND_OK:
+       case PGRES_TUPLES_OK:
+               break;
+       case PGRES_FATAL_ERROR:
+       default:
+               throw DatabaseException(
+                       std::string("PostgreSQL database error: ") +
+                       PQresultErrorMessage(result));
+       }
+
+       if (clear)
+               PQclear(result);
+
+       return result;
+}
+
+void Database_PostgreSQL::createDatabase()
+{
+       PGresult *result = checkResults(PQexec(m_conn,
+               "SELECT relname FROM pg_class WHERE relname='blocks';"),
+               false);
+
+       // If table doesn't exist, create it
+       if (!PQntuples(result)) {
+               static const char* dbcreate_sql = "CREATE TABLE blocks ("
+                       "posX INT NOT NULL,"
+                       "posY INT NOT NULL,"
+                       "posZ INT NOT NULL,"
+                       "data BYTEA,"
+                       "PRIMARY KEY (posX,posY,posZ)"
+               ");";
+               checkResults(PQexec(m_conn, dbcreate_sql));
+       }
+
+       PQclear(result);
+
+       infostream << "PostgreSQL: Game Database was inited." << std::endl;
+}
+
+
+void Database_PostgreSQL::beginSave()
+{
+       verifyDatabase();
+       checkResults(PQexec(m_conn, "BEGIN;"));
+}
+
+void Database_PostgreSQL::endSave()
+{
+       checkResults(PQexec(m_conn, "COMMIT;"));
+}
+
+bool Database_PostgreSQL::saveBlock(const v3s16 &pos,
+               const std::string &data)
+{
+       // Verify if we don't overflow the platform integer with the mapblock size
+       if (data.size() > INT_MAX) {
+               errorstream << "Database_PostgreSQL::saveBlock: Data truncation! "
+                               << "data.size() over 0xFFFF (== " << data.size()
+                               << ")" << std::endl;
+               return false;
+       }
+
+       verifyDatabase();
+
+       s32 x, y, z;
+       x = htonl(pos.X);
+       y = htonl(pos.Y);
+       z = htonl(pos.Z);
+
+       const void *args[] = { &x, &y, &z, data.c_str() };
+       const int argLen[] = {
+               sizeof(x), sizeof(y), sizeof(z), (int)data.size()
+       };
+       const int argFmt[] = { 1, 1, 1, 1 };
+
+       execPrepared("write_block", ARRLEN(args), args, argLen, argFmt);
+       return true;
+}
+
+void Database_PostgreSQL::loadBlock(const v3s16 &pos,
+               std::string *block)
+{
+       verifyDatabase();
+
+       s32 x, y, z;
+       x = htonl(pos.X);
+       y = htonl(pos.Y);
+       z = htonl(pos.Z);
+
+       const void *args[] = { &x, &y, &z };
+       const int argLen[] = { sizeof(x), sizeof(y), sizeof(z) };
+       const int argFmt[] = { 1, 1, 1 };
+
+       PGresult *results = execPrepared("read_block", ARRLEN(args), args,
+                       argLen, argFmt, false);
+
+       *block = "";
+
+       if (PQntuples(results)) {
+               *block = std::string(PQgetvalue(results, 0, 0),
+                               PQgetlength(results, 0, 0));
+       }
+
+       PQclear(results);
+}
+
+bool Database_PostgreSQL::deleteBlock(const v3s16 &pos)
+{
+       verifyDatabase();
+
+       s32 x, y, z;
+       x = htonl(pos.X);
+       y = htonl(pos.Y);
+       z = htonl(pos.Z);
+
+       const void *args[] = { &x, &y, &z };
+       const int argLen[] = { sizeof(x), sizeof(y), sizeof(z) };
+       const int argFmt[] = { 1, 1, 1 };
+
+       execPrepared("read_block", ARRLEN(args), args, argLen, argFmt);
+
+       return true;
+}
+
+void Database_PostgreSQL::listAllLoadableBlocks(std::vector<v3s16> &dst)
+{
+       verifyDatabase();
+
+       PGresult *results = execPrepared("list_all_loadable_blocks", 0,
+                       NULL, NULL, NULL, false, false);
+
+       int numrows = PQntuples(results);
+
+       for (int row = 0; row < numrows; ++row) {
+               dst.push_back(pg_to_v3s16(results, 0, 0));
+       }
+
+       PQclear(results);
+}
+
+#endif // USE_POSTGRESQL
diff --git a/src/database-postgresql.h b/src/database-postgresql.h
new file mode 100644 (file)
index 0000000..1cfa544
--- /dev/null
@@ -0,0 +1,95 @@
+/*
+Minetest
+Copyright (C) 2013 celeron55, Perttu Ahola <celeron55@gmail.com>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation; either version 2.1 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef DATABASE_POSTGRESQL_HEADER
+#define DATABASE_POSTGRESQL_HEADER
+
+#include <string>
+#include <libpq-fe.h>
+#include "database.h"
+#include "util/basic_macros.h"
+
+class Settings;
+
+class Database_PostgreSQL : public Database
+{
+public:
+       Database_PostgreSQL(const Settings &conf);
+       ~Database_PostgreSQL();
+
+       void beginSave();
+       void endSave();
+
+       bool saveBlock(const v3s16 &pos, const std::string &data);
+       void loadBlock(const v3s16 &pos, std::string *block);
+       bool deleteBlock(const v3s16 &pos);
+       void listAllLoadableBlocks(std::vector<v3s16> &dst);
+       bool initialized() const;
+
+private:
+       // Database initialization
+       void connectToDatabase();
+       void initStatements();
+       void createDatabase();
+
+       inline void prepareStatement(const std::string &name, const std::string &sql)
+       {
+               checkResults(PQprepare(m_conn, name.c_str(), sql.c_str(), 0, NULL));
+       }
+
+       // Database connectivity checks
+       void ping();
+       void verifyDatabase();
+
+       // Database usage
+       PGresult *checkResults(PGresult *res, bool clear = true);
+
+       inline PGresult *execPrepared(const char *stmtName, const int paramsNumber,
+                       const void **params,
+                       const int *paramsLengths = NULL, const int *paramsFormats = NULL,
+                       bool clear = true, bool nobinary = true)
+       {
+               return checkResults(PQexecPrepared(m_conn, stmtName, paramsNumber,
+                       (const char* const*) params, paramsLengths, paramsFormats,
+                       nobinary ? 1 : 0), clear);
+       }
+
+       // Conversion helpers
+       inline int pg_to_int(PGresult *res, int row, int col)
+       {
+               return atoi(PQgetvalue(res, row, col));
+       }
+
+       inline v3s16 pg_to_v3s16(PGresult *res, int row, int col)
+       {
+               return v3s16(
+                       pg_to_int(res, row, col),
+                       pg_to_int(res, row, col + 1),
+                       pg_to_int(res, row, col + 2)
+               );
+       }
+
+       // Attributes
+       std::string m_connect_string;
+       PGconn *m_conn;
+       int m_pgversion;
+};
+
+#endif
+
index 848d2ef13255a715dd0f9c9123431c052704b12c..03daf4fa8b2e4f4f9b994b8847eb2ebe82bd8a1b 100644 (file)
@@ -50,6 +50,9 @@ with this program; if not, write to the Free Software Foundation, Inc.,
 #if USE_REDIS
 #include "database-redis.h"
 #endif
+#if USE_POSTGRESQL
+#include "database-postgresql.h"
+#endif
 
 #define PP(x) "("<<(x).X<<","<<(x).Y<<","<<(x).Z<<")"
 
@@ -3240,6 +3243,10 @@ Database *ServerMap::createDatabase(
        else if (name == "redis")
                return new Database_Redis(conf);
        #endif
+       #if USE_POSTGRESQL
+       else if (name == "postgresql")
+               return new Database_PostgreSQL(conf);
+       #endif
        else
                throw BaseException(std::string("Database backend ") + name + " not supported.");
 }
index 58dc42b17db2f485f5a51449ba0e2623f6904701..70037389b4f41c74441bc60c2c2b76e80ba3feea 100755 (executable)
@@ -17,13 +17,15 @@ if [[ $PLATFORM == "Unix" ]]; then
        if [[ $TRAVIS_OS_NAME == "linux" ]]; then
                sudo apt-get install libirrlicht-dev cmake libbz2-dev libpng12-dev \
                        libjpeg-dev libxxf86vm-dev libgl1-mesa-dev libsqlite3-dev \
-                       libhiredis-dev libogg-dev libgmp-dev libvorbis-dev libopenal-dev gettext
+                       libhiredis-dev libogg-dev libgmp-dev libvorbis-dev libopenal-dev \
+                       gettext libpq-dev postgresql-server-dev-all
                # Linking to LevelDB is broken, use a custom build
                wget http://minetest.kitsunemimi.pw/libleveldb-1.18-ubuntu12.04.7z
                sudo 7z x -o/usr libleveldb-1.18-ubuntu12.04.7z
        else
                brew update
                brew install freetype gettext hiredis irrlicht jpeg leveldb libogg libvorbis luajit
+               brew upgrade postgresql
        fi
 elif [[ $PLATFORM == "Win32" ]]; then
        wget http://minetest.kitsunemimi.pw/mingw_w64_i686_ubuntu12.04_4.9.1.7z -O mingw.7z