From cacd64d8635201459e59bf2cd8a2ea8fd0699b84 Mon Sep 17 00:00:00 2001 From: David Barksdale Date: Thu, 13 Apr 2017 21:22:29 -0500 Subject: [PATCH] Rewrite gnunet-datastore to dump to a file --- doc/man/gnunet-datastore.1 | 24 +- src/datastore/gnunet-datastore.c | 452 ++++++++++++++++++++++++------- 2 files changed, 364 insertions(+), 112 deletions(-) diff --git a/doc/man/gnunet-datastore.1 b/doc/man/gnunet-datastore.1 index 14372a49b..1605006b3 100644 --- a/doc/man/gnunet-datastore.1 +++ b/doc/man/gnunet-datastore.1 @@ -1,6 +1,6 @@ -.TH gnunet\-datastore "1" "30 May 2013" "GNUnet" +.TH gnunet\-datastore "1" "13 April 2017" "GNUnet" .SH NAME -gnunet\-datastore \- merge or convert GNUnet datastore databases +gnunet\-datastore \- dump or insert (restore) GNUnet datastore databases .SH SYNOPSIS .B gnunet\-datastore @@ -8,26 +8,32 @@ gnunet\-datastore \- merge or convert GNUnet datastore databases .SH DESCRIPTION .PP -gnunet\-datastore can be used to convert or merge GNUnet datastores. This is useful if a datastore is to be migrated between SQL databases, i.e. from sqlite to postgres or vice versa. gnunet\-datastore basically takes two configuration files (which must specify different databases) and reads in all of the data from the datasource (\-s option) and copies it to the destination (\-c option). Note that replication level information is lost in the process at this time. +gnunet\-datastore can be used to backup and restore or merge GNUnet datastores. This is useful if a datastore is to be migrated between SQL databases, i.e. from sqlite to postgres or vice versa. gnunet\-datastore will dump the entire contents of the database or insert a dump file into the database. .TP \fB\-c \fIFILENAME\fR, \fB\-\-config=FILENAME\fR -configuration file to use for the destination database +configuration file to use +.TP +\fB\-d\fR, \fB\-\-dump\fR +dump all records to a file +.TP +\fB\-f \fIFILENAME\fR, \fB\-\-file=FILENAME\fR +file to dump to or insert from. Otherwise stdin/stdout are used. .TP \fB\-h\fR, \fB\-\-help\fR print help page .TP +\fB\-i\fR, \fB\-\-insert\fR +insert from dump file +.TP \fB\-L \fILOGLEVEL\fR, \fB\-\-loglevel=LOGLEVEL\fR Change the loglevel. Possible values for LOGLEVEL are ERROR, WARNING, INFO and DEBUG. .TP -\fB\-s \fIFILENAME\fR, \fB\-\-sourcecfg=FILENAME\fR -configuration file to use for the source database +\fB\-l \fIFILENAME\fR, \fB\-\-logfile=FILENAME\fR +configure logging to write logs to FILENAME .TP \fB\-v\fR, \fB\-\-version\fR print the version number -.TP -\fB\-V\fR, \fB\-\-verbose\fR -be verbose .SH NOTES diff --git a/src/datastore/gnunet-datastore.c b/src/datastore/gnunet-datastore.c index 9e0ee205e..891343e17 100644 --- a/src/datastore/gnunet-datastore.c +++ b/src/datastore/gnunet-datastore.c @@ -23,101 +23,136 @@ * @brief tool to manipulate datastores * @author Christian Grothoff */ +#include #include "platform.h" #include "gnunet_util_lib.h" #include "gnunet_datastore_service.h" +GNUNET_NETWORK_STRUCT_BEGIN + +struct DataRecord +{ + /** + * Number of bytes in the item (NBO). + */ + uint32_t size GNUNET_PACKED; + + /** + * Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type) + */ + uint32_t type GNUNET_PACKED; + + /** + * Priority of the item (NBO). + */ + uint32_t priority GNUNET_PACKED; + + /** + * Desired anonymity level (NBO). + */ + uint32_t anonymity GNUNET_PACKED; + + /** + * Desired replication level (NBO). + */ + uint32_t replication GNUNET_PACKED; + + /** + * Expiration time (NBO). + */ + struct GNUNET_TIME_AbsoluteNBO expiration; + + /** + * Key under which the item can be found. + */ + struct GNUNET_HashCode key; + +}; +GNUNET_NETWORK_STRUCT_END + /** - * Name of the second configuration file. + * Length of our magic header. */ -static char *alternative_cfg; +static const size_t MAGIC_LEN = 16; /** - * Global return value. + * Magic header bytes. */ -static int ret; +static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1"; + +/** + * Dump the database. + */ +static int dump; /** - * Our offset on 'get'. + * Insert into the database. */ -static uint64_t offset; +static int insert; /** - * First UID ever returned. + * Dump file name. */ -static uint64_t first_uid; +static char *file_name; /** - * Configuration for the source database. + * Dump file handle. */ -static struct GNUNET_CONFIGURATION_Handle *scfg; +static struct GNUNET_DISK_FileHandle *file_handle; /** - * Handle for database source. + * Global return value. */ -static struct GNUNET_DATASTORE_Handle *db_src; +static int ret; /** - * Handle for database destination. + * Handle for datastore. */ -static struct GNUNET_DATASTORE_Handle *db_dst; +static struct GNUNET_DATASTORE_Handle *datastore; /** * Current operation. */ static struct GNUNET_DATASTORE_QueueEntry *qe; +/** + * Record count. + */ +static uint64_t record_count; + static void do_shutdown (void *cls) { if (NULL != qe) GNUNET_DATASTORE_cancel (qe); - GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO); - GNUNET_DATASTORE_disconnect (db_dst, GNUNET_NO); - GNUNET_CONFIGURATION_destroy (scfg); + if (NULL != datastore) + GNUNET_DATASTORE_disconnect (datastore, GNUNET_NO); + if (NULL != file_handle) + GNUNET_DISK_file_close (file_handle); } /** - * Perform next GET operation. + * Begin dumping the database. */ static void -do_get (void); +start_dump (void); /** - * Continuation called to notify client about result of the - * operation. - * - * @param cls closure - * @param success GNUNET_SYSERR on failure (including timeout/queue drop) - * GNUNET_NO if content was already there - * GNUNET_YES (or other positive value) on success - * @param min_expiration minimum expiration time required for 0-priority content to be stored - * by the datacache at this time, zero for unknown, forever if we have no - * space for 0-priority content - * @param msg NULL on success, otherwise an error message + * Begin inserting into the database. */ static void -do_finish (void *cls, - int32_t success, - struct GNUNET_TIME_Absolute min_expiration, - const char *msg) -{ - qe = NULL; - if (GNUNET_SYSERR == success) - { - fprintf (stderr, - _("Failed to store item: %s, aborting\n"), - msg); - ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - do_get (); -} +start_insert (void); + + +/** + * Perform next GET operation. + */ +static void +do_get (const uint64_t next_uid); /** @@ -136,7 +171,7 @@ do_finish (void *cls, * maybe 0 if no unique identifier is available */ static void -do_put (void *cls, +get_cb (void *cls, const struct GNUNET_HashCode *key, size_t size, const void *data, @@ -144,33 +179,63 @@ do_put (void *cls, uint32_t priority, uint32_t anonymity, uint32_t replication, - struct GNUNET_TIME_Absolute - expiration, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) { qe = NULL; - if ( (0 != offset) && - (uid == first_uid) ) + if (NULL == key) { + FPRINTF (stderr, + _("Dumped %" PRIu64 " records\n"), + record_count); + GNUNET_DISK_file_close (file_handle); + file_handle = NULL; + if (insert) + start_insert(); + else + { + ret = 0; + GNUNET_SCHEDULER_shutdown (); + } + return; + } + + struct DataRecord dr; + dr.size = htonl ((uint32_t) size); + dr.type = htonl (type); + dr.priority = htonl (priority); + dr.anonymity = htonl (anonymity); + dr.replication = htonl (replication); + dr.expiration = GNUNET_TIME_absolute_hton (expiration); + dr.key = *key; + + ssize_t len; + len = GNUNET_DISK_file_write (file_handle, &dr, sizeof (dr)); + if (sizeof (dr) != len) + { + FPRINTF (stderr, + _("Short write to file: %zd bytes expecting %zd\n"), + len, + sizeof (dr)); + ret = 1; GNUNET_SCHEDULER_shutdown (); return; } - if (0 == offset) - first_uid = uid; - qe = GNUNET_DATASTORE_put (db_dst, - 0, - key, - size, - data, - type, - priority, - anonymity, - replication, - expiration, - 0, - 1, - &do_finish, - NULL); + + len = GNUNET_DISK_file_write (file_handle, data, size); + if (size != len) + { + FPRINTF (stderr, + _("Short write to file: %zd bytes expecting %zd\n"), + len, + size); + ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } + + record_count++; + do_get(uid + 1); } @@ -178,64 +243,236 @@ do_put (void *cls, * Perform next GET operation. */ static void -do_get () +do_get (const uint64_t next_uid) { - qe = GNUNET_DATASTORE_get_key (db_src, - 0, false, - NULL, GNUNET_BLOCK_TYPE_ANY, - 0, 1, - &do_put, NULL); + GNUNET_assert (NULL == qe); + qe = GNUNET_DATASTORE_get_key (datastore, + next_uid, + false /* random */, + NULL /* key */, + GNUNET_BLOCK_TYPE_ANY, + 0 /* queue_priority */, + 1 /* max_queue_size */, + &get_cb, + NULL /* proc_cls */); + if (NULL == qe) + { + FPRINTF (stderr, + _("Error queueing datastore GET operation\n")); + ret = 1; + GNUNET_SCHEDULER_shutdown (); + } } +/** + * Begin dumping the database. + */ +static void +start_dump () +{ + record_count = 0; + + if (NULL != file_name) + { + file_handle = GNUNET_DISK_file_open (file_name, + GNUNET_DISK_OPEN_WRITE | + GNUNET_DISK_OPEN_TRUNCATE | + GNUNET_DISK_OPEN_CREATE, + GNUNET_DISK_PERM_USER_READ | + GNUNET_DISK_PERM_USER_WRITE); + if (NULL == file_handle) + { + FPRINTF (stderr, + _("Unable to open dump file: %s\n"), + file_name); + ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } + } + else + { + file_handle = GNUNET_DISK_get_handle_from_int_fd (STDOUT_FILENO); + } + GNUNET_DISK_file_write (file_handle, MAGIC_BYTES, MAGIC_LEN); + do_get(0); +} + /** - * Main function that will be run by the scheduler. + * Continuation called to notify client about result of the + * operation. * * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used - * @param cfg configuration -- for destination datastore + * @param success GNUNET_SYSERR on failure (including timeout/queue drop) + * GNUNET_NO if content was already there + * GNUNET_YES (or other positive value) on success + * @param min_expiration minimum expiration time required for 0-priority content to be stored + * by the datacache at this time, zero for unknown, forever if we have no + * space for 0-priority content + * @param msg NULL on success, otherwise an error message */ static void -run (void *cls, char *const *args, const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *cfg) +put_cb (void *cls, + int32_t success, + struct GNUNET_TIME_Absolute min_expiration, + const char *msg) { - if (NULL == alternative_cfg) - return; /* nothing to be done */ - if (0 == strcmp (cfgfile, alternative_cfg)) + qe = NULL; + if (GNUNET_SYSERR == success) { - fprintf (stderr, - _("Cannot use the same configuration for source and destination\n")); + FPRINTF (stderr, + _("Failed to store item: %s, aborting\n"), + msg); ret = 1; + GNUNET_SCHEDULER_shutdown (); return; } - scfg = GNUNET_CONFIGURATION_create (); - if (GNUNET_OK != - GNUNET_CONFIGURATION_load (scfg, - alternative_cfg)) + + struct DataRecord dr; + ssize_t len; + + len = GNUNET_DISK_file_read (file_handle, &dr, sizeof (dr)); + if (0 == len) + { + FPRINTF (stderr, + _("Inserted %" PRIu64 " records\n"), + record_count); + ret = 0; + GNUNET_SCHEDULER_shutdown (); + return; + } + else if (sizeof (dr) != len) { - GNUNET_CONFIGURATION_destroy (scfg); + FPRINTF (stderr, + _("Short read from file: %zd bytes expecting %zd\n"), + len, + sizeof (dr)); ret = 1; + GNUNET_SCHEDULER_shutdown (); return; } - db_src = GNUNET_DATASTORE_connect (scfg); - if (NULL == db_src) + + const size_t size = ntohl (dr.size); + uint8_t data[size]; + len = GNUNET_DISK_file_read (file_handle, data, size); + if (size != len) { - GNUNET_CONFIGURATION_destroy (scfg); + FPRINTF (stderr, + _("Short read from file: %zd bytes expecting %zd\n"), + len, + size); ret = 1; + GNUNET_SCHEDULER_shutdown (); return; } - db_dst = GNUNET_DATASTORE_connect (cfg); - if (NULL == db_dst) + + record_count++; + qe = GNUNET_DATASTORE_put (datastore, + 0, + &dr.key, + size, + data, + ntohl (dr.type), + ntohl (dr.priority), + ntohl (dr.anonymity), + ntohl (dr.replication), + GNUNET_TIME_absolute_ntoh (dr.expiration), + 0, + 1, + &put_cb, + NULL); + if (NULL == qe) { - GNUNET_DATASTORE_disconnect (db_src, GNUNET_NO); - GNUNET_CONFIGURATION_destroy (scfg); + FPRINTF (stderr, + _("Error queueing datastore PUT operation\n")); ret = 1; + GNUNET_SCHEDULER_shutdown (); + } +} + + +/** + * Begin inserting into the database. + */ +static void +start_insert () +{ + record_count = 0; + + if (NULL != file_name) + { + file_handle = GNUNET_DISK_file_open (file_name, + GNUNET_DISK_OPEN_READ, + GNUNET_DISK_PERM_NONE); + if (NULL == file_handle) + { + FPRINTF (stderr, + _("Unable to open dump file: %s\n"), + file_name); + ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } + } + else + { + file_handle = GNUNET_DISK_get_handle_from_int_fd (STDIN_FILENO); + } + + uint8_t buf[MAGIC_LEN]; + ssize_t len; + + len = GNUNET_DISK_file_read (file_handle, buf, MAGIC_LEN); + if (len != MAGIC_LEN || + 0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN)) + { + FPRINTF (stderr, + _("Input file is not of a supported format\n")); return; } + put_cb (NULL, GNUNET_YES, GNUNET_TIME_UNIT_ZERO_ABS, NULL); +} + + +/** + * Main function that will be run by the scheduler. + * + * @param cls closure + * @param args remaining command-line arguments + * @param cfgfile name of the configuration file used + * @param cfg configuration + */ +static void +run (void *cls, + char *const *args, + const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL); - do_get (); + datastore = GNUNET_DATASTORE_connect (cfg); + if (NULL == datastore) + { + FPRINTF (stderr, + _("Failed connecting to the datastore.\n")); + ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } + if (dump) + start_dump(); + else if (insert) + start_insert(); + else + { + FPRINTF (stderr, + _("Please choose at least one operation: %s, %s\n"), + "dump", + "insert"); + ret = 1; + GNUNET_SCHEDULER_shutdown (); + } } @@ -247,14 +484,23 @@ run (void *cls, char *const *args, const char *cfgfile, * @return 0 ok, 1 on error */ int -main (int argc, char *const *argv) +main (int argc, + char *const *argv) { struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_filename ('s', - "sourcecfg", + GNUNET_GETOPT_option_flag ('d', + "dump", + gettext_noop ("Dump all records from the datastore"), + &dump), + GNUNET_GETOPT_option_flag ('i', + "insert", + gettext_noop ("Insert records into the datastore"), + &insert), + GNUNET_GETOPT_option_filename ('f', + "file", "FILENAME", - gettext_noop ("specifies the configuration to use to access an alternative datastore; will merge that datastore into our current datastore"), - &alternative_cfg), + gettext_noop ("File to dump or insert"), + &file_name), GNUNET_GETOPT_OPTION_END }; if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) -- 2.25.1