From 16642c4cb25faae335591b39795c71dedbeb37f1 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sun, 17 Apr 2016 15:09:23 +0000 Subject: [PATCH] adding libgnunetcurl --- configure.ac | 1 + po/POTFILES.in | 1 + src/Makefile.am | 7 + src/curl/Makefile.am | 33 ++ src/curl/curl.c | 574 ++++++++++++++++++++++++++++++++++ src/include/Makefile.am | 1 + src/include/gnunet_curl_lib.h | 155 +++++++++ src/json/json.c | 2 + 8 files changed, 774 insertions(+) create mode 100644 src/curl/Makefile.am create mode 100644 src/curl/curl.c create mode 100644 src/include/gnunet_curl_lib.h diff --git a/configure.ac b/configure.ac index 2b90bc1ee..369ad99ee 100644 --- a/configure.ac +++ b/configure.ac @@ -1534,6 +1534,7 @@ src/consensus/Makefile src/consensus/consensus.conf src/conversation/Makefile src/conversation/conversation.conf +src/curl/Makefile src/datacache/Makefile src/datastore/Makefile src/datastore/datastore.conf diff --git a/po/POTFILES.in b/po/POTFILES.in index 84bf8735e..06d180231 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -74,6 +74,7 @@ src/core/gnunet-service-core_kx.c src/core/gnunet-service-core_neighbours.c src/core/gnunet-service-core_sessions.c src/core/gnunet-service-core_typemap.c +src/curl/curl.c src/datacache/datacache.c src/datacache/plugin_datacache_heap.c src/datacache/plugin_datacache_postgres.c diff --git a/src/Makefile.am b/src/Makefile.am index f38dd19d3..5244c795d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -24,6 +24,13 @@ endif if HAVE_JSON JSON_DIR = json +if HAVE_LIBGNURL + JSON_DIR += curl +else +if HAVE_LIBCURL + JSON_DIR += curl +endif +endif endif if BUILD_PULSE_HELPERS diff --git a/src/curl/Makefile.am b/src/curl/Makefile.am new file mode 100644 index 000000000..4bdd03039 --- /dev/null +++ b/src/curl/Makefile.am @@ -0,0 +1,33 @@ +# This Makefile.am is in the public domain +AM_CPPFLAGS = -I$(top_srcdir)/src/include + +if USE_COVERAGE + AM_CFLAGS = --coverage -O0 + XLIB = -lgcov +endif + +lib_LTLIBRARIES = \ + libgnunetcurl.la + +libgnunetcurl_la_LDFLAGS = \ + -version-info 0:0:0 \ + -no-undefined +libgnunetcurl_la_SOURCES = \ + curl.c +libgnunetcurl_la_LIBADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + -ljansson \ + $(XLIB) + +#check_PROGRAMS = \ +# test_curl + +#TESTS = \ +# $(check_PROGRAMS) + +#test_curl_SOURCES = \ +# test_curl.c +#test_curl_LDADD = \ +# libgnunetcurl.la \ +# $(top_builddir)/src/util/libgnunetutil.la \ +# -ljansson -lcurl diff --git a/src/curl/curl.c b/src/curl/curl.c new file mode 100644 index 000000000..10ba7f719 --- /dev/null +++ b/src/curl/curl.c @@ -0,0 +1,574 @@ +/* + This file is part of GNUnet + Copyright (C) 2014, 2015, 2016 GNUnet e.V. + + GNUnet is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + GNUnet; see the file COPYING. If not, If not, see + +*/ +/** + * @file curl/curl.c + * @brief API for downloading JSON via CURL + * @author Sree Harsha Totakura + * @author Christian Grothoff + */ +#include "platform.h" +#include +#include +#include "gnunet_curl_lib.h" + + +/** + * Log error related to CURL operations. + * + * @param type log level + * @param function which function failed to run + * @param code what was the curl error code + */ +#define CURL_STRERROR(type, function, code) \ + GNUNET_log (type, \ + "Curl function `%s' has failed at `%s:%d' with error: %s\n", \ + function, __FILE__, __LINE__, curl_easy_strerror (code)); + +/** + * Print JSON parsing related error information + */ +#define JSON_WARN(error) \ + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, \ + "JSON parsing failed at %s:%u: %s (%s)\n", \ + __FILE__, __LINE__, error.text, error.source) + + +/** + * Failsafe flag. Raised if our constructor fails to initialize + * the Curl library. + */ +static int curl_fail; + + +/** + * @brief Buffer data structure we use to buffer the HTTP download + * before giving it to the JSON parser. + */ +struct DownloadBuffer +{ + + /** + * Download buffer + */ + void *buf; + + /** + * The size of the download buffer + */ + size_t buf_size; + + /** + * Error code (based on libc errno) if we failed to download + * (i.e. response too large). + */ + int eno; + +}; + + +/** + * Jobs are CURL requests running within a `struct GNUNET_CURL_Context`. + */ +struct GNUNET_CURL_Job +{ + + /** + * We keep jobs in a DLL. + */ + struct GNUNET_CURL_Job *next; + + /** + * We keep jobs in a DLL. + */ + struct GNUNET_CURL_Job *prev; + + /** + * Easy handle of the job. + */ + CURL *easy_handle; + + /** + * Context this job runs in. + */ + struct GNUNET_CURL_Context *ctx; + + /** + * Function to call upon completion. + */ + GNUNET_CURL_JobCompletionCallback jcc; + + /** + * Closure for @e jcc. + */ + void *jcc_cls; + + /** + * Buffer for response received from CURL. + */ + struct DownloadBuffer db; + +}; + + +/** + * Context + */ +struct GNUNET_CURL_Context +{ + /** + * Curl multi handle + */ + CURLM *multi; + + /** + * Curl share handle + */ + CURLSH *share; + + /** + * We keep jobs in a DLL. + */ + struct GNUNET_CURL_Job *jobs_head; + + /** + * We keep jobs in a DLL. + */ + struct GNUNET_CURL_Job *jobs_tail; + + /** + * HTTP header "application/json", created once and used + * for all requests that need it. + */ + struct curl_slist *json_header; + +}; + + +/** + * Initialise this library. This function should be called before using any of + * the following functions. + * + * @return library context + */ +struct GNUNET_CURL_Context * +GNUNET_CURL_init () +{ + struct GNUNET_CURL_Context *ctx; + CURLM *multi; + CURLSH *share; + + if (curl_fail) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Curl was not initialised properly\n"); + return NULL; + } + if (NULL == (multi = curl_multi_init ())) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to create a Curl multi handle\n"); + return NULL; + } + if (NULL == (share = curl_share_init ())) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Failed to create a Curl share handle\n"); + return NULL; + } + ctx = GNUNET_new (struct GNUNET_CURL_Context); + ctx->multi = multi; + ctx->share = share; + GNUNET_assert (NULL != (ctx->json_header = + curl_slist_append (NULL, + "Content-Type: application/json"))); + return ctx; +} + + +/** + * Callback used when downloading the reply to an HTTP request. + * Just appends all of the data to the `buf` in the + * `struct DownloadBuffer` for further processing. The size of + * the download is limited to #GNUNET_MAX_MALLOC_CHECKED, if + * the download exceeds this size, we abort with an error. + * + * @param bufptr data downloaded via HTTP + * @param size size of an item in @a bufptr + * @param nitems number of items in @a bufptr + * @param cls the `struct DownloadBuffer` + * @return number of bytes processed from @a bufptr + */ +static size_t +download_cb (char *bufptr, + size_t size, + size_t nitems, + void *cls) +{ + struct DownloadBuffer *db = cls; + size_t msize; + void *buf; + + if (0 == size * nitems) + { + /* Nothing (left) to do */ + return 0; + } + msize = size * nitems; + if ( (msize + db->buf_size) >= GNUNET_MAX_MALLOC_CHECKED) + { + db->eno = ENOMEM; + return 0; /* signals an error to curl */ + } + db->buf = GNUNET_realloc (db->buf, + db->buf_size + msize); + buf = db->buf + db->buf_size; + memcpy (buf, bufptr, msize); + db->buf_size += msize; + return msize; +} + + +/** + * Schedule a CURL request to be executed and call the given @a jcc + * upon its completion. Note that the context will make use of the + * CURLOPT_PRIVATE facility of the CURL @a eh. + * + * This function modifies the CURL handle to add the + * "Content-Type: application/json" header if @a add_json is set. + * + * @param ctx context to execute the job in + * @param eh curl easy handle for the request, will + * be executed AND cleaned up + * @param add_json add "application/json" content type header + * @param jcc callback to invoke upon completion + * @param jcc_cls closure for @a jcc + * @return NULL on error (in this case, @eh is still released!) + */ +struct GNUNET_CURL_Job * +GNUNET_CURL_job_add (struct GNUNET_CURL_Context *ctx, + CURL *eh, + int add_json, + GNUNET_CURL_JobCompletionCallback jcc, + void *jcc_cls) +{ + struct GNUNET_CURL_Job *job; + + if (GNUNET_YES == add_json) + if (CURLE_OK != + curl_easy_setopt (eh, + CURLOPT_HTTPHEADER, + ctx->json_header)) + { + GNUNET_break (0); + curl_easy_cleanup (eh); + return NULL; + } + + job = GNUNET_new (struct GNUNET_CURL_Job); + if ( (CURLE_OK != + curl_easy_setopt (eh, + CURLOPT_PRIVATE, + job)) || + (CURLE_OK != + curl_easy_setopt (eh, + CURLOPT_WRITEFUNCTION, + &download_cb)) || + (CURLE_OK != + curl_easy_setopt (eh, + CURLOPT_WRITEDATA, + &job->db)) || + (CURLE_OK != + curl_easy_setopt (eh, + CURLOPT_SHARE, + ctx->share)) || + (CURLM_OK != + curl_multi_add_handle (ctx->multi, + eh)) ) + { + GNUNET_break (0); + GNUNET_free (job); + curl_easy_cleanup (eh); + return NULL; + } + + job->easy_handle = eh; + job->ctx = ctx; + job->jcc = jcc; + job->jcc_cls = jcc_cls; + GNUNET_CONTAINER_DLL_insert (ctx->jobs_head, + ctx->jobs_tail, + job); + return job; +} + + +/** + * Cancel a job. Must only be called before the job completion + * callback is called for the respective job. + * + * @param job job to cancel + */ +void +GNUNET_CURL_job_cancel (struct GNUNET_CURL_Job *job) +{ + struct GNUNET_CURL_Context *ctx = job->ctx; + + GNUNET_CONTAINER_DLL_remove (ctx->jobs_head, + ctx->jobs_tail, + job); + GNUNET_break (CURLM_OK == + curl_multi_remove_handle (ctx->multi, + job->easy_handle)); + curl_easy_cleanup (job->easy_handle); + GNUNET_free_non_null (job->db.buf); + GNUNET_free (job); +} + + +/** + * Obtain information about the final result about the + * HTTP download. If the download was successful, parses + * the JSON in the @a db and returns it. Also returns + * the HTTP @a response_code. If the download failed, + * the return value is NULL. The response code is set + * in any case, on download errors to zero. + * + * Calling this function also cleans up @a db. + * + * @param db download buffer + * @param eh CURL handle (to get the response code) + * @param[out] response_code set to the HTTP response code + * (or zero if we aborted the download, i.e. + * because the response was too big, or if + * the JSON we received was malformed). + * @return NULL if downloading a JSON reply failed + */ +static json_t * +download_get_result (struct DownloadBuffer *db, + CURL *eh, + long *response_code) +{ + json_t *json; + json_error_t error; + char *ct; + + if ( (CURLE_OK != + curl_easy_getinfo (eh, + CURLINFO_CONTENT_TYPE, + &ct)) || + (NULL == ct) || + (0 != strcasecmp (ct, + "application/json")) ) + { + /* No content type or explicitly not JSON, refuse to parse + (but keep response code) */ + if (CURLE_OK != + curl_easy_getinfo (eh, + CURLINFO_RESPONSE_CODE, + response_code)) + { + /* unexpected error... */ + GNUNET_break (0); + *response_code = 0; + } + return NULL; + } + + json = NULL; + if (0 == db->eno) + { + json = json_loadb (db->buf, + db->buf_size, + JSON_REJECT_DUPLICATES | JSON_DISABLE_EOF_CHECK, + &error); + if (NULL == json) + { + JSON_WARN (error); + *response_code = 0; + } + } + GNUNET_free_non_null (db->buf); + db->buf = NULL; + db->buf_size = 0; + if (NULL != json) + { + if (CURLE_OK != + curl_easy_getinfo (eh, + CURLINFO_RESPONSE_CODE, + response_code)) + { + /* unexpected error... */ + GNUNET_break (0); + *response_code = 0; + } + } + return json; +} + + +/** + * Run the main event loop for the Taler interaction. + * + * @param ctx the library context + */ +void +GNUNET_CURL_perform (struct GNUNET_CURL_Context *ctx) +{ + CURLMsg *cmsg; + struct GNUNET_CURL_Job *job; + int n_running; + int n_completed; + long response_code; + json_t *j; + + (void) curl_multi_perform (ctx->multi, + &n_running); + while (NULL != (cmsg = curl_multi_info_read (ctx->multi, + &n_completed))) + { + /* Only documented return value is CURLMSG_DONE */ + GNUNET_break (CURLMSG_DONE == cmsg->msg); + GNUNET_assert (CURLE_OK == + curl_easy_getinfo (cmsg->easy_handle, + CURLINFO_PRIVATE, + (char **) &job)); + GNUNET_assert (job->ctx == ctx); + j = download_get_result (&job->db, + job->easy_handle, + &response_code); + job->jcc (job->jcc_cls, + response_code, + j); + GNUNET_CURL_job_cancel (job); + } +} + + +/** + * Obtain the information for a select() call to wait until + * #GNUNET_CURL_perform() is ready again. Note that calling + * any other GNUNET_CURL-API may also imply that the library + * is again ready for #GNUNET_CURL_perform(). + * + * Basically, a client should use this API to prepare for select(), + * then block on select(), then call #GNUNET_CURL_perform() and then + * start again until the work with the context is done. + * + * This function will NOT zero out the sets and assumes that @a max_fd + * and @a timeout are already set to minimal applicable values. It is + * safe to give this API FD-sets and @a max_fd and @a timeout that are + * already initialized to some other descriptors that need to go into + * the select() call. + * + * @param ctx context to get the event loop information for + * @param read_fd_set will be set for any pending read operations + * @param write_fd_set will be set for any pending write operations + * @param except_fd_set is here because curl_multi_fdset() has this argument + * @param max_fd set to the highest FD included in any set; + * if the existing sets have no FDs in it, the initial + * value should be "-1". (Note that `max_fd + 1` will need + * to be passed to select().) + * @param timeout set to the timeout in milliseconds (!); -1 means + * no timeout (NULL, blocking forever is OK), 0 means to + * proceed immediately with #GNUNET_CURL_perform(). + */ +void +GNUNET_CURL_get_select_info (struct GNUNET_CURL_Context *ctx, + fd_set *read_fd_set, + fd_set *write_fd_set, + fd_set *except_fd_set, + int *max_fd, + long *timeout) +{ + long to; + int m; + + m = -1; + GNUNET_assert (CURLM_OK == + curl_multi_fdset (ctx->multi, + read_fd_set, + write_fd_set, + except_fd_set, + &m)); + to = *timeout; + *max_fd = GNUNET_MAX (m, *max_fd); + GNUNET_assert (CURLM_OK == + curl_multi_timeout (ctx->multi, + &to)); + + /* Only if what we got back from curl is smaller than what we + already had (-1 == infinity!), then update timeout */ + if ( (to < *timeout) && + (-1 != to) ) + *timeout = to; + if ( (-1 == (*timeout)) && + (NULL != ctx->jobs_head) ) + *timeout = to; +} + + +/** + * Cleanup library initialisation resources. This function should be called + * after using this library to cleanup the resources occupied during library's + * initialisation. + * + * @param ctx the library context + */ +void +GNUNET_CURL_fini (struct GNUNET_CURL_Context *ctx) +{ + /* all jobs must have been cancelled at this time, assert this */ + GNUNET_assert (NULL == ctx->jobs_head); + curl_share_cleanup (ctx->share); + curl_multi_cleanup (ctx->multi); + curl_slist_free_all (ctx->json_header); + GNUNET_free (ctx); +} + + +/** + * Initial global setup logic, specifically runs the Curl setup. + */ +__attribute__ ((constructor)) +void +GNUNET_CURL_constructor__ (void) +{ + CURLcode ret; + + if (CURLE_OK != (ret = curl_global_init (CURL_GLOBAL_DEFAULT))) + { + CURL_STRERROR (GNUNET_ERROR_TYPE_ERROR, + "curl_global_init", + ret); + curl_fail = 1; + } +} + + +/** + * Cleans up after us, specifically runs the Curl cleanup. + */ +__attribute__ ((destructor)) +void +GNUNET_CURL_destructor__ (void) +{ + if (curl_fail) + return; + curl_global_cleanup (); +} + +/* end of curl.c */ diff --git a/src/include/Makefile.am b/src/include/Makefile.am index 82fa9006b..92893f7c7 100644 --- a/src/include/Makefile.am +++ b/src/include/Makefile.am @@ -45,6 +45,7 @@ gnunetinclude_HEADERS = \ gnunet_conversation_service.h \ gnunet_core_service.h \ gnunet_crypto_lib.h \ + gnunet_curl_lib.h \ gnunet_datacache_lib.h \ gnunet_datacache_plugin.h \ gnunet_datastore_service.h \ diff --git a/src/include/gnunet_curl_lib.h b/src/include/gnunet_curl_lib.h new file mode 100644 index 000000000..098b4dc37 --- /dev/null +++ b/src/include/gnunet_curl_lib.h @@ -0,0 +1,155 @@ +/* + This file is part of GNUnet + Copyright (C) 2014, 2015, 2016 GNUnet e.V. + + GNUnet is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + GNUnet; see the file COPYING. If not, If not, see + +*/ +/** + * @file src/include/gnunet_curl_lib.h + * @brief library to make it easy to download JSON replies over HTTP + * @author Sree Harsha Totakura + * @author Christian Grothoff + * + * @defgroup curl CURL integration library + * Download JSON using libcurl. + * @{ + */ +#ifndef GNUNET_CURL_LIB_H +#define GNUNET_CURL_LIB_H +#include +#include +#include + + +/** + * Initialise this library. This function should be called before using any of + * the following functions. + * + * @return library context + */ +struct GNUNET_CURL_Context * +GNUNET_CURL_init (void); + + +/** + * Obtain the information for a select() call to wait until + * #GNUNET_CURL_perform() is ready again. Note that calling + * any other TALER_EXCHANGE-API may also imply that the library + * is again ready for #GNUNET_CURL_perform(). + * + * Basically, a client should use this API to prepare for select(), + * then block on select(), then call #GNUNET_CURL_perform() and then + * start again until the work with the context is done. + * + * This function will NOT zero out the sets and assumes that @a max_fd + * and @a timeout are already set to minimal applicable values. It is + * safe to give this API FD-sets and @a max_fd and @a timeout that are + * already initialized to some other descriptors that need to go into + * the select() call. + * + * @param ctx context to get the event loop information for + * @param read_fd_set will be set for any pending read operations + * @param write_fd_set will be set for any pending write operations + * @param except_fd_set is here because curl_multi_fdset() has this argument + * @param max_fd set to the highest FD included in any set; + * if the existing sets have no FDs in it, the initial + * value should be "-1". (Note that `max_fd + 1` will need + * to be passed to select().) + * @param timeout set to the timeout in milliseconds (!); -1 means + * no timeout (NULL, blocking forever is OK), 0 means to + * proceed immediately with #GNUNET_CURL_perform(). + */ +void +GNUNET_CURL_get_select_info (struct GNUNET_CURL_Context *ctx, + fd_set *read_fd_set, + fd_set *write_fd_set, + fd_set *except_fd_set, + int *max_fd, + long *timeout); + + +/** + * Run the main event loop for the Taler interaction. + * + * @param ctx the library context + */ +void +GNUNET_CURL_perform (struct GNUNET_CURL_Context *ctx); + + +/** + * Cleanup library initialisation resources. This function should be called + * after using this library to cleanup the resources occupied during library's + * initialisation. + * + * @param ctx the library context + */ +void +GNUNET_CURL_fini (struct GNUNET_CURL_Context *ctx); + + +/** + * Entry in the context's job queue. + */ +struct GNUNET_CURL_Job; + +/** + * Function to call upon completion of a job. + * + * @param cls closure + * @param response_code HTTP response code from server, 0 on hard error + * @param json response, NULL if response was not in JSON format + */ +typedef void +(*GNUNET_CURL_JobCompletionCallback)(void *cls, + long response_code, + json_t *json); + + +/** + * Schedule a CURL request to be executed and call the given @a jcc + * upon its completion. Note that the context will make use of the + * CURLOPT_PRIVATE facility of the CURL @a eh. + * + * This function modifies the CURL handle to add the + * "Content-Type: application/json" header if @a add_json is set. + * + * @param ctx context to execute the job in + * @param eh curl easy handle for the request, will + * be executed AND cleaned up + * @param add_json add "application/json" content type header + * @param jcc callback to invoke upon completion + * @param jcc_cls closure for @a jcc + * @return NULL on error (in this case, @eh is still released!) + */ +struct GNUNET_CURL_Job * +GNUNET_CURL_job_add (struct GNUNET_CURL_Context *ctx, + CURL *eh, + int add_json, + GNUNET_CURL_JobCompletionCallback jcc, + void *jcc_cls); + + +/** + * Cancel a job. Must only be called before the job completion + * callback is called for the respective job. + * + * @param job job to cancel + */ +void +GNUNET_CURL_job_cancel (struct GNUNET_CURL_Job *job); + +#endif +/** @} */ /* end of group */ + +/* end of gnunet_curl_lib.h */ diff --git a/src/json/json.c b/src/json/json.c index aa74bfd48..a2d1a9608 100644 --- a/src/json/json.c +++ b/src/json/json.c @@ -46,6 +46,8 @@ GNUNET_JSON_parse (const json_t *root, unsigned int i; json_t *pos; + if (NULL == root) + return GNUNET_SYSERR; for (i=0;NULL != spec[i].parser;i++) { if (NULL == spec[i].field) -- 2.25.1