From 782565417337605572b66758bf13d7123e26f744 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Fri, 3 May 2019 16:18:59 +0200 Subject: [PATCH] support compressed JSON uploads --- src/json/.gitignore | 1 + src/json/Makefile.am | 5 +- src/json/json_mhd.c | 122 +++++++++++++++++++++++++++++++++++++++ src/json/test_json_mhd.c | 47 +++++++++++++-- 4 files changed, 169 insertions(+), 6 deletions(-) diff --git a/src/json/.gitignore b/src/json/.gitignore index 6709c749d..347bffd7b 100644 --- a/src/json/.gitignore +++ b/src/json/.gitignore @@ -1 +1,2 @@ test_json +test_json_mhd diff --git a/src/json/Makefile.am b/src/json/Makefile.am index f030c3016..dfe185d5e 100644 --- a/src/json/Makefile.am +++ b/src/json/Makefile.am @@ -22,7 +22,9 @@ libgnunetjson_la_LIBADD = \ $(top_builddir)/src/util/libgnunetutil.la \ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ -ljansson \ - $(XLIB) + -lmicrohttpd \ + $(XLIB) \ + $(Z_LIBS) check_PROGRAMS = \ test_json \ @@ -57,6 +59,7 @@ test_json_mhd_LDADD = \ $(top_builddir)/src/util/libgnunetutil.la \ -ljansson \ -lmicrohttpd \ + $(Z_LIBS) \ $(LIB_GNURL) test_json_mhd_CPPFLAGS = \ $(CPP_GNURL) $(AM_CPPFLAGS) diff --git a/src/json/json_mhd.c b/src/json/json_mhd.c index 30b29b88e..b6ab2d116 100644 --- a/src/json/json_mhd.c +++ b/src/json/json_mhd.c @@ -26,6 +26,7 @@ */ #include "platform.h" #include "gnunet_json_lib.h" +#include /** @@ -55,6 +56,11 @@ struct Buffer * Number of allocated bytes in buffer. */ size_t alloc; + + /** + * Maximum buffer size allowed. + */ + size_t max; }; @@ -80,7 +86,10 @@ buffer_init (struct Buffer *buf, if (data_size > alloc_size) alloc_size = data_size; buf->data = GNUNET_malloc (alloc_size); + buf->alloc = alloc_size; GNUNET_memcpy (buf->data, data, data_size); + buf->fill = data_size; + buf->max = max_size; return GNUNET_OK; } @@ -137,6 +146,99 @@ buffer_append (struct Buffer *buf, } +/** + * Decompress data in @a buf. + * + * @param buf input data to inflate + * @return result code indicating the status of the operation + */ +static enum GNUNET_JSON_PostResult +inflate_data (struct Buffer *buf) +{ + z_stream z; + char *tmp; + size_t tmp_size; + int ret; + + memset (&z, 0, sizeof (z)); + z.next_in = (Bytef *) buf->data; + z.avail_in = buf->fill; + tmp_size = GNUNET_MIN (buf->max, buf->fill * 4); + tmp = GNUNET_malloc (tmp_size); + z.next_out = (Bytef *) tmp; + z.avail_out = tmp_size; + ret = inflateInit (&z); + switch (ret) + { + case Z_MEM_ERROR: + GNUNET_break (0); + return GNUNET_JSON_PR_OUT_OF_MEMORY; + case Z_STREAM_ERROR: + GNUNET_break_op (0); + return GNUNET_JSON_PR_JSON_INVALID; + case Z_OK: + break; + } + while (1) + { + ret = inflate (&z, 0); + switch (ret) + { + case Z_MEM_ERROR: + GNUNET_break (0); + GNUNET_break (Z_OK == inflateEnd (&z)); + GNUNET_free (tmp); + return GNUNET_JSON_PR_OUT_OF_MEMORY; + case Z_DATA_ERROR: + GNUNET_break (0); + GNUNET_break (Z_OK == inflateEnd (&z)); + GNUNET_free (tmp); + return GNUNET_JSON_PR_JSON_INVALID; + case Z_NEED_DICT: + GNUNET_break (0); + GNUNET_break (Z_OK == inflateEnd (&z)); + GNUNET_free (tmp); + return GNUNET_JSON_PR_JSON_INVALID; + case Z_OK: + if ((0 < z.avail_out) && (0 == z.avail_in)) + { + /* truncated input stream */ + GNUNET_break (0); + GNUNET_break (Z_OK == inflateEnd (&z)); + GNUNET_free (tmp); + return GNUNET_JSON_PR_JSON_INVALID; + } + if (0 < z.avail_out) + continue; /* just call it again */ + /* output buffer full, can we grow it? */ + if (tmp_size == buf->max) + { + /* already at max */ + GNUNET_break (0); + GNUNET_break (Z_OK == inflateEnd (&z)); + GNUNET_free (tmp); + return GNUNET_JSON_PR_OUT_OF_MEMORY; + } + if (tmp_size * 2 < tmp_size) + tmp_size = buf->max; + else + tmp_size = GNUNET_MIN (buf->max, tmp_size * 2); + tmp = GNUNET_realloc (tmp, tmp_size); + z.next_out = (Bytef *) &tmp[z.total_out]; + continue; + case Z_STREAM_END: + /* decompression successful, make 'tmp' the new 'data' */ + GNUNET_free (buf->data); + buf->data = tmp; + buf->alloc = tmp_size; + buf->fill = z.total_out; + GNUNET_break (Z_OK == inflateEnd (&z)); + return GNUNET_JSON_PR_SUCCESS; /* at least for now */ + } + } /* while (1) */ +} + + /** * Process a POST request containing a JSON object. This function * realizes an MHD POST processor that will (incrementally) process @@ -161,10 +263,13 @@ GNUNET_JSON_post_parser (size_t buffer_max, json_t **json) { struct Buffer *r = *con_cls; + const char *ce; + int ret; *json = NULL; if (NULL == *con_cls) { + /* We are seeing a fresh POST request. */ r = GNUNET_new (struct Buffer); if (GNUNET_OK != buffer_init (r, @@ -202,12 +307,29 @@ GNUNET_JSON_post_parser (size_t buffer_max, } /* We have seen the whole request. */ + ce = MHD_lookup_connection_value (connection, + MHD_HEADER_KIND, + MHD_HTTP_HEADER_CONTENT_ENCODING); + if ((NULL != ce) && (0 == strcasecmp ("deflate", ce))) + { + ret = inflate_data (r); + if (GNUNET_JSON_PR_SUCCESS != ret) + { + buffer_deinit (r); + GNUNET_free (r); + *con_cls = NULL; + return ret; + } + } *json = json_loadb (r->data, r->fill, 0, NULL); if (NULL == *json) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to parse JSON request body\n"); + buffer_deinit (r); + GNUNET_free (r); + *con_cls = NULL; return GNUNET_JSON_PR_JSON_INVALID; } buffer_deinit (r); diff --git a/src/json/test_json_mhd.c b/src/json/test_json_mhd.c index 665fd140e..13338b32c 100644 --- a/src/json/test_json_mhd.c +++ b/src/json/test_json_mhd.c @@ -27,6 +27,7 @@ #include "gnunet_util_lib.h" #include "gnunet_json_lib.h" #include "gnunet_curl_lib.h" +#include #define MAX_SIZE 1024 * 1024 @@ -69,7 +70,7 @@ access_handler_cb (void *cls, global_ret = 6; } json_decref (json); - resp = MHD_create_response_from_buffer (2, "OK", MHD_RESPMEM_PERSISTENT); + resp = MHD_create_response_from_buffer (3, "OK\n", MHD_RESPMEM_PERSISTENT); ret = MHD_queue_response (connection, MHD_HTTP_OK, resp); MHD_destroy_response (resp); return ret; @@ -100,8 +101,12 @@ main (int argc, const char *const argv[]) uint16_t port; CURL *easy; char *url; + char *str; + size_t slen; long post_data_size; void *post_data; + uLongf dlen; + struct curl_slist *json_header; GNUNET_log_setup ("test-json-mhd", "WARNING", NULL); global_ret = 2; @@ -123,28 +128,60 @@ main (int argc, const char *const argv[]) GNUNET_snprintf (tmp, sizeof (tmp), "%u", i); json_object_set_new (bigj, tmp, json_string (tmp)); } - post_data = json_dumps (bigj, JSON_INDENT (2)); - post_data_size = strlen (post_data); - + str = json_dumps (bigj, JSON_INDENT (2)); + slen = strlen (str); + +#ifdef compressBound + dlen = compressBound (slen); +#else + dlen = slen + slen / 100 + 20; + /* documentation says 100.1% oldSize + 12 bytes, but we + * should be able to overshoot by more to be safe */ +#endif + post_data = GNUNET_malloc (dlen); + if (Z_OK != + compress2 ((Bytef *) post_data, &dlen, (const Bytef *) str, slen, 9)) + { + GNUNET_break (0); + MHD_stop_daemon (daemon); + GNUNET_free (url); + json_decref (bigj); + GNUNET_free (post_data); + GNUNET_free (str); + return 1; + } + post_data_size = (long) dlen; port = MHD_get_daemon_info (daemon, MHD_DAEMON_INFO_BIND_PORT)->port; easy = curl_easy_init (); GNUNET_asprintf (&url, "http://localhost:%u/", (unsigned int) port); - curl_easy_setopt (easy, CURLOPT_VERBOSE, 1); + curl_easy_setopt (easy, CURLOPT_VERBOSE, 0); curl_easy_setopt (easy, CURLOPT_URL, url); curl_easy_setopt (easy, CURLOPT_POST, 1); curl_easy_setopt (easy, CURLOPT_POSTFIELDS, post_data); curl_easy_setopt (easy, CURLOPT_POSTFIELDSIZE, post_data_size); + + json_header = curl_slist_append (NULL, "Content-Type: application/json"); + json_header = curl_slist_append (json_header, "Content-Encoding: deflate"); + curl_easy_setopt (easy, CURLOPT_HTTPHEADER, json_header); if (0 != curl_easy_perform (easy)) { GNUNET_break (0); MHD_stop_daemon (daemon); GNUNET_free (url); json_decref (bigj); + GNUNET_free (post_data); + GNUNET_free (str); + curl_slist_free_all (json_header); + curl_easy_cleanup (easy); return 1; } MHD_stop_daemon (daemon); GNUNET_free (url); json_decref (bigj); + GNUNET_free (post_data); + GNUNET_free (str); + curl_slist_free_all (json_header); + curl_easy_cleanup (easy); return global_ret; } -- 2.25.1