2 This file is part of GNUnet
3 Copyright (C) 2014, 2015, 2016, 2018 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
22 * @brief API for downloading JSON via CURL
23 * @author Sree Harsha Totakura <sreeharsha@totakura.in>
24 * @author Christian Grothoff
28 #include "gnunet_curl_lib.h"
31 #include "../util/benchmark.h"
36 * Log error related to CURL operations.
38 * @param type log level
39 * @param function which function failed to run
40 * @param code what was the curl error code
42 #define CURL_STRERROR(type, function, code) \
44 "Curl function `%s' has failed at `%s:%d' with error: %s\n", \
48 curl_easy_strerror (code));
51 * Print JSON parsing related error information
53 #define JSON_WARN(error) \
54 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, \
55 "JSON parsing failed at %s:%u: %s (%s)\n", \
63 * Failsafe flag. Raised if our constructor fails to initialize
69 * Jobs are CURL requests running within a `struct GNUNET_CURL_Context`.
71 struct GNUNET_CURL_Job
74 * We keep jobs in a DLL.
76 struct GNUNET_CURL_Job *next;
79 * We keep jobs in a DLL.
81 struct GNUNET_CURL_Job *prev;
84 * Easy handle of the job.
89 * Context this job runs in.
91 struct GNUNET_CURL_Context *ctx;
94 * Function to call upon completion.
96 GNUNET_CURL_JobCompletionCallback jcc;
104 * Function to call upon completion.
106 GNUNET_CURL_RawJobCompletionCallback jcc_raw;
109 * Closure for @e jcc_raw.
114 * Buffer for response received from CURL.
116 struct GNUNET_CURL_DownloadBuffer db;
119 * Headers used for this job, the list needs to be freed
120 * after the job has finished.
122 struct curl_slist *job_headers;
129 struct GNUNET_CURL_Context
142 * We keep jobs in a DLL.
144 struct GNUNET_CURL_Job *jobs_head;
147 * We keep jobs in a DLL.
149 struct GNUNET_CURL_Job *jobs_tail;
152 * Headers common for all requests in the context.
154 struct curl_slist *common_headers;
157 * If non-NULL, the async scope ID is sent in a request
158 * header of this name.
160 const char *async_scope_id_header;
163 * Function we need to call whenever the event loop's
164 * socket set changed.
166 GNUNET_CURL_RescheduleCallback cb;
176 * Initialise this library. This function should be called before using any of
177 * the following functions.
179 * @param cb function to call when rescheduling is required
180 * @param cb_cls closure for @a cb
181 * @return library context
183 struct GNUNET_CURL_Context *
184 GNUNET_CURL_init (GNUNET_CURL_RescheduleCallback cb, void *cb_cls)
186 struct GNUNET_CURL_Context *ctx;
192 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
193 "Curl was not initialised properly\n");
196 if (NULL == (multi = curl_multi_init ()))
198 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
199 "Failed to create a Curl multi handle\n");
202 if (NULL == (share = curl_share_init ()))
204 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
205 "Failed to create a Curl share handle\n");
208 ctx = GNUNET_new (struct GNUNET_CURL_Context);
210 ctx->cb_cls = cb_cls;
218 * Enable sending the async scope ID as a header.
220 * @param ctx the context to enable this for
221 * @param header_name name of the header to send.
224 GNUNET_CURL_enable_async_scope_header (struct GNUNET_CURL_Context *ctx,
225 const char *header_name)
227 ctx->async_scope_id_header = header_name;
232 * Return #GNUNET_YES if given a valid scope ID and
233 * #GNUNET_NO otherwise. See #setup_job_headers,
235 * #GNUNET_CURL_enable_async_scope_header() for the
236 * code that generates such a @a scope_id.
238 * @returns #GNUNET_YES iff given a valid scope ID
241 GNUNET_CURL_is_valid_scope_id (const char *scope_id)
243 if (strlen (scope_id) >= 64)
245 for (size_t i = 0; i < strlen (scope_id); i++)
246 if (! (isalnum (scope_id[i]) || (scope_id[i] == '-')))
253 * Callback used when downloading the reply to an HTTP request.
254 * Just appends all of the data to the `buf` in the
255 * `struct DownloadBuffer` for further processing. The size of
256 * the download is limited to #GNUNET_MAX_MALLOC_CHECKED, if
257 * the download exceeds this size, we abort with an error.
259 * @param bufptr data downloaded via HTTP
260 * @param size size of an item in @a bufptr
261 * @param nitems number of items in @a bufptr
262 * @param cls the `struct DownloadBuffer`
263 * @return number of bytes processed from @a bufptr
266 download_cb (char *bufptr, size_t size, size_t nitems, void *cls)
268 struct GNUNET_CURL_DownloadBuffer *db = cls;
272 if (0 == size * nitems)
274 /* Nothing (left) to do */
277 msize = size * nitems;
278 if ((msize + db->buf_size) >= GNUNET_MAX_MALLOC_CHECKED)
281 return 0; /* signals an error to curl */
283 db->buf = GNUNET_realloc (db->buf, db->buf_size + msize);
284 buf = db->buf + db->buf_size;
285 GNUNET_memcpy (buf, bufptr, msize);
286 db->buf_size += msize;
292 * Create the HTTP headers for the request
294 * @param ctx context we run in
295 * @param job_headers job-specific headers
296 * @return all headers to use
298 static struct curl_slist *
299 setup_job_headers (struct GNUNET_CURL_Context *ctx,
300 const struct curl_slist *job_headers)
302 struct curl_slist *all_headers = NULL;
304 for (const struct curl_slist *curr = job_headers; curr != NULL;
307 GNUNET_assert (NULL !=
308 (all_headers = curl_slist_append (all_headers, curr->data)));
311 for (const struct curl_slist *curr = ctx->common_headers; curr != NULL;
314 GNUNET_assert (NULL !=
315 (all_headers = curl_slist_append (all_headers, curr->data)));
318 if (NULL != ctx->async_scope_id_header)
320 struct GNUNET_AsyncScopeSave scope;
322 GNUNET_async_scope_get (&scope);
323 if (GNUNET_YES == scope.have_scope)
325 char *aid_header = NULL;
327 GNUNET_STRINGS_data_to_string_alloc (&scope.scope_id,
329 struct GNUNET_AsyncScopeId));
330 GNUNET_assert (NULL != aid_header);
331 GNUNET_assert (NULL != curl_slist_append (all_headers, aid_header));
332 GNUNET_free (aid_header);
342 * @param eh easy handle to use
343 * @param ctx context to run the job in
344 * @param all_headers HTTP client headers to use (free'd)
345 * @return NULL on error
347 static struct GNUNET_CURL_Job *
349 struct GNUNET_CURL_Context *ctx,
350 struct curl_slist *all_headers)
352 struct GNUNET_CURL_Job *job;
355 curl_easy_setopt (eh, CURLOPT_HTTPHEADER, all_headers))
358 curl_slist_free_all (all_headers);
359 curl_easy_cleanup (eh);
362 job = GNUNET_new (struct GNUNET_CURL_Job);
363 job->job_headers = all_headers;
365 if ((CURLE_OK != curl_easy_setopt (eh, CURLOPT_PRIVATE, job)) ||
367 curl_easy_setopt (eh, CURLOPT_WRITEFUNCTION, &download_cb)) ||
368 (CURLE_OK != curl_easy_setopt (eh, CURLOPT_WRITEDATA, &job->db)) ||
369 (CURLE_OK != curl_easy_setopt (eh, CURLOPT_SHARE, ctx->share)) ||
370 (CURLM_OK != curl_multi_add_handle (ctx->multi, eh)))
374 curl_easy_cleanup (eh);
377 job->easy_handle = eh;
379 GNUNET_CONTAINER_DLL_insert (ctx->jobs_head, ctx->jobs_tail, job);
385 * Schedule a CURL request to be executed and call the given @a jcc
386 * upon its completion. Note that the context will make use of the
387 * CURLOPT_PRIVATE facility of the CURL @a eh. Used to download
388 * resources that are NOT in JSON. The raw body will be returned.
390 * @param ctx context to execute the job in
391 * @param eh curl easy handle for the request, will
392 * be executed AND cleaned up
393 * @param job_headers extra headers to add for this request
394 * @param max_reply_size largest acceptable response body
395 * @param jcc callback to invoke upon completion
396 * @param jcc_cls closure for @a jcc
397 * @return NULL on error (in this case, @eh is still released!)
399 struct GNUNET_CURL_Job *
400 GNUNET_CURL_job_add_raw (struct GNUNET_CURL_Context *ctx,
402 const struct curl_slist *job_headers,
403 GNUNET_CURL_RawJobCompletionCallback jcc,
406 struct GNUNET_CURL_Job *job;
407 struct curl_slist *all_headers;
409 GNUNET_assert (NULL != jcc);
410 all_headers = setup_job_headers (ctx,
412 if (NULL == (job = setup_job (eh,
417 job->jcc_raw_cls = jcc_cls;
418 ctx->cb (ctx->cb_cls);
424 * Schedule a CURL request to be executed and call the given @a jcc
425 * upon its completion. Note that the context will make use of the
426 * CURLOPT_PRIVATE facility of the CURL @a eh.
428 * This function modifies the CURL handle to add the
429 * "Content-Type: application/json" header if @a add_json is set.
431 * @param ctx context to execute the job in
432 * @param eh curl easy handle for the request, will be executed AND
433 * cleaned up. NOTE: the handle should _never_ have gotten
434 * any headers list, as that would then be ovverridden by
435 * @a jcc. Therefore, always pass custom headers as the
436 * @a job_headers parameter.
437 * @param job_headers extra headers to add for this request
438 * @param jcc callback to invoke upon completion
439 * @param jcc_cls closure for @a jcc
440 * @return NULL on error (in this case, @eh is still released!)
442 struct GNUNET_CURL_Job *
443 GNUNET_CURL_job_add2 (struct GNUNET_CURL_Context *ctx,
445 const struct curl_slist *job_headers,
446 GNUNET_CURL_JobCompletionCallback jcc,
449 struct GNUNET_CURL_Job *job;
450 struct curl_slist *all_headers;
452 GNUNET_assert (NULL != jcc);
453 all_headers = setup_job_headers (ctx,
455 if (NULL == (job = setup_job (eh,
461 job->jcc_cls = jcc_cls;
462 ctx->cb (ctx->cb_cls);
468 * Schedule a CURL request to be executed and call the given @a jcc
469 * upon its completion. Note that the context will make use of the
470 * CURLOPT_PRIVATE facility of the CURL @a eh.
472 * This function modifies the CURL handle to add the
473 * "Content-Type: application/json" header if @a add_json is set.
475 * @param ctx context to execute the job in
476 * @param eh curl easy handle for the request, will
477 * be executed AND cleaned up
478 * @param add_json add "application/json" content type header
479 * @param jcc callback to invoke upon completion
480 * @param jcc_cls closure for @a jcc
481 * @return NULL on error (in this case, @eh is still released!)
483 struct GNUNET_CURL_Job *
484 GNUNET_CURL_job_add (struct GNUNET_CURL_Context *ctx,
487 GNUNET_CURL_JobCompletionCallback jcc,
490 struct GNUNET_CURL_Job *job;
491 struct curl_slist *job_headers = NULL;
493 if (GNUNET_YES == add_json)
496 NULL != (job_headers =
497 curl_slist_append (NULL, "Content-Type: application/json")));
500 job = GNUNET_CURL_job_add2 (ctx, eh, job_headers, jcc, jcc_cls);
501 curl_slist_free_all (job_headers);
507 * Cancel a job. Must only be called before the job completion
508 * callback is called for the respective job.
510 * @param job job to cancel
513 GNUNET_CURL_job_cancel (struct GNUNET_CURL_Job *job)
515 struct GNUNET_CURL_Context *ctx = job->ctx;
517 GNUNET_CONTAINER_DLL_remove (ctx->jobs_head, ctx->jobs_tail, job);
518 GNUNET_break (CURLM_OK ==
519 curl_multi_remove_handle (ctx->multi, job->easy_handle));
520 curl_easy_cleanup (job->easy_handle);
521 GNUNET_free_non_null (job->db.buf);
522 curl_slist_free_all (job->job_headers);
523 ctx->cb (ctx->cb_cls);
529 * Obtain information about the final result about the
530 * HTTP download. If the download was successful, parses
531 * the JSON in the @a db and returns it. Also returns
532 * the HTTP @a response_code. If the download failed,
533 * the return value is NULL. The response code is set
534 * in any case, on download errors to zero.
536 * Calling this function also cleans up @a db.
538 * @param db download buffer
539 * @param eh CURL handle (to get the response code)
540 * @param[out] response_code set to the HTTP response code
541 * (or zero if we aborted the download, i.e.
542 * because the response was too big, or if
543 * the JSON we received was malformed).
544 * @return NULL if downloading a JSON reply failed.
547 GNUNET_CURL_download_get_result_ (struct GNUNET_CURL_DownloadBuffer *db,
555 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
556 "Downloaded body: %.*s\n",
561 curl_easy_getinfo (eh,
562 CURLINFO_CONTENT_TYPE,
565 (0 != strcasecmp (ct,
566 "application/json")))
568 /* No content type or explicitly not JSON, refuse to parse
569 (but keep response code) */
571 curl_easy_getinfo (eh,
572 CURLINFO_RESPONSE_CODE,
575 /* unexpected error... */
579 if (0 != db->buf_size)
580 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
581 "Did NOT detect response `%.*s' as JSON\n",
583 (const char *) db->buf);
589 json = json_loadb (db->buf,
591 JSON_REJECT_DUPLICATES | JSON_DISABLE_EOF_CHECK,
599 GNUNET_free_non_null (db->buf);
605 curl_easy_getinfo (eh,
606 CURLINFO_RESPONSE_CODE,
609 /* unexpected error... */
619 * Add custom request header.
621 * @param ctx cURL context.
622 * @param header header string; will be given to the context AS IS.
623 * @return #GNUNET_OK if no errors occurred, #GNUNET_SYSERR otherwise.
626 GNUNET_CURL_append_header (struct GNUNET_CURL_Context *ctx, const char *header)
628 ctx->common_headers = curl_slist_append (ctx->common_headers, header);
629 if (NULL == ctx->common_headers)
630 return GNUNET_SYSERR;
638 do_benchmark (CURLMsg *cmsg)
641 double total_as_double = 0;
642 struct GNUNET_TIME_Relative total;
643 struct UrlRequestData *urd;
644 /* Some care required, as curl is using data types (long vs curl_off_t vs
645 * double) inconsistently to store byte count. */
646 curl_off_t size_curl = 0;
648 uint64_t bytes_sent = 0;
649 uint64_t bytes_received = 0;
651 GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
654 total.rel_value_us = total_as_double * 1000 * 1000;
656 GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
657 CURLINFO_EFFECTIVE_URL,
660 /* HEADER_SIZE + SIZE_DOWNLOAD_T is hopefully the total
661 number of bytes received, not clear from curl docs. */
663 GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
664 CURLINFO_HEADER_SIZE,
666 bytes_received += size_long;
668 GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
669 CURLINFO_SIZE_DOWNLOAD_T,
671 bytes_received += size_curl;
673 /* REQUEST_SIZE + SIZE_UPLOAD_T is hopefully the total number of bytes
674 sent, again docs are not completely clear. */
676 GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
677 CURLINFO_REQUEST_SIZE,
679 bytes_sent += size_long;
681 /* We obtain this value to check an invariant, but never use it otherwise. */
682 GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
683 CURLINFO_SIZE_UPLOAD_T,
686 /* CURLINFO_SIZE_UPLOAD_T <= CURLINFO_REQUEST_SIZE should
689 curl -w "foo%{size_request} -XPOST --data "ABC" $URL
690 the CURLINFO_REQUEST_SIZE should be the whole size of the request
691 including headers and body.
692 */GNUNET_break (size_curl <= size_long);
694 urd = get_url_benchmark_data (url, (unsigned int) response_code);
696 urd->time = GNUNET_TIME_relative_add (urd->time, total);
697 urd->time_max = GNUNET_TIME_relative_max (total, urd->time_max);
698 urd->bytes_sent += bytes_sent;
699 urd->bytes_received += bytes_received;
707 * Run the main event loop for the HTTP interaction.
709 * @param ctx the library context
710 * @param rp parses the raw response returned from
712 * @param rc cleans/frees the response
715 GNUNET_CURL_perform2 (struct GNUNET_CURL_Context *ctx,
716 GNUNET_CURL_RawParser rp,
717 GNUNET_CURL_ResponseCleaner rc)
723 (void) curl_multi_perform (ctx->multi,
725 while (NULL != (cmsg = curl_multi_info_read (ctx->multi, &n_completed)))
727 struct GNUNET_CURL_Job *job;
731 /* Only documented return value is CURLMSG_DONE */
732 GNUNET_break (CURLMSG_DONE == cmsg->msg);
733 GNUNET_assert (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
736 GNUNET_assert (job->ctx == ctx);
738 if (NULL != job->jcc_raw)
740 /* RAW mode, no parsing */
741 GNUNET_break (CURLE_OK ==
742 curl_easy_getinfo (job->easy_handle,
743 CURLINFO_RESPONSE_CODE,
745 job->jcc_raw (job->jcc_raw_cls,
752 /* to be parsed via 'rp' */
753 response = rp (&job->db,
756 job->jcc (job->jcc_cls,
764 GNUNET_CURL_job_cancel (job);
770 * Run the main event loop for the HTTP interaction.
772 * @param ctx the library context
775 GNUNET_CURL_perform (struct GNUNET_CURL_Context *ctx)
777 GNUNET_CURL_perform2 (ctx,
778 &GNUNET_CURL_download_get_result_,
779 (GNUNET_CURL_ResponseCleaner) & json_decref);
784 * Obtain the information for a select() call to wait until
785 * #GNUNET_CURL_perform() is ready again. Note that calling
786 * any other GNUNET_CURL-API may also imply that the library
787 * is again ready for #GNUNET_CURL_perform().
789 * Basically, a client should use this API to prepare for select(),
790 * then block on select(), then call #GNUNET_CURL_perform() and then
791 * start again until the work with the context is done.
793 * This function will NOT zero out the sets and assumes that @a max_fd
794 * and @a timeout are already set to minimal applicable values. It is
795 * safe to give this API FD-sets and @a max_fd and @a timeout that are
796 * already initialized to some other descriptors that need to go into
799 * @param ctx context to get the event loop information for
800 * @param read_fd_set will be set for any pending read operations
801 * @param write_fd_set will be set for any pending write operations
802 * @param except_fd_set is here because curl_multi_fdset() has this argument
803 * @param max_fd set to the highest FD included in any set;
804 * if the existing sets have no FDs in it, the initial
805 * value should be "-1". (Note that `max_fd + 1` will need
806 * to be passed to select().)
807 * @param timeout set to the timeout in milliseconds (!); -1 means
808 * no timeout (NULL, blocking forever is OK), 0 means to
809 * proceed immediately with #GNUNET_CURL_perform().
812 GNUNET_CURL_get_select_info (struct GNUNET_CURL_Context *ctx,
814 fd_set *write_fd_set,
815 fd_set *except_fd_set,
823 GNUNET_assert (CURLM_OK == curl_multi_fdset (ctx->multi,
829 *max_fd = GNUNET_MAX (m, *max_fd);
830 GNUNET_assert (CURLM_OK == curl_multi_timeout (ctx->multi, &to));
832 /* Only if what we got back from curl is smaller than what we
833 already had (-1 == infinity!), then update timeout */
834 if ((to < *timeout) && (-1 != to))
836 if ((-1 == (*timeout)) && (NULL != ctx->jobs_head))
842 * Cleanup library initialisation resources. This function should be called
843 * after using this library to cleanup the resources occupied during library's
846 * @param ctx the library context
849 GNUNET_CURL_fini (struct GNUNET_CURL_Context *ctx)
851 /* all jobs must have been cancelled at this time, assert this */
852 GNUNET_assert (NULL == ctx->jobs_head);
853 curl_share_cleanup (ctx->share);
854 curl_multi_cleanup (ctx->multi);
855 curl_slist_free_all (ctx->common_headers);
861 * Initial global setup logic, specifically runs the Curl setup.
863 __attribute__ ((constructor)) void
864 GNUNET_CURL_constructor__ (void)
868 if (CURLE_OK != (ret = curl_global_init (CURL_GLOBAL_DEFAULT)))
870 CURL_STRERROR (GNUNET_ERROR_TYPE_ERROR, "curl_global_init", ret);
877 * Cleans up after us, specifically runs the Curl cleanup.
879 __attribute__ ((destructor)) void
880 GNUNET_CURL_destructor__ (void)
884 curl_global_cleanup ();