2 This file is part of GNUnet
3 Copyright (C) 2014, 2015, 2016, 2018 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
22 * @brief API for downloading JSON via CURL
23 * @author Sree Harsha Totakura <sreeharsha@totakura.in>
24 * @author Christian Grothoff
28 #include "gnunet_curl_lib.h"
31 #include "../util/benchmark.h"
36 * Log error related to CURL operations.
38 * @param type log level
39 * @param function which function failed to run
40 * @param code what was the curl error code
42 #define CURL_STRERROR(type, function, code) \
44 "Curl function `%s' has failed at `%s:%d' with error: %s\n", \
48 curl_easy_strerror(code));
51 * Print JSON parsing related error information
53 #define JSON_WARN(error) \
54 GNUNET_log(GNUNET_ERROR_TYPE_WARNING, \
55 "JSON parsing failed at %s:%u: %s (%s)\n", \
63 * Failsafe flag. Raised if our constructor fails to initialize
69 * Jobs are CURL requests running within a `struct GNUNET_CURL_Context`.
71 struct GNUNET_CURL_Job {
73 * We keep jobs in a DLL.
75 struct GNUNET_CURL_Job *next;
78 * We keep jobs in a DLL.
80 struct GNUNET_CURL_Job *prev;
83 * Easy handle of the job.
88 * Context this job runs in.
90 struct GNUNET_CURL_Context *ctx;
93 * Function to call upon completion.
95 GNUNET_CURL_JobCompletionCallback jcc;
103 * Buffer for response received from CURL.
105 struct GNUNET_CURL_DownloadBuffer db;
108 * Headers used for this job, the list needs to be freed
109 * after the job has finished.
111 struct curl_slist *job_headers;
118 struct GNUNET_CURL_Context {
130 * We keep jobs in a DLL.
132 struct GNUNET_CURL_Job *jobs_head;
135 * We keep jobs in a DLL.
137 struct GNUNET_CURL_Job *jobs_tail;
140 * Headers common for all requests in the context.
142 struct curl_slist *common_headers;
145 * If non-NULL, the async scope ID is sent in a request
146 * header of this name.
148 const char *async_scope_id_header;
151 * Function we need to call whenever the event loop's
152 * socket set changed.
154 GNUNET_CURL_RescheduleCallback cb;
164 * Initialise this library. This function should be called before using any of
165 * the following functions.
167 * @param cb function to call when rescheduling is required
168 * @param cb_cls closure for @a cb
169 * @return library context
171 struct GNUNET_CURL_Context *
172 GNUNET_CURL_init(GNUNET_CURL_RescheduleCallback cb, void *cb_cls)
174 struct GNUNET_CURL_Context *ctx;
180 GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Curl was not initialised properly\n");
183 if (NULL == (multi = curl_multi_init()))
185 GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
186 "Failed to create a Curl multi handle\n");
189 if (NULL == (share = curl_share_init()))
191 GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
192 "Failed to create a Curl share handle\n");
195 ctx = GNUNET_new(struct GNUNET_CURL_Context);
197 ctx->cb_cls = cb_cls;
205 * Enable sending the async scope ID as a header.
207 * @param ctx the context to enable this for
208 * @param header_name name of the header to send.
211 GNUNET_CURL_enable_async_scope_header(struct GNUNET_CURL_Context *ctx,
212 const char *header_name)
214 ctx->async_scope_id_header = header_name;
219 * Callback used when downloading the reply to an HTTP request.
220 * Just appends all of the data to the `buf` in the
221 * `struct DownloadBuffer` for further processing. The size of
222 * the download is limited to #GNUNET_MAX_MALLOC_CHECKED, if
223 * the download exceeds this size, we abort with an error.
225 * @param bufptr data downloaded via HTTP
226 * @param size size of an item in @a bufptr
227 * @param nitems number of items in @a bufptr
228 * @param cls the `struct DownloadBuffer`
229 * @return number of bytes processed from @a bufptr
232 download_cb(char *bufptr, size_t size, size_t nitems, void *cls)
234 struct GNUNET_CURL_DownloadBuffer *db = cls;
238 if (0 == size * nitems)
240 /* Nothing (left) to do */
243 msize = size * nitems;
244 if ((msize + db->buf_size) >= GNUNET_MAX_MALLOC_CHECKED)
247 return 0; /* signals an error to curl */
249 db->buf = GNUNET_realloc(db->buf, db->buf_size + msize);
250 buf = db->buf + db->buf_size;
251 GNUNET_memcpy(buf, bufptr, msize);
252 db->buf_size += msize;
258 * Schedule a CURL request to be executed and call the given @a jcc
259 * upon its completion. Note that the context will make use of the
260 * CURLOPT_PRIVATE facility of the CURL @a eh.
262 * This function modifies the CURL handle to add the
263 * "Content-Type: application/json" header if @a add_json is set.
265 * @param ctx context to execute the job in
266 * @param eh curl easy handle for the request, will be executed AND
267 * cleaned up. NOTE: the handle should _never_ have gotten
268 * any headers list, as that would then be ovverridden by
269 * @a jcc. Therefore, always pass custom headers as the
270 * @a job_headers parameter.
271 * @param job_headers extra headers to add for this request
272 * @param jcc callback to invoke upon completion
273 * @param jcc_cls closure for @a jcc
274 * @return NULL on error (in this case, @eh is still released!)
276 struct GNUNET_CURL_Job *
277 GNUNET_CURL_job_add2(struct GNUNET_CURL_Context *ctx,
279 const struct curl_slist *job_headers,
280 GNUNET_CURL_JobCompletionCallback jcc,
283 struct GNUNET_CURL_Job *job;
284 struct curl_slist *all_headers = NULL;
286 for (const struct curl_slist *curr = job_headers; curr != NULL;
289 GNUNET_assert(NULL !=
290 (all_headers = curl_slist_append(all_headers, curr->data)));
293 for (const struct curl_slist *curr = ctx->common_headers; curr != NULL;
296 GNUNET_assert(NULL !=
297 (all_headers = curl_slist_append(all_headers, curr->data)));
300 if (NULL != ctx->async_scope_id_header)
302 struct GNUNET_AsyncScopeSave scope;
304 GNUNET_async_scope_get(&scope);
305 if (GNUNET_YES == scope.have_scope)
307 char *aid_header = NULL;
309 GNUNET_STRINGS_data_to_string_alloc(&scope.scope_id,
311 struct GNUNET_AsyncScopeId));
312 GNUNET_assert(NULL != aid_header);
313 GNUNET_assert(NULL != curl_slist_append(all_headers, aid_header));
314 GNUNET_free(aid_header);
318 if (CURLE_OK != curl_easy_setopt(eh, CURLOPT_HTTPHEADER, all_headers))
321 curl_slist_free_all(all_headers);
322 curl_easy_cleanup(eh);
326 job = GNUNET_new(struct GNUNET_CURL_Job);
327 job->job_headers = all_headers;
329 if ((CURLE_OK != curl_easy_setopt(eh, CURLOPT_PRIVATE, job)) ||
331 curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, &download_cb)) ||
332 (CURLE_OK != curl_easy_setopt(eh, CURLOPT_WRITEDATA, &job->db)) ||
333 (CURLE_OK != curl_easy_setopt(eh, CURLOPT_SHARE, ctx->share)) ||
334 (CURLM_OK != curl_multi_add_handle(ctx->multi, eh)))
338 curl_easy_cleanup(eh);
342 job->easy_handle = eh;
345 job->jcc_cls = jcc_cls;
346 GNUNET_CONTAINER_DLL_insert(ctx->jobs_head, ctx->jobs_tail, job);
347 ctx->cb(ctx->cb_cls);
353 * Schedule a CURL request to be executed and call the given @a jcc
354 * upon its completion. Note that the context will make use of the
355 * CURLOPT_PRIVATE facility of the CURL @a eh.
357 * This function modifies the CURL handle to add the
358 * "Content-Type: application/json" header if @a add_json is set.
360 * @param ctx context to execute the job in
361 * @param eh curl easy handle for the request, will
362 * be executed AND cleaned up
363 * @param add_json add "application/json" content type header
364 * @param jcc callback to invoke upon completion
365 * @param jcc_cls closure for @a jcc
366 * @return NULL on error (in this case, @eh is still released!)
368 struct GNUNET_CURL_Job *
369 GNUNET_CURL_job_add(struct GNUNET_CURL_Context *ctx,
372 GNUNET_CURL_JobCompletionCallback jcc,
375 struct GNUNET_CURL_Job *job;
376 struct curl_slist *job_headers = NULL;
378 if (GNUNET_YES == add_json)
381 NULL != (job_headers =
382 curl_slist_append(NULL, "Content-Type: application/json")));
385 job = GNUNET_CURL_job_add2(ctx, eh, job_headers, jcc, jcc_cls);
386 curl_slist_free_all(job_headers);
392 * Cancel a job. Must only be called before the job completion
393 * callback is called for the respective job.
395 * @param job job to cancel
398 GNUNET_CURL_job_cancel(struct GNUNET_CURL_Job *job)
400 struct GNUNET_CURL_Context *ctx = job->ctx;
402 GNUNET_CONTAINER_DLL_remove(ctx->jobs_head, ctx->jobs_tail, job);
403 GNUNET_break(CURLM_OK ==
404 curl_multi_remove_handle(ctx->multi, job->easy_handle));
405 curl_easy_cleanup(job->easy_handle);
406 GNUNET_free_non_null(job->db.buf);
407 curl_slist_free_all(job->job_headers);
413 * Obtain information about the final result about the
414 * HTTP download. If the download was successful, parses
415 * the JSON in the @a db and returns it. Also returns
416 * the HTTP @a response_code. If the download failed,
417 * the return value is NULL. The response code is set
418 * in any case, on download errors to zero.
420 * Calling this function also cleans up @a db.
422 * @param db download buffer
423 * @param eh CURL handle (to get the response code)
424 * @param[out] response_code set to the HTTP response code
425 * (or zero if we aborted the download, i.e.
426 * because the response was too big, or if
427 * the JSON we received was malformed).
428 * @return NULL if downloading a JSON reply failed.
431 GNUNET_CURL_download_get_result_(struct GNUNET_CURL_DownloadBuffer *db,
439 GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
440 "Downloaded body: %.*s\n",
444 if ((CURLE_OK != curl_easy_getinfo(eh, CURLINFO_CONTENT_TYPE, &ct)) ||
445 (NULL == ct) || (0 != strcasecmp(ct, "application/json")))
447 /* No content type or explicitly not JSON, refuse to parse
448 (but keep response code) */
450 curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, response_code))
452 /* unexpected error... */
456 if (0 != db->buf_size)
457 GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
458 "Did NOT detect response as JSON\n");
464 json = json_loadb(db->buf,
466 JSON_REJECT_DUPLICATES | JSON_DISABLE_EOF_CHECK,
474 GNUNET_free_non_null(db->buf);
480 curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, response_code))
482 /* unexpected error... */
492 * Add custom request header.
494 * @param ctx cURL context.
495 * @param header header string; will be given to the context AS IS.
496 * @return #GNUNET_OK if no errors occurred, #GNUNET_SYSERR otherwise.
499 GNUNET_CURL_append_header(struct GNUNET_CURL_Context *ctx, const char *header)
501 ctx->common_headers = curl_slist_append(ctx->common_headers, header);
502 if (NULL == ctx->common_headers)
503 return GNUNET_SYSERR;
510 * Run the main event loop for the Taler interaction.
512 * @param ctx the library context
513 * @param rp parses the raw response returned from
515 * @param rc cleans/frees the response
518 GNUNET_CURL_perform2(struct GNUNET_CURL_Context *ctx,
519 GNUNET_CURL_RawParser rp,
520 GNUNET_CURL_ResponseCleaner rc)
526 (void)curl_multi_perform(ctx->multi, &n_running);
527 while (NULL != (cmsg = curl_multi_info_read(ctx->multi, &n_completed)))
529 struct GNUNET_CURL_Job *job;
533 /* Only documented return value is CURLMSG_DONE */
534 GNUNET_break(CURLMSG_DONE == cmsg->msg);
535 GNUNET_assert(CURLE_OK == curl_easy_getinfo(cmsg->easy_handle,
538 GNUNET_assert(job->ctx == ctx);
540 response = rp(&job->db, job->easy_handle, &response_code);
544 double total_as_double = 0;
545 struct GNUNET_TIME_Relative total;
546 struct UrlRequestData *urd;
547 /* Some care required, as curl is using data types (long vs curl_off_t vs
548 * double) inconsistently to store byte count. */
549 curl_off_t size_curl = 0;
551 uint64_t bytes_sent = 0;
552 uint64_t bytes_received = 0;
554 GNUNET_break(CURLE_OK == curl_easy_getinfo(cmsg->easy_handle,
557 total.rel_value_us = total_as_double * 1000 * 1000;
559 GNUNET_break(CURLE_OK == curl_easy_getinfo(cmsg->easy_handle,
560 CURLINFO_EFFECTIVE_URL,
563 /* HEADER_SIZE + SIZE_DOWNLOAD_T is hopefully the total
564 number of bytes received, not clear from curl docs. */
566 GNUNET_break(CURLE_OK == curl_easy_getinfo(cmsg->easy_handle,
567 CURLINFO_HEADER_SIZE,
569 bytes_received += size_long;
571 GNUNET_break(CURLE_OK == curl_easy_getinfo(cmsg->easy_handle,
572 CURLINFO_SIZE_DOWNLOAD_T,
574 bytes_received += size_curl;
576 /* REQUEST_SIZE + SIZE_UPLOAD_T is hopefully the total number of bytes
577 sent, again docs are not completely clear. */
579 GNUNET_break(CURLE_OK == curl_easy_getinfo(cmsg->easy_handle,
580 CURLINFO_REQUEST_SIZE,
582 bytes_sent += size_long;
584 /* We obtain this value to check an invariant, but never use it otherwise. */
585 GNUNET_break(CURLE_OK == curl_easy_getinfo(cmsg->easy_handle,
586 CURLINFO_SIZE_UPLOAD_T,
589 /* CURLINFO_SIZE_UPLOAD_T <= CURLINFO_REQUEST_SIZE should
592 curl -w "foo%{size_request} -XPOST --data "ABC" $URL
593 the CURLINFO_REQUEST_SIZE should be the whole size of the request
594 including headers and body.
596 GNUNET_break(size_curl <= size_long);
598 urd = get_url_benchmark_data(url, (unsigned int)response_code);
600 urd->time = GNUNET_TIME_relative_add(urd->time, total);
601 urd->time_max = GNUNET_TIME_relative_max(total, urd->time_max);
602 urd->bytes_sent += bytes_sent;
603 urd->bytes_received += bytes_received;
606 job->jcc(job->jcc_cls, response_code, response);
608 GNUNET_CURL_job_cancel(job);
614 * Run the main event loop for the Taler interaction.
616 * @param ctx the library context
619 GNUNET_CURL_perform(struct GNUNET_CURL_Context *ctx)
621 GNUNET_CURL_perform2(ctx,
622 &GNUNET_CURL_download_get_result_,
623 (GNUNET_CURL_ResponseCleaner) & json_decref);
628 * Obtain the information for a select() call to wait until
629 * #GNUNET_CURL_perform() is ready again. Note that calling
630 * any other GNUNET_CURL-API may also imply that the library
631 * is again ready for #GNUNET_CURL_perform().
633 * Basically, a client should use this API to prepare for select(),
634 * then block on select(), then call #GNUNET_CURL_perform() and then
635 * start again until the work with the context is done.
637 * This function will NOT zero out the sets and assumes that @a max_fd
638 * and @a timeout are already set to minimal applicable values. It is
639 * safe to give this API FD-sets and @a max_fd and @a timeout that are
640 * already initialized to some other descriptors that need to go into
643 * @param ctx context to get the event loop information for
644 * @param read_fd_set will be set for any pending read operations
645 * @param write_fd_set will be set for any pending write operations
646 * @param except_fd_set is here because curl_multi_fdset() has this argument
647 * @param max_fd set to the highest FD included in any set;
648 * if the existing sets have no FDs in it, the initial
649 * value should be "-1". (Note that `max_fd + 1` will need
650 * to be passed to select().)
651 * @param timeout set to the timeout in milliseconds (!); -1 means
652 * no timeout (NULL, blocking forever is OK), 0 means to
653 * proceed immediately with #GNUNET_CURL_perform().
656 GNUNET_CURL_get_select_info(struct GNUNET_CURL_Context *ctx,
658 fd_set *write_fd_set,
659 fd_set *except_fd_set,
667 GNUNET_assert(CURLM_OK == curl_multi_fdset(ctx->multi,
673 *max_fd = GNUNET_MAX(m, *max_fd);
674 GNUNET_assert(CURLM_OK == curl_multi_timeout(ctx->multi, &to));
676 /* Only if what we got back from curl is smaller than what we
677 already had (-1 == infinity!), then update timeout */
678 if ((to < *timeout) && (-1 != to))
680 if ((-1 == (*timeout)) && (NULL != ctx->jobs_head))
686 * Cleanup library initialisation resources. This function should be called
687 * after using this library to cleanup the resources occupied during library's
690 * @param ctx the library context
693 GNUNET_CURL_fini(struct GNUNET_CURL_Context *ctx)
695 /* all jobs must have been cancelled at this time, assert this */
696 GNUNET_assert(NULL == ctx->jobs_head);
697 curl_share_cleanup(ctx->share);
698 curl_multi_cleanup(ctx->multi);
699 curl_slist_free_all(ctx->common_headers);
705 * Initial global setup logic, specifically runs the Curl setup.
707 __attribute__ ((constructor)) void
708 GNUNET_CURL_constructor__(void)
712 if (CURLE_OK != (ret = curl_global_init(CURL_GLOBAL_DEFAULT)))
714 CURL_STRERROR(GNUNET_ERROR_TYPE_ERROR, "curl_global_init", ret);
721 * Cleans up after us, specifically runs the Curl cleanup.
723 __attribute__ ((destructor)) void
724 GNUNET_CURL_destructor__(void)
728 curl_global_cleanup();