#include <gnunet_util_lib.h>
#include <microhttpd.h>
#include <curl/curl.h>
+#include <regex.h>
#include "gns_proxy_proto.h"
#include "gns.h"
#define BUF_WAIT_FOR_CURL 0
#define BUF_WAIT_FOR_MHD 1
+#define HTML_HDR_CONTENT "Content-Type: text/html\r\n"
+#define RE_DOTPLUS "<a href=\"http://(([A-Za-z]+[.])+)([+])"
+#define RE_N_MATCHES 4
+
+#define HTTP_PORT 80
+#define HTTPS_PORT 443
struct ProxyCurlTask
{
struct ProxyCurlTask *next;
CURL *curl;
+ char url[2048];
char buffer[CURL_MAX_WRITE_SIZE];
+ char *buffer_ptr;
int buf_status;
unsigned int bytes_downloaded;
unsigned int bytes_in_buffer;
int download_successful;
int download_error;
struct MHD_Connection *connection;
+ int parse_content;
+ int is_postprocessing;
+
+ GNUNET_SCHEDULER_TaskIdentifier pp_task;
+
+ char pp_buf[256];
};
struct ProxyCurlTask *ctasks_head;
struct ProxyCurlTask *ctasks_tail;
+static regex_t re_dotplus;
+
+/**
+ * Read HTTP request header field 'Host'
+ *
+ * @param cls buffer to write to
+ * @param kind value kind
+ * @param key field key
+ * @param value field value
+ * @return MHD_NO when Host found
+ */
static int
con_val_iter (void *cls,
enum MHD_ValueKind kind,
return MHD_YES;
}
+
+/**
+ * Check HTTP response header for mime
+ *
+ * @param buffer curl buffer
+ * @param size curl blocksize
+ * @param nmemb curl blocknumber
+ * @param cls handle
+ * @return size of read bytes
+ */
+static size_t
+curl_check_hdr (void *buffer, size_t size, size_t nmemb, void *cls)
+{
+ size_t bytes = size * nmemb;
+ struct ProxyCurlTask *ctask = cls;
+ char hdr[bytes+1];
+
+ memcpy (hdr, buffer, bytes);
+ hdr[bytes] = '\0';
+
+ if (0 == strcmp (hdr, HTML_HDR_CONTENT))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Got HTML HTTP response header\n");
+ ctask->parse_content = GNUNET_YES;
+ }
+
+ return bytes;
+}
+
+/**
+ * schedule mhd
+ */
+static void
+run_httpd (void);
+
+
/**
* Process cURL download bits
*
size_t total;
struct ProxyCurlTask *ctask = ctx;
+ MHD_run (httpd);
+
total = size*nmemb;
ctask->bytes_downloaded += total;
if (total > sizeof (ctask->buffer))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- "cURL gave us too much data to handle (%d)!\n",
+ "CURL gave us too much data to handle (%d)!\n",
total);
return 0;
}
-
+
if (ctask->buf_status == BUF_WAIT_FOR_MHD)
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Waiting for MHD\n");
+ "CURL: Waiting for MHD (%s)\n", ctask->url);
return CURL_WRITEFUNC_PAUSE;
}
+
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "CURL: Copying to MHD (%s, %d)\n", ctask->url, total);
memcpy (ctask->buffer, cbuf, total);
ctask->bytes_in_buffer = total;
+ ctask->buffer_ptr = ctask->buffer;
ctask->buf_status = BUF_WAIT_FOR_MHD;
return total;
}
+
+
+/**
+ * Callback to free content
+ *
+ * @param cls content to free
+ */
+static void
+mhd_content_free (void *cls)
+{
+ struct ProxyCurlTask *ctask = cls;
+
+ if (ctask->curl != NULL)
+ curl_easy_cleanup (ctask->curl);
+
+ GNUNET_free (ctask);
+
+}
+
+static void
+postprocess_name (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+ struct ProxyCurlTask *ctask = cls;
+ char tmp[strlen(ctask->pp_buf)];
+
+ sprintf ( tmp, "<a href=http://%sxxx.gnunet", ctask->pp_buf);
+
+ strcpy (ctask->pp_buf, tmp);
+
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Created %s\n", tmp);
+
+ run_httpd();
+ //MHD_run (httpd);
+}
+
/**
* Callback for MHD response
*
size_t max)
{
struct ProxyCurlTask *ctask = cls;
+ ssize_t copied = 0;
+ size_t bytes_to_copy;
+ int nomatch;
+ char *hostptr;
+ regmatch_t m[RE_N_MATCHES];
+
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "MHD: content cb\n");
if (ctask->download_successful &&
(ctask->buf_status == BUF_WAIT_FOR_CURL))
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "MHD: sending response\n");
+ "MHD: sending response for %s\n", ctask->url);
ctask->download_in_progress = GNUNET_NO;
+ curl_multi_remove_handle (curl_multi, ctask->curl);
+ curl_easy_cleanup (ctask->curl);
return MHD_CONTENT_READER_END_OF_STREAM;
}
(ctask->buf_status == BUF_WAIT_FOR_CURL))
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "MHD: error sending response\n");
+ "MHD: sending error response\n");
ctask->download_in_progress = GNUNET_NO;
+ curl_multi_remove_handle (curl_multi, ctask->curl);
+ curl_easy_cleanup (ctask->curl);
return MHD_CONTENT_READER_END_WITH_ERROR;
}
if ( ctask->buf_status == BUF_WAIT_FOR_CURL )
- {
return 0;
- }
- if ( ctask->bytes_in_buffer > max )
+ bytes_to_copy = ctask->bytes_in_buffer;
+
+ if (ctask->parse_content == GNUNET_YES)
{
- GNUNET_log ( GNUNET_ERROR_TYPE_ERROR,
- "MHD: buffer in response too small!\n");
- return MHD_CONTENT_READER_END_WITH_ERROR;
+
+ GNUNET_log ( GNUNET_ERROR_TYPE_DEBUG,
+ "MHD: We need to parse the HTML %s\n", ctask->buffer_ptr);
+
+ nomatch = regexec ( &re_dotplus, ctask->buffer_ptr, RE_N_MATCHES, m, 0);
+
+ if (!nomatch)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "MHD RE: Match\n");
+
+ GNUNET_assert (m[1].rm_so != -1);
+
+ hostptr = ctask->buffer_ptr+m[1].rm_so;
+
+ if (m[0].rm_so > 0)
+ {
+ bytes_to_copy = m[0].rm_so;
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Copying %d bytes.\n", m[0].rm_so);
+
+
+ }
+ else
+ {
+ if (ctask->is_postprocessing == GNUNET_YES)
+ {
+
+ /*Done?*/
+ if ( 0 == strcmp (ctask->pp_buf, "") )
+ return 0;
+
+ ctask->is_postprocessing = GNUNET_NO;
+
+ ctask->bytes_in_buffer -= m[0].rm_eo;//(m[1].rm_eo-m[1].rm_so);
+ ctask->buffer_ptr += m[0].rm_eo;//(m[1].rm_eo-m[1].rm_so);
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Skipping next %d bytes in buffer\n", m[0].rm_eo);
+
+ if ( strlen (ctask->pp_buf) <= max )
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Copying postprocessed %s.\n", ctask->pp_buf);
+ memcpy ( buf, ctask->pp_buf, strlen (ctask->pp_buf) );
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Done %s.\n", buf);
+ ctask->is_postprocessing = GNUNET_NO;
+ return strlen (ctask->pp_buf);
+ }
+
+ return 0;
+ }
+
+ memset (ctask->pp_buf, 0, sizeof(ctask->pp_buf));
+ memcpy (ctask->pp_buf, hostptr, (m[1].rm_eo-m[1].rm_so));
+
+ ctask->is_postprocessing = GNUNET_YES;
+
+ postprocess_name(ctask, NULL);
+ //ctask->pp_task = GNUNET_SCHEDULER_add_now (&postprocess_name, ctask);
+
+ return 0;
+ }
+ }
}
- if ( 0 != ctask->bytes_in_buffer )
+ if ( bytes_to_copy > max )
+ {
+ GNUNET_log ( GNUNET_ERROR_TYPE_DEBUG,
+ "MHD: buffer in response too small! (%s)\n",
+ ctask->url);
+ memcpy ( buf, ctask->buffer_ptr, max);
+ ctask->bytes_in_buffer -= max;
+ ctask->buffer_ptr += max;
+ copied = max;
+ }
+ else
{
GNUNET_log ( GNUNET_ERROR_TYPE_DEBUG,
"MHD: copying %d bytes to mhd response at offset %d\n",
- ctask->bytes_in_buffer, pos);
- memcpy ( buf, ctask->buffer, ctask->bytes_in_buffer );
+ bytes_to_copy, pos);
+
+ memcpy ( buf, ctask->buffer_ptr, bytes_to_copy );
+ copied = bytes_to_copy;
+ if (bytes_to_copy < ctask->bytes_in_buffer)
+ {
+ ctask->bytes_in_buffer -= bytes_to_copy;
+ ctask->buffer_ptr += bytes_to_copy;
+ }
+ else
+ {
+ ctask->bytes_in_buffer = 0;
+ ctask->buf_status = BUF_WAIT_FOR_CURL;
+ ctask->buffer_ptr = ctask->buffer;
+ curl_easy_pause (ctask->curl, CURLPAUSE_CONT);
+ }
}
-
- ctask->buf_status = BUF_WAIT_FOR_CURL;
- curl_easy_pause (ctask->curl, CURLPAUSE_CONT);
- return ctask->bytes_in_buffer;
+ return copied;
}
-/**
- * schedule mhd
- */
-static void
-run_httpd (void);
/**
* Task that is run when we are ready to receive more data
rtime = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, to);
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "cURL multi fds: max=%d\n", max);
+ "cURL multi fds: max=%d timeout=%llu\n", max, to);
grs = GNUNET_NETWORK_fdset_create ();
gws = GNUNET_NETWORK_fdset_create ();
struct CURLMsg *msg;
CURLMcode mret;
struct ProxyCurlTask *ctask;
+ int num_ctasks;
curl_download_task = GNUNET_SCHEDULER_NO_TASK;
do
{
running = 0;
+ num_ctasks = 0;
mret = curl_multi_perform (curl_multi, &running);
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Running curl tasks: %d\n", running);
+
+ ctask = ctasks_head;
+ for (; ctask != NULL; ctask = ctask->next)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "CTask: %s\n", ctask->url);
+ num_ctasks++;
+ }
+
+ if (num_ctasks != running)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "%d ctasks, %d curl running\n", num_ctasks, running);
+ }
+
do
{
ctask = ctasks_head;
if ((msg->data.result != CURLE_OK) &&
(msg->data.result != CURLE_GOT_NOTHING))
{
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Download curl failed %s\n",
- curl_easy_strerror (msg->data.result));
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Download curl failed");
for (; ctask != NULL; ctask = ctask->next)
{
- if (memcmp (msg->easy_handle, ctask->curl, sizeof (CURL)) == 0)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "cURL task found.\n");
- ctask->download_successful = GNUNET_NO;
- ctask->download_error = GNUNET_YES;
- curl_multi_remove_handle (curl_multi, ctask->curl);
- curl_easy_cleanup (ctask->curl);
- GNUNET_CONTAINER_DLL_remove (ctasks_head, ctasks_tail,
- ctask);
- break;
- }
+ if (memcmp (msg->easy_handle, ctask->curl, sizeof (CURL)) != 0)
+ continue;
+
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Download curl failed for task %s: %s.\n",
+ ctask->url,
+ curl_easy_strerror (msg->data.result));
+ ctask->download_successful = GNUNET_NO;
+ ctask->download_error = GNUNET_YES;
+ //curl_multi_remove_handle (curl_multi, ctask->curl);
+ //curl_easy_cleanup (ctask->curl);
+ GNUNET_CONTAINER_DLL_remove (ctasks_head, ctasks_tail,
+ ctask);
+ break;
}
+ GNUNET_assert (ctask != NULL);
}
else
{
for (; ctask != NULL; ctask = ctask->next)
{
- if (memcmp (msg->easy_handle, ctask->curl, sizeof (CURL)) == 0)
- {
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "cURL task found.\n");
- ctask->download_successful = GNUNET_YES;
- curl_multi_remove_handle (curl_multi, ctask->curl);
- curl_easy_cleanup (ctask->curl);
- GNUNET_CONTAINER_DLL_remove (ctasks_head, ctasks_tail,
- ctask);
- break;
- }
- else
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "cURL task skipped.\n");
+ if (memcmp (msg->easy_handle, ctask->curl, sizeof (CURL)) != 0)
+ continue;
+
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "cURL task %s found.\n", ctask->url);
+ ctask->download_successful = GNUNET_YES;
+ //curl_multi_remove_handle (curl_multi, ctask->curl);
+ //curl_easy_cleanup (ctask->curl);
+ GNUNET_CONTAINER_DLL_remove (ctasks_head, ctasks_tail,
+ ctask);
+ break;
}
- run_httpd ();
- //TODO iterate list, find ctask
+ GNUNET_assert (ctask != NULL);
}
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "curl end %s\n", curl_easy_strerror(msg->data.result));
break;
default:
+ GNUNET_assert (0);
break;
}
} while (msgnum > 0);
- } while (mret == CURLM_CALL_MULTI_PERFORM);
+
+ num_ctasks=0;
+ for (ctask=ctasks_head; ctask != NULL; ctask = ctask->next)
+ {
+ num_ctasks++;
+ }
+
+ if (num_ctasks != running)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "%d ctasks, %d curl running\n", num_ctasks, running);
+ }
+
+ GNUNET_assert ( num_ctasks == running );
+
+ run_httpd ();
+ } while (mret == CURLM_CALL_MULTI_PERFORM);
+
+
if (mret != CURLM_OK)
{
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%s failed at %s:%d: `%s'\n",
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s failed at %s:%d: `%s'\n",
"curl_multi_perform", __FILE__, __LINE__,
curl_multi_strerror (mret));
- //TODO cleanup
}
curl_download_prepare();
}
* Main MHD callback for handling requests.
*
* @param cls unused
- * @param connection MHD connection handle
- * @param method the HTTP method used ("GET", "PUT", etc.)
- * @param version the HTTP version string (i.e. "HTTP/1.1")
+ * @param con MHD connection handle
+ * @param meth the HTTP method used ("GET", "PUT", etc.)
+ * @param ver the HTTP version string (i.e. "HTTP/1.1")
* @param upload_data the data being uploaded (excluding HEADERS,
* for a POST that fits into memory and that is encoded
* with a supported encoding, the POST data will NOT be
* @param upload_data_size set initially to the size of the
* upload_data provided; the method must update this
* value to the number of bytes NOT processed;
- * @param ptr pointer to location where we store the 'struct Request'
+ * @param con_cls pointer to location where we store the 'struct Request'
* @return MHD_YES if the connection was handled successfully,
* MHD_NO if the socket must be closed due to a serious
* error while handling the request
ctask->connection = con;
ctask->buf_status = BUF_WAIT_FOR_CURL;
ctask->bytes_in_buffer = 0;
+ ctask->parse_content = GNUNET_NO;
+ curl_easy_setopt (ctask->curl, CURLOPT_HEADERFUNCTION, &curl_check_hdr);
+ curl_easy_setopt (ctask->curl, CURLOPT_HEADERDATA, ctask);
curl_easy_setopt (ctask->curl, CURLOPT_WRITEFUNCTION, &callback_download);
curl_easy_setopt (ctask->curl, CURLOPT_WRITEDATA, ctask);
curl_easy_setopt (ctask->curl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt (ctask->curl, CURLOPT_MAXREDIRS, 4);
/* no need to abort if the above failed */
sprintf (curlurl, "http://%s%s", host, url);
+ strcpy (ctask->url, curlurl);
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Adding new curl task for %s\n", curlurl);
curl_easy_setopt (ctask->curl, CURLOPT_URL, curlurl);
curl_easy_setopt (ctask->curl, CURLOPT_FAILONERROR, 1);
- curl_easy_setopt (ctask->curl, CURLOPT_CONNECTTIMEOUT, 60L);
- curl_easy_setopt (ctask->curl, CURLOPT_TIMEOUT, 60L);
+ curl_easy_setopt (ctask->curl, CURLOPT_CONNECTTIMEOUT, 600L);
+ curl_easy_setopt (ctask->curl, CURLOPT_TIMEOUT, 600L);
mret = curl_multi_add_handle (curl_multi, ctask->curl);
"%s failed at %s:%d: `%s'\n",
"curl_multi_add_handle", __FILE__, __LINE__,
curl_multi_strerror (mret));
+ response = MHD_create_response_from_buffer (strlen (page),
+ (void*)page,
+ MHD_RESPMEM_PERSISTENT);
+ ret = MHD_queue_response (con,
+ MHD_HTTP_OK,
+ response);
+ MHD_destroy_response (response);
+
curl_easy_cleanup (ctask->curl);
GNUNET_free (ctask);
- //TODO maybe error display here
return ret;
}
response = MHD_create_response_from_callback (-1, -1,
&mhd_content_cb,
ctask,
- NULL); //TODO Destroy resp here
+ &mhd_content_free);
ret = MHD_queue_response (con, MHD_HTTP_OK, response);
+
+ //MHD_destroy_response (response);
return ret;
}
if ((NULL != tc->read_ready) &&
(GNUNET_NETWORK_fdset_isset (tc->write_ready, s5r->remote_sock)) &&
- (len = GNUNET_NETWORK_socket_send (s5r->remote_sock, s5r->rbuf,
- s5r->rbuf_len)))
+ ((len = GNUNET_NETWORK_socket_send (s5r->remote_sock, s5r->rbuf,
+ s5r->rbuf_len)>0)))
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Successfully sent %d bytes to remote socket\n",
if ((NULL != tc->read_ready) &&
(GNUNET_NETWORK_fdset_isset (tc->write_ready, s5r->sock)) &&
- (len = GNUNET_NETWORK_socket_send (s5r->sock, s5r->wbuf,
- s5r->wbuf_len)))
+ ((len = GNUNET_NETWORK_socket_send (s5r->sock, s5r->wbuf,
+ s5r->wbuf_len)>0)))
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Successfully sent %d bytes to socket\n",
}
}
+/**
+ * Compiles a regex for us
+ *
+ * @param re ptr to re struct
+ * @param rt the expression to compile
+ * @return 0 on success
+ */
+static int
+compile_regex (regex_t *re, const char* rt)
+{
+ int status;
+ char err[1024];
+
+ status = regcomp (re, rt, REG_EXTENDED|REG_NEWLINE);
+ if (status)
+ {
+ regerror (status, re, err, 1024);
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Regex error compiling '%s': %s\n", rt, err);
+ return 1;
+ }
+ return 0;
+}
+
/**
* Main function that will be run
*
{
struct sockaddr_in sa;
+ compile_regex (&re_dotplus, (char*) RE_DOTPLUS);
+
memset (&sa, 0, sizeof (sa));
sa.sin_family = AF_INET;
sa.sin_port = htons (port);
projects / oweals / gnunet.git / blobdiff