};
+/**
+ * Command-line argument specifying desired size of the hash map with
+ * all of our pending names. Usually, we use an automatically growing
+ * map, but this is only OK up to about a million entries. Above that
+ * number, the user must explicitly specify the size at startup.
+ */
+static unsigned int map_size = 1024;
+
/**
* Handle to the identity service.
*/
static struct GNUNET_TIME_Relative total_dns_latency;
/**
- * Number of NAMESTORE requests counted in latency total.
+ * Number of records processed (DNS lookup, no NAMESTORE) in total.
+ */
+static uint64_t total_reg_proc_dns;
+
+/**
+ * Number of records processed (DNS lookup, with NAMESTORE) in total.
+ */
+static uint64_t total_reg_proc_dns_ns;
+
+/**
+ * Start time of the regular processing.
+ */
+static struct GNUNET_TIME_Absolute start_time_reg_proc;
+
+/**
+ * Last time we worked before going idle.
*/
-static uint64_t total_ns_latency_cnt;
+static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
/**
- * Sum of NAMESTORE latencies observed.
+ * Time we slept just waiting for work.
*/
-static struct GNUNET_TIME_Relative total_ns_latency;
+static struct GNUNET_TIME_Relative idle_time;
/**
{
if (NULL != t)
GNUNET_SCHEDULER_cancel (t);
+ sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
t = GNUNET_SCHEDULER_add_at (req->expires,
&process_queue,
NULL);
const char *emsg)
{
static struct GNUNET_TIME_Absolute last;
- static unsigned int pdot;
struct Request *req = cls;
req->qe = NULL;
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Stored record set in database (%d)\n",
- success);
- pending_rs--;
- {
- struct GNUNET_TIME_Relative ns_latency;
-
- ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
- total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
- ns_latency);
- total_ns_latency_cnt++;
- if (0 == (total_ns_latency_cnt % 1000))
- {
- GNUNET_STATISTICS_update (stats,
- "# average NAMESTORE PUT latency (μs)",
- total_ns_latency.rel_value_us / total_ns_latency_cnt,
- GNUNET_NO);
- GNUNET_STATISTICS_update (stats,
- "# NAMESTORE PUTs",
- total_ns_latency_cnt,
- GNUNET_NO);
- }
- }
-
- if (NULL == t)
- t = GNUNET_SCHEDULER_add_now (&process_queue,
- NULL);
if (GNUNET_SYSERR == success)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
else
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Stored records under `%s'\n",
- req->hostname);
- if (0 == pdot)
+ "Stored records under `%s' (%d)\n",
+ req->hostname,
+ success);
+ }
+ total_reg_proc_dns_ns++; /* finished regular processing */
+ pending_rs--;
+ free_records (req);
+ /* compute NAMESTORE statistics */
+ {
+ static uint64_t total_ns_latency_cnt;
+ static struct GNUNET_TIME_Relative total_ns_latency;
+ struct GNUNET_TIME_Relative ns_latency;
+
+ ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
+ total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
+ ns_latency);
+ if (0 == total_ns_latency_cnt)
last = GNUNET_TIME_absolute_get ();
- pdot++;
- if (0 == pdot % 1000)
+ total_ns_latency_cnt++;
+ if (0 == (total_ns_latency_cnt % 1000))
{
struct GNUNET_TIME_Relative delta;
"Processed 1000 records in %s\n",
GNUNET_STRINGS_relative_time_to_string (delta,
GNUNET_YES));
+ GNUNET_STATISTICS_set (stats,
+ "# average NAMESTORE PUT latency (μs)",
+ total_ns_latency.rel_value_us / total_ns_latency_cnt,
+ GNUNET_NO);
}
}
- free_records (req);
+ /* compute and publish overall velocity */
+ if (0 == (total_reg_proc_dns_ns % 100) )
+ {
+ struct GNUNET_TIME_Relative runtime;
+
+ runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
+ runtime = GNUNET_TIME_relative_subtract (runtime,
+ idle_time);
+ runtime = GNUNET_TIME_relative_divide (runtime,
+ total_reg_proc_dns + total_reg_proc_dns_ns);
+ GNUNET_STATISTICS_set (stats,
+ "# Regular processing completed without NAMESTORE",
+ total_reg_proc_dns,
+ GNUNET_NO);
+ GNUNET_STATISTICS_set (stats,
+ "# Regular processing completed with NAMESTORE PUT",
+ total_reg_proc_dns_ns,
+ GNUNET_NO);
+ GNUNET_STATISTICS_set (stats,
+ "# average request processing latency (μs)",
+ runtime.rel_value_us,
+ GNUNET_NO);
+ GNUNET_STATISTICS_set (stats,
+ "# total time spent idle (μs)",
+ idle_time.rel_value_us,
+ GNUNET_NO);
+ }
+
+ if (NULL == t)
+ {
+ sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
+ t = GNUNET_SCHEDULER_add_now (&process_queue,
+ NULL);
+ }
}
req);
pending--;
if (NULL == t)
+ {
+ sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
t = GNUNET_SCHEDULER_add_now (&process_queue,
NULL);
+ }
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Stub gave up on DNS reply for `%s'\n",
req->hostname);
GNUNET_NO);
return;
}
+ total_reg_proc_dns++;
req->rs = NULL;
insert_sorted (req);
return;
1,
GNUNET_NO);
if (NULL == t)
+ {
+ sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
t = GNUNET_SCHEDULER_add_now (&process_queue,
NULL);
+ }
if (req->issue_num > MAX_RETRIES)
{
failures++;
total_dns_latency_cnt++;
if (0 == (total_dns_latency_cnt % 1000))
{
- GNUNET_STATISTICS_update (stats,
- "# average DNS latency (μs)",
- total_dns_latency.rel_value_us / total_dns_latency_cnt,
- GNUNET_NO);
- GNUNET_STATISTICS_update (stats,
- "# DNS replies",
- total_dns_latency_cnt,
- GNUNET_NO);
+ GNUNET_STATISTICS_set (stats,
+ "# average DNS lookup latency (μs)",
+ total_dns_latency.rel_value_us / total_dns_latency_cnt,
+ GNUNET_NO);
}
}
rd_count = 0;
/* Instead of going for SOA, simplified for now to look each
day in case we got an empty response */
if (0 == rd_count)
+ {
req->expires
= GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
+ GNUNET_STATISTICS_update (stats,
+ "# empty DNS replies (usually NXDOMAIN)",
+ 1,
+ GNUNET_NO);
+ }
else
+ {
record_sets++;
+ }
/* convert records to namestore import format */
{
struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)];
unsigned int series;
void *raw;
size_t raw_size;
+ struct GNUNET_TIME_Relative delay;
(void) cls;
+ delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
+ idle_time = GNUNET_TIME_relative_add (idle_time,
+ delay);
series = 0;
t = NULL;
while (pending + pending_rs < THRESH)
req->hostname);
if (NULL != t)
GNUNET_SCHEDULER_cancel (t);
+ sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
t = GNUNET_SCHEDULER_add_at (req->expires,
&process_queue,
NULL);
"Throttling\n");
if (NULL != t)
GNUNET_SCHEDULER_cancel (t);
+ sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
&process_queue,
NULL);
{
ns_iterator_trigger_next = NS_BATCH_SIZE;
GNUNET_STATISTICS_update (stats,
- "# NAMESTORE records requested",
- ns_iterator_trigger_next,
- GNUNET_NO);
+ "# NAMESTORE records requested from cache",
+ ns_iterator_trigger_next,
+ GNUNET_NO);
GNUNET_NAMESTORE_zone_iterator_next (zone_it,
ns_iterator_trigger_next);
}
last->domain);
/* subtract left-overs from previous iteration */
GNUNET_STATISTICS_update (stats,
- "# NAMESTORE records requested",
+ "# NAMESTORE records requested from cache",
(long long) (- ns_iterator_trigger_next),
GNUNET_NO);
ns_iterator_trigger_next = 0;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Finished all NAMESTORE iterations!\n");
GNUNET_STATISTICS_set (stats,
- "# NAMESTORE lookups without reply",
+ "# Domain names without cached reply",
GNUNET_CONTAINER_multihashmap_size (ns_pending),
GNUNET_NO);
GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
NULL);
GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
ns_pending = NULL;
+ start_time_reg_proc = GNUNET_TIME_absolute_get ();
+ total_reg_proc_dns = 0;
+ total_reg_proc_dns_ns = 0;
return;
}
if (NULL == last)
last->domain);
/* subtract left-overs from previous iteration */
GNUNET_STATISTICS_update (stats,
- "# NAMESTORE records requested",
+ "# NAMESTORE records requested from cache",
1,
GNUNET_NO);
ns_iterator_trigger_next = 1;
process_stdin (void *cls)
{
static struct GNUNET_TIME_Absolute last;
- static unsigned int pdot;
+ static uint64_t idot;
char hn[256];
(void) cls;
{
if (strlen(hn) > 0)
hn[strlen(hn)-1] = '\0'; /* eat newline */
- if (0 == pdot)
+ if (0 == idot)
last = GNUNET_TIME_absolute_get ();
- pdot++;
- if (0 == pdot % 1000)
+ idot++;
+ if (0 == idot % 10000)
{
struct GNUNET_TIME_Relative delta;
delta = GNUNET_TIME_absolute_get_duration (last);
last = GNUNET_TIME_absolute_get ();
fprintf (stderr,
- "Imported 1000 records in %s\n",
+ "Imported 10000 records in %s\n",
GNUNET_STRINGS_relative_time_to_string (delta,
GNUNET_YES));
GNUNET_STATISTICS_set (stats,
"# domain names provided",
- pdot,
+ idot,
GNUNET_NO);
}
queue (hn);
}
- fprintf (stderr, "\n");
+ fprintf (stderr,
+ "Done reading %llu domain names\n",
+ (unsigned long long) idot);
GNUNET_STATISTICS_set (stats,
"# domain names provided",
- pdot,
+ idot,
GNUNET_NO);
iterate_zones (NULL);
}
stats = GNUNET_STATISTICS_create ("zoneimport",
cfg);
req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
- ns_pending = GNUNET_CONTAINER_multihashmap_create (1024,
+ ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size,
GNUNET_NO);
+ if (NULL == ns_pending)
+ {
+ fprintf (stderr,
+ "Failed to allocate memory for main hash map\n");
+ return;
+ }
ctx = GNUNET_DNSSTUB_start (256);
if (NULL == ctx)
{
char *const*argv)
{
struct GNUNET_GETOPT_CommandLineOption options[] = {
+ GNUNET_GETOPT_option_uint ('s',
+ "size",
+ "MAPSIZE",
+ gettext_noop ("size to use for the main hash map"),
+ &map_size),
GNUNET_GETOPT_OPTION_END
};
GNUNET_CONTAINER_multihashmap_create (unsigned int len,
int do_not_copy_keys)
{
- struct GNUNET_CONTAINER_MultiHashMap *map;
+ struct GNUNET_CONTAINER_MultiHashMap *hm;
GNUNET_assert (len > 0);
- map = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap);
- map->map = GNUNET_malloc (len * sizeof (union MapEntry));
- map->map_length = len;
- map->use_small_entries = do_not_copy_keys;
- return map;
+ hm = GNUNET_new (struct GNUNET_CONTAINER_MultiHashMap);
+ if (len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED)
+ {
+ size_t s;
+ /* application *explicitly* requested very large map, hopefully
+ it checks the return value... */
+ s = len * sizeof (union MapEntry);
+ if ( (s / sizeof (union MapEntry)) != len)
+ return NULL; /* integer overflow on multiplication */
+ if (NULL == (hm->map = GNUNET_malloc_large (s)))
+ {
+ /* out of memory */
+ GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+ "Out of memory allocating large hash map (%u entries)\n",
+ len);
+ GNUNET_free (hm);
+ return NULL;
+ }
+ }
+ else
+ {
+ hm->map = GNUNET_new_array (len,
+ union MapEntry);
+ }
+ hm->map_length = len;
+ hm->use_small_entries = do_not_copy_keys;
+ return hm;
}
GNUNET_CONTAINER_multihashmap_destroy (struct GNUNET_CONTAINER_MultiHashMap
*map)
{
- unsigned int i;
- union MapEntry me;
-
- for (i = 0; i < map->map_length; i++)
+ for (unsigned int i = 0; i < map->map_length; i++)
{
+ union MapEntry me;
+
me = map->map[i];
if (map->use_small_entries)
{
* @return the number of key value pairs
*/
unsigned int
-GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap
- *map)
+GNUNET_CONTAINER_multihashmap_size (const struct GNUNET_CONTAINER_MultiHashMap *map)
{
return map->size;
}
unsigned int old_len;
unsigned int new_len;
unsigned int idx;
- unsigned int i;
map->modification_counter++;
old_map = map->map;
old_len = map->map_length;
new_len = old_len * 2;
- new_map = GNUNET_malloc (sizeof (union MapEntry) * new_len);
+ /* if we would exceed heap size limit for the _first_ time,
+ try staying just below the limit */
+ if ( (new_len * sizeof (union MapEntry) > GNUNET_MAX_MALLOC_CHECKED) &&
+ ((old_len+1) * sizeof (union MapEntry) < GNUNET_MAX_MALLOC_CHECKED) )
+ new_len = GNUNET_MAX_MALLOC_CHECKED / sizeof (union MapEntry);
+ new_map = GNUNET_new_array (new_len,
+ union MapEntry);
map->map_length = new_len;
map->map = new_map;
- for (i = 0; i < old_len; i++)
+ for (unsigned int i = 0; i < old_len; i++)
{
if (map->use_small_entries)
{
* The initial interval in milliseconds btween puts in
* a zone iteration
*/
-#define INITIAL_PUT_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
+#define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
/**
* The upper bound for the zone iteration interval
* and the total number of record sets we have (so far)
* observed in the zone.
*/
-static struct GNUNET_TIME_Relative next_put_interval;
+static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
/**
* Minimum relative expiration time of records seem during the current
/**
- * Calculate #next_put_interval.
+ * Calculate #target_iteration_velocity_per_record.
*/
static void
calculate_put_interval ()
* we can safely set the interval to the value for a single
* record
*/
- next_put_interval = zone_publish_time_window;
+ target_iteration_velocity_per_record = zone_publish_time_window;
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
"No records in namestore database.\n");
}
= GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
PUBLISH_OPS_PER_EXPIRATION),
zone_publish_time_window_default);
- next_put_interval
+ target_iteration_velocity_per_record
= GNUNET_TIME_relative_divide (zone_publish_time_window,
last_num_public_records);
}
- next_put_interval
- = GNUNET_TIME_relative_min (next_put_interval,
+ target_iteration_velocity_per_record
+ = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
MAXIMUM_ZONE_ITERATION_INTERVAL);
GNUNET_STATISTICS_set (statistics,
- "Minimum relative record expiration (in ms)",
- last_min_relative_record_time.rel_value_us / 1000LL,
- GNUNET_NO);
+ "Minimum relative record expiration (in μs)",
+ last_min_relative_record_time.rel_value_us,
+ GNUNET_NO);
GNUNET_STATISTICS_set (statistics,
- "Zone publication time window (in ms)",
- zone_publish_time_window.rel_value_us / 1000LL,
+ "Zone publication time window (in μs)",
+ zone_publish_time_window.rel_value_us,
GNUNET_NO);
GNUNET_STATISTICS_set (statistics,
"Target zone iteration velocity (μs)",
- next_put_interval.rel_value_us,
+ target_iteration_velocity_per_record.rel_value_us,
GNUNET_NO);
}
}
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Desired global zone iteration interval is %s/record!\n",
- GNUNET_STRINGS_relative_time_to_string (next_put_interval,
+ GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
GNUNET_YES));
/* Tell statistics actual vs. desired speed */
GNUNET_NO);
/* update "sub_delta" based on difference, taking
previous sub_delta into account! */
- if (next_put_interval.rel_value_us > delta.rel_value_us)
+ if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
{
/* We were too fast, reduce sub_delta! */
struct GNUNET_TIME_Relative corr;
- corr = GNUNET_TIME_relative_subtract (next_put_interval,
+ corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
delta);
if (sub_delta.rel_value_us > delta.rel_value_us)
{
sub_delta = GNUNET_TIME_UNIT_ZERO;
}
}
- else if (next_put_interval.rel_value_us < delta.rel_value_us)
+ else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
{
/* We were too slow, increase sub_delta! */
struct GNUNET_TIME_Relative corr;
corr = GNUNET_TIME_relative_subtract (delta,
- next_put_interval);
+ target_iteration_velocity_per_record);
sub_delta = GNUNET_TIME_relative_add (sub_delta,
corr);
- if (sub_delta.rel_value_us > next_put_interval.rel_value_us)
+ if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
{
/* CPU overload detected, we cannot go at desired speed,
as this would mean using a negative delay. */
/* compute how much faster we would want to be for
the desired velocity */
- if (0 == next_put_interval.rel_value_us)
+ if (0 == target_iteration_velocity_per_record.rel_value_us)
pct = UINT64_MAX; /* desired speed is infinity ... */
else
pct = (sub_delta.rel_value_us -
- next_put_interval.rel_value_us) * 100LLU
- / next_put_interval.rel_value_us;
- sub_delta = next_put_interval;
+ target_iteration_velocity_per_record.rel_value_us) * 100LLU
+ / target_iteration_velocity_per_record.rel_value_us;
+ sub_delta = target_iteration_velocity_per_record;
}
}
GNUNET_STATISTICS_set (statistics,
return; /* current NAMESTORE iteration not yet done */
update_velocity (put_cnt);
put_cnt = 0;
- delay = GNUNET_TIME_relative_subtract (next_put_interval,
+ delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
sub_delta);
/* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
per-record delay calculated so far with the #NS_BLOCK_SIZE */
PUBLISH_OPS_PER_EXPIRATION);
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Zone iteration finished. Adjusted zone iteration interval to %s\n",
- GNUNET_STRINGS_relative_time_to_string (next_put_interval,
+ GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
GNUNET_YES));
GNUNET_STATISTICS_set (statistics,
- "Current zone iteration interval (in ms)",
- next_put_interval.rel_value_us / 1000LL,
+ "Target zone iteration velocity (μs)",
+ target_iteration_velocity_per_record.rel_value_us,
GNUNET_NO);
GNUNET_STATISTICS_set (statistics,
"Number of public records in DHT",
GNUNET_NO);
GNUNET_assert (NULL == zone_publish_task);
if (0 == last_num_public_records)
- zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval,
+ {
+ zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
&publish_zone_dht_start,
NULL);
+ }
else
+ {
zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
NULL);
+ }
}
min_relative_record_time
= GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
PUBLISH_OPS_PER_EXPIRATION);
- next_put_interval = INITIAL_PUT_INTERVAL;
+ target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
namestore_handle = GNUNET_NAMESTORE_connect (c);
if (NULL == namestore_handle)
{
}
/* Schedule periodic put for our records. */
- first_zone_iteration = GNUNET_YES;\
+ first_zone_iteration = GNUNET_YES;
statistics = GNUNET_STATISTICS_create ("zonemaster",
c);
GNUNET_STATISTICS_set (statistics,
"Target zone iteration velocity (μs)",
- next_put_interval.rel_value_us,
+ target_iteration_velocity_per_record.rel_value_us,
GNUNET_NO);
zmon = GNUNET_NAMESTORE_zone_monitor_start (c,
NULL,
projects / oweals / gnunet.git / commitdiff