From: Christian Grothoff Date: Sun, 25 Oct 2015 23:47:39 +0000 (+0000) Subject: -use hash in prd query to reduce memory consumption of CP request_map X-Git-Tag: initial-import-from-subversion-38251~1211 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=7c55f94e78c72d3b78a5c6007e3658ae30e1f64b;p=oweals%2Fgnunet.git -use hash in prd query to reduce memory consumption of CP request_map --- diff --git a/src/fs/gnunet-service-fs_cp.c b/src/fs/gnunet-service-fs_cp.c index 7066b0698..1d52f28f9 100644 --- a/src/fs/gnunet-service-fs_cp.c +++ b/src/fs/gnunet-service-fs_cp.c @@ -250,7 +250,7 @@ struct GSF_ConnectedPeer struct GNUNET_SCHEDULER_Task *rc_delay_task; /** - * Active requests from this neighbour, map of query to 'struct PeerRequest'. + * Active requests from this neighbour, map of query to `struct PeerRequest`. */ struct GNUNET_CONTAINER_MultiHashMap *request_map; @@ -646,7 +646,8 @@ GSF_peer_connect_handler_ (const struct GNUNET_PeerIdentity *peer, cp->rc = GNUNET_ATS_reserve_bandwidth (GSF_ats, peer, DBLOCK_SIZE, &ats_reserve_callback, cp); - cp->request_map = GNUNET_CONTAINER_multihashmap_create (128, GNUNET_NO); + cp->request_map = GNUNET_CONTAINER_multihashmap_create (128, + GNUNET_YES); GNUNET_break (GNUNET_OK == GNUNET_CONTAINER_multipeermap_put (cp_map, GSF_connected_peer_get_identity2_ (cp), @@ -811,8 +812,10 @@ free_pending_request (struct PeerRequest *peerreq, GNUNET_SCHEDULER_cancel (peerreq->kill_task); peerreq->kill_task = NULL; } - GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# P2P searches active"), - -1, GNUNET_NO); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# P2P searches active"), + -1, + GNUNET_NO); GNUNET_break (GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove (cp->request_map, query, @@ -1288,6 +1291,7 @@ GSF_handle_p2p_query_ (const struct GNUNET_PeerIdentity *other, uint32_t ttl_decrement; struct TestExistClosure tec; GNUNET_PEER_Id spid; + const struct GSF_PendingRequestData *prd; msize = ntohs (message->size); if (msize < sizeof (struct GetMessage)) @@ -1452,20 +1456,25 @@ GSF_handle_p2p_query_ (const struct GNUNET_PeerIdentity *other, spid, GNUNET_PEER_intern (other), NULL, 0, /* replies_seen */ - &handle_p2p_reply, peerreq); + &handle_p2p_reply, + peerreq); GNUNET_assert (NULL != pr); + prd = GSF_pending_request_get_data_ (pr); peerreq->pr = pr; GNUNET_break (GNUNET_OK == GNUNET_CONTAINER_multihashmap_put (cp->request_map, - &gm->query, + &prd->query, peerreq, GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE)); GNUNET_STATISTICS_update (GSF_stats, gettext_noop - ("# P2P query messages received and processed"), 1, + ("# P2P query messages received and processed"), + 1, + GNUNET_NO); + GNUNET_STATISTICS_update (GSF_stats, + gettext_noop ("# P2P searches active"), + 1, GNUNET_NO); - GNUNET_STATISTICS_update (GSF_stats, gettext_noop ("# P2P searches active"), - 1, GNUNET_NO); return pr; } diff --git a/src/fs/gnunet-service-fs_pe.c b/src/fs/gnunet-service-fs_pe.c index ede109c2d..9743d1cb2 100644 --- a/src/fs/gnunet-service-fs_pe.c +++ b/src/fs/gnunet-service-fs_pe.c @@ -59,7 +59,7 @@ struct PeerPlan; * to a given plan entry.) * * Similarly head and tail of the "PR" MDLL are stored - * with the 'struct GSF_PendingRequest'. (We need + * with the `struct GSF_PendingRequest`. (We need * to be able to lookup all plan entries corresponding * to a given pending request.) */ @@ -87,12 +87,14 @@ struct GSF_PendingRequestPlanBijection struct GSF_PendingRequestPlanBijection *prev_PE; /** - * Associated request plan. + * Associated request plan (tells us one of the peers that + * we plan to forward the request to). */ struct GSF_RequestPlan *rp; /** - * Associated pending request. + * Associated pending request (identifies request details + * and one of the origins of the request). */ struct GSF_PendingRequest *pr; @@ -101,8 +103,8 @@ struct GSF_PendingRequestPlanBijection /** * Information we keep per request per peer. This is a doubly-linked - * list (with head and tail in the 'struct GSF_PendingRequestData') - * with one entry in each heap of each 'struct PeerPlan'. Each + * list (with head and tail in the `struct GSF_PendingRequestData`) + * with one entry in each heap of each `struct PeerPlan`. Each * entry tracks information relevant for this request and this peer. */ struct GSF_RequestPlan @@ -129,7 +131,9 @@ struct GSF_RequestPlan struct PeerPlan *pp; /** - * Head of list of associated pending requests. + * Head of list of associated pending requests. This tells us + * which incoming requests from other peers this plan entry + * corresponds to. */ struct GSF_PendingRequestPlanBijection *pe_head; @@ -167,19 +171,20 @@ struct GSF_RequestPlan struct PeerPlan { /** - * Heap with pending queries (struct GSF_RequestPlan), higher weights mean higher priority. + * Heap with pending queries (`struct GSF_RequestPlan`), higher weights mean higher priority. */ struct GNUNET_CONTAINER_Heap *priority_heap; /** - * Heap with pending queries (struct GSF_RequestPlan), by transmission time, lowest first. + * Heap with pending queries (`struct GSF_RequestPlan`), by transmission time, lowest first. */ struct GNUNET_CONTAINER_Heap *delay_heap; /** - * Map of queries to plan entries. All entries in the priority_heap or delay_heap - * should be in the plan map. Note that it IS possible for the plan map to have - * multiple entries for the same query. + * Map of queries to plan entries. All entries in the @e priority_heap + * or @e delay_heap should be in the @e plan_map. Note that it is + * possible for the @e plan_map to have multiple entries for the same + * query. */ struct GNUNET_CONTAINER_MultiHashMap *plan_map; @@ -196,7 +201,7 @@ struct PeerPlan /** * Current task for executing the plan. */ - struct GNUNET_SCHEDULER_Task * task; + struct GNUNET_SCHEDULER_Task *task; }; @@ -222,7 +227,7 @@ static unsigned long long plan_count; * requests for the same key and we just return _one_ of them; this * particular one might complete while another one might still be * active, hence the lifetime of the returned hash code is NOT - * necessarily identical to that of the 'struct GSF_RequestPlan' + * necessarily identical to that of the `struct GSF_RequestPlan` * given. * * @param rp a request plan @@ -253,7 +258,8 @@ schedule_peer_transmission (void *cls, * @param rp request to plan */ static void -plan (struct PeerPlan *pp, struct GSF_RequestPlan *rp) +plan (struct PeerPlan *pp, + struct GSF_RequestPlan *rp) { #define N ((double)128.0) /** @@ -366,8 +372,7 @@ get_latest (const struct GSF_RequestPlan *rp) for (bi = bi->next_PE; NULL != bi; bi = bi->next_PE) { prd = GSF_pending_request_get_data_ (bi->pr); - if (prd->ttl.abs_value_us > - rprd->ttl.abs_value_us) + if (prd->ttl.abs_value_us > rprd->ttl.abs_value_us) { ret = bi->pr; rprd = prd; @@ -383,10 +388,12 @@ get_latest (const struct GSF_RequestPlan *rp) * @param cls closure * @param buf_size number of bytes available in @a buf * @param buf where to copy the message, NULL on error (peer disconnect) - * @return number of bytes copied to 'buf', can be 0 (without indicating an error) + * @return number of bytes copied to @a buf, can be 0 (without indicating an error) */ static size_t -transmit_message_callback (void *cls, size_t buf_size, void *buf) +transmit_message_callback (void *cls, + size_t buf_size, + void *buf) { struct PeerPlan *pp = cls; struct GSF_RequestPlan *rp; @@ -414,7 +421,9 @@ transmit_message_callback (void *cls, size_t buf_size, void *buf) pp->task = GNUNET_SCHEDULER_add_now (&schedule_peer_transmission, pp); return 0; } - msize = GSF_pending_request_get_message_ (get_latest (rp), buf_size, buf); + msize = GSF_pending_request_get_message_ (get_latest (rp), + buf_size, + buf); if (msize > buf_size) { if (NULL != pp->task) @@ -476,7 +485,8 @@ schedule_peer_transmission (void *cls, rp = GNUNET_CONTAINER_heap_peek (pp->delay_heap); if (NULL == rp) { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "No active requests for plan %p.\n", + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "No active requests for plan %p.\n", pp); return; /* both queues empty */ } @@ -486,11 +496,14 @@ schedule_peer_transmission (void *cls, GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES), pp); - GNUNET_STATISTICS_set (GSF_stats, gettext_noop ("# delay heap timeout (ms)"), + GNUNET_STATISTICS_set (GSF_stats, + gettext_noop ("# delay heap timeout (ms)"), delay.rel_value_us / 1000LL, GNUNET_NO); pp->task = - GNUNET_SCHEDULER_add_delayed (delay, &schedule_peer_transmission, pp); + GNUNET_SCHEDULER_add_delayed (delay, + &schedule_peer_transmission, + pp); return; } #if INSANE_STATISTICS @@ -503,8 +516,10 @@ schedule_peer_transmission (void *cls, GNUNET_assert (NULL != rp); msize = GSF_pending_request_get_message_ (get_latest (rp), 0, NULL); pp->pth = - GSF_peer_transmit_ (pp->cp, GNUNET_YES, rp->priority, - GNUNET_TIME_UNIT_FOREVER_REL, msize, + GSF_peer_transmit_ (pp->cp, GNUNET_YES, + rp->priority, + GNUNET_TIME_UNIT_FOREVER_REL, + msize, &transmit_message_callback, pp); GNUNET_assert (NULL != pp->pth); } @@ -516,8 +531,14 @@ schedule_peer_transmission (void *cls, struct MergeContext { + /** + * Request we are trying to merge. + */ struct GSF_PendingRequest *pr; + /** + * Set to #GNUNET_YES if we succeeded to merge. + */ int merged; }; diff --git a/src/fs/gnunet-service-fs_pr.c b/src/fs/gnunet-service-fs_pr.c index b507992b4..21b00e80e 100644 --- a/src/fs/gnunet-service-fs_pr.c +++ b/src/fs/gnunet-service-fs_pr.c @@ -289,7 +289,8 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, enum GNUNET_BLOCK_Type type, const struct GNUNET_HashCode *query, const struct GNUNET_PeerIdentity *target, - const char *bf_data, size_t bf_size, + const char *bf_data, + size_t bf_size, uint32_t mingle, uint32_t anonymity_level, uint32_t priority, @@ -325,7 +326,9 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, if (NULL != target) { pr->public_data.target = (struct GNUNET_PeerIdentity *) eptr; - memcpy (eptr, target, sizeof (struct GNUNET_PeerIdentity)); + memcpy (eptr, + target, + sizeof (struct GNUNET_PeerIdentity)); } pr->public_data.anonymity_level = anonymity_level; pr->public_data.priority = priority; @@ -397,7 +400,9 @@ GSF_pending_request_create_ (enum GSF_PendingRequestOptions options, UINT32_MAX, GNUNET_TIME_UNIT_FOREVER_ABS, GNUNET_TIME_UNIT_FOREVER_ABS, - GNUNET_BLOCK_TYPE_ANY, NULL, 0); + GNUNET_BLOCK_TYPE_ANY, + NULL, + 0); GSF_pending_request_cancel_ (dpr, GNUNET_YES); } diff --git a/src/fs/gnunet-service-fs_pr.h b/src/fs/gnunet-service-fs_pr.h index d022d8640..547595b67 100644 --- a/src/fs/gnunet-service-fs_pr.h +++ b/src/fs/gnunet-service-fs_pr.h @@ -179,25 +179,22 @@ struct GSF_PendingRequestData * @param eval evaluation of the result * @param pr handle to the original pending request * @param reply_anonymity_level anonymity level for the reply, UINT32_MAX for "unknown" - * @param expiration when does 'data' expire? + * @param expiration when does @a data expire? * @param last_transmission the last time we've tried to get this block (FOREVER if unknown) * @param type type of the block * @param data response data, NULL on request expiration - * @param data_len number of bytes in data + * @param data_len number of bytes in @a data */ -typedef void (*GSF_PendingRequestReplyHandler) (void *cls, - enum - GNUNET_BLOCK_EvaluationResult - eval, - struct GSF_PendingRequest * pr, - uint32_t reply_anonymity_level, - struct GNUNET_TIME_Absolute - expiration, - struct GNUNET_TIME_Absolute - last_transmission, - enum GNUNET_BLOCK_Type type, - const void *data, - size_t data_len); +typedef void +(*GSF_PendingRequestReplyHandler) (void *cls, + enum GNUNET_BLOCK_EvaluationResult eval, + struct GSF_PendingRequest *pr, + uint32_t reply_anonymity_level, + struct GNUNET_TIME_Absolute expiration, + struct GNUNET_TIME_Absolute last_transmission, + enum GNUNET_BLOCK_Type type, + const void *data, + size_t data_len); /**