2 This file is part of GNUnet
3 (C) 2004, 2006, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fragmentation/fragmentation.c
22 * @brief fragmentation and defragmentation, this code allows
23 * sending and receiving messages that are larger than
24 * the MTU of the transport. Messages are still limited
25 * to a maximum size of 65535 bytes, which is a good
26 * idea because otherwise we may need ungainly fragmentation
27 * buffers. Each connected peer can have at most one
28 * fragmented packet at any given point in time (prevents
29 * DoS attacks). Fragmented messages that have not been
30 * completed after a certain amount of time are discarded.
31 * @author Christian Grothoff
35 #include "gnunet_fragmentation_lib.h"
36 #include "gnunet_protocols.h"
37 #include "gnunet_util_lib.h"
39 * Message fragment. This header is followed
40 * by the actual data of the fragment.
45 struct GNUNET_MessageHeader header;
50 uint32_t off GNUNET_PACKED;
53 * "unique" id for the fragment
55 uint64_t id GNUNET_PACKED;
62 struct GNUNET_FRAGEMENT_Ctxbuffer{
63 struct GNUNET_FRAGEMENT_Ctxbuffer *next;
68 struct GNUNET_TIME_Absolute receivedTime;
69 struct GNUNET_PeerIdentity *peerID;
75 * Defragmentation context.
77 struct GNUNET_FRAGMENT_Context
80 struct GNUNET_FRAGEMENT_Ctxbuffer *buffer;
81 GNUNET_FRAGMENT_MessageProcessor proc;
87 * Fragment an over-sized message.
89 * @param msg the message to fragment
90 * @param mtu the maximum message size
91 * @param proc function to call for each fragment
92 * @param proc_cls closure for proc
95 GNUNET_FRAGMENT_fragment (const struct GNUNET_MessageHeader *msg,
97 GNUNET_FRAGMENT_MessageProcessor proc,
100 uint32_t id = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, 256);
101 size_t size = sizeof(struct Fragment);
102 if(ntohs(msg->size) > mtu-size){
107 lastSize = ntohs(msg->size) % (mtu-size);
108 num = ntohs(msg->size) / (mtu - size);
113 for(i = 0; i<actualNum; i++)
115 struct Fragment *frag;
116 if(actualNum != num){
118 frag = GNUNET_malloc(mtu);
121 frag = GNUNET_malloc(lastSize+size);
125 frag = GNUNET_malloc(mtu);
127 frag->header.type = htons(GNUNET_MESSAGE_TYPE_FRAGMENT);
128 frag->id = htonl(id);
129 frag->off = htons((mtu-size)*i);
130 frag->mtu = htons(mtu);
131 frag->totalNum = htons(actualNum);
132 frag->totalSize = msg->size;
133 if(actualNum != num){
135 frag->header.size = htons(mtu);
136 memcpy(&frag[1], msg + ntohs(frag->off), mtu - size);
139 frag->header.size = htons(lastSize+size);
140 memcpy(&frag[1], msg + ntohs(frag->off), lastSize);
144 frag->header.size = htons(mtu);
145 memcpy(&frag[1], msg + ntohs(frag->off), mtu - size);
147 proc(proc_cls, &frag->header);
155 * Create a defragmentation context.
157 * @param stats statistics context
158 * @param proc function to call with defragmented messages
159 * @param proc_cls closure for proc
160 * @return the defragmentation context
162 struct GNUNET_FRAGMENT_Context *
163 GNUNET_FRAGMENT_context_create (struct GNUNET_STATISTICS_Handle *stats,
164 GNUNET_FRAGMENT_MessageProcessor proc,
167 struct GNUNET_FRAGMENT_Context *ctx = (struct GNUNET_FRAGMENT_Context*)GNUNET_malloc(sizeof(struct GNUNET_FRAGMENT_Context));
170 ctx->proc_cls = proc_cls;
177 * Destroy the given defragmentation context.
180 GNUNET_FRAGMENT_context_destroy (struct GNUNET_FRAGMENT_Context *ctx)
182 struct GNUNET_FRAGEMENT_Ctxbuffer *buffer;
183 for(buffer = ctx->buffer; buffer!=NULL; buffer = buffer->next){
184 GNUNET_free(buffer->num);
193 * We have received a fragment. Process it.
195 * @param ctx the context
196 * @param sender who transmitted the fragment
197 * @param msg the message that was received
200 GNUNET_FRAGMENT_process (struct GNUNET_FRAGMENT_Context *ctx,
201 const struct GNUNET_PeerIdentity *sender,
202 const struct GNUNET_MessageHeader *msg)
204 uint16_t type = ntohs(msg->type);
205 int exist = 0, received = 0;
206 if(type!=GNUNET_MESSAGE_TYPE_FRAGMENT){
209 struct Fragment *frag = (struct Fragment *)msg;
210 struct GNUNET_FRAGEMENT_Ctxbuffer* buffer;
211 struct GNUNET_FRAGEMENT_Ctxbuffer* prev;
213 buffer = ctx->buffer;
214 while (buffer != NULL)
216 //for(buffer = ctx->buffer; buffer != NULL; buffer = buffer->next){
217 if(buffer->id == ntohl(frag->id)&&(buffer->peerID==sender)){
222 buffer = buffer->next;
228 for(i = 0; i<ntohs(frag->totalNum); i++){
229 if(buffer->num[i]==ntohs(frag->off)/(ntohs(frag->mtu)-sizeof(struct Fragment))){
237 buffer = GNUNET_malloc(sizeof(struct GNUNET_FRAGEMENT_Ctxbuffer));
238 buffer->num = (int*)GNUNET_malloc(ntohs(frag->totalNum)*sizeof(int));
240 for(j = 0; j<ntohs(frag->totalNum); j++){
241 buffer->num[j] = -10;
243 buffer->peerID = sender;
244 buffer->id = ntohl(frag->id);
245 buffer->receivedTime = GNUNET_TIME_absolute_get ();
246 uint64_t si = ntohs(frag->totalSize);
248 buffer->buff = (char *)GNUNET_malloc(si);
249 buffer->next = ctx->buffer;
250 ctx->buffer = buffer;
254 buffer->num[buffer->counter++]=ntohs(frag->off)/(ntohs(frag->mtu)-sizeof(struct Fragment));
255 uint16_t sizeoffrag = ntohs(frag->header.size) - sizeof(struct Fragment);
256 memcpy(&buffer->buff[ntohs(frag->off)], &frag[1], sizeoffrag);
257 buffer->receivedTime = GNUNET_TIME_absolute_get ();
260 if(buffer->counter == ntohs(frag->totalNum))
262 ctx->proc(ctx->proc_cls, (struct GNUNET_MessageHeader *)buffer->buff);
264 ctx->buffer = buffer->next;
267 prev->next = buffer->next;
279 * How many buckets does the fragment hash table
282 #define DEFRAG_BUCKET_COUNT 16
285 * After how long do fragments time out?
287 #ifndef DEFRAGMENTATION_TIMEOUT
288 #define DEFRAGMENTATION_TIMEOUT (3 * GNUNET_CRON_MINUTES)
292 * Entry in the linked list of fragments.
297 P2P_fragmentation_MESSAGE *frag;
301 * Entry in the GNUNET_hash table of fragments.
307 GNUNET_PeerIdentity sender;
312 #define FRAGSIZE(fl) ((ntohs(fl->frag->header.size)-sizeof(P2P_fragmentation_MESSAGE)))
314 static GNUNET_CoreAPIForPlugins *coreAPI;
316 static GNUNET_Stats_ServiceAPI *stats;
318 static int stat_defragmented;
320 static int stat_fragmented;
322 static int stat_discarded;
325 * Hashtable *with* collision management!
327 static FC *defragmentationCache[DEFRAG_BUCKET_COUNT];
330 * Lock for the defragmentation cache.
332 static struct GNUNET_Mutex *defragCacheLock;
335 freeFL (FL * fl, int c)
341 stats->change (stat_discarded, c);
342 GNUNET_free (fl->frag);
349 * This cron job ensures that we purge buffers of fragments
350 * that have timed out. It can run in much longer intervals
351 * than the defragmentationCron, e.g. every 60s.
353 * This method goes through the hashtable, finds entries that
354 * have timed out and removes them (and all the fragments that
355 * belong to the entry). It's a bit more complicated as the
356 * collision list is also collapsed.
359 defragmentationPurgeCron (void *unused)
366 GNUNET_mutex_lock (defragCacheLock);
367 for (i = 0; i < DEFRAG_BUCKET_COUNT; i++)
370 smf = defragmentationCache[i];
373 if (smf->ttl < GNUNET_get_time ())
375 /* free linked list of fragments */
376 freeFL (smf->head, 1);
380 defragmentationCache[i] = next;
390 } /* while smf != NULL */
391 } /* for all buckets */
392 GNUNET_mutex_unlock (defragCacheLock);
396 * Check if this fragment-list is complete. If yes, put it together,
397 * process and free all buffers. Does not free the pep
398 * itself (but sets the TTL to 0 to have the cron free it
399 * in the next iteration).
401 * @param pep the entry in the GNUNET_hash table
404 checkComplete (FC * pep)
411 GNUNET_GE_ASSERT (NULL, pep != NULL);
415 len = ntohs (pos->frag->len);
417 goto CLEANUP; /* really bad error! */
419 while ((pos != NULL) && (ntohs (pos->frag->off) <= off))
421 if (off >= off + FRAGSIZE (pos))
422 goto CLEANUP; /* error! */
423 if (ntohs (pos->frag->off) + FRAGSIZE (pos) > off)
424 off = ntohs (pos->frag->off) + FRAGSIZE (pos);
426 goto CLEANUP; /* error! */
430 return; /* some fragment is still missing */
432 msg = GNUNET_malloc (len);
436 memcpy (&msg[ntohs (pos->frag->off)], &pos->frag[1], FRAGSIZE (pos));
440 stats->change (stat_defragmented, 1);
442 printf ("Finished defragmentation!\n");
444 /* handle message! */
445 coreAPI->loopback_send (&pep->sender, msg, len, GNUNET_YES, NULL);
448 /* free fragment buffers */
449 freeFL (pep->head, 0);
455 * See if the new fragment is a part of this entry and join them if
456 * yes. Return GNUNET_SYSERR if the fragments do not match. Return GNUNET_OK if
457 * the fragments do match and the fragment has been processed. The
458 * defragCacheLock is already acquired by the caller whenever this
459 * method is called.<p>
461 * @param entry the entry in the cache
462 * @param pep the new entry
463 * @param packet the ip part in the new entry
467 const GNUNET_PeerIdentity * sender,
468 const P2P_fragmentation_MESSAGE * packet)
470 /* frame before ours; may end in the middle of
471 our frame or before it starts; NULL if we are
472 the earliest position we have received so far */
474 /* frame after ours; may start in the middle of
475 our frame or after it; NULL if we are the last
476 fragment we have received so far */
478 /* current position in the frame-list */
480 /* the new entry that we're inserting */
485 GNUNET_GE_ASSERT (NULL, entry != NULL);
486 if (0 != memcmp (sender, &entry->sender, sizeof (GNUNET_PeerIdentity)))
487 return GNUNET_SYSERR; /* wrong fragment list, try another! */
488 if (ntohl (packet->id) != entry->id)
489 return GNUNET_SYSERR; /* wrong fragment list, try another! */
491 printf ("Received fragment %u from %u to %u\n",
494 ntohs (packet->off) + ntohs (packet->header.size) -
495 sizeof (P2P_fragmentation_MESSAGE));
498 if ((pos != NULL) && (packet->len != pos->frag->len))
499 return GNUNET_SYSERR; /* wrong fragment size */
502 /* find the before-frame */
503 while ((pos != NULL) && (ntohs (pos->frag->off) < ntohs (packet->off)))
509 /* find the after-frame */
511 ntohs (packet->off) + ntohs (packet->header.size) -
512 sizeof (P2P_fragmentation_MESSAGE);
513 if (end <= ntohs (packet->off))
516 GNUNET_GE_DEVELOPER | GNUNET_GE_DEBUG | GNUNET_GE_BULK,
517 "Received invalid fragment at %s:%d\n", __FILE__,
519 return GNUNET_SYSERR; /* yuck! integer overflow! */
526 while ((after != NULL) && (ntohs (after->frag->off) < end))
529 if ((before != NULL) && (before == after))
531 /* this implies after or before != NULL and thereby the new
532 fragment is redundant as it is fully enclosed in an earlier
535 stats->change (stat_defragmented, 1);
536 return GNUNET_OK; /* drop, there is a packet that spans our range! */
539 if ((before != NULL) &&
541 ((htons (before->frag->off) +
542 FRAGSIZE (before)) >= htons (after->frag->off)))
544 /* this implies that the fragment that starts before us and the
545 fragment that comes after this one leave no space in the middle
546 or even overlap; thus we can drop this redundant piece */
548 stats->change (stat_defragmented, 1);
553 pep = GNUNET_malloc (sizeof (FC));
554 pep->frag = GNUNET_malloc (ntohs (packet->header.size));
555 memcpy (pep->frag, packet, ntohs (packet->header.size));
565 GNUNET_free (pos->frag);
571 /* end of insert first */
576 /* insert last: find the end, free everything after it */
577 freeFL (before->link, 1);
582 /* ok, we are filling the middle between two fragments; insert. If
583 there is anything else in the middle, it can be dropped as we're
584 bigger & cover that area as well */
585 /* free everything between before and after */
590 GNUNET_free (pos->frag);
598 entry->ttl = GNUNET_get_time () + DEFRAGMENTATION_TIMEOUT;
599 checkComplete (entry);
604 * Defragment the given fragment and pass to handler once
605 * defragmentation is complete.
607 * @param frag the packet to defragment
608 * @return GNUNET_SYSERR if the fragment is invalid
611 processFragment (const GNUNET_PeerIdentity * sender,
612 const GNUNET_MessageHeader * frag)
617 if (ntohs (frag->size) < sizeof (P2P_fragmentation_MESSAGE))
618 return GNUNET_SYSERR;
620 GNUNET_mutex_lock (defragCacheLock);
621 hash = sender->hashPubKey.bits[0] % DEFRAG_BUCKET_COUNT;
622 smf = defragmentationCache[hash];
626 tryJoin (smf, sender, (P2P_fragmentation_MESSAGE *) frag))
628 GNUNET_mutex_unlock (defragCacheLock);
631 if (0 == memcmp (sender, &smf->sender, sizeof (GNUNET_PeerIdentity)))
633 freeFL (smf->head, 1);
640 smf = GNUNET_malloc (sizeof (FC));
641 smf->next = defragmentationCache[hash];
642 defragmentationCache[hash] = smf;
643 smf->ttl = GNUNET_get_time () + DEFRAGMENTATION_TIMEOUT;
644 smf->sender = *sender;
646 smf->id = ntohl (((P2P_fragmentation_MESSAGE *) frag)->id);
647 smf->head = GNUNET_malloc (sizeof (FL));
648 smf->head->link = NULL;
649 smf->head->frag = GNUNET_malloc (ntohs (frag->size));
650 memcpy (smf->head->frag, frag, ntohs (frag->size));
652 GNUNET_mutex_unlock (defragCacheLock);
658 GNUNET_PeerIdentity sender;
659 /* maximums size of each fragment */
661 /** how long is this message part expected to be? */
663 /** when did we intend to transmit? */
664 GNUNET_CronTime transmissionTime;
668 * Send a message that had to be fragmented (right now!). First grabs
669 * the first part of the message (obtained from ctx->se) and stores
670 * that in a P2P_fragmentation_MESSAGE envelope. The remaining fragments are
671 * added to the send queue with GNUNET_EXTREME_PRIORITY (to ensure that they
672 * will be transmitted next). The logic here is that if the priority
673 * for the first fragment was sufficiently high, the priority should
674 * also have been sufficiently high for all of the other fragments (at
675 * this time) since they have the same priority. And we want to make
676 * sure that we send all of them since just sending the first fragment
677 * and then going to other messages of equal priority would not be
678 * such a great idea (i.e. would just waste bandwidth).
681 fragmentBMC (void *buf, void *cls, unsigned short len)
683 FragmentBMC *ctx = cls;
684 static int idGen = 0;
685 P2P_fragmentation_MESSAGE *frag;
690 if ((len < ctx->mtu) || (buf == NULL))
693 return GNUNET_SYSERR;
696 stats->change (stat_fragmented, 1);
697 id = (idGen++) + GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK, 512);
698 /* write first fragment to buf */
699 frag = (P2P_fragmentation_MESSAGE *) buf;
700 frag->header.size = htons (len);
701 frag->header.type = htons (GNUNET_P2P_PROTO_MESSAGE_FRAGMENT);
703 frag->off = htons (0);
704 frag->len = htons (ctx->len);
705 memcpy (&frag[1], &ctx[1], len - sizeof (P2P_fragmentation_MESSAGE));
707 /* create remaining fragments, add to queue! */
708 pos = len - sizeof (P2P_fragmentation_MESSAGE);
709 frag = GNUNET_malloc (ctx->mtu);
710 while (pos < ctx->len)
712 mlen = sizeof (P2P_fragmentation_MESSAGE) + ctx->len - pos;
715 GNUNET_GE_ASSERT (NULL, mlen > sizeof (P2P_fragmentation_MESSAGE));
716 frag->header.size = htons (mlen);
717 frag->header.type = htons (GNUNET_P2P_PROTO_MESSAGE_FRAGMENT);
719 frag->off = htons (pos);
720 frag->len = htons (ctx->len);
722 &((char *) (&ctx[1]))[pos],
723 mlen - sizeof (P2P_fragmentation_MESSAGE));
724 coreAPI->ciphertext_send (&ctx->sender,
726 GNUNET_EXTREME_PRIORITY,
727 ctx->transmissionTime - GNUNET_get_time ());
728 pos += mlen - sizeof (P2P_fragmentation_MESSAGE);
730 GNUNET_GE_ASSERT (NULL, pos == ctx->len);
737 * The given message must be fragmented. Produce a placeholder that
738 * corresponds to the first fragment. Once that fragment is scheduled
739 * for transmission, the placeholder should automatically add all of
740 * the other fragments (with very high priority).
743 fragment (const GNUNET_PeerIdentity * peer,
746 unsigned int targetTime,
747 unsigned int len, GNUNET_BuildMessageCallback bmc, void *bmcClosure)
752 GNUNET_GE_ASSERT (NULL, len > mtu);
753 GNUNET_GE_ASSERT (NULL, mtu > sizeof (P2P_fragmentation_MESSAGE));
754 fbmc = GNUNET_malloc (sizeof (FragmentBMC) + len);
756 fbmc->sender = *peer;
757 fbmc->transmissionTime = targetTime;
761 memcpy (&fbmc[1], bmcClosure, len);
762 GNUNET_free (bmcClosure);
766 if (GNUNET_SYSERR == bmc (&fbmc[1], bmcClosure, len))
772 xlen = mtu - sizeof (P2P_fragmentation_MESSAGE);
773 coreAPI->ciphertext_send_with_callback (peer, &fragmentBMC, fbmc, mtu, prio * xlen / len, /* compute new priority */
778 * Initialize Fragmentation module.
780 GNUNET_Fragmentation_ServiceAPI *
781 provide_module_fragmentation (GNUNET_CoreAPIForPlugins * capi)
783 static GNUNET_Fragmentation_ServiceAPI ret;
787 stats = coreAPI->service_request ("stats");
791 stats->create (gettext_noop ("# messages defragmented"));
793 stats->create (gettext_noop ("# messages fragmented"));
794 stat_discarded = stats->create (gettext_noop ("# fragments discarded"));
796 for (i = 0; i < DEFRAG_BUCKET_COUNT; i++)
797 defragmentationCache[i] = NULL;
798 defragCacheLock = GNUNET_mutex_create (GNUNET_NO);
799 GNUNET_cron_add_job (coreAPI->cron,
800 &defragmentationPurgeCron,
801 60 * GNUNET_CRON_SECONDS, 60 * GNUNET_CRON_SECONDS,
803 GNUNET_GE_LOG (capi->ectx,
804 GNUNET_GE_INFO | GNUNET_GE_USER | GNUNET_GE_REQUEST,
805 _("`%s' registering handler %d\n"), "fragmentation",
806 GNUNET_P2P_PROTO_MESSAGE_FRAGMENT);
807 capi->p2p_ciphertext_handler_register (GNUNET_P2P_PROTO_MESSAGE_FRAGMENT,
810 ret.fragment = &fragment;
815 * Shutdown fragmentation.
818 release_module_fragmentation ()
822 coreAPI->p2p_ciphertext_handler_unregister
823 (GNUNET_P2P_PROTO_MESSAGE_FRAGMENT, &processFragment);
824 GNUNET_cron_del_job (coreAPI->cron, &defragmentationPurgeCron,
825 60 * GNUNET_CRON_SECONDS, NULL);
826 for (i = 0; i < DEFRAG_BUCKET_COUNT; i++)
828 FC *pos = defragmentationCache[i];
831 FC *next = pos->next;
832 freeFL (pos->head, 1);
839 coreAPI->service_release (stats);
842 GNUNET_mutex_destroy (defragCacheLock);
843 defragCacheLock = NULL;
849 /* end of fragmentation.c */