2 This file is part of GNUnet
3 (C) 2004, 2006, 2009 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
21 * @file fragmentation/fragmentation.c
22 * @brief fragmentation and defragmentation, this code allows
23 * sending and receiving messages that are larger than
24 * the MTU of the transport. Messages are still limited
25 * to a maximum size of 65535 bytes, which is a good
26 * idea because otherwise we may need ungainly fragmentation
27 * buffers. Each connected peer can have at most one
28 * fragmented packet at any given point in time (prevents
29 * DoS attacks). Fragmented messages that have not been
30 * completed after a certain amount of time are discarded.
35 #include "gnunet_fragmentation_lib.h"
36 #include "gnunet_protocols.h"
37 #include "gnunet_util_lib.h"
40 * Message fragment. This header is followed
41 * by the actual data of the fragment.
47 struct GNUNET_MessageHeader header;
52 uint16_t off GNUNET_PACKED;
55 * "unique" id for the fragment
57 uint32_t id GNUNET_PACKED;
64 struct GNUNET_FRAGEMENT_Ctxbuffer{
65 struct GNUNET_FRAGEMENT_Ctxbuffer *next;
70 struct GNUNET_TIME_Absolute receivedTime;
71 struct GNUNET_PeerIdentity peerID;
77 * Defragmentation context.
79 struct GNUNET_FRAGMENT_Context
82 struct GNUNET_FRAGEMENT_Ctxbuffer *buffer;
83 GNUNET_FRAGMENT_MessageProcessor proc;
89 * Fragment an over-sized message.
91 * @param msg the message to fragment
92 * @param mtu the maximum message size
93 * @param proc function to call for each fragment
94 * @param proc_cls closure for proc
97 GNUNET_FRAGMENT_fragment (const struct GNUNET_MessageHeader *msg,
99 GNUNET_FRAGMENT_MessageProcessor proc,
102 uint32_t id = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, 256);
103 size_t size = sizeof(struct Fragment);
105 if(ntohs(msg->size) > mtu-size){
110 lastSize = ntohs(msg->size) % (mtu-size);
111 num = ntohs(msg->size) / (mtu - size);
116 for(i = 0; i<actualNum; i++)
118 struct Fragment *frag;
119 if(actualNum != num){
121 frag = (struct Fragment *)GNUNET_malloc(mtu);
124 frag = (struct Fragment *)GNUNET_malloc(lastSize+size);
128 frag = (struct Fragment *)GNUNET_malloc(mtu);
130 frag->header.type = htons(GNUNET_MESSAGE_TYPE_FRAGMENT);
131 frag->id = htonl(id);
132 frag->off = htons((mtu-size)*i);
133 frag->mtu = htons(mtu);
134 frag->totalNum = htons(actualNum);
135 frag->totalSize = msg->size;
136 char *tmpMsg = (char *)msg;
137 if(actualNum != num){
139 frag->header.size = htons(mtu);
140 memcpy(&frag[1], tmpMsg + (mtu-size)*i, mtu - size);
143 frag->header.size = htons(lastSize+size);
144 memcpy(&frag[1], tmpMsg + (mtu-size)*i, lastSize);
148 frag->header.size = htons(mtu);
149 memcpy(&frag[1], tmpMsg + (mtu-size)*i, mtu - size);
151 proc(proc_cls, &frag->header);
158 * Create a defragmentation context.
160 * @param stats statistics context
161 * @param proc function to call with defragmented messages
162 * @param proc_cls closure for proc
163 * @return the defragmentation context
165 struct GNUNET_FRAGMENT_Context *
166 GNUNET_FRAGMENT_context_create (struct GNUNET_STATISTICS_Handle *stats,
167 GNUNET_FRAGMENT_MessageProcessor proc,
170 struct GNUNET_FRAGMENT_Context *ctx = (struct GNUNET_FRAGMENT_Context*)GNUNET_malloc(sizeof(struct GNUNET_FRAGMENT_Context));
173 ctx->proc_cls = proc_cls;
180 * Destroy the given defragmentation context.
183 GNUNET_FRAGMENT_context_destroy (struct GNUNET_FRAGMENT_Context *ctx)
185 struct GNUNET_FRAGEMENT_Ctxbuffer *buffer;
186 struct GNUNET_FRAGEMENT_Ctxbuffer *temp;
187 buffer = ctx->buffer;
189 while (buffer != NULL)
192 GNUNET_free(buffer->num);
202 * We have received a fragment. Process it.
204 * @param ctx the context
205 * @param sender who transmitted the fragment
206 * @param msg the message that was received
209 GNUNET_FRAGMENT_process (struct GNUNET_FRAGMENT_Context *ctx,
210 const struct GNUNET_PeerIdentity *sender,
211 const struct GNUNET_MessageHeader *msg)
213 uint16_t type = ntohs(msg->type);
214 int exist = 0, received = 0;
215 if(type!=GNUNET_MESSAGE_TYPE_FRAGMENT){
218 struct Fragment *frag = (struct Fragment *)msg;
219 struct GNUNET_FRAGEMENT_Ctxbuffer* buffer;
220 struct GNUNET_FRAGEMENT_Ctxbuffer* prev;
222 buffer = ctx->buffer;
223 while (buffer != NULL)
225 if ((buffer->id == ntohl(frag->id))&&(0 == memcmp (&buffer->peerID,
226 sender, sizeof (struct GNUNET_PeerIdentity)))){
231 buffer = buffer->next;
237 for(i = 0; i<ntohs(frag->totalNum); i++){
238 if(buffer->num[i]==ntohs(frag->off)/(ntohs(frag->mtu)-sizeof(struct Fragment))){
246 buffer = GNUNET_malloc(sizeof(struct GNUNET_FRAGEMENT_Ctxbuffer));
247 buffer->num = GNUNET_malloc(ntohs(frag->totalNum)*sizeof(int));
249 for(j = 0; j<ntohs(frag->totalNum); j++){
250 buffer->num[j] = -10;
252 buffer->peerID = *sender;
253 buffer->id = ntohl(frag->id);
254 buffer->receivedTime = GNUNET_TIME_absolute_get ();
255 uint16_t si = ntohs(frag->totalSize);
257 buffer->buff = GNUNET_malloc(si);
258 buffer->next = ctx->buffer;
259 ctx->buffer = buffer;
263 buffer->num[buffer->counter++]=ntohs(frag->off)/(ntohs(frag->mtu)-sizeof(struct Fragment));
264 uint16_t sizeoffrag = ntohs(frag->header.size) - sizeof(struct Fragment);
265 memcpy(&buffer->buff[ntohs(frag->off)], &frag[1], sizeoffrag);
266 buffer->receivedTime = GNUNET_TIME_absolute_get ();
269 if(buffer->counter == ntohs(frag->totalNum))
271 ctx->proc(ctx->proc_cls, (struct GNUNET_MessageHeader *)buffer->buff);
273 ctx->buffer = buffer->next;
276 prev->next = buffer->next;
288 * How many buckets does the fragment hash table
291 #define DEFRAG_BUCKET_COUNT 16
294 * After how long do fragments time out?
296 #ifndef DEFRAGMENTATION_TIMEOUT
297 #define DEFRAGMENTATION_TIMEOUT (3 * GNUNET_CRON_MINUTES)
301 * Entry in the linked list of fragments.
306 P2P_fragmentation_MESSAGE *frag;
310 * Entry in the GNUNET_hash table of fragments.
316 GNUNET_PeerIdentity sender;
321 #define FRAGSIZE(fl) ((ntohs(fl->frag->header.size)-sizeof(P2P_fragmentation_MESSAGE)))
323 static GNUNET_CoreAPIForPlugins *coreAPI;
325 static GNUNET_Stats_ServiceAPI *stats;
327 static int stat_defragmented;
329 static int stat_fragmented;
331 static int stat_discarded;
334 * Hashtable *with* collision management!
336 static FC *defragmentationCache[DEFRAG_BUCKET_COUNT];
339 * Lock for the defragmentation cache.
341 static struct GNUNET_Mutex *defragCacheLock;
344 freeFL (FL * fl, int c)
350 stats->change (stat_discarded, c);
351 GNUNET_free (fl->frag);
358 * This cron job ensures that we purge buffers of fragments
359 * that have timed out. It can run in much longer intervals
360 * than the defragmentationCron, e.g. every 60s.
362 * This method goes through the hashtable, finds entries that
363 * have timed out and removes them (and all the fragments that
364 * belong to the entry). It's a bit more complicated as the
365 * collision list is also collapsed.
368 defragmentationPurgeCron (void *unused)
375 GNUNET_mutex_lock (defragCacheLock);
376 for (i = 0; i < DEFRAG_BUCKET_COUNT; i++)
379 smf = defragmentationCache[i];
382 if (smf->ttl < GNUNET_get_time ())
384 /* free linked list of fragments */
385 freeFL (smf->head, 1);
389 defragmentationCache[i] = next;
399 } /* while smf != NULL */
400 } /* for all buckets */
401 GNUNET_mutex_unlock (defragCacheLock);
405 * Check if this fragment-list is complete. If yes, put it together,
406 * process and free all buffers. Does not free the pep
407 * itself (but sets the TTL to 0 to have the cron free it
408 * in the next iteration).
410 * @param pep the entry in the GNUNET_hash table
413 checkComplete (FC * pep)
420 GNUNET_GE_ASSERT (NULL, pep != NULL);
424 len = ntohs (pos->frag->len);
426 goto CLEANUP; /* really bad error! */
428 while ((pos != NULL) && (ntohs (pos->frag->off) <= off))
430 if (off >= off + FRAGSIZE (pos))
431 goto CLEANUP; /* error! */
432 if (ntohs (pos->frag->off) + FRAGSIZE (pos) > off)
433 off = ntohs (pos->frag->off) + FRAGSIZE (pos);
435 goto CLEANUP; /* error! */
439 return; /* some fragment is still missing */
441 msg = GNUNET_malloc (len);
445 memcpy (&msg[ntohs (pos->frag->off)], &pos->frag[1], FRAGSIZE (pos));
449 stats->change (stat_defragmented, 1);
451 printf ("Finished defragmentation!\n");
453 /* handle message! */
454 coreAPI->loopback_send (&pep->sender, msg, len, GNUNET_YES, NULL);
457 /* free fragment buffers */
458 freeFL (pep->head, 0);
464 * See if the new fragment is a part of this entry and join them if
465 * yes. Return GNUNET_SYSERR if the fragments do not match. Return GNUNET_OK if
466 * the fragments do match and the fragment has been processed. The
467 * defragCacheLock is already acquired by the caller whenever this
468 * method is called.<p>
470 * @param entry the entry in the cache
471 * @param pep the new entry
472 * @param packet the ip part in the new entry
476 const GNUNET_PeerIdentity * sender,
477 const P2P_fragmentation_MESSAGE * packet)
479 /* frame before ours; may end in the middle of
480 our frame or before it starts; NULL if we are
481 the earliest position we have received so far */
483 /* frame after ours; may start in the middle of
484 our frame or after it; NULL if we are the last
485 fragment we have received so far */
487 /* current position in the frame-list */
489 /* the new entry that we're inserting */
494 GNUNET_GE_ASSERT (NULL, entry != NULL);
495 if (0 != memcmp (sender, &entry->sender, sizeof (GNUNET_PeerIdentity)))
496 return GNUNET_SYSERR; /* wrong fragment list, try another! */
497 if (ntohl (packet->id) != entry->id)
498 return GNUNET_SYSERR; /* wrong fragment list, try another! */
500 printf ("Received fragment %u from %u to %u\n",
503 ntohs (packet->off) + ntohs (packet->header.size) -
504 sizeof (P2P_fragmentation_MESSAGE));
507 if ((pos != NULL) && (packet->len != pos->frag->len))
508 return GNUNET_SYSERR; /* wrong fragment size */
511 /* find the before-frame */
512 while ((pos != NULL) && (ntohs (pos->frag->off) < ntohs (packet->off)))
518 /* find the after-frame */
520 ntohs (packet->off) + ntohs (packet->header.size) -
521 sizeof (P2P_fragmentation_MESSAGE);
522 if (end <= ntohs (packet->off))
525 GNUNET_GE_DEVELOPER | GNUNET_GE_DEBUG | GNUNET_GE_BULK,
526 "Received invalid fragment at %s:%d\n", __FILE__,
528 return GNUNET_SYSERR; /* yuck! integer overflow! */
535 while ((after != NULL) && (ntohs (after->frag->off) < end))
538 if ((before != NULL) && (before == after))
540 /* this implies after or before != NULL and thereby the new
541 fragment is redundant as it is fully enclosed in an earlier
544 stats->change (stat_defragmented, 1);
545 return GNUNET_OK; /* drop, there is a packet that spans our range! */
548 if ((before != NULL) &&
550 ((htons (before->frag->off) +
551 FRAGSIZE (before)) >= htons (after->frag->off)))
553 /* this implies that the fragment that starts before us and the
554 fragment that comes after this one leave no space in the middle
555 or even overlap; thus we can drop this redundant piece */
557 stats->change (stat_defragmented, 1);
562 pep = GNUNET_malloc (sizeof (FC));
563 pep->frag = GNUNET_malloc (ntohs (packet->header.size));
564 memcpy (pep->frag, packet, ntohs (packet->header.size));
574 GNUNET_free (pos->frag);
580 /* end of insert first */
585 /* insert last: find the end, free everything after it */
586 freeFL (before->link, 1);
591 /* ok, we are filling the middle between two fragments; insert. If
592 there is anything else in the middle, it can be dropped as we're
593 bigger & cover that area as well */
594 /* free everything between before and after */
599 GNUNET_free (pos->frag);
607 entry->ttl = GNUNET_get_time () + DEFRAGMENTATION_TIMEOUT;
608 checkComplete (entry);
613 * Defragment the given fragment and pass to handler once
614 * defragmentation is complete.
616 * @param frag the packet to defragment
617 * @return GNUNET_SYSERR if the fragment is invalid
620 processFragment (const GNUNET_PeerIdentity * sender,
621 const GNUNET_MessageHeader * frag)
626 if (ntohs (frag->size) < sizeof (P2P_fragmentation_MESSAGE))
627 return GNUNET_SYSERR;
629 GNUNET_mutex_lock (defragCacheLock);
630 hash = sender->hashPubKey.bits[0] % DEFRAG_BUCKET_COUNT;
631 smf = defragmentationCache[hash];
635 tryJoin (smf, sender, (P2P_fragmentation_MESSAGE *) frag))
637 GNUNET_mutex_unlock (defragCacheLock);
640 if (0 == memcmp (sender, &smf->sender, sizeof (GNUNET_PeerIdentity)))
642 freeFL (smf->head, 1);
649 smf = GNUNET_malloc (sizeof (FC));
650 smf->next = defragmentationCache[hash];
651 defragmentationCache[hash] = smf;
652 smf->ttl = GNUNET_get_time () + DEFRAGMENTATION_TIMEOUT;
653 smf->sender = *sender;
655 smf->id = ntohl (((P2P_fragmentation_MESSAGE *) frag)->id);
656 smf->head = GNUNET_malloc (sizeof (FL));
657 smf->head->link = NULL;
658 smf->head->frag = GNUNET_malloc (ntohs (frag->size));
659 memcpy (smf->head->frag, frag, ntohs (frag->size));
661 GNUNET_mutex_unlock (defragCacheLock);
667 GNUNET_PeerIdentity sender;
668 /* maximums size of each fragment */
670 /** how long is this message part expected to be? */
672 /** when did we intend to transmit? */
673 GNUNET_CronTime transmissionTime;
677 * Send a message that had to be fragmented (right now!). First grabs
678 * the first part of the message (obtained from ctx->se) and stores
679 * that in a P2P_fragmentation_MESSAGE envelope. The remaining fragments are
680 * added to the send queue with GNUNET_EXTREME_PRIORITY (to ensure that they
681 * will be transmitted next). The logic here is that if the priority
682 * for the first fragment was sufficiently high, the priority should
683 * also have been sufficiently high for all of the other fragments (at
684 * this time) since they have the same priority. And we want to make
685 * sure that we send all of them since just sending the first fragment
686 * and then going to other messages of equal priority would not be
687 * such a great idea (i.e. would just waste bandwidth).
690 fragmentBMC (void *buf, void *cls, unsigned short len)
692 FragmentBMC *ctx = cls;
693 static int idGen = 0;
694 P2P_fragmentation_MESSAGE *frag;
699 if ((len < ctx->mtu) || (buf == NULL))
702 return GNUNET_SYSERR;
705 stats->change (stat_fragmented, 1);
706 id = (idGen++) + GNUNET_random_u32 (GNUNET_RANDOM_QUALITY_WEAK, 512);
707 /* write first fragment to buf */
708 frag = (P2P_fragmentation_MESSAGE *) buf;
709 frag->header.size = htons (len);
710 frag->header.type = htons (GNUNET_P2P_PROTO_MESSAGE_FRAGMENT);
712 frag->off = htons (0);
713 frag->len = htons (ctx->len);
714 memcpy (&frag[1], &ctx[1], len - sizeof (P2P_fragmentation_MESSAGE));
716 /* create remaining fragments, add to queue! */
717 pos = len - sizeof (P2P_fragmentation_MESSAGE);
718 frag = GNUNET_malloc (ctx->mtu);
719 while (pos < ctx->len)
721 mlen = sizeof (P2P_fragmentation_MESSAGE) + ctx->len - pos;
724 GNUNET_GE_ASSERT (NULL, mlen > sizeof (P2P_fragmentation_MESSAGE));
725 frag->header.size = htons (mlen);
726 frag->header.type = htons (GNUNET_P2P_PROTO_MESSAGE_FRAGMENT);
728 frag->off = htons (pos);
729 frag->len = htons (ctx->len);
731 &((char *) (&ctx[1]))[pos],
732 mlen - sizeof (P2P_fragmentation_MESSAGE));
733 coreAPI->ciphertext_send (&ctx->sender,
735 GNUNET_EXTREME_PRIORITY,
736 ctx->transmissionTime - GNUNET_get_time ());
737 pos += mlen - sizeof (P2P_fragmentation_MESSAGE);
739 GNUNET_GE_ASSERT (NULL, pos == ctx->len);
746 * The given message must be fragmented. Produce a placeholder that
747 * corresponds to the first fragment. Once that fragment is scheduled
748 * for transmission, the placeholder should automatically add all of
749 * the other fragments (with very high priority).
752 fragment (const GNUNET_PeerIdentity * peer,
755 unsigned int targetTime,
756 unsigned int len, GNUNET_BuildMessageCallback bmc, void *bmcClosure)
761 GNUNET_GE_ASSERT (NULL, len > mtu);
762 GNUNET_GE_ASSERT (NULL, mtu > sizeof (P2P_fragmentation_MESSAGE));
763 fbmc = GNUNET_malloc (sizeof (FragmentBMC) + len);
765 fbmc->sender = *peer;
766 fbmc->transmissionTime = targetTime;
770 memcpy (&fbmc[1], bmcClosure, len);
771 GNUNET_free (bmcClosure);
775 if (GNUNET_SYSERR == bmc (&fbmc[1], bmcClosure, len))
781 xlen = mtu - sizeof (P2P_fragmentation_MESSAGE);
782 coreAPI->ciphertext_send_with_callback (peer, &fragmentBMC, fbmc, mtu, prio * xlen / len, /* compute new priority */
787 * Initialize Fragmentation module.
789 GNUNET_Fragmentation_ServiceAPI *
790 provide_module_fragmentation (GNUNET_CoreAPIForPlugins * capi)
792 static GNUNET_Fragmentation_ServiceAPI ret;
796 stats = coreAPI->service_request ("stats");
800 stats->create (gettext_noop ("# messages defragmented"));
802 stats->create (gettext_noop ("# messages fragmented"));
803 stat_discarded = stats->create (gettext_noop ("# fragments discarded"));
805 for (i = 0; i < DEFRAG_BUCKET_COUNT; i++)
806 defragmentationCache[i] = NULL;
807 defragCacheLock = GNUNET_mutex_create (GNUNET_NO);
808 GNUNET_cron_add_job (coreAPI->cron,
809 &defragmentationPurgeCron,
810 60 * GNUNET_CRON_SECONDS, 60 * GNUNET_CRON_SECONDS,
812 GNUNET_GE_LOG (capi->ectx,
813 GNUNET_GE_INFO | GNUNET_GE_USER | GNUNET_GE_REQUEST,
814 _("`%s' registering handler %d\n"), "fragmentation",
815 GNUNET_P2P_PROTO_MESSAGE_FRAGMENT);
816 capi->p2p_ciphertext_handler_register (GNUNET_P2P_PROTO_MESSAGE_FRAGMENT,
819 ret.fragment = &fragment;
824 * Shutdown fragmentation.
827 release_module_fragmentation ()
831 coreAPI->p2p_ciphertext_handler_unregister
832 (GNUNET_P2P_PROTO_MESSAGE_FRAGMENT, &processFragment);
833 GNUNET_cron_del_job (coreAPI->cron, &defragmentationPurgeCron,
834 60 * GNUNET_CRON_SECONDS, NULL);
835 for (i = 0; i < DEFRAG_BUCKET_COUNT; i++)
837 FC *pos = defragmentationCache[i];
840 FC *next = pos->next;
841 freeFL (pos->head, 1);
848 coreAPI->service_release (stats);
851 GNUNET_mutex_destroy (defragCacheLock);
852 defragCacheLock = NULL;
858 /* end of fragmentation.c */