src/testing/test_testing_topology.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 2, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20 /**
  21  * @file testing/test_testing_topology.c
  22  * @brief base testcase for testing all the topologies provided
  23  */
  24 #include "platform.h"
  25 #include "gnunet_testing_lib.h"
  26 #include "gnunet_core_service.h"
  27
  28 #define VERBOSE GNUNET_YES
  29
  30 /**
  31  * How long until we fail the whole testcase?
  32  */
  33 #define TEST_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 600)
  34
  35 /**
  36  * How long until we give up on connecting the peers?
  37  */
  38 #define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
  39
  40 #define DEFAULT_NUM_PEERS 4
  41
  42 #define MAX_OUTSTANDING_CONNECTIONS 300
  43
  44 static float fail_percentage = 0.05;
  45
  46 static int ok;
  47
  48 static unsigned long long num_peers;
  49
  50 static unsigned int total_connections;
  51
  52 static unsigned int failed_connections;
  53
  54 static unsigned int total_server_connections;
  55
  56 static unsigned int total_messages_received;
  57
  58 static unsigned int expected_messages;
  59
  60 static unsigned int expected_connections;
  61
  62 static unsigned long long peers_left;
  63
  64 static struct GNUNET_TESTING_PeerGroup *pg;
  65
  66 static struct GNUNET_SCHEDULER_Handle *sched;
  67
  68 const struct GNUNET_CONFIGURATION_Handle *main_cfg;
  69
  70 GNUNET_SCHEDULER_TaskIdentifier die_task;
  71
  72 static char *dotOutFileName = "topology.dot";
  73
  74 static FILE *dotOutFile;
  75
  76 static char *topology_string;
  77
  78 static int transmit_ready_scheduled;
  79
  80 static int transmit_ready_failed;
  81
  82 static int transmit_ready_called;
  83
  84 #define MTYPE 12345
  85
  86 struct GNUNET_TestMessage
  87 {
  88   /**
  89    * Header of the message
  90    */
  91   struct GNUNET_MessageHeader header;
  92
  93   /**
  94    * Unique identifier for this message.
  95    */
  96   uint32_t uid;
  97 };
  98
  99 struct TestMessageContext
 100 {
 101   /* This is a linked list */
 102   struct TestMessageContext *next;
 103
 104   /* Handle to the sending peer core */
 105   struct GNUNET_CORE_Handle *peer1handle;
 106
 107   /* Handle to the receiving peer core */
 108   struct GNUNET_CORE_Handle *peer2handle;
 109
 110   /* Handle to the sending peer daemon */
 111   struct GNUNET_TESTING_Daemon *peer1;
 112
 113   /* Handle to the receiving peer daemon */
 114   struct GNUNET_TESTING_Daemon *peer2;
 115
 116   /* Identifier for this message, so we don't disconnect other peers! */
 117   uint32_t uid;
 118
 119 };
 120
 121 static struct TestMessageContext *test_messages;
 122
 123 static void
 124 finish_testing ()
 125 {
 126   GNUNET_assert (pg != NULL);
 127   struct TestMessageContext *pos;
 128   struct TestMessageContext *free_pos;
 129 #if VERBOSE
 130   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 131               "Called finish testing, stopping daemons.\n");
 132 #endif
 133   int count;
 134   count = 0;
 135   pos = test_messages;
 136   while (pos != NULL)
 137     {
 138       if (pos->peer1handle != NULL)
 139         {
 140           GNUNET_CORE_disconnect(pos->peer1handle);
 141           pos->peer1handle = NULL;
 142         }
 143       if (pos->peer2handle != NULL)
 144         {
 145           GNUNET_CORE_disconnect(pos->peer2handle);
 146           pos->peer2handle = NULL;
 147         }
 148       free_pos = pos;
 149       pos = pos->next;
 150       GNUNET_free(free_pos);
 151     }
 152 #if VERBOSE
 153           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 154                       "transmit_ready's scheduled %d, failed %d, transmit_ready's called %d\n", transmit_ready_scheduled, transmit_ready_failed, transmit_ready_called);
 155 #endif
 156   sleep(1);
 157 #if VERBOSE
 158           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 159                       "Calling daemons_stop\n");
 160 #endif
 161   GNUNET_TESTING_daemons_stop (pg);
 162 #if VERBOSE
 163           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 164                       "daemons_stop finished\n");
 165 #endif
 166   if (dotOutFile != NULL)
 167     {
 168       fprintf(dotOutFile, "}");
 169       fclose(dotOutFile);
 170     }
 171
 172   ok = 0;
 173 }
 174
 175
 176 static void
 177 disconnect_cores (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 178 {
 179   struct TestMessageContext *pos = cls;
 180
 181   /* Disconnect from the respective cores */
 182 #if VERBOSE
 183   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 184               "Disconnecting from peer 1 `%4s'\n", GNUNET_i2s (&pos->peer1->id));
 185 #endif
 186   if (pos->peer1handle != NULL)
 187     GNUNET_CORE_disconnect(pos->peer1handle);
 188 #if VERBOSE
 189   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 190               "Disconnecting from peer 2 `%4s'\n", GNUNET_i2s (&pos->peer2->id));
 191 #endif
 192   if (pos->peer2handle != NULL)
 193     GNUNET_CORE_disconnect(pos->peer2handle);
 194   /* Set handles to NULL so test case can be ended properly */
 195   pos->peer1handle = NULL;
 196   pos->peer2handle = NULL;
 197   /* Decrement total connections so new can be established */
 198   total_server_connections -= 2;
 199 }
 200
 201 static int
 202 process_mtype (void *cls,
 203                const struct GNUNET_PeerIdentity *peer,
 204                const struct GNUNET_MessageHeader *message,
 205                struct GNUNET_TIME_Relative latency,
 206                uint32_t distance)
 207 {
 208   struct TestMessageContext *pos = cls;
 209   struct GNUNET_TestMessage *msg = (struct GNUNET_TestMessage *)message;
 210   if (pos->uid != ntohl(msg->uid))
 211     return GNUNET_OK;
 212
 213   total_messages_received++;
 214 #if VERBOSE
 215   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 216               "Received message from `%4s', type %d.\n", GNUNET_i2s (peer), ntohs(message->type));
 217   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 218               "Total messages received %d, expected %d.\n", total_messages_received, expected_messages);
 219 #endif
 220
 221   if (total_messages_received == expected_messages)
 222     {
 223       GNUNET_SCHEDULER_cancel (sched, die_task);
 224       GNUNET_SCHEDULER_add_now (sched, &finish_testing, NULL);
 225     }
 226   else
 227     {
 228       GNUNET_SCHEDULER_add_now(sched, &disconnect_cores, pos);
 229     }
 230
 231   return GNUNET_OK;
 232 }
 233
 234 static void
 235 end_badly (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 236 {
 237   char *msg = cls;
 238   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 239               "End badly was called (%s)... stopping daemons.\n", msg);
 240   struct TestMessageContext *pos;
 241   struct TestMessageContext *free_pos;
 242
 243   pos = test_messages;
 244   while (pos != NULL)
 245     {
 246       if (pos->peer1handle != NULL)
 247         {
 248           GNUNET_CORE_disconnect(pos->peer1handle);
 249           pos->peer1handle = NULL;
 250         }
 251       if (pos->peer2handle != NULL)
 252         {
 253           GNUNET_CORE_disconnect(pos->peer2handle);
 254           pos->peer2handle = NULL;
 255         }
 256       free_pos = pos;
 257       pos = pos->next;
 258       GNUNET_free(free_pos);
 259     }
 260
 261   if (pg != NULL)
 262     {
 263       GNUNET_TESTING_daemons_stop (pg);
 264       ok = 7331;                /* Opposite of leet */
 265     }
 266   else
 267     ok = 401;                   /* Never got peers started */
 268
 269   if (dotOutFile != NULL)
 270     {
 271       fprintf(dotOutFile, "}");
 272       fclose(dotOutFile);
 273     }
 274 }
 275
 276
 277
 278 static size_t
 279 transmit_ready (void *cls, size_t size, void *buf)
 280 {
 281   struct GNUNET_TestMessage *m;
 282   struct TestMessageContext *pos = cls;
 283
 284   GNUNET_assert (buf != NULL);
 285   m = (struct GNUNET_TestMessage *) buf;
 286   m->header.type = htons (MTYPE);
 287   m->header.size = htons (sizeof (struct GNUNET_TestMessage));
 288   m->uid = htonl(pos->uid);
 289   transmit_ready_called++;
 290 #if VERBOSE
 291   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 292               "transmit ready for peer %s\ntransmit_ready's scheduled %d, transmit_ready's called %d\n", GNUNET_i2s(&pos->peer1->id), transmit_ready_scheduled, transmit_ready_called);
 293 #endif
 294   return sizeof (struct GNUNET_TestMessage);
 295 }
 296
 297
 298 static struct GNUNET_CORE_MessageHandler no_handlers[] = {
 299   {NULL, 0, 0}
 300 };
 301
 302 static struct GNUNET_CORE_MessageHandler handlers[] = {
 303   {&process_mtype, MTYPE, sizeof (struct GNUNET_TestMessage)},
 304   {NULL, 0, 0}
 305 };
 306
 307 static void
 308 init_notify_peer2 (void *cls,
 309              struct GNUNET_CORE_Handle *server,
 310              const struct GNUNET_PeerIdentity *my_identity,
 311              const struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded *publicKey)
 312 {
 313   struct TestMessageContext *pos = cls;
 314
 315 #if VERBOSE
 316   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 317               "Core connection to `%4s' established, scheduling message send\n",
 318               GNUNET_i2s (my_identity));
 319 #endif
 320   total_server_connections++;
 321
 322   if (NULL == GNUNET_CORE_notify_transmit_ready (pos->peer1handle,
 323                                                  0,
 324                                                  TIMEOUT,
 325                                                  &pos->peer2->id,
 326                                                  sizeof (struct GNUNET_TestMessage),
 327                                                  &transmit_ready, pos))
 328     {
 329       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 330                   "RECEIVED NULL when asking core (1) for transmission to peer `%4s'\n",
 331                   GNUNET_i2s (&pos->peer2->id));
 332       transmit_ready_failed++;
 333     }
 334   else
 335     {
 336       transmit_ready_scheduled++;
 337     }
 338 }
 339
 340
 341 static void
 342 init_notify_peer1 (void *cls,
 343              struct GNUNET_CORE_Handle *server,
 344              const struct GNUNET_PeerIdentity *my_identity,
 345              const struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded *publicKey)
 346 {
 347   struct TestMessageContext *pos = cls;
 348   total_server_connections++;
 349
 350 #if VERBOSE
 351   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 352               "Core connection to `%4s' established, setting up handles\n",
 353               GNUNET_i2s (my_identity));
 354 #endif
 355
 356   /*
 357    * Connect to the receiving peer
 358    */
 359   pos->peer2handle = GNUNET_CORE_connect (sched,
 360                        pos->peer2->cfg,
 361                        TIMEOUT,
 362                        pos,
 363                        &init_notify_peer2,
 364                        NULL,
 365                        NULL,
 366                        NULL,
 367                        GNUNET_YES, NULL, GNUNET_YES, handlers);
 368
 369 }
 370
 371
 372 static void
 373 send_test_messages (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 374 {
 375   struct TestMessageContext *pos = cls;
 376
 377   if ((tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN) || (cls == NULL))
 378     return;
 379
 380   if (die_task == GNUNET_SCHEDULER_NO_TASK)
 381     {
 382       die_task = GNUNET_SCHEDULER_add_delayed (sched,
 383                                                TEST_TIMEOUT,
 384                                                &end_badly, "from create topology (timeout)");
 385     }
 386
 387   if (total_server_connections >= MAX_OUTSTANDING_CONNECTIONS)
 388     {
 389       GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1),
 390                                     &send_test_messages, pos);
 391       return; /* Otherwise we'll double schedule messages here! */
 392     }
 393
 394   /*
 395    * Connect to the sending peer
 396    */
 397   pos->peer1handle = GNUNET_CORE_connect (sched,
 398                                           pos->peer1->cfg,
 399                                           TIMEOUT,
 400                                           pos,
 401                                           &init_notify_peer1,
 402                                           NULL,
 403                                           NULL,
 404                                           NULL,
 405                                           GNUNET_NO, NULL, GNUNET_NO, no_handlers);
 406
 407   GNUNET_assert(pos->peer1handle != NULL);
 408
 409   if (total_server_connections < MAX_OUTSTANDING_CONNECTIONS)
 410     {
 411       GNUNET_SCHEDULER_add_now (sched,
 412                                 &send_test_messages, pos->next);
 413     }
 414   else
 415     {
 416       GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1),
 417                                     &send_test_messages, pos->next);
 418     }
 419 }
 420
 421
 422 void
 423 topology_callback (void *cls,
 424                    const struct GNUNET_PeerIdentity *first,
 425                    const struct GNUNET_PeerIdentity *second,
 426                    const struct GNUNET_CONFIGURATION_Handle *first_cfg,
 427                    const struct GNUNET_CONFIGURATION_Handle *second_cfg,
 428                    struct GNUNET_TESTING_Daemon *first_daemon,
 429                    struct GNUNET_TESTING_Daemon *second_daemon,
 430                    const char *emsg)
 431 {
 432   struct TestMessageContext *temp_context;
 433   if (emsg == NULL)
 434     {
 435       total_connections++;
 436 #if VERBOSE
 437       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s\n",
 438                first_daemon->shortname,
 439                second_daemon->shortname);
 440 #endif
 441       temp_context = GNUNET_malloc(sizeof(struct TestMessageContext));
 442       temp_context->peer1 = first_daemon;
 443       temp_context->peer2 = second_daemon;
 444       temp_context->next = test_messages;
 445       temp_context->uid = total_connections;
 446       test_messages = temp_context;
 447
 448       expected_messages++;
 449       if (dotOutFile != NULL)
 450         fprintf(dotOutFile, "\tn%s -- n%s;\n", first_daemon->shortname, second_daemon->shortname);
 451     }
 452 #if VERBOSE
 453   else
 454     {
 455       failed_connections++;
 456       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n",
 457                first_daemon->shortname,
 458                second_daemon->shortname, emsg);
 459     }
 460 #endif
 461
 462   if (total_connections == expected_connections)
 463     {
 464 #if VERBOSE
 465       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 466                   "Created %d total connections, which is our target number!  Calling send messages.\n",
 467                   total_connections);
 468 #endif
 469
 470       GNUNET_SCHEDULER_cancel (sched, die_task);
 471       die_task = GNUNET_SCHEDULER_NO_TASK;
 472       GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1), &send_test_messages, test_messages);
 473     }
 474   else if (total_connections + failed_connections == expected_connections)
 475     {
 476       if (failed_connections < (unsigned int)(fail_percentage * total_connections))
 477         {
 478           GNUNET_SCHEDULER_cancel (sched, die_task);
 479           die_task = GNUNET_SCHEDULER_NO_TASK;
 480           GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1), &send_test_messages, test_messages);
 481         }
 482       else
 483         {
 484           GNUNET_SCHEDULER_cancel (sched, die_task);
 485           die_task = GNUNET_SCHEDULER_add_now (sched,
 486                                                &end_badly, "from topology_callback (too many failed connections)");
 487         }
 488     }
 489   else
 490     {
 491 #if VERBOSE
 492       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 493                   "Have %d total connections, %d failed connections, Want %d (at least %d)\n",
 494                   total_connections, failed_connections, expected_connections, expected_connections - (unsigned int)(fail_percentage * expected_connections));
 495 #endif
 496     }
 497 }
 498
 499
 500 static void
 501 create_topology ()
 502 {
 503   expected_connections = -1;
 504   if ((pg != NULL) && (peers_left == 0))
 505     {
 506       /* create_topology will read the topology information from
 507          the config already contained in the peer group, so should
 508          we have create_topology called from start peers?  I think
 509          maybe this way is best so that the client can know both
 510          when peers are started, and when they are connected.
 511        */
 512       expected_connections = GNUNET_TESTING_create_topology (pg);
 513 #if VERBOSE
 514       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 515                   "Have %d expected connections\n", expected_connections);
 516 #endif
 517     }
 518
 519   GNUNET_SCHEDULER_cancel (sched, die_task);
 520   if (expected_connections == GNUNET_SYSERR)
 521     {
 522       die_task = GNUNET_SCHEDULER_add_now (sched,
 523                                            &end_badly, "from create topology (bad return)");
 524     }
 525   die_task = GNUNET_SCHEDULER_add_delayed (sched,
 526                                            TEST_TIMEOUT,
 527                                            &end_badly, "from create topology (timeout)");
 528 }
 529
 530
 531 static void
 532 my_cb (void *cls,
 533        const struct GNUNET_PeerIdentity *id,
 534        const struct GNUNET_CONFIGURATION_Handle *cfg,
 535        struct GNUNET_TESTING_Daemon *d, const char *emsg)
 536 {
 537   GNUNET_assert (id != NULL);
 538 #if VERBOSE
 539   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
 540               (num_peers - peers_left) + 1, num_peers);
 541 #endif
 542   peers_left--;
 543   if (peers_left == 0)
 544     {
 545 #if VERBOSE
 546       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 547                   "All %d daemons started, now creating topology!\n",
 548                   num_peers);
 549 #endif
 550       GNUNET_SCHEDULER_cancel (sched, die_task);
 551       /* Set up task in case topology creation doesn't finish
 552        * within a reasonable amount of time */
 553       die_task = GNUNET_SCHEDULER_add_delayed (sched,
 554                                                GNUNET_TIME_relative_multiply
 555                                                (GNUNET_TIME_UNIT_MINUTES, 5),
 556                                                &end_badly, "from my_cb");
 557       create_topology ();
 558       ok = 0;
 559     }
 560 }
 561
 562
 563 static void
 564 run (void *cls,
 565      struct GNUNET_SCHEDULER_Handle *s,
 566      char *const *args,
 567      const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
 568 {
 569   sched = s;
 570   ok = 1;
 571
 572   dotOutFile = fopen (dotOutFileName, "w");
 573   if (dotOutFile != NULL)
 574     {
 575       fprintf (dotOutFile, "strict graph G {\n");
 576     }
 577
 578 #if VERBOSE
 579   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 580               "Starting daemons based on config file %s\n", cfgfile);
 581 #endif
 582   if (GNUNET_SYSERR ==
 583       GNUNET_CONFIGURATION_get_value_number (cfg, "testing", "num_peers",
 584                                              &num_peers))
 585     num_peers = DEFAULT_NUM_PEERS;
 586
 587   main_cfg = cfg;
 588
 589   peers_left = num_peers;
 590
 591   /* Set up a task to end testing if peer start fails */
 592   die_task = GNUNET_SCHEDULER_add_delayed (sched,
 593                                            GNUNET_TIME_relative_multiply
 594                                            (GNUNET_TIME_UNIT_MINUTES, 5),
 595                                            &end_badly, "didn't start all daemons in reasonable amount of time!!!");
 596
 597   pg = GNUNET_TESTING_daemons_start (sched, cfg,
 598                                      peers_left, &my_cb, NULL,
 599                                      &topology_callback, NULL, NULL);
 600
 601 }
 602
 603 static int
 604 check ()
 605 {
 606   char *binary_name;
 607   char *config_file_name;
 608   GNUNET_asprintf(&binary_name, "test-testing-topology-%s", topology_string);
 609   GNUNET_asprintf(&config_file_name, "test_testing_data_topology_%s.conf", topology_string);
 610   char *const argv[] = {binary_name,
 611     "-c",
 612     config_file_name,
 613 #if VERBOSE
 614     "-L", "DEBUG",
 615 #endif
 616     NULL
 617   };
 618   struct GNUNET_GETOPT_CommandLineOption options[] = {
 619     GNUNET_GETOPT_OPTION_END
 620   };
 621   GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
 622                       argv, binary_name, "nohelp",
 623                       options, &run, &ok);
 624   GNUNET_free(binary_name);
 625   GNUNET_free(config_file_name);
 626   return ok;
 627 }
 628
 629 int
 630 main (int argc, char *argv[])
 631 {
 632   int ret;
 633   char *binary_start_pos;
 634   char *our_binary_name;
 635
 636   binary_start_pos = rindex(argv[0], '/');
 637   topology_string = strstr (binary_start_pos,
 638                             "_topology");
 639   GNUNET_assert (topology_string != NULL);
 640   topology_string++;
 641   topology_string = strstr (topology_string, "_");
 642   GNUNET_assert (topology_string != NULL);
 643   topology_string++;
 644
 645   GNUNET_asprintf(&our_binary_name, "test-testing-topology_%s", topology_string);
 646   GNUNET_log_setup (our_binary_name,
 647 #if VERBOSE
 648                     "DEBUG",
 649 #else
 650                     "WARNING",
 651 #endif
 652                     NULL);
 653   ret = check ();
 654
 655   /**
 656    * Need to remove base directory, subdirectories taken care
 657    * of by the testing framework.
 658    */
 659   GNUNET_DISK_directory_remove ("/tmp/test-gnunet-testing");
 660   GNUNET_free(our_binary_name);
 661   return ret;
 662 }
 663
 664 /* end of test_testing_group.c */