src/tests/proctests.cc

   1 #include <cassert>
   2 #include <iostream>
   3 #include <list>
   4 #include <utility>
   5 #include <string>
   6 #include <sstream>
   7
   8 #include "service.h"
   9 #include "proc-service.h"
  10
  11 // Tests of process-service related functionality.
  12 //
  13 // These tests work mostly by completely mocking out the base_process_service class. The mock
  14 // implementations can be found in test-baseproc.cc.
  15
  16 extern eventloop_t event_loop;
  17
  18 constexpr static auto REG = dependency_type::REGULAR;
  19 constexpr static auto WAITS = dependency_type::WAITS_FOR;
  20
  21 // Friend interface to access base_process_service private/protected members.
  22 class base_process_service_test
  23 {
  24     public:
  25     static void exec_succeeded(base_process_service *bsp)
  26     {
  27         bsp->waiting_for_execstat = false;
  28         bsp->exec_succeeded();
  29     }
  30
  31     static void exec_failed(base_process_service *bsp, int errcode)
  32     {
  33         run_proc_err err;
  34         err.stage = exec_stage::DO_EXEC;
  35         err.st_errno = errcode;
  36         bsp->waiting_for_execstat = false;
  37         bsp->exec_failed(err);
  38     }
  39
  40     static void handle_exit(base_process_service *bsp, int exit_status)
  41     {
  42         bsp->pid = -1;
  43         bsp->handle_exit_status(bp_sys::exit_status(true, false, exit_status));
  44     }
  45
  46     static void handle_signal_exit(base_process_service *bsp, int signo)
  47     {
  48         bsp->pid = -1;
  49         bsp->handle_exit_status(bp_sys::exit_status(false, true, signo));
  50     }
  51
  52     static int get_notification_fd(base_process_service *bsp)
  53     {
  54         return bsp->notification_fd;
  55     }
  56 };
  57
  58 namespace bp_sys {
  59     // last signal sent:
  60     extern int last_sig_sent;
  61     extern pid_t last_forked_pid;
  62 }
  63
  64 static void init_service_defaults(base_process_service &ps)
  65 {
  66     ps.set_restart_interval(time_val(10,0), 3);
  67     ps.set_restart_delay(time_val(0, 200000000)); // 200 milliseconds
  68     ps.set_stop_timeout(time_val(10,0));
  69 }
  70
  71 // Regular service start
  72 void test_proc_service_start()
  73 {
  74     using namespace std;
  75
  76     service_set sset;
  77
  78     string command = "test-command";
  79     list<pair<unsigned,unsigned>> command_offsets;
  80     command_offsets.emplace_back(0, command.length());
  81     std::list<prelim_dep> depends;
  82
  83     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
  84     init_service_defaults(p);
  85     sset.add_service(&p);
  86
  87     p.start(true);
  88     sset.process_queues();
  89
  90     assert(p.get_state() == service_state_t::STARTING);
  91
  92     base_process_service_test::exec_succeeded(&p);
  93     sset.process_queues();
  94
  95     assert(p.get_state() == service_state_t::STARTED);
  96     assert(event_loop.active_timers.size() == 0);
  97
  98     sset.remove_service(&p);
  99 }
 100
 101 // Test start with readiness notification
 102 void test_proc_notify_start()
 103 {
 104     using namespace std;
 105
 106     service_set sset;
 107
 108     string command = "test-command";
 109     list<pair<unsigned,unsigned>> command_offsets;
 110     command_offsets.emplace_back(0, command.length());
 111     std::list<prelim_dep> depends;
 112
 113     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 114     init_service_defaults(p);
 115     p.set_notification_fd(3);
 116     sset.add_service(&p);
 117
 118     p.start(true);
 119     sset.process_queues();
 120
 121     assert(p.get_state() == service_state_t::STARTING);
 122
 123     base_process_service_test::exec_succeeded(&p);
 124     sset.process_queues();
 125
 126     assert(p.get_state() == service_state_t::STARTING);
 127
 128     int nfd = base_process_service_test::get_notification_fd(&p);
 129     assert(nfd > 0);
 130
 131     char notifystr[] = "ok started\n";
 132     std::vector<char> rnotifystr;
 133     rnotifystr.insert(rnotifystr.end(), notifystr, notifystr + sizeof(notifystr));
 134     bp_sys::supply_read_data(nfd, std::move(rnotifystr));
 135
 136     event_loop.regd_fd_watchers[nfd]->fd_event(event_loop, nfd, dasynq::IN_EVENTS);
 137
 138     assert(p.get_state() == service_state_t::STARTED);
 139     assert(event_loop.active_timers.size() == 0);
 140
 141     sset.remove_service(&p);
 142 }
 143
 144 // Unexpected termination
 145 void test_proc_unexpected_term()
 146 {
 147     using namespace std;
 148
 149     service_set sset;
 150
 151     string command = "test-command";
 152     list<pair<unsigned,unsigned>> command_offsets;
 153     command_offsets.emplace_back(0, command.length());
 154     std::list<prelim_dep> depends;
 155
 156     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 157     init_service_defaults(p);
 158     sset.add_service(&p);
 159
 160     p.start(true);
 161     sset.process_queues();
 162
 163     base_process_service_test::exec_succeeded(&p);
 164     sset.process_queues();
 165
 166     assert(p.get_state() == service_state_t::STARTED);
 167
 168     base_process_service_test::handle_exit(&p, 0);
 169
 170     assert(p.get_state() == service_state_t::STOPPED);
 171     assert(p.get_stop_reason() == stopped_reason_t::TERMINATED);
 172     assert(event_loop.active_timers.size() == 0);
 173
 174     sset.remove_service(&p);
 175 }
 176
 177 // Unexpected termination with restart
 178 void test_proc_term_restart()
 179 {
 180     using namespace std;
 181
 182     service_set sset;
 183
 184     string command = "test-command";
 185     list<pair<unsigned,unsigned>> command_offsets;
 186     command_offsets.emplace_back(0, command.length());
 187     std::list<prelim_dep> depends;
 188
 189     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 190     init_service_defaults(p);
 191     p.set_auto_restart(true);
 192     sset.add_service(&p);
 193
 194     p.start(true);
 195     sset.process_queues();
 196
 197     base_process_service_test::exec_succeeded(&p);
 198     sset.process_queues();
 199
 200     assert(p.get_state() == service_state_t::STARTED);
 201     assert(event_loop.active_timers.size() == 0);
 202
 203     base_process_service_test::handle_exit(&p, 0);
 204     sset.process_queues();
 205
 206     // Starting, restart timer should be armed:
 207     assert(p.get_state() == service_state_t::STARTING);
 208     assert(event_loop.active_timers.size() == 1);
 209
 210     event_loop.advance_time(time_val(0, 200000000));
 211     assert(event_loop.active_timers.size() == 0);
 212
 213     sset.process_queues();
 214     base_process_service_test::exec_succeeded(&p);
 215     sset.process_queues();
 216
 217     assert(p.get_state() == service_state_t::STARTED);
 218     assert(event_loop.active_timers.size() == 0);
 219
 220     sset.remove_service(&p);
 221 }
 222
 223 void test_proc_term_restart2()
 224 {
 225     using namespace std;
 226
 227     service_set sset;
 228
 229     string command = "test-command";
 230     list<pair<unsigned,unsigned>> command_offsets;
 231     command_offsets.emplace_back(0, command.length());
 232     std::list<prelim_dep> depends;
 233
 234     service_record b {&sset, "boot"};
 235     sset.add_service(&b);
 236
 237     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 238     init_service_defaults(p);
 239     p.set_auto_restart(true);
 240     sset.add_service(&p);
 241
 242     b.add_dep(&p, WAITS);
 243
 244     b.start(true);
 245     sset.process_queues();
 246
 247     assert(p.get_state() == service_state_t::STARTING);
 248
 249     base_process_service_test::exec_succeeded(&p);
 250     sset.process_queues();
 251
 252     assert(p.get_state() == service_state_t::STARTED);
 253     assert(event_loop.active_timers.size() == 0);
 254
 255     // simulate process terminating, should then be restarted:
 256     base_process_service_test::handle_exit(&p, 0);
 257     sset.process_queues();
 258
 259     // Starting, restart timer should be armed:
 260     assert(p.get_state() == service_state_t::STARTING);
 261     assert(event_loop.active_timers.size() == 1);
 262
 263     event_loop.advance_time(time_val(0, 200000000));
 264     assert(event_loop.active_timers.size() == 0);
 265
 266     sset.process_queues();
 267     base_process_service_test::exec_succeeded(&p);
 268     sset.process_queues();
 269
 270     assert(p.get_state() == service_state_t::STARTED);
 271     assert(event_loop.active_timers.size() == 0);
 272
 273     assert(sset.count_active_services() == 2);
 274
 275     // Request stop, this time it should not restart:
 276     p.stop(true);
 277     sset.process_queues();
 278     base_process_service_test::handle_exit(&p, 0);
 279     sset.process_queues();
 280
 281     assert(p.get_state() == service_state_t::STOPPED);
 282     assert(event_loop.active_timers.size() == 0);
 283     assert(sset.count_active_services() == 1);
 284
 285     // simulate terminate dinit
 286     sset.stop_all_services();
 287
 288     //base_process_service_test::handle_exit(&p, 0);
 289     sset.process_queues();
 290
 291     assert(p.get_state() == service_state_t::STOPPED);
 292     assert(event_loop.active_timers.size() == 0);
 293     assert(sset.count_active_services() == 0);
 294
 295     sset.remove_service(&p);
 296     sset.remove_service(&b);
 297 }
 298
 299 // Termination via stop request
 300 void test_term_via_stop()
 301 {
 302     using namespace std;
 303
 304     service_set sset;
 305
 306     string command = "test-command";
 307     list<pair<unsigned,unsigned>> command_offsets;
 308     command_offsets.emplace_back(0, command.length());
 309     std::list<prelim_dep> depends;
 310
 311     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 312     init_service_defaults(p);
 313     sset.add_service(&p);
 314
 315     p.start(true);
 316     sset.process_queues();
 317
 318     base_process_service_test::exec_succeeded(&p);
 319     sset.process_queues();
 320
 321     assert(p.get_state() == service_state_t::STARTED);
 322     assert(event_loop.active_timers.size() == 0);
 323
 324     p.stop(true);
 325     sset.process_queues();
 326
 327     assert(p.get_state() == service_state_t::STOPPING);
 328     assert(event_loop.active_timers.size() == 1);
 329
 330     base_process_service_test::handle_exit(&p, 0);
 331     sset.process_queues();
 332
 333     assert(p.get_state() == service_state_t::STOPPED);
 334     assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
 335     assert(event_loop.active_timers.size() == 0);
 336
 337     sset.remove_service(&p);
 338 }
 339
 340 // Termination via stop request, ensure reason is reset:
 341 void test_term_via_stop2()
 342 {
 343     using namespace std;
 344
 345     service_set sset;
 346
 347     string command = "test-command";
 348     list<pair<unsigned,unsigned>> command_offsets;
 349     command_offsets.emplace_back(0, command.length());
 350     std::list<prelim_dep> depends;
 351
 352     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 353     init_service_defaults(p);
 354     sset.add_service(&p);
 355
 356     p.start(true);
 357     sset.process_queues();
 358
 359     // first set it up with failure reason:
 360
 361     base_process_service_test::exec_failed(&p, ENOENT);
 362     sset.process_queues();
 363
 364     assert(p.get_state() == service_state_t::STOPPED);
 365     assert(p.get_stop_reason() == stopped_reason_t::EXECFAILED);
 366
 367     // now restart clean:
 368
 369     p.start(true);
 370     sset.process_queues();
 371
 372     base_process_service_test::exec_succeeded(&p);
 373     sset.process_queues();
 374
 375     assert(p.get_state() == service_state_t::STARTED);
 376     assert(event_loop.active_timers.size() == 0);
 377
 378     p.stop(true);
 379     sset.process_queues();
 380     assert(p.get_state() == service_state_t::STOPPING);
 381
 382     base_process_service_test::handle_exit(&p, 0);
 383     sset.process_queues();
 384     assert(p.get_state() == service_state_t::STOPPED);
 385     assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
 386     assert(event_loop.active_timers.size() == 0);
 387
 388     sset.remove_service(&p);
 389 }
 390
 391 // Time-out during start
 392 void test_proc_start_timeout()
 393 {
 394     using namespace std;
 395
 396     service_set sset;
 397
 398     string command = "test-command";
 399     list<pair<unsigned,unsigned>> command_offsets;
 400     command_offsets.emplace_back(0, command.length());
 401     std::list<prelim_dep> depends;
 402
 403     scripted_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 404     init_service_defaults(p);
 405     p.set_start_timeout(time_val(10,0));
 406     sset.add_service(&p);
 407
 408     p.start(true);
 409     sset.process_queues();
 410
 411     assert(p.get_state() == service_state_t::STARTING);
 412
 413     event_loop.advance_time(time_val(10,0));
 414     sset.process_queues();
 415
 416     assert(p.get_state() == service_state_t::STOPPING);
 417
 418     base_process_service_test::handle_signal_exit(&p, SIGTERM);
 419     sset.process_queues();
 420
 421     // We set no stop script, so state should now be STOPPED with no timer set
 422     assert(p.get_state() == service_state_t::STOPPED);
 423     assert(p.get_stop_reason() == stopped_reason_t::TIMEDOUT);
 424     assert(event_loop.active_timers.size() == 0);
 425
 426     sset.remove_service(&p);
 427 }
 428
 429 // Test that a timeout doesn't stop a "waits for" dependent to fail to start
 430 void test_proc_start_timeout2()
 431 {
 432     using namespace std;
 433
 434     service_set sset;
 435
 436     string command = "test-command";
 437     list<pair<unsigned,unsigned>> command_offsets;
 438     command_offsets.emplace_back(0, command.length());
 439     std::list<prelim_dep> depends;
 440
 441     scripted_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 442     p.set_start_timeout(time_val {1,0});
 443     init_service_defaults(p);
 444     sset.add_service(&p);
 445
 446     service_record ts {&sset, "test-service-1", service_type_t::INTERNAL,
 447         {{&p, dependency_type::WAITS_FOR}} };
 448
 449     ts.start(true);
 450     sset.process_queues();
 451
 452     assert(p.get_state() == service_state_t::STARTING);
 453     assert(ts.get_state() == service_state_t::STARTING);
 454
 455     event_loop.advance_time(time_val {1,0}); // start timer should expire
 456     sset.process_queues();
 457
 458     assert(p.get_state() == service_state_t::STOPPING);
 459
 460     base_process_service_test::handle_exit(&p, 0);
 461     sset.process_queues();
 462
 463     assert(p.get_state() == service_state_t::STOPPED);
 464     assert(p.get_stop_reason() == stopped_reason_t::TIMEDOUT);
 465     assert(ts.get_state() == service_state_t::STARTED);
 466     assert(event_loop.active_timers.size() == 0);
 467
 468     sset.remove_service(&p);
 469 }
 470
 471 // Test exec() failure for process service start.
 472 void test_proc_start_execfail()
 473 {
 474     using namespace std;
 475
 476     service_set sset;
 477
 478     string command = "test-command";
 479     list<pair<unsigned,unsigned>> command_offsets;
 480     command_offsets.emplace_back(0, command.length());
 481     std::list<prelim_dep> depends;
 482
 483     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 484     init_service_defaults(p);
 485     sset.add_service(&p);
 486
 487     p.start(true);
 488     sset.process_queues();
 489
 490     assert(p.get_state() == service_state_t::STARTING);
 491
 492     base_process_service_test::exec_failed(&p, ENOENT);
 493     sset.process_queues();
 494
 495     assert(p.get_state() == service_state_t::STOPPED);
 496     assert(p.get_stop_reason() == stopped_reason_t::EXECFAILED);
 497     assert(event_loop.active_timers.size() == 0);
 498
 499     sset.remove_service(&p);
 500 }
 501
 502 // Test no ready notification before process terminates
 503 void test_proc_notify_fail()
 504 {
 505     using namespace std;
 506
 507     service_set sset;
 508
 509     string command = "test-command";
 510     list<pair<unsigned,unsigned>> command_offsets;
 511     command_offsets.emplace_back(0, command.length());
 512     std::list<prelim_dep> depends;
 513
 514     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 515     init_service_defaults(p);
 516     p.set_notification_fd(3);
 517     sset.add_service(&p);
 518
 519     p.start(true);
 520     sset.process_queues();
 521
 522     assert(p.get_state() == service_state_t::STARTING);
 523
 524     base_process_service_test::exec_succeeded(&p);
 525     sset.process_queues();
 526
 527     assert(p.get_state() == service_state_t::STARTING);
 528
 529     int nfd = base_process_service_test::get_notification_fd(&p);
 530     assert(nfd > 0);
 531
 532     // Signal EOF on notify fd:
 533     event_loop.regd_fd_watchers[nfd]->fd_event(event_loop, nfd, dasynq::IN_EVENTS);
 534
 535     assert(p.get_state() == service_state_t::STOPPING);
 536
 537     base_process_service_test::handle_exit(&p, 0);
 538     sset.process_queues();
 539
 540     assert(p.get_state() == service_state_t::STOPPED);
 541     assert(event_loop.active_timers.size() == 0);
 542
 543     sset.remove_service(&p);
 544 }
 545
 546 // Test stop timeout
 547 void test_proc_stop_timeout()
 548 {
 549     using namespace std;
 550
 551     service_set sset;
 552
 553     string command = "test-command";
 554     list<pair<unsigned,unsigned>> command_offsets;
 555     command_offsets.emplace_back(0, command.length());
 556     std::list<prelim_dep> depends;
 557
 558     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 559     init_service_defaults(p);
 560     p.set_stop_timeout(time_val {10, 0});
 561     sset.add_service(&p);
 562
 563     p.start(true);
 564     sset.process_queues();
 565
 566     assert(p.get_state() == service_state_t::STARTING);
 567
 568     base_process_service_test::exec_succeeded(&p);
 569     sset.process_queues();
 570
 571     assert(p.get_state() == service_state_t::STARTED);
 572
 573     p.stop(true);
 574     sset.process_queues();
 575
 576     assert(p.get_state() == service_state_t::STOPPING);
 577     assert(bp_sys::last_sig_sent == SIGTERM);
 578
 579     event_loop.advance_time(time_val {10, 0}); // expire stop timer
 580     sset.process_queues();
 581
 582     // kill signal (SIGKILL) should have been sent; process not dead until it's dead, however
 583     assert(p.get_state() == service_state_t::STOPPING);
 584     assert(bp_sys::last_sig_sent == SIGKILL);
 585
 586     base_process_service_test::handle_exit(&p, 0);
 587     sset.process_queues();
 588
 589     assert(p.get_state() == service_state_t::STOPPED);
 590     assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
 591
 592     assert(event_loop.active_timers.size() == 0);
 593
 594     sset.remove_service(&p);
 595 }
 596
 597 // Smooth recovery
 598 void test_proc_smooth_recovery1()
 599 {
 600     using namespace std;
 601
 602     service_set sset;
 603
 604     string command = "test-command";
 605     list<pair<unsigned,unsigned>> command_offsets;
 606     command_offsets.emplace_back(0, command.length());
 607     std::list<prelim_dep> depends;
 608
 609     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 610     init_service_defaults(p);
 611     p.set_smooth_recovery(true);
 612     p.set_restart_delay(time_val {0, 1000});
 613     sset.add_service(&p);
 614
 615     p.start(true);
 616     sset.process_queues();
 617
 618     base_process_service_test::exec_succeeded(&p);
 619     sset.process_queues();
 620
 621     pid_t first_instance = bp_sys::last_forked_pid;
 622
 623     assert(p.get_state() == service_state_t::STARTED);
 624
 625     base_process_service_test::handle_exit(&p, 0);
 626     sset.process_queues();
 627
 628     // since time hasn't been changed, we expect that the process has not yet been re-launched:
 629     assert(first_instance == bp_sys::last_forked_pid);
 630     assert(p.get_state() == service_state_t::STARTED);
 631
 632     event_loop.advance_time(time_val {0, 1000});
 633     sset.process_queues();
 634
 635     // Now a new process should've been launched:
 636     assert(first_instance + 1 == bp_sys::last_forked_pid);
 637     assert(p.get_state() == service_state_t::STARTED);
 638
 639     assert(event_loop.active_timers.size() == 0);
 640
 641     sset.remove_service(&p);
 642 }
 643
 644 // Smooth recovery without restart delay
 645 void test_proc_smooth_recovery2()
 646 {
 647     using namespace std;
 648
 649     service_set sset;
 650
 651     string command = "test-command";
 652     list<pair<unsigned,unsigned>> command_offsets;
 653     command_offsets.emplace_back(0, command.length());
 654     std::list<prelim_dep> depends;
 655
 656     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 657     init_service_defaults(p);
 658     p.set_smooth_recovery(true);
 659     p.set_restart_delay(time_val(0, 0));
 660     sset.add_service(&p);
 661
 662     p.start(true);
 663     sset.process_queues();
 664
 665     base_process_service_test::exec_succeeded(&p);
 666     sset.process_queues();
 667
 668     pid_t first_instance = bp_sys::last_forked_pid;
 669
 670     assert(p.get_state() == service_state_t::STARTED);
 671     assert(event_loop.active_timers.size() == 0);
 672
 673     base_process_service_test::handle_exit(&p, 0);
 674     sset.process_queues();
 675
 676     // no restart delay, process should restart immediately:
 677     assert(first_instance + 1 == bp_sys::last_forked_pid);
 678     assert(p.get_state() == service_state_t::STARTED);
 679     assert(event_loop.active_timers.size() == 0);
 680
 681     sset.remove_service(&p);
 682 }
 683
 684 void test_bgproc_smooth_recover()
 685 {
 686     using namespace std;
 687
 688     service_set sset;
 689
 690     string command = "test-command";
 691     list<pair<unsigned,unsigned>> command_offsets;
 692     command_offsets.emplace_back(0, command.length());
 693     std::list<prelim_dep> depends;
 694
 695     bgproc_service p {&sset, "testproc", std::move(command), command_offsets, depends};
 696     init_service_defaults(p);
 697     p.set_smooth_recovery(true);
 698     p.set_restart_delay(time_val {0, 1000});
 699     p.set_pid_file("/run/daemon.pid");
 700     sset.add_service(&p);
 701
 702     p.start(true);
 703     sset.process_queues();
 704
 705     base_process_service_test::exec_succeeded(&p);
 706     sset.process_queues();
 707
 708     // pid_t first_instance = bp_sys::last_forked_pid;
 709     pid_t daemon_instance = ++bp_sys::last_forked_pid;
 710
 711     // Set up the pid file content with the pid of the daemon
 712     stringstream str;
 713     str << daemon_instance << std::flush;
 714     string pid_file_content = str.str();
 715     vector<char> pid_file_content_v(pid_file_content.begin(), pid_file_content.end());
 716     bp_sys::supply_file_content("/run/daemon.pid", std::move(pid_file_content_v));
 717
 718     assert(p.get_state() == service_state_t::STARTING);
 719
 720     base_process_service_test::handle_exit(&p, 0); // exit the launch process
 721     sset.process_queues();
 722
 723     // daemon process has been started now, state should be STARTED
 724     assert(p.get_state() == service_state_t::STARTED);
 725     assert(daemon_instance == bp_sys::last_forked_pid);
 726
 727     base_process_service_test::handle_exit(&p, 0); // exit the daemon process
 728
 729     // since time hasn't been changed, we expect that the process has not yet been re-launched:
 730     assert(p.get_state() == service_state_t::STARTED);
 731     assert(daemon_instance == bp_sys::last_forked_pid);
 732
 733     event_loop.advance_time(time_val {0, 1000});
 734     sset.process_queues();
 735
 736     // Now a new process should've been launched:
 737     assert(event_loop.active_timers.size() == 0);
 738     assert(daemon_instance + 1 == bp_sys::last_forked_pid);
 739     assert(p.get_state() == service_state_t::STARTED);
 740     assert(event_loop.active_timers.size() == 0);
 741
 742     base_process_service_test::exec_succeeded(&p);
 743     sset.process_queues();
 744
 745     assert(event_loop.active_timers.size() == 0);
 746
 747     daemon_instance = ++bp_sys::last_forked_pid;
 748     str.str("");
 749     str << daemon_instance << std::flush;
 750     pid_file_content = str.str();
 751     pid_file_content_v = std::vector<char>(pid_file_content.begin(), pid_file_content.end());
 752     bp_sys::supply_file_content("/run/daemon.pid", std::move(pid_file_content_v));
 753
 754     base_process_service_test::handle_exit(&p, 0);
 755     sset.process_queues();
 756
 757     assert(p.get_state() == service_state_t::STARTED);
 758
 759     assert(event_loop.active_timers.size() == 0);
 760
 761     // Now run through it again
 762
 763     base_process_service_test::handle_exit(&p, 0); // exit the daemon process
 764
 765     // since time hasn't been changed, we expect that the process has not yet been re-launched:
 766     assert(p.get_state() == service_state_t::STARTED);
 767     assert(daemon_instance == bp_sys::last_forked_pid);
 768
 769     event_loop.advance_time(time_val {0, 1000});
 770     sset.process_queues();
 771
 772     // Now a new process should've been launched:
 773     assert(event_loop.active_timers.size() == 0);
 774     assert(daemon_instance + 1 == bp_sys::last_forked_pid);
 775     assert(p.get_state() == service_state_t::STARTED);
 776     assert(event_loop.active_timers.size() == 0);
 777
 778     base_process_service_test::exec_succeeded(&p);
 779     sset.process_queues();
 780
 781     assert(event_loop.active_timers.size() == 0);
 782
 783     daemon_instance = ++bp_sys::last_forked_pid;
 784     str.str("");
 785     str << daemon_instance << std::flush;
 786     pid_file_content = str.str();
 787     pid_file_content_v = std::vector<char>(pid_file_content.begin(), pid_file_content.end());
 788     bp_sys::supply_file_content("/run/daemon.pid", std::move(pid_file_content_v));
 789
 790     base_process_service_test::handle_exit(&p, 0);
 791     sset.process_queues();
 792
 793     assert(p.get_state() == service_state_t::STARTED);
 794
 795     assert(event_loop.active_timers.size() == 0);
 796
 797     sset.remove_service(&p);
 798 }
 799
 800 // Test stop timeout
 801 void test_scripted_stop_timeout()
 802 {
 803     using namespace std;
 804
 805     service_set sset;
 806
 807     string command = "test-command";
 808     string stopcommand = "stop-command";
 809     list<pair<unsigned,unsigned>> command_offsets;
 810     command_offsets.emplace_back(0, command.length());
 811     std::list<prelim_dep> depends;
 812
 813     scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
 814     init_service_defaults(p);
 815     p.set_stop_command(stopcommand, command_offsets);
 816     p.set_stop_timeout(time_val {10, 0});
 817     sset.add_service(&p);
 818
 819     p.start(true);
 820     sset.process_queues();
 821
 822     assert(p.get_state() == service_state_t::STARTING);
 823
 824     base_process_service_test::exec_succeeded(&p);
 825     sset.process_queues();
 826     base_process_service_test::handle_exit(&p, 0);
 827     sset.process_queues();
 828
 829     assert(p.get_state() == service_state_t::STARTED);
 830
 831     p.stop(true);
 832     sset.process_queues();
 833
 834     assert(p.get_state() == service_state_t::STOPPING);
 835
 836     base_process_service_test::exec_succeeded(&p);
 837     sset.process_queues();
 838
 839     // should still be stopping:
 840     assert(p.get_state() == service_state_t::STOPPING);
 841
 842     event_loop.advance_time(time_val {10, 0}); // expire stop timer
 843     sset.process_queues();
 844
 845     // kill signal (SIGKILL) should have been sent; process not dead until it's dead, however
 846     assert(p.get_state() == service_state_t::STOPPING);
 847     assert(bp_sys::last_sig_sent == SIGKILL);
 848
 849     base_process_service_test::handle_exit(&p, SIGKILL);
 850     sset.process_queues();
 851
 852     assert(p.get_state() == service_state_t::STOPPED);
 853     assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
 854
 855     assert(event_loop.active_timers.size() == 0);
 856
 857     sset.remove_service(&p);
 858 }
 859
 860 void test_scripted_start_fail()
 861 {
 862     using namespace std;
 863
 864     service_set sset;
 865
 866     string command = "test-command";
 867     string stopcommand = "stop-command";
 868     list<pair<unsigned,unsigned>> command_offsets;
 869     command_offsets.emplace_back(0, command.length());
 870     std::list<prelim_dep> depends;
 871
 872     scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
 873     init_service_defaults(p);
 874     p.set_stop_command(stopcommand, command_offsets);
 875     sset.add_service(&p);
 876
 877     service_record *s2 = new service_record(&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}});
 878     service_record *s3 = new service_record(&sset, "test-service-3",
 879             service_type_t::INTERNAL, {{&p, REG}, {s2, REG}});
 880     sset.add_service(s2);
 881     sset.add_service(s3);
 882
 883     s3->start(true);
 884     sset.process_queues();
 885
 886     assert(p.get_state() == service_state_t::STARTING);
 887
 888     base_process_service_test::exec_succeeded(&p);
 889     sset.process_queues();
 890     base_process_service_test::handle_exit(&p, 0x1);  // exit fail
 891     sset.process_queues();
 892
 893     // failed to start:
 894     assert(p.get_state() == service_state_t::STOPPED);
 895     assert(s2->get_state() == service_state_t::STOPPED);
 896     assert(s3->get_state() == service_state_t::STOPPED);
 897     assert(p.get_stop_reason() == stopped_reason_t::FAILED);
 898     assert(s2->get_stop_reason() == stopped_reason_t::DEPFAILED);
 899     assert(s3->get_stop_reason() == stopped_reason_t::DEPFAILED);
 900
 901     event_loop.active_timers.clear();
 902     sset.remove_service(&p);
 903
 904     assert(sset.count_active_services() == 0);
 905 }
 906
 907 void test_scripted_stop_fail()
 908 {
 909     using namespace std;
 910
 911     service_set sset;
 912
 913     string command = "test-command";
 914     string stopcommand = "stop-command";
 915     list<pair<unsigned,unsigned>> command_offsets;
 916     command_offsets.emplace_back(0, command.length());
 917     std::list<prelim_dep> depends;
 918
 919     scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
 920     init_service_defaults(p);
 921     p.set_stop_command(stopcommand, command_offsets);
 922     sset.add_service(&p);
 923
 924     service_record *s2 = new service_record(&sset, "test-service-2", service_type_t::INTERNAL, {});
 925     service_record *s3 = new service_record(&sset, "test-service-3", service_type_t::INTERNAL,
 926             {{s2, REG}, {&p, REG}});
 927     service_record *s4 = new service_record(&sset, "test-service-4", service_type_t::INTERNAL,
 928             {{&p, REG}, {s3, REG}});
 929     sset.add_service(s2);
 930     sset.add_service(s3);
 931     sset.add_service(s4);
 932
 933     s4->start(true);
 934     sset.process_queues();
 935
 936     base_process_service_test::exec_succeeded(&p);
 937     sset.process_queues();
 938     base_process_service_test::handle_exit(&p, 0x0);  // success
 939     sset.process_queues();
 940
 941     assert(p.get_state() == service_state_t::STARTED);
 942     assert(s2->get_state() == service_state_t::STARTED);
 943     assert(s3->get_state() == service_state_t::STARTED);
 944     assert(s4->get_state() == service_state_t::STARTED);
 945
 946     pid_t last_forked = bp_sys::last_forked_pid;
 947
 948     s4->stop(true);
 949     sset.process_queues();
 950
 951     base_process_service_test::exec_succeeded(&p);
 952     sset.process_queues();
 953     base_process_service_test::handle_exit(&p, 0x1);  // failure
 954     sset.process_queues();
 955
 956     // The stop command should be executed once:
 957     assert((bp_sys::last_forked_pid - last_forked) == 1);
 958
 959     assert(p.get_state() == service_state_t::STOPPED);
 960     assert(s2->get_state() == service_state_t::STOPPED);
 961     assert(s3->get_state() == service_state_t::STOPPED);
 962     assert(s4->get_state() == service_state_t::STOPPED);
 963
 964     event_loop.active_timers.clear();
 965     sset.remove_service(&p);
 966 }
 967
 968 void test_scripted_start_skip()
 969 {
 970     using namespace std;
 971
 972     service_set sset;
 973
 974     string command = "test-command";
 975     list<pair<unsigned,unsigned>> command_offsets;
 976     command_offsets.emplace_back(0, command.length());
 977     std::list<prelim_dep> depends;
 978
 979     scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
 980     init_service_defaults(p);
 981     service_flags_t sflags;
 982     sflags.skippable = true;
 983     p.set_flags(sflags);
 984     sset.add_service(&p);
 985
 986     service_record *s2 = new service_record(&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}});
 987     sset.add_service(s2);
 988
 989     s2->start(true);
 990     sset.process_queues();
 991     assert(p.get_state() == service_state_t::STARTING);
 992
 993     base_process_service_test::exec_succeeded(&p);
 994     sset.process_queues();
 995     assert(p.get_state() == service_state_t::STARTING);
 996
 997     base_process_service_test::handle_signal_exit(&p, SIGINT); // interrupted
 998     sset.process_queues();
 999
1000     assert(p.get_state() == service_state_t::STARTED);
1001     assert(s2->get_state() == service_state_t::STARTED);
1002     assert(p.was_start_skipped());
1003     assert(! s2->was_start_skipped());
1004     assert(sset.count_active_services() == 2);
1005
1006     s2->stop(true);
1007     sset.process_queues();
1008
1009     assert(p.get_state() == service_state_t::STOPPED);
1010     assert(s2->get_state() == service_state_t::STOPPED);
1011     assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
1012     assert(s2->get_stop_reason() == stopped_reason_t::NORMAL);
1013     assert(sset.count_active_services() == 0);
1014
1015     event_loop.active_timers.clear();
1016     sset.remove_service(&p);
1017 }
1018
1019 // Test interrupting start of a service marked skippable
1020 void test_scripted_start_skip2()
1021 {
1022     using namespace std;
1023
1024     service_set sset;
1025
1026     string command = "test-command";
1027     list<pair<unsigned,unsigned>> command_offsets;
1028     command_offsets.emplace_back(0, command.length());
1029     std::list<prelim_dep> depends;
1030
1031     scripted_service p {&sset, "testscripted", std::move(command), command_offsets, depends};
1032     init_service_defaults(p);
1033     service_flags_t sflags;
1034     sflags.skippable = true;
1035     sflags.start_interruptible = true;
1036     p.set_flags(sflags);
1037     sset.add_service(&p);
1038
1039     service_record *s2 = new service_record(&sset, "test-service-2", service_type_t::INTERNAL, {{&p, REG}});
1040     sset.add_service(s2);
1041
1042     s2->start(true);
1043     sset.process_queues();
1044     assert(p.get_state() == service_state_t::STARTING);
1045
1046     base_process_service_test::exec_succeeded(&p);
1047     sset.process_queues();
1048     assert(p.get_state() == service_state_t::STARTING);
1049
1050     s2->stop(true);  // abort startup; p should be cancelled
1051     sset.process_queues();
1052
1053     assert(p.get_state() == service_state_t::STOPPING);
1054
1055     base_process_service_test::handle_signal_exit(&p, SIGINT); // interrupted
1056     sset.process_queues();
1057
1058     assert(p.get_state() == service_state_t::STOPPED);
1059     assert(s2->get_state() == service_state_t::STOPPED);
1060     assert(p.get_stop_reason() == stopped_reason_t::NORMAL);
1061     assert(s2->get_stop_reason() == stopped_reason_t::NORMAL);
1062     assert(sset.count_active_services() == 0);
1063
1064     event_loop.active_timers.clear();
1065     sset.remove_service(&p);
1066 }
1067
1068 // Test that starting a service with a waits-for dependency on another - currently stopping - service,
1069 // causes that service to re-start.
1070 void test_waitsfor_restart()
1071 {
1072     using namespace std;
1073
1074     service_set sset;
1075
1076     string command = "test-command";
1077     list<pair<unsigned,unsigned>> command_offsets;
1078     command_offsets.emplace_back(0, command.length());
1079     std::list<prelim_dep> depends;
1080
1081     process_service p {&sset, "testproc", std::move(command), command_offsets, depends};
1082     init_service_defaults(p);
1083     sset.add_service(&p);
1084
1085     service_record tp {&sset, "test-service", service_type_t::INTERNAL, {{&p, WAITS}}};
1086     sset.add_service(&tp);
1087
1088     // start p:
1089
1090     p.start(true);
1091     sset.process_queues();
1092
1093     assert(p.get_state() == service_state_t::STARTING);
1094
1095     base_process_service_test::exec_succeeded(&p);
1096     sset.process_queues();
1097
1098     assert(p.get_state() == service_state_t::STARTED);
1099     assert(event_loop.active_timers.size() == 0);
1100
1101     // begin stopping p:
1102
1103     p.stop(true);
1104     sset.process_queues();
1105
1106     assert(p.get_state() == service_state_t::STOPPING);
1107
1108     // start tp (which waits-for p):
1109
1110     tp.start(true);
1111     sset.process_queues();
1112
1113     assert(tp.get_state() == service_state_t::STARTING);
1114     assert(p.get_state() == service_state_t::STOPPING);
1115
1116     // p terminates (finishes stopping). Then it should re-start...
1117     base_process_service_test::handle_signal_exit(&p, SIGTERM);
1118     sset.process_queues();
1119
1120     assert(tp.get_state() == service_state_t::STARTING);
1121     assert(p.get_state() == service_state_t::STARTING);
1122
1123     base_process_service_test::exec_succeeded(&p);
1124     sset.process_queues();
1125
1126     assert(tp.get_state() == service_state_t::STARTED);
1127     assert(p.get_state() == service_state_t::STARTED);
1128
1129     sset.remove_service(&tp);
1130     sset.remove_service(&p);
1131 }
1132
1133
1134 #define RUN_TEST(name, spacing) \
1135     std::cout << #name "..." spacing << std::flush; \
1136     name(); \
1137     std::cout << "PASSED" << std::endl;
1138
1139 int main(int argc, char **argv)
1140 {
1141     RUN_TEST(test_proc_service_start, "   ");
1142     RUN_TEST(test_proc_notify_start, "    ");
1143     RUN_TEST(test_proc_unexpected_term, " ");
1144     RUN_TEST(test_proc_term_restart, "    ");
1145     RUN_TEST(test_proc_term_restart2, "   ");
1146     RUN_TEST(test_term_via_stop, "        ");
1147     RUN_TEST(test_term_via_stop2, "       ");
1148     RUN_TEST(test_proc_start_timeout, "   ");
1149     RUN_TEST(test_proc_start_timeout2, "  ");
1150     RUN_TEST(test_proc_start_execfail, "  ");
1151     RUN_TEST(test_proc_notify_fail, "     ");
1152     RUN_TEST(test_proc_stop_timeout, "    ");
1153     RUN_TEST(test_proc_smooth_recovery1, "");
1154     RUN_TEST(test_proc_smooth_recovery2, "");
1155     RUN_TEST(test_bgproc_smooth_recover, "");
1156     RUN_TEST(test_scripted_stop_timeout, "");
1157     RUN_TEST(test_scripted_start_fail, "  ");
1158     RUN_TEST(test_scripted_stop_fail, "   ");
1159     RUN_TEST(test_scripted_start_skip, "  ");
1160     RUN_TEST(test_scripted_start_skip2, " ");
1161     RUN_TEST(test_waitsfor_restart, "     ");
1162 }