From c5f7a99645aa1718e226e6d867efcd5cabcbd610 Mon Sep 17 00:00:00 2001 From: Pauli Date: Mon, 15 Apr 2019 09:53:53 +1000 Subject: [PATCH] Test skip option. Provide C test cases with the option to skip tests and subtests. Reviewed-by: Matt Caswell (Merged from https://github.com/openssl/openssl/pull/8695) --- test/test_test.c | 22 ++++++++++++++++ test/testutil.h | 5 ++++ test/testutil/driver.c | 55 ++++++++++++++++------------------------ test/testutil/tests.c | 23 +++++++++++++++++ test/testutil/tu_local.h | 2 ++ 5 files changed, 74 insertions(+), 33 deletions(-) diff --git a/test/test_test.c b/test/test_test.c index 3741e17122..ba66a158ad 100644 --- a/test/test_test.c +++ b/test/test_test.c @@ -531,6 +531,25 @@ static int test_bn_output(int n) return 1; } +static int test_skip_one(void) +{ + return TEST_skip("skip test"); +} + +static int test_skip_many(int n) +{ + return TEST_skip("skip tests: %d", n); +} + +static int test_skip_null(void) +{ + /* + * This is not a recommended way of skipping a test, a reason or + * description should be included. + */ + return TEST_skip(NULL); +} + int setup_tests(void) { ADD_TEST(test_int); @@ -553,5 +572,8 @@ int setup_tests(void) ADD_TEST(test_single_eval); ADD_TEST(test_output); ADD_ALL_TESTS(test_bn_output, OSSL_NELEM(bn_output_tests)); + ADD_TEST(test_skip_one); + ADD_TEST(test_skip_null); + ADD_ALL_TESTS(test_skip_many, 3); return 1; } diff --git a/test/testutil.h b/test/testutil.h index 7f5db601cd..3a5c4866da 100644 --- a/test/testutil.h +++ b/test/testutil.h @@ -346,6 +346,9 @@ void test_info(const char *file, int line, const char *desc, ...) PRINTF_FORMAT(3, 4); void test_info_c90(const char *desc, ...) PRINTF_FORMAT(1, 2); void test_note(const char *desc, ...) PRINTF_FORMAT(1, 2); +int test_skip(const char *file, int line, const char *desc, ...) + PRINTF_FORMAT(3, 4); +int test_skip_c90(const char *desc, ...) PRINTF_FORMAT(1, 2); void test_openssl_errors(void); void test_perror(const char *s); @@ -463,9 +466,11 @@ void test_perror(const char *s); # if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L # define TEST_error test_error_c90 # define TEST_info test_info_c90 +# define TEST_skip test_skip_c90 # else # define TEST_error(...) test_error(__FILE__, __LINE__, __VA_ARGS__) # define TEST_info(...) test_info(__FILE__, __LINE__, __VA_ARGS__) +# define TEST_skip(...) test_skip(__FILE__, __LINE__, __VA_ARGS__) # endif # define TEST_note test_note # define TEST_openssl_errors test_openssl_errors diff --git a/test/testutil/driver.c b/test/testutil/driver.c index 10d74e2ea4..2da83ed727 100644 --- a/test/testutil/driver.c +++ b/test/testutil/driver.c @@ -280,20 +280,20 @@ void set_test_title(const char *title) test_title = title == NULL ? NULL : strdup(title); } -PRINTF_FORMAT(2, 3) static void test_verdict(int pass, const char *extra, ...) +PRINTF_FORMAT(2, 3) static void test_verdict(int verdict, + const char *description, ...) { va_list ap; test_flush_stdout(); test_flush_stderr(); - test_printf_stdout("%*s%s", level, "", pass ? "ok" : "not ok"); - if (extra != NULL) { - test_printf_stdout(" "); - va_start(ap, extra); - test_vprintf_stdout(extra, ap); - va_end(ap); - } + test_printf_stdout("%*s%s ", level, "", verdict != 0 ? "ok" : "not ok"); + va_start(ap, description); + test_vprintf_stdout(description, ap); + va_end(ap); + if (verdict == TEST_SKIP_CODE) + test_printf_stdout(" # skipped"); test_printf_stdout("\n"); test_flush_stdout(); } @@ -349,20 +349,14 @@ int run_tests(const char *test_prog_name) } test_flush_stdout(); } else if (all_tests[i].num == -1) { - int ret = 0; - set_test_title(all_tests[i].test_case_name); - ret = all_tests[i].test_fn(); - verdict = 1; - if (!ret) { - verdict = 0; - ++num_failed; - } + verdict = all_tests[i].test_fn(); test_verdict(verdict, "%d - %s", ii + 1, test_title); - finalize(ret); + finalize(verdict != 0); } else { int num_failed_inner = 0; + verdict = TEST_SKIP_CODE; level += 4; if (all_tests[i].subtest && single_iter == -1) { test_printf_stdout("%*s# Subtest: %s\n", level, "", @@ -381,39 +375,34 @@ int run_tests(const char *test_prog_name) while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1); for (jj = 0; jj < all_tests[i].num; jj++) { - int ret; + int v; j = (j + jstep) % all_tests[i].num; if (single_iter != -1 && ((jj + 1) != single_iter)) continue; set_test_title(NULL); - ret = all_tests[i].param_test_fn(j); + v = all_tests[i].param_test_fn(j); - if (!ret) + if (v == 0) { ++num_failed_inner; + verdict = 0; + } else if (v != TEST_SKIP_CODE && verdict != 0) { + verdict = 1; + } - finalize(ret); + finalize(v != 0); if (all_tests[i].subtest) { - verdict = 1; - if (!ret) { - verdict = 0; - ++num_failed_inner; - } if (test_title != NULL) - test_verdict(verdict, "%d - %s", jj + 1, test_title); + test_verdict(v, "%d - %s", jj + 1, test_title); else - test_verdict(verdict, "%d - iteration %d", - jj + 1, j + 1); + test_verdict(v, "%d - iteration %d", jj + 1, j + 1); } } level -= 4; - verdict = 1; - if (num_failed_inner) { - verdict = 0; + if (verdict == 0) ++num_failed; - } test_verdict(verdict, "%d - %s", ii + 1, all_tests[i].test_case_name); } diff --git a/test/testutil/tests.c b/test/testutil/tests.c index 05b0688880..56177cd999 100644 --- a/test/testutil/tests.c +++ b/test/testutil/tests.c @@ -157,6 +157,29 @@ void test_note(const char *fmt, ...) test_flush_stderr(); } + +int test_skip(const char *file, int line, const char *desc, ...) +{ + va_list ap; + + va_start(ap, desc); + test_fail_message_va("SKIP", file, line, NULL, NULL, NULL, NULL, desc, ap); + va_end(ap); + return TEST_SKIP_CODE; +} + +int test_skip_c90(const char *desc, ...) +{ + va_list ap; + + va_start(ap, desc); + test_fail_message_va("SKIP", NULL, -1, NULL, NULL, NULL, NULL, desc, ap); + va_end(ap); + test_printf_stderr("\n"); + return TEST_SKIP_CODE; +} + + void test_openssl_errors(void) { ERR_print_errors_cb(openssl_error_cb, NULL); diff --git a/test/testutil/tu_local.h b/test/testutil/tu_local.h index 049d7b1bc2..ede4546948 100644 --- a/test/testutil/tu_local.h +++ b/test/testutil/tu_local.h @@ -12,6 +12,8 @@ #include #include "../testutil.h" +#define TEST_SKIP_CODE 123 + int subtest_level(void); int openssl_error_cb(const char *str, size_t len, void *u); const BIO_METHOD *BIO_f_tap(void); -- 2.25.1