From: Richard Levitte Date: Tue, 18 Apr 2017 14:27:27 +0000 (+0200) Subject: Refactor the test framework testutil X-Git-Tag: OpenSSL_1_1_1-pre1~1709 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=4db40c94c38ee94fbef820a9816ac2d68f65506e;p=oweals%2Fopenssl.git Refactor the test framework testutil It's now built as a static library, and greatly simplified for test programs, which no longer need to include test_main_custom.h or test_main.h and link with the corresponding object files. Reviewed-by: Rich Salz (Merged from https://github.com/openssl/openssl/pull/3243) --- diff --git a/test/README b/test/README index 7831c860f5..fc9f7d03b1 100644 --- a/test/README +++ b/test/README @@ -99,14 +99,13 @@ test): to modify the include paths and source files if you don't want to use the basic test framework: - SOURCE[{name}]={name}.c testutil.c test_main.c + SOURCE[{name}]={name}.c INCLUDE[{name}]=.. ../include - DEPEND[{name}]=../libcrypto + DEPEND[{name}]=../libcrypto libtestutil.a Generic form of C test executables ================================== - #include "test_main.h" #include "testutil.h" static int my_test(void) diff --git a/test/build.info b/test/build.info index 5a2e03e607..f75b85a641 100644 --- a/test/build.info +++ b/test/build.info @@ -5,11 +5,16 @@ my ($base, $files) = @_; return join(" ", map { "$base/$_" } split(/\s+/, $files)); } - our $apps_extra = - $config{target} =~ /^vms-/ ? "../apps/vms_term_sock.c" : ""; "" -} IF[{- !$disabled{tests} -}] + LIBS_NO_INST=libtestutil.a + SOURCE[libtestutil.a]=testutil/basic_output.c testutil/driver.c \ + testutil/tests.c testutil/test_main.c testutil/main.c \ + {- rebase_files("../apps", $target{apps_aux_src}) -} + INCLUDE[libtestutil.a]=../include + DEPEND[libtestutil.a]=../libcrypto + PROGRAMS_NO_INST=\ aborttest test_test \ sanitytest exdatatest bntest \ diff --git a/test/test_main.c b/test/test_main.c deleted file mode 100644 index 99b21acc65..0000000000 --- a/test/test_main.c +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the OpenSSL license (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "test_main.h" -#include "testutil.h" - -#include - -int main(int argc, char *argv[]) -{ - int ret; - if (argc > 1) - printf("Warning: ignoring extra command-line arguments.\n"); - - setup_test(); - register_tests(); - ret = run_tests(argv[0]); - - return finish_test(ret); -} diff --git a/test/test_main.h b/test/test_main.h deleted file mode 100644 index ce0d06a407..0000000000 --- a/test/test_main.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the OpenSSL license (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef HEADER_TEST_MAIN_H -# define HEADER_TEST_MAIN_H - -/* - * Simple unit tests should implement register_tests() and link to test_main.c. - */ -extern void register_tests(void); - -#endif /* HEADER_TEST_MAIN_H */ diff --git a/test/test_main_custom.c b/test/test_main_custom.c deleted file mode 100644 index e10f2795df..0000000000 --- a/test/test_main_custom.c +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the OpenSSL license (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "test_main_custom.h" -#include "testutil.h" - -#include -#include - -int main(int argc, char *argv[]) -{ - int ret; - setup_test(); - - ret = test_main(argc, argv); - - return finish_test(ret); -} diff --git a/test/test_main_custom.h b/test/test_main_custom.h deleted file mode 100644 index 22bc8f51a6..0000000000 --- a/test/test_main_custom.h +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the OpenSSL license (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#ifndef HEADER_TEST_MAIN_CUSTOM_H -# define HEADER_TEST_MAIN_CUSTOM_H - -/* - * Unit tests that need a custom main() should implement test_main and link to - * test_main_custom.c - * test_main() should return the result of run_tests(). - */ -extern int test_main(int argc, char *argv[]); - -#endif /* HEADER_TEST_MAIN_CUSTOM_H */ diff --git a/test/testutil.c b/test/testutil.c deleted file mode 100644 index ae98e1096d..0000000000 --- a/test/testutil.c +++ /dev/null @@ -1,469 +0,0 @@ -/* - * Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. - * - * Licensed under the OpenSSL license (the "License"). You may not use - * this file except in compliance with the License. You can obtain a copy - * in the file LICENSE in the source distribution or at - * https://www.openssl.org/source/license.html - */ - -#include "testutil.h" - -#include -#include -#include -#include -#include "e_os.h" - -#include -#include -#include - -/* The size of memory buffers to display on failure */ -#define MEM_BUFFER_SIZE (21) - -/* - * Declares the structures needed to register each test case function. - */ -typedef struct test_info { - const char *test_case_name; - int (*test_fn) (); - int (*param_test_fn)(int idx); - int num; -} TEST_INFO; - -static TEST_INFO all_tests[1024]; -static int num_tests = 0; -/* - * A parameterised tests runs a loop of test cases. - * |num_test_cases| counts the total number of test cases - * across all tests. - */ -static int num_test_cases = 0; - -void add_test(const char *test_case_name, int (*test_fn) ()) -{ - assert(num_tests != OSSL_NELEM(all_tests)); - all_tests[num_tests].test_case_name = test_case_name; - all_tests[num_tests].test_fn = test_fn; - all_tests[num_tests].num = -1; - ++num_test_cases; - ++num_tests; -} - -void add_all_tests(const char *test_case_name, int(*test_fn)(int idx), - int num) -{ - assert(num_tests != OSSL_NELEM(all_tests)); - all_tests[num_tests].test_case_name = test_case_name; - all_tests[num_tests].param_test_fn = test_fn; - all_tests[num_tests].num = num; - ++num_tests; - num_test_cases += num; -} - -#ifndef OPENSSL_NO_CRYPTO_MDEBUG -static int should_report_leaks() -{ - /* - * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0 - * can be used to disable leak checking at runtime. - * Note this only works when running the test binary manually; - * the test harness always enables OPENSSL_DEBUG_MEMORY. - */ - char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY"); - - return mem_debug_env == NULL - || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, "")); -} -#endif - - -void setup_test() -{ -#ifndef OPENSSL_NO_CRYPTO_MDEBUG - if (should_report_leaks()) { - CRYPTO_set_mem_debug(1); - CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON); - } -#endif -} - -int finish_test(int ret) -{ -#ifndef OPENSSL_NO_CRYPTO_MDEBUG - if (should_report_leaks() && CRYPTO_mem_leaks_fp(stderr) <= 0) - return EXIT_FAILURE; -#endif - return ret; -} - -static void finalize(int success) -{ - if (success) - ERR_clear_error(); - else - ERR_print_errors_fp(stderr); -} - -int run_tests(const char *test_prog_name) -{ - int num_failed = 0; - - int i, j; - - printf("%s: %d test case%s\n", test_prog_name, num_test_cases, - num_test_cases == 1 ? "" : "s"); - fflush(stdout); - - for (i = 0; i != num_tests; ++i) { - if (all_tests[i].num == -1) { - int ret = all_tests[i].test_fn(); - - if (!ret) { - printf("** %s failed **\n--------\n", - all_tests[i].test_case_name); - fflush(stdout); - ++num_failed; - } - finalize(ret); - } else { - for (j = 0; j < all_tests[i].num; j++) { - int ret = all_tests[i].param_test_fn(j); - - if (!ret) { - printf("** %s failed test %d\n--------\n", - all_tests[i].test_case_name, j); - fflush(stdout); - ++num_failed; - } - finalize(ret); - } - } - } - - if (num_failed != 0) { - printf("%s: %d test%s failed (out of %d)\n", test_prog_name, - num_failed, num_failed != 1 ? "s" : "", num_test_cases); - fflush(stdout); - return EXIT_FAILURE; - } - printf(" All tests passed.\n"); - fflush(stdout); - return EXIT_SUCCESS; -} - -/* - * A common routine to output test failure messages. Generally this should not - * be called directly, rather it should be called by the following functions. - * - * |desc| is a printf formatted description with arguments |args| that is - * supplied by the user and |desc| can be NULL. |type| is the data type - * that was tested (int, char, ptr, ...). |fmt| is a system provided - * printf format with following arguments that spell out the failure - * details i.e. the actual values compared and the operator used. - * - * The typical use for this is from an utility test function: - * - * int test6(const char *file, int line, int n) { - * if (n != 6) { - * test_fail_message(1, file, line, "int", "value %d is not %d", n, 6); - * return 0; - * } - * return 1; - * } - * - * calling test6(3, "oops") will return 0 and produce out along the lines of: - * FAIL oops: (int) value 3 is not 6\n - * - * It general, test_fail_message should not be called directly. - */ -static void test_fail_message(const char *prefix, const char *file, int line, - const char *type, const char *fmt, ...) - PRINTF_FORMAT(5, 6); - -static void test_fail_message_va(const char *prefix, const char *file, int line, - const char *type, const char *fmt, va_list ap) -{ - fputs(prefix != NULL ? prefix : "ERROR", stderr); - fputs(":", stderr); - if (type) - fprintf(stderr, " (%s)", type); - if (fmt != NULL) { - fputc(' ', stderr); - vfprintf(stderr, fmt, ap); - } - if (file != NULL) { - fprintf(stderr, " @ %s:%d", file, line); - } - fputc('\n', stderr); -} - -static void test_fail_message(const char *prefix, const char *file, int line, - const char *type, const char *fmt, ...) -{ - va_list ap; - va_start(ap, fmt); - test_fail_message_va(prefix, file, line, type, fmt, ap); - va_end(ap); -} - -void test_info_c90(const char *desc, ...) -{ - va_list ap; - - va_start(ap, desc); - test_fail_message_va("INFO", NULL, -1, NULL, desc, ap); - va_end(ap); -} - -void test_info(const char *file, int line, const char *desc, ...) -{ - va_list ap; - - va_start(ap, desc); - test_fail_message_va("INFO", file, line, NULL, desc, ap); - va_end(ap); -} - -void test_error_c90(const char *desc, ...) -{ - va_list ap; - - va_start(ap, desc); - test_fail_message(NULL, NULL, -1, NULL, desc, ap); - va_end(ap); -} - -void test_error(const char *file, int line, const char *desc, ...) -{ - va_list ap; - - va_start(ap, desc); - test_fail_message_va(NULL, file, line, NULL, desc, ap); - va_end(ap); -} - -/* - * Define some comparisons between pairs of various types. - * These functions return 1 if the test is true. - * Otherwise, they return 0 and pretty-print diagnostics. - * - * In each case the functions produced are: - * int test_name_eq(const type t1, const type t2, const char *desc, ...); - * int test_name_ne(const type t1, const type t2, const char *desc, ...); - * int test_name_lt(const type t1, const type t2, const char *desc, ...); - * int test_name_le(const type t1, const type t2, const char *desc, ...); - * int test_name_gt(const type t1, const type t2, const char *desc, ...); - * int test_name_ge(const type t1, const type t2, const char *desc, ...); - * - * The t1 and t2 arguments are to be compared for equality, inequality, - * less than, less than or equal to, greater than and greater than or - * equal to respectively. If the specified condition holds, the functions - * return 1. If the condition does not hold, the functions print a diagnostic - * message and return 0. - * - * The desc argument is a printf format string followed by its arguments and - * this is included in the output if the condition being tested for is false. - */ -#define DEFINE_COMPARISON(type, name, opname, op, fmt) \ - int test_ ## name ## _ ## opname(const char *file, int line, \ - const char *s1, const char *s2, \ - const type t1, const type t2) \ - { \ - if (t1 op t2) \ - return 1; \ - test_fail_message(NULL, file, line, #type, \ - "%s [" fmt "] " #op " %s [" fmt "]", \ - s1, t1, s2, t2); \ - return 0; \ - } - -#define DEFINE_COMPARISONS(type, name, fmt) \ - DEFINE_COMPARISON(type, name, eq, ==, fmt) \ - DEFINE_COMPARISON(type, name, ne, !=, fmt) \ - DEFINE_COMPARISON(type, name, lt, <, fmt) \ - DEFINE_COMPARISON(type, name, le, <=, fmt) \ - DEFINE_COMPARISON(type, name, gt, >, fmt) \ - DEFINE_COMPARISON(type, name, ge, >=, fmt) - -DEFINE_COMPARISONS(int, int, "%d") -DEFINE_COMPARISONS(unsigned int, uint, "%u") -DEFINE_COMPARISONS(char, char, "%c") -DEFINE_COMPARISONS(unsigned char, uchar, "%u") -DEFINE_COMPARISONS(long, long, "%ld") -DEFINE_COMPARISONS(unsigned long, ulong, "%lu") -DEFINE_COMPARISONS(size_t, size_t, "%" OSSLzu) - -DEFINE_COMPARISON(void *, ptr, eq, ==, "%p") -DEFINE_COMPARISON(void *, ptr, ne, !=, "%p") - -int test_ptr_null(const char *file, int line, const char *s, const void *p) -{ - if (p == NULL) - return 1; - test_fail_message(NULL, file, line, "ptr", "%s [%p] == NULL", s, p); - return 0; -} - -int test_ptr(const char *file, int line, const char *s, const void *p) -{ - if (p != NULL) - return 1; - test_fail_message(NULL, file, line, "ptr", "%s [%p] != NULL", s, p); - return 0; -} - -int test_true(const char *file, int line, const char *s, int b) -{ - if (b) - return 1; - test_fail_message(NULL, file, line, "bool", "%s [false] == true", s); - return 0; -} - -int test_false(const char *file, int line, const char *s, int b) -{ - if (!b) - return 1; - test_fail_message(NULL, file, line, "bool", "%s [true] == false", s); - return 0; -} - -static const char *print_string_maybe_null(const char *s) -{ - return s == NULL ? "(NULL)" : s; -} - -int test_str_eq(const char *file, int line, const char *st1, const char *st2, - const char *s1, const char *s2) -{ - if (s1 == NULL && s2 == NULL) - return 1; - if (s1 == NULL || s2 == NULL || strcmp(s1, s2) != 0) { - test_fail_message(NULL, file, line, "string", "%s [%s] == %s [%s]", - st1, print_string_maybe_null(s1), - st2, print_string_maybe_null(s2)); - return 0; - } - return 1; -} - -int test_str_ne(const char *file, int line, const char *st1, const char *st2, - const char *s1, const char *s2) -{ - if ((s1 == NULL) ^ (s2 == NULL)) - return 1; - if (s1 == NULL || strcmp(s1, s2) == 0) { - test_fail_message(NULL, file, line, "string", "%s [%s] != %s [%s]", - st1, print_string_maybe_null(s1), - st2, print_string_maybe_null(s2)); - return 0; - } - return 1; -} - -int test_strn_eq(const char *file, int line, const char *st1, const char *st2, - const char *s1, const char *s2, size_t len) -{ - int prec = (int)len; - - if (s1 == NULL && s2 == NULL) - return 1; - if (s1 == NULL || s2 == NULL || strncmp(s1, s2, len) != 0) { - test_fail_message(NULL, file, line, "string", "%.s [%.*s] == %s [%.*s]", - st1, prec, print_string_maybe_null(s1), - st2, prec, print_string_maybe_null(s2)); - return 0; - } - return 1; -} - -int test_strn_ne(const char *file, int line, const char *st1, const char *st2, - const char *s1, const char *s2, size_t len) -{ - int prec = (int)len; - - if ((s1 == NULL) ^ (s2 == NULL)) - return 1; - if (s1 == NULL || strncmp(s1, s2, len) == 0) { - test_fail_message(NULL, file, line, "string", "%s [%.*s] != %s [%.*s]", - st1, prec, print_string_maybe_null(s1), - st2, prec, print_string_maybe_null(s2)); - return 0; - } - return 1; -} - -/* - * We could use OPENSSL_buf2hexstr() to do this but trying to allocate memory - * in a failure state isn't generally a great idea. - */ -static const char *print_mem_maybe_null(const void *s, size_t n, - char out[MEM_BUFFER_SIZE]) -{ - size_t i; - const unsigned char *p = (const unsigned char *)s; - int pad = 2*n >= MEM_BUFFER_SIZE; - - if (s == NULL) - return "(NULL)"; - if (pad) - n = MEM_BUFFER_SIZE-4; - - for (i=0; i<2*n; i++) { - unsigned char c = (i & 1) != 0 ? p[i / 2] & 15 : p[i / 2] >> 4; - out[i] = "0123456789abcdef"[c]; - } - if (pad) { - out[i++] = '.'; - out[i++] = '.'; - out[i++] = '.'; - } - out[i] = '\0'; - - return out; -} - -int test_mem_eq(const char *file, int line, const char *st1, const char *st2, - const void *s1, size_t n1, const void *s2, size_t n2) -{ - char b1[MEM_BUFFER_SIZE], b2[MEM_BUFFER_SIZE]; - - if (s1 == NULL && s2 == NULL) - return 1; - if (n1 != n2) { - test_fail_message(NULL, file, line, "memory", - "size mismatch %s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]", - st1, print_mem_maybe_null(s1, n1, b1), n1, - st2, print_mem_maybe_null(s2, n2, b2), n2); - return 0; - } - if (s1 == NULL || s2 == NULL || memcmp(s1, s2, n1) != 0) { - test_fail_message(NULL, file, line, "memory", - "%s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]", - st1, print_mem_maybe_null(s1, n1, b1), n1, - st2, print_mem_maybe_null(s2, n2, b2), n2); - return 0; - } - return 1; -} - -int test_mem_ne(const char *file, int line, const char *st1, const char *st2, - const void *s1, size_t n1, const void *s2, size_t n2) -{ - char b1[MEM_BUFFER_SIZE], b2[MEM_BUFFER_SIZE]; - - if ((s1 == NULL) ^ (s2 == NULL)) - return 1; - if (n1 != n2) - return 1; - if (s1 == NULL || memcmp(s1, s2, n1) == 0) { - test_fail_message(NULL, file, line, "memory", - "%s %s [%"OSSLzu"] != %s %s [%"OSSLzu"]", - st1, print_mem_maybe_null(s1, n1, b1), n1, - st2, print_mem_maybe_null(s2, n2, b2), n2); - return 0; - } - return 1; -} diff --git a/test/testutil.h b/test/testutil.h index e7478a8ade..d266fd5288 100644 --- a/test/testutil.h +++ b/test/testutil.h @@ -16,12 +16,9 @@ #include /*- - * Simple unit tests should implement register_tests() from test_main.h - * and link against test_main.c. + * Simple unit tests should implement register_tests(). * To register tests, call ADD_TEST or ADD_ALL_TESTS: * - * #include "test_main.h" - * * void register_tests(void) * { * ADD_TEST(test_foo); @@ -29,8 +26,7 @@ * } * * Tests that need to perform custom setup or read command-line arguments should - * implement test_main() from test_main_custom.h and link against - * test_main_custom.c: + * implement test_main(): * * int test_main(int argc, char *argv[]) * { @@ -138,14 +134,27 @@ void add_test(const char *test_case_name, int (*test_fn) ()); void add_all_tests(const char *test_case_name, int (*test_fn)(int idx), int num); __owur int run_tests(const char *test_prog_name); +/* + * Declarations for user defined functions + */ +void register_tests(void); +int test_main(int argc, char *argv[]); + + /* * Test assumption verification helpers. */ -# if defined(__GNUC__) -#define PRINTF_FORMAT(a, b) __attribute__ ((format(printf, a, b))) -# else #define PRINTF_FORMAT(a, b) +#if defined(__GNUC__) && defined(__STDC_VERSION__) + /* + * Because we support the 'z' modifier, which made its appearance in C99, + * we can't use __attribute__ with pre C99 dialects. + */ +# if __STDC_VERSION__ >= 199901L +# undef PRINTF_FORMAT +# define PRINTF_FORMAT(a, b) __attribute__ ((format(printf, a, b))) +# endif #endif # define DECLARE_COMPARISON(type, name, opname) \ @@ -335,3 +344,22 @@ void test_info_c90(const char *desc, ...) PRINTF_FORMAT(1, 2); } \ } while (0) #endif /* HEADER_TESTUTIL_H */ + + +/* + * The basic I/O functions used by the test framework. These can be + * overriden when needed. Note that if one is, then all must be. + */ +void test_open_streams(void); +void test_close_streams(void); +/* The following ALL return the number of characters written */ +int test_puts_stdout(const char *str); +int test_puts_stderr(const char *str); +int test_vprintf_stdout(const char *fmt, va_list ap); +int test_vprintf_stderr(const char *fmt, va_list ap); +/* These return failure or success */ +int test_flush_stdout(void); +int test_flush_stderr(void); + +extern BIO *bio_out; +extern BIO *bio_err; diff --git a/test/testutil/basic_output.c b/test/testutil/basic_output.c new file mode 100644 index 0000000000..ac413a6a4a --- /dev/null +++ b/test/testutil/basic_output.c @@ -0,0 +1,80 @@ +/* + * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../testutil.h" + +#include +#include + +BIO *bio_out = NULL; +BIO *bio_err = NULL; + +#ifdef OPENSSL_USE_APPLINK +/* + * Using BIO_new_fd() obligates the use of applinks on platforms where it's + * relevant. Because it becomes a module of the libtestutil library and would + * be disregarded if not actively referred to, we have this dummy that does + * exactly this. For any module that uses the rest of the routines here, + * OPENSSL_Applink should tag along for sure. + */ +void Applink_dummy(void); +void Applink_dummy(void) +{ + OPENSSL_EXTERN void OPENSSL_Applink(void); + + OPENSSL_Applink(); +} +/* Generate an error for anyone who tries to actually use this dummy */ +# define Applink_dummy "DON'T USE THIS" +#endif + +void test_open_streams(void) +{ + bio_out = BIO_new_fd(1, 0); + bio_err = BIO_new_fd(2, 0); + + OPENSSL_assert(bio_out != NULL); + OPENSSL_assert(bio_err != NULL); +} + +void test_close_streams(void) +{ + BIO_free(bio_out); + BIO_free(bio_err); +} + +int test_puts_stdout(const char *str) +{ + return BIO_puts(bio_out, str); +} + +int test_puts_stderr(const char *str) +{ + return BIO_puts(bio_err, str); +} + +int test_vprintf_stdout(const char *fmt, va_list ap) +{ + return BIO_vprintf(bio_out, fmt, ap); +} + +int test_vprintf_stderr(const char *fmt, va_list ap) +{ + return BIO_vprintf(bio_err, fmt, ap); +} + +int test_flush_stdout(void) +{ + return BIO_flush(bio_out); +} + +int test_flush_stderr(void) +{ + return BIO_flush(bio_err); +} diff --git a/test/testutil/driver.c b/test/testutil/driver.c new file mode 100644 index 0000000000..e70fd21615 --- /dev/null +++ b/test/testutil/driver.c @@ -0,0 +1,167 @@ +/* + * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../testutil.h" + +#include +#include + +#include "../../e_os.h" +#include + +/* + * Declares the structures needed to register each test case function. + */ +typedef struct test_info { + const char *test_case_name; + int (*test_fn) (); + int (*param_test_fn)(int idx); + int num; +} TEST_INFO; + +static TEST_INFO all_tests[1024]; +static int num_tests = 0; +/* + * A parameterised tests runs a loop of test cases. + * |num_test_cases| counts the total number of test cases + * across all tests. + */ +static int num_test_cases = 0; + +void add_test(const char *test_case_name, int (*test_fn) ()) +{ + assert(num_tests != OSSL_NELEM(all_tests)); + all_tests[num_tests].test_case_name = test_case_name; + all_tests[num_tests].test_fn = test_fn; + all_tests[num_tests].num = -1; + ++num_tests; + ++num_test_cases; +} + +void add_all_tests(const char *test_case_name, int(*test_fn)(int idx), + int num) +{ + assert(num_tests != OSSL_NELEM(all_tests)); + all_tests[num_tests].test_case_name = test_case_name; + all_tests[num_tests].param_test_fn = test_fn; + all_tests[num_tests].num = num; + ++num_tests; + num_test_cases += num; +} + +#ifndef OPENSSL_NO_CRYPTO_MDEBUG +static int should_report_leaks() +{ + /* + * When compiled with enable-crypto-mdebug, OPENSSL_DEBUG_MEMORY=0 + * can be used to disable leak checking at runtime. + * Note this only works when running the test binary manually; + * the test harness always enables OPENSSL_DEBUG_MEMORY. + */ + char *mem_debug_env = getenv("OPENSSL_DEBUG_MEMORY"); + + return mem_debug_env == NULL + || (strcmp(mem_debug_env, "0") && strcmp(mem_debug_env, "")); +} +#endif + + +static int err_cb(const char *str, size_t len, void *u) +{ + return test_puts_stderr(str); +} + +void setup_test() +{ + test_open_streams(); + +#ifndef OPENSSL_NO_CRYPTO_MDEBUG + if (should_report_leaks()) { + CRYPTO_set_mem_debug(1); + CRYPTO_mem_ctrl(CRYPTO_MEM_CHECK_ON); + } +#endif +} + +int finish_test(int ret) +{ +#ifndef OPENSSL_NO_CRYPTO_MDEBUG + if (should_report_leaks() && CRYPTO_mem_leaks_cb(err_cb, NULL) <= 0) + return EXIT_FAILURE; +#endif + + test_close_streams(); + + return ret; +} + +static void finalize(int success) +{ + if (success) + ERR_clear_error(); + else + ERR_print_errors_cb(err_cb, NULL); +} + +static void helper_printf_stdout(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + test_vprintf_stdout(fmt, ap); + va_end(ap); +} + +int run_tests(const char *test_prog_name) +{ + int num_failed = 0; + int i, j; + + helper_printf_stdout("%s: %d test case%s\n", test_prog_name, num_test_cases, + num_test_cases == 1 ? "" : "s"); + test_flush_stdout(); + + for (i = 0; i != num_tests; ++i) { + if (all_tests[i].num == -1) { + int ret = all_tests[i].test_fn(); + + if (!ret) { + helper_printf_stdout("** %s failed **\n--------\n", + all_tests[i].test_case_name); + test_flush_stdout(); + ++num_failed; + } + finalize(ret); + } else { + for (j = 0; j < all_tests[i].num; j++) { + int ret = all_tests[i].param_test_fn(j); + + if (!ret) { + helper_printf_stdout("** %s failed test %d\n--------\n", + all_tests[i].test_case_name, j); + test_flush_stdout(); + ++num_failed; + } + finalize(ret); + } + } + } + + if (num_failed != 0) { + helper_printf_stdout("%s: %d test%s failed (out of %d)\n", + test_prog_name, num_failed, + num_failed != 1 ? "s" : "", num_test_cases); + test_flush_stdout(); + return EXIT_FAILURE; + } + helper_printf_stdout(" All tests passed.\n"); + test_flush_stdout(); + return EXIT_SUCCESS; +} + diff --git a/test/testutil/main.c b/test/testutil/main.c new file mode 100644 index 0000000000..435a3585ce --- /dev/null +++ b/test/testutil/main.c @@ -0,0 +1,20 @@ +/* + * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../testutil.h" + +int main(int argc, char *argv[]) +{ + int ret; + setup_test(); + + ret = test_main(argc, argv); + + return finish_test(ret); +} diff --git a/test/testutil/test_main.c b/test/testutil/test_main.c new file mode 100644 index 0000000000..0152421fe1 --- /dev/null +++ b/test/testutil/test_main.c @@ -0,0 +1,21 @@ +/* + * Copyright 2016-2017 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../testutil.h" + +#include + +int test_main(int argc, char *argv[]) +{ + if (argc > 1) + test_puts_stderr("Warning: ignoring extra command-line arguments.\n"); + + register_tests(); + return run_tests(argv[0]); +} diff --git a/test/testutil/tests.c b/test/testutil/tests.c new file mode 100644 index 0000000000..f00ec6c19e --- /dev/null +++ b/test/testutil/tests.c @@ -0,0 +1,342 @@ +/* + * Copyright 2017 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include "../testutil.h" + +#include +#include "../../e_os.h" + +/* The size of memory buffers to display on failure */ +#define MEM_BUFFER_SIZE (21) + +/* + * A common routine to output test failure messages. Generally this should not + * be called directly, rather it should be called by the following functions. + * + * |desc| is a printf formatted description with arguments |args| that is + * supplied by the user and |desc| can be NULL. |type| is the data type + * that was tested (int, char, ptr, ...). |fmt| is a system provided + * printf format with following arguments that spell out the failure + * details i.e. the actual values compared and the operator used. + * + * The typical use for this is from an utility test function: + * + * int test6(const char *file, int line, int n) { + * if (n != 6) { + * test_fail_message(1, file, line, "int", "value %d is not %d", n, 6); + * return 0; + * } + * return 1; + * } + * + * calling test6(3, "oops") will return 0 and produce out along the lines of: + * FAIL oops: (int) value 3 is not 6\n + * + * It general, test_fail_message should not be called directly. + */ +static void test_fail_message(const char *prefix, const char *file, int line, + const char *type, const char *fmt, ...) + PRINTF_FORMAT(5, 6); + +static void helper_printf_stderr(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + test_vprintf_stderr(fmt, ap); + va_end(ap); +} + +static void test_fail_message_va(const char *prefix, const char *file, int line, + const char *type, const char *fmt, va_list ap) +{ + test_puts_stderr(prefix != NULL ? prefix : "ERROR"); + test_puts_stderr(":"); + if (type) + helper_printf_stderr(" (%s)", type); + if (fmt != NULL) { + test_puts_stderr(" "); + test_vprintf_stderr(fmt, ap); + } + if (file != NULL) { + helper_printf_stderr(" @ %s:%d", file, line); + } + test_puts_stderr("\n"); + test_flush_stderr(); +} + +static void test_fail_message(const char *prefix, const char *file, int line, + const char *type, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + test_fail_message_va(prefix, file, line, type, fmt, ap); + va_end(ap); +} + +void test_info_c90(const char *desc, ...) +{ + va_list ap; + + va_start(ap, desc); + test_fail_message_va("INFO", NULL, -1, NULL, desc, ap); + va_end(ap); +} + +void test_info(const char *file, int line, const char *desc, ...) +{ + va_list ap; + + va_start(ap, desc); + test_fail_message_va("INFO", file, line, NULL, desc, ap); + va_end(ap); +} + +void test_error_c90(const char *desc, ...) +{ + va_list ap; + + va_start(ap, desc); + test_fail_message(NULL, NULL, -1, NULL, desc, ap); + va_end(ap); +} + +void test_error(const char *file, int line, const char *desc, ...) +{ + va_list ap; + + va_start(ap, desc); + test_fail_message_va(NULL, file, line, NULL, desc, ap); + va_end(ap); +} + +/* + * Define some comparisons between pairs of various types. + * These functions return 1 if the test is true. + * Otherwise, they return 0 and pretty-print diagnostics. + * + * In each case the functions produced are: + * int test_name_eq(const type t1, const type t2, const char *desc, ...); + * int test_name_ne(const type t1, const type t2, const char *desc, ...); + * int test_name_lt(const type t1, const type t2, const char *desc, ...); + * int test_name_le(const type t1, const type t2, const char *desc, ...); + * int test_name_gt(const type t1, const type t2, const char *desc, ...); + * int test_name_ge(const type t1, const type t2, const char *desc, ...); + * + * The t1 and t2 arguments are to be compared for equality, inequality, + * less than, less than or equal to, greater than and greater than or + * equal to respectively. If the specified condition holds, the functions + * return 1. If the condition does not hold, the functions print a diagnostic + * message and return 0. + * + * The desc argument is a printf format string followed by its arguments and + * this is included in the output if the condition being tested for is false. + */ +#define DEFINE_COMPARISON(type, name, opname, op, fmt) \ + int test_ ## name ## _ ## opname(const char *file, int line, \ + const char *s1, const char *s2, \ + const type t1, const type t2) \ + { \ + if (t1 op t2) \ + return 1; \ + test_fail_message(NULL, file, line, #type, \ + "%s [" fmt "] " #op " %s [" fmt "]", \ + s1, t1, s2, t2); \ + return 0; \ + } + +#define DEFINE_COMPARISONS(type, name, fmt) \ + DEFINE_COMPARISON(type, name, eq, ==, fmt) \ + DEFINE_COMPARISON(type, name, ne, !=, fmt) \ + DEFINE_COMPARISON(type, name, lt, <, fmt) \ + DEFINE_COMPARISON(type, name, le, <=, fmt) \ + DEFINE_COMPARISON(type, name, gt, >, fmt) \ + DEFINE_COMPARISON(type, name, ge, >=, fmt) + +DEFINE_COMPARISONS(int, int, "%d") +DEFINE_COMPARISONS(unsigned int, uint, "%u") +DEFINE_COMPARISONS(char, char, "%c") +DEFINE_COMPARISONS(unsigned char, uchar, "%u") +DEFINE_COMPARISONS(long, long, "%ld") +DEFINE_COMPARISONS(unsigned long, ulong, "%lu") +DEFINE_COMPARISONS(size_t, size_t, "%zu") + +DEFINE_COMPARISON(void *, ptr, eq, ==, "%p") +DEFINE_COMPARISON(void *, ptr, ne, !=, "%p") + +int test_ptr_null(const char *file, int line, const char *s, const void *p) +{ + if (p == NULL) + return 1; + test_fail_message(NULL, file, line, "ptr", "%s [%p] == NULL", s, p); + return 0; +} + +int test_ptr(const char *file, int line, const char *s, const void *p) +{ + if (p != NULL) + return 1; + test_fail_message(NULL, file, line, "ptr", "%s [%p] != NULL", s, p); + return 0; +} + +int test_true(const char *file, int line, const char *s, int b) +{ + if (b) + return 1; + test_fail_message(NULL, file, line, "bool", "%s [false] == true", s); + return 0; +} + +int test_false(const char *file, int line, const char *s, int b) +{ + if (!b) + return 1; + test_fail_message(NULL, file, line, "bool", "%s [true] == false", s); + return 0; +} + +static const char *print_string_maybe_null(const char *s) +{ + return s == NULL ? "(NULL)" : s; +} + +int test_str_eq(const char *file, int line, const char *st1, const char *st2, + const char *s1, const char *s2) +{ + if (s1 == NULL && s2 == NULL) + return 1; + if (s1 == NULL || s2 == NULL || strcmp(s1, s2) != 0) { + test_fail_message(NULL, file, line, "string", "%s [%s] == %s [%s]", + st1, print_string_maybe_null(s1), + st2, print_string_maybe_null(s2)); + return 0; + } + return 1; +} + +int test_str_ne(const char *file, int line, const char *st1, const char *st2, + const char *s1, const char *s2) +{ + if ((s1 == NULL) ^ (s2 == NULL)) + return 1; + if (s1 == NULL || strcmp(s1, s2) == 0) { + test_fail_message(NULL, file, line, "string", "%s [%s] != %s [%s]", + st1, print_string_maybe_null(s1), + st2, print_string_maybe_null(s2)); + return 0; + } + return 1; +} + +int test_strn_eq(const char *file, int line, const char *st1, const char *st2, + const char *s1, const char *s2, size_t len) +{ + int prec = (int)len; + + if (s1 == NULL && s2 == NULL) + return 1; + if (s1 == NULL || s2 == NULL || strncmp(s1, s2, len) != 0) { + test_fail_message(NULL, file, line, "string", "%.s [%.*s] == %s [%.*s]", + st1, prec, print_string_maybe_null(s1), + st2, prec, print_string_maybe_null(s2)); + return 0; + } + return 1; +} + +int test_strn_ne(const char *file, int line, const char *st1, const char *st2, + const char *s1, const char *s2, size_t len) +{ + int prec = (int)len; + + if ((s1 == NULL) ^ (s2 == NULL)) + return 1; + if (s1 == NULL || strncmp(s1, s2, len) == 0) { + test_fail_message(NULL, file, line, "string", "%s [%.*s] != %s [%.*s]", + st1, prec, print_string_maybe_null(s1), + st2, prec, print_string_maybe_null(s2)); + return 0; + } + return 1; +} + +/* + * We could use OPENSSL_buf2hexstr() to do this but trying to allocate memory + * in a failure state isn't generally a great idea. + */ +static const char *print_mem_maybe_null(const void *s, size_t n, + char out[MEM_BUFFER_SIZE]) +{ + size_t i; + const unsigned char *p = (const unsigned char *)s; + int pad = 2*n >= MEM_BUFFER_SIZE; + + if (s == NULL) + return "(NULL)"; + if (pad) + n = MEM_BUFFER_SIZE-4; + + for (i=0; i<2*n; i++) { + unsigned char c = (i & 1) != 0 ? p[i / 2] & 15 : p[i / 2] >> 4; + out[i] = "0123456789abcdef"[c]; + } + if (pad) { + out[i++] = '.'; + out[i++] = '.'; + out[i++] = '.'; + } + out[i] = '\0'; + + return out; +} + +int test_mem_eq(const char *file, int line, const char *st1, const char *st2, + const void *s1, size_t n1, const void *s2, size_t n2) +{ + char b1[MEM_BUFFER_SIZE], b2[MEM_BUFFER_SIZE]; + + if (s1 == NULL && s2 == NULL) + return 1; + if (n1 != n2) { + test_fail_message(NULL, file, line, "memory", + "size mismatch %s %s [%zu] != %s %s [%zu]", + st1, print_mem_maybe_null(s1, n1, b1), n1, + st2, print_mem_maybe_null(s2, n2, b2), n2); + return 0; + } + if (s1 == NULL || s2 == NULL || memcmp(s1, s2, n1) != 0) { + test_fail_message(NULL, file, line, "memory", + "%s %s [%zu] != %s %s [%zu]", + st1, print_mem_maybe_null(s1, n1, b1), n1, + st2, print_mem_maybe_null(s2, n2, b2), n2); + return 0; + } + return 1; +} + +int test_mem_ne(const char *file, int line, const char *st1, const char *st2, + const void *s1, size_t n1, const void *s2, size_t n2) +{ + char b1[MEM_BUFFER_SIZE], b2[MEM_BUFFER_SIZE]; + + if ((s1 == NULL) ^ (s2 == NULL)) + return 1; + if (n1 != n2) + return 1; + if (s1 == NULL || memcmp(s1, s2, n1) == 0) { + test_fail_message(NULL, file, line, "memory", + "%s %s [%zu] != %s %s [%zu]", + st1, print_mem_maybe_null(s1, n1, b1), n1, + st2, print_mem_maybe_null(s2, n2, b2), n2); + return 0; + } + return 1; +}