2 * Copyright 2016-2018 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
10 #include "../testutil.h"
17 #include "internal/nelem.h"
18 #include <openssl/bio.h>
20 #include "platform.h" /* From libapps */
23 # define strdup _strdup
28 * Declares the structures needed to register each test case function.
30 typedef struct test_info {
31 const char *test_case_name;
32 int (*test_fn) (void);
33 int (*param_test_fn)(int idx);
40 static TEST_INFO all_tests[1024];
41 static int num_tests = 0;
42 static int show_list = 0;
43 static int single_test = -1;
44 static int single_iter = -1;
48 * A parameterised test runs a loop of test cases.
49 * |num_test_cases| counts the total number of test cases
52 static int num_test_cases = 0;
54 static int process_shared_options(void);
57 void add_test(const char *test_case_name, int (*test_fn) (void))
59 assert(num_tests != OSSL_NELEM(all_tests));
60 all_tests[num_tests].test_case_name = test_case_name;
61 all_tests[num_tests].test_fn = test_fn;
62 all_tests[num_tests].num = -1;
67 void add_all_tests(const char *test_case_name, int(*test_fn)(int idx),
70 assert(num_tests != OSSL_NELEM(all_tests));
71 all_tests[num_tests].test_case_name = test_case_name;
72 all_tests[num_tests].param_test_fn = test_fn;
73 all_tests[num_tests].num = num;
74 all_tests[num_tests].subtest = subtest;
76 num_test_cases += num;
79 int subtest_level(void)
84 static int gcd(int a, int b)
94 static void set_seed(int s)
98 seed = (int)time(NULL);
99 test_printf_stdout("%*s# RAND SEED %d\n", subtest_level(), "", seed);
101 test_random_seed(seed);
105 int setup_test_framework(int argc, char *argv[])
107 char *test_seed = getenv("OPENSSL_TEST_RAND_ORDER");
108 char *TAP_levels = getenv("HARNESS_OSSL_LEVEL");
110 if (TAP_levels != NULL)
111 level = 4 * atoi(TAP_levels);
112 if (test_seed != NULL)
113 set_seed(atoi(test_seed));
115 #if defined(OPENSSL_SYS_VMS) && defined(__DECC)
116 argv = copy_argv(&argc, argv);
117 #elif defined(_WIN32)
119 * Replace argv[] with UTF-8 encoded strings.
121 win32_utf8argv(&argc, &argv);
124 if (!opt_init(argc, argv, test_get_options()))
131 * This can only be called after setup() has run, since num_tests and
132 * all_tests[] are setup at this point
134 static int check_single_test_params(char *name, char *testname, char *itname)
138 for (i = 0; i < num_tests; ++i) {
139 if (strcmp(name, all_tests[i].test_case_name) == 0) {
145 single_test = atoi(name);
149 /* if only iteration is specified, assume we want the first test */
150 if (single_test == -1 && single_iter != -1)
153 if (single_test != -1) {
154 if (single_test < 1 || single_test > num_tests) {
155 test_printf_stderr("Invalid -%s value "
156 "(Value must be a valid test name OR a value between %d..%d)\n",
157 testname, 1, num_tests);
161 if (single_iter != -1) {
162 if (all_tests[single_test - 1].num == -1) {
163 test_printf_stderr("-%s option is not valid for test %d:%s\n",
166 all_tests[single_test - 1].test_case_name);
168 } else if (single_iter < 1
169 || single_iter > all_tests[single_test - 1].num) {
170 test_printf_stderr("Invalid -%s value for test %d:%s\t"
171 "(Value must be in the range %d..%d)\n",
173 all_tests[single_test - 1].test_case_name,
174 1, all_tests[single_test - 1].num);
181 static int process_shared_options(void)
183 OPTION_CHOICE_DEFAULT o;
186 char *flag_test = "";
187 char *flag_iter = "";
188 char *testname = NULL;
191 while ((o = opt_next()) != OPT_EOF) {
193 /* Ignore any test options at this level */
199 opt_help(test_get_options());
204 case OPT_TEST_SINGLE:
205 flag_test = opt_flag();
206 testname = opt_arg();
208 case OPT_TEST_ITERATION:
209 flag_iter = opt_flag();
210 if (!opt_int(opt_arg(), &single_iter))
213 case OPT_TEST_INDENT:
214 if (!opt_int(opt_arg(), &value))
219 if (!opt_int(opt_arg(), &value))
225 if (!check_single_test_params(testname, flag_test, flag_iter))
233 int pulldown_test_framework(int ret)
235 set_test_title(NULL);
239 static void finalize(int success)
244 ERR_print_errors_cb(openssl_error_cb, NULL);
247 static char *test_title = NULL;
249 void set_test_title(const char *title)
252 test_title = title == NULL ? NULL : strdup(title);
255 PRINTF_FORMAT(2, 3) static void test_verdict(int verdict,
256 const char *description, ...)
263 test_printf_stdout("%*s%s ", level, "", verdict != 0 ? "ok" : "not ok");
264 va_start(ap, description);
265 test_vprintf_stdout(description, ap);
267 if (verdict == TEST_SKIP_CODE)
268 test_printf_stdout(" # skipped");
269 test_printf_stdout("\n");
273 int run_tests(const char *test_prog_name)
277 int ii, i, jj, j, jstep;
278 int permute[OSSL_NELEM(all_tests)];
280 i = process_shared_options();
287 test_printf_stdout("%*s1..0 # Skipped: %s\n", level, "",
289 } else if (show_list == 0 && single_test == -1) {
291 test_printf_stdout("%*s# Subtest: %s\n", level, "", test_prog_name);
292 test_printf_stdout("%*s1..%d\n", level, "", num_tests);
297 for (i = 0; i < num_tests; i++)
300 for (i = num_tests - 1; i >= 1; i--) {
301 j = test_random() % (1 + i);
303 permute[j] = permute[i];
307 for (ii = 0; ii != num_tests; ++ii) {
310 if (single_test != -1 && ((i+1) != single_test)) {
313 else if (show_list) {
314 if (all_tests[i].num != -1) {
315 test_printf_stdout("%d - %s (%d..%d)\n", ii + 1,
316 all_tests[i].test_case_name, 1,
319 test_printf_stdout("%d - %s\n", ii + 1,
320 all_tests[i].test_case_name);
323 } else if (all_tests[i].num == -1) {
324 set_test_title(all_tests[i].test_case_name);
325 verdict = all_tests[i].test_fn();
326 test_verdict(verdict, "%d - %s", ii + 1, test_title);
327 finalize(verdict != 0);
331 int num_failed_inner = 0;
333 verdict = TEST_SKIP_CODE;
335 if (all_tests[i].subtest && single_iter == -1) {
336 test_printf_stdout("%*s# Subtest: %s\n", level, "",
337 all_tests[i].test_case_name);
338 test_printf_stdout("%*s%d..%d\n", level, "", 1,
344 if (seed == 0 || all_tests[i].num < 3)
348 jstep = test_random() % all_tests[i].num;
349 while (jstep == 0 || gcd(all_tests[i].num, jstep) != 1);
351 for (jj = 0; jj < all_tests[i].num; jj++) {
354 j = (j + jstep) % all_tests[i].num;
355 if (single_iter != -1 && ((jj + 1) != single_iter))
357 set_test_title(NULL);
358 v = all_tests[i].param_test_fn(j);
363 } else if (v != TEST_SKIP_CODE && verdict != 0) {
369 if (all_tests[i].subtest) {
370 if (test_title != NULL)
371 test_verdict(v, "%d - %s", jj + 1, test_title);
373 test_verdict(v, "%d - iteration %d", jj + 1, j + 1);
380 test_verdict(verdict, "%d - %s", ii + 1,
381 all_tests[i].test_case_name);
390 * Glue an array of strings together and return it as an allocated string.
391 * Optionally return the whole length of this string in |out_len|
393 char *glue_strings(const char *list[], size_t *out_len)
399 for (i = 0; list[i] != NULL; i++)
400 len += strlen(list[i]);
405 if (!TEST_ptr(ret = p = OPENSSL_malloc(len + 1)))
408 for (i = 0; list[i] != NULL; i++)
409 p += strlen(strcpy(p, list[i]));
414 char *test_mk_file_path(const char *dir, const char *file)
416 # ifndef OPENSSL_SYS_VMS
417 const char *sep = "/";
419 const char *sep = "";
421 size_t len = strlen(dir) + strlen(sep) + strlen(file) + 1;
422 char *full_file = OPENSSL_zalloc(len);
424 if (full_file != NULL) {
425 OPENSSL_strlcpy(full_file, dir, len);
426 OPENSSL_strlcat(full_file, sep, len);
427 OPENSSL_strlcat(full_file, file, len);