diff --git a/doc/develop/test/ztest.rst b/doc/develop/test/ztest.rst index f9255ecad2a7..c2d7cd0923e7 100644 --- a/doc/develop/test/ztest.rst +++ b/doc/develop/test/ztest.rst @@ -197,15 +197,15 @@ function can be written as follows: /* Only suites that use a predicate checking for phase == PWR_PHASE_0 will run. */ state.phase = PWR_PHASE_0; - ztest_run_all(&state); + ztest_run_all(&state, false, 1, 1); /* Only suites that use a predicate checking for phase == PWR_PHASE_1 will run. */ state.phase = PWR_PHASE_1; - ztest_run_all(&state); + ztest_run_all(&state, false, 1, 1); /* Only suites that use a predicate checking for phase == PWR_PHASE_2 will run. */ state.phase = PWR_PHASE_2; - ztest_run_all(&state); + ztest_run_all(&state, false, 1, 1); /* Check that all the suites in this binary ran at least once. */ ztest_verify_all_test_suites_ran(); diff --git a/subsys/testsuite/ztest/CMakeLists.txt b/subsys/testsuite/ztest/CMakeLists.txt index 7817bc2f30b8..4d29ac033c3b 100644 --- a/subsys/testsuite/ztest/CMakeLists.txt +++ b/subsys/testsuite/ztest/CMakeLists.txt @@ -26,6 +26,8 @@ zephyr_library_sources_ifdef(CONFIG_ZTRESS src/ztress.c) if(CONFIG_ARCH_POSIX) zephyr_library_sources(src/ztest_posix.c) +elseif(CONFIG_ZTEST_SHELL) + zephyr_library_sources(src/ztest_shell.c) else() zephyr_library_sources(src/ztest_defaults.c) endif() diff --git a/subsys/testsuite/ztest/Kconfig b/subsys/testsuite/ztest/Kconfig index 977f882d838d..e453d2864632 100644 --- a/subsys/testsuite/ztest/Kconfig +++ b/subsys/testsuite/ztest/Kconfig @@ -23,6 +23,15 @@ config ZTEST_TEST_DELAY_MS Add a delay between between tests to manage output on the console on systems that can't handle the rapid output rate. +config ZTEST_SHELL + bool "Ztest with shell support" + select SHELL + select SHELL_THREAD_PRIORITY_OVERRIDE + select GETOPT_LONG + select SHELL_GETOPT + help + Enable shell to manage test execution and selection. + config ZTEST_CPU_HOLD_TIME_MS int "Time in milliseconds to hold other CPUs for 1cpu type tests" default 3000 diff --git a/subsys/testsuite/ztest/include/zephyr/ztest_test.h b/subsys/testsuite/ztest/include/zephyr/ztest_test.h index aac7a7b2ad88..120848f26a73 100644 --- a/subsys/testsuite/ztest/include/zephyr/ztest_test.h +++ b/subsys/testsuite/ztest/include/zephyr/ztest_test.h @@ -232,8 +232,11 @@ extern struct ztest_suite_node _ztest_suite_node_list_end[]; * Default entry point for running or listing registered unit tests. * * @param state The current state of the machine as it relates to the test executable. + * @param shuffle Shuffle tests + * @param suite_iter Test suite repetitions. + * @param case_iter Test case repetitions. */ -void ztest_run_all(const void *state); +void ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter); /** * The result of the current running test. It's possible that the setup function sets the result @@ -265,18 +268,25 @@ enum ztest_phase { * Run the registered unit tests which return true from their predicate function. * * @param state The current state of the machine as it relates to the test executable. + * @param shuffle Shuffle tests + * @param suite_iter Test suite repetitions. + * @param case_iter Test case repetitions. * @return The number of tests that ran. */ #ifdef ZTEST_UNITTEST -int z_impl_ztest_run_test_suites(const void *state); -static inline int ztest_run_test_suites(const void *state) +int z_impl_ztest_run_test_suites(const void *state, bool shuffle, + int suite_iter, int case_iter); + +static inline int ztest_run_test_suites(const void *state, bool shuffle, + int suite_iter, int case_iter) { - return z_impl_ztest_run_test_suites(state); + return z_impl_ztest_run_test_suites(state, shuffle, suite_iter, case_iter); } #else -__syscall int ztest_run_test_suites(const void *state); +__syscall int ztest_run_test_suites(const void *state, bool shuffle, + int suite_iter, int case_iter); #endif #ifdef ZTEST_UNITTEST @@ -315,9 +325,12 @@ void ztest_verify_all_test_suites_ran(void); * checks for fast failures and initialization. * * @param name The name of the suite to run. + * @param shuffle Shuffle tests + * @param suite_iter Test suite repetitions. + * @param case_iter Test case repetitions. * @return Negative value if the test suite never ran; otherwise, return the number of failures. */ -int z_ztest_run_test_suite(const char *name); +int z_ztest_run_test_suite(const char *name, bool shuffle, int suite_iter, int case_iter); /** * @brief Returns next test within suite. @@ -534,14 +547,15 @@ void ztest_simple_1cpu_after(void *data); * * @param suite Test suite to run. */ -#define ztest_run_test_suite(suite) z_ztest_run_test_suite(STRINGIFY(suite)) +#define ztest_run_test_suite(suite) z_ztest_run_test_suite(STRINGIFY(suite), \ + int suite_iter, int case_iter) /** * @brief Structure for architecture specific APIs * */ struct ztest_arch_api { - void (*run_all)(const void *state); + void (*run_all)(const void *state, bool shuffle, int suite_iter, int case_iter); bool (*should_suite_run)(const void *state, struct ztest_suite_node *suite); bool (*should_test_run)(const char *suite, const char *test); }; diff --git a/subsys/testsuite/ztest/src/ztest.c b/subsys/testsuite/ztest/src/ztest.c index d49a1ef9c110..5c44829acd42 100644 --- a/subsys/testsuite/ztest/src/ztest.c +++ b/subsys/testsuite/ztest/src/ztest.c @@ -18,11 +18,15 @@ static struct k_thread ztest_thread; #endif static bool failed_expectation; +#ifdef CONFIG_ZTEST_SHELL +#include +#endif + #ifdef CONFIG_ZTEST_SHUFFLE #include #include - #include + #define NUM_ITER_PER_SUITE CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT #define NUM_ITER_PER_TEST CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT #else @@ -651,30 +655,33 @@ struct ztest_unit_test *z_ztest_get_next_test(const char *suite, struct ztest_un return NULL; } -#ifdef CONFIG_ZTEST_SHUFFLE -static void z_ztest_shuffle(void *dest[], intptr_t start, size_t num_items, size_t element_size) +#if CONFIG_ZTEST_SHUFFLE +static void z_ztest_shuffle(bool shuffle, void *dest[], intptr_t start, + size_t num_items, size_t element_size) { - void *tmp; - /* Initialize dest array */ for (size_t i = 0; i < num_items; ++i) { dest[i] = (void *)(start + (i * element_size)); } + void *tmp; /* Shuffle dest array */ - for (size_t i = num_items - 1; i > 0; i--) { - int j = sys_rand32_get() % (i + 1); - - if (i != j) { - tmp = dest[j]; - dest[j] = dest[i]; - dest[i] = tmp; + if (shuffle) { + for (size_t i = num_items - 1; i > 0; i--) { + int j = sys_rand32_get() % (i + 1); + + if (i != j) { + tmp = dest[j]; + dest[j] = dest[i]; + dest[i] = tmp; + } } } } -#endif /* CONFIG_ZTEST_SHUFFLE */ +#endif -static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) +static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite, + bool shuffle, int suite_iter, int case_iter) { struct ztest_unit_test *test = NULL; void *data = NULL; @@ -714,14 +721,13 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) data = suite->setup(); } - for (int i = 0; i < NUM_ITER_PER_TEST; i++) { - fail = 0; - + for (int i = 0; i < case_iter; i++) { #ifdef CONFIG_ZTEST_SHUFFLE struct ztest_unit_test *tests_to_run[ZTEST_TEST_COUNT]; memset(tests_to_run, 0, ZTEST_TEST_COUNT * sizeof(struct ztest_unit_test *)); - z_ztest_shuffle((void **)tests_to_run, (intptr_t)_ztest_unit_test_list_start, + z_ztest_shuffle(shuffle, (void **)tests_to_run, + (intptr_t)_ztest_unit_test_list_start, ZTEST_TEST_COUNT, sizeof(struct ztest_unit_test)); for (size_t j = 0; j < ZTEST_TEST_COUNT; ++j) { test = tests_to_run[j]; @@ -771,7 +777,6 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) } } #endif - if (test_status == ZTEST_STATUS_OK && fail != 0) { test_status = ZTEST_STATUS_HAS_FAILURE; } @@ -786,9 +791,10 @@ static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) return fail; } -int z_ztest_run_test_suite(const char *name) +int z_ztest_run_test_suite(const char *name, bool shuffle, int suite_iter, int case_iter) { - return z_ztest_run_test_suite_ptr(ztest_find_test_suite(name)); + return z_ztest_run_test_suite_ptr(ztest_find_test_suite(name), + shuffle, suite_iter, case_iter); } #ifdef CONFIG_USERSPACE @@ -942,14 +948,15 @@ static void __ztest_show_suite_summary(void) flush_log(); } -static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *state) +static int __ztest_run_test_suite(struct ztest_suite_node *ptr, + const void *state, bool shuffle, int suite_iter, int case_iter) { struct ztest_suite_stats *stats = ptr->stats; int count = 0; - for (int i = 0; i < NUM_ITER_PER_SUITE; i++) { + for (int i = 0; i < suite_iter; i++) { if (ztest_api.should_suite_run(state, ptr)) { - int fail = z_ztest_run_test_suite_ptr(ptr); + int fail = z_ztest_run_test_suite_ptr(ptr, shuffle, suite_iter, case_iter); count++; stats->run_count++; @@ -962,7 +969,7 @@ static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *stat return count; } -int z_impl_ztest_run_test_suites(const void *state) +int z_impl_ztest_run_test_suites(const void *state, bool shuffle, int suite_iter, int case_iter) { int count = 0; @@ -974,31 +981,32 @@ int z_impl_ztest_run_test_suites(const void *state) struct ztest_suite_node *suites_to_run[ZTEST_SUITE_COUNT]; memset(suites_to_run, 0, ZTEST_SUITE_COUNT * sizeof(struct ztest_suite_node *)); - z_ztest_shuffle((void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start, + z_ztest_shuffle(shuffle, (void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start, ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node)); for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) { __ztest_init_unit_test_result_for_suite(suites_to_run[i]); } for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) { - count += __ztest_run_test_suite(suites_to_run[i], state); + count += __ztest_run_test_suite(suites_to_run[i], + state, shuffle, suite_iter, case_iter); /* Stop running tests if we have a critical error or if we have a failure and * FAIL_FAST was set */ if (test_status == ZTEST_STATUS_CRITICAL_ERROR || - (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { + (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { break; } } #else for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start; - ptr < _ztest_suite_node_list_end; ++ptr) { + ptr < _ztest_suite_node_list_end; ++ptr) { __ztest_init_unit_test_result_for_suite(ptr); - count += __ztest_run_test_suite(ptr, state); + count += __ztest_run_test_suite(ptr, state, shuffle, suite_iter, case_iter); /* Stop running tests if we have a critical error or if we have a failure and * FAIL_FAST was set */ if (test_status == ZTEST_STATUS_CRITICAL_ERROR || - (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { + (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { break; } } @@ -1070,12 +1078,18 @@ void ztest_verify_all_test_suites_ran(void) } } -void ztest_run_all(const void *state) { ztest_api.run_all(state); } +void ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter) +{ + ztest_api.run_all(state, shuffle, suite_iter, case_iter); +} void __weak test_main(void) { - ztest_run_all(NULL); - +#if CONFIG_ZTEST_SHUFFLE + ztest_run_all(NULL, true, NUM_ITER_PER_SUITE, NUM_ITER_PER_TEST); +#else + ztest_run_all(NULL, false, NUM_ITER_PER_SUITE, NUM_ITER_PER_TEST); +#endif ztest_verify_all_test_suites_ran(); } @@ -1100,6 +1114,179 @@ int main(void) return test_status; } #else + +/* Shell */ + +#ifdef CONFIG_ZTEST_SHELL +static int cmd_list_suites(const struct shell *sh, size_t argc, char **argv) +{ + struct ztest_suite_node *suite; + + for (suite = _ztest_suite_node_list_start; suite < _ztest_suite_node_list_end; ++suite) { + shell_print(sh, "%s", suite->name); + } + return 0; +} + +static int cmd_list_cases(const struct shell *sh, size_t argc, char **argv) +{ + struct ztest_suite_node *ptr; + struct ztest_unit_test *test = NULL; + int test_count = 0; + + for (ptr = _ztest_suite_node_list_start; ptr < _ztest_suite_node_list_end; ++ptr) { + test = NULL; + while ((test = z_ztest_get_next_test(ptr->name, test)) != NULL) { + shell_print(sh, "%s::%s", test->test_suite_name, test->name); + test_count++; + } + } + return 0; +} +extern void ztest_set_test_args(char *argv); +extern void ztest_reset_test_args(void); + +static int cmd_runall(const struct shell *sh, size_t argc, char **argv) +{ + ztest_reset_test_args(); + ztest_run_all(NULL, false, 1, 1); + end_report(); + return 0; +} + +#ifdef CONFIG_ZTEST_SHUFFLE +static int cmd_shuffle(const struct shell *sh, size_t argc, char **argv) +{ + + struct getopt_state *state; + int opt; + static struct option long_options[] = {{"suite_iter", required_argument, 0, 's'}, + {"case_iter", required_argument, 0, 'c'}, + {0, 0, 0, 0}}; + int opt_index = 0; + int val; + int opt_num = 0; + + int suite_iter = 1; + int case_iter = 1; + + while ((opt = getopt_long(argc, argv, "s:c:", long_options, &opt_index)) != -1) { + state = getopt_state_get(); + switch (opt) { + case 's': + val = atoi(state->optarg); + if (val < 1) { + shell_fprintf(sh, SHELL_ERROR, + "Invalid number of suite interations\n"); + return -ENOEXEC; + } + suite_iter = val; + opt_num++; + break; + case 'c': + val = atoi(state->optarg); + if (val < 1) { + shell_fprintf(sh, SHELL_ERROR, + "Invalid number of case interations\n"); + return -ENOEXEC; + } + case_iter = val; + opt_num++; + break; + default: + shell_fprintf(sh, SHELL_ERROR, + "Invalid option or option usage: %s\n", argv[opt_index + 1]); + return -ENOEXEC; + } + } + ztest_reset_test_args(); + ztest_run_all(NULL, true, suite_iter, case_iter); + end_report(); + return 0; +} +#endif + +static int cmd_run_suite(const struct shell *sh, size_t argc, char **argv) +{ + int count = 0; + bool shuffle = false; + + ztest_set_test_args(argv[1]); + + for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start; + ptr < _ztest_suite_node_list_end; ++ptr) { + __ztest_init_unit_test_result_for_suite(ptr); + count += __ztest_run_test_suite(ptr, NULL, shuffle, 1, 1); + if (test_status == ZTEST_STATUS_CRITICAL_ERROR || + (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { + break; + } + } + return 0; +} + +static void testsuite_list_get(size_t idx, struct shell_static_entry *entry); + +SHELL_DYNAMIC_CMD_CREATE(testsuite_names, testsuite_list_get); + +static size_t testsuite_get_all_static(struct ztest_suite_node const **suites) +{ + *suites = _ztest_suite_node_list_start; + return _ztest_suite_node_list_end - _ztest_suite_node_list_start; +} + +static const struct ztest_suite_node *suite_lookup(size_t idx, const char *prefix) +{ + size_t match_idx = 0; + const struct ztest_suite_node *suite; + size_t len = testsuite_get_all_static(&suite); + const struct ztest_suite_node *suite_end = suite + len; + + while (suite < suite_end) { + if ((suite->name != NULL) && (strlen(suite->name) != 0) && + ((prefix == NULL) || + (strncmp(prefix, suite->name, strlen(prefix)) == 0))) { + if (match_idx == idx) { + return suite; + } + ++match_idx; + } + ++suite; + } + + return NULL; +} + +static void testsuite_list_get(size_t idx, struct shell_static_entry *entry) +{ + const struct ztest_suite_node *suite = suite_lookup(idx, ""); + + entry->syntax = (suite != NULL) ? suite->name : NULL; + entry->handler = NULL; + entry->help = NULL; + entry->subcmd = NULL; +} + + SHELL_STATIC_SUBCMD_SET_CREATE( + sub_ztest_cmds, + SHELL_CMD_ARG(run-all, NULL, "Run all tests", cmd_runall, 0, 0), +#ifdef CONFIG_ZTEST_SHUFFLE + SHELL_COND_CMD_ARG(CONFIG_ZTEST_SHUFFLE, shuffle, NULL, + "Shuffle tests", cmd_shuffle, 0, 2), +#endif + SHELL_CMD_ARG(list-testsuites, NULL, + "List all test suites", cmd_list_suites, 0, 0), + SHELL_CMD_ARG(list-testcases, NULL, + "List all test cases", cmd_list_cases, 0, 0), + SHELL_CMD_ARG(run-testsuite, &testsuite_names, + "Run test suite", cmd_run_suite, 2, 0), + SHELL_CMD_ARG(run-testcase, NULL, "Run testcase", cmd_run_suite, 2, 0), + SHELL_SUBCMD_SET_END /* Array terminated. */ + ); + +SHELL_CMD_REGISTER(ztest, &sub_ztest_cmds, "Ztest commands", NULL); +#endif /* CONFIG_ZTEST_SHELL */ + int main(void) { #ifdef CONFIG_USERSPACE @@ -1116,6 +1303,7 @@ int main(void) #endif /* CONFIG_USERSPACE */ z_init_mock(); +#ifndef CONFIG_ZTEST_SHELL test_main(); end_report(); flush_log(); @@ -1152,7 +1340,8 @@ int main(void) ; /* Spin */ } irq_unlock(key); -#endif +#endif /* CONFIG_ZTEST_NO_YIELD */ +#endif /* CONFIG_ZTEST_SHELL */ return 0; } #endif diff --git a/subsys/testsuite/ztest/src/ztest_defaults.c b/subsys/testsuite/ztest/src/ztest_defaults.c index eaf59c8a096c..12569d4126a8 100644 --- a/subsys/testsuite/ztest/src/ztest_defaults.c +++ b/subsys/testsuite/ztest/src/ztest_defaults.c @@ -27,9 +27,9 @@ const char *ztest_relative_filename(const char *file) * * @param state The current state of the machine as it relates to the test executable. */ -void z_ztest_run_all(const void *state) +void z_ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter) { - ztest_run_test_suites(state); + ztest_run_test_suites(state, shuffle, suite_iter, case_iter); } /** diff --git a/subsys/testsuite/ztest/src/ztest_new.c b/subsys/testsuite/ztest/src/ztest_new.c deleted file mode 100644 index d49a1ef9c110..000000000000 --- a/subsys/testsuite/ztest/src/ztest_new.c +++ /dev/null @@ -1,1158 +0,0 @@ -/* - * Copyright (c) 2016 Intel Corporation - * - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include -#ifdef CONFIG_USERSPACE -#include -#endif -#include -#include - -#ifdef KERNEL -static struct k_thread ztest_thread; -#endif -static bool failed_expectation; - -#ifdef CONFIG_ZTEST_SHUFFLE -#include -#include - -#include -#define NUM_ITER_PER_SUITE CONFIG_ZTEST_SHUFFLE_SUITE_REPEAT_COUNT -#define NUM_ITER_PER_TEST CONFIG_ZTEST_SHUFFLE_TEST_REPEAT_COUNT -#else -#define NUM_ITER_PER_SUITE 1 -#define NUM_ITER_PER_TEST 1 -#endif - -/* ZTEST_DMEM and ZTEST_BMEM are used for the application shared memory test */ - -/** - * @brief The current status of the test binary - */ -enum ztest_status { - ZTEST_STATUS_OK, - ZTEST_STATUS_HAS_FAILURE, - ZTEST_STATUS_CRITICAL_ERROR -}; - -/** - * @brief Tracks the current phase that ztest is operating in. - */ -ZTEST_DMEM enum ztest_phase cur_phase = TEST_PHASE_FRAMEWORK; - -static ZTEST_BMEM enum ztest_status test_status = ZTEST_STATUS_OK; - -extern ZTEST_DMEM const struct ztest_arch_api ztest_api; - -static void __ztest_show_suite_summary(void); - -static void end_report(void) -{ - __ztest_show_suite_summary(); - if (test_status) { - TC_END_REPORT(TC_FAIL); - } else { - TC_END_REPORT(TC_PASS); - } -} - -static int cleanup_test(struct ztest_unit_test *test) -{ - int ret = TC_PASS; - int mock_status; - - mock_status = z_cleanup_mock(); - -#ifdef KERNEL - /* we need to remove the ztest_thread information from the timeout_q. - * Because we reuse the same k_thread structure this would - * causes some problems. - */ - if (IS_ENABLED(CONFIG_MULTITHREADING)) { - k_thread_abort(&ztest_thread); - } -#endif - - if (!ret && mock_status == 1) { - PRINT("Test %s failed: Unused mock parameter values\n", test->name); - ret = TC_FAIL; - } else if (!ret && mock_status == 2) { - PRINT("Test %s failed: Unused mock return values\n", test->name); - ret = TC_FAIL; - } else { - ; - } - - return ret; -} - -#ifdef KERNEL - -#if defined(CONFIG_SMP) && (CONFIG_MP_MAX_NUM_CPUS > 1) -#define MAX_NUM_CPUHOLD (CONFIG_MP_MAX_NUM_CPUS - 1) -#define CPUHOLD_STACK_SZ (512 + CONFIG_TEST_EXTRA_STACK_SIZE) -static struct k_thread cpuhold_threads[MAX_NUM_CPUHOLD]; -K_KERNEL_STACK_ARRAY_DEFINE(cpuhold_stacks, MAX_NUM_CPUHOLD, CPUHOLD_STACK_SZ); - -static struct k_sem cpuhold_sem; -volatile int cpuhold_active; - -/* "Holds" a CPU for use with the "1cpu" test cases. Note that we - * can't use tools like the cpumask feature because we have tests that - * may need to control that configuration themselves. We do this at - * the lowest level, but locking interrupts directly and spinning. - */ -static void cpu_hold(void *arg1, void *arg2, void *arg3) -{ - ARG_UNUSED(arg1); - ARG_UNUSED(arg2); - ARG_UNUSED(arg3); - - unsigned int key = arch_irq_lock(); - uint32_t dt, start_ms = k_uptime_get_32(); - - k_sem_give(&cpuhold_sem); - -#if (defined(CONFIG_ARM64) || defined(CONFIG_RISCV)) && defined(CONFIG_FPU_SHARING) - /* - * We'll be spinning with IRQs disabled. The flush-your-FPU request - * IPI will never be serviced during that time. Therefore we flush - * the FPU preemptively here to prevent any other CPU waiting after - * this CPU forever and deadlock the system. - */ - k_float_disable(_current_cpu->arch.fpu_owner); -#endif - - while (cpuhold_active) { - k_busy_wait(1000); - } - - /* Holding the CPU via spinning is expensive, and abusing this - * for long-running test cases tends to overload the CI system - * (qemu runs separate CPUs in different threads, but the CI - * logic views it as one "job") and cause other test failures. - */ - dt = k_uptime_get_32() - start_ms; - zassert_true(dt < CONFIG_ZTEST_CPU_HOLD_TIME_MS, - "1cpu test took too long (%d ms)", dt); - arch_irq_unlock(key); -} -#endif /* CONFIG_SMP && (CONFIG_MP_MAX_NUM_CPUS > 1) */ - -void z_impl_z_test_1cpu_start(void) -{ -#if defined(CONFIG_SMP) && (CONFIG_MP_MAX_NUM_CPUS > 1) - unsigned int num_cpus = arch_num_cpus(); - - cpuhold_active = 1; - char tname[CONFIG_THREAD_MAX_NAME_LEN]; - - k_sem_init(&cpuhold_sem, 0, 999); - - /* Spawn N-1 threads to "hold" the other CPUs, waiting for - * each to signal us that it's locked and spinning. - */ - for (int i = 0; i < num_cpus - 1; i++) { - k_thread_create(&cpuhold_threads[i], cpuhold_stacks[i], CPUHOLD_STACK_SZ, - cpu_hold, NULL, NULL, NULL, K_HIGHEST_THREAD_PRIO, - 0, K_NO_WAIT); - if (IS_ENABLED(CONFIG_THREAD_NAME)) { - snprintk(tname, CONFIG_THREAD_MAX_NAME_LEN, "cpuhold%02d", i); - k_thread_name_set(&cpuhold_threads[i], tname); - } - k_sem_take(&cpuhold_sem, K_FOREVER); - } -#endif -} - -void z_impl_z_test_1cpu_stop(void) -{ -#if defined(CONFIG_SMP) && (CONFIG_MP_MAX_NUM_CPUS > 1) - unsigned int num_cpus = arch_num_cpus(); - - cpuhold_active = 0; - - for (int i = 0; i < num_cpus - 1; i++) { - k_thread_abort(&cpuhold_threads[i]); - } -#endif -} - -#ifdef CONFIG_USERSPACE -void z_vrfy_z_test_1cpu_start(void) { z_impl_z_test_1cpu_start(); } -#include - -void z_vrfy_z_test_1cpu_stop(void) { z_impl_z_test_1cpu_stop(); } -#include -#endif /* CONFIG_USERSPACE */ -#endif - -__maybe_unused static void run_test_rules(bool is_before, struct ztest_unit_test *test, void *data) -{ - for (struct ztest_test_rule *rule = _ztest_test_rule_list_start; - rule < _ztest_test_rule_list_end; ++rule) { - if (is_before && rule->before_each) { - rule->before_each(test, data); - } else if (!is_before && rule->after_each) { - rule->after_each(test, data); - } - } -} - -static void run_test_functions(struct ztest_suite_node *suite, struct ztest_unit_test *test, - void *data) -{ - __ztest_set_test_phase(TEST_PHASE_TEST); - test->test(data); -} - -COND_CODE_1(KERNEL, (ZTEST_BMEM), ()) static enum ztest_result test_result; - -static int get_final_test_result(const struct ztest_unit_test *test, int ret) -{ - enum ztest_expected_result expected_result = -1; - - for (struct ztest_expected_result_entry *expectation = - _ztest_expected_result_entry_list_start; - expectation < _ztest_expected_result_entry_list_end; ++expectation) { - if (strcmp(expectation->test_name, test->name) == 0 && - strcmp(expectation->test_suite_name, test->test_suite_name) == 0) { - expected_result = expectation->expected_result; - break; - } - } - - if (expected_result == ZTEST_EXPECTED_RESULT_FAIL) { - /* Expected a failure: - * - If we got a failure, return TC_PASS - * - Otherwise force a failure - */ - return (ret == TC_FAIL) ? TC_PASS : TC_FAIL; - } - if (expected_result == ZTEST_EXPECTED_RESULT_SKIP) { - /* Expected a skip: - * - If we got a skip, return TC_PASS - * - Otherwise force a failure - */ - return (ret == TC_SKIP) ? TC_PASS : TC_FAIL; - } - /* No expectation was made, no change is needed. */ - return ret; -} - -/** - * @brief Get a friendly name string for a given test phrase. - * - * @param phase an enum ztest_phase value describing the desired test phase - * @returns a string name for `phase` - */ -static inline const char *get_friendly_phase_name(enum ztest_phase phase) -{ - switch (phase) { - case TEST_PHASE_SETUP: - return "setup"; - case TEST_PHASE_BEFORE: - return "before"; - case TEST_PHASE_TEST: - return "test"; - case TEST_PHASE_AFTER: - return "after"; - case TEST_PHASE_TEARDOWN: - return "teardown"; - case TEST_PHASE_FRAMEWORK: - return "framework"; - default: - return "(unknown)"; - } -} - -static bool current_test_failed_assumption; -void ztest_skip_failed_assumption(void) -{ - if (IS_ENABLED(CONFIG_ZTEST_FAIL_ON_ASSUME)) { - current_test_failed_assumption = true; - } - ztest_test_skip(); -} - -#ifndef KERNEL - -/* Static code analysis tool can raise a violation that the standard header - * shall not be used. - * - * setjmp is using in a test code, not in a runtime code, it is acceptable. - * It is a deliberate deviation. - */ -#include /* parasoft-suppress MISRAC2012-RULE_21_4-a MISRAC2012-RULE_21_4-b*/ -#include -#include -#include - -#define FAIL_FAST 0 - -static jmp_buf test_fail; -static jmp_buf test_pass; -static jmp_buf test_skip; -static jmp_buf stack_fail; -static jmp_buf test_suite_fail; - -void ztest_test_fail(void) -{ - switch (cur_phase) { - case TEST_PHASE_SETUP: - PRINT(" at %s function\n", get_friendly_phase_name(cur_phase)); - longjmp(test_suite_fail, 1); - case TEST_PHASE_BEFORE: - case TEST_PHASE_TEST: - PRINT(" at %s function\n", get_friendly_phase_name(cur_phase)); - longjmp(test_fail, 1); - case TEST_PHASE_AFTER: - case TEST_PHASE_TEARDOWN: - case TEST_PHASE_FRAMEWORK: - PRINT(" ERROR: cannot fail in test phase '%s()', bailing\n", - get_friendly_phase_name(cur_phase)); - longjmp(stack_fail, 1); - } -} - -void ztest_test_pass(void) -{ - if (cur_phase == TEST_PHASE_TEST) { - longjmp(test_pass, 1); - } - PRINT(" ERROR: cannot pass in test phase '%s()', bailing\n", - get_friendly_phase_name(cur_phase)); - longjmp(stack_fail, 1); -} - -void ztest_test_skip(void) -{ - switch (cur_phase) { - case TEST_PHASE_SETUP: - case TEST_PHASE_BEFORE: - case TEST_PHASE_TEST: - longjmp(test_skip, 1); - default: - PRINT(" ERROR: cannot skip in test phase '%s()', bailing\n", - get_friendly_phase_name(cur_phase)); - longjmp(stack_fail, 1); - } -} - -void ztest_test_expect_fail(void) -{ - failed_expectation = true; - - switch (cur_phase) { - case TEST_PHASE_SETUP: - PRINT(" at %s function\n", get_friendly_phase_name(cur_phase)); - break; - case TEST_PHASE_BEFORE: - case TEST_PHASE_TEST: - PRINT(" at %s function\n", get_friendly_phase_name(cur_phase)); - break; - case TEST_PHASE_AFTER: - case TEST_PHASE_TEARDOWN: - case TEST_PHASE_FRAMEWORK: - PRINT(" ERROR: cannot fail in test phase '%s()', bailing\n", - get_friendly_phase_name(cur_phase)); - longjmp(stack_fail, 1); - } -} - -static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test, void *data) -{ - int ret = TC_PASS; - - TC_START(test->name); - __ztest_set_test_phase(TEST_PHASE_BEFORE); - - if (test_result == ZTEST_RESULT_SUITE_FAIL) { - ret = TC_FAIL; - goto out; - } - - if (setjmp(test_fail)) { - ret = TC_FAIL; - goto out; - } - - if (setjmp(test_pass)) { - ret = TC_PASS; - goto out; - } - - if (setjmp(test_skip)) { - ret = TC_SKIP; - goto out; - } - - run_test_rules(/*is_before=*/true, test, data); - if (suite->before) { - suite->before(data); - } - run_test_functions(suite, test, data); -out: - if (failed_expectation) { - failed_expectation = false; - ret = TC_FAIL; - } - - __ztest_set_test_phase(TEST_PHASE_AFTER); - if (test_result != ZTEST_RESULT_SUITE_FAIL) { - if (suite->after != NULL) { - suite->after(data); - } - run_test_rules(/*is_before=*/false, test, data); - } - __ztest_set_test_phase(TEST_PHASE_FRAMEWORK); - ret |= cleanup_test(test); - - ret = get_final_test_result(test, ret); - Z_TC_END_RESULT(ret, test->name); - if (ret == TC_SKIP && current_test_failed_assumption) { - test_status = 1; - } - - return ret; -} - -#else /* KERNEL */ - -/* Zephyr's probably going to cause all tests to fail if one test fails, so - * skip the rest of tests if one of them fails - */ -#ifdef CONFIG_ZTEST_FAIL_FAST -#define FAIL_FAST 1 -#else -#define FAIL_FAST 0 -#endif - -K_THREAD_STACK_DEFINE(ztest_thread_stack, CONFIG_ZTEST_STACK_SIZE + CONFIG_TEST_EXTRA_STACK_SIZE); - -static void test_finalize(void) -{ - if (IS_ENABLED(CONFIG_MULTITHREADING)) { - k_thread_abort(&ztest_thread); - if (k_is_in_isr()) { - return; - } - - k_thread_abort(k_current_get()); - CODE_UNREACHABLE; - } -} - -void ztest_test_fail(void) -{ - switch (cur_phase) { - case TEST_PHASE_SETUP: - __ztest_set_test_result(ZTEST_RESULT_SUITE_FAIL); - break; - case TEST_PHASE_BEFORE: - case TEST_PHASE_TEST: - __ztest_set_test_result(ZTEST_RESULT_FAIL); - test_finalize(); - break; - default: - PRINT(" ERROR: cannot fail in test phase '%s()', bailing\n", - get_friendly_phase_name(cur_phase)); - test_status = ZTEST_STATUS_CRITICAL_ERROR; - break; - } -} - -void ztest_test_pass(void) -{ - switch (cur_phase) { - case TEST_PHASE_TEST: - __ztest_set_test_result(ZTEST_RESULT_PASS); - test_finalize(); - break; - default: - PRINT(" ERROR: cannot pass in test phase '%s()', bailing\n", - get_friendly_phase_name(cur_phase)); - test_status = ZTEST_STATUS_CRITICAL_ERROR; - if (cur_phase == TEST_PHASE_BEFORE) { - test_finalize(); - } - } -} - -void ztest_test_skip(void) -{ - switch (cur_phase) { - case TEST_PHASE_SETUP: - __ztest_set_test_result(ZTEST_RESULT_SUITE_SKIP); - break; - case TEST_PHASE_BEFORE: - case TEST_PHASE_TEST: - __ztest_set_test_result(ZTEST_RESULT_SKIP); - test_finalize(); - break; - default: - PRINT(" ERROR: cannot skip in test phase '%s()', bailing\n", - get_friendly_phase_name(cur_phase)); - test_status = ZTEST_STATUS_CRITICAL_ERROR; - break; - } -} - -void ztest_test_expect_fail(void) -{ - failed_expectation = true; -} - -void ztest_simple_1cpu_before(void *data) -{ - ARG_UNUSED(data); - z_test_1cpu_start(); -} - -void ztest_simple_1cpu_after(void *data) -{ - ARG_UNUSED(data); - z_test_1cpu_stop(); -} - -static void test_cb(void *a, void *b, void *c) -{ - struct ztest_suite_node *suite = a; - struct ztest_unit_test *test = b; - const bool config_user_mode = FIELD_GET(K_USER, test->thread_options) != 0; - - if (!IS_ENABLED(CONFIG_USERSPACE) || !k_is_user_context()) { - __ztest_set_test_result(ZTEST_RESULT_PENDING); - run_test_rules(/*is_before=*/true, test, /*data=*/c); - if (suite->before) { - suite->before(/*data=*/c); - } - if (IS_ENABLED(CONFIG_USERSPACE) && config_user_mode) { - k_thread_user_mode_enter(test_cb, a, b, c); - } - } - run_test_functions(suite, test, c); - __ztest_set_test_result(ZTEST_RESULT_PASS); -} - -static int run_test(struct ztest_suite_node *suite, struct ztest_unit_test *test, void *data) -{ - int ret = TC_PASS; - -#if CONFIG_ZTEST_TEST_DELAY_MS > 0 - k_busy_wait(CONFIG_ZTEST_TEST_DELAY_MS * USEC_PER_MSEC); -#endif - TC_START(test->name); - - __ztest_set_test_phase(TEST_PHASE_BEFORE); - - /* If the suite's setup function marked us as skipped, don't bother - * running the tests. - */ - if (IS_ENABLED(CONFIG_MULTITHREADING)) { - get_start_time_cyc(); - k_thread_create(&ztest_thread, ztest_thread_stack, - K_THREAD_STACK_SIZEOF(ztest_thread_stack), - test_cb, suite, test, data, - CONFIG_ZTEST_THREAD_PRIORITY, - K_INHERIT_PERMS, K_FOREVER); - - k_thread_access_grant(&ztest_thread, suite, test, suite->stats); - if (test->name != NULL) { - k_thread_name_set(&ztest_thread, test->name); - } - /* Only start the thread if we're not skipping the suite */ - if (test_result != ZTEST_RESULT_SUITE_SKIP && - test_result != ZTEST_RESULT_SUITE_FAIL) { - k_thread_start(&ztest_thread); - k_thread_join(&ztest_thread, K_FOREVER); - } - } else if (test_result != ZTEST_RESULT_SUITE_SKIP && - test_result != ZTEST_RESULT_SUITE_FAIL) { - __ztest_set_test_result(ZTEST_RESULT_PENDING); - get_start_time_cyc(); - run_test_rules(/*is_before=*/true, test, data); - if (suite->before) { - suite->before(data); - } - run_test_functions(suite, test, data); - } - - __ztest_set_test_phase(TEST_PHASE_AFTER); - if (suite->after != NULL) { - suite->after(data); - } - run_test_rules(/*is_before=*/false, test, data); - - get_test_duration_ms(); - if (tc_spend_time > test->stats->duration_worst_ms) { - test->stats->duration_worst_ms = tc_spend_time; - } - - __ztest_set_test_phase(TEST_PHASE_FRAMEWORK); - - /* Flush all logs in case deferred mode and default logging thread are used. */ - while (IS_ENABLED(CONFIG_TEST_LOGGING_FLUSH_AFTER_TEST) && - IS_ENABLED(CONFIG_LOG_PROCESS_THREAD) && log_data_pending()) { - k_msleep(100); - } - - if (test_result == ZTEST_RESULT_FAIL || test_result == ZTEST_RESULT_SUITE_FAIL || - failed_expectation) { - ret = TC_FAIL; - failed_expectation = false; - } else if (test_result == ZTEST_RESULT_SKIP || test_result == ZTEST_RESULT_SUITE_SKIP) { - ret = TC_SKIP; - } - - if (test_result == ZTEST_RESULT_PASS || !FAIL_FAST) { - ret |= cleanup_test(test); - } - - ret = get_final_test_result(test, ret); - Z_TC_END_RESULT(ret, test->name); - if (ret == TC_SKIP && current_test_failed_assumption) { - test_status = 1; - } - - return ret; -} - -#endif /* !KERNEL */ - -static struct ztest_suite_node *ztest_find_test_suite(const char *name) -{ - struct ztest_suite_node *node; - - for (node = _ztest_suite_node_list_start; node < _ztest_suite_node_list_end; ++node) { - if (strcmp(name, node->name) == 0) { - return node; - } - } - - return NULL; -} - -struct ztest_unit_test *z_ztest_get_next_test(const char *suite, struct ztest_unit_test *prev) -{ - struct ztest_unit_test *test = (prev == NULL) ? _ztest_unit_test_list_start : prev + 1; - - for (; test < _ztest_unit_test_list_end; ++test) { - if (strcmp(suite, test->test_suite_name) == 0) { - return test; - } - } - return NULL; -} - -#ifdef CONFIG_ZTEST_SHUFFLE -static void z_ztest_shuffle(void *dest[], intptr_t start, size_t num_items, size_t element_size) -{ - void *tmp; - - /* Initialize dest array */ - for (size_t i = 0; i < num_items; ++i) { - dest[i] = (void *)(start + (i * element_size)); - } - - /* Shuffle dest array */ - for (size_t i = num_items - 1; i > 0; i--) { - int j = sys_rand32_get() % (i + 1); - - if (i != j) { - tmp = dest[j]; - dest[j] = dest[i]; - dest[i] = tmp; - } - } -} -#endif /* CONFIG_ZTEST_SHUFFLE */ - -static int z_ztest_run_test_suite_ptr(struct ztest_suite_node *suite) -{ - struct ztest_unit_test *test = NULL; - void *data = NULL; - int fail = 0; - int tc_result = TC_PASS; - - if (FAIL_FAST && test_status != ZTEST_STATUS_OK) { - return test_status; - } - - if (suite == NULL) { - test_status = ZTEST_STATUS_CRITICAL_ERROR; - return -1; - } - -#ifndef KERNEL - if (setjmp(stack_fail)) { - PRINT("TESTSUITE crashed.\n"); - test_status = ZTEST_STATUS_CRITICAL_ERROR; - end_report(); - exit(1); - } -#else - k_object_access_all_grant(&ztest_thread); -#endif - - TC_SUITE_START(suite->name); - current_test_failed_assumption = false; - __ztest_set_test_result(ZTEST_RESULT_PENDING); - __ztest_set_test_phase(TEST_PHASE_SETUP); -#ifndef KERNEL - if (setjmp(test_suite_fail)) { - __ztest_set_test_result(ZTEST_RESULT_SUITE_FAIL); - } -#endif - if (test_result != ZTEST_RESULT_SUITE_FAIL && suite->setup != NULL) { - data = suite->setup(); - } - - for (int i = 0; i < NUM_ITER_PER_TEST; i++) { - fail = 0; - -#ifdef CONFIG_ZTEST_SHUFFLE - struct ztest_unit_test *tests_to_run[ZTEST_TEST_COUNT]; - - memset(tests_to_run, 0, ZTEST_TEST_COUNT * sizeof(struct ztest_unit_test *)); - z_ztest_shuffle((void **)tests_to_run, (intptr_t)_ztest_unit_test_list_start, - ZTEST_TEST_COUNT, sizeof(struct ztest_unit_test)); - for (size_t j = 0; j < ZTEST_TEST_COUNT; ++j) { - test = tests_to_run[j]; - /* Make sure that the test belongs to this suite */ - if (strcmp(suite->name, test->test_suite_name) != 0) { - continue; - } - if (ztest_api.should_test_run(suite->name, test->name)) { - test->stats->run_count++; - tc_result = run_test(suite, test, data); - if (tc_result == TC_PASS) { - test->stats->pass_count++; - } else if (tc_result == TC_SKIP) { - test->stats->skip_count++; - } else if (tc_result == TC_FAIL) { - test->stats->fail_count++; - } - if (tc_result == TC_FAIL) { - fail++; - } - } - - if ((fail && FAIL_FAST) || test_status == ZTEST_STATUS_CRITICAL_ERROR) { - break; - } - } -#else - while (((test = z_ztest_get_next_test(suite->name, test)) != NULL)) { - if (ztest_api.should_test_run(suite->name, test->name)) { - test->stats->run_count++; - tc_result = run_test(suite, test, data); - if (tc_result == TC_PASS) { - test->stats->pass_count++; - } else if (tc_result == TC_SKIP) { - test->stats->skip_count++; - } else if (tc_result == TC_FAIL) { - test->stats->fail_count++; - } - - if (tc_result == TC_FAIL) { - fail++; - } - } - - if ((fail && FAIL_FAST) || test_status == ZTEST_STATUS_CRITICAL_ERROR) { - break; - } - } -#endif - - if (test_status == ZTEST_STATUS_OK && fail != 0) { - test_status = ZTEST_STATUS_HAS_FAILURE; - } - } - - TC_SUITE_END(suite->name, (fail > 0 ? TC_FAIL : TC_PASS)); - __ztest_set_test_phase(TEST_PHASE_TEARDOWN); - if (suite->teardown != NULL) { - suite->teardown(data); - } - - return fail; -} - -int z_ztest_run_test_suite(const char *name) -{ - return z_ztest_run_test_suite_ptr(ztest_find_test_suite(name)); -} - -#ifdef CONFIG_USERSPACE -K_APPMEM_PARTITION_DEFINE(ztest_mem_partition); -#endif - -static void __ztest_init_unit_test_result_for_suite(struct ztest_suite_node *suite) -{ - struct ztest_unit_test *test = NULL; - - while (((test = z_ztest_get_next_test(suite->name, test)) != NULL)) { - test->stats->run_count = 0; - test->stats->skip_count = 0; - test->stats->fail_count = 0; - test->stats->pass_count = 0; - test->stats->duration_worst_ms = 0; - } -} - -static void flush_log(void) -{ - if (IS_ENABLED(CONFIG_LOG_PROCESS_THREAD)) { - while (log_data_pending()) { - k_sleep(K_MSEC(10)); - } - k_sleep(K_MSEC(10)); - } else { - while (LOG_PROCESS()) { - } - } -} - -/* Show one line summary for a test suite. - */ -static void __ztest_show_suite_summary_oneline(struct ztest_suite_node *suite) -{ - int distinct_pass = 0, distinct_fail = 0, distinct_skip = 0, distinct_total = 0; - int effective_total = 0; - int expanded_pass = 0, expanded_passrate = 0; - int passrate_major = 0, passrate_minor = 0, passrate_tail = 0; - int suite_result = TC_PASS; - - struct ztest_unit_test *test = NULL; - unsigned int suite_duration_worst_ms = 0; - - /** summary of disctinct run */ - while (((test = z_ztest_get_next_test(suite->name, test)) != NULL)) { - distinct_total++; - suite_duration_worst_ms += test->stats->duration_worst_ms; - if (test->stats->skip_count == test->stats->run_count) { - distinct_skip++; - } else if (test->stats->pass_count == test->stats->run_count) { - distinct_pass++; - } else { - distinct_fail++; - } - } - - if (distinct_skip == distinct_total) { - suite_result = TC_SKIP; - passrate_major = passrate_minor = 0; - } else { - suite_result = (distinct_fail > 0) ? TC_FAIL : TC_PASS; - effective_total = distinct_total - distinct_skip; - expanded_pass = distinct_pass * 100000; - expanded_passrate = expanded_pass / effective_total; - passrate_major = expanded_passrate / 1000; - passrate_minor = (expanded_passrate - passrate_major * 1000) / 10; - passrate_tail = expanded_passrate - passrate_major * 1000 - passrate_minor * 10; - if (passrate_tail >= 5) { /* rounding */ - passrate_minor++; - } - } - - TC_SUMMARY_PRINT("SUITE %s - %3d.%02d%% [%s]: pass = %d, fail = %d, " - "skip = %d, total = %d duration = %u.%03u seconds\n", - TC_RESULT_TO_STR(suite_result), - passrate_major, passrate_minor, - suite->name, distinct_pass, distinct_fail, - distinct_skip, distinct_total, - suite_duration_worst_ms / 1000, suite_duration_worst_ms % 1000); - flush_log(); -} - -static void __ztest_show_suite_summary_verbose(struct ztest_suite_node *suite) -{ - struct ztest_unit_test *test = NULL; - int tc_result = TC_PASS; - int flush_frequency = 0; - - if (IS_ENABLED(CONFIG_ZTEST_VERBOSE_SUMMARY) == 0) { - return; - } - - while (((test = z_ztest_get_next_test(suite->name, test)) != NULL)) { - if (test->stats->skip_count == test->stats->run_count) { - tc_result = TC_SKIP; - } else if (test->stats->pass_count == test->stats->run_count) { - tc_result = TC_PASS; - } else if (test->stats->pass_count == 0) { - tc_result = TC_FAIL; - } else { - tc_result = TC_FLAKY; - } - - if (tc_result == TC_FLAKY) { - TC_SUMMARY_PRINT(" - %s - [%s.%s] - (Failed %d of %d attempts)" - " - duration = %u.%03u seconds\n", - TC_RESULT_TO_STR(tc_result), - test->test_suite_name, test->name, - test->stats->run_count - test->stats->pass_count, - test->stats->run_count, - test->stats->duration_worst_ms / 1000, - test->stats->duration_worst_ms % 1000); - } else { - TC_SUMMARY_PRINT(" - %s - [%s.%s] duration = %u.%03u seconds\n", - TC_RESULT_TO_STR(tc_result), - test->test_suite_name, test->name, - test->stats->duration_worst_ms / 1000, - test->stats->duration_worst_ms % 1000); - } - - if (flush_frequency % 3 == 0) { - /** Reduce the flush frequencey a bit to speed up the output */ - flush_log(); - } - flush_frequency++; - } - TC_SUMMARY_PRINT("\n"); - flush_log(); -} - -static void __ztest_show_suite_summary(void) -{ - if (IS_ENABLED(CONFIG_ZTEST_SUMMARY) == 0) { - return; - } - /* Flush the log a lot to ensure that no summary content - * is dropped if it goes through the logging subsystem. - */ - flush_log(); - TC_SUMMARY_PRINT("\n------ TESTSUITE SUMMARY START ------\n\n"); - flush_log(); - for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start; - ptr < _ztest_suite_node_list_end; ++ptr) { - - __ztest_show_suite_summary_oneline(ptr); - __ztest_show_suite_summary_verbose(ptr); - } - TC_SUMMARY_PRINT("------ TESTSUITE SUMMARY END ------\n\n"); - flush_log(); -} - -static int __ztest_run_test_suite(struct ztest_suite_node *ptr, const void *state) -{ - struct ztest_suite_stats *stats = ptr->stats; - int count = 0; - - for (int i = 0; i < NUM_ITER_PER_SUITE; i++) { - if (ztest_api.should_suite_run(state, ptr)) { - int fail = z_ztest_run_test_suite_ptr(ptr); - - count++; - stats->run_count++; - stats->fail_count += (fail != 0) ? 1 : 0; - } else { - stats->skip_count++; - } - } - - return count; -} - -int z_impl_ztest_run_test_suites(const void *state) -{ - int count = 0; - - if (test_status == ZTEST_STATUS_CRITICAL_ERROR) { - return count; - } - -#ifdef CONFIG_ZTEST_SHUFFLE - struct ztest_suite_node *suites_to_run[ZTEST_SUITE_COUNT]; - - memset(suites_to_run, 0, ZTEST_SUITE_COUNT * sizeof(struct ztest_suite_node *)); - z_ztest_shuffle((void **)suites_to_run, (intptr_t)_ztest_suite_node_list_start, - ZTEST_SUITE_COUNT, sizeof(struct ztest_suite_node)); - for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) { - __ztest_init_unit_test_result_for_suite(suites_to_run[i]); - } - for (size_t i = 0; i < ZTEST_SUITE_COUNT; ++i) { - count += __ztest_run_test_suite(suites_to_run[i], state); - /* Stop running tests if we have a critical error or if we have a failure and - * FAIL_FAST was set - */ - if (test_status == ZTEST_STATUS_CRITICAL_ERROR || - (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { - break; - } - } -#else - for (struct ztest_suite_node *ptr = _ztest_suite_node_list_start; - ptr < _ztest_suite_node_list_end; ++ptr) { - __ztest_init_unit_test_result_for_suite(ptr); - count += __ztest_run_test_suite(ptr, state); - /* Stop running tests if we have a critical error or if we have a failure and - * FAIL_FAST was set - */ - if (test_status == ZTEST_STATUS_CRITICAL_ERROR || - (test_status == ZTEST_STATUS_HAS_FAILURE && FAIL_FAST)) { - break; - } - } -#endif - - return count; -} - -void z_impl___ztest_set_test_result(enum ztest_result new_result) -{ - test_result = new_result; -} - -void z_impl___ztest_set_test_phase(enum ztest_phase new_phase) -{ - cur_phase = new_phase; -} - -#ifdef CONFIG_USERSPACE -void z_vrfy___ztest_set_test_result(enum ztest_result new_result) -{ - z_impl___ztest_set_test_result(new_result); -} -#include - -void z_vrfy___ztest_set_test_phase(enum ztest_phase new_phase) -{ - z_impl___ztest_set_test_phase(new_phase); -} -#include -#endif /* CONFIG_USERSPACE */ - -void ztest_verify_all_test_suites_ran(void) -{ - bool all_tests_run = true; - struct ztest_suite_node *suite; - struct ztest_unit_test *test; - - if (IS_ENABLED(CONFIG_ZTEST_VERIFY_RUN_ALL)) { - for (suite = _ztest_suite_node_list_start; suite < _ztest_suite_node_list_end; - ++suite) { - if (suite->stats->run_count < 1) { - PRINT("ERROR: Test suite '%s' did not run.\n", suite->name); - all_tests_run = false; - } - } - - for (test = _ztest_unit_test_list_start; test < _ztest_unit_test_list_end; ++test) { - suite = ztest_find_test_suite(test->test_suite_name); - if (suite == NULL) { - PRINT("ERROR: Test '%s' assigned to test suite '%s' which doesn't " - "exist\n", - test->name, test->test_suite_name); - all_tests_run = false; - } - } - - if (!all_tests_run) { - test_status = ZTEST_STATUS_HAS_FAILURE; - } - } - - for (test = _ztest_unit_test_list_start; test < _ztest_unit_test_list_end; ++test) { - if (test->stats->fail_count + test->stats->pass_count + test->stats->skip_count != - test->stats->run_count) { - PRINT("Bad stats for %s.%s\n", test->test_suite_name, test->name); - test_status = 1; - } - } -} - -void ztest_run_all(const void *state) { ztest_api.run_all(state); } - -void __weak test_main(void) -{ - ztest_run_all(NULL); - - ztest_verify_all_test_suites_ran(); -} - -#ifndef KERNEL -int main(void) -{ - z_init_mock(); - test_main(); - end_report(); -#ifdef CONFIG_ZTEST_NO_YIELD - /* - * Rather than yielding to idle thread, keep the part awake so debugger can - * still access it, since some SOCs cannot be debugged in low power states. - */ - uint32_t key = irq_lock(); - - while (1) { - ; /* Spin */ - } - irq_unlock(key); -#endif - return test_status; -} -#else -int main(void) -{ -#ifdef CONFIG_USERSPACE - /* Partition containing globals tagged with ZTEST_DMEM and ZTEST_BMEM - * macros. Any variables that user code may reference need to be - * placed in this partition if no other memory domain configuration - * is made. - */ - k_mem_domain_add_partition(&k_mem_domain_default, &ztest_mem_partition); -#ifdef Z_MALLOC_PARTITION_EXISTS - /* Allow access to malloc() memory */ - k_mem_domain_add_partition(&k_mem_domain_default, &z_malloc_partition); -#endif -#endif /* CONFIG_USERSPACE */ - - z_init_mock(); - test_main(); - end_report(); - flush_log(); - LOG_PANIC(); - if (IS_ENABLED(CONFIG_ZTEST_RETEST_IF_PASSED)) { - static __noinit struct { - uint32_t magic; - uint32_t boots; - } state; - const uint32_t magic = 0x152ac523; - - if (state.magic != magic) { - state.magic = magic; - state.boots = 0; - } - state.boots += 1; - if (test_status == 0) { - PRINT("Reset board #%u to test again\n", state.boots); - k_msleep(10); - sys_reboot(SYS_REBOOT_COLD); - } else { - PRINT("Failed after %u attempts\n", state.boots); - state.boots = 0; - } - } -#ifdef CONFIG_ZTEST_NO_YIELD - /* - * Rather than yielding to idle thread, keep the part awake so debugger can - * still access it, since some SOCs cannot be debugged in low power states. - */ - uint32_t key = irq_lock(); - - while (1) { - ; /* Spin */ - } - irq_unlock(key); -#endif - return 0; -} -#endif diff --git a/subsys/testsuite/ztest/src/ztest_posix.c b/subsys/testsuite/ztest/src/ztest_posix.c index d2c1857c04cd..585bbeb98375 100644 --- a/subsys/testsuite/ztest/src/ztest_posix.c +++ b/subsys/testsuite/ztest/src/ztest_posix.c @@ -95,6 +95,8 @@ const char *ztest_get_test_args(void) return test_args; } + + /** * @brief Lists registered unit tests in this binary, one per line * @@ -126,12 +128,12 @@ int z_ztest_list_tests(void) * * @param state The current state of the machine as it relates to the test executable. */ -void z_ztest_run_all(const void *state) +void z_ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter) { if (z_ztest_get_list_test()) { z_ztest_list_tests(); } else { - ztest_run_test_suites(state); + ztest_run_test_suites(state, shuffle, suite_iter, case_iter); } } diff --git a/subsys/testsuite/ztest/src/ztest_shell.c b/subsys/testsuite/ztest/src/ztest_shell.c new file mode 100644 index 000000000000..969f0fcd9d40 --- /dev/null +++ b/subsys/testsuite/ztest/src/ztest_shell.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2022 Google LLC + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + + +static const char *test_args; + +/** + * @brief Try to shorten a filename by removing the current directory + * + * This helps to reduce the very long filenames in assertion failures. It + * removes the current directory from the filename and returns the rest. + * This makes assertions a lot more readable, and sometimes they fit on one + * line. + * + * @param file Filename to check + * @returns Shortened filename, or @file if it could not be shortened + */ +const char *ztest_relative_filename(const char *file) +{ + return file; +} + +/** + * Default entry point for running registered unit tests. + * + * @param state The current state of the machine as it relates to the test executable. + */ +void z_ztest_run_all(const void *state, bool shuffle, int suite_iter, int case_iter) +{ + ztest_run_test_suites(state, shuffle, suite_iter, case_iter); +} + +void ztest_reset_test_args(void) +{ + test_args = NULL; +} + +void ztest_set_test_args(char *args) +{ + test_args = strdup(args); +} + +/** + * @brief Helper function to get command line test arguments + * + * @return const char* + */ +char *ztest_get_test_args(void) +{ + if (test_args != NULL) { + return strdup(test_args); + } else { + return NULL; + } +} + + +/** + * @brief Checks if the test_args contains the suite/test name. + * + * @param suite_name + * @param test_name + * @return true + * @return false + */ +static bool z_ztest_testargs_contains(const char *suite_name, const char *test_name) +{ + bool found = false; + char *test_args_local = ztest_get_test_args(); + char *suite_test_pair; + char *last_suite_test_pair; + char *suite_arg; + char *test_arg; + char *last_arg; + + if (test_args_local == NULL) { + return true; + } + suite_test_pair = strtok_r(test_args_local, ",", &last_suite_test_pair); + + while (suite_test_pair && !found) { + suite_arg = strtok_r(suite_test_pair, ":", &last_arg); + test_arg = strtok_r(NULL, ":", &last_arg); + + found = !strcmp(suite_arg, suite_name); + if (test_name && test_arg) { + found &= !strcmp(test_arg, "*") || + !strcmp(test_arg, test_name); + } + + suite_test_pair = strtok_r(NULL, ",", &last_suite_test_pair); + } + return found; +} + +/** + * @brief Determines if the test suite should run based on test cases listed + * in the command line argument. + * + * Overrides implementation in ztest.c + * + * @param state The current state of the machine as it relates to the test + * executable. + * @param suite Pointer to ztest_suite_node + * @return true + * @return false + */ +bool z_ztest_should_suite_run(const void *state, struct ztest_suite_node *suite) +{ + char *test_args_local = ztest_get_test_args(); + bool run_suite = true; + + if (test_args_local != NULL && !z_ztest_testargs_contains(suite->name, NULL)) { + run_suite = false; + suite->stats->run_count++; + } else if (suite->predicate != NULL) { + run_suite = suite->predicate(state); + } + + return run_suite; +} + + + +/** + * @brief Determines if the test case should run based on test cases listed + * in the command line argument. Run all tests for non-posix builds + * + * @param suite - name of test suite + * @param test - name of unit test + * @return true + * @return false + */ +bool z_ztest_should_test_run(const char *suite, const char *test) +{ + bool run_test = false; + + run_test = z_ztest_testargs_contains(suite, test); + + return run_test; +} + +ZTEST_DMEM const struct ztest_arch_api ztest_api = { + .run_all = z_ztest_run_all, + .should_suite_run = z_ztest_should_suite_run, + .should_test_run = z_ztest_should_test_run +}; diff --git a/tests/bluetooth/host/keys/bt_keys_get_addr/src/main.c b/tests/bluetooth/host/keys/bt_keys_get_addr/src/main.c index 062134631815..cebbf1923e91 100644 --- a/tests/bluetooth/host/keys/bt_keys_get_addr/src/main.c +++ b/tests/bluetooth/host/keys/bt_keys_get_addr/src/main.c @@ -181,11 +181,11 @@ void test_main(void) { /* Only startup suite will run. */ all_startup_checks_executed = false; - ztest_run_all(NULL); + ztest_run_all(NULL, false, 1, 1); /* All other suites, except startup suite, will run. */ all_startup_checks_executed = true; - ztest_run_all(NULL); + ztest_run_all(NULL, false, 1, 1); /* Check that all the suites in this binary ran at least once. */ ztest_verify_all_test_suites_ran(); diff --git a/tests/drivers/eeprom/api/src/main.c b/tests/drivers/eeprom/api/src/main.c index e9ddd3ec76f5..5a14565cccb3 100644 --- a/tests/drivers/eeprom/api/src/main.c +++ b/tests/drivers/eeprom/api/src/main.c @@ -178,7 +178,7 @@ static void run_tests_on_eeprom(const struct device *dev) printk("Running tests on device \"%s\"\n", eeprom->name); k_object_access_grant(eeprom, k_current_get()); - ztest_run_all(NULL); + ztest_run_all(NULL, false, 1, 1); } void test_main(void) diff --git a/tests/kernel/sleep/src/main.c b/tests/kernel/sleep/src/main.c index 89da43b82ca8..9ef991b8647c 100644 --- a/tests/kernel/sleep/src/main.c +++ b/tests/kernel/sleep/src/main.c @@ -72,8 +72,6 @@ static void test_objects_init(void) k_sem_init(&test_thread_sem, 0, UINT_MAX); k_sem_init(&helper_thread_sem, 0, UINT_MAX); k_sem_init(&task_sem, 0, UINT_MAX); - - TC_PRINT("Kernel objects initialized\n"); } static void align_to_tick_boundary(void) @@ -111,7 +109,6 @@ static void test_thread(void *p1, void *p2, void *p3) k_sem_take(&test_thread_sem, K_FOREVER); - TC_PRINT("Testing normal expiration of k_sleep()\n"); align_to_tick_boundary(); start_tick = k_uptime_get_32(); @@ -125,7 +122,6 @@ static void test_thread(void *p1, void *p2, void *p3) return; } - TC_PRINT("Testing: test thread sleep + helper thread wakeup test\n"); k_sem_give(&helper_thread_sem); /* Activate helper thread */ align_to_tick_boundary(); @@ -139,7 +135,6 @@ static void test_thread(void *p1, void *p2, void *p3) return; } - TC_PRINT("Testing: test thread sleep + isr offload wakeup test\n"); k_sem_give(&helper_thread_sem); /* Activate helper thread */ align_to_tick_boundary(); @@ -153,7 +148,6 @@ static void test_thread(void *p1, void *p2, void *p3) return; } - TC_PRINT("Testing: test thread sleep + main wakeup test thread\n"); k_sem_give(&task_sem); /* Activate task */ align_to_tick_boundary(); @@ -216,16 +210,12 @@ ZTEST(sleep, test_sleep) 0, 0, NULL, TEST_THREAD_PRIORITY, 0, K_NO_WAIT); - TC_PRINT("Test thread started: id = %p\n", test_thread_id); - helper_thread_id = k_thread_create(&helper_thread_data, helper_thread_stack, THREAD_STACK, helper_thread, 0, 0, NULL, HELPER_THREAD_PRIORITY, 0, K_NO_WAIT); - TC_PRINT("Helper thread started: id = %p\n", helper_thread_id); - /* Activate test_thread */ k_sem_give(&test_thread_sem); @@ -237,7 +227,6 @@ ZTEST(sleep, test_sleep) zassert_false(test_failure, "test failure"); - TC_PRINT("Testing kernel k_sleep()\n"); align_to_tick_boundary(); start_tick = k_uptime_get_32(); k_sleep(K_SECONDS(1)); diff --git a/tests/kernel/sleep/src/usleep.c b/tests/kernel/sleep/src/usleep.c index 67037e593c90..b7c1144505a2 100644 --- a/tests/kernel/sleep/src/usleep.c +++ b/tests/kernel/sleep/src/usleep.c @@ -89,7 +89,6 @@ ZTEST_USER(sleep, test_usleep) } } - printk("elapsed_ms = %" PRId64 "\n", elapsed_ms); zassert_true(elapsed_ms >= LOWER_BOUND_MS, "short sleep"); zassert_true(elapsed_ms <= UPPER_BOUND_MS, "overslept"); } diff --git a/tests/kernel/threads/thread_apis/src/main.c b/tests/kernel/threads/thread_apis/src/main.c index b68c43311019..2e4a4b90f456 100644 --- a/tests/kernel/threads/thread_apis/src/main.c +++ b/tests/kernel/threads/thread_apis/src/main.c @@ -43,7 +43,7 @@ static ZTEST_DMEM int tp = 10; */ ZTEST(threads_lifecycle, test_systhreads_main) { - zassert_true(main_prio == CONFIG_MAIN_THREAD_PRIORITY); + zassert_true(main_prio == CONFIG_MAIN_THREAD_PRIORITY, "%d", CONFIG_MAIN_THREAD_PRIORITY); } /** @@ -185,7 +185,7 @@ ZTEST_USER(threads_lifecycle, test_thread_name_user_get_set) zassert_equal(ret, -EINVAL, "not a thread object"); ret = k_thread_name_copy(&z_main_thread, thread_name, sizeof(thread_name)); - zassert_equal(ret, 0, "couldn't get main thread name"); + zassert_equal(ret, 0, "couldn't get main thread name: %s (%d)", thread_name, ret); LOG_DBG("Main thread name is '%s'", thread_name); /* Set and get child thread's name */ diff --git a/tests/subsys/mgmt/mcumgr/cb_notifications/src/main.c b/tests/subsys/mgmt/mcumgr/cb_notifications/src/main.c index 50ac67d43b23..99dee33e380e 100644 --- a/tests/subsys/mgmt/mcumgr/cb_notifications/src/main.c +++ b/tests/subsys/mgmt/mcumgr/cb_notifications/src/main.c @@ -248,7 +248,7 @@ static void cleanup_test(void *p) void test_main(void) { while (test_state.test_set < CB_NOTIFICATION_TEST_SET_COUNT) { - ztest_run_all(&test_state); + ztest_run_all(&test_state, false, 1, 1); ++test_state.test_set; } diff --git a/tests/subsys/mgmt/mcumgr/os_mgmt_datetime/src/main.c b/tests/subsys/mgmt/mcumgr/os_mgmt_datetime/src/main.c index c387c3b4812a..c3a3c71d2b2b 100644 --- a/tests/subsys/mgmt/mcumgr/os_mgmt_datetime/src/main.c +++ b/tests/subsys/mgmt/mcumgr/os_mgmt_datetime/src/main.c @@ -1458,7 +1458,7 @@ static void cleanup_test(void *p) void test_main(void) { while (test_state.test_set < OS_MGMT_DATETIME_TEST_SET_COUNT) { - ztest_run_all(&test_state); + ztest_run_all(&test_state, false, 1, 1); ++test_state.test_set; } diff --git a/tests/subsys/mgmt/mcumgr/os_mgmt_info/src/main.c b/tests/subsys/mgmt/mcumgr/os_mgmt_info/src/main.c index 8ffc2a354b80..d07b8ed6d4de 100644 --- a/tests/subsys/mgmt/mcumgr/os_mgmt_info/src/main.c +++ b/tests/subsys/mgmt/mcumgr/os_mgmt_info/src/main.c @@ -1499,7 +1499,7 @@ static void cleanup_test(void *p) void test_main(void) { while (test_state.test_set < OS_MGMT_TEST_SET_COUNT) { - ztest_run_all(&test_state); + ztest_run_all(&test_state, false, 1, 1); ++test_state.test_set; } diff --git a/tests/ztest/error_hook/src/main.c b/tests/ztest/error_hook/src/main.c index e38b3111cf3c..dfa998e0075d 100644 --- a/tests/ztest/error_hook/src/main.c +++ b/tests/ztest/error_hook/src/main.c @@ -424,7 +424,7 @@ ZTEST(fail_assume_in_test, test_to_skip) void test_main(void) { - ztest_run_test_suites(NULL); + ztest_run_test_suites(NULL, false, 1, 1); /* Can't run ztest_verify_all_test_suites_ran() since some tests are * skipped by design. */