From 3e5d0784b9b5296bda801add034b057ad68237f7 Mon Sep 17 00:00:00 2001 From: "Alexander V. Chernikov" Date: Fri, 14 Apr 2023 15:25:50 +0000 Subject: [PATCH] Testing: add framework for the kernel unit tests. This changes intends to reduce the bar to the kernel unit-testing by introducing a new kernel-testing framework ("ktest") based on Netlink, loadable test modules and python test suite integration. This framework provides the following features: * Integration to the FreeBSD test suite * Automatic test discovery * Automatic test module loading * Minimal boiler-plate code in both kernel and userland * Passing any metadata to the test * Convenient environment pre-setup using python testing framework * Streaming messages from the kernel to the userland * Running tests in the dedicated taskqueues * Skipping or parametrizing tests Differential Revision: https://reviews.freebsd.org/D39385 MFC after: 2 weeks --- sys/modules/ktest/Makefile | 7 + sys/modules/ktest/ktest/Makefile | 14 + sys/modules/ktest/ktest_example/Makefile | 13 + sys/tests/ktest.c | 414 ++++++++++++++++++ sys/tests/ktest.h | 141 ++++++ sys/tests/ktest_example.c | 134 ++++++ tests/atf_python/Makefile | 2 +- tests/atf_python/atf_pytest.py | 6 + tests/atf_python/ktest.py | 173 ++++++++ tests/atf_python/sys/netlink/attrs.py | 2 + tests/atf_python/sys/netlink/base_headers.py | 7 + tests/atf_python/sys/netlink/netlink.py | 2 +- .../atf_python/sys/netlink/netlink_generic.py | 118 +++++ tests/atf_python/utils.py | 5 + tests/conftest.py | 6 + tests/examples/Makefile | 1 + tests/examples/test_ktest_example.py | 35 ++ 17 files changed, 1078 insertions(+), 2 deletions(-) create mode 100644 sys/modules/ktest/Makefile create mode 100644 sys/modules/ktest/ktest/Makefile create mode 100644 sys/modules/ktest/ktest_example/Makefile create mode 100644 sys/tests/ktest.c create mode 100644 sys/tests/ktest.h create mode 100644 sys/tests/ktest_example.c create mode 100644 tests/atf_python/ktest.py create mode 100644 tests/examples/test_ktest_example.py diff --git a/sys/modules/ktest/Makefile b/sys/modules/ktest/Makefile new file mode 100644 index 000000000000..21c94caabc30 --- /dev/null +++ b/sys/modules/ktest/Makefile @@ -0,0 +1,7 @@ +SYSDIR?=${SRCTOP}/sys +.include "${SYSDIR}/conf/kern.opts.mk" + +SUBDIR= ktest \ + ktest_example + +.include diff --git a/sys/modules/ktest/ktest/Makefile b/sys/modules/ktest/ktest/Makefile new file mode 100644 index 000000000000..86ed957ac2b7 --- /dev/null +++ b/sys/modules/ktest/ktest/Makefile @@ -0,0 +1,14 @@ +# $FreeBSD$ + +PACKAGE= tests + +SYSDIR?=${SRCTOP}/sys +.include "${SYSDIR}/conf/kern.opts.mk" + +.PATH: ${SYSDIR}/tests + +KMOD= ktest +SRCS= ktest.c +SRCS+= opt_netlink.h + +.include diff --git a/sys/modules/ktest/ktest_example/Makefile b/sys/modules/ktest/ktest_example/Makefile new file mode 100644 index 000000000000..b4a3e778e2ed --- /dev/null +++ b/sys/modules/ktest/ktest_example/Makefile @@ -0,0 +1,13 @@ +# $FreeBSD$ + +PACKAGE= tests + +SYSDIR?=${SRCTOP}/sys +.include "${SYSDIR}/conf/kern.opts.mk" + +.PATH: ${SYSDIR}/tests + +KMOD= ktest_example +SRCS= ktest_example.c + +.include diff --git a/sys/tests/ktest.c b/sys/tests/ktest.c new file mode 100644 index 000000000000..fcb40130bcef --- /dev/null +++ b/sys/tests/ktest.c @@ -0,0 +1,414 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2023 Alexander V. Chernikov + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "opt_netlink.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +struct mtx ktest_mtx; +#define KTEST_LOCK() mtx_lock(&ktest_mtx) +#define KTEST_UNLOCK() mtx_unlock(&ktest_mtx) +#define KTEST_LOCK_ASSERT() mtx_assert(&ktest_mtx, MA_OWNED) + +MTX_SYSINIT(ktest_mtx, &ktest_mtx, "ktest mutex", MTX_DEF); + +struct ktest_module { + struct ktest_module_info *info; + volatile u_int refcount; + TAILQ_ENTRY(ktest_module) entries; +}; +static TAILQ_HEAD(, ktest_module) module_list = TAILQ_HEAD_INITIALIZER(module_list); + +struct nl_ktest_parsed { + char *mod_name; + char *test_name; + struct nlattr *test_meta; +}; + +#define _IN(_field) offsetof(struct genlmsghdr, _field) +#define _OUT(_field) offsetof(struct nl_ktest_parsed, _field) + +static const struct nlattr_parser nla_p_get[] = { + { .type = KTEST_ATTR_MOD_NAME, .off = _OUT(mod_name), .cb = nlattr_get_string }, + { .type = KTEST_ATTR_TEST_NAME, .off = _OUT(test_name), .cb = nlattr_get_string }, + { .type = KTEST_ATTR_TEST_META, .off = _OUT(test_meta), .cb = nlattr_get_nla }, +}; +static const struct nlfield_parser nlf_p_get[] = { +}; +NL_DECLARE_PARSER(ktest_parser, struct genlmsghdr, nlf_p_get, nla_p_get); +#undef _IN +#undef _OUT + +static bool +create_reply(struct nl_writer *nw, struct nlmsghdr *hdr, int cmd) +{ + if (!nlmsg_reply(nw, hdr, sizeof(struct genlmsghdr))) + return (false); + + struct genlmsghdr *ghdr_new = nlmsg_reserve_object(nw, struct genlmsghdr); + ghdr_new->cmd = cmd; + ghdr_new->version = 0; + ghdr_new->reserved = 0; + + return (true); +} + +static int +dump_mod_test(struct nlmsghdr *hdr, struct nl_pstate *npt, + struct ktest_module *mod, const struct ktest_test_info *test_info) +{ + struct nl_writer *nw = npt->nw; + + if (!create_reply(nw, hdr, KTEST_CMD_NEWTEST)) + goto enomem; + + nlattr_add_string(nw, KTEST_ATTR_MOD_NAME, mod->info->name); + nlattr_add_string(nw, KTEST_ATTR_TEST_NAME, test_info->name); + nlattr_add_string(nw, KTEST_ATTR_TEST_DESCR, test_info->desc); + + if (nlmsg_end(nw)) + return (0); +enomem: + nlmsg_abort(nw); + return (ENOMEM); +} + +static int +dump_mod_tests(struct nlmsghdr *hdr, struct nl_pstate *npt, + struct ktest_module *mod, struct nl_ktest_parsed *attrs) +{ + for (int i = 0; i < mod->info->num_tests; i++) { + const struct ktest_test_info *test_info = &mod->info->tests[i]; + if (attrs->test_name != NULL && strcmp(attrs->test_name, test_info->name)) + continue; + int error = dump_mod_test(hdr, npt, mod, test_info); + if (error != 0) + return (error); + } + + return (0); +} + +static int +dump_tests(struct nlmsghdr *hdr, struct nl_pstate *npt) +{ + struct nl_ktest_parsed attrs = { }; + struct ktest_module *mod; + int error; + + error = nl_parse_nlmsg(hdr, &ktest_parser, npt, &attrs); + if (error != 0) + return (error); + + hdr->nlmsg_flags |= NLM_F_MULTI; + + KTEST_LOCK(); + TAILQ_FOREACH(mod, &module_list, entries) { + if (attrs.mod_name && strcmp(attrs.mod_name, mod->info->name)) + continue; + error = dump_mod_tests(hdr, npt, mod, &attrs); + if (error != 0) + break; + } + KTEST_UNLOCK(); + + if (!nlmsg_end_dump(npt->nw, error, hdr)) { + //NL_LOG(LOG_DEBUG, "Unable to finalize the dump"); + return (ENOMEM); + } + + return (error); +} + +static int +run_test(struct nlmsghdr *hdr, struct nl_pstate *npt) +{ + struct nl_ktest_parsed attrs = { }; + struct ktest_module *mod; + int error; + + error = nl_parse_nlmsg(hdr, &ktest_parser, npt, &attrs); + if (error != 0) + return (error); + + if (attrs.mod_name == NULL) { + nlmsg_report_err_msg(npt, "KTEST_ATTR_MOD_NAME not set"); + return (EINVAL); + } + + if (attrs.test_name == NULL) { + nlmsg_report_err_msg(npt, "KTEST_ATTR_TEST_NAME not set"); + return (EINVAL); + } + + const struct ktest_test_info *test = NULL; + + KTEST_LOCK(); + TAILQ_FOREACH(mod, &module_list, entries) { + if (strcmp(attrs.mod_name, mod->info->name)) + continue; + + const struct ktest_module_info *info = mod->info; + + for (int i = 0; i < info->num_tests; i++) { + const struct ktest_test_info *test_info = &info->tests[i]; + + if (!strcmp(attrs.test_name, test_info->name)) { + test = test_info; + break; + } + } + break; + } + if (test != NULL) + refcount_acquire(&mod->refcount); + KTEST_UNLOCK(); + + if (test == NULL) + return (ESRCH); + + /* Run the test */ + struct ktest_test_context ctx = { + .npt = npt, + .hdr = hdr, + .buf = npt_alloc(npt, KTEST_MAX_BUF), + .bufsize = KTEST_MAX_BUF, + }; + + if (ctx.buf == NULL) { + //NL_LOG(LOG_DEBUG, "unable to allocate temporary buffer"); + return (ENOMEM); + } + + if (test->parse != NULL && attrs.test_meta != NULL) { + error = test->parse(&ctx, attrs.test_meta); + if (error != 0) + return (error); + } + + hdr->nlmsg_flags |= NLM_F_MULTI; + + KTEST_LOG_LEVEL(&ctx, LOG_INFO, "start running %s", test->name); + error = test->func(&ctx); + KTEST_LOG_LEVEL(&ctx, LOG_INFO, "end running %s", test->name); + + refcount_release(&mod->refcount); + + if (!nlmsg_end_dump(npt->nw, error, hdr)) { + //NL_LOG(LOG_DEBUG, "Unable to finalize the dump"); + return (ENOMEM); + } + + return (error); +} + + +/* USER API */ +static void +register_test_module(struct ktest_module_info *info) +{ + struct ktest_module *mod = malloc(sizeof(*mod), M_TEMP, M_WAITOK | M_ZERO); + + mod->info = info; + info->module_ptr = mod; + KTEST_LOCK(); + TAILQ_INSERT_TAIL(&module_list, mod, entries); + KTEST_UNLOCK(); +} + +static void +unregister_test_module(struct ktest_module_info *info) +{ + struct ktest_module *mod = info->module_ptr; + + info->module_ptr = NULL; + + KTEST_LOCK(); + TAILQ_REMOVE(&module_list, mod, entries); + KTEST_UNLOCK(); + + free(mod, M_TEMP); +} + +static bool +can_unregister(struct ktest_module_info *info) +{ + struct ktest_module *mod = info->module_ptr; + + return (refcount_load(&mod->refcount) == 0); +} + +int +ktest_default_modevent(module_t mod, int type, void *arg) +{ + struct ktest_module_info *info = (struct ktest_module_info *)arg; + int error = 0; + + switch (type) { + case MOD_LOAD: + register_test_module(info); + break; + case MOD_UNLOAD: + if (!can_unregister(info)) + return (EBUSY); + unregister_test_module(info); + break; + default: + error = EOPNOTSUPP; + break; + } + return (error); +} + +bool +ktest_start_msg(struct ktest_test_context *ctx) +{ + return (create_reply(ctx->npt->nw, ctx->hdr, KTEST_CMD_NEWMESSAGE)); +} + +void +ktest_add_msg_meta(struct ktest_test_context *ctx, const char *func, + const char *fname, int line) +{ + struct nl_writer *nw = ctx->npt->nw; + struct timespec ts; + + nanouptime(&ts); + nlattr_add(nw, KTEST_MSG_ATTR_TS, sizeof(ts), &ts); + + nlattr_add_string(nw, KTEST_MSG_ATTR_FUNC, func); + nlattr_add_string(nw, KTEST_MSG_ATTR_FILE, fname); + nlattr_add_u32(nw, KTEST_MSG_ATTR_LINE, line); +} + +void +ktest_add_msg_text(struct ktest_test_context *ctx, int msg_level, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vsnprintf(ctx->buf, ctx->bufsize, fmt, ap); + va_end(ap); + + nlattr_add_u8(ctx->npt->nw, KTEST_MSG_ATTR_LEVEL, msg_level); + nlattr_add_string(ctx->npt->nw, KTEST_MSG_ATTR_TEXT, ctx->buf); +} + +void +ktest_end_msg(struct ktest_test_context *ctx) +{ + nlmsg_end(ctx->npt->nw); +} + +/* Module glue */ + +static const struct nlhdr_parser *all_parsers[] = { &ktest_parser }; + +static const struct genl_cmd ktest_cmds[] = { + { + .cmd_num = KTEST_CMD_LIST, + .cmd_name = "KTEST_CMD_LIST", + .cmd_cb = dump_tests, + .cmd_flags = GENL_CMD_CAP_DO | GENL_CMD_CAP_DUMP | GENL_CMD_CAP_HASPOL, + }, + { + .cmd_num = KTEST_CMD_RUN, + .cmd_name = "KTEST_CMD_RUN", + .cmd_cb = run_test, + .cmd_flags = GENL_CMD_CAP_DO | GENL_CMD_CAP_HASPOL, + .cmd_priv = PRIV_KLD_LOAD, + }, +}; + +static void +ktest_nl_register(void) +{ + bool ret __diagused; + int family_id __diagused; + + NL_VERIFY_PARSERS(all_parsers); + family_id = genl_register_family(KTEST_FAMILY_NAME, 0, 1, KTEST_CMD_MAX); + MPASS(family_id != 0); + + ret = genl_register_cmds(KTEST_FAMILY_NAME, ktest_cmds, NL_ARRAY_LEN(ktest_cmds)); + MPASS(ret); +} + +static void +ktest_nl_unregister(void) +{ + MPASS(TAILQ_EMPTY(&module_list)); + + genl_unregister_family(KTEST_FAMILY_NAME); +} + +static int +ktest_modevent(module_t mod, int type, void *unused) +{ + int error = 0; + + switch (type) { + case MOD_LOAD: + ktest_nl_register(); + break; + case MOD_UNLOAD: + ktest_nl_unregister(); + break; + default: + error = EOPNOTSUPP; + break; + } + return (error); +} + +static moduledata_t ktestmod = { + "ktest", + ktest_modevent, + 0 +}; + +DECLARE_MODULE(ktestmod, ktestmod, SI_SUB_PSEUDO, SI_ORDER_ANY); +MODULE_VERSION(ktestmod, 1); +MODULE_DEPEND(ktestmod, netlink, 1, 1, 1); + diff --git a/sys/tests/ktest.h b/sys/tests/ktest.h new file mode 100644 index 000000000000..feadb800551b --- /dev/null +++ b/sys/tests/ktest.h @@ -0,0 +1,141 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2023 Alexander V. Chernikov + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef SYS_TESTS_KTEST_H_ +#define SYS_TESTS_KTEST_H_ + +#ifdef _KERNEL + +#include +#include +#include +#include + +struct nlattr; +struct nl_pstate; +struct nlmsghdr; + +struct ktest_test_context { + void *arg; + struct nl_pstate *npt; + struct nlmsghdr *hdr; + char *buf; + size_t bufsize; +}; + +typedef int (*ktest_run_t)(struct ktest_test_context *ctx); +typedef int (*ktest_parse_t)(struct ktest_test_context *ctx, struct nlattr *container); + +struct ktest_test_info { + const char *name; + const char *desc; + ktest_run_t func; + ktest_parse_t parse; +}; + +struct ktest_module_info { + const char *name; + const struct ktest_test_info *tests; + int num_tests; + void *module_ptr; +}; + +int ktest_default_modevent(module_t mod, int type, void *arg); + +bool ktest_start_msg(struct ktest_test_context *ctx); +void ktest_add_msg_meta(struct ktest_test_context *ctx, const char *func, + const char *fname, int line); +void ktest_add_msg_text(struct ktest_test_context *ctx, int msg_level, + const char *fmt, ...); +void ktest_end_msg(struct ktest_test_context *ctx); + +#define KTEST_LOG_LEVEL(_ctx, _l, _fmt, ...) { \ + if (ktest_start_msg(_ctx)) { \ + ktest_add_msg_meta(_ctx, __func__, __FILE__, __LINE__); \ + ktest_add_msg_text(_ctx, _l, _fmt, ## __VA_ARGS__); \ + ktest_end_msg(_ctx); \ + } \ +} + +#define KTEST_LOG(_ctx, _fmt, ...) \ + KTEST_LOG_LEVEL(_ctx, LOG_DEBUG, _fmt, ## __VA_ARGS__) + +#define KTEST_MAX_BUF 512 + +#define KTEST_MODULE_DECLARE(_n, _t) \ +static struct ktest_module_info _module_info = { \ + .name = #_n, \ + .tests = _t, \ + .num_tests = nitems(_t), \ +}; \ + \ +static moduledata_t _module_data = { \ + "__" #_n "_module", \ + ktest_default_modevent, \ + &_module_info, \ +}; \ + \ +DECLARE_MODULE(ktest_##_n, _module_data, SI_SUB_PSEUDO, SI_ORDER_ANY); \ +MODULE_VERSION(ktest_##_n, 1); \ +MODULE_DEPEND(ktest_##_n, ktestmod, 1, 1, 1); \ + +#endif /* _KERNEL */ + +/* genetlink definitions */ +#define KTEST_FAMILY_NAME "ktest" + +/* commands */ +enum { + KTEST_CMD_UNSPEC = 0, + KTEST_CMD_LIST = 1, + KTEST_CMD_RUN = 2, + KTEST_CMD_NEWTEST = 3, + KTEST_CMD_NEWMESSAGE = 4, + __KTEST_CMD_MAX, +}; +#define KTEST_CMD_MAX (__KTEST_CMD_MAX - 1) + +enum ktest_attr_type_t { + KTEST_ATTR_UNSPEC, + KTEST_ATTR_MOD_NAME = 1, /* string: test module name */ + KTEST_ATTR_TEST_NAME = 2, /* string: test name */ + KTEST_ATTR_TEST_DESCR = 3, /* string: test description */ + KTEST_ATTR_TEST_META = 4, /* nested: container with test-specific metadata */ +}; + +enum ktest_msg_attr_type_t { + KTEST_MSG_ATTR_UNSPEC, + KTEST_MSG_ATTR_TS = 1, /* struct timespec */ + KTEST_MSG_ATTR_FUNC = 2, /* string: function name */ + KTEST_MSG_ATTR_FILE = 3, /* string: file name */ + KTEST_MSG_ATTR_LINE = 4, /* u32: line in the file */ + KTEST_MSG_ATTR_TEXT = 5, /* string: actual message data */ + KTEST_MSG_ATTR_LEVEL = 6, /* u8: syslog loglevel */ + KTEST_MSG_ATTR_META = 7, /* nested: message metadata */ +}; + +#endif diff --git a/sys/tests/ktest_example.c b/sys/tests/ktest_example.c new file mode 100644 index 000000000000..7cccaad7a855 --- /dev/null +++ b/sys/tests/ktest_example.c @@ -0,0 +1,134 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2023 Alexander V. Chernikov + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + + +static int +test_something(struct ktest_test_context *ctx) +{ + KTEST_LOG(ctx, "I'm here, [%s]", __func__); + + pause("sleeping...", hz / 10); + + KTEST_LOG(ctx, "done"); + + return (0); +} + +static int +test_something_else(struct ktest_test_context *ctx) +{ + return (0); +} + +static int +test_failed(struct ktest_test_context *ctx) +{ + return (EBUSY); +} + +static int +test_failed2(struct ktest_test_context *ctx) +{ + KTEST_LOG(ctx, "failed because it always fails"); + return (EBUSY); +} + +#include +#include +#include + +struct test1_attrs { + uint32_t arg1; + uint32_t arg2; + char *text; +}; + +#define _OUT(_field) offsetof(struct test1_attrs, _field) +static const struct nlattr_parser nla_p_test1[] = { + { .type = 1, .off = _OUT(arg1), .cb = nlattr_get_uint32 }, + { .type = 2, .off = _OUT(arg2), .cb = nlattr_get_uint32 }, + { .type = 3, .off = _OUT(text), .cb = nlattr_get_string }, +}; +#undef _OUT +NL_DECLARE_ATTR_PARSER(test1_parser, nla_p_test1); + +static int +test_with_params_parser(struct ktest_test_context *ctx, struct nlattr *nla) +{ + struct test1_attrs *attrs = npt_alloc(ctx->npt, sizeof(*attrs)); + + ctx->arg = attrs; + if (attrs != NULL) + return (nl_parse_nested(nla, &test1_parser, ctx->npt, attrs)); + return (ENOMEM); +} + +static int +test_with_params(struct ktest_test_context *ctx) +{ + struct test1_attrs *attrs = ctx->arg; + + if (attrs->text != NULL) + KTEST_LOG(ctx, "Get '%s'", attrs->text); + KTEST_LOG(ctx, "%u + %u = %u", attrs->arg1, attrs->arg2, + attrs->arg1 + attrs->arg2); + return (0); +} + +static const struct ktest_test_info tests[] = { + { + .name = "test_something", + .desc = "example description", + .func = &test_something, + }, + { + .name = "test_something_else", + .desc = "example description 2", + .func = &test_something_else, + }, + { + .name = "test_failed", + .desc = "always failing test", + .func = &test_failed, + }, + { + .name = "test_failed2", + .desc = "always failing test", + .func = &test_failed2, + }, + { + .name = "test_with_params", + .desc = "test summing integers", + .func = &test_with_params, + .parse = &test_with_params_parser, + }, +}; +KTEST_MODULE_DECLARE(ktest_example, tests); diff --git a/tests/atf_python/Makefile b/tests/atf_python/Makefile index 1a2fec387eda..889cdcdf9592 100644 --- a/tests/atf_python/Makefile +++ b/tests/atf_python/Makefile @@ -2,7 +2,7 @@ .PATH: ${.CURDIR} -FILES= __init__.py atf_pytest.py utils.py +FILES= __init__.py atf_pytest.py ktest.py utils.py SUBDIR= sys .include diff --git a/tests/atf_python/atf_pytest.py b/tests/atf_python/atf_pytest.py index 0dd3a225b73d..19b5f88fa200 100644 --- a/tests/atf_python/atf_pytest.py +++ b/tests/atf_python/atf_pytest.py @@ -6,6 +6,7 @@ from typing import NamedTuple from typing import Optional from typing import Tuple +from atf_python.ktest import generate_ktests from atf_python.utils import nodeid_to_method_name import pytest @@ -42,6 +43,8 @@ class ATFTestObj(object): def _get_test_description(self, obj): """Returns first non-empty line from func docstring or func name""" + if getattr(obj, "descr", None) is not None: + return getattr(obj, "descr") docstr = obj.function.__doc__ if docstr: for line in docstr.split("\n"): @@ -163,6 +166,9 @@ class ATFHandler(object): items.clear() items.extend(new_items) + def expand_tests(self, collector, name, obj): + return generate_ktests(collector, name, obj) + def modify_tests(self, items, config): if config.option.atf_cleanup: self._generate_test_cleanups(items) diff --git a/tests/atf_python/ktest.py b/tests/atf_python/ktest.py new file mode 100644 index 000000000000..4cd9970aaec1 --- /dev/null +++ b/tests/atf_python/ktest.py @@ -0,0 +1,173 @@ +import logging +import time +from typing import NamedTuple + +import pytest +from atf_python.sys.netlink.attrs import NlAttrNested +from atf_python.sys.netlink.attrs import NlAttrStr +from atf_python.sys.netlink.netlink import NetlinkMultipartIterator +from atf_python.sys.netlink.netlink import NlHelper +from atf_python.sys.netlink.netlink import Nlsock +from atf_python.sys.netlink.netlink_generic import KtestAttrType +from atf_python.sys.netlink.netlink_generic import KtestInfoMessage +from atf_python.sys.netlink.netlink_generic import KtestLogMsgType +from atf_python.sys.netlink.netlink_generic import KtestMsgAttrType +from atf_python.sys.netlink.netlink_generic import KtestMsgType +from atf_python.sys.netlink.netlink_generic import timespec +from atf_python.sys.netlink.utils import NlConst +from atf_python.utils import BaseTest +from atf_python.utils import libc +from atf_python.utils import nodeid_to_method_name + + +datefmt = "%H:%M:%S" +fmt = "%(asctime)s.%(msecs)03d %(filename)s:%(funcName)s:%(lineno)d %(message)s" +logging.basicConfig(level=logging.DEBUG, format=fmt, datefmt=datefmt) +logger = logging.getLogger("ktest") + + +NETLINK_FAMILY = "ktest" + + +class KtestItem(pytest.Item): + def __init__(self, *, descr, kcls, **kwargs): + super().__init__(**kwargs) + self.descr = descr + self._kcls = kcls + + def runtest(self): + self._kcls().runtest() + + +class KtestCollector(pytest.Class): + def collect(self): + obj = self.obj + exclude_names = set([n for n in dir(obj) if not n.startswith("_")]) + + autoload = obj.KTEST_MODULE_AUTOLOAD + module_name = obj.KTEST_MODULE_NAME + loader = KtestLoader(module_name, autoload) + ktests = loader.load_ktests() + if not ktests: + return + + orig = pytest.Class.from_parent(self.parent, name=self.name, obj=obj) + for py_test in orig.collect(): + yield py_test + + for ktest in ktests: + name = ktest["name"] + descr = ktest["desc"] + if name in exclude_names: + continue + yield KtestItem.from_parent(self, name=name, descr=descr, kcls=obj) + + +class KtestLoader(object): + def __init__(self, module_name: str, autoload: bool): + self.module_name = module_name + self.autoload = autoload + self.helper = NlHelper() + self.nlsock = Nlsock(NlConst.NETLINK_GENERIC, self.helper) + self.family_id = self._get_family_id() + + def _get_family_id(self): + try: + family_id = self.nlsock.get_genl_family_id(NETLINK_FAMILY) + except ValueError: + if self.autoload: + libc.kldload(self.module_name) + family_id = self.nlsock.get_genl_family_id(NETLINK_FAMILY) + else: + raise + return family_id + + def _load_ktests(self): + msg = KtestInfoMessage(self.helper, self.family_id, KtestMsgType.KTEST_CMD_LIST) + msg.set_request() + msg.add_nla(NlAttrStr(KtestAttrType.KTEST_ATTR_MOD_NAME, self.module_name)) + self.nlsock.write_message(msg, verbose=False) + nlmsg_seq = msg.nl_hdr.nlmsg_seq + + ret = [] + for rx_msg in NetlinkMultipartIterator(self.nlsock, nlmsg_seq, self.family_id): + # test_msg.print_message() + tst = { + "mod_name": rx_msg.get_nla(KtestAttrType.KTEST_ATTR_MOD_NAME).text, + "name": rx_msg.get_nla(KtestAttrType.KTEST_ATTR_TEST_NAME).text, + "desc": rx_msg.get_nla(KtestAttrType.KTEST_ATTR_TEST_DESCR).text, + } + ret.append(tst) + return ret + + def load_ktests(self): + ret = self._load_ktests() + if not ret and self.autoload: + libc.kldload(self.module_name) + ret = self._load_ktests() + return ret + + +def generate_ktests(collector, name, obj): + if getattr(obj, "KTEST_MODULE_NAME", None) is not None: + return KtestCollector.from_parent(collector, name=name, obj=obj) + return None + + +class BaseKernelTest(BaseTest): + KTEST_MODULE_AUTOLOAD = True + KTEST_MODULE_NAME = None + + def _get_record_time(self, msg) -> float: + timespec = msg.get_nla(KtestMsgAttrType.KTEST_MSG_ATTR_TS).ts + epoch_ktime = timespec.tv_sec * 1.0 + timespec.tv_nsec * 1.0 / 1000000000 + if not hasattr(self, "_start_epoch"): + self._start_ktime = epoch_ktime + self._start_time = time.time() + epoch_time = self._start_time + else: + epoch_time = time.time() - self._start_time + epoch_ktime + return epoch_time + + def _log_message(self, msg): + # Convert syslog-type l + syslog_level = msg.get_nla(KtestMsgAttrType.KTEST_MSG_ATTR_LEVEL).u8 + if syslog_level <= 6: + loglevel = logging.INFO + else: + loglevel = logging.DEBUG + rec = logging.LogRecord( + self.KTEST_MODULE_NAME, + loglevel, + msg.get_nla(KtestMsgAttrType.KTEST_MSG_ATTR_FILE).text, + msg.get_nla(KtestMsgAttrType.KTEST_MSG_ATTR_LINE).u32, + "%s", + (msg.get_nla(KtestMsgAttrType.KTEST_MSG_ATTR_TEXT).text), + None, + msg.get_nla(KtestMsgAttrType.KTEST_MSG_ATTR_FUNC).text, + None, + ) + rec.created = self._get_record_time(msg) + logger.handle(rec) + + def _runtest_name(self, test_name: str, test_data): + module_name = self.KTEST_MODULE_NAME + # print("Running kernel test {} for module {}".format(test_name, module_name)) + helper = NlHelper() + nlsock = Nlsock(NlConst.NETLINK_GENERIC, helper) + family_id = nlsock.get_genl_family_id(NETLINK_FAMILY) + msg = KtestInfoMessage(helper, family_id, KtestMsgType.KTEST_CMD_RUN) + msg.set_request() + msg.add_nla(NlAttrStr(KtestAttrType.KTEST_ATTR_MOD_NAME, module_name)) + msg.add_nla(NlAttrStr(KtestAttrType.KTEST_ATTR_TEST_NAME, test_name)) + if test_data is not None: + msg.add_nla(NlAttrNested(KtestAttrType.KTEST_ATTR_TEST_META, test_data)) + nlsock.write_message(msg, verbose=False) + + for log_msg in NetlinkMultipartIterator( + nlsock, msg.nl_hdr.nlmsg_seq, family_id + ): + self._log_message(log_msg) + + def runtest(self, test_data=None): + self._runtest_name(nodeid_to_method_name(self.test_id), test_data) diff --git a/tests/atf_python/sys/netlink/attrs.py b/tests/atf_python/sys/netlink/attrs.py index f6fe9ee43c98..58fbab7fc8db 100644 --- a/tests/atf_python/sys/netlink/attrs.py +++ b/tests/atf_python/sys/netlink/attrs.py @@ -7,6 +7,8 @@ from atf_python.sys.netlink.utils import enum_or_int class NlAttr(object): + HDR_LEN = 4 # sizeof(struct nlattr) + def __init__(self, nla_type, data): if isinstance(nla_type, Enum): self._nla_type = nla_type.value diff --git a/tests/atf_python/sys/netlink/base_headers.py b/tests/atf_python/sys/netlink/base_headers.py index 759d8827fb3c..71771a249b3d 100644 --- a/tests/atf_python/sys/netlink/base_headers.py +++ b/tests/atf_python/sys/netlink/base_headers.py @@ -15,6 +15,13 @@ class Nlmsghdr(Structure): ] +class Nlattr(Structure): + _fields_ = [ + ("nla_len", c_ushort), + ("nla_type", c_ushort), + ] + + class NlMsgType(Enum): NLMSG_NOOP = 1 NLMSG_ERROR = 2 diff --git a/tests/atf_python/sys/netlink/netlink.py b/tests/atf_python/sys/netlink/netlink.py index f813727d55b4..4bdefc2d5014 100644 --- a/tests/atf_python/sys/netlink/netlink.py +++ b/tests/atf_python/sys/netlink/netlink.py @@ -22,8 +22,8 @@ from atf_python.sys.netlink.message import BaseNetlinkMessage from atf_python.sys.netlink.message import NlMsgCategory from atf_python.sys.netlink.message import NlMsgProps from atf_python.sys.netlink.message import StdNetlinkMessage -from atf_python.sys.netlink.netlink_generic import GenlCtrlMsgType from atf_python.sys.netlink.netlink_generic import GenlCtrlAttrType +from atf_python.sys.netlink.netlink_generic import GenlCtrlMsgType from atf_python.sys.netlink.netlink_generic import handler_classes as genl_classes from atf_python.sys.netlink.netlink_route import handler_classes as rt_classes from atf_python.sys.netlink.utils import align4 diff --git a/tests/atf_python/sys/netlink/netlink_generic.py b/tests/atf_python/sys/netlink/netlink_generic.py index ee75d5bf37f3..06dc8704fe07 100644 --- a/tests/atf_python/sys/netlink/netlink_generic.py +++ b/tests/atf_python/sys/netlink/netlink_generic.py @@ -1,10 +1,16 @@ #!/usr/local/bin/python3 +from ctypes import c_int64 +from ctypes import c_long from ctypes import sizeof +from ctypes import Structure from enum import Enum +import struct +from atf_python.sys.netlink.attrs import NlAttr from atf_python.sys.netlink.attrs import NlAttrStr from atf_python.sys.netlink.attrs import NlAttrU16 from atf_python.sys.netlink.attrs import NlAttrU32 +from atf_python.sys.netlink.attrs import NlAttrU8 from atf_python.sys.netlink.base_headers import GenlMsgHdr from atf_python.sys.netlink.message import NlMsgCategory from atf_python.sys.netlink.message import NlMsgProps @@ -105,6 +111,118 @@ class NetlinkGenlCtrlMessage(NetlinkGenlMessage): family_name = GenlCtrlFamilyName +KtestFamilyName = "ktest" + + +class KtestMsgType(Enum): + KTEST_CMD_UNSPEC = 0 + KTEST_CMD_LIST = 1 + KTEST_CMD_RUN = 2 + KTEST_CMD_NEWTEST = 3 + KTEST_CMD_NEWMESSAGE = 4 + + +class KtestAttrType(Enum): + KTEST_ATTR_MOD_NAME = 1 + KTEST_ATTR_TEST_NAME = 2 + KTEST_ATTR_TEST_DESCR = 3 + KTEST_ATTR_TEST_META = 4 + + +class KtestLogMsgType(Enum): + KTEST_MSG_START = 1 + KTEST_MSG_END = 2 + KTEST_MSG_LOG = 3 + KTEST_MSG_FAIL = 4 + + +class KtestMsgAttrType(Enum): + KTEST_MSG_ATTR_TS = 1 + KTEST_MSG_ATTR_FUNC = 2 + KTEST_MSG_ATTR_FILE = 3 + KTEST_MSG_ATTR_LINE = 4 + KTEST_MSG_ATTR_TEXT = 5 + KTEST_MSG_ATTR_LEVEL = 6 + KTEST_MSG_ATTR_META = 7 + + +class timespec(Structure): + _fields_ = [ + ("tv_sec", c_int64), + ("tv_nsec", c_long), + ] + + +class NlAttrTS(NlAttr): + DATA_LEN = sizeof(timespec) + + def __init__(self, nla_type, val): + self.ts = val + super().__init__(nla_type, b"") + + @property + def nla_len(self): + return NlAttr.HDR_LEN + self.DATA_LEN + + def _print_attr_value(self): + return " tv_sec={} tv_nsec={}".format(self.ts.tv_sec, self.ts.tv_nsec) + + @staticmethod + def _validate(data): + assert len(data) == NlAttr.HDR_LEN + NlAttrTS.DATA_LEN + nla_len, nla_type = struct.unpack("@HH", data[:NlAttr.HDR_LEN]) + assert nla_len == NlAttr.HDR_LEN + NlAttrTS.DATA_LEN + + @classmethod + def _parse(cls, data): + nla_len, nla_type = struct.unpack("@HH", data[:NlAttr.HDR_LEN]) + val = timespec.from_buffer_copy(data[NlAttr.HDR_LEN:]) + return cls(nla_type, val) + + def __bytes__(self): + return self._to_bytes(bytes(self.ts)) + + +ktest_info_attrs = prepare_attrs_map( + [ + AttrDescr(KtestAttrType.KTEST_ATTR_MOD_NAME, NlAttrStr), + AttrDescr(KtestAttrType.KTEST_ATTR_TEST_NAME, NlAttrStr), + AttrDescr(KtestAttrType.KTEST_ATTR_TEST_DESCR, NlAttrStr), + ] +) + + +ktest_msg_attrs = prepare_attrs_map( + [ + AttrDescr(KtestMsgAttrType.KTEST_MSG_ATTR_FUNC, NlAttrStr), + AttrDescr(KtestMsgAttrType.KTEST_MSG_ATTR_FILE, NlAttrStr), + AttrDescr(KtestMsgAttrType.KTEST_MSG_ATTR_LINE, NlAttrU32), + AttrDescr(KtestMsgAttrType.KTEST_MSG_ATTR_TEXT, NlAttrStr), + AttrDescr(KtestMsgAttrType.KTEST_MSG_ATTR_LEVEL, NlAttrU8), + AttrDescr(KtestMsgAttrType.KTEST_MSG_ATTR_TS, NlAttrTS), + ] +) + + +class KtestInfoMessage(NetlinkGenlMessage): + messages = [ + NlMsgProps(KtestMsgType.KTEST_CMD_LIST, NlMsgCategory.GET), + NlMsgProps(KtestMsgType.KTEST_CMD_RUN, NlMsgCategory.NEW), + NlMsgProps(KtestMsgType.KTEST_CMD_NEWTEST, NlMsgCategory.NEW), + ] + nl_attrs_map = ktest_info_attrs + family_name = KtestFamilyName + + +class KtestMsgMessage(NetlinkGenlMessage): + messages = [ + NlMsgProps(KtestMsgType.KTEST_CMD_NEWMESSAGE, NlMsgCategory.NEW), + ] + nl_attrs_map = ktest_msg_attrs + family_name = KtestFamilyName + + handler_classes = { GenlCtrlFamilyName: [NetlinkGenlCtrlMessage], + KtestFamilyName: [KtestInfoMessage, KtestMsgMessage], } diff --git a/tests/atf_python/utils.py b/tests/atf_python/utils.py index 591a532ca476..1c0a68dad383 100644 --- a/tests/atf_python/utils.py +++ b/tests/atf_python/utils.py @@ -28,6 +28,11 @@ class LibCWrapper(object): return get_errno() return 0 + def kldload(self, kld_name: str) -> int: + if self._libc.kldload(bytes(kld_name, encoding="ascii")) == -1: + return get_errno() + return 0 + def jail_attach(self, jid: int) -> int: if self._libc.jail_attach(jid) != 0: return get_errno() diff --git a/tests/conftest.py b/tests/conftest.py index 5d319863af73..8e3c004b74d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -99,6 +99,12 @@ def pytest_configure(config): handler.setup_configure() +def pytest_pycollect_makeitem(collector, name, obj): + if PLUGIN_ENABLED: + handler = get_handler() + return handler.expand_tests(collector, name, obj) + + def pytest_collection_modifyitems(session, config, items): """If cleanup is requested, replace collected tests with their cleanups (if any)""" if PLUGIN_ENABLED: diff --git a/tests/examples/Makefile b/tests/examples/Makefile index 7a5d84a98dfe..6bb87b300ee7 100644 --- a/tests/examples/Makefile +++ b/tests/examples/Makefile @@ -5,6 +5,7 @@ PACKAGE= tests TESTSDIR= ${TESTSBASE}/examples ATF_TESTS_PYTEST += test_examples.py +ATF_TESTS_PYTEST += test_ktest_example.py .include diff --git a/tests/examples/test_ktest_example.py b/tests/examples/test_ktest_example.py new file mode 100644 index 000000000000..c11f178cb054 --- /dev/null +++ b/tests/examples/test_ktest_example.py @@ -0,0 +1,35 @@ +import pytest + +from atf_python.ktest import BaseKernelTest + +from atf_python.sys.netlink.attrs import NlAttrStr +from atf_python.sys.netlink.attrs import NlAttrU32 + + +class TestExample(BaseKernelTest): + KTEST_MODULE_NAME = "ktest_example" + + @pytest.mark.parametrize( + "numbers", + [ + pytest.param([1, 2], id="1_2_Sum"), + pytest.param([3, 4], id="3_4_Sum"), + ], + ) + def test_with_params(self, numbers): + """override to parametrize""" + + test_meta = [ + NlAttrU32(1, numbers[0]), + NlAttrU32(2, numbers[1]), + NlAttrStr(3, "test string"), + ] + self.runtest(test_meta) + + @pytest.mark.skip(reason="comment me ( or delete the func) to run the test") + def test_failed(self): + pass + + @pytest.mark.skip(reason="comment me ( or delete the func) to run the test") + def test_failed2(self): + pass