kyua: Add FreeBSD Jail execution environment support

A new Kyua concept is added -- "execution environment". A test can be
configured to be run within a specific environment. The test case
lifecycle is extended respectively:
- execenv init (creates a jail or does nothing for default
  execenv="host")
- test exec
- cleanup exec (optional)
- execenv cleanup (removes a jail or does nothing for default
  execenv="host")

The following new functionality is provided, from bottom to top:

1 ATF based tests

- The new "execenv" metadata property can be set to explicitly ask for
  an execution environment: "host" or "jail". If it's not defined, as
  all existing tests do, then it implicitly means "host".

- The new "execenv.jail.params" metadata property can be optionally
  defined to ask Kyua to use specific jail(8) parameters during creation
  of a temporary jail. An example is "vnet allow.raw_sockets".

  Kyua implicitly adds "children.max" to "execenv_jail_params"
  parameters with the maximum possible value. A test case can override
  it.

2 Kyuafile

- The same new metadata properties can be defined on Kyuafile level:
  "execenv" and "execenv_jail_params".

- Note that historically ATF uses dotted style of metadata naming, while
  Kyua uses underscore style. Hence "execenv.jail.params" vs.
  "execenv_jail_params".

3 kyua.conf, kyua CLI

- The new "execenvs" engine configuration variable can be set to a list
  of execution environments to run only tests designed for. Tests of not
  listed environments are skipped.

- By default, this variable lists all execution environments supported
  by a Kyua binary, e.g. execenvs="host jail".

- This variable can be changed via "kyua.conf" or via kyua CLI's "-v"
  parameter. For example, "kyua -v execenvs=host test" will run only
  host-based tests and skip jail-based ones.

- Current value of this variable can be examined with "kyua config".

[markj] This feature has not landed upstream yet.
See the discussion in freebsd/kyua#224 .
Having the ability to automatically jail tests allows many network tests
to run in parallel, giving a drastic speedup.  So, let's import the
feature and start using it in main.

Signed-off-by:  Igor Ostapenko <pm@igoro.pro>
Reviewed by:    markj, kp
Tested by:      markj, kp
MFC after:	3 months
Differential Revision:  https://reviews.freebsd.org/D45865

contrib/kyua/AUTHORS

@@ -10,3 +10,4 @@
 
 * The FreeBSD Foundation
 * Google Inc.
+* Igor Ostapenko <pm@igoro.pro>

contrib/kyua/cli/cmd_config_test.cpp

@@ -61,6 +61,7 @@ fake_config(void)
 {
     config::tree user_config = engine::default_config();
     user_config.set_string("architecture", "the-architecture");
+    user_config.set_string("execenvs", "the-env");
     user_config.set_string("parallelism", "128");
     user_config.set_string("platform", "the-platform");
     //user_config.set_string("unprivileged_user", "");
@@ -83,12 +84,13 @@ ATF_TEST_CASE_BODY(all)
     cmdline::ui_mock ui;
     ATF_REQUIRE_EQ(EXIT_SUCCESS, cmd.main(&ui, args, fake_config()));
 
-    ATF_REQUIRE_EQ(5, ui.out_log().size());
+    ATF_REQUIRE_EQ(6, ui.out_log().size());
     ATF_REQUIRE_EQ("architecture = the-architecture", ui.out_log()[0]);
-    ATF_REQUIRE_EQ("parallelism = 128", ui.out_log()[1]);
-    ATF_REQUIRE_EQ("platform = the-platform", ui.out_log()[2]);
-    ATF_REQUIRE_EQ("test_suites.foo.bar = first", ui.out_log()[3]);
-    ATF_REQUIRE_EQ("test_suites.foo.baz = second", ui.out_log()[4]);
+    ATF_REQUIRE_EQ("execenvs = the-env", ui.out_log()[1]);
+    ATF_REQUIRE_EQ("parallelism = 128", ui.out_log()[2]);
+    ATF_REQUIRE_EQ("platform = the-platform", ui.out_log()[3]);
+    ATF_REQUIRE_EQ("test_suites.foo.bar = first", ui.out_log()[4]);
+    ATF_REQUIRE_EQ("test_suites.foo.baz = second", ui.out_log()[5]);
     ATF_REQUIRE(ui.err_log().empty());
 }
 

contrib/kyua/doc/kyua.conf.5.in

@@ -1,4 +1,4 @@
-.\" Copyright 2012 The Kyua Authors.
+.\" Copyright 2012-2024 The Kyua Authors.
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -25,7 +25,7 @@
 .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.Dd February 20, 2015
+.Dd March 22, 2024
 .Dt KYUA.CONF 5
 .Os
 .Sh NAME
@@ -36,6 +36,7 @@
 .Pp
 Variables:
 .Va architecture ,
+.Va execenvs ,
 .Va platform ,
 .Va test_suites ,
 .Va unprivileged_user .
@@ -72,6 +73,14 @@ The following variables are internally recognized by
 .Bl -tag -width XX -offset indent
 .It Va architecture
 Name of the system architecture (aka processor type).
+.It Va execenvs
+Whitespace-separated list of execution environment names.
+.Pp
+Only tests which require one of the given execution environments will be run.
+.Pp
+See
+.Xr kyuafile 5
+for the list of possible execution environments.
 .It Va parallelism
 Maximum number of test cases to execute concurrently.
 .It Va platform

contrib/kyua/doc/kyuafile.5.in

@@ -1,4 +1,4 @@
-.\" Copyright 2012 The Kyua Authors.
+.\" Copyright 2012-2024 The Kyua Authors.
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -25,7 +25,7 @@
 .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.Dd July 3, 2015
+.Dd March 23, 2024
 .Dt KYUAFILE 5
 .Os
 .Sh NAME
@@ -173,6 +173,75 @@ Refer to the
 section below for clarification.
 .It Va description
 Textual description of the test.
+.It Va execenv
+The name of the execution environment to be used for running the test.
+If empty or not defined, the
+.Sq host
+execution environment is meant.
+The possible values are:
+.Bl -tag -width xUnnnnnnn
+.It host
+The default environment which runs the test as a usual child process.
+.It jail
+The
+.Fx
+.Xr jail 8
+environment.
+It creates a temporary jail to run the test and its optional cleanup logic
+within.
+.Pp
+This feature requires
+.Xr kyua 1
+to be running with superuser privileges.
+.Pp
+The difference between
+.Va security.jail.children.max
+and
+.Va security.jail.children.cur
+sysctl of the jail
+.Xr kyua 1
+is running within must have a value high enough for the jail based tests
+planned to be run.
+For instance, the value 1 should be enough for a sequential run of simple
+tests.
+Otherwise, such aspects as parallel test execution and sub-jails spawned
+by specific test cases should be considered.
+.Pp
+The formula of a temporary jail name is
+.Sq kyua
++
+.Va test program path
++
+.Sq _
++
+.Va test case name .
+All non-alphanumeric characters are replaced with
+.Sq _ .
+.Sq kyua_usr_tests_sys_netpfil_pf_pass_block_v4
+is an example for /usr/tests/sys/netpfil/pf/pass_block:v4 test case.
+.El
+.It Va execenv_jail_params
+Additional test-specific whitespace-separated parameters of
+.Fx
+.Xr jail 8
+to create a temporary jail within which the test is run.
+It makes sense only if execenv is set to
+.Sq jail .
+.sp
+.Xr kyua 1
+implicitly passes
+.Sq children.max
+parameter to
+.Xr jail 8
+for a temporary jail with the maximum possible value according to
+the jail
+.Xr kyua 1
+itself is running within.
+It allows tests to easily spawn their own sub-jails without additional
+configuration.
+It can be overridden via
+.Va execenv_jail_params
+if needed.
 .It Va is_exclusive
 If true, indicates that this test program cannot be executed along any other
 programs at the same time.
@@ -360,6 +429,36 @@ test_suite('FreeBSD')
 plain_test_program{name='the_test',
                    ['custom.FreeBSD-Bug-Id']='category/12345'}
 .Ed
+.Ss FreeBSD jail execution environment
+The following example configures the test to be run within a temporary jail
+with
+.Xr vnet 9
+support and the permission to create raw sockets:
+.Bd -literal -offset indent
+syntax(2)
+
+test_suite('FreeBSD')
+
+atf_test_program{name='network_test',
+                 execenv='jail',
+                 execenv_jail_params='vnet allow.raw_sockets',
+                 required_user='root'}
+.Ed
+.Pp
+A test case itself may have no requirements in superuser privileges,
+but required_user='root' metadata property reminds that the jail execution
+environment requires
+.Xr kyua 1
+being running with root privileges, and the test is skipped otherwise with
+the respective message. The combination of
+.Va execenv
+set to
+.Sq jail
+and
+.Va required_user
+set to
+.Sq unprivileged
+does not work respectively.
 .Ss Connecting disjoint test suites
 Now suppose you had various test suites on your file system and you would
 like to connect them together so that they could be executed and treated as

contrib/kyua/drivers/report_junit_test.cpp

@@ -63,6 +63,8 @@ static const char* const default_metadata =
     "allowed_architectures is empty\n"
     "allowed_platforms is empty\n"
     "description is empty\n"
+    "execenv is empty\n"
+    "execenv_jail_params is empty\n"
     "has_cleanup = false\n"
     "is_exclusive = false\n"
     "required_configs is empty\n"
@@ -80,6 +82,8 @@ static const char* const overriden_metadata =
     "allowed_architectures is empty\n"
     "allowed_platforms is empty\n"
     "description = Textual description\n"
+    "execenv is empty\n"
+    "execenv_jail_params is empty\n"
     "has_cleanup = false\n"
     "is_exclusive = false\n"
     "required_configs is empty\n"
@@ -199,6 +203,8 @@ ATF_TEST_CASE_BODY(junit_metadata__overrides)
         .add_allowed_architecture("arch1")
         .add_allowed_platform("platform1")
         .set_description("This is a test")
+        .set_execenv("jail")
+        .set_execenv_jail_params("vnet")
         .set_has_cleanup(true)
         .set_is_exclusive(true)
         .add_required_config("config1")
@@ -215,6 +221,8 @@ ATF_TEST_CASE_BODY(junit_metadata__overrides)
         + "allowed_architectures = arch1\n"
         + "allowed_platforms = platform1\n"
         + "description = This is a test\n"
+        + "execenv = jail\n"
+        + "execenv_jail_params = vnet\n"
         + "has_cleanup = true\n"
         + "is_exclusive = true\n"
         + "required_configs = config1\n"

contrib/kyua/engine/atf.cpp

@@ -39,6 +39,7 @@ extern "C" {
 #include "engine/atf_list.hpp"
 #include "engine/atf_result.hpp"
 #include "engine/exceptions.hpp"
+#include "engine/execenv/execenv.hpp"
 #include "model/test_case.hpp"
 #include "model/test_program.hpp"
 #include "model/test_result.hpp"
@@ -54,6 +55,7 @@ extern "C" {
 #include "utils/stream.hpp"
 
 namespace config = utils::config;
+namespace execenv = engine::execenv;
 namespace fs = utils::fs;
 namespace process = utils::process;
 
@@ -190,7 +192,10 @@ engine::atf_interface::exec_test(const model::test_program& test_program,
 
     args.push_back(F("-r%s") % (control_directory / result_name));
     args.push_back(test_case_name);
-    process::exec(test_program.absolute_path(), args);
+
+    auto e = execenv::get(test_program, test_case_name);
+    e->init();
+    e->exec(args);
 }
 
 
@@ -219,7 +224,9 @@ engine::atf_interface::exec_cleanup(
     }
 
     args.push_back(F("%s:cleanup") % test_case_name);
-    process::exec(test_program.absolute_path(), args);
+
+    auto e = execenv::get(test_program, test_case_name);
+    e->exec(args);
 }
 
 

contrib/kyua/engine/atf_list.cpp

@@ -121,6 +121,10 @@ engine::parse_atf_metadata(const model::properties_map& props)
                 mdbuilder.set_string("has_cleanup", value);
             } else if (name == "require.arch") {
                 mdbuilder.set_string("allowed_architectures", value);
+            } else if (name == "execenv") {
+                mdbuilder.set_string("execenv", value);
+            } else if (name == "execenv.jail.params") {
+                mdbuilder.set_string("execenv_jail_params", value);
             } else if (name == "require.config") {
                 mdbuilder.set_string("required_configs", value);
             } else if (name == "require.files") {

contrib/kyua/engine/config.cpp

@@ -35,6 +35,7 @@
 #include <stdexcept>
 
 #include "engine/exceptions.hpp"
+#include "engine/execenv/execenv.hpp"
 #include "utils/config/exceptions.hpp"
 #include "utils/config/parser.hpp"
 #include "utils/config/tree.ipp"
@@ -43,6 +44,7 @@
 #include "utils/text/operations.ipp"
 
 namespace config = utils::config;
+namespace execenv = engine::execenv;
 namespace fs = utils::fs;
 namespace passwd = utils::passwd;
 namespace text = utils::text;
@@ -59,6 +61,7 @@ static void
 init_tree(config::tree& tree)
 {
     tree.define< config::string_node >("architecture");
+    tree.define< config::strings_set_node >("execenvs");
     tree.define< config::positive_int_node >("parallelism");
     tree.define< config::string_node >("platform");
     tree.define< engine::user_node >("unprivileged_user");
@@ -74,6 +77,14 @@ static void
 set_defaults(config::tree& tree)
 {
     tree.set< config::string_node >("architecture", KYUA_ARCHITECTURE);
+
+    std::set< std::string > supported;
+    for (auto em : execenv::execenvs())
+        if (em->is_supported())
+            supported.insert(em->name());
+    supported.insert(execenv::default_execenv_name);
+    tree.set< config::strings_set_node >("execenvs", supported);
+
     // TODO(jmmv): Automatically derive this from the number of CPUs in the
     // machine and forcibly set to a value greater than 1.  Still testing
     // the new parallel implementation as of 2015-02-27 though.
@@ -229,6 +240,13 @@ engine::empty_config(void)
 {
     config::tree tree(false);
     init_tree(tree);
+
+    // Tests of Kyua itself tend to use an empty config, i.e. default
+    // execution environment is used. Let's allow it.
+    std::set< std::string > supported;
+    supported.insert(engine::execenv::default_execenv_name);
+    tree.set< config::strings_set_node >("execenvs", supported);
+
     return tree;
 }