Add FreeBSD Jail execution environment support

A new Kyua concept is added -- "execution environment". A test can be
configured to be run within a specific environment. The test case lifecycle
is extended respectively:
- execenv init (creates a jail or does nothing for default execenv="host")
- test exec
- cleanup exec (optional)
- execenv cleanup (removes a jail or does nothing for default execenv="host")

The following new functionality is provided, from bottom to top:

1 ATF based tests

- The new "execenv" metadata property can be set to explicitly ask for an
  execution environment: "host" or "jail". If it's not defined, as all
  existing tests do, then it implicitly means "host".

- The new "execenv.jail" metadata property can be optionally defined to ask
  Kyua to use specific jail(8) parameters during creation of a temporary
  jail. An example is "vnet allow.raw_sockets".

  Kyua implicitly adds "children.max" to "execenv_jail" parameters with the
  maximum possible value. A test case can override it.

2 Kyuafile

- The same new metadata properties can be defined on Kyuafile level:
  "execenv" and "execenv_jail".

- Note that historically ATF uses dotted style of metadata naming, while
  Kyua uses underscore style. Hence "execenv.jail" vs. "execenv_jail".

3 kyua.conf, kyua CLI

- The new "execenv" engine configuration variable can be set to a list of
  execution environments to run only tests designed for. Tests of not listed
  environments are skipped.

- By default, this variable lists all execution environments supported by a
  Kyua binary, e.g. execenv="host jail".

- This variable can be changed via "kyua.conf" or via kyua CLI's "-v"
  parameter. For example, "kyua -v execenv=host test" will run only
  host-based tests and skip jail-based ones.

- Current value of this variable can be examined with "kyua config".

Signed-off-by: Igor Ostapenko <pm@igoro.pro>

AUTHORS

@@ -10,3 +10,4 @@
 
 * The FreeBSD Foundation
 * Google Inc.
+* Igor Ostapenko <pm@igoro.pro>

Makefile.am

@@ -59,6 +59,7 @@ include doc/Makefile.am.inc
 include drivers/Makefile.am.inc
 include engine/Makefile.am.inc
 include examples/Makefile.am.inc
+include freebsd/Makefile.am.inc
 include integration/Makefile.am.inc
 include misc/Makefile.am.inc
 include model/Makefile.am.inc
@@ -68,7 +69,7 @@ include utils/Makefile.am.inc
 bin_PROGRAMS = kyua
 kyua_SOURCES = main.cpp
 kyua_CXXFLAGS = $(CLI_CFLAGS) $(ENGINE_CFLAGS) $(UTILS_CFLAGS)
-kyua_LDADD = $(CLI_LIBS) $(ENGINE_LIBS) $(UTILS_LIBS)
+kyua_LDADD = $(CLI_LIBS) $(ENGINE_LIBS) $(FREEBSD_LIBS) $(UTILS_LIBS)
 
 CHECK_ENVIRONMENT = KYUA_CONFDIR="/non-existent" \
                     KYUA_DOCDIR="$(abs_top_srcdir)" \

cli/cmd_config_test.cpp

@@ -61,6 +61,7 @@ fake_config(void)
 {
     config::tree user_config = engine::default_config();
     user_config.set_string("architecture", "the-architecture");
+    user_config.set_string("execenv", "the-env");
     user_config.set_string("parallelism", "128");
     user_config.set_string("platform", "the-platform");
     //user_config.set_string("unprivileged_user", "");
@@ -83,12 +84,13 @@ ATF_TEST_CASE_BODY(all)
     cmdline::ui_mock ui;
     ATF_REQUIRE_EQ(EXIT_SUCCESS, cmd.main(&ui, args, fake_config()));
 
-    ATF_REQUIRE_EQ(5, ui.out_log().size());
+    ATF_REQUIRE_EQ(6, ui.out_log().size());
     ATF_REQUIRE_EQ("architecture = the-architecture", ui.out_log()[0]);
-    ATF_REQUIRE_EQ("parallelism = 128", ui.out_log()[1]);
-    ATF_REQUIRE_EQ("platform = the-platform", ui.out_log()[2]);
-    ATF_REQUIRE_EQ("test_suites.foo.bar = first", ui.out_log()[3]);
-    ATF_REQUIRE_EQ("test_suites.foo.baz = second", ui.out_log()[4]);
+    ATF_REQUIRE_EQ("execenv = the-env", ui.out_log()[1]);
+    ATF_REQUIRE_EQ("parallelism = 128", ui.out_log()[2]);
+    ATF_REQUIRE_EQ("platform = the-platform", ui.out_log()[3]);
+    ATF_REQUIRE_EQ("test_suites.foo.bar = first", ui.out_log()[4]);
+    ATF_REQUIRE_EQ("test_suites.foo.baz = second", ui.out_log()[5]);
     ATF_REQUIRE(ui.err_log().empty());
 }
 

configure.ac

@@ -169,5 +169,6 @@ fi
 AM_CONDITIONAL(TARGET_SRCDIR_EMPTY, [test -z "${target_srcdir}"])
 AC_SUBST([target_srcdir])
 
+AM_CONDITIONAL([FreeBSD], [test "$(uname -o)" = "FreeBSD"])
 
 AC_OUTPUT

doc/kyua.conf.5.in

@@ -1,4 +1,4 @@
-.\" Copyright 2012 The Kyua Authors.
+.\" Copyright 2012-2024 The Kyua Authors.
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -25,7 +25,7 @@
 .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.Dd February 20, 2015
+.Dd March 22, 2024
 .Dt KYUA.CONF 5
 .Os
 .Sh NAME
@@ -36,6 +36,7 @@
 .Pp
 Variables:
 .Va architecture ,
+.Va execenv ,
 .Va platform ,
 .Va test_suites ,
 .Va unprivileged_user .
@@ -72,6 +73,14 @@ The following variables are internally recognized by
 .Bl -tag -width XX -offset indent
 .It Va architecture
 Name of the system architecture (aka processor type).
+.It Va execenv
+Whitespace-separated list of execution environment names.
+.Pp
+Only tests which require one of the given execution environments will be run.
+.Pp
+See
+.Xr kyuafile 5
+for the list of possible execution environments.
 .It Va parallelism
 Maximum number of test cases to execute concurrently.
 .It Va platform

doc/kyuafile.5.in

@@ -1,4 +1,4 @@
-.\" Copyright 2012 The Kyua Authors.
+.\" Copyright 2012-2024 The Kyua Authors.
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -25,7 +25,7 @@
 .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.Dd July 3, 2015
+.Dd March 23, 2024
 .Dt KYUAFILE 5
 .Os
 .Sh NAME
@@ -173,6 +173,75 @@ Refer to the
 section below for clarification.
 .It Va description
 Textual description of the test.
+.It Va execenv
+The name of the execution environment to be used for running the test.
+If empty or not defined, the
+.Sq host
+execution environment is meant.
+The possible values are:
+.Bl -tag -width xUnnnnnnn
+.It host
+The default environment which runs the test as a usual child process.
+.It jail
+The
+.Fx
+.Xr jail 8
+environment.
+It creates a temporary jail to run the test and its optional cleanup logic
+within.
+.Pp
+This feature requires
+.Xr kyua 1
+to be running with superuser privileges.
+.Pp
+The difference between
+.Va security.jail.children.max
+and
+.Va security.jail.children.cur
+sysctl of the jail
+.Xr kyua 1
+is running within must have a value high enough for the jail based tests
+planned to be run.
+For instance, the value 1 should be enough for a sequential run of simple
+tests.
+Otherwise, such aspects as parallel test execution and sub-jails spawned
+by specific test cases should be considered.
+.Pp
+The formula of a temporary jail name is
+.Sq kyua
++
+.Va test program path
++
+.Sq _
++
+.Va test case name .
+All non-alphanumeric characters are replaced with
+.Sq _ .
+.Sq kyua_usr_tests_sys_netpfil_pf_pass_block_v4
+is an example for /usr/tests/sys/netpfil/pf/pass_block:v4 test case.
+.El
+.It Va execenv_jail
+Additional test-specific whitespace-separated parameters of
+.Fx
+.Xr jail 8
+to create a temporary jail within which the test is run.
+It makes sense only if execenv is set to
+.Sq jail .
+.sp
+.Xr kyua 1
+implicitly passes
+.Sq children.max
+parameter to
+.Xr jail 8
+for a temporary jail with the maximum possible value according to
+the jail
+.Xr kyua 1
+itself is running within.
+It allows tests to easily spawn their own sub-jails without additional
+configuration.
+It can be overridden via
+.Va execenv_jail
+if needed.
 .It Va is_exclusive
 If true, indicates that this test program cannot be executed along any other
 programs at the same time.
@@ -360,6 +429,36 @@ test_suite('FreeBSD')
 plain_test_program{name='the_test',
                    ['custom.FreeBSD-Bug-Id']='category/12345'}
 .Ed
+.Ss FreeBSD jail execution environment
+The following example configures the test to be run within a temporary jail
+with
+.Xr vnet 9
+support and the permission to create raw sockets:
+.Bd -literal -offset indent
+syntax(2)
+
+test_suite('FreeBSD')
+
+atf_test_program{name='network_test',
+                 execenv='jail',
+                 execenv_jail='vnet allow.raw_sockets',
+                 required_user='root'}
+.Ed
+.Pp
+A test case itself may have no requirements in superuser privileges,
+but required_user='root' metadata property reminds that the jail execution
+environment requires
+.Xr kyua 1
+being running with root privileges, and the test is skipped otherwise with
+the respective message. The combination of
+.Va execenv
+set to
+.Sq jail
+and
+.Va required_user
+set to
+.Sq unprivileged
+does not work respectively.
 .Ss Connecting disjoint test suites
 Now suppose you had various test suites on your file system and you would
 like to connect them together so that they could be executed and treated as

drivers/report_junit_test.cpp

@@ -63,6 +63,8 @@ static const char* const default_metadata =
     "allowed_architectures is empty\n"
     "allowed_platforms is empty\n"
     "description is empty\n"
+    "execenv is empty\n"
+    "execenv_jail is empty\n"
     "has_cleanup = false\n"
     "is_exclusive = false\n"
     "required_configs is empty\n"
@@ -80,6 +82,8 @@ static const char* const overriden_metadata =
     "allowed_architectures is empty\n"
     "allowed_platforms is empty\n"
     "description = Textual description\n"
+    "execenv is empty\n"
+    "execenv_jail is empty\n"
     "has_cleanup = false\n"
     "is_exclusive = false\n"
     "required_configs is empty\n"
@@ -199,6 +203,8 @@ ATF_TEST_CASE_BODY(junit_metadata__overrides)
         .add_allowed_architecture("arch1")
         .add_allowed_platform("platform1")
         .set_description("This is a test")
+	.set_execenv("jail")
+	.set_execenv_jail("vnet")
         .set_has_cleanup(true)
         .set_is_exclusive(true)
         .add_required_config("config1")
@@ -215,6 +221,8 @@ ATF_TEST_CASE_BODY(junit_metadata__overrides)
         + "allowed_architectures = arch1\n"
         + "allowed_platforms = platform1\n"
         + "description = This is a test\n"
+        + "execenv = jail\n"
+        + "execenv_jail = vnet\n"
         + "has_cleanup = true\n"
         + "is_exclusive = true\n"
         + "required_configs = config1\n"

engine/Makefile.am.inc

@@ -153,3 +153,5 @@ engine_scheduler_test_SOURCES = engine/scheduler_test.cpp
 engine_scheduler_test_CXXFLAGS = $(ENGINE_CFLAGS) $(ATF_CXX_CFLAGS)
 engine_scheduler_test_LDADD = $(ENGINE_LIBS) $(ATF_CXX_LIBS)
 endif
+
+include engine/execenv/Makefile.am.inc

engine/atf.cpp

@@ -39,6 +39,7 @@ extern "C" {
 #include "engine/atf_list.hpp"
 #include "engine/atf_result.hpp"
 #include "engine/exceptions.hpp"
+#include "engine/execenv/execenv.hpp"
 #include "model/test_case.hpp"
 #include "model/test_program.hpp"
 #include "model/test_result.hpp"
@@ -54,6 +55,7 @@ extern "C" {
 #include "utils/stream.hpp"
 
 namespace config = utils::config;
+namespace execenv = engine::execenv;
 namespace fs = utils::fs;
 namespace process = utils::process;
 
@@ -190,7 +192,10 @@ engine::atf_interface::exec_test(const model::test_program& test_program,
 
     args.push_back(F("-r%s") % (control_directory / result_name));
     args.push_back(test_case_name);
-    process::exec(test_program.absolute_path(), args);
+
+    auto e = execenv::get(test_program, test_case_name);
+    e->init();
+    e->exec(args);
 }
 
 
@@ -219,7 +224,9 @@ engine::atf_interface::exec_cleanup(
     }
 
     args.push_back(F("%s:cleanup") % test_case_name);
-    process::exec(test_program.absolute_path(), args);
+
+    auto e = execenv::get(test_program, test_case_name);
+    e->exec(args);
 }
 
 

engine/atf_list.cpp

@@ -121,6 +121,10 @@ engine::parse_atf_metadata(const model::properties_map& props)
                 mdbuilder.set_string("has_cleanup", value);
             } else if (name == "require.arch") {
                 mdbuilder.set_string("allowed_architectures", value);
+            } else if (name == "execenv") {
+                mdbuilder.set_string("execenv", value);
+            } else if (name == "execenv.jail") {
+                mdbuilder.set_string("execenv_jail", value);
             } else if (name == "require.config") {
                 mdbuilder.set_string("required_configs", value);
             } else if (name == "require.files") {

engine/config.cpp

@@ -35,6 +35,7 @@
 #include <stdexcept>
 
 #include "engine/exceptions.hpp"
+#include "engine/execenv/execenv.hpp"
 #include "utils/config/exceptions.hpp"
 #include "utils/config/parser.hpp"
 #include "utils/config/tree.ipp"
@@ -43,6 +44,7 @@
 #include "utils/text/operations.ipp"
 
 namespace config = utils::config;
+namespace execenv = engine::execenv;
 namespace fs = utils::fs;
 namespace passwd = utils::passwd;
 namespace text = utils::text;
@@ -59,6 +61,7 @@ static void
 init_tree(config::tree& tree)
 {
     tree.define< config::string_node >("architecture");
+    tree.define< config::strings_set_node >("execenv");
     tree.define< config::positive_int_node >("parallelism");
     tree.define< config::string_node >("platform");
     tree.define< engine::user_node >("unprivileged_user");
@@ -74,6 +77,14 @@ static void
 set_defaults(config::tree& tree)
 {
     tree.set< config::string_node >("architecture", KYUA_ARCHITECTURE);
+
+    std::set< std::string > supported;
+    for (auto em : execenv::execenvs())
+        if (em->is_supported())
+            supported.insert(em->name());
+    supported.insert(execenv::default_execenv_name);
+    tree.set< config::strings_set_node >("execenv", supported);
+
     // TODO(jmmv): Automatically derive this from the number of CPUs in the
     // machine and forcibly set to a value greater than 1.  Still testing
     // the new parallel implementation as of 2015-02-27 though.
@@ -229,6 +240,13 @@ engine::empty_config(void)
 {
     config::tree tree(false);
     init_tree(tree);
+
+    // Tests of Kyua itself tend to use an empty config, i.e. default
+    // execution environment is used. Let's allow it.
+    std::set< std::string > supported;
+    supported.insert(engine::execenv::default_execenv_name);
+    tree.set< config::strings_set_node >("execenv", supported);
+
     return tree;
 }
 

engine/execenv/Makefile.am.inc

@@ -0,0 +1,32 @@
+# Copyright 2024 The Kyua Authors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+# * Neither the name of Google Inc. nor the names of its contributors
+#   may be used to endorse or promote products derived from this software
+#   without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+libengine_a_SOURCES += engine/execenv/execenv.hpp
+libengine_a_SOURCES += engine/execenv/execenv.cpp
+libengine_a_SOURCES += engine/execenv/execenv_host.hpp
+libengine_a_SOURCES += engine/execenv/execenv_host.cpp