feature: skip_test_env

linux_os/guide/services/base/service_psacct_enabled/rule.yml

@@ -43,3 +43,5 @@ template:
     name: service_enabled
     vars:
         servicename: psacct
+        # needs CAP_SYS_PACCT, as podman is run using normal user, there seems to be no way to elevate
+        skip_test_env: podman-based

linux_os/guide/services/fapolicyd/service_fapolicyd_enabled/rule.yml

@@ -41,3 +41,5 @@ template:
     name: service_enabled
     vars:
         servicename: fapolicyd
+        # needs CAP_DAC_OVERRIDE,CAP_SETGID,CAP_SETUID,CAP_SYS_PTRACE,CAP_SYS_ADMIN,CAP_SYS_NICE,CAP_SYS_RESOURCE,CAP_AUDIT_WRITE, as podman is run using normal user, there seems to be no way to elevate
+        skip_test_env: podman-based

linux_os/guide/system/accounts/accounts-pam/locking_out_password_attempts/account_password_selinux_faillock_dir/tests/correct_value.pass.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 # packages = policycoreutils-python-utils
 # platform = multi_platform_all
+#   selinux / restorecon does not work under podman
+# skip_test_env = podman-based
 
 truncate -s 0 /etc/security/faillock.conf
 echo "dir=/var/log/faillock" > /etc/security/faillock.conf

linux_os/guide/system/accounts/accounts-pam/locking_out_password_attempts/account_password_selinux_faillock_dir/tests/correct_value_multiple_dirs.pass.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 # packages = policycoreutils-python-utils
 # platform = multi_platform_all
+#   selinux / restorecon does not work under podman
+# skip_test_env = podman-based
 
 truncate -s 0 /etc/security/faillock.conf
 echo "dir=/var/log/faillock" > /etc/security/faillock.conf

linux_os/guide/system/accounts/accounts-pam/locking_out_password_attempts/account_password_selinux_faillock_dir/tests/missing_dir.fail.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 # packages = policycoreutils-python-utils
 # platform = multi_platform_all
+#   selinux / restorecon does not work under podman
+# skip_test_env = podman-based
 
 truncate -s 0 /etc/security/faillock.conf
 echo "dir=/var/log/faillock" > /etc/security/faillock.conf

linux_os/guide/system/accounts/accounts-pam/locking_out_password_attempts/account_password_selinux_faillock_dir/tests/wrong_value.fail.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 # packages = policycoreutils-python-utils
 # platform = multi_platform_all
+#   selinux / restorecon does not work under podman
+# skip_test_env = podman-based
 
 truncate -s 0 /etc/security/faillock.conf
 echo "dir=/var/log/faillock" > /etc/security/faillock.conf

linux_os/guide/system/accounts/accounts-pam/locking_out_password_attempts/account_password_selinux_faillock_dir/tests/wrong_value_multiple_dirs.fail.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 # packages = policycoreutils-python-utils
 # platform = multi_platform_all
+#   selinux / restorecon does not work under podman
+# skip_test_env = podman-based
 
 truncate -s 0 /etc/security/faillock.conf
 echo "dir=/var/log/faillock" > /etc/security/faillock.conf

linux_os/guide/system/auditing/service_auditd_enabled/rule.yml

@@ -92,3 +92,5 @@ template:
         packagename@ubuntu1804: auditd
         packagename@ubuntu2004: auditd
         packagename@ubuntu2204: auditd
+        # Testing needs CAP_AUDIT_* and those are not available under podman
+        skip_test_env: podman-based

linux_os/guide/system/permissions/files/dir_perms_world_writable_root_owned/tests/world_writable_dir_on_nonlocal_fs.fail.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 # packages = nfs-utils
+#   This test fails under podman as there seems to be problems setupping nfs
+#   server, nfsd kernel module might not be loaded and so on
+# skip_test_env = podman-based
 
 mkdir -p /tmp/testdir/testdir2
 mkdir /tmp/testmount

linux_os/guide/system/permissions/restrictions/sysctl_kernel_perf_event_paranoid/rule.yml

@@ -44,3 +44,4 @@ template:
         sysctlvar: kernel.perf_event_paranoid
         sysctlval: '2'
         datatype: int
+        skip_test_env: podman-based

linux_os/guide/system/selinux/selinux_all_devicefiles_labeled/tests/regular_file_device_t.pass.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+#   selinux / restorecon does not work under podman
+# skip_test_env = podman-based
 
 touch /dev/foo
 restorecon -F /dev/foo

linux_os/guide/system/selinux/selinux_all_devicefiles_labeled/tests/symlink_with_wrong_label.pass.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+#   selinux / restorecon does not work under podman
+# skip_test_env = podman-based
 
 ln -s /dev/cpu /dev/foo
 restorecon -F /dev/foo

shared/templates/service_enabled/tests/service_disabled.fail.sh

@@ -3,6 +3,7 @@
 # platform = Not Applicable
 {{% endif%}}
 # packages = {{{ PACKAGENAME }}}
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 
 SYSTEMCTL_EXEC='/usr/bin/systemctl'
 "$SYSTEMCTL_EXEC" stop '{{{ DAEMONNAME }}}.service'

shared/templates/service_enabled/tests/service_enabled.pass.sh

@@ -3,6 +3,7 @@
 # platform = Not Applicable
 {{% endif%}}
 # packages = {{{ PACKAGENAME }}}
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 
 SYSTEMCTL_EXEC='/usr/bin/systemctl'
 "$SYSTEMCTL_EXEC" unmask '{{{ DAEMONNAME }}}.service'

shared/templates/sysctl/tests/comment.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/correct_value.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/correct_value_usr_local_lib.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/line_not_there.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/one_sysctl_conf_one_sysctl_d.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/one_sysctl_conf_one_sysctl_d_conflicting.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/symlink_conflicting.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/symlink_different_option.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/symlink_repeated_sysctl_conf.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/symlink_root_duplicate.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/symlink_root_duplicate_conflicting.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/symlink_root_incompliant.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/symlink_same_option.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/symlinks_to_same_file.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/two_sysctls_on_d.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/two_sysctls_on_d_conflicting.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/two_sysctls_on_same_file.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/two_sysctls_on_same_file_name.pass.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/two_sysctls_on_same_file_name_conflicting.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/wrong_runtime.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/wrong_value.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/wrong_value_d_directory.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

shared/templates/sysctl/tests/wrong_value_usr_local_lib.fail.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# skip_test_env = {{{ SKIP_TEST_ENV }}}
 {{% if SYSCTLVAL == "" %}}
 # variables = sysctl_{{{ SYSCTLID }}}_value={{{ SYSCTL_CORRECT_VALUE }}}
 {{% endif %}}

tests/README.md

@@ -188,6 +188,15 @@ The header consists of comments (starting by `#`). Possible keys are:
   remediation breaks test environment (for example unmounting /tmp in a test
   scenario would break test runs, because OpenSCAP generates reports into the
   /tmp directory).
+- `skip_test_env` is a string specifying comma separated list of `test_env`
+  names where the running a test is skipped. Possible values are:
+  `libvirt-based`, `docker-based`, `podman-based`. Main usage is a test case
+  where feature-under-test uses something where there is some kind of (security)
+  limitation in place in testing environment. And the limitation is impossible
+  or not a good idea to remove. Especially you should not give unlimited
+  permissions to containerized testing environments, environments are not
+  superprivileged containers. For example currently containers do not have
+  SELinux, so any test trying to `restorecon` is going to fail.
 - `templates` has no effect at the moment.
 - `variables` is a comma-separated list of XCCDF values that sets a different
   default value for XCCDF variables in a form `<variable name>=<value>`.

tests/ssg_test_suite/rule.py

@@ -524,6 +524,11 @@ def _change_variable_value(self, varname, value):
         os.unlink(xslt_filename)
 
     def _check_rule_scenario(self, scenario, remote_rule_dir, rule_id, remediation_available):
+        if self.test_env.name in scenario.script_params["skip_test_env"]:
+            logging.warning('Script {0} is not applicable on given test_env {1}'
+                            .format(scenario.script, self.test_env.name))
+            return
+
         if not _apply_script(
                 remote_rule_dir, self.test_env, scenario.script):
             logging.error("Environment failed to prepare, skipping test")
@@ -584,6 +589,7 @@ def _parse_parameters(self):
             'platform': ['multi_platform_all'],
             'remediation': ['all'],
             'variables': [],
+            'skip_test_env': [],
         }
 
         for parameter in params: