From b03cc0d4a6306963b7bbd04bcc81758de9fb81fb Mon Sep 17 00:00:00 2001 From: Jiri Jaburek Date: Thu, 28 Mar 2024 17:52:13 +0100 Subject: [PATCH] turn waivers into a directory Signed-off-by: Jiri Jaburek --- README.md | 12 +- WAIVERS.md | 23 ++- conf/waivers-released | 248 ------------------------------- conf/waivers-upstream | 301 -------------------------------------- conf/waivers/10-unknown | 125 ++++++++++++++++ conf/waivers/20-long-term | 139 ++++++++++++++++++ conf/waivers/30-permanent | 63 ++++++++ lib/waive.py | 34 +++-- 8 files changed, 381 insertions(+), 564 deletions(-) delete mode 100644 conf/waivers-released delete mode 100644 conf/waivers-upstream create mode 100644 conf/waivers/10-unknown create mode 100644 conf/waivers/20-long-term create mode 100644 conf/waivers/30-permanent diff --git a/README.md b/README.md index 5da40d51..140885fa 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,14 @@ on Red Hat Enterprise Linux. - `1` (default) is `fail`, `error` and `warn` - `2` or greater to output everything -- `CONTEST_WAIVERS` - - Specify a `conf/waiver-` suffix for a waiver file name inside `conf` to be - used for waiving results. Ie. `CONTEST_WAIVERS=upstream` to use - `conf/waivers-upstream`. Defaults to `released`. +- `CONTEST_WAIVER_DIR` + - Specify a relative path to a waiver directory containing waiver files. + - The directory itself is traversed recursively (may contain further + sub-directories). + - All files and directories are read in a locale-specific sorted order, + and their contents combined to a final list of waiver rules. + - Files and directories starting with `.` are ignored. + - Defaults to `conf/waivers`. - `CONTEST_LEAVE_GUEST_RUNNING` - Set to `1` to break gurantees provided by `class Guest()`, that is make the diff --git a/WAIVERS.md b/WAIVERS.md index 3857532d..53311f54 100644 --- a/WAIVERS.md +++ b/WAIVERS.md @@ -1,9 +1,28 @@ # Waiving known failures +## Directory structure + +The files inside the [waivers directory](conf/waivers) are read in +an alphanumeric order, including (sub)directories and any files inside them, +forming a contiguous list of `
`s (as described below) from their +contents. + +By convention, file names inside the waivers directory have specific meaning: + +- `unknown` - these are for waiving not-yet-known failures that we simply + don't want failing in daily runs while somebody investigates them +- `long-term` - these are for investigated issues, which typically have + bugs or issues filed, and are waiting for a fix +- `permanent` - these are never expected to be fixed, as they are a result + of the test infrastructure specifics, or other test or technology + limitations + - note that they may still disappear over time, ie. by dropping support + for an old OS release, which had the limitation + ## Custom file format -The [waivers file](conf/waivers) uses a custom file format to specify when -a failure is expected. +The files inside the waivers directory use a custom file format to specify +when a failure is expected. This format consists of so-called "sections", optionally separated by any number of empty lines. Any lines beginning with `#` are skipped. diff --git a/conf/waivers-released b/conf/waivers-released deleted file mode 100644 index c1cd2e61..00000000 --- a/conf/waivers-released +++ /dev/null @@ -1,248 +0,0 @@ -# remediation for this is globally disabled in Contest, -# see conf/remediation.py -/hardening/.*/accounts_password_set_max_life_existing - Match(True, sometimes=True) - -# requires running firewalld (firewall-cmd) and NetworkManager, -# which are not available in their final form in the Anaconda environment -# - see https://github.com/ComplianceAsCode/content/issues/9746 -# https://github.com/ComplianceAsCode/content/issues/11318 -/hardening/anaconda(/with-gui)?/[^/]+/firewalld_sshd_port_enabled -/hardening/image-builder(/with-gui)?/[^/]+/firewalld_sshd_port_enabled -# https://github.com/ComplianceAsCode/content/issues/10939 -# https://github.com/ComplianceAsCode/content/issues/10613 -/hardening/anaconda(/with-gui)?/[^/]+/firewalld_loopback_traffic_(restricted|trusted) -# https://github.com/ComplianceAsCode/content/issues/11564 -/hardening/image-builder(/with-gui)?/[^/]+/firewalld_loopback_traffic_(restricted|trusted) - rhel >= 8 - -# rule ordering issue - accounts_password_pam_retry is checked first and passes, -# and a later enable_authselect remediation breaks it -# - see https://github.com/OpenSCAP/openscap/issues/1880 -/hardening/anaconda/.+/accounts_password_pam_retry - rhel >= 8 - -# bz1828871, won't be fixed on rhel7 -# - likely NOT related to rule ordering, double remediation doesn't help -/hardening/anaconda(/with-gui)?/[^/]+/postfix_network_listening_disabled -# https://github.com/ComplianceAsCode/content/issues/10938 -/hardening/host-os/oscap/[^/]+/postfix_network_listening_disabled - Match(rhel == 7, sometimes=True) - -# caused by one of: -# - bz1778661 (abrt) -# - bz2209266 (RHEL-9 gdm) -# - bz2209294 (RHEL-7 gdm, different issue) -/hardening/.+/rpm_verify_(ownership|permissions) - Match(True, sometimes=True) - -# bz1825810 or maybe bz1929805 -# can be reproduced mostly reliably (95%) both via anaconda and oscap CLI, -# but apparently we don't really care about old releases -# -# also re-discovered on RHEL-8 via -# https://github.com/ComplianceAsCode/content/issues/10937 -# and afterwards on other profiles (anssi_bp28_high), but still -# only on GUI -/hardening/[^/]+/with-gui/[^/]+/sysctl_net_ipv4_ip_forward - Match(rhel <= 8, sometimes=True) - -# https://github.com/ComplianceAsCode/content/issues/10424 -# happens on host-os hardening too, probably because Beaker doesn't have -# firewall enabled or even installed -/hardening/anaconda(/with-gui)?/[^/]+/service_nftables_disabled -/hardening/host-os/oscap/[^/]+/service_nftables_disabled - Match(True, sometimes=True) - -# caused by us not caring whether the VM has it installed, -# - remediation should fix it, but doesn't -- possibly caused by -# https://github.com/RHSecurityCompliance/contest/issues/15 -/hardening/oscap(/with-gui)?/[^/]+/package_scap-security-guide_installed - Match(True, sometimes=True) - -# TODO: something new? .. RHEL-8 on e8 and ism_o, RHEL-7 e8 -# - seems to not happen on latest 8.9 nightlies ?? -# - on latest 7.9, but upstream 2023/05 content -/hardening/oscap/[^/]+/package_rear_installed - Match(rhel <= 8, sometimes=True) - -# RHEL-8: https://bugzilla.redhat.com/show_bug.cgi?id=1834716 -# RHEL-9: https://bugzilla.redhat.com/show_bug.cgi?id=1999587 -/hardening/anaconda/with-gui/[^/]+/service_avahi-daemon_disabled - True - -# RHEL-9 is not FIPS certified yet -/hardening/.+/aide_use_fips_hashes - rhel == 9 - -# ssh either doesn't start up, or gets blocked, possibly related -# to new firewalld rules being added? -# https://github.com/ComplianceAsCode/content/pull/10573 -# (happens only with GUI) -# -# or perhaps one of -# https://github.com/ComplianceAsCode/content/issues/10592 -# https://github.com/ComplianceAsCode/content/issues/10593 -# https://github.com/ComplianceAsCode/content/issues/10594 -/hardening/oscap/with-gui/cis_workstation_l[12] - status == 'error' - -# OAA just failed without an error, as usual -# https://issues.redhat.com/browse/OPENSCAP-3321 -# seems to be happening much more reliably with GUI -/hardening/anaconda/with-gui/cis_workstation_l[12] - Match(status == 'error', sometimes=True) - -# happened in Beaker, but uses VMs, so it shouldn't be Beaker-specific -# TODO: investigate, seems to be RHEL-9.3+ but unsure -/hardening/oscap/with-gui/.+/package_aide_installed -/hardening/oscap/with-gui/.+/aide_build_database -/hardening/oscap/with-gui/.+/aide_periodic_cron_checking -/hardening/oscap/with-gui/.+/aide_scan_notification -/hardening/oscap/with-gui/.+/aide_verify_acls -/hardening/oscap/with-gui/.+/aide_verify_ext_attributes - Match(True, sometimes=True) - -# Beaker-specific: -# all Beaker repositories have gpgcheck=0 and they get copied to nested VMs too -/(hardening|scanning)/.+/ensure_gpgcheck_never_disabled -# we don't control partitions on the host OS -/hardening/host-os/oscap/.+/mount_option_(home|opt|srv|var|var_log|var_log_audit)_(noexec|nosuid|nodev|usrquota|grpquota) -/hardening/host-os/oscap/.+/mount_option_boot_efi_nosuid -# likely something caused by restraint / Beaker test env -/hardening/host-os/.+/file_permissions_unauthorized_world_writable -# Beaker and host-os seem to randomly fail any services enabled -# or packages installed - TODO investigate remediation script outputs -# to figure out why -/hardening/host-os/oscap/[^/]+/service_.+_enabled -/hardening/host-os/oscap/[^/]+/timer_.+_enabled -/hardening/host-os/oscap/[^/]+/package_.+_installed -# TODO: unknown, probably worth investigating -/hardening/host-os/oscap/.+/sysctl_net_ipv6_conf_(all|default)_accept_ra -/hardening/host-os/oscap/.+/sysctl_net_ipv4_conf_default_log_martians - Match(True, sometimes=True) - -# Beaker-specific, possibly; -# same for dnf-automatic and rsyslog (??), is this fully random? -/hardening/host-os/oscap/[^/]+/package_dnf-automatic_installed -/hardening/host-os/oscap/[^/]+/timer_dnf-automatic_enabled -/hardening/host-os/oscap/[^/]+/package_rsyslog-gnutls_installed - Match(rhel >= 8, sometimes=True) - -# seems RHEL-8 specific, unknown, TODO investigate -# remediation script says: -# Current configuration is valid. -# Current configuration is valid. -# [error] Unknown profile feature [with-smartcard] -# [error] Unable to activate profile [custom/hardening] [22]: Invalid argument -# Unable to enable feature [22]: Invalid argument -# maybe hardware-specific and our Beaker systems don't have the hardware? -/hardening/host-os/oscap/.+/sssd_enable_smartcards - Match(rhel == 8, sometimes=True) - -# visible on double reboot (double remediation) -# /usr/libexec/abrt-action-install-debuginfo-to-abrt-cache -/hardening/oscap/with-gui/e8/file_ownership_binary_dirs - rhel == 7 - -# TODO: completely unknown, investigate and sort -# -# all RHELs -/hardening/ansible/.+/mount_option_boot_noexec -/hardening/ansible/.+/mount_option_boot_nosuid -/hardening/ansible/.+/mount_option_home_noexec -/hardening/ansible/.+/audit_rules_usergroup_modification - True -# RHEL-9 only -/hardening(/host-os)?/ansible/.+/dnf-automatic_apply_updates -/hardening(/host-os)?/ansible/.+/dnf-automatic_security_updates_only -/hardening(/host-os)?/ansible/.+/accounts_polyinstantiated_tmp -/hardening(/host-os)?/ansible/.+/accounts_polyinstantiated_var_tmp -/hardening(/host-os)?/ansible/.+/force_opensc_card_drivers -/hardening/ansible/with-gui/.+/network_nmcli_permissions - rhel == 9 -# RHEL-8 or 9 -/hardening(/host-os)?/ansible/.+/no_tmux_in_shells -/hardening(/host-os)?/ansible/.+/configure_usbguard_auditbackend -/hardening(/host-os)?/ansible/.+/audit_rules_unsuccessful_file_modification - rhel == 8 or rhel == 9 -# RHEL-8 -/hardening/ansible/with-gui/stig_gui/sysctl_net_ipv4_conf_all_forwarding - rhel == 8 -# RHEL-7 -/hardening(/host-os)?/ansible/.+/sshd_use_strong_ciphers -/hardening(/host-os)?/ansible/.+/sshd_use_strong_macs -/hardening(/host-os)?/ansible/.+/audit_rules_for_ospp -/hardening(/host-os)?/ansible/.+/aide_use_fips_hashes -/hardening(/host-os)?/ansible/.+/smartcard_auth - rhel == 7 - -# unknown as well, but happens only rarely -/hardening/ansible/.+/configure_bashrc_exec_tmux -# home_nosuid failures are just really random across RHEL versions and nightlies -/hardening/ansible/.+/mount_option_home_nosuid - Match(True, sometimes=True) - -# only pci-dss, passes everywhere else -/hardening/ansible(/with-gui)?/pci-dss/audit_rules_login_events - rhel == 8 or rhel == 9 - -# https://bugzilla.redhat.com/show_bug.cgi?id=1797653 WONTFIX -/scanning/oscap-eval/ERROR - rhel <= 8 and note == 'E: oscap: Failed to convert OVAL state to SEXP, id: oval:ssg-state_file_groupowner_var_log_syslog_gid_4_0:ste:1.' - -# https://github.com/ComplianceAsCode/content/issues/10901 -# not sure what enables the service, but second remediation fixes the problem -/hardening/anaconda/with-gui/[^/]+/service_rpcbind_disabled - Match(rhel == 8, sometimes=True) - -# https://github.com/ComplianceAsCode/content/issues/10938 -/hardening/host-os/oscap/anssi_nt28_high/audit_rules_privileged_commands - rhel == 7 and arch == 'ppc64le' - -# Image Builder errors -# -# https://issues.redhat.com/browse/RHEL-24246 -/hardening/image-builder/anssi_bp28_high - status == 'error' and 'TimeoutError' in note -# 'composer-cli compose start' fails with: -# OpenSCAP unsupported profile: xccdf_org.ssgproject.content_profile_ccn_advanced -/hardening/image-builder/ccn_advanced - status == 'error' -# https://github.com/ComplianceAsCode/content/issues/11344 -/hardening/image-builder/cis[^/]* - rhel == 9 and status == 'error' -# https://issues.redhat.com/browse/RHEL-24246 -/hardening/image-builder/cui - rhel == 8 and status == 'error' -# https://issues.redhat.com/browse/RHEL-24441 on 'squid' -/hardening/image-builder/e8 - rhel == 9 and status == 'error' -# https://issues.redhat.com/browse/RHEL-24441 on 'telnet' -/hardening/image-builder/hipaa - rhel == 9 and status == 'error' -# RHEL-9: https://issues.redhat.com/browse/RHEL-24441 on 'snmpd' -/hardening/image-builder/ism_o - rhel == 9 and status == 'error' -# https://issues.redhat.com/browse/RHEL-24246 -/hardening/image-builder/ospp - rhel == 8 and status == 'error' -# RHEL-9: https://issues.redhat.com/browse/RHEL-24441 on 'autofs' -# RHEL-8: https://issues.redhat.com/browse/RHEL-24246 -/hardening/image-builder/stig - status == 'error' - -# Image Builder failures -# -# https://github.com/ComplianceAsCode/content/issues/11565 -/hardening/image-builder/.*/audit_rules_privileged_commands - True -# https://github.com/ComplianceAsCode/content/issues/11566 -/hardening/image-builder/hipaa/sebool_selinuxuser_execstack - rhel == 8 -# https://github.com/ComplianceAsCode/content/issues/11567 -/hardening/image-builder/.*/enable_dracut_fips_module -/hardening/image-builder/.*/enable_fips_mode - True - -# vim: syntax=python diff --git a/conf/waivers-upstream b/conf/waivers-upstream deleted file mode 100644 index 76dd22f4..00000000 --- a/conf/waivers-upstream +++ /dev/null @@ -1,301 +0,0 @@ -# remediation for this is globally disabled in Contest, -# see conf/remediation.py -/hardening/.*/accounts_password_set_max_life_(existing|root) - Match(True, sometimes=True) -# RHEL-7 only, because RHEL-8+ check for rsync-daemon, -# while TMT only needs the 'rsync' command executed via ssh, -# so remove the remediation exception + waiver on RHEL-8+ -/hardening/host-os/.+/[^/]+/package_rsync_removed - rhel == 7 -# the service_sssd_enabled will be failing even if the service is enabled -# because it requires manual configuration which cannot be attained with our rules -/hardening/.+/service_sssd_enabled - True - -# requires running firewalld (firewall-cmd) and NetworkManager, -# which are not available in their final form in the Anaconda environment -# - see https://github.com/ComplianceAsCode/content/issues/9746 -/hardening/anaconda(/with-gui)?/[^/]+/firewalld_sshd_port_enabled -# https://github.com/ComplianceAsCode/content/issues/11625 -/hardening/image-builder(/with-gui)?/[^/]+/firewalld_sshd_port_enabled -# https://github.com/ComplianceAsCode/content/issues/10939 -# https://github.com/ComplianceAsCode/content/issues/10613 -/hardening/anaconda(/with-gui)?/[^/]+/firewalld_loopback_traffic_(restricted|trusted) -# https://github.com/ComplianceAsCode/content/issues/11564 -/hardening/image-builder(/with-gui)?/[^/]+/firewalld_loopback_traffic_(restricted|trusted) - rhel >= 8 - -# rule ordering issue - accounts_password_pam_retry is checked first and passes, -# and a later enable_authselect remediation breaks it -# - see https://github.com/OpenSCAP/openscap/issues/1880 -/hardening/anaconda/.+/accounts_password_pam_retry - rhel >= 8 - -# bz1828871, won't be fixed on rhel7 -# - likely NOT related to rule ordering, double remediation doesn't help -/hardening/anaconda(/with-gui)?/[^/]+/postfix_network_listening_disabled -# https://github.com/ComplianceAsCode/content/issues/10938 -/hardening/host-os/oscap/[^/]+/postfix_network_listening_disabled - Match(rhel == 7, sometimes=True) - -# caused by one of: -# - bz1778661 (abrt) -# - bz2209266 (RHEL-9 gdm) -# - bz2209294 (RHEL-7 gdm, different issue) -/hardening/.+/rpm_verify_(ownership|permissions) - Match(True, sometimes=True) - -# bz1825810 or maybe bz1929805 -# can be reproduced mostly reliably (95%) both via anaconda and oscap CLI, -# but apparently we don't really care about old releases -# -# also re-discovered on RHEL-8 via -# https://github.com/ComplianceAsCode/content/issues/10937 -# and afterwards on other profiles (anssi_bp28_high), but still -# only on GUI -/hardening/[^/]+/with-gui/[^/]+/sysctl_net_ipv4_ip_forward - Match(rhel <= 8, sometimes=True) - -# https://github.com/ComplianceAsCode/content/issues/10424 -# happens on host-os hardening too, probably because Beaker doesn't have -# firewall enabled or even installed -/hardening/anaconda(/with-gui)?/[^/]+/service_nftables_disabled -/hardening/host-os/oscap/[^/]+/service_nftables_disabled - Match(True, sometimes=True) - -# caused by us not caring whether the VM has it installed, -# - remediation should fix it, but doesn't -- possibly caused by -# https://github.com/RHSecurityCompliance/contest/issues/15 -/hardening/oscap(/with-gui)?/[^/]+/package_scap-security-guide_installed - Match(True, sometimes=True) - -# TODO: something new? .. RHEL-8 on e8 and ism_o, RHEL-7 e8 -# - seems to not happen on latest 8.9 nightlies ?? -# - on latest 7.9, but upstream 2023/05 content -/hardening/oscap/[^/]+/package_rear_installed - Match(rhel <= 8, sometimes=True) - -# RHEL-8: https://bugzilla.redhat.com/show_bug.cgi?id=1834716 -# RHEL-9: https://bugzilla.redhat.com/show_bug.cgi?id=1999587 -/hardening/anaconda/with-gui/[^/]+/service_avahi-daemon_disabled - True - -# https://github.com/ComplianceAsCode/content/issues/11498 -# On RHEL 7, this rule accidentally passes due to a bug in OpenSCAP -# https://issues.redhat.com/browse/RHEL-24335 -/hardening/anaconda/with-gui/[^/]+/service_bluetooth_disabled - rhel >= 8 - -# RHEL-9 is not FIPS certified yet -/hardening/.+/aide_use_fips_hashes - rhel == 9 - -# ssh either doesn't start up, or gets blocked, possibly related -# to new firewalld rules being added? -# https://github.com/ComplianceAsCode/content/pull/10573 -# (happens only with GUI) -# -# or perhaps one of -# https://github.com/ComplianceAsCode/content/issues/10592 -# https://github.com/ComplianceAsCode/content/issues/10593 -# https://github.com/ComplianceAsCode/content/issues/10594 -/hardening/oscap/with-gui/cis_workstation_l[12] - status == 'error' - -# OAA just failed without an error, as usual -# https://issues.redhat.com/browse/OPENSCAP-3321 -# seems to be happening much more reliably with GUI -/hardening/anaconda/with-gui/cis_workstation_l[12] - Match(status == 'error', sometimes=True) - -# happened in Beaker, but uses VMs, so it shouldn't be Beaker-specific -# TODO: investigate, seems to be RHEL-9.3+ but unsure -/hardening/oscap/with-gui/.+/package_aide_installed -/hardening/oscap/with-gui/.+/aide_build_database -/hardening/oscap/with-gui/.+/aide_periodic_cron_checking -/hardening/oscap/with-gui/.+/aide_scan_notification -/hardening/oscap/with-gui/.+/aide_verify_acls -/hardening/oscap/with-gui/.+/aide_verify_ext_attributes - Match(True, sometimes=True) - -# Beaker-specific: -# all Beaker repositories have gpgcheck=0 and they get copied to nested VMs too -/(hardening|scanning)/.+/ensure_gpgcheck_never_disabled -# we don't control partitions on the host OS -/hardening/host-os/oscap/.+/mount_option_(home|opt|srv|var|var_log|var_log_audit|tmp)_(noexec|nosuid|nodev|usrquota|grpquota) -/hardening/host-os/oscap/.+/mount_option_boot_efi_nosuid -# likely something caused by restraint / Beaker test env -/hardening/host-os/.+/file_permissions_unauthorized_world_writable -# Beaker and host-os seem to randomly fail any services enabled -# or packages installed - TODO investigate remediation script outputs -# to figure out why -/hardening/host-os/oscap/[^/]+/service_.+_enabled -/hardening/host-os/oscap/[^/]+/timer_.+_enabled -/hardening/host-os/oscap/[^/]+/package_.+_installed -# TODO: unknown, probably worth investigating -/hardening/host-os/oscap/.+/sysctl_net_ipv6_conf_(all|default)_accept_ra -/hardening/host-os/oscap/.+/sysctl_net_ipv4_conf_default_log_martians - Match(True, sometimes=True) - -# Beaker-specific, possibly; -# same for dnf-automatic and rsyslog (??), is this fully random? -/hardening/host-os/oscap/[^/]+/package_dnf-automatic_installed -/hardening/host-os/oscap/[^/]+/timer_dnf-automatic_enabled -/hardening/host-os/oscap/[^/]+/package_rsyslog-gnutls_installed - Match(rhel >= 8, sometimes=True) - -# seems RHEL-8 specific, unknown, TODO investigate -# remediation script says: -# Current configuration is valid. -# Current configuration is valid. -# [error] Unknown profile feature [with-smartcard] -# [error] Unable to activate profile [custom/hardening] [22]: Invalid argument -# Unable to enable feature [22]: Invalid argument -# maybe hardware-specific and our Beaker systems don't have the hardware? -/hardening/host-os/oscap/.+/sssd_enable_smartcards - Match(rhel == 8, sometimes=True) - -# visible on double reboot (double remediation) -# /usr/libexec/abrt-action-install-debuginfo-to-abrt-cache -/hardening/oscap/with-gui/e8/file_ownership_binary_dirs - rhel == 7 - -# TODO: completely unknown, investigate and sort -# -# all RHELs -/hardening/ansible/.+/mount_option_boot_noexec -/hardening/ansible/.+/mount_option_boot_nosuid -/hardening/ansible/.+/mount_option_home_noexec -/hardening/ansible/.+/audit_rules_usergroup_modification - True -# RHEL-9 only -/hardening/ansible/with-gui/.+/network_nmcli_permissions - rhel == 9 -# RHEL-8 or 9 -/hardening(/host-os)?/ansible/.+/no_tmux_in_shells -/hardening(/host-os)?/ansible/.+/configure_usbguard_auditbackend -/hardening(/host-os)?/ansible/.+/audit_rules_unsuccessful_file_modification - rhel == 8 or rhel == 9 -# RHEL-8 -/hardening/ansible/with-gui/stig_gui/sysctl_net_ipv4_conf_all_forwarding - rhel == 8 -# RHEL-7 -/hardening(/host-os)?/ansible/.+/sshd_use_strong_ciphers -/hardening(/host-os)?/ansible/.+/audit_rules_for_ospp -/hardening(/host-os)?/ansible/.+/aide_use_fips_hashes -/hardening(/host-os)?/ansible/.+/smartcard_auth - rhel == 7 -# sssd_enable_pam_services is missing Ansible remediation -/hardening(/host-os)?/ansible/.+/sssd_enable_pam_services - True - -# unknown as well, but happens only rarely -/hardening/ansible/.+/configure_bashrc_exec_tmux -# home_nosuid failures are just really random across RHEL versions and nightlies -/hardening/ansible/.+/mount_option_home_nosuid - Match(True, sometimes=True) - -# only pci-dss, passes everywhere else -/hardening/ansible(/with-gui)?/pci-dss/audit_rules_login_events - rhel == 8 or rhel == 9 - -# https://bugzilla.redhat.com/show_bug.cgi?id=1797653 WONTFIX -/scanning/oscap-eval/ERROR - rhel <= 8 and note == 'E: oscap: Failed to convert OVAL state to SEXP, id: oval:ssg-state_file_groupowner_var_log_syslog_gid_4_0:ste:1.' - -# https://github.com/ComplianceAsCode/content/issues/10901 -# not sure what enables the service, but second remediation fixes the problem -/hardening/anaconda/with-gui/[^/]+/service_rpcbind_disabled - Match(rhel == 8, sometimes=True) - -# Image Builder errors -# -# ignore errors for now, until 'openscap generate fix' syntax stabilizes -# and blocker bugs are resolved -/hardening/image-builder/.+ - Match(status == 'error', sometimes=True) - -# Image Builder failures -# -# https://github.com/ComplianceAsCode/content/issues/11565 -/hardening/image-builder/.*/audit_rules_privileged_commands - True -# https://github.com/ComplianceAsCode/content/issues/11566 -/hardening/image-builder/hipaa/sebool_selinuxuser_execstack - rhel == 8 -# https://github.com/ComplianceAsCode/content/issues/11567 -/hardening/image-builder/.*/enable_dracut_fips_module -/hardening/image-builder/.*/enable_fips_mode - True - -# /per-rule (Automatus rule mode) waivers -# - these intentionally don't apply to /per-rule/from-env (ad-hoc RULE run) -# -# TODO: all of these are unknown and need investigation -/per-rule/[^/]+/accounts_password_set_max_life_root/correct.pass -/per-rule/[^/]+/accounts_password_set_max_life_root/wrong.fail -/per-rule/[^/]+/postfix_client_configure_mail_alias/correct.pass - True -/per-rule/[^/]+/grub2_audit_backlog_limit_argument/correct_grubenv.pass -/per-rule/[^/]+/grub2_password/invalid_username.fail -/per-rule/[^/]+/harden_sshd_ciphers_openssh_conf_crypto_policy/stig_correct.pass -/per-rule/[^/]+/harden_sshd_ciphers_openssh_conf_crypto_policy/stig_correct_followed_by_incorrect_commented.pass -/per-rule/[^/]+/sudo_add_umask/0027_var_multiple_values.pass - rhel == 8 -/per-rule/[^/]+/directory_permissions_var_log_audit/correct_value_0700.pass -/per-rule/[^/]+/directory_permissions_var_log_audit/incorrect_value_0700.fail -/per-rule/[^/]+/dconf_gnome_lock_screen_on_smartcard_removal/wrong_value.fail -/per-rule/[^/]+/file_ownership_var_log_audit_stig/correct_value_default_file.pass -/per-rule/[^/]+/tftpd_uses_secure_mode/correct.pass -/per-rule/[^/]+/tftpd_uses_secure_mode/wrong.fail - rhel == 9 - -# DISA Alignment waivers -# -# https://github.com/ComplianceAsCode/content/issues/9307 (DISA issue) -/scanning/disa-alignment/.*/sysctl_kernel_yama_ptrace_scope -# https://github.com/ComplianceAsCode/content/issues/9308 (DISA issue) -/scanning/disa-alignment/.*/sysctl_kernel_core_pattern -# https://github.com/ComplianceAsCode/content/issues/10044 (DISA issue) -/scanning/disa-alignment/.*/accounts_password_pam_pwhistory_remember_password_auth -# https://github.com/ComplianceAsCode/content/issues/11197 (DISA issue) -/scanning/disa-alignment/.*/display_login_attempts - rhel == 8 -# https://github.com/ComplianceAsCode/content/issues/11649 (DISA issue) -/scanning/disa-alignment/.*/installed_OS_is_vendor_supported -# https://github.com/ComplianceAsCode/content/issues/11650 -/scanning/disa-alignment/.*/kernel_module_tipc_disabled - rhel == 9 - -# HTML links from datastreams waivers -# -# ignore SSL certificate expirations in html-links - these are generally -# harmless (expiration is not MITM) while being the biggest contributor -# to false positives, so just ignore them, avoiding frequent random fails -/static-checks/html-links/.+ - Match("failed: certificate has expired" in note, sometimes=True) -# Inaccessible until form is filled: -/static-checks/html-links/https://docs-prv.pcisecuritystandards.org/PCI%20DSS/Standard/PCI-DSS-v4_0.pdf - True - -# CentOS-specific waivers -# -# Rules checking if OS is FIPS certified -/hardening/host-os/.+/sshd_use_approved_.+ -/hardening/host-os/.+/package_dracut-fips_installed -/hardening/host-os/.+/grub2_enable_fips_mode -/hardening/host-os/.+/aide_use_fips_hashes - rhel.is_centos() - -# Remediations are unselected for CentOS, but they might pass outside Testing Farm -/hardening/host-os/.+/ensure_gpgcheck_globally_activated -/hardening/host-os/.+/ensure_gpgcheck_local_packages -/hardening/host-os/.+/ensure_gpgcheck_never_disabled -/hardening/host-os/.+/ensure_gpgcheck_repo_metadata - Match(rhel.is_centos(), sometimes=True) - -/hardening/host-os/.+/ospp/enable_fips_mode -/hardening/host-os/.*/ospp/configure_crypto_policy - rhel.is_centos() and rhel == 9 - -# vim: syntax=python diff --git a/conf/waivers/10-unknown b/conf/waivers/10-unknown new file mode 100644 index 00000000..7f4d78ab --- /dev/null +++ b/conf/waivers/10-unknown @@ -0,0 +1,125 @@ +# put freshly discovered, un-investigated issues here, to suppress them +# from failing during regular runs +# +# it is expected that somebody will go over these soon(ish), investigating +# their root cause, filing isseus/bugs or fixing tests as appropriate, +# eventually either removing the waivers or moving them to other files + +# TODO: something new? .. RHEL-8 on e8 and ism_o, RHEL-7 e8 +# - seems to not happen on latest 8.9 nightlies ?? +# - on latest 7.9, but upstream 2023/05 content +/hardening/oscap/[^/]+/package_rear_installed + Match(rhel <= 8, sometimes=True) + +# ssh either doesn't start up, or gets blocked, possibly related +# to new firewalld rules being added? +# https://github.com/ComplianceAsCode/content/pull/10573 +# (happens only with GUI) +# +# or perhaps one of +# https://github.com/ComplianceAsCode/content/issues/10592 +# https://github.com/ComplianceAsCode/content/issues/10593 +# https://github.com/ComplianceAsCode/content/issues/10594 +/hardening/oscap/with-gui/cis_workstation_l[12] + status == 'error' + +# happened in Beaker, but uses VMs, so it shouldn't be Beaker-specific +# TODO: investigate, seems to be RHEL-9.3+ but unsure +/hardening/oscap/with-gui/.+/package_aide_installed +/hardening/oscap/with-gui/.+/aide_build_database +/hardening/oscap/with-gui/.+/aide_periodic_cron_checking +/hardening/oscap/with-gui/.+/aide_scan_notification +/hardening/oscap/with-gui/.+/aide_verify_acls +/hardening/oscap/with-gui/.+/aide_verify_ext_attributes + Match(True, sometimes=True) + +# seems RHEL-8 specific, unknown, TODO investigate +# remediation script says: +# Current configuration is valid. +# Current configuration is valid. +# [error] Unknown profile feature [with-smartcard] +# [error] Unable to activate profile [custom/hardening] [22]: Invalid argument +# Unable to enable feature [22]: Invalid argument +# maybe hardware-specific and our Beaker systems don't have the hardware? +/hardening/host-os/oscap/.+/sssd_enable_smartcards + Match(rhel == 8, sometimes=True) + +# Ansible TODO: completely unknown, investigate and sort +# +# all RHELs +/hardening/ansible/.+/mount_option_boot_noexec +/hardening/ansible/.+/mount_option_boot_nosuid +/hardening/ansible/.+/mount_option_home_noexec +/hardening/ansible/.+/audit_rules_usergroup_modification + True +# RHEL-9 only +/hardening/ansible/with-gui/.+/network_nmcli_permissions + rhel == 9 +# RHEL-8 or 9 +/hardening(/host-os)?/ansible/.+/no_tmux_in_shells +/hardening(/host-os)?/ansible/.+/configure_usbguard_auditbackend +/hardening(/host-os)?/ansible/.+/audit_rules_unsuccessful_file_modification + rhel == 8 or rhel == 9 +# RHEL-8 +/hardening/ansible/with-gui/stig_gui/sysctl_net_ipv4_conf_all_forwarding + rhel == 8 +# RHEL-7 +/hardening(/host-os)?/ansible/.+/sshd_use_strong_ciphers +/hardening(/host-os)?/ansible/.+/audit_rules_for_ospp +/hardening(/host-os)?/ansible/.+/aide_use_fips_hashes +/hardening(/host-os)?/ansible/.+/smartcard_auth + rhel == 7 +# unknown as well, but happens only rarely +/hardening/ansible/.+/configure_bashrc_exec_tmux + Match(True, sometimes=True) +# only pci-dss, passes everywhere else +/hardening/ansible(/with-gui)?/pci-dss/audit_rules_login_events + rhel == 8 or rhel == 9 + +# home_nosuid failures are just really random across RHEL versions and nightlies +/hardening/ansible/.+/mount_option_home_nosuid + Match(True, sometimes=True) + +# https://github.com/ComplianceAsCode/content/issues/10901 +# not sure what enables the service, but second remediation fixes the problem +# TODO: we do run double remediation, but this still sometimes fails, +# investigate why +/hardening/anaconda/with-gui/[^/]+/service_rpcbind_disabled + Match(rhel == 8, sometimes=True) + +# /per-rule (Automatus rule mode) waivers +# +# TODO: all of these are unknown and need investigation +/per-rule/[^/]+/accounts_password_set_max_life_root/correct.pass +/per-rule/[^/]+/accounts_password_set_max_life_root/wrong.fail +/per-rule/[^/]+/postfix_client_configure_mail_alias/correct.pass + True +/per-rule/[^/]+/grub2_audit_backlog_limit_argument/correct_grubenv.pass +/per-rule/[^/]+/grub2_password/invalid_username.fail +/per-rule/[^/]+/harden_sshd_ciphers_openssh_conf_crypto_policy/stig_correct.pass +/per-rule/[^/]+/harden_sshd_ciphers_openssh_conf_crypto_policy/stig_correct_followed_by_incorrect_commented.pass +/per-rule/[^/]+/sudo_add_umask/0027_var_multiple_values.pass + rhel == 8 +/per-rule/[^/]+/directory_permissions_var_log_audit/correct_value_0700.pass +/per-rule/[^/]+/directory_permissions_var_log_audit/incorrect_value_0700.fail +/per-rule/[^/]+/dconf_gnome_lock_screen_on_smartcard_removal/wrong_value.fail +/per-rule/[^/]+/file_ownership_var_log_audit_stig/correct_value_default_file.pass +/per-rule/[^/]+/tftpd_uses_secure_mode/correct.pass +/per-rule/[^/]+/tftpd_uses_secure_mode/wrong.fail + rhel == 9 + +# likely something caused by restraint / Beaker test env +# TODO: investigate +/hardening/host-os/.+/file_permissions_unauthorized_world_writable +# Beaker and host-os seem to randomly fail any services enabled +# or packages installed - TODO investigate remediation script outputs +# to figure out why +/hardening/host-os/oscap/[^/]+/service_.+_enabled +/hardening/host-os/oscap/[^/]+/timer_.+_enabled +/hardening/host-os/oscap/[^/]+/package_.+_installed +# TODO: unknown, probably worth investigating +/hardening/host-os/oscap/.+/sysctl_net_ipv6_conf_(all|default)_accept_ra +/hardening/host-os/oscap/.+/sysctl_net_ipv4_conf_default_log_martians + Match(True, sometimes=True) + +# vim: syntax=python diff --git a/conf/waivers/20-long-term b/conf/waivers/20-long-term new file mode 100644 index 00000000..3eaad839 --- /dev/null +++ b/conf/waivers/20-long-term @@ -0,0 +1,139 @@ +# these are either technology limitations that may be fixed in a distant +# future, or other "unfixable" issues caused by the nature of the software +# we use +# +# ie. failures that we don't expect to fix until a big feature is implemented, +# even if it might take a year or two for the feature to be done + +# RHEL-7 only, because RHEL-8+ check for rsync-daemon, +# while TMT only needs the 'rsync' command executed via ssh, +# so remove the remediation exception + waiver on RHEL-8+ +/hardening/host-os/.+/[^/]+/package_rsync_removed + rhel == 7 + +# requires running firewalld (firewall-cmd) and NetworkManager, +# which are not available in their final form in the Anaconda environment +# - see https://github.com/ComplianceAsCode/content/issues/9746 +/hardening/anaconda(/with-gui)?/[^/]+/firewalld_sshd_port_enabled +# https://github.com/ComplianceAsCode/content/issues/11625 +/hardening/image-builder(/with-gui)?/[^/]+/firewalld_sshd_port_enabled +# https://github.com/ComplianceAsCode/content/issues/10939 +# https://github.com/ComplianceAsCode/content/issues/10613 +/hardening/anaconda(/with-gui)?/[^/]+/firewalld_loopback_traffic_(restricted|trusted) +# https://github.com/ComplianceAsCode/content/issues/11564 +/hardening/image-builder(/with-gui)?/[^/]+/firewalld_loopback_traffic_(restricted|trusted) + rhel >= 8 + +# rule ordering issue - accounts_password_pam_retry is checked first and passes, +# and a later enable_authselect remediation breaks it +# - see https://github.com/OpenSCAP/openscap/issues/1880 +/hardening/anaconda/.+/accounts_password_pam_retry + rhel >= 8 + +# bz1828871, won't be fixed on rhel7 +# - likely NOT related to rule ordering, double remediation doesn't help +/hardening/anaconda(/with-gui)?/[^/]+/postfix_network_listening_disabled +# https://github.com/ComplianceAsCode/content/issues/10938 +/hardening/host-os/oscap/[^/]+/postfix_network_listening_disabled + Match(rhel == 7, sometimes=True) + +# caused by one of: +# - bz1778661 (abrt) +# - bz2209266 (RHEL-9 gdm) +# - bz2209294 (RHEL-7 gdm, different issue) +/hardening/.+/rpm_verify_(ownership|permissions) + Match(True, sometimes=True) + +# bz1825810 or maybe bz1929805 +# can be reproduced mostly reliably (95%) both via anaconda and oscap CLI, +# but apparently we don't really care about old releases +# +# also re-discovered on RHEL-8 via +# https://github.com/ComplianceAsCode/content/issues/10937 +# and afterwards on other profiles (anssi_bp28_high), but still +# only on GUI +/hardening/[^/]+/with-gui/[^/]+/sysctl_net_ipv4_ip_forward + Match(rhel <= 8, sometimes=True) + +# https://github.com/ComplianceAsCode/content/issues/10424 +# happens on host-os hardening too, probably because Beaker doesn't have +# firewall enabled or even installed +/hardening/anaconda(/with-gui)?/[^/]+/service_nftables_disabled +/hardening/host-os/oscap/[^/]+/service_nftables_disabled + Match(True, sometimes=True) + +# caused by us not caring whether the VM has it installed, +# - remediation should fix it, but doesn't -- possibly caused by +# https://github.com/RHSecurityCompliance/contest/issues/15 +/hardening/oscap(/with-gui)?/[^/]+/package_scap-security-guide_installed + Match(True, sometimes=True) + +# RHEL-8: https://bugzilla.redhat.com/show_bug.cgi?id=1834716 +# RHEL-9: https://bugzilla.redhat.com/show_bug.cgi?id=1999587 +/hardening/anaconda/with-gui/[^/]+/service_avahi-daemon_disabled + True + +# https://github.com/ComplianceAsCode/content/issues/11498 +# On RHEL 7, this rule accidentally passes due to a bug in OpenSCAP +# https://issues.redhat.com/browse/RHEL-24335 +/hardening/anaconda/with-gui/[^/]+/service_bluetooth_disabled + rhel >= 8 + +# RHEL-9 is not FIPS certified yet +/hardening/.+/aide_use_fips_hashes + rhel == 9 + +# OAA just failed without an error, as usual +# https://issues.redhat.com/browse/OPENSCAP-3321 +# seems to be happening much more reliably with GUI +/hardening/anaconda/with-gui/cis_workstation_l[12] + Match(status == 'error', sometimes=True) + +# visible on double reboot (double remediation) +# /usr/libexec/abrt-action-install-debuginfo-to-abrt-cache +/hardening/oscap/with-gui/e8/file_ownership_binary_dirs + rhel == 7 + +# Image Builder errors +# +# ignore errors for now, until 'openscap generate fix' syntax stabilizes +# and blocker bugs are resolved +/hardening/image-builder/.+ + Match(status == 'error', sometimes=True) + +# Image Builder failures +# +# https://github.com/ComplianceAsCode/content/issues/11565 +/hardening/image-builder/.*/audit_rules_privileged_commands + True +# https://github.com/ComplianceAsCode/content/issues/11566 +/hardening/image-builder/hipaa/sebool_selinuxuser_execstack + rhel == 8 +# https://github.com/ComplianceAsCode/content/issues/11567 +/hardening/image-builder/.*/enable_dracut_fips_module +/hardening/image-builder/.*/enable_fips_mode + True + +# DISA Alignment waivers +# +# https://github.com/ComplianceAsCode/content/issues/9307 (DISA issue) +/scanning/disa-alignment/.*/sysctl_kernel_yama_ptrace_scope +# https://github.com/ComplianceAsCode/content/issues/9308 (DISA issue) +/scanning/disa-alignment/.*/sysctl_kernel_core_pattern +# https://github.com/ComplianceAsCode/content/issues/10044 (DISA issue) +/scanning/disa-alignment/.*/accounts_password_pam_pwhistory_remember_password_auth +# https://github.com/ComplianceAsCode/content/issues/11197 (DISA issue) +/scanning/disa-alignment/.*/display_login_attempts + rhel == 8 +# https://github.com/ComplianceAsCode/content/issues/11649 (DISA issue) +/scanning/disa-alignment/.*/installed_OS_is_vendor_supported +# https://github.com/ComplianceAsCode/content/issues/11650 +/scanning/disa-alignment/.*/kernel_module_tipc_disabled + rhel == 9 + +# sssd_enable_pam_services is missing Ansible remediation +# https://github.com/RHSecurityCompliance/contest/issues/87 +/hardening(/host-os)?/ansible/.+/sssd_enable_pam_services + True + +# vim: syntax=python diff --git a/conf/waivers/30-permanent b/conf/waivers/30-permanent new file mode 100644 index 00000000..20383a9f --- /dev/null +++ b/conf/waivers/30-permanent @@ -0,0 +1,63 @@ +# these tests are always expected to fail, because there's no reasonable way +# to fix them, or because we intentionally expect the failure and a pass should +# trigger alarms and be invetigated + +# remediation for this is globally disabled in Contest, +# see conf/remediation.py +/hardening/.*/accounts_password_set_max_life_(existing|root) + Match(True, sometimes=True) + +# the service_sssd_enabled will be failing even if the service is enabled +# because it requires manual configuration which cannot be attained with our rules +/hardening/.+/service_sssd_enabled + True + +# Beaker-specific: +# all Beaker repositories have gpgcheck=0 and they get copied to nested VMs too +/(hardening|scanning)/.+/ensure_gpgcheck_never_disabled +# we don't control partitions on the host OS +/hardening/host-os/oscap/.+/mount_option_(home|opt|srv|var|var_log|var_log_audit|tmp)_(noexec|nosuid|nodev|usrquota|grpquota) +/hardening/host-os/oscap/.+/mount_option_boot_efi_nosuid + +# Beaker-specific, possibly; +# same for dnf-automatic and rsyslog (??), is this fully random? +/hardening/host-os/oscap/[^/]+/package_dnf-automatic_installed +/hardening/host-os/oscap/[^/]+/timer_dnf-automatic_enabled +/hardening/host-os/oscap/[^/]+/package_rsyslog-gnutls_installed + Match(rhel >= 8, sometimes=True) + +# https://bugzilla.redhat.com/show_bug.cgi?id=1797653 WONTFIX +/scanning/oscap-eval/ERROR + rhel <= 8 and note == 'E: oscap: Failed to convert OVAL state to SEXP, id: oval:ssg-state_file_groupowner_var_log_syslog_gid_4_0:ste:1.' + +# HTML links from datastreams waivers +# +# ignore SSL certificate expirations in html-links - these are generally +# harmless (expiration is not MITM) while being the biggest contributor +# to false positives, so just ignore them, avoiding frequent random fails +/static-checks/html-links/.+ + Match("failed: certificate has expired" in note, sometimes=True) +# Inaccessible until form is filled: +/static-checks/html-links/https://docs-prv.pcisecuritystandards.org/PCI%20DSS/Standard/PCI-DSS-v4_0.pdf + True + +# CentOS-specific waivers +# +# Rules checking if OS is FIPS certified +/hardening/host-os/.+/sshd_use_approved_.+ +/hardening/host-os/.+/package_dracut-fips_installed +/hardening/host-os/.+/grub2_enable_fips_mode +/hardening/host-os/.+/aide_use_fips_hashes + rhel.is_centos() +# Remediations are unselected for CentOS, but they might pass outside Testing Farm +/hardening/host-os/.+/ensure_gpgcheck_globally_activated +/hardening/host-os/.+/ensure_gpgcheck_local_packages +/hardening/host-os/.+/ensure_gpgcheck_never_disabled +/hardening/host-os/.+/ensure_gpgcheck_repo_metadata + Match(rhel.is_centos(), sometimes=True) +# Presumably not valid for CentOS +/hardening/host-os/.+/ospp/enable_fips_mode +/hardening/host-os/.*/ospp/configure_crypto_policy + rhel.is_centos() and rhel == 9 + +# vim: syntax=python diff --git a/lib/waive.py b/lib/waive.py index 26f6390f..8bf129d6 100644 --- a/lib/waive.py +++ b/lib/waive.py @@ -126,13 +126,30 @@ def _parse_waiver_file(stream): return sections -def _open_waiver_file(): - preferred = os.environ.get('CONTEST_WAIVERS', 'released') - waiver_file = Path(util.libdir).parent / 'conf' / f'waivers-{preferred}' - if not waiver_file.exists(): - raise FileNotFoundError(f"{waiver_file.name} doesn't exist in 'conf'") - util.log(f"using {waiver_file} for waiving") - return open(waiver_file) +def _collect_waivers(): + """ + Recursively walk a directory of waiver files/directories, + yielding waiver sections. + """ + # note: we don't use os.walk() because it splits files and directories + # into two lists, breaking sorting - we want to treat both equally, so that + # files can interleave directories in the sorted order + dir_name = os.environ.get('CONTEST_WAIVER_DIR', 'conf/waivers') + dir_path = Path(util.libdir).parent / dir_name + util.log(f"using {dir_path} for waiving") + + def _collect_files(in_dir): + for item in sorted(in_dir.iterdir()): + if item.name.startswith('.'): + continue + if item.is_dir(): + yield from _collect_files(item) + elif item.is_file(): + yield item + + for file in _collect_files(dir_path): + with open(file) as f: + yield from _parse_waiver_file(f) class Match: @@ -151,8 +168,7 @@ def __bool__(self): def match_result(status, name, note): global _sections_cache if _sections_cache is None: - with _open_waiver_file() as f: - _sections_cache = _parse_waiver_file(f) + _sections_cache = list(_collect_waivers()) # make sure "'someting' in name" always works if name is None: