Merge pull request #2039 from jan-cerny/improve_autotailor

Improve the autotailor script

.github/workflows/build.yml

@@ -34,7 +34,7 @@ jobs:
  - name: Install Deps
  run: |
  sudo apt-get update
- sudo apt-get -y install lcov swig xsltproc rpm-common lua5.3 libpcre2-dev libyaml-dev libapt-pkg-dev libdbus-1-dev libdbus-glib-1-dev libcurl4-openssl-dev libgcrypt-dev libselinux1-dev libgconf2-dev libacl1-dev libblkid-dev libcap-dev libxml2-dev libxslt1-dev libxml-parser-perl libxml-xpath-perl libperl-dev librpm-dev librtmp-dev libxmlsec1-dev libxmlsec1-openssl python3-dbusmock
+ sudo apt-get -y install lcov swig xsltproc rpm-common lua5.3 libpcre2-dev libyaml-dev libapt-pkg-dev libdbus-1-dev libdbus-glib-1-dev libcurl4-openssl-dev libgcrypt-dev libselinux1-dev libgconf2-dev libacl1-dev libblkid-dev libcap-dev libxml2-dev libxslt1-dev libxml-parser-perl libxml-xpath-perl libperl-dev librpm-dev librtmp-dev libxmlsec1-dev libxmlsec1-openssl python3-dbusmock python3-pytest
  sudo apt-get -y remove rpm
 
  # Runs a set of commands using the runners shell
@@ -57,7 +57,7 @@ jobs:
  image: fedora:latest
  steps:
  - name: Install Deps
- run: dnf install -y cmake git dbus-devel GConf2-devel libacl-devel libblkid-devel libcap-devel libcurl-devel libgcrypt-devel libselinux-devel libxml2-devel libxslt-devel libattr-devel make openldap-devel pcre2-devel perl-XML-Parser perl-XML-XPath perl-devel python3-devel python3-dbusmock rpm-devel swig bzip2-devel gcc-c++ libyaml-devel xmlsec1-devel xmlsec1-openssl-devel hostname bzip2 lua rpm-build which strace apt-devel
+ run: dnf install -y cmake git dbus-devel GConf2-devel libacl-devel libblkid-devel libcap-devel libcurl-devel libgcrypt-devel libselinux-devel libxml2-devel libxslt-devel libattr-devel make openldap-devel pcre2-devel perl-XML-Parser perl-XML-XPath perl-devel python3-devel python3-dbusmock rpm-devel swig bzip2-devel gcc-c++ libyaml-devel xmlsec1-devel xmlsec1-openssl-devel hostname bzip2 lua rpm-build which strace apt-devel python3-pytest
  - name: Checkout
  uses: actions/checkout@v3
  with:

tests/utils/CMakeLists.txt

@@ -1 +1,7 @@
+add_oscap_test("autotailor_integration_test.sh")
 add_oscap_test("test_utils_args.sh")
+
+add_test(
+ NAME "autotailor-unit-tests"
+ COMMAND ${PYTHON_EXECUTABLE} -m pytest -v "${CMAKE_CURRENT_SOURCE_DIR}/test_autotailor.py"
+)

tests/utils/autotailor_integration_test.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+
+. $builddir/tests/test_common.sh
+
+set -e -o pipefail
+
+autotailor="$top_srcdir/utils/autotailor"
+tailoring="$(mktemp)"
+ds="$srcdir/data_stream.xml"
+stdout="$(mktemp)"
+original_profile="P1"
+result="$(mktemp)"
+
+# the original profile P1 selects rules R1 and R2
+
+# select additional rule R3
+python3 $autotailor --id-namespace "com.example.www" --select R3 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# select additional rules R3, R4
+python3 $autotailor --id-namespace "com.example.www" --select R3 --select R4 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="pass"]'
+
+# unselect rule R2
+python3 $autotailor --id-namespace "com.example.www" --unselect R2 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# unselect rule R2 and select R4
+python3 $autotailor --id-namespace "com.example.www" --unselect R2 --select R4 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="pass"]'
+
+# select additional rule R3 and change its severity to high
+python3 $autotailor --id-namespace "com.example.www" --select R3 --rule-severity R3=high $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @severity="unknown"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @severity="unknown"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @severity="unknown"]'
+
+# don't select rules, don't unselect rules, but change severity of all rules to high
+python3 $autotailor --id-namespace "com.example.www" --rule-severity R1=high --rule-severity R2=high --rule-severity R3=high --rule-severity R4=high $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @severity="high"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @severity="high"]'
+
+
+# select additional rule R4 and change its role to "unchecked"
+python3 $autotailor --id-namespace "com.example.www" --select R4 --rule-role R4=unchecked $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3" and @role="full"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notchecked"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4" and @role="unchecked"]'
+
+
+# select additional rule R3; the customized profile will have a special profile ID
+customized_profile="xccdf_com.pink.elephant_profile_pineapple"
+python3 $autotailor --new-profile-id $customized_profile --id-namespace "com.example.www" --select R3 $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile $customized_profile --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult[@id="xccdf_org.open-scap_testresult_'$customized_profile'"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'
+
+# refine value v1 to 30
+python3 $autotailor --id-namespace "com.example.www" --var-value V1=thirty $ds $original_profile > $tailoring
+$OSCAP xccdf eval --profile P1_customized --progress --tailoring-file $tailoring --results $result $ds
+assert_exists 1 '/Benchmark/TestResult/set-value[@idref="xccdf_com.example.www_value_V1" and text()="thirty"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R1"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R2"]/result[text()="pass"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R3"]/result[text()="notselected"]'
+assert_exists 1 '/Benchmark/TestResult/rule-result[@idref="xccdf_com.example.www_rule_R4"]/result[text()="notselected"]'

tests/utils/data_stream.xml

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="utf-8"?>
+<ds:data-stream-collection xmlns:ds="http://scap.nist.gov/schema/scap/source/1.2" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:cat="urn:oasis:names:tc:entity:xmlns:xml:catalog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="scap_org.open-scap_collection_from_xccdf_test_single_rule.xccdf.xml" schematron-version="1.3" xsi:schemaLocation="http://scap.nist.gov/schema/scap/source/1.2 https://scap.nist.gov/schema/scap/1.3/scap-source-data-stream_1.3.xsd">
+ <ds:data-stream id="scap_org.open-scap_datastream_simple" scap-version="1.3" use-case="OTHER">
+ <ds:checklists>
+ <ds:component-ref id="scap_org.open-scap_cref_test_single_rule.xccdf.xml" xlink:href="#scap_org.open-scap_comp_test_single_rule.xccdf.xml">
+ <cat:catalog>
+ <cat:uri name="test_single_rule.oval.xml" uri="#scap_org.open-scap_cref_test_single_rule.oval.xml"/>
+ </cat:catalog>
+ </ds:component-ref>
+ </ds:checklists>
+ <ds:checks>
+ <ds:component-ref id="scap_org.open-scap_cref_test_single_rule.oval.xml" xlink:href="#scap_org.open-scap_comp_test_single_rule.oval.xml"/>
+ </ds:checks>
+ </ds:data-stream>
+ <ds:component id="scap_org.open-scap_comp_test_single_rule.oval.xml" timestamp="2021-02-01T08:07:06+01:00">
+ <oval_definitions xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:ind-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" xmlns:oval-def="http://oval.mitre.org/XMLSchema/oval-definitions-5" xmlns:oval="http://oval.mitre.org/XMLSchema/oval-common-5" xmlns:win-def="http://oval.mitre.org/XMLSchema/oval-definitions-5#windows" xsi:schemaLocation="http://oval.mitre.org/XMLSchema/oval-definitions-5 oval-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#independent independent-definitions-schema.xsd http://oval.mitre.org/XMLSchema/oval-definitions-5#windows windows-definitions-schema.xsd">
+ <generator>
+ <oval:schema_version>5.11.2</oval:schema_version>
+ <oval:timestamp>2021-02-01T08:07:06+01:00</oval:timestamp>
+ </generator>
+ <definitions>
+ <definition class="compliance" id="oval:x:def:1" version="1">
+ <metadata>
+ <title>PASS</title>
+ <description>pass</description>
+ </metadata>
+ <criteria>
+ <criterion comment="PASS test" test_ref="oval:x:tst:1"/>
+ </criteria>
+ </definition>
+ </definitions>
+ <tests>
+ <variable_test xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" id="oval:x:tst:1" check="all" comment="always pass" version="1">
+ <object object_ref="oval:x:obj:1"/>
+ </variable_test>
+ </tests>
+ <objects>
+ <variable_object xmlns="http://oval.mitre.org/XMLSchema/oval-definitions-5#independent" id="oval:x:obj:1" version="1" comment="x">
+ <var_ref>oval:x:var:1</var_ref>
+ </variable_object>
+ </objects>
+ <variables>
+ <constant_variable id="oval:x:var:1" version="1" comment="x" datatype="int">
+ <value>100</value>
+ </constant_variable>
+ </variables>
+ </oval_definitions>
+ </ds:component>
+ <ds:component id="scap_org.open-scap_comp_test_single_rule.xccdf.xml" timestamp="2021-02-01T08:07:06+01:00">
+ <Benchmark xmlns="http://checklists.nist.gov/xccdf/1.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" id="xccdf_com.example.www_benchmark_dummy" xsi:schemaLocation="http://checklists.nist.gov/xccdf/1.1 xccdf-1.1.4.xsd" resolved="false" xml:lang="en-US">
+ <status date="2021-01-21">accepted</status>
+ <title>Test Benchmark</title>
+ <description>Description</description>
+ <version>1.0</version>
+ <metadata>
+ <dc:contributor xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:contributor>
+ <dc:publisher xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:publisher>
+ <dc:creator xmlns:dc="http://purl.org/dc/elements/1.1/">OpenSCAP</dc:creator>
+ <dc:source xmlns:dc="http://purl.org/dc/elements/1.1/">http://scap.nist.gov</dc:source>
+ </metadata>
+ <Profile id="xccdf_com.example.www_profile_P1">
+ <title>xccdf_test_profile</title>
+ <description>This profile is for testing.</description>
+ <select idref="xccdf_com.example.www_rule_R1" selected="true"/>
+ <select idref="xccdf_com.example.www_rule_R2" selected="true"/>
+ </Profile>
+ <Value id="xccdf_com.example.www_value_V1" operator="equals" type="number">
+ <title>value</title>
+ <description xml:lang="en">cccc</description>
+ <question xml:lang="en">ssss</question>
+ <value>5</value>
+ <value selector="thirty">30</value>
+ </Value>
+ <Rule selected="false" id="xccdf_com.example.www_rule_R1">
+ <title>Rule R1</title>
+ <description>Description</description>
+ <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+ <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+ </check>
+ </Rule>
+ <Rule selected="false" id="xccdf_com.example.www_rule_R2">
+ <title>Rule R2</title>
+ <description>Description</description>
+ <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+ <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+ </check>
+ </Rule>
+ <Rule selected="false" id="xccdf_com.example.www_rule_R3">
+ <title>Rule R3</title>
+ <description>Description</description>
+ <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+ <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+ </check>
+ </Rule>
+ <Rule selected="false" id="xccdf_com.example.www_rule_R4">
+ <title>Rule R4</title>
+ <description>Description</description>
+ <check system="http://oval.mitre.org/XMLSchema/oval-definitions-5">
+ <check-content-ref href="test_single_rule.oval.xml" name="oval:x:def:1"/>
+ </check>
+ </Rule>
+ </Benchmark>
+ </ds:component>
+</ds:data-stream-collection>