From 911872879ecdb072b39729f3904e768dd86e24ed Mon Sep 17 00:00:00 2001 From: Marcus Burghardt Date: Fri, 11 Aug 2023 10:56:02 +0200 Subject: [PATCH 1/4] Remove the file_block_removed_and_added test It was concluded this test is too costy for the benefit it brings. It takes about 10 minutes and search for a very specific and rare pattern with is also pretty easy to be caught during review. --- tests/CMakeLists.txt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7bc37bd8c47..cf287387cad 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -90,19 +90,6 @@ set_tests_properties("fix_rules" PROPERTIES LABELS quick) set_tests_properties("fix_rules" PROPERTIES DEPENDS "test-rule-dir-json") set_tests_properties("fix_rules" PROPERTIES FIXTURES_REQUIRED "rule-dir-json") -if(PY_YAMLPATH) - if(PY_PYTEST) - add_test( - NAME "test-function-check_playbook_file_removed_and_added" - COMMAND "${PYTHON_EXECUTABLE}" -m pytest ${PYTEST_COVERAGE_OPTIONS} "${CMAKE_CURRENT_SOURCE_DIR}/test_check_playbook_file_removed_and_added.py" - ) - endif() - add_test( - NAME "ansible-file-removed-and-added" - COMMAND env "PYTHONPATH=$ENV{PYTHONPATH}" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_ansible_file_removed_and_added.py" --ansible_dir "${CMAKE_BINARY_DIR}/ansible" - ) -endif() - macro(mypy_test SCRIPT) if(PY_MYPY) add_test( From cc51cee4db42504546f6be2fd6820de06e48e422 Mon Sep 17 00:00:00 2001 From: Marcus Burghardt Date: Fri, 11 Aug 2023 10:58:23 +0200 Subject: [PATCH 2/4] Remove scripts used for file_block_removed_and_added tests These scripts are no longer necessary. --- tests/test_ansible_file_removed_and_added.py | 97 ------------------- ...t_check_playbook_file_removed_and_added.py | 39 -------- 2 files changed, 136 deletions(-) delete mode 100644 tests/test_ansible_file_removed_and_added.py delete mode 100644 tests/test_check_playbook_file_removed_and_added.py diff --git a/tests/test_ansible_file_removed_and_added.py b/tests/test_ansible_file_removed_and_added.py deleted file mode 100644 index 23f6f888bda..00000000000 --- a/tests/test_ansible_file_removed_and_added.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/python3 - -import argparse -import os -import sys -from types import SimpleNamespace -from yamlpath import Processor -from yamlpath import YAMLPath -from yamlpath.common import Parsers -from yamlpath.exceptions import YAMLPathException -from yamlpath.wrappers import ConsolePrinter - - -def parse_command_line_args(): - parser = argparse.ArgumentParser( - description="Checks if an Ansible Playbook removes a file and then adds it again.") - parser.add_argument("--ansible_dir", required=True, - help="Directory containing Ansible Playbooks") - args = parser.parse_args() - return args - - -def check_playbook_file_removed_and_added(playbook_path): - playbook_ok = True - - yaml_parser = Parsers.get_yaml_editor() - - logging_args = SimpleNamespace(quiet=False, verbose=False, debug=False) - log = ConsolePrinter(logging_args) - - # Find every path removed by a file Task (also matches tasks within blocks) - files_absent_string = "tasks.**.file[state=absent][parent()].path" - files_absent_yamlpath = YAMLPath(files_absent_string) - path_editing_tasks_yamlpath = "" - - log.info("Info: Evaluating playbook '{}'".format(playbook_path)) - (yaml_data, doc_loaded) = Parsers.get_yaml_data(yaml_parser, log, playbook_path) - if not doc_loaded: - # There was an issue loading the file; an error message has already been - # printed via ConsolePrinter. - return False - - processor = Processor(log, yaml_data) - try: - for node in processor.get_nodes(files_absent_yamlpath, mustexist=False): - path = str(node) - # 'node' is a NodeCoords. - if path == 'None': - continue - elif "{{" in path: - # Identified path is a Jinja expression, unfortunately there is no easy way to get - # the actual path without making this test very complicated - continue - - # Check if this paths is used in any of the following ansible modules - ansible_modules = ["lineinfile", "blockinfile", "copy"] - path_editing_tasks_string = "tasks.**.[.=~/{modules}/][*='{path}'][parent()].name" - path_editing_tasks_yamlpath = YAMLPath(path_editing_tasks_string.format( - modules="|".join(ansible_modules), - path=node) - ) - for task in processor.get_nodes(path_editing_tasks_yamlpath, mustexist=False): - log.info("Error: Task '{}' manipulates a file that is removed by another task" - .format(task)) - playbook_ok = False - except YAMLPathException as ex: - no_file_msg = ("Cannot add PathSegmentTypes.TRAVERSE subreference to lists at 'None' " - "in '{}'.") - if str(ex) == no_file_msg.format(files_absent_string): - log.info("Info: Playbook {} has no 'file' tasks.".format(playbook_path)) - elif path_editing_tasks_yamlpath and str(ex) == no_file_msg.format( - path_editing_tasks_yamlpath): - log.info("Info: Playbook {} has no '{}' tasks.".format( - playbook_path, " ".join(ansible_modules))) - else: - log.info("Error: {}.".format(ex)) - - return playbook_ok - - -def main(): - args = parse_command_line_args() - - all_playbooks_ok = True - for dir_item in os.listdir(args.ansible_dir): - if dir_item.endswith(".yml"): - playbook_path = os.path.join(args.ansible_dir, dir_item) - - if not check_playbook_file_removed_and_added(playbook_path): - all_playbooks_ok = False - - if not all_playbooks_ok: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tests/test_check_playbook_file_removed_and_added.py b/tests/test_check_playbook_file_removed_and_added.py deleted file mode 100644 index 181bb14ed46..00000000000 --- a/tests/test_check_playbook_file_removed_and_added.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import pytest - -from .test_ansible_file_removed_and_added import check_playbook_file_removed_and_added - - -def test_file_removed_and_added(): - playbook_path = os.path.join(os.path.dirname(__file__), - "ansible_file_removed_and_added", - "file_removed_and_added.yml") - assert not check_playbook_file_removed_and_added(playbook_path) - - -def test_file_removed_and_not_added(): - playbook_path = os.path.join(os.path.dirname(__file__), - "ansible_file_removed_and_added", - "file_removed_and_not_added.yml") - assert check_playbook_file_removed_and_added(playbook_path) - - -def test_file_not_removed_and_added(): - playbook_path = os.path.join(os.path.dirname(__file__), - "ansible_file_removed_and_added", - "file_not_removed_and_added.yml") - assert check_playbook_file_removed_and_added(playbook_path) - - -def test_file_block_removed_and_added(): - playbook_path = os.path.join(os.path.dirname(__file__), - "ansible_file_removed_and_added", - "file_block_removed_and_added.yml") - assert not check_playbook_file_removed_and_added(playbook_path) - - -def test_file_block_removed_and_not_added(): - playbook_path = os.path.join(os.path.dirname(__file__), - "ansible_file_removed_and_added", - "file_block_removed_and_not_added.yml") - assert check_playbook_file_removed_and_added(playbook_path) From 7093200c0d0c3ef73beb579f655989fd45c67a29 Mon Sep 17 00:00:00 2001 From: Marcus Burghardt Date: Fri, 11 Aug 2023 10:59:42 +0200 Subject: [PATCH 3/4] Remove reference files used by file_block_removed_and_added test These files are no longer necessary. --- .../file_block_removed_and_added.yml | 69 ------------------- .../file_block_removed_and_not_added.yml | 62 ----------------- .../file_not_removed_and_added.yml | 49 ------------- .../file_removed_and_added.yml | 62 ----------------- .../file_removed_and_not_added.yml | 46 ------------- 5 files changed, 288 deletions(-) delete mode 100644 tests/ansible_file_removed_and_added/file_block_removed_and_added.yml delete mode 100644 tests/ansible_file_removed_and_added/file_block_removed_and_not_added.yml delete mode 100644 tests/ansible_file_removed_and_added/file_not_removed_and_added.yml delete mode 100644 tests/ansible_file_removed_and_added/file_removed_and_added.yml delete mode 100644 tests/ansible_file_removed_and_added/file_removed_and_not_added.yml diff --git a/tests/ansible_file_removed_and_added/file_block_removed_and_added.yml b/tests/ansible_file_removed_and_added/file_block_removed_and_added.yml deleted file mode 100644 index 8863b333129..00000000000 --- a/tests/ansible_file_removed_and_added/file_block_removed_and_added.yml +++ /dev/null @@ -1,69 +0,0 @@ ---- - -- hosts: all - vars: - var_system_crypto_policy: !!str FUTURE - var_sudo_logfile: !!str /var/log/sudo.log - - tasks: - - name: Modify the System Login Banner - add correct banner - lineinfile: - dest: /etc/issue - line: '{{ login_banner_text | regex_replace("^\^(.*)\$$", "\1") | regex_replace("^\((.*)\|.*\)$", - "\1") | regex_replace("\[\\s\\n\]\+"," ") | regex_replace("\(\?:\[\\n\]\+\|\(\?:\\\\n\)\+\)", - "\n") | regex_replace("\\", "") | wordwrap() }}' - create: true - when: ansible_virtualization_type not in ["docker", "lxc", "openvz", "podman", "container"] - tags: - - banner_etc_issue - - low_complexity - - medium_disruption - - medium_severity - - no_reboot_needed - - unknown_strategy - - - name: Test for existence /etc/issue - stat: - path: /etc/issue - register: file_exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - - - name: Ensure permission 0644 on /etc/issue - file: - path: /etc/issue - mode: '0644' - when: file_exists.stat is defined and file_exists.stat.exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - - - block: - - - name: Remove Rsh Trust Files - file: - path: /root/shosts.equiv - state: absent - - - name: Add line to /root/shosts.equiv - lineinfile: - dest: /root/shosts.equiv - line: 'test host' - create: true - tags: - - high_severity - - low_complexity - - low_disruption - - no_reboot_needed - - no_rsh_trust_files - - restrict_strategy - diff --git a/tests/ansible_file_removed_and_added/file_block_removed_and_not_added.yml b/tests/ansible_file_removed_and_added/file_block_removed_and_not_added.yml deleted file mode 100644 index 8391d1bc99b..00000000000 --- a/tests/ansible_file_removed_and_added/file_block_removed_and_not_added.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- - -- hosts: all - vars: - var_system_crypto_policy: !!str FUTURE - var_sudo_logfile: !!str /var/log/sudo.log - - tasks: - - name: Modify the System Login Banner - add correct banner - lineinfile: - dest: /etc/issue - line: '{{ login_banner_text | regex_replace("^\^(.*)\$$", "\1") | regex_replace("^\((.*)\|.*\)$", - "\1") | regex_replace("\[\\s\\n\]\+"," ") | regex_replace("\(\?:\[\\n\]\+\|\(\?:\\\\n\)\+\)", - "\n") | regex_replace("\\", "") | wordwrap() }}' - create: true - when: ansible_virtualization_type not in ["docker", "lxc", "openvz", "podman", "container"] - tags: - - banner_etc_issue - - low_complexity - - medium_disruption - - medium_severity - - no_reboot_needed - - unknown_strategy - - - name: Test for existence /etc/issue - stat: - path: /etc/issue - register: file_exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - - - name: Ensure permission 0644 on /etc/issue - file: - path: /etc/issue - mode: '0644' - when: file_exists.stat is defined and file_exists.stat.exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - - - block: - - name: Remove Rsh Trust Files - file: - path: '/root/shosts.equiv' - state: absent - tags: - - high_severity - - low_complexity - - low_disruption - - no_reboot_needed - - no_rsh_trust_files - - restrict_strategy - diff --git a/tests/ansible_file_removed_and_added/file_not_removed_and_added.yml b/tests/ansible_file_removed_and_added/file_not_removed_and_added.yml deleted file mode 100644 index 3d3e53b958f..00000000000 --- a/tests/ansible_file_removed_and_added/file_not_removed_and_added.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -- hosts: all - vars: - var_system_crypto_policy: !!str FUTURE - var_sudo_logfile: !!str /var/log/sudo.log - - tasks: - - name: Modify the System Login Banner - add correct banner - lineinfile: - dest: /etc/issue - line: '{{ login_banner_text | regex_replace("^\^(.*)\$$", "\1") | regex_replace("^\((.*)\|.*\)$", - "\1") | regex_replace("\[\\s\\n\]\+"," ") | regex_replace("\(\?:\[\\n\]\+\|\(\?:\\\\n\)\+\)", - "\n") | regex_replace("\\", "") | wordwrap() }}' - create: true - when: ansible_virtualization_type not in ["docker", "lxc", "openvz", "podman", "container"] - tags: - - banner_etc_issue - - low_complexity - - medium_disruption - - medium_severity - - no_reboot_needed - - unknown_strategy - - - name: Test for existence /etc/issue - stat: - path: /etc/issue - register: file_exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - - - name: Ensure permission 0644 on /etc/issue - file: - path: /etc/issue - mode: '0644' - when: file_exists.stat is defined and file_exists.stat.exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - diff --git a/tests/ansible_file_removed_and_added/file_removed_and_added.yml b/tests/ansible_file_removed_and_added/file_removed_and_added.yml deleted file mode 100644 index a44c39a9db2..00000000000 --- a/tests/ansible_file_removed_and_added/file_removed_and_added.yml +++ /dev/null @@ -1,62 +0,0 @@ ---- - -- hosts: all - vars: - var_system_crypto_policy: !!str FUTURE - var_sudo_logfile: !!str /var/log/sudo.log - - tasks: - - name: Modify the System Login Banner - remove incorrect banner - file: - state: absent - path: /etc/issue - when: ansible_virtualization_type not in ["docker", "lxc", "openvz", "podman", "container"] - tags: - - banner_etc_issue - - low_complexity - - medium_disruption - - medium_severity - - no_reboot_needed - - unknown_strategy - - - name: Modify the System Login Banner - add correct banner - lineinfile: - dest: /etc/issue - line: '{{ login_banner_text | regex_replace("^\^(.*)\$$", "\1") | regex_replace("^\((.*)\|.*\)$", - "\1") | regex_replace("\[\\s\\n\]\+"," ") | regex_replace("\(\?:\[\\n\]\+\|\(\?:\\\\n\)\+\)", - "\n") | regex_replace("\\", "") | wordwrap() }}' - create: true - when: ansible_virtualization_type not in ["docker", "lxc", "openvz", "podman", "container"] - tags: - - banner_etc_issue - - low_complexity - - medium_disruption - - medium_severity - - no_reboot_needed - - unknown_strategy - - - name: Test for existence /etc/issue - stat: - path: /etc/issue - register: file_exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - - - name: Ensure permission 0644 on /etc/issue - file: - path: /etc/issue - mode: '0644' - when: file_exists.stat is defined and file_exists.stat.exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - diff --git a/tests/ansible_file_removed_and_added/file_removed_and_not_added.yml b/tests/ansible_file_removed_and_added/file_removed_and_not_added.yml deleted file mode 100644 index 08cda7e5063..00000000000 --- a/tests/ansible_file_removed_and_added/file_removed_and_not_added.yml +++ /dev/null @@ -1,46 +0,0 @@ ---- - -- hosts: all - vars: - var_system_crypto_policy: !!str FUTURE - var_sudo_logfile: !!str /var/log/sudo.log - - tasks: - - name: Modify the System Login Banner - remove incorrect banner - file: - state: absent - path: /etc/issue - when: ansible_virtualization_type not in ["docker", "lxc", "openvz", "podman", "container"] - tags: - - banner_etc_issue - - low_complexity - - medium_disruption - - medium_severity - - no_reboot_needed - - unknown_strategy - - - name: Test for existence /etc/issue - stat: - path: /etc/issue - register: file_exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - - - name: Ensure permission 0644 on /etc/issue - file: - path: /etc/issue - mode: '0644' - when: file_exists.stat is defined and file_exists.stat.exists - tags: - - configure_strategy - - file_permissions_etc_issue - - low_complexity - - low_disruption - - medium_severity - - no_reboot_needed - From ab8570f885fdf2f7739a6de4974633cff16f6213 Mon Sep 17 00:00:00 2001 From: Marcus Burghardt Date: Fri, 11 Aug 2023 10:54:49 +0200 Subject: [PATCH 4/4] Remove yamlpath from CMakeLists.txt It was only used by test_ansible_file_removed_and_added.py script. --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d8dd55922d4..96b84d8e12c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,7 +139,6 @@ find_python_module(jinja2 REQUIRED) find_python_module(pytest) find_python_module(pytest_cov) find_python_module(json2html) -find_python_module(yamlpath) find_python_module(mypy) find_python_module(openpyxl) find_python_module(pandas) @@ -235,7 +234,6 @@ message(STATUS "python pytest module (optional): ${PY_PYTEST}") message(STATUS "ansible-playbook module (optional): ${ANSIBLE_PLAYBOOK_EXECUTABLE}") message(STATUS "ansible-lint module (optional): ${ANSIBLE_LINT_EXECUTABLE}") message(STATUS "yamllint module (optional): ${YAMLLINT_EXECUTABLE}") -message(STATUS "yamlpath module (optional): ${PY_YAMLPATH}") message(STATUS "python mypy module (optional): ${PY_MYPY}") message(STATUS "BATS framework (optional): ${BATS_EXECUTABLE}") message(STATUS "python sphinx module (optional): ${PY_SPHINX}")