From 9103b809d1868f9a98c1fe019d006d86b9dd9b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C4=8Cern=C3=BD?= Date: Thu, 29 Jun 2023 09:30:25 +0200 Subject: [PATCH] Introduce a script to compare ComplianceAsCode versions This commit introduces a new script compare_versions.py which can show differences between two ComplianceAsCode versions. Lists added or removed rules, profiles, changes in profile composition and changes in remediations and platforms. For comparison, you can use git tags or ComplianceAsCode JSON manifest files directly. This new feature leverages the ComplianceAsCode JSON manifests, which have been introduced by https://github.com/ComplianceAsCode/content/pull/10761 --- .../developer/05_tools_and_utilities.md | 21 ++ utils/compare_versions.py | 183 ++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 utils/compare_versions.py diff --git a/docs/manual/developer/05_tools_and_utilities.md b/docs/manual/developer/05_tools_and_utilities.md index d3d3db4778..d79b08a9d6 100644 --- a/docs/manual/developer/05_tools_and_utilities.md +++ b/docs/manual/developer/05_tools_and_utilities.md @@ -707,3 +707,24 @@ $ python utils/generate_profile.py -i benchmark.xlsx generate --product-type ocp The `PLACEHOLDER` values must be filled in later, ideally when the rules are provided for each control. + + +### Compare ComplianceAsCode versions - `utils/compare_versions.py` + +Show differences between two ComplianceAsCode versions. +Lists added or removed rules, profiles, changes in profile composition and changes in remediations and platforms. +For comparison, you can use git tags or ComplianceAsCode JSON manifest files directly. + +To compare 2 ComplianceAsCode JSON manifests, provide the manifest files. + +``` +python3 utils/compare_versions.py compare_manifests ~/manifests/old.json ~/manifests/new.json +``` + +To compare 2 upstream versions, you need to specify the version git tags and a product ID. + +``` +$ python3 utils/compare_versions.py compare_tags v0.1.67 v0.1.68 rhel9 +``` + +It will internally clone the upstream project, checkout these tags, generate ComplianceAsCode JSON manifests, compare them and print the output. diff --git a/utils/compare_versions.py b/utils/compare_versions.py new file mode 100644 index 0000000000..795ba97ace --- /dev/null +++ b/utils/compare_versions.py @@ -0,0 +1,183 @@ +#!/usr/bin/python3 + +import argparse +import json +import os +import subprocess +import tempfile + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=""" + Show differences between two ComplianceAsCode versions. + Lists added or removed rules, profiles, changes in profile composition + and changes in remediations and platforms. + For comparison, you can use git tags or ComplianceAsCode JSON manifest + files directly. + """) + subparsers = parser.add_subparsers() + parser_compare_manifests = subparsers.add_parser("compare_manifests") + parser_compare_manifests.set_defaults(func=compare_manifests) + parser_compare_manifests.add_argument( + "manifest1", help="Path to a ComplianceAsCode JSON manifest file") + parser_compare_manifests.add_argument( + "manifest2", help="Path to a ComplianceAsCode JSON manifest file") + parser_compare_tags = subparsers.add_parser("compare_tags") + parser_compare_tags.set_defaults(func=compare_tags) + parser_compare_tags.add_argument("tag1", help="git tag, eg. v0.1.67") + parser_compare_tags.add_argument("tag2", help="git tag, eg. v0.1.68") + parser_compare_tags.add_argument( + "product", help="product ID, eg. 'rhel9'") + return parser.parse_args() + + +def load_manifest(file_path: str) -> dict: + with open(file_path) as f: + manifest = json.load(f) + return manifest + + +def print_set(elements: set) -> None: + for element in sorted(list(elements)): + print(" - " + element) + + +def compare_sets(set1: set, set2: set, title: str, name: str) -> None: + added = set2 - set1 + removed = set1 - set2 + if not added and not removed: + return + print(title) + if added: + print(f"The following {name} were added:") + print_set(added) + if removed: + print(f"The following {name} were removed:") + print_set(removed) + print() + + +class ManifestComparator(): + def __init__(self, manifest1_path: str, manifest2_path: str) -> None: + self.manifest1 = load_manifest(manifest1_path) + self.manifest2 = load_manifest(manifest2_path) + pass + + def compare_products(self) -> None: + product_name1 = self.manifest1["product_name"] + product_name2 = self.manifest2["product_name"] + if product_name1 != product_name2: + print("Product names differ") + + def compare_rules(self) -> None: + rules1 = set(self.manifest1["rules"].keys()) + rules2 = set(self.manifest2["rules"].keys()) + compare_sets(rules1, rules2, "Rules in benchmark:", "rules") + + def _get_reports(self, rules: list) -> list: + rule_reports = [] + for rule_id in rules: + rule1 = self.manifest1["rules"][rule_id] + rule2 = self.manifest2["rules"][rule_id] + content1 = set(rule1["content"]) + platforms1 = set(rule1["platform_names"]) + content2 = set(rule2["content"]) + platforms2 = set(rule2["platform_names"]) + content_added = content2 - content1 + content_removed = content1 - content2 + platforms_added = platforms2 - platforms1 + platforms_removed = platforms1 - platforms2 + if (content_added or content_removed or platforms_added + or platforms_removed): + msgs = [] + if content_added: + msgs.append("adds " + ", ".join(content_added)) + if content_removed: + msgs.append("removes " + ", ".join(content_removed)) + if platforms_added: + msgs.append("adds platform " + ", ".join(platforms_added)) + if platforms_removed: + msgs.append( + "removes platform " + ", ".join(platforms_removed)) + rule_report = "Rule " + rule_id + " " + ", ".join(msgs) + "." + rule_reports.append(rule_report) + return rule_reports + + def compare_rule_details(self) -> None: + rules1 = set(self.manifest1["rules"].keys()) + rules2 = set(self.manifest2["rules"].keys()) + rules_intersection = sorted(rules1 & rules2) + rule_reports = self._get_reports(rules_intersection) + if len(rule_reports) > 0: + print("Differences in rules:") + for report in rule_reports: + print(" - " + report) + print() + + def compare_profiles(self) -> None: + profiles1 = set(self.manifest1["profiles"].keys()) + profiles2 = set(self.manifest2["profiles"].keys()) + compare_sets( + profiles1, profiles2, "Profiles in benchmark:", "profiles") + profiles_intersection = sorted(profiles1 & profiles2) + for profile_id in profiles_intersection: + rules1 = set(self.manifest1["profiles"][profile_id]["rules"]) + rules2 = set(self.manifest2["profiles"][profile_id]["rules"]) + compare_sets( + rules1, rules2, f"Profile {profile_id} differs:", "rules") + + def compare(self) -> None: + self.compare_products() + self.compare_rules() + self.compare_rule_details() + self.compare_profiles() + + +def compare_manifests(args: argparse.Namespace) -> None: + comparator = ManifestComparator(args.manifest1, args.manifest2) + comparator.compare() + + +def clone_git(tag: str, target_directory_path: str) -> None: + cac_uri = "https://github.com/ComplianceAsCode/content.git" + cmd = ["git", "clone", cac_uri, "-b", tag, target_directory_path] + subprocess.run(cmd, check=True, capture_output=True) + + +def build_product(cac_root: str, product: str) -> None: + cmd = ["./build_product", product] + subprocess.run(cmd, cwd=cac_root, check=True, capture_output=True) + + +def generate_manifest(build_root: str, manifest_file_path: str) -> None: + cmd = [ + "python3", "build-scripts/generate_manifest.py", + "--build-root", build_root, "--output", manifest_file_path] + subprocess.run(cmd, check=True, capture_output=True) + + +def prepare(tmpdirname: str, tag: str, product: str) -> str: + git_dir_path = os.path.join(tmpdirname, tag) + clone_git(tag, git_dir_path) + build_product(git_dir_path, product) + build_dir_path = os.path.join(git_dir_path, "build", product) + manifest_file_path = os.path.join(tmpdirname, tag + ".manifest.json") + generate_manifest(build_dir_path, manifest_file_path) + return manifest_file_path + + +def compare_tags(args: argparse.Namespace) -> None: + with tempfile.TemporaryDirectory() as tmpdirname: + manifest1 = prepare(tmpdirname, args.tag1, args.product) + manifest2 = prepare(tmpdirname, args.tag2, args.product) + comparator = ManifestComparator(manifest1, manifest2) + comparator.compare() + + +def main() -> None: + args = parse_args() + args.func(args) + + +if __name__ == "__main__": + main()