Introduce a script to compare ComplianceAsCode versions

This commit introduces a new script compare_versions.py which can show
differences between two ComplianceAsCode versions.  Lists added or
removed rules, profiles, changes in profile composition and changes in
remediations and platforms.  For comparison, you can use git tags or
ComplianceAsCode JSON manifest files directly.  This new feature
leverages the ComplianceAsCode JSON manifests, which have been
introduced by ComplianceAsCode/content#10761

Author: jan-cerny

docs/manual/developer/05_tools_and_utilities.md

@@ -707,3 +707,24 @@ $ python utils/generate_profile.py -i benchmark.xlsx generate --product-type ocp
 
 The `PLACEHOLDER` values must be filled in later, ideally when the rules are
 provided for each control.
+
+
+### Compare ComplianceAsCode versions - `utils/compare_versions.py`
+
+Show differences between two ComplianceAsCode versions.
+Lists added or removed rules, profiles, changes in profile composition and changes in remediations and platforms.
+For comparison, you can use git tags or ComplianceAsCode JSON manifest files directly.
+
+To compare 2 ComplianceAsCode JSON manifests, provide the manifest files.
+
+```
+python3 utils/compare_versions.py compare_manifests ~/manifests/old.json ~/manifests/new.json
+```
+
+To compare 2 upstream versions, you need to specify the version git tags and a product ID.
+
+```
+$ python3 utils/compare_versions.py compare_tags v0.1.67 v0.1.68 rhel9
+```
+
+It will internally clone the upstream project, checkout these tags, generate ComplianceAsCode JSON manifests, compare them and print the output.

utils/compare_versions.py

@@ -0,0 +1,183 @@
+#!/usr/bin/python3
+
+import argparse
+import json
+import os
+import subprocess
+import tempfile
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="""
+        Show differences between two ComplianceAsCode versions.
+        Lists added or removed rules, profiles, changes in profile composition
+        and changes in remediations and platforms.
+        For comparison, you can use git tags or ComplianceAsCode JSON manifest
+        files directly.
+        """)
+    subparsers = parser.add_subparsers()
+    parser_compare_manifests = subparsers.add_parser("compare_manifests")
+    parser_compare_manifests.set_defaults(func=compare_manifests)
+    parser_compare_manifests.add_argument(
+        "manifest1", help="Path to a ComplianceAsCode JSON manifest file")
+    parser_compare_manifests.add_argument(
+        "manifest2", help="Path to a ComplianceAsCode JSON manifest file")
+    parser_compare_tags = subparsers.add_parser("compare_tags")
+    parser_compare_tags.set_defaults(func=compare_tags)
+    parser_compare_tags.add_argument("tag1", help="git tag, eg. v0.1.67")
+    parser_compare_tags.add_argument("tag2", help="git tag, eg. v0.1.68")
+    parser_compare_tags.add_argument(
+        "product", help="product ID, eg. 'rhel9'")
+    return parser.parse_args()
+
+
+def load_manifest(file_path: str) -> dict:
+    with open(file_path) as f:
+        manifest = json.load(f)
+    return manifest
+
+
+def print_set(elements: set) -> None:
+    for element in sorted(list(elements)):
+        print(" - " + element)
+
+
+def compare_sets(set1: set, set2: set, title: str, name: str) -> None:
+    added = set2 - set1
+    removed = set1 - set2
+    if not added and not removed:
+        return
+    print(title)
+    if added:
+        print(f"The following {name} were added:")
+        print_set(added)
+    if removed:
+        print(f"The following {name} were removed:")
+        print_set(removed)
+    print()
+
+
+class ManifestComparator():
+    def __init__(self, manifest1_path: str, manifest2_path: str) -> None:
+        self.manifest1 = load_manifest(manifest1_path)
+        self.manifest2 = load_manifest(manifest2_path)
+        pass
+
+    def compare_products(self) -> None:
+        product_name1 = self.manifest1["product_name"]
+        product_name2 = self.manifest2["product_name"]
+        if product_name1 != product_name2:
+            print("Product names differ")
+
+    def compare_rules(self) -> None:
+        rules1 = set(self.manifest1["rules"].keys())
+        rules2 = set(self.manifest2["rules"].keys())
+        compare_sets(rules1, rules2, "Rules in benchmark:", "rules")
+
+    def _get_reports(self, rules: list) -> list:
+        rule_reports = []
+        for rule_id in rules:
+            rule1 = self.manifest1["rules"][rule_id]
+            rule2 = self.manifest2["rules"][rule_id]
+            content1 = set(rule1["content"])
+            platforms1 = set(rule1["platform_names"])
+            content2 = set(rule2["content"])
+            platforms2 = set(rule2["platform_names"])
+            content_added = content2 - content1
+            content_removed = content1 - content2
+            platforms_added = platforms2 - platforms1
+            platforms_removed = platforms1 - platforms2
+            if (content_added or content_removed or platforms_added
+                    or platforms_removed):
+                msgs = []
+                if content_added:
+                    msgs.append("adds " + ", ".join(content_added))
+                if content_removed:
+                    msgs.append("removes " + ", ".join(content_removed))
+                if platforms_added:
+                    msgs.append("adds platform " + ", ".join(platforms_added))
+                if platforms_removed:
+                    msgs.append(
+                        "removes platform " + ", ".join(platforms_removed))
+                rule_report = "Rule " + rule_id + " " + ", ".join(msgs) + "."
+                rule_reports.append(rule_report)
+        return rule_reports
+
+    def compare_rule_details(self) -> None:
+        rules1 = set(self.manifest1["rules"].keys())
+        rules2 = set(self.manifest2["rules"].keys())
+        rules_intersection = sorted(rules1 & rules2)
+        rule_reports = self._get_reports(rules_intersection)
+        if len(rule_reports) > 0:
+            print("Differences in rules:")
+            for report in rule_reports:
+                print(" - " + report)
+            print()
+
+    def compare_profiles(self) -> None:
+        profiles1 = set(self.manifest1["profiles"].keys())
+        profiles2 = set(self.manifest2["profiles"].keys())
+        compare_sets(
+            profiles1, profiles2, "Profiles in benchmark:", "profiles")
+        profiles_intersection = sorted(profiles1 & profiles2)
+        for profile_id in profiles_intersection:
+            rules1 = set(self.manifest1["profiles"][profile_id]["rules"])
+            rules2 = set(self.manifest2["profiles"][profile_id]["rules"])
+            compare_sets(
+                rules1, rules2, f"Profile {profile_id} differs:", "rules")
+
+    def compare(self) -> None:
+        self.compare_products()
+        self.compare_rules()
+        self.compare_rule_details()
+        self.compare_profiles()
+
+
+def compare_manifests(args: argparse.Namespace) -> None:
+    comparator = ManifestComparator(args.manifest1, args.manifest2)
+    comparator.compare()
+
+
+def clone_git(tag: str, target_directory_path: str) -> None:
+    cac_uri = "https://github.com/ComplianceAsCode/content.git"
+    cmd = ["git", "clone", cac_uri, "-b", tag, target_directory_path]
+    subprocess.run(cmd, check=True, capture_output=True)
+
+
+def build_product(cac_root: str, product: str) -> None:
+    cmd = ["./build_product", product]
+    subprocess.run(cmd, cwd=cac_root, check=True, capture_output=True)
+
+
+def generate_manifest(build_root: str, manifest_file_path: str) -> None:
+    cmd = [
+        "python3", "build-scripts/generate_manifest.py",
+        "--build-root", build_root, "--output", manifest_file_path]
+    subprocess.run(cmd, check=True, capture_output=True)
+
+
+def prepare(tmpdirname: str, tag: str, product: str) -> str:
+    git_dir_path = os.path.join(tmpdirname, tag)
+    clone_git(tag, git_dir_path)
+    build_product(git_dir_path, product)
+    build_dir_path = os.path.join(git_dir_path, "build", product)
+    manifest_file_path = os.path.join(tmpdirname, tag + ".manifest.json")
+    generate_manifest(build_dir_path, manifest_file_path)
+    return manifest_file_path
+
+
+def compare_tags(args: argparse.Namespace) -> None:
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        manifest1 = prepare(tmpdirname, args.tag1, args.product)
+        manifest2 = prepare(tmpdirname, args.tag2, args.product)
+        comparator = ManifestComparator(manifest1, manifest2)
+        comparator.compare()
+
+
+def main() -> None:
+    args = parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()