Expand snapshots part I (#1467)

* Extend snapshot testing. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Remove sdkman step. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Run java and javascript. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Troubleshooting. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Troubleshooting log. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Enable yarn. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Bazel troubleshooting. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Correct bazel build, specify java. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Set java and bazel env variables. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Enable skipping specific project. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Skip jazzer. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Add sdkman env. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Troubleshooting sdkman path. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Troubleshooting sdkman path. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Troubleshooting sdkman path. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Correct escapes. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Revert time logging. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Repo rename. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Re-enable prebuild. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Clean up java prebuilds. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Correction. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Remove expanded python and dotnet repos. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Run all snapshot tests. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Syntax corrections. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Locate go. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Use --migrate-legacy. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Allow bidirectional --migrate-legacy. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Remove testing branch checkout. Signed-off-by: Caroline Russell <caroline@appthreat.dev> * Remove self-hosted from matrix. Signed-off-by: Caroline Russell <caroline@appthreat.dev> --------- Signed-off-by: Caroline Russell <caroline@appthreat.dev>
CycloneDX · Nov 22, 2024 · 6020dac · 6020dac
1 parent 5cfb69f
commit 6020dac
Show file tree

Hide file tree

Showing 4 changed files with 202 additions and 103 deletions.
diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml
@@ -5,6 +5,9 @@ on:
   pull_request:
     branches:
       - master
+  push:
+    branches:
+      - master
 
 
 concurrency:
@@ -14,74 +17,55 @@ concurrency:
 
 jobs:
 
-  generate:
-    runs-on: ubuntu-latest
+  test_non_dotnet:
+    runs-on: ["snapshot-test"]
     steps:
 
       - uses: actions/checkout@v4
 
-      - name: Setup Node.js environment
-        uses: actions/setup-node@v4.0.1
-        with:
-          node-version: '23.x'
+      - name: Set up pnpm
+        uses: pnpm/action-setup@v4
 
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.12
-
-      - uses: pnpm/action-setup@v4
-
-      -  name: cdxgen, sdkman, custom-json-diff installs
+      -  name: cdxgen, custom-json-diff installs
          shell: bash
          env:
            SHELL: bash
          run: |
            cdxgen_tarball=$(pnpm pack | tail -1)
            npm install -g "$cdxgen_tarball"
-           python -m pip install --upgrade pip
-           python -m pip install pytest
-           git clone https://github.com/appthreat/cdxgen-samples.git /home/runner/work/original_snapshots
-           python -m pip install -r test/diff/requirements.txt
-           curl -s "https://get.sdkman.io" | bash
-           source "/home/runner/.sdkman/bin/sdkman-init.sh"
-
-      - name: sdkman install cache
-        uses: actions/cache@v4
-        id: sdkman-cache
-        with:
-          path: /home/runner/.sdkman/candidates
-          key: "${{ runner.os }}-sdkman-${{ hashFiles('test/diff/repos.csv') }}"
+           git clone https://github.com/appthreat/cdxgen-samples.git original_snapshots
+           python3.12 -m venv .venv
+           source .venv/bin/activate && pip install -r test/diff/requirements.txt
 
-      - name: Generate and run scripts
-        env:
-          SDKMAN_DIR: /home/runner/.sdkman
-          CDXGEN_DEBUG_MODE: debug
-          CDXGEN_LOG: /home/runner/work/new_snapshots/generate.log
-          PREFER_MAVEN_DEPS_TREE: false
+      - name: Generate scripts
         run: |
-          mkdir /home/runner/work/new_snapshots
+          source .venv/bin/activate
           python test/diff/generate.py
-          bash /home/runner/work/new_snapshots/sdkman_installs.sh
-          bash /home/runner/work/new_snapshots/cdxgen_commands.sh
 
       - name: Upload shell scripts generated as artifact
         uses: actions/upload-artifact@v4
         with:
           name: scripts
-          path: /home/runner/work/new_snapshots/*.sh
+          path: new_snapshots/*.sh
+
+      - name: Run scripts
+        env:
+          PREFER_MAVEN_DEPS_TREE: false
+        run: |
+          bash new_snapshots/cdxgen_commands.sh
 
       - name: Upload cdxgen boms
         uses: actions/upload-artifact@v4
         with:
           name: cdxgen_boms
           path: |
-            /home/runner/work/new_snapshots
+            new_snapshots
 
       - name: Test BOMs
         run: |
+          source .venv/bin/activate
           python test/diff/diff_tests.py --migrate-legacy
-          if test -f /home/runner/work/new_snapshots/diffs.json; then
+          if test -f new_snapshots/diffs.json; then
             echo "status=FAILED" >> "$GITHUB_ENV"
           fi
 
@@ -90,9 +74,9 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: diffs
-          path: |
-            /home/runner/work/new_snapshots/diffs.json
-            /home/runner/work/new_snapshots/*.html
+          path: | 
+            new_snapshots/diffs.json
+            new_snapshots/*.html
 
       - name: Exit with error
         if: ${{ env.status == 'FAILED' }}

diff --git a/test/diff/diff_tests.py b/test/diff/diff_tests.py
@@ -2,10 +2,13 @@
 import csv
 import logging
 import os
+from typing import Dict, List, Set
 
 from custom_json_diff.lib.custom_diff import compare_dicts, perform_bom_diff, report_results
 from custom_json_diff.lib.custom_diff_classes import Options
-from custom_json_diff.lib.utils import json_dump, json_load
+from custom_json_diff.lib.utils import json_dump,json_load
+
+from generate import filter_repos
 
 logging.disable(logging.INFO)
 
@@ -15,7 +18,7 @@ def build_args():
     parser.add_argument(
         '--directories',
         '-d',
-        default=['/home/runner/work/original_snapshots', '/home/runner/work/new_snapshots'],
+        default=[f'/{os.getenv("GITHUB_WORKSPACE")}/original_snapshots', f'/{os.getenv("GITHUB_WORKSPACE")}/new_snapshots'],
         help='Directories containing the snapshots to compare',
         nargs=2
     )
@@ -25,16 +28,36 @@ def build_args():
         action="store_true",
         help="Migrate legacy snapshots to 1.6 format"
     )
+    parser.add_argument(
+        '--projects',
+        '-p',
+        help='Filter to these projects.',
+        dest='projects',
+    )
+    parser.add_argument(
+        '--types',
+        '-t',
+        help='Filter to these project types.',
+        dest='project_types',
+    )
+    parser.add_argument(
+        '--skip-projects',
+        '-s',
+        help='Skip these projects'
+    )
     return parser.parse_args()
 
 
-def compare_snapshot(dir1, dir2, options, repo, migrate_legacy):
+def compare_snapshot(dir1: str, dir2: str, options: Options, repo: Dict, migrate_legacy: bool):
     bom_1 = f"{dir1}/{repo['project']}-bom.json"
     bom_2 = f"{dir2}/{repo['project']}-bom.json"
     if migrate_legacy:
         bom_data = migrate_to_1_6(bom_1)
         bom_1 = bom_1.replace("bom.json", "bom.migrated.json")
         json_dump(bom_1, bom_data)
+        bom_data = migrate_to_1_6(bom_1)
+        bom_2 = bom_2.replace("bom.json", "bom.migrated.json")
+        json_dump(bom_2, bom_data)
     options.file_1 = bom_1
     options.file_2 = bom_2
     options.output = f'{dir2}/{repo["project"]}-diff.json'
@@ -45,12 +68,11 @@ def compare_snapshot(dir1, dir2, options, repo, migrate_legacy):
         status, result_summary = perform_bom_diff(j1, j2)
         report_results(status, result_summary, options, j1, j2)
         return status, f"{repo['project']} failed.", result_summary
-    return status, None, None
-
+    return status, f"{repo['project']} succeeded.", {}
 
-def perform_snapshot_tests(dir1, dir2, migrate_legacy):
-    repo_data = read_csv()
 
+def perform_snapshot_tests(dir1: str, dir2: str, projects: List, project_types: Set, migrate_legacy: bool, skipped_projects):
+    repo_data = read_csv(projects, project_types, skipped_projects)
     options = Options(
         allow_new_versions=True,
         allow_new_data=True,
@@ -62,11 +84,9 @@ def perform_snapshot_tests(dir1, dir2, migrate_legacy):
     failed_diffs = {}
     for repo in repo_data:
         status, result, summary = compare_snapshot(dir1, dir2, options, repo, migrate_legacy)
-        if result:
-            print(result)
+        print(result)
         if status:
             failed_diffs[repo["project"]] = summary
-
     if failed_diffs:
         diff_file = os.path.join(dir2, 'diffs.json')
         print("Snapshot tests failed.")
@@ -94,14 +114,21 @@ def migrate_to_1_6(bom_file):
     return bom_data
 
 
-def read_csv():
+def read_csv(projects, project_types, skipped_projects):
     csv_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "repos.csv")
     with open(csv_file, 'r', encoding='utf-8') as f:
         reader = csv.DictReader(f)
         repo_data = list(reader)
-    return repo_data
+    return filter_repos(repo_data, projects, project_types, skipped_projects)
 
 
 if __name__ == '__main__':
     args = build_args()
-    perform_snapshot_tests(args.directories[0], args.directories[1], args.migrate_legacy)
+    if args.project_types:
+        if ',' in args.project_types:
+            project_types = set(args.project_types.split(','))
+        else:
+            project_types = {args.project_types}
+    else:
+        project_types = set()
+    perform_snapshot_tests(args.directories[0], args.directories[1], args.projects, project_types, args.migrate_legacy, args.skip_projects)