From d401067f543051fe84725e053101599bfc5e8689 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Mon, 18 Nov 2024 16:35:17 -0500 Subject: [PATCH 01/29] Extend snapshot testing. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 81 ++++++++------- test/diff/diff_tests.py | 45 +++++--- test/diff/generate.py | 147 ++++++++++++++++++++------- test/diff/repos.csv | 23 +++-- 4 files changed, 197 insertions(+), 99 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 60b0abb26..e63d0c1af 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -5,6 +5,9 @@ on: pull_request: branches: - master + push: + branches: + - master concurrency: @@ -14,74 +17,70 @@ concurrency: jobs: - generate: - runs-on: ubuntu-latest + test_non_dotnet: + runs-on: ["snapshot-test"] steps: - uses: actions/checkout@v4 - - name: Setup Node.js environment - uses: actions/setup-node@v4.0.1 - with: - node-version: '23.x' - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.12 + - name: Install uv versions + env: + UV_PYTHON_INSTALL_DIR: $GITHUB_WORKSPACE/pythons + run: | + uv python install 3.9.20 3.10.11 3.11.4 3.12.1 + uv python list - - uses: pnpm/action-setup@v4 + - name: Set up pnpm + uses: pnpm/action-setup@v4 - - name: cdxgen, sdkman, custom-json-diff installs + - name: cdxgen, custom-json-diff installs shell: bash env: SHELL: bash run: | cdxgen_tarball=$(pnpm pack | tail -1) npm install -g "$cdxgen_tarball" - python -m pip install --upgrade pip - python -m pip install pytest - git clone https://github.com/appthreat/cdxgen-samples.git /home/runner/work/original_snapshots - python -m pip install -r test/diff/requirements.txt - curl -s "https://get.sdkman.io" | bash - source "/home/runner/.sdkman/bin/sdkman-init.sh" - - - name: sdkman install cache - uses: actions/cache@v4 - id: sdkman-cache - with: - path: /home/runner/.sdkman/candidates - key: "${{ runner.os }}-sdkman-${{ hashFiles('test/diff/repos.csv') }}" + git clone https://github.com/appthreat/cdxgen-samples.git $GITHUB_WORKSPACE/original_snapshots + python3.12 -m venv .venv + source .venv/bin/activate && pip install -r test/diff/requirements.txt + cd $GITHUB_WORKSPACE/original_snapshots + git checkout feature/expand-snapshots - - name: Generate and run scripts + - name: Generate scripts env: - SDKMAN_DIR: /home/runner/.sdkman CDXGEN_DEBUG_MODE: debug - CDXGEN_LOG: /home/runner/work/new_snapshots/generate.log - PREFER_MAVEN_DEPS_TREE: false + CDXGEN_LOG: $GITHUB_WORKSPACE/new_snapshots/generate.log run: | - mkdir /home/runner/work/new_snapshots - python test/diff/generate.py - bash /home/runner/work/new_snapshots/sdkman_installs.sh - bash /home/runner/work/new_snapshots/cdxgen_commands.sh + mkdir $GITHUB_WORKSPACE/new_snapshots + source .venv/bin/activate + python test/diff/generate.py -t python,java8,java17,javascript - name: Upload shell scripts generated as artifact uses: actions/upload-artifact@v4 with: name: scripts - path: /home/runner/work/new_snapshots/*.sh + path: $GITHUB_WORKSPACE/new_snapshots/*.sh + + - name: Run script + env: + CDXGEN_DEBUG_MODE: debug + CDXGEN_LOG: $GITHUB_WORKSPACE/new_snapshots/generate.log + run: | + cd $GITHUB_WORKSPACE/new_snapshots + bash cdxgen_commands.sh - name: Upload cdxgen boms uses: actions/upload-artifact@v4 with: name: cdxgen_boms path: | - /home/runner/work/new_snapshots + $GITHUB_WORKSPACE/new_snapshots - name: Test BOMs run: | - python test/diff/diff_tests.py --migrate-legacy - if test -f /home/runner/work/new_snapshots/diffs.json; then + source .venv/bin/activate + python test/diff/diff_tests.py -t python,java8,java17,javascript + if test -f $GITHUB_WORKSPACE/new_snapshots/diffs.json; then echo "status=FAILED" >> "$GITHUB_ENV" fi @@ -90,9 +89,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: diffs - path: | - /home/runner/work/new_snapshots/diffs.json - /home/runner/work/new_snapshots/*.html + path: | + $GITHUB_WORKSPACE/new_snapshots/diffs.json + $GITHUB_WORKSPACE/new_snapshots/*.html - name: Exit with error if: ${{ env.status == 'FAILED' }} diff --git a/test/diff/diff_tests.py b/test/diff/diff_tests.py index 5f7dae36e..3453ad156 100644 --- a/test/diff/diff_tests.py +++ b/test/diff/diff_tests.py @@ -2,10 +2,13 @@ import csv import logging import os +from typing import Dict, List, Set from custom_json_diff.lib.custom_diff import compare_dicts, perform_bom_diff, report_results from custom_json_diff.lib.custom_diff_classes import Options -from custom_json_diff.lib.utils import json_dump, json_load +from custom_json_diff.lib.utils import json_dump,json_load + +from generate import filter_repos logging.disable(logging.INFO) @@ -15,7 +18,7 @@ def build_args(): parser.add_argument( '--directories', '-d', - default=['/home/runner/work/original_snapshots', '/home/runner/work/new_snapshots'], + default=[f'/{os.getenv("GITHUB_WORKSPACE")}/original_snapshots', f'/{os.getenv("GITHUB_WORKSPACE")}/new_snapshots'], help='Directories containing the snapshots to compare', nargs=2 ) @@ -25,10 +28,22 @@ def build_args(): action="store_true", help="Migrate legacy snapshots to 1.6 format" ) + parser.add_argument( + '--projects', + '-p', + help='Filter to these projects.', + dest='projects', + ) + parser.add_argument( + '--types', + '-t', + help='Filter to these project types.', + dest='project_types', + ) return parser.parse_args() -def compare_snapshot(dir1, dir2, options, repo, migrate_legacy): +def compare_snapshot(dir1: str, dir2: str, options: Options, repo: Dict, migrate_legacy: bool): bom_1 = f"{dir1}/{repo['project']}-bom.json" bom_2 = f"{dir2}/{repo['project']}-bom.json" if migrate_legacy: @@ -45,12 +60,11 @@ def compare_snapshot(dir1, dir2, options, repo, migrate_legacy): status, result_summary = perform_bom_diff(j1, j2) report_results(status, result_summary, options, j1, j2) return status, f"{repo['project']} failed.", result_summary - return status, None, None - + return status, f"{repo['project']} succeeded.", {} -def perform_snapshot_tests(dir1, dir2, migrate_legacy): - repo_data = read_csv() +def perform_snapshot_tests(dir1: str, dir2: str, projects: List, project_types: Set, migrate_legacy: bool): + repo_data = read_csv(projects, project_types) options = Options( allow_new_versions=True, allow_new_data=True, @@ -62,11 +76,9 @@ def perform_snapshot_tests(dir1, dir2, migrate_legacy): failed_diffs = {} for repo in repo_data: status, result, summary = compare_snapshot(dir1, dir2, options, repo, migrate_legacy) - if result: - print(result) + print(result) if status: failed_diffs[repo["project"]] = summary - if failed_diffs: diff_file = os.path.join(dir2, 'diffs.json') print("Snapshot tests failed.") @@ -94,14 +106,21 @@ def migrate_to_1_6(bom_file): return bom_data -def read_csv(): +def read_csv(projects, project_types): csv_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "repos.csv") with open(csv_file, 'r', encoding='utf-8') as f: reader = csv.DictReader(f) repo_data = list(reader) - return repo_data + return filter_repos(repo_data, projects, project_types) if __name__ == '__main__': args = build_args() - perform_snapshot_tests(args.directories[0], args.directories[1], args.migrate_legacy) + if args.project_types: + if ',' in args.project_types: + project_types = set(args.project_types.split(',')) + else: + project_types = {args.project_types} + else: + project_types = set() + perform_snapshot_tests(args.directories[0], args.directories[1], args.projects, project_types, args.migrate_legacy) diff --git a/test/diff/generate.py b/test/diff/generate.py index 99949e1ce..c6799eae0 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -3,9 +3,12 @@ import os import pathlib import argparse +from copy import deepcopy from pathlib import Path +from custom_json_diff.lib.utils import file_write + def build_args(): """ @@ -22,24 +25,29 @@ def build_args(): parser.add_argument( '--clone-dir', type=Path, - default=Path('/home/runner/work/src_repos'), + default=Path(f'{os.getenv("GITHUB_WORKSPACE")}/src_repos'), help='Path to src_repos', dest='clone_dir' ) parser.add_argument( - '-o', '--output-dir', + '-o', type=Path, - default='/home/runner/work/new_snapshots', + default=f'{os.getenv("GITHUB_WORKSPACE")}/new_snapshots', help='Path to output', dest='output_dir', ) parser.add_argument( - '-p', '--projects', - help='Filter to these sample projects', + '-p', + help='Filter to these projects.', dest='projects', - nargs='*' + ) + parser.add_argument( + '--types', + '-t', + help='Filter to these project types.', + dest='project_types', ) parser.add_argument( '--skip-clone', @@ -133,7 +141,26 @@ def clone_repo(url, repo_dir): return list2cmdline(clone_cmd) -def exec_on_repo(clone, output_dir, skip_build, repo): +def create_python_venvs(repo_data): + """ + Sets the Python version for each Python repository + + Args: + repo_data (list[dict]): Contains the sample repository data + + Returns: + list[dict]: The updated repository data + """ + for r in repo_data: + if r["language"] == "python": + if r["package_manager"] == "poetry": + r["build_cmd"] = f"poetry env use python{r['language_range']} && {r['build_cmd']}" + else: + r["build_cmd"] = f"python{r['language_range']} -m venv .venv; source .venv/bin/activate && {r['build_cmd']}" + return repo_data + + +def exec_on_repo(clone, output_dir, skip_build, repo, cdxgen_log): """ Determines a sequence of commands on a repository. @@ -148,28 +175,68 @@ def exec_on_repo(clone, output_dir, skip_build, repo): str: The sequence of commands to be executed. """ commands = [] - if clone: commands.append(f'{clone_repo(repo["link"], repo["repo_dir"])}') commands.append(f'{list2cmdline(["cd", repo["repo_dir"]])}') commands.append(f'{checkout_commit(repo["commit"])}') - if not skip_build and len(repo['pre_build_cmd']) > 0: - cmds = repo['pre_build_cmd'].split(';') + if not skip_build and repo["pre_build_cmd"]: + cmds = repo["pre_build_cmd"].split(';') cmds = [cmd.lstrip().rstrip() for cmd in cmds] for cmd in cmds: - new_cmd = list(cmd.split(' ')) - commands.append(f'{list2cmdline(new_cmd)}') - if not skip_build and len(repo['build_cmd']) > 0: - cmds = repo['build_cmd'].split(';') + new_cmd = list(cmd.split(" ")) + commands.append(f"{list2cmdline(new_cmd)}") + if not skip_build and repo["build_cmd"]: + cmds = repo["build_cmd"].split(";") cmds = [cmd.lstrip().rstrip() for cmd in cmds] for cmd in cmds: - new_cmd = list(cmd.split(' ')) - commands.append(f'{list2cmdline(new_cmd)}') - commands.append(f'{run_cdxgen(repo, output_dir)}') - commands = '\n'.join(commands) + new_cmd = list(cmd.split(" ")) + # if repo["language"] == "dotnet": + # new_cmd.extend(["-r", f"{repo['language_range']}"]) + commands.append(f"{list2cmdline(new_cmd)}") + # if repo["language"] == "python": + # if repo["package_manager"] == "pip": + # cdxgen_cmd = f"source .venv/bin/activate && {cdxgen_cmd}" + # else: + # cdxgen_cmd = f"poetry env use {repo['language_range']} && {cdxgen_cmd}" + commands.append(f"$(time TIMEFORMAT='{repo['project']}: %E' {run_cdxgen(repo, output_dir)}) >> {cdxgen_log}\n\n") + commands = "\n".join(commands) return commands +def expand_multi_versions(repo_data): + """ + Creates additional entries for repositories testing multiple versions + + Args: + repo_data (list[dict]): Contains the sample repository data + + Returns: + list[dict]: The expanded repository data + """ + new_data = [] + for r in repo_data: + if "," in r["language_range"]: + versions = r["language_range"].split(",") + for version in versions: + new_repo = deepcopy(r) + new_repo["project"] = f"{r['project']}_{version}" + new_repo["language_range"] = version + new_data.append(new_repo) + else: + new_data.append(r) + return create_python_venvs(new_data) + + +def filter_repos(repo_data, projects, project_types): + if projects: + if project_types: + return [repo for repo in repo_data if repo["project"] in projects or repo["language"] in project_types] + return [repo for repo in repo_data if repo["project"] in projects] + if project_types: + return [repo for repo in repo_data if repo["language"] in project_types] + return repo_data + + def generate(args): """ Generate commands for executing a series of tasks on a repository. @@ -182,22 +249,31 @@ def generate(args): """ if args.output_dir == '.': args.output_dir = pathlib.Path.cwd() + + project_types = set() + if args.project_types: + if ',' in args.project_types: + project_types = set(args.project_types.split(',')) + else: + project_types = {args.project_types} + + repo_data = read_csv(args.repo_csv, args.projects, project_types) + processed_repos = add_repo_dirs(args.clone_dir, expand_multi_versions(repo_data)) + if not args.debug_cmds: check_dirs(args.skip_clone, args.clone_dir, args.output_dir) - repo_data = read_csv(args.repo_csv, args.projects, args.clone_dir) - processed_repos = process_repo_data(repo_data, args.clone_dir) - if not args.skip_build: run_pre_builds(repo_data, args.output_dir, args.debug_cmds) - commands = '' + commands = "" + cdxgen_log = os.getenv("CDXGEN_LOG") for repo in processed_repos: - commands += f'\necho {repo["project"]} started at $(date) >> $CDXGEN_LOG\n' - commands += exec_on_repo(args.skip_clone, args.output_dir, args.skip_build, repo) - commands += f'\necho {repo["project"]} finished at $(date) >> $CDXGEN_LOG\n\n' + # commands += f"\necho {repo['project']} started at $(time) >> $CDXGEN_LOG\n" + commands += exec_on_repo(args.skip_clone, args.output_dir, args.skip_build, repo, cdxgen_log) + # commands += f"\necho {repo['project']} finished at $(time) >> $CDXGEN_LOG\n\n" - commands = ''.join(commands) + commands = "".join(commands) sh_path = Path.joinpath(args.output_dir, 'cdxgen_commands.sh') write_script_file(sh_path, commands, args.debug_cmds) @@ -235,7 +311,6 @@ def list2cmdline(seq): # or search http://msdn.microsoft.com for # "Parsing C++ Command-Line Arguments" result = [] - needquote = False for arg in map(os.fsdecode, seq): bs_buf = [] @@ -292,24 +367,21 @@ def process_repo_data(repo_data, clone_dir): return new_data -def read_csv(csv_file, projects, clone_dir): +def read_csv(csv_file, projects, project_types): """ Reads a CSV file and filters the data based on a list of languages. Parameters: csv_file (pathlib.Path): The path to the CSV file. projects (list): A list of projects names to filter on. - clone_dir (pathlib.Path): The directory storing the cloned repositories. - + project_types (set): A set of project types to filter on. Returns: list: A filtered list of repository data. """ with open(csv_file, 'r', encoding='utf-8') as f: reader = csv.DictReader(f) repo_data = list(reader) - if projects: - repo_data = [repo for repo in repo_data if repo['project'] in projects] - return add_repo_dirs(clone_dir, repo_data) + return filter_repos(repo_data, projects, project_types) def run_cdxgen(repo, output_dir): @@ -357,7 +429,7 @@ def run_pre_builds(repo_data, output_dir, debug_cmds): cmds = set(cmds) commands = [c.replace('use', 'install') for c in cmds] - commands.append('sdk install java 23.0.1-tem') + commands.append('sdk install java 23-tem') commands = '\n'.join(commands) sh_path = Path.joinpath(output_dir, 'sdkman_installs.sh') write_script_file(sh_path, commands, debug_cmds) @@ -375,10 +447,9 @@ def write_script_file(file_path, commands, debug_cmds): Returns: None """ - with open(file_path, 'w', encoding='utf-8') as f: - sdkman_path = Path.joinpath(Path('$SDKMAN_DIR'), 'bin', 'sdkman-init.sh') - f.write(f'#!/usr/bin/bash\nsource {sdkman_path}\n\n') - f.write(commands) + sdkman_path = Path.joinpath(Path('$SDKMAN_DIR'), 'bin', 'sdkman-init.sh') + cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\n\n{commands}' + file_write(str(file_path), cmds, success_msg=f"Wrote script to {file_path}.") if debug_cmds: print(commands) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index 4b795a5da..a6100a78e 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -1,7 +1,16 @@ -project,link,language,pre_build_cmd,build_cmd,commit -"django-goat","https://github.com/red-and-black/DjangoGoat.git","python","","python -m venv venv; source venv/bin/activate && pip install -r requirements_app.txt","5e6aaa6d0497bf24abd179304e6ca51295a8091d" -"java-sec-code","https://github.com/JoyChou93/java-sec-code.git","java8","","mvn -B clean compile -DskipTests=true","457d703e8f89bff657c6c51151ada71ebd09a1c6" -"rasa","https://github.com/RasaHQ/rasa.git","python","pipx install poetry","","7807b19ad5fffab73ca1a04dc710f812115a9288" -"restic","https://github.com/restic/restic.git","go","","go run build.go","3786536dc18ef27aedcfa8e4c6953b48353eee79" -"syncthing","https://github.com/syncthing/syncthing.git","go","","go run build.go","ba6ac2f604eb1cd27764460b687537c5e40aaaf8" -"tinydb","https://github.com/msiemens/tinydb.git","python","","python -m venv venv; source venv/bin/activate && pip install .","3dc6a952ef8700706909bf60a1b15cf21af47608" +project,notes,link,language,language_range,config_files,package_manager,pm_version,pre_build_cmd,build_cmd,commit +djangogoat,,https://github.com/red-and-black/DjangoGoat.git,python,3.10,requirements_app.txt,pip,,,pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d +javaseccode,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,,mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 +restic,,https://github.com/restic/restic.git,go,,,,,,go run build.go,3786536dc18ef27aedcfa8e4c6953b48353eee79 +syncthing,,https://github.com/syncthing/syncthing.git,go,,,,,,go run build.go,ba6ac2f604eb1cd27764460b687537c5e40aaaf8 +tinydb_poetry,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12","pyproject.toml,poetry.lock",poetry,,,poetry install,10644a0e07ad180c5b756aba272ee6b0dbd12df8 +tinydb,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12",pyproject.toml,pip,,rm poetry.lock,pip install .,10644a0e07ad180c5b756aba272ee6b0dbd12df8 +numba,with native c lib,https://github.com/numba/numba.git,python,"3.9,3.10,3.11,3.12","setup.py,requirements.txt",pip,,,python setup.py install,53e976f1b0c6683933fa0a93738362914bffc1cd +funcy,,https://github.com/Suor/funcy.git,python,"3.9,3.10",setup.py,pip,,,python setup.py install,859056d039adea75c1c3550286437ce0b612fe92 +github_readme_stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e +prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba +astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 +plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade +plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade +jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,,bazel build,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb +stripe_dotnet,,https://github.com/stripe/stripe-dotnet.git,dotnet,,,,,,"dotnet build src -c Release",e1641fed1e90d40b10ff9250e2a75baec88f0a8c \ No newline at end of file From 935c099453009250ae1bef05d7e98c7c36472d1d Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Mon, 18 Nov 2024 16:57:37 -0500 Subject: [PATCH 02/29] Remove sdkman step. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 6 ++---- test/diff/generate.py | 9 +++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index e63d0c1af..eedc0bcf6 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -51,7 +51,6 @@ jobs: CDXGEN_DEBUG_MODE: debug CDXGEN_LOG: $GITHUB_WORKSPACE/new_snapshots/generate.log run: | - mkdir $GITHUB_WORKSPACE/new_snapshots source .venv/bin/activate python test/diff/generate.py -t python,java8,java17,javascript @@ -61,13 +60,12 @@ jobs: name: scripts path: $GITHUB_WORKSPACE/new_snapshots/*.sh - - name: Run script + - name: Run scripts env: CDXGEN_DEBUG_MODE: debug CDXGEN_LOG: $GITHUB_WORKSPACE/new_snapshots/generate.log run: | - cd $GITHUB_WORKSPACE/new_snapshots - bash cdxgen_commands.sh + bash $GITHUB_WORKSPACE/new_snapshots/cdxgen_commands.sh - name: Upload cdxgen boms uses: actions/upload-artifact@v4 diff --git a/test/diff/generate.py b/test/diff/generate.py index c6799eae0..cc0d9cbce 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -263,8 +263,8 @@ def generate(args): if not args.debug_cmds: check_dirs(args.skip_clone, args.clone_dir, args.output_dir) - if not args.skip_build: - run_pre_builds(repo_data, args.output_dir, args.debug_cmds) + # if not args.skip_build: + # run_pre_builds(repo_data, args.output_dir, args.debug_cmds) commands = "" cdxgen_log = os.getenv("CDXGEN_LOG") @@ -447,8 +447,9 @@ def write_script_file(file_path, commands, debug_cmds): Returns: None """ - sdkman_path = Path.joinpath(Path('$SDKMAN_DIR'), 'bin', 'sdkman-init.sh') - cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\n\n{commands}' + # sdkman_path = Path.joinpath(Path('$SDKMAN_DIR'), 'bin', 'sdkman-init.sh') + # cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\n\n{commands}' + cmds = f'#!/usr/bin/bash\n\n{commands}' file_write(str(file_path), cmds, success_msg=f"Wrote script to {file_path}.") if debug_cmds: print(commands) From 6ab2b09902a5321596e46a8ee9ff912536ccf5c3 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Mon, 18 Nov 2024 17:45:10 -0500 Subject: [PATCH 03/29] Run java and javascript. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index eedc0bcf6..002e9ebd8 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -77,7 +77,7 @@ jobs: - name: Test BOMs run: | source .venv/bin/activate - python test/diff/diff_tests.py -t python,java8,java17,javascript + python test/diff/diff_tests.py -t java8,java17,javascript if test -f $GITHUB_WORKSPACE/new_snapshots/diffs.json; then echo "status=FAILED" >> "$GITHUB_ENV" fi From e170fc86c3247ed36ccf5e56539ac9aa9b2305f2 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Mon, 18 Nov 2024 18:50:27 -0500 Subject: [PATCH 04/29] Troubleshooting. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 29 +++++++++++----------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 002e9ebd8..94cf8b8c0 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -23,13 +23,6 @@ jobs: - uses: actions/checkout@v4 - - name: Install uv versions - env: - UV_PYTHON_INSTALL_DIR: $GITHUB_WORKSPACE/pythons - run: | - uv python install 3.9.20 3.10.11 3.11.4 3.12.1 - uv python list - - name: Set up pnpm uses: pnpm/action-setup@v4 @@ -40,45 +33,45 @@ jobs: run: | cdxgen_tarball=$(pnpm pack | tail -1) npm install -g "$cdxgen_tarball" - git clone https://github.com/appthreat/cdxgen-samples.git $GITHUB_WORKSPACE/original_snapshots + git clone https://github.com/appthreat/cdxgen-samples.git original_snapshots python3.12 -m venv .venv source .venv/bin/activate && pip install -r test/diff/requirements.txt - cd $GITHUB_WORKSPACE/original_snapshots + cd original_snapshots git checkout feature/expand-snapshots - name: Generate scripts env: CDXGEN_DEBUG_MODE: debug - CDXGEN_LOG: $GITHUB_WORKSPACE/new_snapshots/generate.log + CDXGEN_LOG: new_snapshots/generate.log run: | source .venv/bin/activate - python test/diff/generate.py -t python,java8,java17,javascript + python test/diff/generate.py -t java8,java17,javascript - name: Upload shell scripts generated as artifact uses: actions/upload-artifact@v4 with: name: scripts - path: $GITHUB_WORKSPACE/new_snapshots/*.sh + path: new_snapshots/*.sh - name: Run scripts env: CDXGEN_DEBUG_MODE: debug - CDXGEN_LOG: $GITHUB_WORKSPACE/new_snapshots/generate.log + CDXGEN_LOG: new_snapshots/generate.log run: | - bash $GITHUB_WORKSPACE/new_snapshots/cdxgen_commands.sh + bash new_snapshots/cdxgen_commands.sh - name: Upload cdxgen boms uses: actions/upload-artifact@v4 with: name: cdxgen_boms path: | - $GITHUB_WORKSPACE/new_snapshots + new_snapshots - name: Test BOMs run: | source .venv/bin/activate python test/diff/diff_tests.py -t java8,java17,javascript - if test -f $GITHUB_WORKSPACE/new_snapshots/diffs.json; then + if test -f new_snapshots/diffs.json; then echo "status=FAILED" >> "$GITHUB_ENV" fi @@ -88,8 +81,8 @@ jobs: with: name: diffs path: | - $GITHUB_WORKSPACE/new_snapshots/diffs.json - $GITHUB_WORKSPACE/new_snapshots/*.html + new_snapshots/diffs.json + new_snapshots/*.html - name: Exit with error if: ${{ env.status == 'FAILED' }} From a06fe954dd915678c9cc504ed4054b9e67c88f92 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Mon, 18 Nov 2024 19:53:21 -0500 Subject: [PATCH 05/29] Troubleshooting log. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 2 -- test/diff/generate.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 94cf8b8c0..7aba58838 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -42,7 +42,6 @@ jobs: - name: Generate scripts env: CDXGEN_DEBUG_MODE: debug - CDXGEN_LOG: new_snapshots/generate.log run: | source .venv/bin/activate python test/diff/generate.py -t java8,java17,javascript @@ -56,7 +55,6 @@ jobs: - name: Run scripts env: CDXGEN_DEBUG_MODE: debug - CDXGEN_LOG: new_snapshots/generate.log run: | bash new_snapshots/cdxgen_commands.sh diff --git a/test/diff/generate.py b/test/diff/generate.py index cc0d9cbce..c750fa24e 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -267,7 +267,7 @@ def generate(args): # run_pre_builds(repo_data, args.output_dir, args.debug_cmds) commands = "" - cdxgen_log = os.getenv("CDXGEN_LOG") + cdxgen_log = args.output_dir.joinpath("cdxgen.log") for repo in processed_repos: # commands += f"\necho {repo['project']} started at $(time) >> $CDXGEN_LOG\n" commands += exec_on_repo(args.skip_clone, args.output_dir, args.skip_build, repo, cdxgen_log) From 56e46e7ab39a3a09fdd3de3cc10f3cb2b40a5baf Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Mon, 18 Nov 2024 20:08:46 -0500 Subject: [PATCH 06/29] Enable yarn. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index a6100a78e..adda2120f 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -8,7 +8,7 @@ tinydb,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12",pypro numba,with native c lib,https://github.com/numba/numba.git,python,"3.9,3.10,3.11,3.12","setup.py,requirements.txt",pip,,,python setup.py install,53e976f1b0c6683933fa0a93738362914bffc1cd funcy,,https://github.com/Suor/funcy.git,python,"3.9,3.10",setup.py,pip,,,python setup.py install,859056d039adea75c1c3550286437ce0b612fe92 github_readme_stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e -prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba +prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade From 828078811777953acd45ac2d942ce0d3429ac1f6 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 16:36:22 -0500 Subject: [PATCH 07/29] Bazel troubleshooting. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index adda2120f..2a4dd00b2 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -12,5 +12,5 @@ prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade -jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,,bazel build,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb +jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,,bazelisk build,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb stripe_dotnet,,https://github.com/stripe/stripe-dotnet.git,dotnet,,,,,,"dotnet build src -c Release",e1641fed1e90d40b10ff9250e2a75baec88f0a8c \ No newline at end of file From c9271a5ac9aa4b90cebeafcab5e66caeae5fb0c9 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 17:10:38 -0500 Subject: [PATCH 08/29] Correct bazel build, specify java. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index 2a4dd00b2..e2efa519a 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -1,6 +1,6 @@ project,notes,link,language,language_range,config_files,package_manager,pm_version,pre_build_cmd,build_cmd,commit djangogoat,,https://github.com/red-and-black/DjangoGoat.git,python,3.10,requirements_app.txt,pip,,,pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d -javaseccode,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,,mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 +javaseccode,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem,mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 restic,,https://github.com/restic/restic.git,go,,,,,,go run build.go,3786536dc18ef27aedcfa8e4c6953b48353eee79 syncthing,,https://github.com/syncthing/syncthing.git,go,,,,,,go run build.go,ba6ac2f604eb1cd27764460b687537c5e40aaaf8 tinydb_poetry,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12","pyproject.toml,poetry.lock",poetry,,,poetry install,10644a0e07ad180c5b756aba272ee6b0dbd12df8 @@ -10,7 +10,7 @@ funcy,,https://github.com/Suor/funcy.git,python,"3.9,3.10",setup.py,pip,,,python github_readme_stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 -plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade -plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade -jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,,bazelisk build,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb +plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade +plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,sdk use java 17.0.13-tem,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade +jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem,bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb stripe_dotnet,,https://github.com/stripe/stripe-dotnet.git,dotnet,,,,,,"dotnet build src -c Release",e1641fed1e90d40b10ff9250e2a75baec88f0a8c \ No newline at end of file From 03ecbe5681d45b9a7d356a87ec39d1b90c3c8345 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 17:54:01 -0500 Subject: [PATCH 09/29] Set java and bazel env variables. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index e2efa519a..878751754 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -1,6 +1,6 @@ project,notes,link,language,language_range,config_files,package_manager,pm_version,pre_build_cmd,build_cmd,commit djangogoat,,https://github.com/red-and-black/DjangoGoat.git,python,3.10,requirements_app.txt,pip,,,pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d -javaseccode,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem,mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 +javaseccode,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem),mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 restic,,https://github.com/restic/restic.git,go,,,,,,go run build.go,3786536dc18ef27aedcfa8e4c6953b48353eee79 syncthing,,https://github.com/syncthing/syncthing.git,go,,,,,,go run build.go,ba6ac2f604eb1cd27764460b687537c5e40aaaf8 tinydb_poetry,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12","pyproject.toml,poetry.lock",poetry,,,poetry install,10644a0e07ad180c5b756aba272ee6b0dbd12df8 @@ -10,7 +10,7 @@ funcy,,https://github.com/Suor/funcy.git,python,"3.9,3.10",setup.py,pip,,,python github_readme_stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 -plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade -plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,sdk use java 17.0.13-tem,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade -jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem,bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb +plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem),gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade +plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,sdk use java 17.0.13-tem;export JAVA_HOME=$(sdk home java 17.0.13-tem),gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade +jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem);export BAZEL_CMD="bazelisk";export BAZEL_TARGET="//:jazzer_release",bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb stripe_dotnet,,https://github.com/stripe/stripe-dotnet.git,dotnet,,,,,,"dotnet build src -c Release",e1641fed1e90d40b10ff9250e2a75baec88f0a8c \ No newline at end of file From 72a493ebf38f46f78c48f364d846c6722ed0226e Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 18:56:00 -0500 Subject: [PATCH 10/29] Enable skipping specific project. Signed-off-by: Caroline Russell --- test/diff/diff_tests.py | 15 ++++++++++----- test/diff/generate.py | 19 +++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/test/diff/diff_tests.py b/test/diff/diff_tests.py index 3453ad156..d87e1f7a3 100644 --- a/test/diff/diff_tests.py +++ b/test/diff/diff_tests.py @@ -40,6 +40,11 @@ def build_args(): help='Filter to these project types.', dest='project_types', ) + parser.add_argument( + '--skip-projects', + '-s', + help='Skip these projects' + ) return parser.parse_args() @@ -63,8 +68,8 @@ def compare_snapshot(dir1: str, dir2: str, options: Options, repo: Dict, migrate return status, f"{repo['project']} succeeded.", {} -def perform_snapshot_tests(dir1: str, dir2: str, projects: List, project_types: Set, migrate_legacy: bool): - repo_data = read_csv(projects, project_types) +def perform_snapshot_tests(dir1: str, dir2: str, projects: List, project_types: Set, migrate_legacy: bool, skipped_projects): + repo_data = read_csv(projects, project_types, skipped_projects) options = Options( allow_new_versions=True, allow_new_data=True, @@ -106,12 +111,12 @@ def migrate_to_1_6(bom_file): return bom_data -def read_csv(projects, project_types): +def read_csv(projects, project_types, skipped_projects): csv_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "repos.csv") with open(csv_file, 'r', encoding='utf-8') as f: reader = csv.DictReader(f) repo_data = list(reader) - return filter_repos(repo_data, projects, project_types) + return filter_repos(repo_data, projects, project_types, skipped_projects) if __name__ == '__main__': @@ -123,4 +128,4 @@ def read_csv(projects, project_types): project_types = {args.project_types} else: project_types = set() - perform_snapshot_tests(args.directories[0], args.directories[1], args.projects, project_types, args.migrate_legacy) + perform_snapshot_tests(args.directories[0], args.directories[1], args.projects, project_types, args.migrate_legacy, args.skip_projects) diff --git a/test/diff/generate.py b/test/diff/generate.py index c750fa24e..064a08938 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -69,6 +69,11 @@ def build_args(): default=False, help='Skip building the samples and just run cdxgen. Should be used with --skip-clone' ) + parser.add_argument( + '--skip-projects', + '-s', + help='Skip these projects', + ) return parser.parse_args() @@ -227,8 +232,10 @@ def expand_multi_versions(repo_data): return create_python_venvs(new_data) -def filter_repos(repo_data, projects, project_types): - if projects: +def filter_repos(repo_data, projects, project_types, skipped_projects): + if skipped_projects: + repo_data = [repo for repo in repo_data if repo["project"] not in skipped_projects] + elif projects: if project_types: return [repo for repo in repo_data if repo["project"] in projects or repo["language"] in project_types] return [repo for repo in repo_data if repo["project"] in projects] @@ -257,7 +264,7 @@ def generate(args): else: project_types = {args.project_types} - repo_data = read_csv(args.repo_csv, args.projects, project_types) + repo_data = read_csv(args.repo_csv, args.projects, project_types, args.skip_projects) processed_repos = add_repo_dirs(args.clone_dir, expand_multi_versions(repo_data)) if not args.debug_cmds: @@ -267,7 +274,7 @@ def generate(args): # run_pre_builds(repo_data, args.output_dir, args.debug_cmds) commands = "" - cdxgen_log = args.output_dir.joinpath("cdxgen.log") + cdxgen_log = args.output_dir.joinpath("generate.log") for repo in processed_repos: # commands += f"\necho {repo['project']} started at $(time) >> $CDXGEN_LOG\n" commands += exec_on_repo(args.skip_clone, args.output_dir, args.skip_build, repo, cdxgen_log) @@ -367,7 +374,7 @@ def process_repo_data(repo_data, clone_dir): return new_data -def read_csv(csv_file, projects, project_types): +def read_csv(csv_file, projects, project_types, skipped_projects): """ Reads a CSV file and filters the data based on a list of languages. @@ -381,7 +388,7 @@ def read_csv(csv_file, projects, project_types): with open(csv_file, 'r', encoding='utf-8') as f: reader = csv.DictReader(f) repo_data = list(reader) - return filter_repos(repo_data, projects, project_types) + return filter_repos(repo_data, projects, project_types, skipped_projects) def run_cdxgen(repo, output_dir): From d23ead7e404910e4d0267747422ba33aa0394b22 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 18:56:57 -0500 Subject: [PATCH 11/29] Skip jazzer. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 7aba58838..8b8445f7a 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -44,7 +44,7 @@ jobs: CDXGEN_DEBUG_MODE: debug run: | source .venv/bin/activate - python test/diff/generate.py -t java8,java17,javascript + python test/diff/generate.py -t java8,java17,javascript -s jazzer - name: Upload shell scripts generated as artifact uses: actions/upload-artifact@v4 @@ -68,7 +68,7 @@ jobs: - name: Test BOMs run: | source .venv/bin/activate - python test/diff/diff_tests.py -t java8,java17,javascript + python test/diff/diff_tests.py -t java8,java17,javascript -s jazzer if test -f new_snapshots/diffs.json; then echo "status=FAILED" >> "$GITHUB_ENV" fi From 3a921fad8c5970b50c00d3da84bb794bff2a8635 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 19:43:40 -0500 Subject: [PATCH 12/29] Add sdkman env. Signed-off-by: Caroline Russell --- test/diff/generate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/diff/generate.py b/test/diff/generate.py index 064a08938..7c116f7f5 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -454,9 +454,9 @@ def write_script_file(file_path, commands, debug_cmds): Returns: None """ - # sdkman_path = Path.joinpath(Path('$SDKMAN_DIR'), 'bin', 'sdkman-init.sh') - # cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\n\n{commands}' - cmds = f'#!/usr/bin/bash\n\n{commands}' + sdkman_path = Path.joinpath(Path('$SDKMAN_DIR'), 'bin', 'sdkman-init.sh') + cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\n\n{commands}' + # cmds = f'#!/usr/bin/bash\n\n{commands}' file_write(str(file_path), cmds, success_msg=f"Wrote script to {file_path}.") if debug_cmds: print(commands) From af9d734700a75788b2a3395ccd7c9b273b030408 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 20:19:13 -0500 Subject: [PATCH 13/29] Troubleshooting sdkman path. Signed-off-by: Caroline Russell --- test/diff/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/diff/generate.py b/test/diff/generate.py index 7c116f7f5..1d3db0719 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -454,7 +454,7 @@ def write_script_file(file_path, commands, debug_cmds): Returns: None """ - sdkman_path = Path.joinpath(Path('$SDKMAN_DIR'), 'bin', 'sdkman-init.sh') + sdkman_path = Path.joinpath(Path(os.getenv("SDKMAN_DIR")), "bin", "sdkman-init.sh") cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\n\n{commands}' # cmds = f'#!/usr/bin/bash\n\n{commands}' file_write(str(file_path), cmds, success_msg=f"Wrote script to {file_path}.") From 0f404c322a79a336b99b4de743a0eaf9425af736 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 21:08:06 -0500 Subject: [PATCH 14/29] Troubleshooting sdkman path. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 8b8445f7a..e72d7f5c7 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -42,6 +42,7 @@ jobs: - name: Generate scripts env: CDXGEN_DEBUG_MODE: debug + SDKMAN_DIR: "/home/snapshot1/.sdkman" run: | source .venv/bin/activate python test/diff/generate.py -t java8,java17,javascript -s jazzer @@ -55,6 +56,7 @@ jobs: - name: Run scripts env: CDXGEN_DEBUG_MODE: debug + SDKMAN_DIR: "/home/snapshot1/.sdkman" run: | bash new_snapshots/cdxgen_commands.sh From 87f15159a39782539bfdbdea0422d1cd2ce638d7 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 22:35:09 -0500 Subject: [PATCH 15/29] Troubleshooting sdkman path. Signed-off-by: Caroline Russell --- test/diff/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/diff/generate.py b/test/diff/generate.py index 1d3db0719..286e3bf50 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -454,7 +454,7 @@ def write_script_file(file_path, commands, debug_cmds): Returns: None """ - sdkman_path = Path.joinpath(Path(os.getenv("SDKMAN_DIR")), "bin", "sdkman-init.sh") + sdkman_path = Path.joinpath(Path("/home/snapshot1/.sdkman"), "bin", "sdkman-init.sh") cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\n\n{commands}' # cmds = f'#!/usr/bin/bash\n\n{commands}' file_write(str(file_path), cmds, success_msg=f"Wrote script to {file_path}.") From 097036ce4fcb6c0d50028ed3a4764d7be5f92ec1 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 23:40:57 -0500 Subject: [PATCH 16/29] Correct escapes. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index 878751754..0a527f5ad 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -8,9 +8,9 @@ tinydb,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12",pypro numba,with native c lib,https://github.com/numba/numba.git,python,"3.9,3.10,3.11,3.12","setup.py,requirements.txt",pip,,,python setup.py install,53e976f1b0c6683933fa0a93738362914bffc1cd funcy,,https://github.com/Suor/funcy.git,python,"3.9,3.10",setup.py,pip,,,python setup.py install,859056d039adea75c1c3550286437ce0b612fe92 github_readme_stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e -prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba +prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable yarn;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem),gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,sdk use java 17.0.13-tem;export JAVA_HOME=$(sdk home java 17.0.13-tem),gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade -jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem);export BAZEL_CMD="bazelisk";export BAZEL_TARGET="//:jazzer_release",bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb +jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem);export BAZEL_CMD='bazelisk';export BAZEL_TARGET='//:jazzer_release',bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb stripe_dotnet,,https://github.com/stripe/stripe-dotnet.git,dotnet,,,,,,"dotnet build src -c Release",e1641fed1e90d40b10ff9250e2a75baec88f0a8c \ No newline at end of file From 5c1ea7d1d84a5cd1e9aec68d7477f01ed4937b73 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Tue, 19 Nov 2024 23:41:22 -0500 Subject: [PATCH 17/29] Revert time logging. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 7 ++----- test/diff/generate.py | 6 +++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index e72d7f5c7..eff389d50 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -40,9 +40,6 @@ jobs: git checkout feature/expand-snapshots - name: Generate scripts - env: - CDXGEN_DEBUG_MODE: debug - SDKMAN_DIR: "/home/snapshot1/.sdkman" run: | source .venv/bin/activate python test/diff/generate.py -t java8,java17,javascript -s jazzer @@ -55,8 +52,8 @@ jobs: - name: Run scripts env: - CDXGEN_DEBUG_MODE: debug - SDKMAN_DIR: "/home/snapshot1/.sdkman" + CDXGEN_LOG: $GITHUB_WORKSPACE/new_snapshots/generate.log + PREFER_MAVEN_DEPS_TREE: false run: | bash new_snapshots/cdxgen_commands.sh diff --git a/test/diff/generate.py b/test/diff/generate.py index 286e3bf50..943ca503d 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -203,7 +203,7 @@ def exec_on_repo(clone, output_dir, skip_build, repo, cdxgen_log): # cdxgen_cmd = f"source .venv/bin/activate && {cdxgen_cmd}" # else: # cdxgen_cmd = f"poetry env use {repo['language_range']} && {cdxgen_cmd}" - commands.append(f"$(time TIMEFORMAT='{repo['project']}: %E' {run_cdxgen(repo, output_dir)}) >> {cdxgen_log}\n\n") + commands.append(run_cdxgen(repo, output_dir)) commands = "\n".join(commands) return commands @@ -276,9 +276,9 @@ def generate(args): commands = "" cdxgen_log = args.output_dir.joinpath("generate.log") for repo in processed_repos: - # commands += f"\necho {repo['project']} started at $(time) >> $CDXGEN_LOG\n" + commands += f"\necho {repo['project']} started: $(date) >> /home/snapshot1/actions-runner/_work/cdxgen/cdxgen/new_snapshots/generate.log\n" commands += exec_on_repo(args.skip_clone, args.output_dir, args.skip_build, repo, cdxgen_log) - # commands += f"\necho {repo['project']} finished at $(time) >> $CDXGEN_LOG\n\n" + commands += f"\necho {repo['project']} finished: $(date) >> /home/snapshot1/actions-runner/_work/cdxgen/cdxgen/new_snapshots/generate.log\n\n" commands = "".join(commands) sh_path = Path.joinpath(args.output_dir, 'cdxgen_commands.sh') From d1fb063c6a1078095645e6a5d2b086a57177ce87 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 01:56:35 -0500 Subject: [PATCH 18/29] Repo rename. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index 0a527f5ad..fbae2bf82 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -1,13 +1,13 @@ project,notes,link,language,language_range,config_files,package_manager,pm_version,pre_build_cmd,build_cmd,commit -djangogoat,,https://github.com/red-and-black/DjangoGoat.git,python,3.10,requirements_app.txt,pip,,,pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d -javaseccode,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem),mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 +django-goat,,https://github.com/red-and-black/DjangoGoat.git,python,3.10,requirements_app.txt,pip,,,pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d +java-sec-code,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem),mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 restic,,https://github.com/restic/restic.git,go,,,,,,go run build.go,3786536dc18ef27aedcfa8e4c6953b48353eee79 syncthing,,https://github.com/syncthing/syncthing.git,go,,,,,,go run build.go,ba6ac2f604eb1cd27764460b687537c5e40aaaf8 tinydb_poetry,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12","pyproject.toml,poetry.lock",poetry,,,poetry install,10644a0e07ad180c5b756aba272ee6b0dbd12df8 tinydb,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12",pyproject.toml,pip,,rm poetry.lock,pip install .,10644a0e07ad180c5b756aba272ee6b0dbd12df8 numba,with native c lib,https://github.com/numba/numba.git,python,"3.9,3.10,3.11,3.12","setup.py,requirements.txt",pip,,,python setup.py install,53e976f1b0c6683933fa0a93738362914bffc1cd funcy,,https://github.com/Suor/funcy.git,python,"3.9,3.10",setup.py,pip,,,python setup.py install,859056d039adea75c1c3550286437ce0b612fe92 -github_readme_stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e +github-readme-stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable yarn;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem),gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade From 794b0760e264d89c6d19d9e7e4c4ad7a8a40e03b Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 06:43:01 -0500 Subject: [PATCH 19/29] Re-enable prebuild. Signed-off-by: Caroline Russell --- test/diff/generate.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/test/diff/generate.py b/test/diff/generate.py index 943ca503d..01b604ba6 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -74,6 +74,11 @@ def build_args(): '-s', help='Skip these projects', ) + parser.add_argument( + '--sdkman-sh', + help='Location to activate sdkman.', + default='~/.sdkman/bin/sdkman-init.sh' + ) return parser.parse_args() @@ -165,7 +170,7 @@ def create_python_venvs(repo_data): return repo_data -def exec_on_repo(clone, output_dir, skip_build, repo, cdxgen_log): +def exec_on_repo(clone, output_dir, skip_build, repo): """ Determines a sequence of commands on a repository. @@ -270,19 +275,19 @@ def generate(args): if not args.debug_cmds: check_dirs(args.skip_clone, args.clone_dir, args.output_dir) - # if not args.skip_build: - # run_pre_builds(repo_data, args.output_dir, args.debug_cmds) + if not args.skip_build: + run_pre_builds(repo_data, args.output_dir, args.debug_cmds) commands = "" cdxgen_log = args.output_dir.joinpath("generate.log") for repo in processed_repos: - commands += f"\necho {repo['project']} started: $(date) >> /home/snapshot1/actions-runner/_work/cdxgen/cdxgen/new_snapshots/generate.log\n" - commands += exec_on_repo(args.skip_clone, args.output_dir, args.skip_build, repo, cdxgen_log) - commands += f"\necho {repo['project']} finished: $(date) >> /home/snapshot1/actions-runner/_work/cdxgen/cdxgen/new_snapshots/generate.log\n\n" + commands += f"\necho {repo['project']} started: $(date) >> {cdxgen_log}\n" + commands += exec_on_repo(args.skip_clone, args.output_dir, args.skip_build, repo) + commands += f"\necho {repo['project']} finished: $(date) >> {cdxgen_log}\n\n" commands = "".join(commands) sh_path = Path.joinpath(args.output_dir, 'cdxgen_commands.sh') - write_script_file(sh_path, commands, args.debug_cmds) + write_script_file(sh_path, commands, args.debug_cmds, args.sdkman_sh) def list2cmdline(seq): @@ -442,7 +447,7 @@ def run_pre_builds(repo_data, output_dir, debug_cmds): write_script_file(sh_path, commands, debug_cmds) -def write_script_file(file_path, commands, debug_cmds): +def write_script_file(file_path, commands, debug_cmds, sdkman_path): """ Write a script to execute a series of commands in a file. @@ -454,9 +459,7 @@ def write_script_file(file_path, commands, debug_cmds): Returns: None """ - sdkman_path = Path.joinpath(Path("/home/snapshot1/.sdkman"), "bin", "sdkman-init.sh") cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\n\n{commands}' - # cmds = f'#!/usr/bin/bash\n\n{commands}' file_write(str(file_path), cmds, success_msg=f"Wrote script to {file_path}.") if debug_cmds: print(commands) From 1002be5c42b455de44a5e6b0449ee2007e688c3e Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 06:43:21 -0500 Subject: [PATCH 20/29] Clean up java prebuilds. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index fbae2bf82..f1874d2fd 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -1,6 +1,6 @@ project,notes,link,language,language_range,config_files,package_manager,pm_version,pre_build_cmd,build_cmd,commit django-goat,,https://github.com/red-and-black/DjangoGoat.git,python,3.10,requirements_app.txt,pip,,,pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d -java-sec-code,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem),mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 +java-sec-code,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem,mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 restic,,https://github.com/restic/restic.git,go,,,,,,go run build.go,3786536dc18ef27aedcfa8e4c6953b48353eee79 syncthing,,https://github.com/syncthing/syncthing.git,go,,,,,,go run build.go,ba6ac2f604eb1cd27764460b687537c5e40aaaf8 tinydb_poetry,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12","pyproject.toml,poetry.lock",poetry,,,poetry install,10644a0e07ad180c5b756aba272ee6b0dbd12df8 @@ -10,7 +10,7 @@ funcy,,https://github.com/Suor/funcy.git,python,"3.9,3.10",setup.py,pip,,,python github-readme-stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable yarn;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 -plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem),gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade -plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,sdk use java 17.0.13-tem;export JAVA_HOME=$(sdk home java 17.0.13-tem),gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade -jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem;export JAVA_HOME=$(sdk home java 8.0.432-tem);export BAZEL_CMD='bazelisk';export BAZEL_TARGET='//:jazzer_release',bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb +plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade +plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade +jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem;export BAZEL_CMD='bazelisk';export BAZEL_TARGET='//:jazzer_release',bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb stripe_dotnet,,https://github.com/stripe/stripe-dotnet.git,dotnet,,,,,,"dotnet build src -c Release",e1641fed1e90d40b10ff9250e2a75baec88f0a8c \ No newline at end of file From b34efef531be8e8b9361f517b8317d195a73a807 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 06:49:02 -0500 Subject: [PATCH 21/29] Correction. Signed-off-by: Caroline Russell --- test/diff/generate.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/diff/generate.py b/test/diff/generate.py index 01b604ba6..d064cd05f 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -276,7 +276,7 @@ def generate(args): check_dirs(args.skip_clone, args.clone_dir, args.output_dir) if not args.skip_build: - run_pre_builds(repo_data, args.output_dir, args.debug_cmds) + run_pre_builds(repo_data, args.output_dir, args.debug_cmds, args.sdkman_sh) commands = "" cdxgen_log = args.output_dir.joinpath("generate.log") @@ -419,7 +419,7 @@ def run_cdxgen(repo, output_dir): return list2cmdline(cdxgen_cmd) -def run_pre_builds(repo_data, output_dir, debug_cmds): +def run_pre_builds(repo_data, output_dir, debug_cmds, sdkman_sh): """ Generates a list of commands to be executed before the build process. @@ -444,7 +444,7 @@ def run_pre_builds(repo_data, output_dir, debug_cmds): commands.append('sdk install java 23-tem') commands = '\n'.join(commands) sh_path = Path.joinpath(output_dir, 'sdkman_installs.sh') - write_script_file(sh_path, commands, debug_cmds) + write_script_file(sh_path, commands, debug_cmds, sdkman_sh) def write_script_file(file_path, commands, debug_cmds, sdkman_path): From 47029a9df6865594034026bbd0dd64bb09e0dbfa Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 07:01:15 -0500 Subject: [PATCH 22/29] Remove expanded python and dotnet repos. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index f1874d2fd..c2eaf4fd5 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -1,16 +1,13 @@ project,notes,link,language,language_range,config_files,package_manager,pm_version,pre_build_cmd,build_cmd,commit -django-goat,,https://github.com/red-and-black/DjangoGoat.git,python,3.10,requirements_app.txt,pip,,,pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d +django-goat,https://github.com/red-and-black/DjangoGoat.git,python,,python -m venv venv; source venv/bin/activate && pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d java-sec-code,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem,mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 +rasa,https://github.com/RasaHQ/rasa.git,python,pipx install poetry,,7807b19ad5fffab73ca1a04dc710f812115a9288 restic,,https://github.com/restic/restic.git,go,,,,,,go run build.go,3786536dc18ef27aedcfa8e4c6953b48353eee79 syncthing,,https://github.com/syncthing/syncthing.git,go,,,,,,go run build.go,ba6ac2f604eb1cd27764460b687537c5e40aaaf8 -tinydb_poetry,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12","pyproject.toml,poetry.lock",poetry,,,poetry install,10644a0e07ad180c5b756aba272ee6b0dbd12df8 -tinydb,,https://github.com/msiemens/tinydb.git,python,"3.9,3.10,3.11,3.12",pyproject.toml,pip,,rm poetry.lock,pip install .,10644a0e07ad180c5b756aba272ee6b0dbd12df8 -numba,with native c lib,https://github.com/numba/numba.git,python,"3.9,3.10,3.11,3.12","setup.py,requirements.txt",pip,,,python setup.py install,53e976f1b0c6683933fa0a93738362914bffc1cd -funcy,,https://github.com/Suor/funcy.git,python,"3.9,3.10",setup.py,pip,,,python setup.py install,859056d039adea75c1c3550286437ce0b612fe92 github-readme-stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable yarn;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem;export BAZEL_CMD='bazelisk';export BAZEL_TARGET='//:jazzer_release',bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb -stripe_dotnet,,https://github.com/stripe/stripe-dotnet.git,dotnet,,,,,,"dotnet build src -c Release",e1641fed1e90d40b10ff9250e2a75baec88f0a8c \ No newline at end of file +tinydb,https://github.com/msiemens/tinydb.git,python,,python -m venv venv; source venv/bin/activate && pip install .,3dc6a952ef8700706909bf60a1b15cf21af47608 \ No newline at end of file From b2ec32849de7ed6439368c286abcab0ce8d78d59 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 07:02:22 -0500 Subject: [PATCH 23/29] Run all snapshot tests. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index eff389d50..5e0fc4c6d 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -42,7 +42,7 @@ jobs: - name: Generate scripts run: | source .venv/bin/activate - python test/diff/generate.py -t java8,java17,javascript -s jazzer + python test/diff/generate.py - name: Upload shell scripts generated as artifact uses: actions/upload-artifact@v4 @@ -52,7 +52,6 @@ jobs: - name: Run scripts env: - CDXGEN_LOG: $GITHUB_WORKSPACE/new_snapshots/generate.log PREFER_MAVEN_DEPS_TREE: false run: | bash new_snapshots/cdxgen_commands.sh @@ -67,7 +66,7 @@ jobs: - name: Test BOMs run: | source .venv/bin/activate - python test/diff/diff_tests.py -t java8,java17,javascript -s jazzer + python test/diff/diff_tests.py if test -f new_snapshots/diffs.json; then echo "status=FAILED" >> "$GITHUB_ENV" fi From d9a96ca2cd6dc2be53a9d29fa5723583b9c897b4 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 07:25:55 -0500 Subject: [PATCH 24/29] Syntax corrections. Signed-off-by: Caroline Russell --- test/diff/repos.csv | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/diff/repos.csv b/test/diff/repos.csv index c2eaf4fd5..29cef5659 100644 --- a/test/diff/repos.csv +++ b/test/diff/repos.csv @@ -1,13 +1,13 @@ -project,notes,link,language,language_range,config_files,package_manager,pm_version,pre_build_cmd,build_cmd,commit -django-goat,https://github.com/red-and-black/DjangoGoat.git,python,,python -m venv venv; source venv/bin/activate && pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d -java-sec-code,,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem,mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 -rasa,https://github.com/RasaHQ/rasa.git,python,pipx install poetry,,7807b19ad5fffab73ca1a04dc710f812115a9288 -restic,,https://github.com/restic/restic.git,go,,,,,,go run build.go,3786536dc18ef27aedcfa8e4c6953b48353eee79 -syncthing,,https://github.com/syncthing/syncthing.git,go,,,,,,go run build.go,ba6ac2f604eb1cd27764460b687537c5e40aaaf8 -github-readme-stats,,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e -prettier,,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable yarn;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba -astro,,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 -plantuml,,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade -plantuml_17,,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade -jazzer,,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem;export BAZEL_CMD='bazelisk';export BAZEL_TARGET='//:jazzer_release',bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb -tinydb,https://github.com/msiemens/tinydb.git,python,,python -m venv venv; source venv/bin/activate && pip install .,3dc6a952ef8700706909bf60a1b15cf21af47608 \ No newline at end of file +project,link,language,language_range,config_files,package_manager,pm_version,pre_build_cmd,build_cmd,commit +django-goat,https://github.com/red-and-black/DjangoGoat.git,python,,,,,,python -m venv venv; source venv/bin/activate && pip install -r requirements_app.txt,5e6aaa6d0497bf24abd179304e6ca51295a8091d +java-sec-code,https://github.com/JoyChou93/java-sec-code.git,java8,8,,,,sdk use java 8.0.432-tem,mvn -B clean compile -DskipTests=true,457d703e8f89bff657c6c51151ada71ebd09a1c6 +rasa,https://github.com/RasaHQ/rasa.git,python,,,,,pipx install poetry,,7807b19ad5fffab73ca1a04dc710f812115a9288 +restic,https://github.com/restic/restic.git,go,,,,,,go run build.go,3786536dc18ef27aedcfa8e4c6953b48353eee79 +syncthing,https://github.com/syncthing/syncthing.git,go,,,,,,go run build.go,ba6ac2f604eb1cd27764460b687537c5e40aaaf8 +github-readme-stats,https://github.com/anuraghazra/github-readme-stats.git,javascript,>=18,package-lock.json,npm,,,npm install .,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e +prettier,https://github.com/prettier/prettier.git,javascript,>=18,"package.json,yarn.lock",yarn,4.5.0,,corepack enable yarn;yarn install,9cf9079f75a30f1088529e0cae6296aeb71205ba +astro,https://github.com/withastro/astro.git,javascript,>=18.17.1,"package.json,pnpm-lock.yaml",pnpm,9.12.1,,pnpm install,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63 +plantuml,https://github.com/plantuml/plantuml.git,java8,8,,gradle,,sdk use java 8.0.432-tem,gradle clean build -x javaDoc -PjavacRelease=8,8eb791f39478778788fd47a9195dc1b2feb3eade +plantuml_17,https://github.com/plantuml/plantuml.git,java17,17,,gradle,,,gradle clean build -x javaDoc -PjavacRelease=17,8eb791f39478778788fd47a9195dc1b2feb3eade +jazzer,https://github.com/CodeIntelligenceTesting/jazzer.git,java8,8,,bazelisk,7.3.0,sdk use java 8.0.432-tem;export BAZEL_CMD='bazelisk';export BAZEL_TARGET='//:jazzer_release',bazelisk build //:jazzer_release,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb +tinydb,https://github.com/msiemens/tinydb.git,python,,,,,,python -m venv venv; source venv/bin/activate && pip install .,3dc6a952ef8700706909bf60a1b15cf21af47608 \ No newline at end of file From 242b89bfc78586f7a2e439ea8f5323b9e2e83081 Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 08:32:20 -0500 Subject: [PATCH 25/29] Locate go. Signed-off-by: Caroline Russell --- test/diff/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/diff/generate.py b/test/diff/generate.py index d064cd05f..ce76e0151 100644 --- a/test/diff/generate.py +++ b/test/diff/generate.py @@ -459,7 +459,7 @@ def write_script_file(file_path, commands, debug_cmds, sdkman_path): Returns: None """ - cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\n\n{commands}' + cmds = f'#!/usr/bin/bash\nsource {sdkman_path}\nexport PATH=$PATH:/usr/local/go/bin\n\n{commands}' file_write(str(file_path), cmds, success_msg=f"Wrote script to {file_path}.") if debug_cmds: print(commands) From 08abc43cb5bc8e09d66216d4d7a1956ec4a500bd Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 12:18:44 -0500 Subject: [PATCH 26/29] Use --migrate-legacy. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 5e0fc4c6d..73edf1db1 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -66,7 +66,7 @@ jobs: - name: Test BOMs run: | source .venv/bin/activate - python test/diff/diff_tests.py + python test/diff/diff_tests.py --migrate-legacy if test -f new_snapshots/diffs.json; then echo "status=FAILED" >> "$GITHUB_ENV" fi From cf637e63f459ea01a2f0c855bd483f936dcac74f Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 12:33:06 -0500 Subject: [PATCH 27/29] Allow bidirectional --migrate-legacy. Signed-off-by: Caroline Russell --- test/diff/diff_tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/diff/diff_tests.py b/test/diff/diff_tests.py index d87e1f7a3..06ea62466 100644 --- a/test/diff/diff_tests.py +++ b/test/diff/diff_tests.py @@ -55,6 +55,9 @@ def compare_snapshot(dir1: str, dir2: str, options: Options, repo: Dict, migrate bom_data = migrate_to_1_6(bom_1) bom_1 = bom_1.replace("bom.json", "bom.migrated.json") json_dump(bom_1, bom_data) + bom_data = migrate_to_1_6(bom_1) + bom_2 = bom_2.replace("bom.json", "bom.migrated.json") + json_dump(bom_2, bom_data) options.file_1 = bom_1 options.file_2 = bom_2 options.output = f'{dir2}/{repo["project"]}-diff.json' From 3028eb7bd5715f8b775a512765d45c9c3663655a Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Wed, 20 Nov 2024 13:00:52 -0500 Subject: [PATCH 28/29] Remove testing branch checkout. Signed-off-by: Caroline Russell --- .github/workflows/snapshot-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml index 73edf1db1..d5e345c28 100644 --- a/.github/workflows/snapshot-tests.yml +++ b/.github/workflows/snapshot-tests.yml @@ -36,8 +36,6 @@ jobs: git clone https://github.com/appthreat/cdxgen-samples.git original_snapshots python3.12 -m venv .venv source .venv/bin/activate && pip install -r test/diff/requirements.txt - cd original_snapshots - git checkout feature/expand-snapshots - name: Generate scripts run: | From 84f2f744c59124285d6f8edd6f948cda83c96acc Mon Sep 17 00:00:00 2001 From: Caroline Russell Date: Thu, 21 Nov 2024 18:12:41 -0500 Subject: [PATCH 29/29] Remove self-hosted from matrix. Signed-off-by: Caroline Russell --- .github/workflows/repotests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml index ae8c31026..a4bd6aff6 100644 --- a/.github/workflows/repotests.yml +++ b/.github/workflows/repotests.yml @@ -15,7 +15,7 @@ jobs: fail-fast: true matrix: node-version: ['23.x'] - os: ['ubuntu-latest', 'windows-latest', 'self-hosted'] + os: ['ubuntu-latest', 'windows-latest'] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4