container snapshot tests (#1535)

* First cut Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Script to create container snapshot tests data Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Lint fixes Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> * Lint fixes Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com> --------- Signed-off-by: Prabhu Subramanian <prabhu@appthreat.com>
CycloneDX · Jan 7, 2025 · 0f1a142 · 0f1a142
1 parent 743b31e
commit 0f1a142
Show file tree

Hide file tree

Showing 3 changed files with 216 additions and 0 deletions.
diff --git a/contrib/bulk-generate/README.md b/contrib/bulk-generate/README.md
@@ -0,0 +1,16 @@
+# Introduction
+
+This is a script to generate SBOMs for multiple git repos using cdxgen container images.
+
+## Usage
+
+```shell
+node index.js <csv file> <output directory>
+```
+
+## Example csv file
+
+```
+project,link,commit,image,language,cdxgen_vars
+astro,https://github.com/withastro/astro.git,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63,ghcr.io/cyclonedx/cdxgen-node20:v11,js,,
+```
diff --git a/contrib/bulk-generate/index.js b/contrib/bulk-generate/index.js
@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+
+import { spawnSync } from "node:child_process";
+import {
+  copyFileSync,
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, dirname, join } from "node:path";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+
+let url = import.meta.url;
+if (!url.startsWith("file://")) {
+  url = new URL(`file://${import.meta.url}`).toString();
+}
+const dirName = import.meta ? dirname(fileURLToPath(url)) : __dirname;
+
+const DOCKER_CMD = process.env.DOCKER_CMD || "docker";
+
+function gitClone(name, repoUrl, commit, cloneDir) {
+  const repoDir = join(cloneDir, name);
+  const gitArgs = ["clone", repoUrl, repoDir];
+  console.log(`Cloning Repo ${name} to ${repoDir}`);
+  let result = spawnSync("git", gitArgs, {
+    encoding: "utf-8",
+    shell: false,
+    cwd: cloneDir,
+  });
+  if (result.status !== 0) {
+    console.log(result.stderr);
+    process.exit(1);
+  }
+  if (!existsSync(repoDir)) {
+    console.log(`${repoDir} doesn't exist!`);
+    process.exit(1);
+  }
+  if (commit) {
+    result = spawnSync("git", ["checkout", commit], {
+      encoding: "utf-8",
+      shell: false,
+      cwd: repoDir,
+    });
+    if (result.status !== 0) {
+      console.log(`Unable to checkout ${commit}`);
+      console.log(result.stderr);
+      process.exit(1);
+    }
+  }
+  if (existsSync(join(repoDir, ".gitmodules"))) {
+    result = spawnSync("git", ["submodule", "update", "--init"], {
+      encoding: "utf-8",
+      shell: false,
+      cwd: repoDir,
+    });
+    if (result.status !== 0) {
+      console.log(`Unable to checkout ${commit}`);
+      console.log(result.stderr);
+      process.exit(1);
+    }
+  }
+  return repoDir;
+}
+
+function runWithDocker(args, options = {}) {
+  console.log(`Executing ${DOCKER_CMD} ${args.join(" ")}`);
+  const result = spawnSync(DOCKER_CMD, args, {
+    ...options,
+    encoding: "utf-8",
+    stdio: "inherit",
+    stderr: "inherit",
+    env: process.env,
+  });
+  if (result.status !== 0) {
+    console.log(result.stdout, result.stderr, result.error);
+  }
+  return result;
+}
+
+function readcsv(acsv, outputDir) {
+  const argsList = [];
+  if (!existsSync(outputDir)) {
+    mkdirSync(outputDir, {
+      recursive: true,
+    });
+  }
+  const csvLines = readFileSync(acsv, { encoding: "utf-8" })
+    .split("\n")
+    .slice(1);
+  for (const aline of csvLines) {
+    const values = aline.split(/[,|]/);
+    argsList.push({
+      project: values[0],
+      link: values[1],
+      commit: values[2],
+      image: values[3],
+      language: values[4],
+      cdxgen_vars: values[5],
+    });
+  }
+  return argsList;
+}
+
+function main(argvs) {
+  if (argvs.length !== 2) {
+    console.log("USAGE: node index.js <csv file> <output directory>");
+    process.exit(1);
+  }
+  const tempDir = mkdtempSync(join(tmpdir(), "bulk-generate-"));
+  const reposList = readcsv(argvs[0], argvs[1]);
+  for (const repoArgs of reposList) {
+    if (!repoArgs?.project?.length) {
+      continue;
+    }
+    const repoDir = gitClone(
+      repoArgs.project,
+      repoArgs.link,
+      repoArgs.commit,
+      tempDir,
+    );
+    const repoOutputDir = join(argvs[1], repoArgs.project, repoArgs.commit);
+    const envVars = ["-e", "CDXGEN_DEBUG_MODE=debug"];
+    if (repoArgs.cdxgen_vars) {
+      for (const avaluePair of repoArgs.cdxgen_vars(" ")) {
+        if (avaluePair.includes("=")) {
+          envVars.push("-e");
+          envVars.push(avaluePair);
+        }
+      }
+    }
+    if (!existsSync(repoOutputDir)) {
+      mkdirSync(repoOutputDir, {
+        recursive: true,
+      });
+    }
+    const bomFile = join(
+      repoOutputDir,
+      `bom-${repoArgs.language.replaceAll(" ", "-")}.json`,
+    );
+    const dockerArgs = [
+      "run",
+      "--rm",
+      "--pull=always",
+      ...envVars,
+      "-v",
+      "/tmp:/tmp",
+      "-v",
+      `${repoDir}:/app:rw`,
+      "-w",
+      "/app",
+      "-it",
+      repoArgs.image,
+      "-r",
+      "/app",
+      "-o",
+      "/app/bom.json",
+    ];
+    for (const alang of repoArgs.language.split(" ")) {
+      dockerArgs.push("-t");
+      dockerArgs.push(alang);
+    }
+    runWithDocker(dockerArgs, { cwd: repoDir });
+    if (existsSync(join(repoDir, "bom.json"))) {
+      copyFileSync(join(repoDir, "bom.json"), bomFile);
+    } else {
+      console.log(
+        join(repoDir, "bom.json"),
+        "was not found! Check if the image used is valid for this project:",
+        repoArgs.image,
+        repoArgs.language,
+      );
+      process.exit(1);
+    }
+  }
+  if (tempDir.startsWith(tmpdir())) {
+    rmSync(tempDir, { recursive: true, force: true });
+  }
+}
+
+main(process.argv.slice(2));
diff --git a/test/diff/container-tests-repos.csv b/test/diff/container-tests-repos.csv
@@ -0,0 +1,14 @@
+project,link,commit,image,language,cdxgen_vars
+java-sec-code,https://github.com/JoyChou93/java-sec-code.git,457d703e8f89bff657c6c51151ada71ebd09a1c6,ghcr.io/cyclonedx/cdxgen:master,java8,,
+jazzer,https://github.com/CodeIntelligenceTesting/jazzer.git,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb,ghcr.io/cyclonedx/cdxgen:master,java8,,
+plantuml,https://github.com/plantuml/plantuml.git,8eb791f39478778788fd47a9195dc1b2feb3eade,ghcr.io/cyclonedx/cdxgen:master,java8,,
+syncthing,https://github.com/syncthing/syncthing.git,ba6ac2f604eb1cd27764460b687537c5e40aaaf8,ghcr.io/cyclonedx/cdxgen:master,go,,
+restic,https://github.com/restic/restic.git,3786536dc18ef27aedcfa8e4c6953b48353eee79,ghcr.io/cyclonedx/cdxgen:master,go,,
+astro,https://github.com/withastro/astro.git,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63,ghcr.io/cyclonedx/cdxgen-node20:v11,js,,
+funcy,https://github.com/Suor/funcy.git,859056d039adea75c1c3550286437ce0b612fe92,ghcr.io/cyclonedx/cdxgen-python310:v11,python,,
+numpy,https://github.com/numpy/numpy.git,93fdebfcb4bc4cd53c959ccd0117a612d5f13f1a,ghcr.io/cyclonedx/cdxgen-python311:v11,python,,
+requests,https://github.com/psf/requests.git,23540c93cac97c763fe59e843a08fa2825aa80fd,ghcr.io/cyclonedx/cdxgen-python311:v11,python,,
+genforce,https://github.com/genforce/genforce.git,197feee82101b78266521c8470648bbb9b7f31f4,ghcr.io/cyclonedx/cdxgen-python310:v11,python,,
+tinydb,https://github.com/msiemens/tinydb.git,10644a0e07ad180c5b756aba272ee6b0dbd12df8,ghcr.io/cyclonedx/cdxgen-python311:v11,python,,
+github-readme-stats,https://github.com/anuraghazra/github-readme-stats.git,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e,ghcr.io/cyclonedx/cdxgen-node20:v11,js,,
+prettier,https://github.com/prettier/prettier.git,9cf9079f75a30f1088529e0cae6296aeb71205ba,ghcr.io/cyclonedx/cdxgen-node20:v11,js,,