diff --git a/contrib/bulk-generate/README.md b/contrib/bulk-generate/README.md new file mode 100644 index 0000000000..0db32b5dff --- /dev/null +++ b/contrib/bulk-generate/README.md @@ -0,0 +1,16 @@ +# Introduction + +This is a script to generate SBOMs for multiple git repos using cdxgen container images. + +## Usage + +```shell +node index.js +``` + +## Example csv file + +``` +project,link,commit,image,language,cdxgen_vars +astro,https://github.com/withastro/astro.git,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63,ghcr.io/cyclonedx/cdxgen-node20:v11,js,, +``` diff --git a/contrib/bulk-generate/index.js b/contrib/bulk-generate/index.js new file mode 100644 index 0000000000..b380e12898 --- /dev/null +++ b/contrib/bulk-generate/index.js @@ -0,0 +1,186 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { + copyFileSync, + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +let url = import.meta.url; +if (!url.startsWith("file://")) { + url = new URL(`file://${import.meta.url}`).toString(); +} +const dirName = import.meta ? dirname(fileURLToPath(url)) : __dirname; + +const DOCKER_CMD = process.env.DOCKER_CMD || "docker"; + +function gitClone(name, repoUrl, commit, cloneDir) { + const repoDir = join(cloneDir, name); + const gitArgs = ["clone", repoUrl, repoDir]; + console.log(`Cloning Repo ${name} to ${repoDir}`); + let result = spawnSync("git", gitArgs, { + encoding: "utf-8", + shell: false, + cwd: cloneDir, + }); + if (result.status !== 0) { + console.log(result.stderr); + process.exit(1); + } + if (!existsSync(repoDir)) { + console.log(`${repoDir} doesn't exist!`); + process.exit(1); + } + if (commit) { + result = spawnSync("git", ["checkout", commit], { + encoding: "utf-8", + shell: false, + cwd: repoDir, + }); + if (result.status !== 0) { + console.log(`Unable to checkout ${commit}`); + console.log(result.stderr); + process.exit(1); + } + } + if (existsSync(join(repoDir, ".gitmodules"))) { + result = spawnSync("git", ["submodule", "update", "--init"], { + encoding: "utf-8", + shell: false, + cwd: repoDir, + }); + if (result.status !== 0) { + console.log(`Unable to checkout ${commit}`); + console.log(result.stderr); + process.exit(1); + } + } + return repoDir; +} + +function runWithDocker(args, options = {}) { + console.log(`Executing ${DOCKER_CMD} ${args.join(" ")}`); + const result = spawnSync(DOCKER_CMD, args, { + ...options, + encoding: "utf-8", + stdio: "inherit", + stderr: "inherit", + env: process.env, + }); + if (result.status !== 0) { + console.log(result.stdout, result.stderr, result.error); + } + return result; +} + +function readcsv(acsv, outputDir) { + const argsList = []; + if (!existsSync(outputDir)) { + mkdirSync(outputDir, { + recursive: true, + }); + } + const csvLines = readFileSync(acsv, { encoding: "utf-8" }) + .split("\n") + .slice(1); + for (const aline of csvLines) { + const values = aline.split(/[,|]/); + argsList.push({ + project: values[0], + link: values[1], + commit: values[2], + image: values[3], + language: values[4], + cdxgen_vars: values[5], + }); + } + return argsList; +} + +function main(argvs) { + if (argvs.length !== 2) { + console.log("USAGE: node index.js "); + process.exit(1); + } + const tempDir = mkdtempSync(join(tmpdir(), "bulk-generate-")); + const reposList = readcsv(argvs[0], argvs[1]); + for (const repoArgs of reposList) { + if (!repoArgs?.project?.length) { + continue; + } + const repoDir = gitClone( + repoArgs.project, + repoArgs.link, + repoArgs.commit, + tempDir, + ); + const repoOutputDir = join(argvs[1], repoArgs.project, repoArgs.commit); + const envVars = ["-e", "CDXGEN_DEBUG_MODE=debug"]; + if (repoArgs.cdxgen_vars) { + for (const avaluePair of repoArgs.cdxgen_vars(" ")) { + if (avaluePair.includes("=")) { + envVars.push("-e"); + envVars.push(avaluePair); + } + } + } + if (!existsSync(repoOutputDir)) { + mkdirSync(repoOutputDir, { + recursive: true, + }); + } + const bomFile = join( + repoOutputDir, + `bom-${repoArgs.language.replaceAll(" ", "-")}.json`, + ); + const dockerArgs = [ + "run", + "--rm", + "--pull=always", + ...envVars, + "-v", + "/tmp:/tmp", + "-v", + `${repoDir}:/app:rw`, + "-w", + "/app", + "-it", + repoArgs.image, + "-r", + "/app", + "-o", + "/app/bom.json", + ]; + for (const alang of repoArgs.language.split(" ")) { + dockerArgs.push("-t"); + dockerArgs.push(alang); + } + runWithDocker(dockerArgs, { cwd: repoDir }); + if (existsSync(join(repoDir, "bom.json"))) { + copyFileSync(join(repoDir, "bom.json"), bomFile); + } else { + console.log( + join(repoDir, "bom.json"), + "was not found! Check if the image used is valid for this project:", + repoArgs.image, + repoArgs.language, + ); + process.exit(1); + } + } + if (tempDir.startsWith(tmpdir())) { + rmSync(tempDir, { recursive: true, force: true }); + } +} + +main(process.argv.slice(2)); diff --git a/test/diff/container-tests-repos.csv b/test/diff/container-tests-repos.csv new file mode 100644 index 0000000000..ce91789fa7 --- /dev/null +++ b/test/diff/container-tests-repos.csv @@ -0,0 +1,14 @@ +project,link,commit,image,language,cdxgen_vars +java-sec-code,https://github.com/JoyChou93/java-sec-code.git,457d703e8f89bff657c6c51151ada71ebd09a1c6,ghcr.io/cyclonedx/cdxgen:master,java8,, +jazzer,https://github.com/CodeIntelligenceTesting/jazzer.git,3947707d7db7e5cae0c8cfaeb10bdfeb06fc32bb,ghcr.io/cyclonedx/cdxgen:master,java8,, +plantuml,https://github.com/plantuml/plantuml.git,8eb791f39478778788fd47a9195dc1b2feb3eade,ghcr.io/cyclonedx/cdxgen:master,java8,, +syncthing,https://github.com/syncthing/syncthing.git,ba6ac2f604eb1cd27764460b687537c5e40aaaf8,ghcr.io/cyclonedx/cdxgen:master,go,, +restic,https://github.com/restic/restic.git,3786536dc18ef27aedcfa8e4c6953b48353eee79,ghcr.io/cyclonedx/cdxgen:master,go,, +astro,https://github.com/withastro/astro.git,9d6bcdb88fcb9df0c5c70e2b591bcf962ce55f63,ghcr.io/cyclonedx/cdxgen-node20:v11,js,, +funcy,https://github.com/Suor/funcy.git,859056d039adea75c1c3550286437ce0b612fe92,ghcr.io/cyclonedx/cdxgen-python310:v11,python,, +numpy,https://github.com/numpy/numpy.git,93fdebfcb4bc4cd53c959ccd0117a612d5f13f1a,ghcr.io/cyclonedx/cdxgen-python311:v11,python,, +requests,https://github.com/psf/requests.git,23540c93cac97c763fe59e843a08fa2825aa80fd,ghcr.io/cyclonedx/cdxgen-python311:v11,python,, +genforce,https://github.com/genforce/genforce.git,197feee82101b78266521c8470648bbb9b7f31f4,ghcr.io/cyclonedx/cdxgen-python310:v11,python,, +tinydb,https://github.com/msiemens/tinydb.git,10644a0e07ad180c5b756aba272ee6b0dbd12df8,ghcr.io/cyclonedx/cdxgen-python311:v11,python,, +github-readme-stats,https://github.com/anuraghazra/github-readme-stats.git,9a0d9ae2c17e007cbb8e9f32654941e1f0a8268e,ghcr.io/cyclonedx/cdxgen-node20:v11,js,, +prettier,https://github.com/prettier/prettier.git,9cf9079f75a30f1088529e0cae6296aeb71205ba,ghcr.io/cyclonedx/cdxgen-node20:v11,js,, \ No newline at end of file