diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 580719d474..96e7f52f63 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ - + ### Risk assessment diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b0b90b659..1c2a103678 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs. +## 3.30.3 - 10 Sep 2025 + +No user facing changes. + ## 3.30.2 - 09 Sep 2025 - Fixed a bug which could cause language autodetection to fail. [#3084](https://github.com/github/codeql-action/pull/3084) diff --git a/lib/analyze-action-post.js b/lib/analyze-action-post.js index 8c535f5e1a..312e02135d 100644 --- a/lib/analyze-action-post.js +++ b/lib/analyze-action-post.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -117686,7 +117686,7 @@ function withGroup(groupName, f) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -118251,7 +118251,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", diff --git a/lib/analyze-action.js b/lib/analyze-action.js index 59f583235e..ce00934a08 100644 --- a/lib/analyze-action.js +++ b/lib/analyze-action.js @@ -32287,7 +32287,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -90811,7 +90811,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -92814,7 +92814,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", @@ -93707,7 +93706,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, async function runInterpretResultsFor(analysis, language, queries, enableDebugLogging) { logger.info(`Interpreting ${analysis.name} results for ${language}`); let category = automationDetailsId; - if (dbAnalysisConfig.kind === "code-quality" /* CodeQuality */) { + if (analysis.kind === "code-quality" /* CodeQuality */) { category = fixCodeQualityCategory(logger, automationDetailsId); } const sarifFile = path16.join( @@ -95613,6 +95612,10 @@ async function uploadSpecifiedFiles(sarifPaths, checkoutPath, category, features validateUniqueCategory(sarif, uploadTarget.sentinelPrefix); logger.debug(`Serializing SARIF for upload`); const sarifPayload = JSON.stringify(sarif); + const dumpDir = process.env["CODEQL_ACTION_SARIF_DUMP_DIR" /* SARIF_DUMP_DIR */]; + if (dumpDir) { + dumpSarifFile(sarifPayload, dumpDir, logger, uploadTarget); + } logger.debug(`Compressing serialized SARIF`); const zippedSarif = import_zlib.default.gzipSync(sarifPayload).toString("base64"); const checkoutURI = url.pathToFileURL(checkoutPath).href; @@ -95651,6 +95654,21 @@ async function uploadSpecifiedFiles(sarifPaths, checkoutPath, category, features sarifID }; } +function dumpSarifFile(sarifPayload, outputDir, logger, uploadTarget) { + if (!fs18.existsSync(outputDir)) { + fs18.mkdirSync(outputDir, { recursive: true }); + } else if (!fs18.lstatSync(outputDir).isDirectory()) { + throw new ConfigurationError( + `The path specified by the ${"CODEQL_ACTION_SARIF_DUMP_DIR" /* SARIF_DUMP_DIR */} environment variable exists and is not a directory: ${outputDir}` + ); + } + const outputFile = path18.resolve( + outputDir, + `upload${uploadTarget.sarifExtension}` + ); + logger.info(`Dumping processed SARIF file to ${outputFile}`); + fs18.writeFileSync(outputFile, sarifPayload); +} var STATUS_CHECK_FREQUENCY_MILLISECONDS = 5 * 1e3; var STATUS_CHECK_TIMEOUT_MILLISECONDS = 2 * 60 * 1e3; async function waitForProcessing(repositoryNwo, sarifID, logger, options = { diff --git a/lib/autobuild-action.js b/lib/autobuild-action.js index ba16dbdf04..38c842dce4 100644 --- a/lib/autobuild-action.js +++ b/lib/autobuild-action.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -78426,7 +78426,7 @@ function getActionsLogger() { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -79286,7 +79286,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", diff --git a/lib/init-action-post.js b/lib/init-action-post.js index da3dd17080..1cb50a92d3 100644 --- a/lib/init-action-post.js +++ b/lib/init-action-post.js @@ -32287,7 +32287,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -129016,7 +129016,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -130724,7 +130724,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", @@ -133050,6 +133049,10 @@ async function uploadSpecifiedFiles(sarifPaths, checkoutPath, category, features validateUniqueCategory(sarif, uploadTarget.sentinelPrefix); logger.debug(`Serializing SARIF for upload`); const sarifPayload = JSON.stringify(sarif); + const dumpDir = process.env["CODEQL_ACTION_SARIF_DUMP_DIR" /* SARIF_DUMP_DIR */]; + if (dumpDir) { + dumpSarifFile(sarifPayload, dumpDir, logger, uploadTarget); + } logger.debug(`Compressing serialized SARIF`); const zippedSarif = import_zlib.default.gzipSync(sarifPayload).toString("base64"); const checkoutURI = url.pathToFileURL(checkoutPath).href; @@ -133088,6 +133091,21 @@ async function uploadSpecifiedFiles(sarifPaths, checkoutPath, category, features sarifID }; } +function dumpSarifFile(sarifPayload, outputDir, logger, uploadTarget) { + if (!fs17.existsSync(outputDir)) { + fs17.mkdirSync(outputDir, { recursive: true }); + } else if (!fs17.lstatSync(outputDir).isDirectory()) { + throw new ConfigurationError( + `The path specified by the ${"CODEQL_ACTION_SARIF_DUMP_DIR" /* SARIF_DUMP_DIR */} environment variable exists and is not a directory: ${outputDir}` + ); + } + const outputFile = path17.resolve( + outputDir, + `upload${uploadTarget.sarifExtension}` + ); + logger.info(`Dumping processed SARIF file to ${outputFile}`); + fs17.writeFileSync(outputFile, sarifPayload); +} var STATUS_CHECK_FREQUENCY_MILLISECONDS = 5 * 1e3; var STATUS_CHECK_TIMEOUT_MILLISECONDS = 2 * 60 * 1e3; async function waitForProcessing(repositoryNwo, sarifID, logger, options = { diff --git a/lib/init-action.js b/lib/init-action.js index 895f5dae2d..804f24af12 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -32287,7 +32287,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -86417,7 +86417,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -89408,7 +89408,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", diff --git a/lib/resolve-environment-action.js b/lib/resolve-environment-action.js index ce24fc14f2..edc2cc242a 100644 --- a/lib/resolve-environment-action.js +++ b/lib/resolve-environment-action.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -78419,7 +78419,7 @@ function getActionsLogger() { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -78986,7 +78986,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", diff --git a/lib/start-proxy-action-post.js b/lib/start-proxy-action-post.js index 4484fd1c70..f4ae973df6 100644 --- a/lib/start-proxy-action-post.js +++ b/lib/start-proxy-action-post.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -117158,7 +117158,7 @@ function getActionsLogger() { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; // src/tools-features.ts diff --git a/lib/start-proxy-action.js b/lib/start-proxy-action.js index 3f73083708..ad4e5c882c 100644 --- a/lib/start-proxy-action.js +++ b/lib/start-proxy-action.js @@ -44966,7 +44966,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { diff --git a/lib/upload-lib.js b/lib/upload-lib.js index 01a7cec67b..d816e145f8 100644 --- a/lib/upload-lib.js +++ b/lib/upload-lib.js @@ -33584,7 +33584,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -89114,7 +89114,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -90552,7 +90552,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", @@ -92422,6 +92421,10 @@ async function uploadSpecifiedFiles(sarifPaths, checkoutPath, category, features validateUniqueCategory(sarif, uploadTarget.sentinelPrefix); logger.debug(`Serializing SARIF for upload`); const sarifPayload = JSON.stringify(sarif); + const dumpDir = process.env["CODEQL_ACTION_SARIF_DUMP_DIR" /* SARIF_DUMP_DIR */]; + if (dumpDir) { + dumpSarifFile(sarifPayload, dumpDir, logger, uploadTarget); + } logger.debug(`Compressing serialized SARIF`); const zippedSarif = import_zlib.default.gzipSync(sarifPayload).toString("base64"); const checkoutURI = url.pathToFileURL(checkoutPath).href; @@ -92460,6 +92463,21 @@ async function uploadSpecifiedFiles(sarifPaths, checkoutPath, category, features sarifID }; } +function dumpSarifFile(sarifPayload, outputDir, logger, uploadTarget) { + if (!fs13.existsSync(outputDir)) { + fs13.mkdirSync(outputDir, { recursive: true }); + } else if (!fs13.lstatSync(outputDir).isDirectory()) { + throw new ConfigurationError( + `The path specified by the ${"CODEQL_ACTION_SARIF_DUMP_DIR" /* SARIF_DUMP_DIR */} environment variable exists and is not a directory: ${outputDir}` + ); + } + const outputFile = path14.resolve( + outputDir, + `upload${uploadTarget.sarifExtension}` + ); + logger.info(`Dumping processed SARIF file to ${outputFile}`); + fs13.writeFileSync(outputFile, sarifPayload); +} var STATUS_CHECK_FREQUENCY_MILLISECONDS = 5 * 1e3; var STATUS_CHECK_TIMEOUT_MILLISECONDS = 2 * 60 * 1e3; async function waitForProcessing(repositoryNwo, sarifID, logger, options = { diff --git a/lib/upload-sarif-action-post.js b/lib/upload-sarif-action-post.js index 726d9f8b5c..76753057d2 100644 --- a/lib/upload-sarif-action-post.js +++ b/lib/upload-sarif-action-post.js @@ -26438,7 +26438,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -117319,7 +117319,7 @@ function withGroup(groupName, f) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; // src/tools-features.ts diff --git a/lib/upload-sarif-action.js b/lib/upload-sarif-action.js index de591db0da..3335f82c2c 100644 --- a/lib/upload-sarif-action.js +++ b/lib/upload-sarif-action.js @@ -32287,7 +32287,7 @@ var require_package = __commonJS({ "package.json"(exports2, module2) { module2.exports = { name: "codeql", - version: "3.30.2", + version: "3.30.3", private: true, description: "CodeQL action", scripts: { @@ -89108,7 +89108,7 @@ function formatDuration(durationMs) { // src/overlay-database-utils.ts var CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; -var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6e3; +var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15e3; var OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1e6; async function writeBaseDatabaseOidsFile(config, sourceRoot) { const gitFileOids = await getFileOidsUnderPath(sourceRoot); @@ -91253,7 +91253,6 @@ ${output}` "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", @@ -93123,6 +93122,10 @@ async function uploadSpecifiedFiles(sarifPaths, checkoutPath, category, features validateUniqueCategory(sarif, uploadTarget.sentinelPrefix); logger.debug(`Serializing SARIF for upload`); const sarifPayload = JSON.stringify(sarif); + const dumpDir = process.env["CODEQL_ACTION_SARIF_DUMP_DIR" /* SARIF_DUMP_DIR */]; + if (dumpDir) { + dumpSarifFile(sarifPayload, dumpDir, logger, uploadTarget); + } logger.debug(`Compressing serialized SARIF`); const zippedSarif = import_zlib.default.gzipSync(sarifPayload).toString("base64"); const checkoutURI = url.pathToFileURL(checkoutPath).href; @@ -93161,6 +93164,21 @@ async function uploadSpecifiedFiles(sarifPaths, checkoutPath, category, features sarifID }; } +function dumpSarifFile(sarifPayload, outputDir, logger, uploadTarget) { + if (!fs14.existsSync(outputDir)) { + fs14.mkdirSync(outputDir, { recursive: true }); + } else if (!fs14.lstatSync(outputDir).isDirectory()) { + throw new ConfigurationError( + `The path specified by the ${"CODEQL_ACTION_SARIF_DUMP_DIR" /* SARIF_DUMP_DIR */} environment variable exists and is not a directory: ${outputDir}` + ); + } + const outputFile = path15.resolve( + outputDir, + `upload${uploadTarget.sarifExtension}` + ); + logger.info(`Dumping processed SARIF file to ${outputFile}`); + fs14.writeFileSync(outputFile, sarifPayload); +} var STATUS_CHECK_FREQUENCY_MILLISECONDS = 5 * 1e3; var STATUS_CHECK_TIMEOUT_MILLISECONDS = 2 * 60 * 1e3; async function waitForProcessing(repositoryNwo, sarifID, logger, options = { diff --git a/package-lock.json b/package-lock.json index f55f7f9a71..12ab1133b1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "codeql", - "version": "3.30.2", + "version": "3.30.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "codeql", - "version": "3.30.2", + "version": "3.30.3", "license": "MIT", "dependencies": { "@actions/artifact": "^2.3.1", diff --git a/package.json b/package.json index 2796020742..dacadc7321 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "codeql", - "version": "3.30.2", + "version": "3.30.3", "private": true, "description": "CodeQL action", "scripts": { diff --git a/src/analyze.ts b/src/analyze.ts index 153b00a1f4..b22e9c23da 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -780,7 +780,7 @@ export async function runQueries( // If this is a Code Quality analysis, correct the category to one // accepted by the Code Quality backend. let category = automationDetailsId; - if (dbAnalysisConfig.kind === analyses.AnalysisKind.CodeQuality) { + if (analysis.kind === analyses.AnalysisKind.CodeQuality) { category = fixCodeQualityCategory(logger, automationDetailsId); } diff --git a/src/codeql.test.ts b/src/codeql.test.ts index ece6ab878a..36775f6530 100644 --- a/src/codeql.test.ts +++ b/src/codeql.test.ts @@ -953,7 +953,7 @@ test("runTool recognizes fatal internal errors", async (t) => { await codeqlObject.databaseRunQueries(stubConfig.dbLocation, []), { instanceOf: CliError, - message: `Encountered a fatal error while running "codeql-for-testing database run-queries --intra-layer-parallelism --min-disk-free=1024 -v". Exit code was 1 and error was: Oops! A fatal internal error occurred. Details: + message: `Encountered a fatal error while running "codeql-for-testing database run-queries --min-disk-free=1024 -v". Exit code was 1 and error was: Oops! A fatal internal error occurred. Details: com.semmle.util.exception.CatastrophicError: An error occurred while evaluating ControlFlowGraph::ControlFlow::Root.isRootOf/1#dispred#f610e6ed/2@86282cc8 Severe disk cache trouble (corruption or out of space) at /home/runner/work/_temp/codeql_databases/go/db-go/default/cache/pages/28/33.pack: Failed to write item to disk. See the logs for more details.`, }, diff --git a/src/codeql.ts b/src/codeql.ts index 567eb80875..35c3cc625f 100644 --- a/src/codeql.ts +++ b/src/codeql.ts @@ -789,7 +789,6 @@ export async function getCodeQLForCmd( "run-queries", ...flags, databasePath, - "--intra-layer-parallelism", "--min-disk-free=1024", // Try to leave at least 1GB free "-v", ...queries, diff --git a/src/environment.ts b/src/environment.ts index f25e7270da..e78c367244 100644 --- a/src/environment.ts +++ b/src/environment.ts @@ -119,4 +119,10 @@ export enum EnvVar { * Whether to enable experimental extractors for CodeQL. */ EXPERIMENTAL_FEATURES = "CODEQL_ENABLE_EXPERIMENTAL_FEATURES", + + /** + * Whether and where to dump the processed SARIF file that would be uploaded, regardless of + * whether the upload is disabled. This is intended for testing and debugging purposes. + */ + SARIF_DUMP_DIR = "CODEQL_ACTION_SARIF_DUMP_DIR", } diff --git a/src/overlay-database-utils.ts b/src/overlay-database-utils.ts index c4f6ae7c00..ea43abcaa3 100644 --- a/src/overlay-database-utils.ts +++ b/src/overlay-database-utils.ts @@ -22,18 +22,23 @@ export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.22.3"; /** * The maximum (uncompressed) size of the overlay base database that we will - * upload. Actions Cache has an overall capacity of 10 GB, and the Actions Cache - * client library uses zstd compression. + * upload. By default, the Actions Cache has an overall capacity of 10 GB, and + * the Actions Cache client library uses zstd compression. * * Ideally we would apply a size limit to the compressed overlay-base database, * but we cannot do so because compression is handled transparently by the * Actions Cache client library. Instead we place a limit on the uncompressed * size of the overlay-base database. * - * Assuming 2.5:1 compression ratio, the 6 GB limit on uncompressed data would - * translate to a limit of around 2.4 GB after compression. + * Assuming 2.5:1 compression ratio, the 15 GB limit on uncompressed data would + * translate to a limit of around 6 GB after compression. This is a high limit + * compared to the default 10GB Actions Cache capacity, but enforcement of Actions + * Cache quotas is not immediate. + * + * TODO: revisit this limit before removing the restriction for overlay analysis + * to the `github` and `dsp-testing` orgs. */ -const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 6000; +const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB = 15000; const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES = OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_MB * 1_000_000; diff --git a/src/upload-lib.ts b/src/upload-lib.ts index 8939e16944..650e7a803a 100644 --- a/src/upload-lib.ts +++ b/src/upload-lib.ts @@ -696,6 +696,12 @@ export async function uploadSpecifiedFiles( validateUniqueCategory(sarif, uploadTarget.sentinelPrefix); logger.debug(`Serializing SARIF for upload`); const sarifPayload = JSON.stringify(sarif); + + const dumpDir = process.env[EnvVar.SARIF_DUMP_DIR]; + if (dumpDir) { + dumpSarifFile(sarifPayload, dumpDir, logger, uploadTarget); + } + logger.debug(`Compressing serialized SARIF`); const zippedSarif = zlib.gzipSync(sarifPayload).toString("base64"); const checkoutURI = url.pathToFileURL(checkoutPath).href; @@ -742,6 +748,30 @@ export async function uploadSpecifiedFiles( }; } +/** + * Dumps the given processed SARIF file contents to `outputDir`. + */ +function dumpSarifFile( + sarifPayload: string, + outputDir: string, + logger: Logger, + uploadTarget: analyses.AnalysisConfig, +) { + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } else if (!fs.lstatSync(outputDir).isDirectory()) { + throw new ConfigurationError( + `The path specified by the ${EnvVar.SARIF_DUMP_DIR} environment variable exists and is not a directory: ${outputDir}`, + ); + } + const outputFile = path.resolve( + outputDir, + `upload${uploadTarget.sarifExtension}`, + ); + logger.info(`Dumping processed SARIF file to ${outputFile}`); + fs.writeFileSync(outputFile, sarifPayload); +} + const STATUS_CHECK_FREQUENCY_MILLISECONDS = 5 * 1000; const STATUS_CHECK_TIMEOUT_MILLISECONDS = 2 * 60 * 1000;