diff --git a/scripts/build/replace-build-env.js b/scripts/build/replace-build-env.js index 0f38785a4f..6b62cb3839 100644 --- a/scripts/build/replace-build-env.js +++ b/scripts/build/replace-build-env.js @@ -1,5 +1,6 @@ const glob = require('glob') -const { printLog, runMain, modifyFile } = require('../lib/utils') +const { printLog, runMain } = require('../lib/execution-utils') +const { modifyFile } = require('../lib/files-utils') const buildEnv = require('../lib/build-env') /** diff --git a/scripts/check-licenses.js b/scripts/check-licenses.js index c649ba5609..e16a3fec0e 100644 --- a/scripts/check-licenses.js +++ b/scripts/check-licenses.js @@ -3,7 +3,8 @@ const fs = require('fs') const path = require('path') const readline = require('readline') -const { printLog, printError, runMain, findBrowserSdkPackageJsonFiles } = require('./lib/utils') +const { printLog, printError, runMain } = require('./lib/execution-utils') +const { findBrowserSdkPackageJsonFiles } = require('./lib/files-utils') const LICENSE_FILE = 'LICENSE-3rdparty.csv' diff --git a/scripts/deploy/deploy.js b/scripts/deploy/deploy.js index 43a3971b1e..0c96d738dc 100644 --- a/scripts/deploy/deploy.js +++ b/scripts/deploy/deploy.js @@ -1,6 +1,7 @@ 'use strict' -const { printLog, command, runMain } = require('../lib/utils') +const { printLog, runMain } = require('../lib/execution-utils') +const { command } = require('../lib/command') const { buildRootUploadPath, buildDatacenterUploadPath, diff --git a/scripts/deploy/upload-source-maps.js b/scripts/deploy/upload-source-maps.js index c22dfb4b6b..b95bca14f2 100644 --- a/scripts/deploy/upload-source-maps.js +++ b/scripts/deploy/upload-source-maps.js @@ -1,8 +1,10 @@ 'use strict' const path = require('path') -const { getSecretKey, command, printLog, runMain } = require('../lib/utils') +const { printLog, runMain } = require('../lib/execution-utils') +const { command } = require('../lib/command') const { SDK_VERSION } = require('../lib/build-env') +const { getTelemetryOrgApiKey } = require('../lib/secrets') const { buildRootUploadPath, buildDatacenterUploadPath, @@ -66,9 +68,6 @@ function renameFilesWithVersionSuffix(packageName, bundleFolder) { function uploadSourceMaps(packageName, service, prefix, bundleFolder, sites) { for (const site of sites) { - const normalizedSite = site.replaceAll('.', '-') - const apiKey = getSecretKey(`ci.browser-sdk.source-maps.${normalizedSite}.ci_api_key`) - printLog(`Uploading ${packageName} source maps with prefix ${prefix} for ${site}...`) command` @@ -80,7 +79,7 @@ function uploadSourceMaps(packageName, service, prefix, bundleFolder, sites) { --repository-url https://www.github.com/datadog/browser-sdk ` .withEnvironment({ - DATADOG_API_KEY: apiKey, + DATADOG_API_KEY: getTelemetryOrgApiKey(site), DATADOG_SITE: site, }) .run() diff --git a/scripts/dev-server.js b/scripts/dev-server.js index b70eb1b173..68d33b8045 100644 --- a/scripts/dev-server.js +++ b/scripts/dev-server.js @@ -6,7 +6,7 @@ const webpack = require('webpack') const logsConfig = require('../packages/logs/webpack.config') const rumSlimConfig = require('../packages/rum-slim/webpack.config') const rumConfig = require('../packages/rum/webpack.config') -const { printLog } = require('./lib/utils') +const { printLog } = require('./lib/execution-utils') const port = 8080 const app = express() diff --git a/scripts/generate-schema-types.js b/scripts/generate-schema-types.js index e1abc99e10..a21e909bff 100644 --- a/scripts/generate-schema-types.js +++ b/scripts/generate-schema-types.js @@ -2,7 +2,7 @@ const fs = require('fs') const path = require('path') const { compileFromFile } = require('json-schema-to-typescript') const prettier = require('prettier') -const { printLog, runMain } = require('./lib/utils') +const { printLog, runMain } = require('./lib/execution-utils') const schemasDirectoryPath = path.join(__dirname, '../rum-events-format/schemas') const prettierConfigPath = path.join(__dirname, '../.prettierrc.yml') diff --git a/scripts/lib/utils.js b/scripts/lib/command.js similarity index 53% rename from scripts/lib/utils.js rename to scripts/lib/command.js index 58b9a35008..70e0c7ef03 100644 --- a/scripts/lib/utils.js +++ b/scripts/lib/command.js @@ -1,80 +1,6 @@ -const path = require('path') -const os = require('os') -const fsPromises = require('fs/promises') -const fs = require('fs') const childProcess = require('child_process') -const spawn = require('child_process').spawn -// node-fetch v3.x only support ESM syntax. -// Todo: Remove node-fetch when node v18 LTS is released with fetch out of the box -const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args)) -const CI_FILE = '.gitlab-ci.yml' - -function getSecretKey(name) { - return command` - aws ssm get-parameter --region=us-east-1 --with-decryption --query=Parameter.Value --out=text --name=${name} - ` - .run() - .trim() -} - -function initGitConfig(repository) { - const githubDeployKey = getSecretKey('ci.browser-sdk.github_deploy_key') - const homedir = os.homedir() - - // ssh-add expects a new line at the end of the PEM-formatted private key - // https://stackoverflow.com/a/59595773 - command`ssh-add -`.withInput(`${githubDeployKey}\n`).run() - command`mkdir -p ${homedir}/.ssh`.run() - command`chmod 700 ${homedir}/.ssh`.run() - const sshHost = command`ssh-keyscan -H github.com`.run() - fs.appendFileSync(`${homedir}/.ssh/known_hosts`, sshHost) - command`git config user.email ci.browser-sdk@datadoghq.com`.run() - command`git config user.name ci.browser-sdk`.run() - command`git remote set-url origin ${repository}`.run() -} - -function readCiFileVariable(variableName) { - const regexp = new RegExp(`${variableName}: (.*)`) - const ciFileContent = fs.readFileSync(CI_FILE, { encoding: 'utf-8' }) - return regexp.exec(ciFileContent)?.[1] -} - -async function replaceCiFileVariable(variableName, value) { - await modifyFile(CI_FILE, (content) => - content.replace(new RegExp(`${variableName}: .*`), `${variableName}: ${value}`) - ) -} - -/** - * @param filePath {string} - * @param modifier {(content: string) => string} - */ -async function modifyFile(filePath, modifier) { - const content = await fsPromises.readFile(filePath, { encoding: 'utf-8' }) - const modifiedContent = modifier(content) - if (content !== modifiedContent) { - await fsPromises.writeFile(filePath, modifiedContent) - return true - } - return false -} - -/** - * Helper to run executables asynchronously, in a shell. This function does not prevent Shell - * injections[0], so please use carefully. Only use it to run commands with trusted arguments. - * Prefer the `command` helper for most use cases. - * - * [0]: https://matklad.github.io/2021/07/30/shell-injection.html - */ -async function spawnCommand(command, args) { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { stdio: 'inherit', shell: true }) - child.on('error', reject) - child.on('close', resolve) - child.on('exit', resolve) - }) -} +const { printError } = require('./execution-utils') /** * Helper to run executables. This has been introduced to work around Shell injections[0] while @@ -205,66 +131,6 @@ function parseCommandTemplateArguments(templateStrings, ...templateVariables) { return parsedArguments } -function runMain(mainFunction) { - Promise.resolve() - // The main function can be either synchronous or asynchronous, so let's wrap it in an async - // callback that will catch both thrown errors and rejected promises - .then(() => mainFunction()) - .catch((error) => { - printError('\nScript exited with error:') - printError(error) - process.exit(1) - }) -} - -const resetColor = '\x1b[0m' - -function printError(...params) { - const redColor = '\x1b[31;1m' - console.log(redColor, ...params, resetColor) -} - -function printLog(...params) { - const greenColor = '\x1b[32;1m' - console.log(greenColor, ...params, resetColor) -} - -async function fetchWrapper(url, options) { - const response = await fetch(url, options) - if (!response.ok) { - throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`) - } - - return response.text() -} - -function findBrowserSdkPackageJsonFiles() { - const manifestPaths = command`git ls-files -- package.json */package.json`.run() - return manifestPaths - .trim() - .split('\n') - .map((manifestPath) => { - const absoluteManifestPath = path.join(__dirname, '../..', manifestPath) - return { - relativePath: manifestPath, - path: absoluteManifestPath, - content: require(absoluteManifestPath), - } - }) -} - module.exports = { - CI_FILE, - getSecretKey, - initGitConfig, command, - spawnCommand, - printError, - printLog, - runMain, - readCiFileVariable, - replaceCiFileVariable, - fetch: fetchWrapper, - modifyFile, - findBrowserSdkPackageJsonFiles, } diff --git a/scripts/lib/execution-utils.js b/scripts/lib/execution-utils.js new file mode 100644 index 0000000000..c476ed49ea --- /dev/null +++ b/scripts/lib/execution-utils.js @@ -0,0 +1,61 @@ +const spawn = require('child_process').spawn +// node-fetch v3.x only support ESM syntax. +// Todo: Remove node-fetch when node v18 LTS is released with fetch out of the box +const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args)) + +/** + * Helper to run executables asynchronously, in a shell. This function does not prevent Shell + * injections[0], so please use carefully. Only use it to run commands with trusted arguments. + * Prefer the `command` helper for most use cases. + * + * [0]: https://matklad.github.io/2021/07/30/shell-injection.html + */ +async function spawnCommand(command, args) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: 'inherit', shell: true }) + child.on('error', reject) + child.on('close', resolve) + child.on('exit', resolve) + }) +} + +function runMain(mainFunction) { + Promise.resolve() + // The main function can be either synchronous or asynchronous, so let's wrap it in an async + // callback that will catch both thrown errors and rejected promises + .then(() => mainFunction()) + .catch((error) => { + printError('\nScript exited with error:') + printError(error) + process.exit(1) + }) +} + +const resetColor = '\x1b[0m' + +function printError(...params) { + const redColor = '\x1b[31;1m' + console.log(redColor, ...params, resetColor) +} + +function printLog(...params) { + const greenColor = '\x1b[32;1m' + console.log(greenColor, ...params, resetColor) +} + +async function fetchWrapper(url, options) { + const response = await fetch(url, options) + if (!response.ok) { + throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`) + } + + return response.text() +} + +module.exports = { + spawnCommand, + printError, + printLog, + runMain, + fetch: fetchWrapper, +} diff --git a/scripts/lib/files-utils.js b/scripts/lib/files-utils.js new file mode 100644 index 0000000000..6aa5dca42d --- /dev/null +++ b/scripts/lib/files-utils.js @@ -0,0 +1,56 @@ +const fs = require('fs') +const path = require('path') +const fsPromises = require('fs/promises') + +const { command } = require('./command') + +const CI_FILE = '.gitlab-ci.yml' + +function readCiFileVariable(variableName) { + const regexp = new RegExp(`${variableName}: (.*)`) + const ciFileContent = fs.readFileSync(CI_FILE, { encoding: 'utf-8' }) + return regexp.exec(ciFileContent)?.[1] +} + +async function replaceCiFileVariable(variableName, value) { + await modifyFile(CI_FILE, (content) => + content.replace(new RegExp(`${variableName}: .*`), `${variableName}: ${value}`) + ) +} + +/** + * @param filePath {string} + * @param modifier {(content: string) => string} + */ +async function modifyFile(filePath, modifier) { + const content = await fsPromises.readFile(filePath, { encoding: 'utf-8' }) + const modifiedContent = modifier(content) + if (content !== modifiedContent) { + await fsPromises.writeFile(filePath, modifiedContent) + return true + } + return false +} + +function findBrowserSdkPackageJsonFiles() { + const manifestPaths = command`git ls-files -- package.json */package.json`.run() + return manifestPaths + .trim() + .split('\n') + .map((manifestPath) => { + const absoluteManifestPath = path.join(__dirname, '../..', manifestPath) + return { + relativePath: manifestPath, + path: absoluteManifestPath, + content: require(absoluteManifestPath), + } + }) +} + +module.exports = { + CI_FILE, + readCiFileVariable, + replaceCiFileVariable, + modifyFile, + findBrowserSdkPackageJsonFiles, +} diff --git a/scripts/lib/git-utils.js b/scripts/lib/git-utils.js new file mode 100644 index 0000000000..0c9090ccd6 --- /dev/null +++ b/scripts/lib/git-utils.js @@ -0,0 +1,24 @@ +const os = require('os') +const fs = require('fs') + +const { command } = require('../lib/command') +const { getGithubDeployKey } = require('./secrets') + +function initGitConfig(repository) { + const homedir = os.homedir() + + // ssh-add expects a new line at the end of the PEM-formatted private key + // https://stackoverflow.com/a/59595773 + command`ssh-add -`.withInput(`${getGithubDeployKey()}\n`).run() + command`mkdir -p ${homedir}/.ssh`.run() + command`chmod 700 ${homedir}/.ssh`.run() + const sshHost = command`ssh-keyscan -H github.com`.run() + fs.appendFileSync(`${homedir}/.ssh/known_hosts`, sshHost) + command`git config user.email ci.browser-sdk@datadoghq.com`.run() + command`git config user.name ci.browser-sdk`.run() + command`git remote set-url origin ${repository}`.run() +} + +module.exports = { + initGitConfig, +} diff --git a/scripts/lib/secrets.js b/scripts/lib/secrets.js new file mode 100644 index 0000000000..a88f1f64ef --- /dev/null +++ b/scripts/lib/secrets.js @@ -0,0 +1,33 @@ +const { command } = require('../lib/command') + +function getGithubDeployKey() { + return getSecretKey('ci.browser-sdk.github_deploy_key') +} + +function getGithubAccessToken() { + return getSecretKey('ci.browser-sdk.github_access_token') +} + +function getOrg2ApiKey() { + return getSecretKey('ci.browser-sdk.datadog_ci_api_key') +} + +function getTelemetryOrgApiKey(site) { + const normalizedSite = site.replaceAll('.', '-') + return getSecretKey(`ci.browser-sdk.source-maps.${normalizedSite}.ci_api_key`) +} + +function getSecretKey(name) { + return command` + aws ssm get-parameter --region=us-east-1 --with-decryption --query=Parameter.Value --out=text --name=${name} + ` + .run() + .trim() +} + +module.exports = { + getGithubDeployKey, + getGithubAccessToken, + getOrg2ApiKey, + getTelemetryOrgApiKey, +} diff --git a/scripts/release/check-release.js b/scripts/release/check-release.js index 8b26f74d9a..531f8ffe1f 100644 --- a/scripts/release/check-release.js +++ b/scripts/release/check-release.js @@ -3,7 +3,9 @@ const fs = require('fs') const path = require('path') const { version: releaseVersion } = require('../../lerna.json') -const { findBrowserSdkPackageJsonFiles, printLog, runMain, command } = require('../lib/utils') +const { printLog, runMain } = require('../lib/execution-utils') +const { command } = require('../lib/command') +const { findBrowserSdkPackageJsonFiles } = require('../lib/files-utils') runMain(() => { checkGitTag() diff --git a/scripts/release/generate-changelog.js b/scripts/release/generate-changelog.js index 809402f7f7..87fd1f3e02 100644 --- a/scripts/release/generate-changelog.js +++ b/scripts/release/generate-changelog.js @@ -6,7 +6,9 @@ const readFile = util.promisify(require('fs').readFile) const emojiNameMap = require('emoji-name-map') const lernaConfig = require('../../lerna.json') -const { command, spawnCommand, printError, runMain, modifyFile } = require('../lib/utils') +const { spawnCommand, printError, runMain } = require('../lib/execution-utils') +const { command } = require('../lib/command') +const { modifyFile } = require('../lib/files-utils') const CHANGELOG_FILE = 'CHANGELOG.md' const CONTRIBUTING_FILE = 'CONTRIBUTING.md' @@ -78,12 +80,11 @@ function getChangesList() { .filter(isNotMaintenanceEntry) .join('\n') - const changesWithPullRequestLinks = allowedChanges.replace( + // changes with pull request links + return allowedChanges.replace( /\(#(\d+)\)/gm, (_, id) => `([#${id}](https://github.com/DataDog/browser-sdk/pull/${id}))` ) - - return changesWithPullRequestLinks } function isNotVersionEntry(line) { diff --git a/scripts/release/update-peer-dependency-versions.js b/scripts/release/update-peer-dependency-versions.js index b8b7d32303..a3c5c98f8f 100644 --- a/scripts/release/update-peer-dependency-versions.js +++ b/scripts/release/update-peer-dependency-versions.js @@ -1,5 +1,6 @@ const lernaConfig = require('../../lerna.json') -const { runMain, modifyFile } = require('../lib/utils') +const { runMain } = require('../lib/execution-utils') +const { modifyFile } = require('../lib/files-utils') // This script updates the peer dependency versions between rum and logs packages to match the new // version during a release. diff --git a/scripts/staging-ci/check-squash-into-staging.js b/scripts/staging-ci/check-squash-into-staging.js index 5e5422ca9a..191683457c 100644 --- a/scripts/staging-ci/check-squash-into-staging.js +++ b/scripts/staging-ci/check-squash-into-staging.js @@ -1,6 +1,8 @@ 'use strict' -const { initGitConfig, command, printLog, printError, runMain } = require('../lib/utils') +const { printLog, printError, runMain } = require('../lib/execution-utils') +const { command } = require('../lib/command') +const { initGitConfig } = require('../lib/git-utils') const REPOSITORY = process.env.GIT_REPOSITORY const CI_COMMIT_SHA = process.env.CI_COMMIT_SHA diff --git a/scripts/staging-ci/check-staging-merge.js b/scripts/staging-ci/check-staging-merge.js index 28ffd84fc9..6807555224 100644 --- a/scripts/staging-ci/check-staging-merge.js +++ b/scripts/staging-ci/check-staging-merge.js @@ -1,6 +1,8 @@ 'use strict' -const { initGitConfig, command, printLog, printError, runMain } = require('../lib/utils') +const { printLog, printError, runMain } = require('../lib/execution-utils') +const { command } = require('../lib/command') +const { initGitConfig } = require('../lib/git-utils') const REPOSITORY = process.env.GIT_REPOSITORY const CI_COMMIT_SHA = process.env.CI_COMMIT_SHA diff --git a/scripts/staging-ci/merge-into-staging.js b/scripts/staging-ci/merge-into-staging.js index e31d35f2e1..8bab277fcc 100644 --- a/scripts/staging-ci/merge-into-staging.js +++ b/scripts/staging-ci/merge-into-staging.js @@ -1,6 +1,8 @@ 'use strict' -const { initGitConfig, command, printLog, printError, runMain } = require('../lib/utils') +const { printLog, printError, runMain } = require('../lib/execution-utils') +const { command } = require('../lib/command') +const { initGitConfig } = require('../lib/git-utils') const REPOSITORY = process.env.GIT_REPOSITORY const CURRENT_STAGING_BRANCH = process.env.CURRENT_STAGING diff --git a/scripts/staging-ci/staging-reset.js b/scripts/staging-ci/staging-reset.js index 82ae7403cb..da1d8dce94 100755 --- a/scripts/staging-ci/staging-reset.js +++ b/scripts/staging-ci/staging-reset.js @@ -1,15 +1,10 @@ 'use strict' const fs = require('fs') -const { - CI_FILE, - initGitConfig, - command, - printLog, - runMain, - replaceCiFileVariable, - readCiFileVariable, -} = require('../lib/utils') +const { printLog, runMain } = require('../lib/execution-utils') +const { command } = require('../lib/command') +const { CI_FILE, replaceCiFileVariable, readCiFileVariable } = require('../lib/files-utils') +const { initGitConfig } = require('../lib/git-utils') const REPOSITORY = process.env.GIT_REPOSITORY const MAIN_BRANCH = process.env.MAIN_BRANCH diff --git a/scripts/test/bs-wrapper.js b/scripts/test/bs-wrapper.js index 62435b5595..a4de88966e 100644 --- a/scripts/test/bs-wrapper.js +++ b/scripts/test/bs-wrapper.js @@ -14,7 +14,8 @@ // after killing it. There might be a better way of prematurely aborting the test command if we need // to in the future. -const { spawnCommand, printLog, runMain, command, fetch } = require('../lib/utils') +const { spawnCommand, printLog, runMain, fetch } = require('../lib/execution-utils') +const { command } = require('../lib/command') const AVAILABILITY_CHECK_DELAY = 30_000 const BS_USERNAME = process.env.BS_USERNAME diff --git a/scripts/test/bump-chrome-driver-version.js b/scripts/test/bump-chrome-driver-version.js index 2684f5bda9..65a2ad185b 100644 --- a/scripts/test/bump-chrome-driver-version.js +++ b/scripts/test/bump-chrome-driver-version.js @@ -1,17 +1,11 @@ 'use strict' const fs = require('fs') -const { - printLog, - printError, - runMain, - command, - replaceCiFileVariable, - initGitConfig, - getSecretKey, - fetch, - CI_FILE, -} = require('../lib/utils') +const { printLog, printError, runMain, fetch } = require('../lib/execution-utils') +const { command } = require('../lib/command') +const { CI_FILE, replaceCiFileVariable } = require('../lib/files-utils') +const { initGitConfig } = require('../lib/git-utils') +const { getGithubAccessToken } = require('../lib/secrets') const REPOSITORY = process.env.GIT_REPOSITORY const MAIN_BRANCH = process.env.MAIN_BRANCH @@ -94,8 +88,7 @@ function getMajor(version) { } function createPullRequest() { - const githubAccessToken = getSecretKey('ci.browser-sdk.github_access_token') - command`gh auth login --with-token`.withInput(githubAccessToken).run() + command`gh auth login --with-token`.withInput(getGithubAccessToken()).run() const pullRequestUrl = command`gh pr create --fill --base ${MAIN_BRANCH}`.run() return pullRequestUrl.trim() } diff --git a/scripts/test/export-test-result.js b/scripts/test/export-test-result.js index 0b6d604ae8..010bd7ef8a 100644 --- a/scripts/test/export-test-result.js +++ b/scripts/test/export-test-result.js @@ -1,6 +1,8 @@ 'use strict' -const { getSecretKey, command, printLog, runMain } = require('../lib/utils') +const { printLog, runMain } = require('../lib/execution-utils') +const { command } = require('../lib/command') +const { getOrg2ApiKey } = require('../lib/secrets') /** * Upload test result to datadog @@ -12,11 +14,9 @@ const testType = process.argv[2] const resultFolder = `test-report/${testType}/` runMain(() => { - const DATADOG_API_KEY = getSecretKey('ci.browser-sdk.datadog_ci_api_key') - command`datadog-ci junit upload --service browser-sdk --env ci --tags test.type:${testType} ${resultFolder}` .withEnvironment({ - DATADOG_API_KEY, + DATADOG_API_KEY: getOrg2ApiKey(), }) .run()