Skip to content

Commit

Permalink
♻️[RUMF-1517] split scripts utils (#2102)
Browse files Browse the repository at this point in the history
* ♻️ extract files.utils.js

* ♻️ extract git.utils.js

* ♻️ extract secrets.js

* ♻️ rename utils.js in execution.utils.js

* 👌 rename `*.utils` in `*-utils`

* 👌 extract command.js
  • Loading branch information
bcaudan authored Mar 28, 2023
1 parent 8fc4fd9 commit 63ad678
Show file tree
Hide file tree
Showing 21 changed files with 222 additions and 181 deletions.
3 changes: 2 additions & 1 deletion scripts/build/replace-build-env.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const glob = require('glob')
const { printLog, runMain, modifyFile } = require('../lib/utils')
const { printLog, runMain } = require('../lib/execution-utils')
const { modifyFile } = require('../lib/files-utils')
const buildEnv = require('../lib/build-env')

/**
Expand Down
3 changes: 2 additions & 1 deletion scripts/check-licenses.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
const fs = require('fs')
const path = require('path')
const readline = require('readline')
const { printLog, printError, runMain, findBrowserSdkPackageJsonFiles } = require('./lib/utils')
const { printLog, printError, runMain } = require('./lib/execution-utils')
const { findBrowserSdkPackageJsonFiles } = require('./lib/files-utils')

const LICENSE_FILE = 'LICENSE-3rdparty.csv'

Expand Down
3 changes: 2 additions & 1 deletion scripts/deploy/deploy.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use strict'

const { printLog, command, runMain } = require('../lib/utils')
const { printLog, runMain } = require('../lib/execution-utils')
const { command } = require('../lib/command')
const {
buildRootUploadPath,
buildDatacenterUploadPath,
Expand Down
9 changes: 4 additions & 5 deletions scripts/deploy/upload-source-maps.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
'use strict'

const path = require('path')
const { getSecretKey, command, printLog, runMain } = require('../lib/utils')
const { printLog, runMain } = require('../lib/execution-utils')
const { command } = require('../lib/command')
const { SDK_VERSION } = require('../lib/build-env')
const { getTelemetryOrgApiKey } = require('../lib/secrets')
const {
buildRootUploadPath,
buildDatacenterUploadPath,
Expand Down Expand Up @@ -66,9 +68,6 @@ function renameFilesWithVersionSuffix(packageName, bundleFolder) {

function uploadSourceMaps(packageName, service, prefix, bundleFolder, sites) {
for (const site of sites) {
const normalizedSite = site.replaceAll('.', '-')
const apiKey = getSecretKey(`ci.browser-sdk.source-maps.${normalizedSite}.ci_api_key`)

printLog(`Uploading ${packageName} source maps with prefix ${prefix} for ${site}...`)

command`
Expand All @@ -80,7 +79,7 @@ function uploadSourceMaps(packageName, service, prefix, bundleFolder, sites) {
--repository-url https://www.github.com/datadog/browser-sdk
`
.withEnvironment({
DATADOG_API_KEY: apiKey,
DATADOG_API_KEY: getTelemetryOrgApiKey(site),
DATADOG_SITE: site,
})
.run()
Expand Down
2 changes: 1 addition & 1 deletion scripts/dev-server.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const webpack = require('webpack')
const logsConfig = require('../packages/logs/webpack.config')
const rumSlimConfig = require('../packages/rum-slim/webpack.config')
const rumConfig = require('../packages/rum/webpack.config')
const { printLog } = require('./lib/utils')
const { printLog } = require('./lib/execution-utils')

const port = 8080
const app = express()
Expand Down
2 changes: 1 addition & 1 deletion scripts/generate-schema-types.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ const fs = require('fs')
const path = require('path')
const { compileFromFile } = require('json-schema-to-typescript')
const prettier = require('prettier')
const { printLog, runMain } = require('./lib/utils')
const { printLog, runMain } = require('./lib/execution-utils')

const schemasDirectoryPath = path.join(__dirname, '../rum-events-format/schemas')
const prettierConfigPath = path.join(__dirname, '../.prettierrc.yml')
Expand Down
136 changes: 1 addition & 135 deletions scripts/lib/utils.js → scripts/lib/command.js
Original file line number Diff line number Diff line change
@@ -1,80 +1,6 @@
const path = require('path')
const os = require('os')
const fsPromises = require('fs/promises')
const fs = require('fs')
const childProcess = require('child_process')
const spawn = require('child_process').spawn
// node-fetch v3.x only support ESM syntax.
// Todo: Remove node-fetch when node v18 LTS is released with fetch out of the box
const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args))

const CI_FILE = '.gitlab-ci.yml'

function getSecretKey(name) {
return command`
aws ssm get-parameter --region=us-east-1 --with-decryption --query=Parameter.Value --out=text --name=${name}
`
.run()
.trim()
}

function initGitConfig(repository) {
const githubDeployKey = getSecretKey('ci.browser-sdk.github_deploy_key')
const homedir = os.homedir()

// ssh-add expects a new line at the end of the PEM-formatted private key
// https://stackoverflow.com/a/59595773
command`ssh-add -`.withInput(`${githubDeployKey}\n`).run()
command`mkdir -p ${homedir}/.ssh`.run()
command`chmod 700 ${homedir}/.ssh`.run()
const sshHost = command`ssh-keyscan -H github.com`.run()
fs.appendFileSync(`${homedir}/.ssh/known_hosts`, sshHost)
command`git config user.email ci.browser-sdk@datadoghq.com`.run()
command`git config user.name ci.browser-sdk`.run()
command`git remote set-url origin ${repository}`.run()
}

function readCiFileVariable(variableName) {
const regexp = new RegExp(`${variableName}: (.*)`)
const ciFileContent = fs.readFileSync(CI_FILE, { encoding: 'utf-8' })
return regexp.exec(ciFileContent)?.[1]
}

async function replaceCiFileVariable(variableName, value) {
await modifyFile(CI_FILE, (content) =>
content.replace(new RegExp(`${variableName}: .*`), `${variableName}: ${value}`)
)
}

/**
* @param filePath {string}
* @param modifier {(content: string) => string}
*/
async function modifyFile(filePath, modifier) {
const content = await fsPromises.readFile(filePath, { encoding: 'utf-8' })
const modifiedContent = modifier(content)
if (content !== modifiedContent) {
await fsPromises.writeFile(filePath, modifiedContent)
return true
}
return false
}

/**
* Helper to run executables asynchronously, in a shell. This function does not prevent Shell
* injections[0], so please use carefully. Only use it to run commands with trusted arguments.
* Prefer the `command` helper for most use cases.
*
* [0]: https://matklad.github.io/2021/07/30/shell-injection.html
*/
async function spawnCommand(command, args) {
return new Promise((resolve, reject) => {
const child = spawn(command, args, { stdio: 'inherit', shell: true })
child.on('error', reject)
child.on('close', resolve)
child.on('exit', resolve)
})
}
const { printError } = require('./execution-utils')

/**
* Helper to run executables. This has been introduced to work around Shell injections[0] while
Expand Down Expand Up @@ -205,66 +131,6 @@ function parseCommandTemplateArguments(templateStrings, ...templateVariables) {
return parsedArguments
}

function runMain(mainFunction) {
Promise.resolve()
// The main function can be either synchronous or asynchronous, so let's wrap it in an async
// callback that will catch both thrown errors and rejected promises
.then(() => mainFunction())
.catch((error) => {
printError('\nScript exited with error:')
printError(error)
process.exit(1)
})
}

const resetColor = '\x1b[0m'

function printError(...params) {
const redColor = '\x1b[31;1m'
console.log(redColor, ...params, resetColor)
}

function printLog(...params) {
const greenColor = '\x1b[32;1m'
console.log(greenColor, ...params, resetColor)
}

async function fetchWrapper(url, options) {
const response = await fetch(url, options)
if (!response.ok) {
throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`)
}

return response.text()
}

function findBrowserSdkPackageJsonFiles() {
const manifestPaths = command`git ls-files -- package.json */package.json`.run()
return manifestPaths
.trim()
.split('\n')
.map((manifestPath) => {
const absoluteManifestPath = path.join(__dirname, '../..', manifestPath)
return {
relativePath: manifestPath,
path: absoluteManifestPath,
content: require(absoluteManifestPath),
}
})
}

module.exports = {
CI_FILE,
getSecretKey,
initGitConfig,
command,
spawnCommand,
printError,
printLog,
runMain,
readCiFileVariable,
replaceCiFileVariable,
fetch: fetchWrapper,
modifyFile,
findBrowserSdkPackageJsonFiles,
}
61 changes: 61 additions & 0 deletions scripts/lib/execution-utils.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
const spawn = require('child_process').spawn
// node-fetch v3.x only support ESM syntax.
// Todo: Remove node-fetch when node v18 LTS is released with fetch out of the box
const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args))

/**
* Helper to run executables asynchronously, in a shell. This function does not prevent Shell
* injections[0], so please use carefully. Only use it to run commands with trusted arguments.
* Prefer the `command` helper for most use cases.
*
* [0]: https://matklad.github.io/2021/07/30/shell-injection.html
*/
async function spawnCommand(command, args) {
return new Promise((resolve, reject) => {
const child = spawn(command, args, { stdio: 'inherit', shell: true })
child.on('error', reject)
child.on('close', resolve)
child.on('exit', resolve)
})
}

function runMain(mainFunction) {
Promise.resolve()
// The main function can be either synchronous or asynchronous, so let's wrap it in an async
// callback that will catch both thrown errors and rejected promises
.then(() => mainFunction())
.catch((error) => {
printError('\nScript exited with error:')
printError(error)
process.exit(1)
})
}

const resetColor = '\x1b[0m'

function printError(...params) {
const redColor = '\x1b[31;1m'
console.log(redColor, ...params, resetColor)
}

function printLog(...params) {
const greenColor = '\x1b[32;1m'
console.log(greenColor, ...params, resetColor)
}

async function fetchWrapper(url, options) {
const response = await fetch(url, options)
if (!response.ok) {
throw new Error(`HTTP Error Response: ${response.status} ${response.statusText}`)
}

return response.text()
}

module.exports = {
spawnCommand,
printError,
printLog,
runMain,
fetch: fetchWrapper,
}
56 changes: 56 additions & 0 deletions scripts/lib/files-utils.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
const fs = require('fs')
const path = require('path')
const fsPromises = require('fs/promises')

const { command } = require('./command')

const CI_FILE = '.gitlab-ci.yml'

function readCiFileVariable(variableName) {
const regexp = new RegExp(`${variableName}: (.*)`)
const ciFileContent = fs.readFileSync(CI_FILE, { encoding: 'utf-8' })
return regexp.exec(ciFileContent)?.[1]
}

async function replaceCiFileVariable(variableName, value) {
await modifyFile(CI_FILE, (content) =>
content.replace(new RegExp(`${variableName}: .*`), `${variableName}: ${value}`)
)
}

/**
* @param filePath {string}
* @param modifier {(content: string) => string}
*/
async function modifyFile(filePath, modifier) {
const content = await fsPromises.readFile(filePath, { encoding: 'utf-8' })
const modifiedContent = modifier(content)
if (content !== modifiedContent) {
await fsPromises.writeFile(filePath, modifiedContent)
return true
}
return false
}

function findBrowserSdkPackageJsonFiles() {
const manifestPaths = command`git ls-files -- package.json */package.json`.run()
return manifestPaths
.trim()
.split('\n')
.map((manifestPath) => {
const absoluteManifestPath = path.join(__dirname, '../..', manifestPath)
return {
relativePath: manifestPath,
path: absoluteManifestPath,
content: require(absoluteManifestPath),
}
})
}

module.exports = {
CI_FILE,
readCiFileVariable,
replaceCiFileVariable,
modifyFile,
findBrowserSdkPackageJsonFiles,
}
24 changes: 24 additions & 0 deletions scripts/lib/git-utils.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
const os = require('os')
const fs = require('fs')

const { command } = require('../lib/command')
const { getGithubDeployKey } = require('./secrets')

function initGitConfig(repository) {
const homedir = os.homedir()

// ssh-add expects a new line at the end of the PEM-formatted private key
// https://stackoverflow.com/a/59595773
command`ssh-add -`.withInput(`${getGithubDeployKey()}\n`).run()
command`mkdir -p ${homedir}/.ssh`.run()
command`chmod 700 ${homedir}/.ssh`.run()
const sshHost = command`ssh-keyscan -H github.com`.run()
fs.appendFileSync(`${homedir}/.ssh/known_hosts`, sshHost)
command`git config user.email ci.browser-sdk@datadoghq.com`.run()
command`git config user.name ci.browser-sdk`.run()
command`git remote set-url origin ${repository}`.run()
}

module.exports = {
initGitConfig,
}
Loading

0 comments on commit 63ad678

Please sign in to comment.