diff --git a/.github/workflows/release.script-utils.yml b/.github/workflows/release.script-utils.yml new file mode 100644 index 00000000..1688bf64 --- /dev/null +++ b/.github/workflows/release.script-utils.yml @@ -0,0 +1,18 @@ +name: Package release to NPM -> script-utils +on: + pull_request: + types: + - closed + branches: + - main + paths: + - 'packages/app/script-utils/**' + +jobs: + call-build-flow: + uses: lokalise/shared-ts-libs/.github/workflows/release.package.yml@main + with: + working_directory: 'packages/app/script-utils' + package_name: 'script-utils' + secrets: + npm_token: ${{ secrets.NPM_TOKEN }} diff --git a/packages/app/api-common/vite.config.ts b/packages/app/api-common/vite.config.ts index e220dfa8..7eebe9e5 100644 --- a/packages/app/api-common/vite.config.ts +++ b/packages/app/api-common/vite.config.ts @@ -1,4 +1,4 @@ -import { resolve } from 'path' +import { resolve } from 'node:path' import defineConfig from '@lokalise/package-vite-config/package' diff --git a/packages/app/events-common/vite.config.ts b/packages/app/events-common/vite.config.ts index 757f21b3..490254dc 100644 --- a/packages/app/events-common/vite.config.ts +++ b/packages/app/events-common/vite.config.ts @@ -1,4 +1,4 @@ -import { resolve } from 'path' +import { resolve } from 'node:path' import defineConfig from '@lokalise/package-vite-config/package' diff --git a/packages/app/id-utils/vite.config.ts b/packages/app/id-utils/vite.config.ts index 70b3e84f..e55d8b19 100644 --- a/packages/app/id-utils/vite.config.ts +++ b/packages/app/id-utils/vite.config.ts @@ -1,4 +1,4 @@ -import { resolve } from 'path' +import { resolve } from 'node:path' import defineConfig from '@lokalise/package-vite-config/package' diff --git a/packages/app/non-translatable-markup/vite.config.ts b/packages/app/non-translatable-markup/vite.config.ts index e220dfa8..7eebe9e5 100644 --- a/packages/app/non-translatable-markup/vite.config.ts +++ b/packages/app/non-translatable-markup/vite.config.ts @@ -1,4 +1,4 @@ -import { resolve } from 'path' +import { resolve } from 'node:path' import defineConfig from '@lokalise/package-vite-config/package' diff --git a/packages/app/prisma-utils/vite.config.ts b/packages/app/prisma-utils/vite.config.ts index 70b3e84f..e55d8b19 100644 --- a/packages/app/prisma-utils/vite.config.ts +++ b/packages/app/prisma-utils/vite.config.ts @@ -1,4 +1,4 @@ -import { resolve } from 'path' +import { resolve } from 'node:path' import defineConfig from '@lokalise/package-vite-config/package' diff --git a/packages/app/script-utils/.eslintignore b/packages/app/script-utils/.eslintignore new file mode 100644 index 00000000..b9470778 --- /dev/null +++ b/packages/app/script-utils/.eslintignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/packages/app/script-utils/.eslintrc.json b/packages/app/script-utils/.eslintrc.json new file mode 100644 index 00000000..b7179966 --- /dev/null +++ b/packages/app/script-utils/.eslintrc.json @@ -0,0 +1,11 @@ +{ + "extends": ["@lokalise/eslint-config/shared-package"], + "overrides": [ + { + "files": ["*.ts", "*.tsx"], + "parserOptions": { + "project": "./tsconfig.lint.json" + } + } + ] +} diff --git a/packages/app/script-utils/.gitignore b/packages/app/script-utils/.gitignore new file mode 100644 index 00000000..03f97ad9 --- /dev/null +++ b/packages/app/script-utils/.gitignore @@ -0,0 +1,4 @@ +node_modules +dist +coverage +.eslintcache diff --git a/packages/app/script-utils/.prettierignore b/packages/app/script-utils/.prettierignore new file mode 100644 index 00000000..ba2a97b5 --- /dev/null +++ b/packages/app/script-utils/.prettierignore @@ -0,0 +1,2 @@ +node_modules +coverage diff --git a/packages/app/script-utils/LICENSE.md b/packages/app/script-utils/LICENSE.md new file mode 100644 index 00000000..463dc9dc --- /dev/null +++ b/packages/app/script-utils/LICENSE.md @@ -0,0 +1,13 @@ +Copyright 2024 Lokalise, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/app/script-utils/README.md b/packages/app/script-utils/README.md new file mode 100644 index 00000000..8ab93fcf --- /dev/null +++ b/packages/app/script-utils/README.md @@ -0,0 +1,20 @@ +# Script utils + +# Usage + +## Syncing vault secrets with .env file + +```typescript +//sync-with-vault.ts +import { synchronizeEnvFileWithVault } from '@lokalise/script-utils' + +//Use this function to sync .env file with vault secrets, provide all params to the function. +``` + +`tsx` is recommended to be used as a script runner, so you can add the following script to your `package.json` file: + +``` +"scripts": { + "sync-with-vault": "tsx sync-with-vault.ts" +} +``` diff --git a/packages/app/script-utils/package.json b/packages/app/script-utils/package.json new file mode 100644 index 00000000..30b82bbb --- /dev/null +++ b/packages/app/script-utils/package.json @@ -0,0 +1,51 @@ +{ + "name": "@lokalise/script-utils", + "version": "1.0.0", + "type": "module", + "files": [ + "dist", + "README.md", + "LICENSE.md" + ], + "license": "Apache-2.0", + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "homepage": "https://github.com/lokalise/shared-ts-libs", + "repository": { + "type": "git", + "url": "git://github.com/lokalise/shared-ts-libs.git" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "require": "./dist/index.cjs", + "import": "./dist/index.js" + } + }, + "scripts": { + "build": "rimraf dist && vite build", + "dev": "vite watch", + "clean": "rimraf dist .eslintcache", + "lint": "eslint --cache --max-warnings=0 && prettier --check --log-level warn src \"**/*.{json,md}\" && tsc --noEmit", + "lint:fix": "eslint --fix && prettier --write src \"**/*.{json,md}\"", + "test:ci": "vitest run --coverage", + "prepublishOnly": "npm run build", + "package-version": "echo $npm_package_version" + }, + "dependencies": { + "@lokalise/node-core": "^10.0.0" + }, + "devDependencies": { + "@lokalise/eslint-config": "latest", + "@lokalise/package-vite-config": "latest", + "@lokalise/prettier-config": "latest", + "@vitest/coverage-v8": "^1.6.0", + "prettier": "^3.3.1", + "rimraf": "^5.0.7", + "typescript": "5.4.5", + "vite": "^5.2.13", + "vitest": "^1.6.0" + }, + "prettier": "@lokalise/prettier-config" +} diff --git a/packages/app/script-utils/src/index.ts b/packages/app/script-utils/src/index.ts new file mode 100644 index 00000000..e4eb9826 --- /dev/null +++ b/packages/app/script-utils/src/index.ts @@ -0,0 +1,2 @@ +/* c8 ignore next */ +export { synchronizeEnvFileWithVault } from './vault/syncEnvWithVault' diff --git a/packages/app/script-utils/src/vault/syncEnvWithVault.spec.ts b/packages/app/script-utils/src/vault/syncEnvWithVault.spec.ts new file mode 100644 index 00000000..8d4936dd --- /dev/null +++ b/packages/app/script-utils/src/vault/syncEnvWithVault.spec.ts @@ -0,0 +1,87 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { updateEnvFile } from './syncEnvWithVault' +import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs' + +const DOT_ENV_PATH = __dirname + '/test_env' + +function readDotEnvFile() { + return readFileSync(DOT_ENV_PATH, { encoding: 'utf8' }).trim().split('\n') +} + +function putToDotEnvFile(lines: string[]) { + writeFileSync(DOT_ENV_PATH, lines.join('\n')) +} + +describe('sync env with vault', () => { + afterEach(() => { + if (existsSync(DOT_ENV_PATH)) { + unlinkSync(DOT_ENV_PATH) + } + }) + + it('should add env vars to file', () => { + updateEnvFile( + { + var1: 'value1', + var2: 'value2', + }, + DOT_ENV_PATH, + ) + + const content = readDotEnvFile() + + expect(content).toEqual(['var1=value1', 'var2=value2']) + }) + + it('should merge existing data in file with the one from input', () => { + putToDotEnvFile(['var0=value0', 'var3=value3']) + + updateEnvFile( + { + var1: 'value1', + var2: 'value2', + }, + DOT_ENV_PATH, + ) + + const content = readDotEnvFile() + + expect(content).toEqual(['var0=value0', 'var3=value3', 'var1=value1', 'var2=value2']) + }) + + it('should update value if exists in file', () => { + putToDotEnvFile(['var0=value0', 'var3=value3']) + + updateEnvFile( + { + var0: 'value1', + }, + DOT_ENV_PATH, + ) + + const content = readDotEnvFile() + + expect(content).toEqual(['var0=value1', 'var3=value3']) + }) + + it('should do nothing if provided env vars are empty', () => { + updateEnvFile({}, DOT_ENV_PATH) + + expect(existsSync(DOT_ENV_PATH)).toEqual(false) + }) + + it('should replace value of key if in file it is empty', () => { + putToDotEnvFile(['var0=', 'var3=value3']) + + updateEnvFile( + { + var0: 'value0', + }, + DOT_ENV_PATH, + ) + + const content = readDotEnvFile() + + expect(content).toEqual(['var0=value0', 'var3=value3']) + }) +}) diff --git a/packages/app/script-utils/src/vault/syncEnvWithVault.ts b/packages/app/script-utils/src/vault/syncEnvWithVault.ts new file mode 100644 index 00000000..eb73acb9 --- /dev/null +++ b/packages/app/script-utils/src/vault/syncEnvWithVault.ts @@ -0,0 +1,88 @@ +import { existsSync, readFileSync, writeFileSync } from 'node:fs' +import { parseEnv } from 'node:util' +import { globalLogger } from '@lokalise/node-core' +import { vaultGetVars, vaultLogin } from './vault' + +/** + * This function updates the contents of an .env file so the passed `key` has + * a new `value` while preserving the rest of the contents. If the key is not + * present, it will be appended to the end of the .env text. + * + * @param key The env variable to be updated + * @param value The new value for the env variable + * @param envContents String contents of the .env file + * @param parsedEnv Record of .env file key-values (parsed using `dotenv.parse`) + */ +const upsertEnvValue = ( + key: string, + value: string, + envContents: string, + parsedEnv: Record, +) => { + /* c8 ignore next */ + const formattedValue = value.includes('\n') ? `"${value.trim()}"` : value + + // Append to the end + if (!Object.keys(parsedEnv).includes(key)) { + const existingContent = envContents.trim().length > 0 ? `${envContents.trimEnd()}\n` : '' + return `${existingContent}${key}=${formattedValue}\n` + } + + // If variable is empty - set it + if (parsedEnv[key] === '') { + return envContents.replace(`${key}=`, `${key}=${formattedValue}`) + } + + // Else replace the value itself with adding quotes + return envContents.replace(parsedEnv[key], value.trim()) +} + +/** + * Updates the .env file with new variables. + * + * @param envVars Record of variable key-values + * @param file Path to the .env file + */ +export const updateEnvFile = (envVars: Record, file: string) => { + if (Object.entries(envVars).length === 0) { + globalLogger.info(`Skipping env file ${file}`) + return + } + let env = existsSync(file) ? readFileSync(file, { encoding: 'utf-8' }) : '' + const parsedEnv = parseEnv(env) as Record + + for (const [key, value] of Object.entries(envVars)) { + env = upsertEnvValue(key, value, env, parsedEnv) + } + + globalLogger.info(`Writing ${file}`) + writeFileSync(file, env, { encoding: 'utf-8' }) +} + +/* c8 ignore start */ +export const synchronizeEnvFileWithVault = ({ + dryRun, + vault, + vaultNamespace, + vaultUrl, + dotenvFilePath, +}: { + dryRun: boolean + vault: boolean + vaultNamespace: string + vaultUrl: string + dotenvFilePath: string +}) => { + if (vault) { + vaultLogin(vaultUrl) + } + const envVarsLocal = vaultGetVars(vaultNamespace) + + if (dryRun) { + globalLogger.info(envVarsLocal) + globalLogger.info(`// ${dotenvFilePath}`) + } else { + updateEnvFile(envVarsLocal, dotenvFilePath) + } +} +/* c8 ignore end */ diff --git a/packages/app/script-utils/src/vault/vault.ts b/packages/app/script-utils/src/vault/vault.ts new file mode 100644 index 00000000..e13ecc3a --- /dev/null +++ b/packages/app/script-utils/src/vault/vault.ts @@ -0,0 +1,108 @@ +/* c8 ignore start */ +import type { SpawnSyncReturns } from 'node:child_process' +import { execSync } from 'node:child_process' + +import { globalLogger } from '@lokalise/node-core' + +type VaultGetTokenResponse = { + request_id: string + data: { + id: string + expire_time: string + } +} + +type VaultGetVarsResponse = { + request_id: string + data: { + data: Record + metadata: { + version: number + } + } +} + +const isExecError = (e: unknown): e is Error & SpawnSyncReturns => + typeof e === 'object' && e !== null && 'stderr' in e + +const installVaultCli = () => { + const osType = process.platform + if (osType === 'linux') { + try { + execSync('which vault') + } catch (_e) { + // if it fails means vault is not found -> install + execSync('sudo apt update && sudo apt install gpg') + execSync( + 'wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg', + ) + execSync( + 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list', + ) + execSync('sudo apt update && sudo apt install vault') + } + } else if (osType === 'darwin') { + // macOS + execSync('brew list vault || brew install vault') // installing only if not installed + } else { + globalLogger.warn( + `You are running ${osType}, which doesn't support autoinstall, please ensure that Vault is installed and is accessible globally via "vault" command.`, + ) + } +} + +// eslint-disable-next-line max-statements +export const vaultLogin = (vaultUrl: string) => { + process.env.VAULT_ADDR = vaultUrl + + let tokenResponse: VaultGetTokenResponse | null = null + + const loginCommand = 'vault login -method=oidc -path=gsuite -no-print' + const tokenCommand = 'vault token lookup -format=json' + try { + installVaultCli() + try { + // If the token doesn't exist or is expired --> login + tokenResponse = JSON.parse(execSync(tokenCommand).toString()) as VaultGetTokenResponse + + if (tokenResponse && new Date(tokenResponse.data.expire_time) <= new Date()) { + execSync(loginCommand) + tokenResponse = null + } + } catch (_e) { + execSync(loginCommand) + } + + if (!tokenResponse) { + tokenResponse = JSON.parse(execSync(tokenCommand).toString()) as VaultGetTokenResponse + } + } catch (e: unknown) { + if (e instanceof Error) { + globalLogger.info(`Vault login error -> ${e.message}`) + } + } + + process.env.VAULT_TOKEN = tokenResponse?.data.id ?? '' +} + +export const vaultGetVars = (service: string): Record => { + if (!process.env.VAULT_TOKEN) { + return {} + } + + try { + const responseJson = JSON.parse( + execSync(`vault kv get -format=json "kv/${service}/dev"`).toString(), + ) as VaultGetVarsResponse + + globalLogger.info(`Vault ${service} version: ${responseJson.data.metadata.version}`) + + return responseJson.data.data + } catch (e: unknown) { + if (isExecError(e)) { + globalLogger.error(`Vault ${service} error downloading env vars -> ${e.stderr.toString()}`) + } + return {} + } +} +/* c8 ignore end */ diff --git a/packages/app/script-utils/tsconfig.json b/packages/app/script-utils/tsconfig.json new file mode 100644 index 00000000..b0199dfa --- /dev/null +++ b/packages/app/script-utils/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "outDir": "dist", + "module": "ESNext", + "target": "ES2022", + "lib": ["ES2022", "dom"], + "sourceMap": false, + "declaration": true, + "declarationMap": false, + "types": ["vitest/globals"], + "skipLibCheck": true, + "strict": true, + "moduleResolution": "node", + "noUnusedLocals": false, + "noUnusedParameters": false, + "noFallthroughCasesInSwitch": true, + "strictNullChecks": true, + "importHelpers": true, + "baseUrl": "./", + "allowSyntheticDefaultImports": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.spec.ts", "src/**/*.test.ts"] +} diff --git a/packages/app/script-utils/tsconfig.lint.json b/packages/app/script-utils/tsconfig.lint.json new file mode 100644 index 00000000..a88cd87f --- /dev/null +++ b/packages/app/script-utils/tsconfig.lint.json @@ -0,0 +1,5 @@ +{ + "extends": ["./tsconfig.json"], + "include": ["src/**/*", "vite.config.ts"], + "exclude": [] +} diff --git a/packages/app/script-utils/vite.config.ts b/packages/app/script-utils/vite.config.ts new file mode 100644 index 00000000..e55d8b19 --- /dev/null +++ b/packages/app/script-utils/vite.config.ts @@ -0,0 +1,11 @@ +import { resolve } from 'node:path' + +import defineConfig from '@lokalise/package-vite-config/package' + +import packageJson from './package.json' + +/* eslint-disable import/no-default-export */ +export default defineConfig({ + entry: resolve(__dirname, 'src/index.ts'), + dependencies: Object.keys(packageJson.dependencies), +}) diff --git a/packages/app/websockets-common/vite.config.ts b/packages/app/websockets-common/vite.config.ts index cdf0cb21..4fd145b2 100644 --- a/packages/app/websockets-common/vite.config.ts +++ b/packages/app/websockets-common/vite.config.ts @@ -1,4 +1,4 @@ -import { resolve } from 'path' +import { resolve } from 'node:path' import defineConfig from '@lokalise/package-vite-config/package'