AP-3965 Vault fetch (#177)

* AP-3965 Script utility for fetching variables from vault from .env file.

.github/workflows/release.script-utils.yml

@@ -0,0 +1,18 @@
+name: Package release to NPM -> script-utils
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+    paths:
+      - 'packages/app/script-utils/**'
+
+jobs:
+  call-build-flow:
+    uses: lokalise/shared-ts-libs/.github/workflows/release.package.yml@main
+    with:
+      working_directory: 'packages/app/script-utils'
+      package_name: 'script-utils'
+    secrets:
+      npm_token: ${{ secrets.NPM_TOKEN }}

packages/app/api-common/vite.config.ts

@@ -1,4 +1,4 @@
-import { resolve } from 'path'
+import { resolve } from 'node:path'
 
 import defineConfig from '@lokalise/package-vite-config/package'
 

packages/app/events-common/vite.config.ts

@@ -1,4 +1,4 @@
-import { resolve } from 'path'
+import { resolve } from 'node:path'
 
 import defineConfig from '@lokalise/package-vite-config/package'
 

packages/app/id-utils/vite.config.ts

@@ -1,4 +1,4 @@
-import { resolve } from 'path'
+import { resolve } from 'node:path'
 
 import defineConfig from '@lokalise/package-vite-config/package'
 

packages/app/non-translatable-markup/vite.config.ts

@@ -1,4 +1,4 @@
-import { resolve } from 'path'
+import { resolve } from 'node:path'
 
 import defineConfig from '@lokalise/package-vite-config/package'
 

packages/app/prisma-utils/vite.config.ts

@@ -1,4 +1,4 @@
-import { resolve } from 'path'
+import { resolve } from 'node:path'
 
 import defineConfig from '@lokalise/package-vite-config/package'
 

packages/app/script-utils/.eslintignore

@@ -0,0 +1,2 @@
+node_modules/
+dist/

packages/app/script-utils/.eslintrc.json

@@ -0,0 +1,11 @@
+{
+	"extends": ["@lokalise/eslint-config/shared-package"],
+	"overrides": [
+		{
+			"files": ["*.ts", "*.tsx"],
+			"parserOptions": {
+				"project": "./tsconfig.lint.json"
+			}
+		}
+	]
+}

packages/app/script-utils/.gitignore

@@ -0,0 +1,4 @@
+node_modules
+dist
+coverage
+.eslintcache

packages/app/script-utils/.prettierignore

@@ -0,0 +1,2 @@
+node_modules
+coverage

packages/app/script-utils/LICENSE.md

@@ -0,0 +1,13 @@
+Copyright 2024 Lokalise, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

packages/app/script-utils/README.md

@@ -0,0 +1,20 @@
+# Script utils
+
+# Usage
+
+## Syncing vault secrets with .env file
+
+```typescript
+//sync-with-vault.ts
+import { synchronizeEnvFileWithVault } from '@lokalise/script-utils'
+
+//Use this function to sync .env file with vault secrets, provide all params to the function.
+```
+
+`tsx` is recommended to be used as a script runner, so you can add the following script to your `package.json` file:
+
+```
+"scripts": {
+    "sync-with-vault": "tsx sync-with-vault.ts"
+}
+```

packages/app/script-utils/package.json

@@ -0,0 +1,51 @@
+{
+	"name": "@lokalise/script-utils",
+	"version": "1.0.0",
+	"type": "module",
+	"files": [
+		"dist",
+		"README.md",
+		"LICENSE.md"
+	],
+	"license": "Apache-2.0",
+	"main": "./dist/index.cjs",
+	"module": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"homepage": "https://github.com/lokalise/shared-ts-libs",
+	"repository": {
+		"type": "git",
+		"url": "git://github.com/lokalise/shared-ts-libs.git"
+	},
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"require": "./dist/index.cjs",
+			"import": "./dist/index.js"
+		}
+	},
+	"scripts": {
+		"build": "rimraf dist && vite build",
+		"dev": "vite watch",
+		"clean": "rimraf dist .eslintcache",
+		"lint": "eslint --cache --max-warnings=0  && prettier --check --log-level warn src \"**/*.{json,md}\" && tsc --noEmit",
+		"lint:fix": "eslint  --fix && prettier --write src \"**/*.{json,md}\"",
+		"test:ci": "vitest run --coverage",
+		"prepublishOnly": "npm run build",
+		"package-version": "echo $npm_package_version"
+	},
+	"dependencies": {
+		"@lokalise/node-core": "^10.0.0"
+	},
+	"devDependencies": {
+		"@lokalise/eslint-config": "latest",
+		"@lokalise/package-vite-config": "latest",
+		"@lokalise/prettier-config": "latest",
+		"@vitest/coverage-v8": "^1.6.0",
+		"prettier": "^3.3.1",
+		"rimraf": "^5.0.7",
+		"typescript": "5.4.5",
+		"vite": "^5.2.13",
+		"vitest": "^1.6.0"
+	},
+	"prettier": "@lokalise/prettier-config"
+}

packages/app/script-utils/src/index.ts

@@ -0,0 +1,2 @@
+/* c8 ignore next */
+export { synchronizeEnvFileWithVault } from './vault/syncEnvWithVault'

packages/app/script-utils/src/vault/syncEnvWithVault.spec.ts

@@ -0,0 +1,87 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { updateEnvFile } from './syncEnvWithVault'
+import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'
+
+const DOT_ENV_PATH = __dirname + '/test_env'
+
+function readDotEnvFile() {
+	return readFileSync(DOT_ENV_PATH, { encoding: 'utf8' }).trim().split('\n')
+}
+
+function putToDotEnvFile(lines: string[]) {
+	writeFileSync(DOT_ENV_PATH, lines.join('\n'))
+}
+
+describe('sync env with vault', () => {
+	afterEach(() => {
+		if (existsSync(DOT_ENV_PATH)) {
+			unlinkSync(DOT_ENV_PATH)
+		}
+	})
+
+	it('should add env vars to file', () => {
+		updateEnvFile(
+			{
+				var1: 'value1',
+				var2: 'value2',
+			},
+			DOT_ENV_PATH,
+		)
+
+		const content = readDotEnvFile()
+
+		expect(content).toEqual(['var1=value1', 'var2=value2'])
+	})
+
+	it('should merge existing data in file with the one from input', () => {
+		putToDotEnvFile(['var0=value0', 'var3=value3'])
+
+		updateEnvFile(
+			{
+				var1: 'value1',
+				var2: 'value2',
+			},
+			DOT_ENV_PATH,
+		)
+
+		const content = readDotEnvFile()
+
+		expect(content).toEqual(['var0=value0', 'var3=value3', 'var1=value1', 'var2=value2'])
+	})
+
+	it('should update value if exists in file', () => {
+		putToDotEnvFile(['var0=value0', 'var3=value3'])
+
+		updateEnvFile(
+			{
+				var0: 'value1',
+			},
+			DOT_ENV_PATH,
+		)
+
+		const content = readDotEnvFile()
+
+		expect(content).toEqual(['var0=value1', 'var3=value3'])
+	})
+
+	it('should do nothing if provided env vars are empty', () => {
+		updateEnvFile({}, DOT_ENV_PATH)
+
+		expect(existsSync(DOT_ENV_PATH)).toEqual(false)
+	})
+
+	it('should replace value of key if in file it is empty', () => {
+		putToDotEnvFile(['var0=', 'var3=value3'])
+
+		updateEnvFile(
+			{
+				var0: 'value0',
+			},
+			DOT_ENV_PATH,
+		)
+
+		const content = readDotEnvFile()
+
+		expect(content).toEqual(['var0=value0', 'var3=value3'])
+	})
+})

packages/app/script-utils/src/vault/syncEnvWithVault.ts

@@ -0,0 +1,88 @@
+import { existsSync, readFileSync, writeFileSync } from 'node:fs'
+import { parseEnv } from 'node:util'
+import { globalLogger } from '@lokalise/node-core'
+import { vaultGetVars, vaultLogin } from './vault'
+
+/**
+ * This function updates the contents of an .env file so the passed `key` has
+ * a new `value` while preserving the rest of the contents. If the key is not
+ * present, it will be appended to the end of the .env text.
+ *
+ * @param key The env variable to be updated
+ * @param value The new value for the env variable
+ * @param envContents String contents of the .env file
+ * @param parsedEnv Record of .env file key-values (parsed using `dotenv.parse`)
+ */
+const upsertEnvValue = (
+	key: string,
+	value: string,
+	envContents: string,
+	parsedEnv: Record<string, string>,
+) => {
+	/* c8 ignore next */
+	const formattedValue = value.includes('\n') ? `"${value.trim()}"` : value
+
+	// Append to the end
+	if (!Object.keys(parsedEnv).includes(key)) {
+		const existingContent = envContents.trim().length > 0 ? `${envContents.trimEnd()}\n` : ''
+		return `${existingContent}${key}=${formattedValue}\n`
+	}
+
+	// If variable is empty - set it
+	if (parsedEnv[key] === '') {
+		return envContents.replace(`${key}=`, `${key}=${formattedValue}`)
+	}
+
+	// Else replace the value itself with adding quotes
+	return envContents.replace(parsedEnv[key], value.trim())
+}
+
+/**
+ * Updates the .env file with new variables.
+ *
+ * @param envVars Record of variable key-values
+ * @param file Path to the .env file
+ */
+export const updateEnvFile = (envVars: Record<string, string>, file: string) => {
+	if (Object.entries(envVars).length === 0) {
+		globalLogger.info(`Skipping env file ${file}`)
+		return
+	}
+	let env = existsSync(file) ? readFileSync(file, { encoding: 'utf-8' }) : ''
+	const parsedEnv = parseEnv(env) as Record<string, string>
+
+	for (const [key, value] of Object.entries(envVars)) {
+		env = upsertEnvValue(key, value, env, parsedEnv)
+	}
+
+	globalLogger.info(`Writing ${file}`)
+	writeFileSync(file, env, { encoding: 'utf-8' })
+}
+
+/* c8 ignore start */
+export const synchronizeEnvFileWithVault = ({
+	dryRun,
+	vault,
+	vaultNamespace,
+	vaultUrl,
+	dotenvFilePath,
+}: {
+	dryRun: boolean
+	vault: boolean
+	vaultNamespace: string
+	vaultUrl: string
+	dotenvFilePath: string
+}) => {
+	if (vault) {
+		vaultLogin(vaultUrl)
+	}
+	const envVarsLocal = vaultGetVars(vaultNamespace)
+
+	if (dryRun) {
+		globalLogger.info(envVarsLocal)
+		globalLogger.info(`// ${dotenvFilePath}`)
+	} else {
+		updateEnvFile(envVarsLocal, dotenvFilePath)
+	}
+}
+/* c8 ignore end */