From 3affd6bbed3f1668fa551e9da743bd42ce04a425 Mon Sep 17 00:00:00 2001 From: Alok Swamy Date: Tue, 27 Feb 2024 15:50:40 -0500 Subject: [PATCH] Add `dotenv` package to cli-kit --- .changeset/perfect-beans-repair.md | 5 + packages/cli-kit/package.json | 2 +- .../cli-kit/src/public/node/dot-env.test.ts | 145 ++++++++++++++++- packages/cli-kit/src/public/node/dot-env.ts | 77 +++++++-- pnpm-lock.yaml | 146 ++++++++++++++++-- 5 files changed, 352 insertions(+), 23 deletions(-) create mode 100644 .changeset/perfect-beans-repair.md diff --git a/.changeset/perfect-beans-repair.md b/.changeset/perfect-beans-repair.md new file mode 100644 index 0000000000..64ba8097fa --- /dev/null +++ b/.changeset/perfect-beans-repair.md @@ -0,0 +1,5 @@ +--- +'@shopify/cli-kit': minor +--- + +Support parsing multiline environment variables in .env file diff --git a/packages/cli-kit/package.json b/packages/cli-kit/package.json index fc9a8229bc..16fbcebd8b 100644 --- a/packages/cli-kit/package.json +++ b/packages/cli-kit/package.json @@ -123,8 +123,8 @@ "cross-zip": "4.0.0", "deepmerge": "4.3.1", "del": "6.1.1", + "dotenv": "16.4.5", "env-paths": "3.0.0", - "envfile": "6.18.0", "execa": "7.2.0", "fast-glob": "3.3.1", "figures": "5.0.0", diff --git a/packages/cli-kit/src/public/node/dot-env.test.ts b/packages/cli-kit/src/public/node/dot-env.test.ts index 96f51d6f11..3982e5e361 100644 --- a/packages/cli-kit/src/public/node/dot-env.test.ts +++ b/packages/cli-kit/src/public/node/dot-env.test.ts @@ -1,4 +1,4 @@ -import {patchEnvFile, readAndParseDotEnv, writeDotEnv} from './dot-env.js' +import {patchEnvFile, readAndParseDotEnv, writeDotEnv, createDotEnvFileLine} from './dot-env.js' import {inTemporaryDirectory, writeFile, readFile} from './fs.js' import {joinPath} from './path.js' import {describe, expect, test} from 'vitest' @@ -26,6 +26,22 @@ describe('readAndParseDotEnv', () => { expect(got.variables.FOO).toEqual('BAR') }) }) + + test('ensures newline characters are parsed from .env file', async () => { + await inTemporaryDirectory(async (tmpDir) => { + // Given + const dotEnvPath = joinPath(tmpDir, '.env') + + await writeFile(dotEnvPath, `FOO="BAR\nBAR\nBAR"`) + + // When + const got = await readAndParseDotEnv(dotEnvPath) + + // Then + expect(got.path).toEqual(dotEnvPath) + expect(got.variables.FOO).toEqual('BAR\nBAR\nBAR') + }) + }) }) describe('writeDotEnv', () => { @@ -49,6 +65,27 @@ describe('writeDotEnv', () => { }) }) + test('creates a file with multiline env vars', async () => { + await inTemporaryDirectory(async (tmpDir) => { + // Given + const dotEnvPath = joinPath(tmpDir, '.env') + + // When + await writeDotEnv({ + path: dotEnvPath, + variables: { + FOO: 'BAR', + MULTI: 'LINE\nVARIABLE', + }, + }) + const got = await readAndParseDotEnv(dotEnvPath) + + // Then + expect(got.path).toEqual(dotEnvPath) + expect(got.variables.MULTI).toEqual('LINE\nVARIABLE') + }) + }) + test('overrides any existing file', async () => { await inTemporaryDirectory(async (tmpDir) => { // Given @@ -112,4 +149,110 @@ describe('patchEnvFile', () => { expect(patchedContent).toEqual('FOO=BAR\nABC=123\n#Wow!\n\n DEF =GHI\r\nWIN=DOWS') }) }) + + test('patches an environment file containing newline characters', async () => { + await inTemporaryDirectory(async (tmpDir) => { + // Given + const dotEnvPath = joinPath(tmpDir, '.env') + await writeFile(dotEnvPath, 'FOO="BAR\nBAR\nBAR"\nABC =XYZ\n#Wow!\n\n DEF =GHI') + + // When + const got = await readAndParseDotEnv(dotEnvPath) + expect(got.variables).toEqual({ + FOO: 'BAR\nBAR\nBAR', + ABC: 'XYZ', + DEF: 'GHI', + }) + + // Then + const patchedContent = patchEnvFile(await readFile(dotEnvPath), {ABC: '123'}) + expect(patchedContent).toEqual('FOO="BAR\nBAR\nBAR"\nABC=123\n#Wow!\n\n DEF =GHI') + }) + }) + + test('patches env var with newline characters', async () => { + await inTemporaryDirectory(async (tmpDir) => { + // Given + const dotEnvPath = joinPath(tmpDir, '.env') + await writeFile(dotEnvPath, 'FOO="BAR\nBAR\nBAR"\nABC =XYZ\n#Wow!\n\n DEF =GHI') + + // When + const got = await readAndParseDotEnv(dotEnvPath) + expect(got.variables).toEqual({ + FOO: 'BAR\nBAR\nBAR', + ABC: 'XYZ', + DEF: 'GHI', + }) + + // Then + const patchedContent = patchEnvFile(await readFile(dotEnvPath), {FOO: 'BAZ\nBAZ\nBAZ'}) + expect(patchedContent).toEqual('FOO="BAZ\nBAZ\nBAZ"\nABC =XYZ\n#Wow!\n\n DEF =GHI') + }) + }) + + test('patches an environment file and creates a new env var with newline characters', async () => { + await inTemporaryDirectory(async (tmpDir) => { + // Given + const dotEnvPath = joinPath(tmpDir, '.env') + await writeFile(dotEnvPath, 'FOO=BAR\nABC =XYZ\n#Wow!\n\n DEF =GHI') + + // When + const got = await readAndParseDotEnv(dotEnvPath) + expect(got.variables).toEqual({ + FOO: 'BAR', + ABC: 'XYZ', + DEF: 'GHI', + }) + + // Then + const patchedContent = patchEnvFile(await readFile(dotEnvPath), {MULTI: 'LINE\nVARIABLE'}) + expect(patchedContent).toEqual('FOO=BAR\nABC =XYZ\n#Wow!\n\n DEF =GHI\nMULTI="LINE\nVARIABLE"') + }) + }) + + test(`throws error when multiline environment variable isn't closed`, async () => { + await inTemporaryDirectory(async (tmpDir) => { + // Given + const dotEnvPath = joinPath(tmpDir, '.env') + await writeFile(dotEnvPath, 'FOO=BAR\nABC ="XYZ\n#Wow!\n\n DEF =GHI') + + // Then + await expect(async () => { + patchEnvFile(await readFile(dotEnvPath), {MULTI: 'LINE\nVARIABLE'}) + }).rejects.toThrow(`Multi-line environment variable 'ABC' is not properly enclosed.`) + }) + }) +}) + +describe('createDotEnvFileLine', () => { + test('creates an env var for a .env file', () => { + const line = createDotEnvFileLine('FOO', 'BAR') + + expect(line).toEqual('FOO=BAR') + }) + + test('creates a multiline env var for a .env file', () => { + const line = createDotEnvFileLine('FOO', 'BAR\nBAR\nBAR') + + expect(line).toEqual('FOO="BAR\nBAR\nBAR"') + }) + + test('creates a multiline env var for a .env file with double-quotes', () => { + const line = createDotEnvFileLine('FOO', 'BAR\n"BAR"\nBAR') + + expect(line).toEqual(`FOO='BAR\n"BAR"\nBAR'`) + }) + + test('creates a multiline env var for a .env file with double-quotes and single-quotes', () => { + const line = createDotEnvFileLine('FOO', `BAR\n"BAR"\n'BAR'`) + + expect(line).toEqual(`FOO=\`BAR\n"BAR"\n'BAR'\``) + }) + + test('throws AbortError when trying to create a multiline env var with single-quote, double-quote and tilde', async () => { + const value = `\`BAR\`\n"BAR"\n'BAR'` + await expect(async () => { + createDotEnvFileLine('FOO', value) + }).rejects.toThrow(`The environment file patch has an env value that can't be surrounded by quotes: ${value}`) + }) }) diff --git a/packages/cli-kit/src/public/node/dot-env.ts b/packages/cli-kit/src/public/node/dot-env.ts index eabc8ea56b..e16ededcfe 100644 --- a/packages/cli-kit/src/public/node/dot-env.ts +++ b/packages/cli-kit/src/public/node/dot-env.ts @@ -1,9 +1,7 @@ import {AbortError} from './error.js' import {fileExists, readFile, writeFile} from './fs.js' import {outputDebug, outputContent, outputToken} from '../../public/node/output.js' -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-ignore -import {parse, stringify} from 'envfile' +import {parse} from 'dotenv' /** * This interface represents a .env file. @@ -41,7 +39,11 @@ export async function readAndParseDotEnv(path: string): Promise { * @param file - .env file to be written. */ export async function writeDotEnv(file: DotEnvFile): Promise { - await writeFile(file.path, stringify(file.variables)) + const fileContent = Object.entries(file.variables) + .map(([key, value]) => createDotEnvFileLine(key, value)) + .join('\n') + + await writeFile(file.path, fileContent) } /** @@ -55,33 +57,90 @@ export function patchEnvFile( updatedValues: {[key: string]: string | undefined}, ): string { const outputLines: string[] = [] - const lines = envFileContent === null ? [] : envFileContent.split('\n') + const envFileLines = envFileContent === null ? [] : envFileContent.split('\n') const alreadyPresentKeys: string[] = [] - const toLine = (key: string, value?: string) => `${key}=${value}` + let multilineVariable: + | { + key: string + value: string + quote: string + } + | undefined + + for (const line of envFileLines) { + if (multilineVariable) { + if (line.endsWith(multilineVariable.quote)) { + let lineToWrite = createDotEnvFileLine( + multilineVariable.key, + multilineVariable.value + line.slice(0, -1), + multilineVariable.quote, + ) + const newValue = updatedValues[multilineVariable.key] + if (newValue) { + alreadyPresentKeys.push(multilineVariable.key) + lineToWrite = createDotEnvFileLine(multilineVariable.key, newValue) + } + outputLines.push(lineToWrite) + multilineVariable = undefined + } else { + multilineVariable.value += `${line}\n` + } + continue + } - for (const line of lines) { const match = line.match(/^([^=:#]+?)[=:](.*)/) let lineToWrite = line if (match) { const key = match[1]!.trim() + const value = (match[2] || '')!.trim() + + if (/^["'`]/.test(value) && !value.endsWith(value[0]!)) { + multilineVariable = { + key, + value: `${value.slice(1)}\n`, + quote: value[0]!, + } + continue + } + const newValue = updatedValues[key] if (newValue) { alreadyPresentKeys.push(key) - lineToWrite = toLine(key, newValue) + lineToWrite = createDotEnvFileLine(key, newValue) } } outputLines.push(lineToWrite) } + if (multilineVariable) { + throw new AbortError(`Multi-line environment variable '${multilineVariable.key}' is not properly enclosed.`) + } + for (const [patchKey, updatedValue] of Object.entries(updatedValues)) { if (!alreadyPresentKeys.includes(patchKey)) { - outputLines.push(toLine(patchKey, updatedValue)) + outputLines.push(createDotEnvFileLine(patchKey, updatedValue)) } } return outputLines.join('\n') } + +export function createDotEnvFileLine(key: string, value?: string, quote?: string): string { + if (quote) { + return `${key}=${quote}${value}${quote}` + } + if (value && value.includes('\n')) { + const quoteCharacter = ['"', "'", '`'].find((char) => !value.includes(char)) + + if (!quoteCharacter) { + throw new AbortError(`The environment file patch has an env value that can't be surrounded by quotes: ${value}`) + } + + return `${key}=${quoteCharacter}${value}${quoteCharacter}` + } + return `${key}=${value}` +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7d086715a7..d3d0b8f4bd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -384,12 +384,12 @@ importers: del: specifier: 6.1.1 version: 6.1.1 + dotenv: + specifier: 16.4.5 + version: 16.4.5 env-paths: specifier: 3.0.0 version: 3.0.0 - envfile: - specifier: 6.18.0 - version: 6.18.0 execa: specifier: 7.2.0 version: 7.2.0 @@ -552,10 +552,10 @@ importers: version: 5.2.2 vite: specifier: 4.4.9 - version: 4.4.9(@types/node@18.19.3)(sass@1.64.2) + version: 4.4.9(@types/node@18.19.3) vitest: specifier: ^0.34.3 - version: 0.34.4(jsdom@20.0.3) + version: 0.34.4 packages/create-app: dependencies: @@ -5534,7 +5534,7 @@ packages: istanbul-reports: 3.1.5 picocolors: 1.0.0 test-exclude: 6.0.0 - vitest: 0.34.4(jsdom@20.0.3) + vitest: 0.34.4 transitivePeerDependencies: - supports-color dev: true @@ -7406,6 +7406,11 @@ packages: engines: {node: '>=12'} dev: true + /dotenv@16.4.5: + resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==} + engines: {node: '>=12'} + dev: false + /duplexer@0.1.2: resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} dev: true @@ -7471,12 +7476,6 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: false - /envfile@6.18.0: - resolution: {integrity: sha512-IsYv64dtlNXTm4huvCBpbXsdZQurYUju9WoYCkSj+SDYpO3v4/dq346QsCnNZ3JcnWw0G3E6+saVkVtmPw98Gg==} - engines: {node: '>=10'} - hasBin: true - dev: false - /err-code@2.0.3: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} dev: true @@ -14019,6 +14018,28 @@ packages: replace-ext: 1.0.1 dev: true + /vite-node@0.34.4(@types/node@18.19.3): + resolution: {integrity: sha512-ho8HtiLc+nsmbwZMw8SlghESEE3KxJNp04F/jPUCLVvaURwt0d+r9LxEqCX5hvrrOQ0GSyxbYr5ZfRYhQ0yVKQ==} + engines: {node: '>=v14.18.0'} + hasBin: true + dependencies: + cac: 6.7.14 + debug: 4.3.4(supports-color@8.1.1) + mlly: 1.4.0 + pathe: 1.1.1 + picocolors: 1.0.0 + vite: 4.4.9(@types/node@18.19.3) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - stylus + - sugarss + - supports-color + - terser + dev: true + /vite-node@0.34.4(@types/node@18.19.3)(sass@1.64.2): resolution: {integrity: sha512-ho8HtiLc+nsmbwZMw8SlghESEE3KxJNp04F/jPUCLVvaURwt0d+r9LxEqCX5hvrrOQ0GSyxbYr5ZfRYhQ0yVKQ==} engines: {node: '>=v14.18.0'} @@ -14054,6 +14075,42 @@ packages: - supports-color dev: true + /vite@4.4.9(@types/node@18.19.3): + resolution: {integrity: sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==} + engines: {node: ^14.18.0 || >=16.0.0} + hasBin: true + peerDependencies: + '@types/node': '>= 14' + less: '*' + lightningcss: ^1.21.0 + sass: '*' + stylus: '*' + sugarss: '*' + terser: ^5.4.0 + peerDependenciesMeta: + '@types/node': + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + dependencies: + '@types/node': 18.19.3 + esbuild: 0.18.17 + postcss: 8.4.27 + rollup: 3.28.0 + optionalDependencies: + fsevents: 2.3.3 + dev: true + /vite@4.4.9(@types/node@18.19.3)(sass@1.64.2): resolution: {integrity: sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==} engines: {node: ^14.18.0 || >=16.0.0} @@ -14090,6 +14147,71 @@ packages: optionalDependencies: fsevents: 2.3.3 + /vitest@0.34.4: + resolution: {integrity: sha512-SE/laOsB6995QlbSE6BtkpXDeVNLJc1u2LHRG/OpnN4RsRzM3GQm4nm3PQCK5OBtrsUqnhzLdnT7se3aeNGdlw==} + engines: {node: '>=v14.18.0'} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@vitest/browser': '*' + '@vitest/ui': '*' + happy-dom: '*' + jsdom: '*' + playwright: '*' + safaridriver: '*' + webdriverio: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + playwright: + optional: true + safaridriver: + optional: true + webdriverio: + optional: true + dependencies: + '@types/chai': 4.3.5 + '@types/chai-subset': 1.3.3 + '@types/node': 18.19.3 + '@vitest/expect': 0.34.4 + '@vitest/runner': 0.34.4 + '@vitest/snapshot': 0.34.4 + '@vitest/spy': 0.34.4 + '@vitest/utils': 0.34.4 + acorn: 8.9.0 + acorn-walk: 8.2.0 + cac: 6.7.14 + chai: 4.3.7 + debug: 4.3.4(supports-color@8.1.1) + local-pkg: 0.4.3 + magic-string: 0.30.2 + pathe: 1.1.1 + picocolors: 1.0.0 + std-env: 3.3.3 + strip-literal: 1.0.1 + tinybench: 2.5.0 + tinypool: 0.7.0 + vite: 4.4.9(@types/node@18.19.3) + vite-node: 0.34.4(@types/node@18.19.3) + why-is-node-running: 2.2.2 + transitivePeerDependencies: + - less + - lightningcss + - sass + - stylus + - sugarss + - supports-color + - terser + dev: true + /vitest@0.34.4(jsdom@20.0.3): resolution: {integrity: sha512-SE/laOsB6995QlbSE6BtkpXDeVNLJc1u2LHRG/OpnN4RsRzM3GQm4nm3PQCK5OBtrsUqnhzLdnT7se3aeNGdlw==} engines: {node: '>=v14.18.0'}