Support simple variable substitution in .env files. (#4267)

(for #3275)

Notable:

* only simple substitution is supported (via `${NAME}`; no nesting)
* substitution can be made relative to the current env vars
* recursion (setting var relative to itself) is supported
* stopped using `dotenv` module (doesn't preserve order)
* any invalid substitution causes value to be left as-is
* adds the `ENVFILE_VARIABLE_SUBSTITUTION` telemetry event

.github/test_plan.md

@@ -83,6 +83,7 @@ print('Hello,', os.environ.get('WHO'), '!')
 # .env
 WHO=world
 PYTHONPATH=some/path/somewhere
+SPAM='hello ${WHO}'
 ````
 
 **ALWAYS**:
@@ -92,6 +93,7 @@ PYTHONPATH=some/path/somewhere
 - [ ] Environment variables in a `.env` file are exposed when running under the debugger
 - [ ] `"python.envFile"` allows for specifying an environment file manually (e.g. Jedi picks up `PYTHONPATH` changes)
 - [ ] `envFile` in a `launch.json` configuration works
+- [ ] simple variable substitution works
 
 #### [Debugging](https://code.visualstudio.com/docs/python/environments#_python-interpreter-for-debugging)
 

news/1 Enhancements/3275.md

@@ -0,0 +1 @@
+Support simple variable substitution in .env files.

package.json

@@ -1920,7 +1920,6 @@
         "arch": "^2.1.0",
         "azure-storage": "^2.10.1",
         "diff-match-patch": "^1.0.0",
-        "dotenv": "^5.0.1",
         "file-matcher": "^1.3.0",
         "fs-extra": "^4.0.3",
         "fuzzy": "^0.1.3",
@@ -1974,7 +1973,6 @@
         "@types/copy-webpack-plugin": "^4.4.2",
         "@types/del": "^3.0.0",
         "@types/diff-match-patch": "^1.0.32",
-        "@types/dotenv": "^4.0.3",
         "@types/download": "^6.2.2",
         "@types/enzyme": "^3.1.14",
         "@types/enzyme-adapter-react-16": "^1.0.3",

src/client/common/variables/environment.ts

@@ -1,10 +1,11 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-import * as dotenv from 'dotenv';
 import * as fs from 'fs-extra';
 import { inject, injectable } from 'inversify';
 import * as path from 'path';
+import { sendTelemetryEvent } from '../../telemetry';
+import { EventName } from '../../telemetry/constants';
 import { IPathUtils } from '../types';
 import { EnvironmentVariables, IEnvironmentVariablesService } from './types';
 
@@ -14,14 +15,14 @@ export class EnvironmentVariablesService implements IEnvironmentVariablesService
     constructor(@inject(IPathUtils) pathUtils: IPathUtils) {
         this.pathVariable = pathUtils.getPathVariableName();
     }
-    public async parseFile(filePath?: string): Promise<EnvironmentVariables | undefined> {
+    public async parseFile(filePath?: string, baseVars?: EnvironmentVariables): Promise<EnvironmentVariables | undefined> {
         if (!filePath || !await fs.pathExists(filePath)) {
             return;
         }
         if (!fs.lstatSync(filePath).isFile()) {
             return;
         }
-        return dotenv.parse(await fs.readFile(filePath));
+        return parseEnvFile(await fs.readFile(filePath), baseVars);
     }
     public mergeVariables(source: EnvironmentVariables, target: EnvironmentVariables) {
         if (!target) {
@@ -61,3 +62,77 @@ export class EnvironmentVariablesService implements IEnvironmentVariablesService
         return vars;
     }
 }
+
+export function parseEnvFile(
+    lines: string | Buffer,
+    baseVars?: EnvironmentVariables
+): EnvironmentVariables {
+    const globalVars = baseVars ? baseVars : {};
+    const vars = {};
+    lines.toString().split('\n').forEach((line, idx) => {
+        const [name, value] = parseEnvLine(line);
+        if (name === '') {
+            return;
+        }
+        vars[name] = substituteEnvVars(value, vars, globalVars);
+    });
+    return vars;
+}
+
+function parseEnvLine(line: string): [string, string] {
+    // Most of the following is an adaptation of the dotenv code:
+    //   https://github.com/motdotla/dotenv/blob/master/lib/main.js#L32
+    // We don't use dotenv here because it loses ordering, which is
+    // significant for substitution.
+    const match = line.match(/^\s*([a-zA-Z]\w*)\s*=\s*(.*?)?\s*$/);
+    if (!match) {
+        return ['', ''];
+    }
+
+    const name = match[1];
+    let value = match[2];
+    if (value && value !== '') {
+        if (value[0] === '\'' && value[value.length - 1] === '\'') {
+            value = value.substring(1, value.length - 1);
+            value = value.replace(/\\n/gm, '\n');
+        } else if (value[0] === '"' && value[value.length - 1] === '"') {
+            value = value.substring(1, value.length - 1);
+            value = value.replace(/\\n/gm, '\n');
+        }
+    } else {
+        value = '';
+    }
+
+    return [name, value];
+}
+
+const SUBST_REGEX = /\${([a-zA-Z]\w*)?([^}\w].*)?}/g;
+
+function substituteEnvVars(
+    value: string,
+    localVars: EnvironmentVariables,
+    globalVars: EnvironmentVariables,
+    missing = ''
+): string {
+    // Substitution here is inspired a little by dotenv-expand:
+    //   https://github.com/motdotla/dotenv-expand/blob/master/lib/main.js
+
+    let invalid = false;
+    let replacement = value;
+    replacement = replacement.replace(SUBST_REGEX, (match, substName, bogus, offset, orig) => {
+        if (offset > 0 && orig[offset - 1] === '\\') {
+            return match;
+        }
+        if ((bogus && bogus !== '') || !substName || substName === '') {
+            invalid = true;
+            return match;
+        }
+        return localVars[substName] || globalVars[substName] || missing;
+    });
+    if (!invalid && replacement !== value) {
+        value = replacement;
+        sendTelemetryEvent(EventName.ENVFILE_VARIABLE_SUBSTITUTION);
+    }
+
+    return value.replace(/\\\$/g, '$');
+}

src/client/common/variables/environmentVariablesProvider.ts

@@ -44,7 +44,7 @@ export class EnvironmentVariablesProvider implements IEnvironmentVariablesProvid
         const workspaceFolderUri = this.getWorkspaceFolderUri(resource);
         this.trackedWorkspaceFolders.add(workspaceFolderUri ? workspaceFolderUri.fsPath : '');
         this.createFileWatcher(settings.envFile, workspaceFolderUri);
-        let mergedVars = await this.envVarsService.parseFile(settings.envFile);
+        let mergedVars = await this.envVarsService.parseFile(settings.envFile, this.process.env);
         if (!mergedVars) {
             mergedVars = {};
         }

src/client/common/variables/types.ts

@@ -8,7 +8,7 @@ export type EnvironmentVariables = Object & Record<string, string | undefined>;
 export const IEnvironmentVariablesService = Symbol('IEnvironmentVariablesService');
 
 export interface IEnvironmentVariablesService {
-    parseFile(filePath?: string): Promise<EnvironmentVariables | undefined>;
+    parseFile(filePath?: string, baseVars?: EnvironmentVariables): Promise<EnvironmentVariables | undefined>;
     mergeVariables(source: EnvironmentVariables, target: EnvironmentVariables): void;
     appendPythonPath(vars: EnvironmentVariables, ...pythonPaths: string[]): void;
     appendPath(vars: EnvironmentVariables, ...paths: string[]): void;

src/client/debugger/debugAdapter/DebugClients/helper.ts

@@ -9,9 +9,9 @@ export class DebugClientHelper {
         const pathVariableName = this.pathUtils.getPathVariableName();
 
         // Merge variables from both .env file and env json variables.
-        const envFileVars = await this.envParser.parseFile(args.envFile);
         // tslint:disable-next-line:no-any
         const debugLaunchEnvVars: Record<string, string> = (args.env && Object.keys(args.env).length > 0) ? { ...args.env } as any : {} as any;
+        const envFileVars = await this.envParser.parseFile(args.envFile, debugLaunchEnvVars);
         const env = envFileVars ? { ...envFileVars! } : {};
         this.envParser.mergeVariables(debugLaunchEnvVars, env);
 

src/client/telemetry/constants.ts

@@ -28,6 +28,7 @@ export enum EventName {
     PYTHON_INTERPRETER_ACTIVATION_ENVIRONMENT_VARIABLES = 'PYTHON_INTERPRETER_ACTIVATION_ENVIRONMENT_VARIABLES',
     PYTHON_INTERPRETER_ACTIVATION_FOR_RUNNING_CODE = 'PYTHON_INTERPRETER_ACTIVATION_FOR_RUNNING_CODE',
     PYTHON_INTERPRETER_ACTIVATION_FOR_TERMINAL = 'PYTHON_INTERPRETER_ACTIVATION_FOR_TERMINAL',
+    ENVFILE_VARIABLE_SUBSTITUTION = 'ENVFILE_VARIABLE_SUBSTITUTION',
     WORKSPACE_SYMBOLS_BUILD = 'WORKSPACE_SYMBOLS.BUILD',
     WORKSPACE_SYMBOLS_GO_TO = 'WORKSPACE_SYMBOLS.GO_TO',
     EXECUTION_CODE = 'EXECUTION_CODE',

src/client/telemetry/index.ts

@@ -253,6 +253,7 @@ interface IEventNamePropertyMapping {
     [EventName.DIAGNOSTICS_ACTION]: DiagnosticsAction;
     [EventName.DIAGNOSTICS_MESSAGE]: DiagnosticsMessages;
     [EventName.EDITOR_LOAD]: EditorLoadTelemetry;
+    [EventName.ENVFILE_VARIABLE_SUBSTITUTION]: never | undefined;
     [EventName.EXECUTION_CODE]: CodeExecutionTelemetry;
     [EventName.EXECUTION_DJANGO]: CodeExecutionTelemetry;
     [EventName.FORMAT]: FormatTelemetry;