Fix backslash escape, multiple linters output, .pyenv/versions folder search

src/client/common/markdown/restTextConverter.ts

@@ -36,7 +36,8 @@ export class RestTextConverter {
  return text
  .replace(/\#/g, '\\#')
  .replace(/\*/g, '\\*')
- .replace(/\_/g, '\\_');
+ .replace(/\ _/g, ' \\_')
+ .replace(/^_/, '\\_');
  }
 
  private transformLines(docstring: string): string {

src/client/interpreter/locators/services/globalVirtualEnvService.ts

@@ -42,9 +42,11 @@ export class GlobalVirtualEnvironmentsSearchPathProvider implements IVirtualEnvi
  folders.push(pyenvRoot);
  folders.push(path.join(pyenvRoot, 'versions'));
  } else {
+ // Check if .pyenv/versions is in the list
  const pyenvVersions = path.join('.pyenv', 'versions');
  if (venvFolders.indexOf('.pyenv') >= 0 && venvFolders.indexOf(pyenvVersions) < 0) {
- folders.push(pyenvVersions);
+ // if .pyenv is in the list, but .pyenv/versions is not, add it.
+ folders.push(path.join(homedir, pyenvVersions));
  }
  }
  return folders;

src/client/linters/linterCommands.ts

@@ -79,9 +79,9 @@ export class LinterCommands implements vscode.Disposable {
  }
  }
 
- public runLinting(): void {
+ public runLinting(): Promise<vscode.DiagnosticCollection> {
  const engine = this.serviceContainer.get<ILintingEngine>(ILintingEngine);
- engine.lintOpenPythonFiles();
+ return engine.lintOpenPythonFiles();
  }
 
  private get settingsUri(): vscode.Uri | undefined {

src/client/linters/lintingEngine.ts

@@ -7,6 +7,7 @@ import * as path from 'path';
 import * as vscode from 'vscode';
 import { IDocumentManager, IWorkspaceService } from '../common/application/types';
 import { LinterErrors, PythonLanguage, STANDARD_OUTPUT_CHANNEL } from '../common/constants';
+import { IFileSystem } from '../common/platform/types';
 import { IConfigurationService, IOutputChannel } from '../common/types';
 import { IServiceContainer } from '../ioc/types';
 import { JupyterProvider } from '../jupyter/provider';
@@ -40,6 +41,7 @@ export class LintingEngine implements ILintingEngine {
  private diagnosticCollection: vscode.DiagnosticCollection;
  private pendingLintings = new Map<string, vscode.CancellationTokenSource>();
  private outputChannel: vscode.OutputChannel;
+ private fileSystem: IFileSystem;
 
  constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer) {
  this.documentHasJupyterCodeCells = (a, b) => Promise.resolve(false);
@@ -48,35 +50,23 @@ export class LintingEngine implements ILintingEngine {
  this.configurationService = serviceContainer.get<IConfigurationService>(IConfigurationService);
  this.outputChannel = serviceContainer.get<vscode.OutputChannel>(IOutputChannel, STANDARD_OUTPUT_CHANNEL);
  this.linterManager = serviceContainer.get<ILinterManager>(ILinterManager);
+ this.fileSystem = serviceContainer.get<IFileSystem>(IFileSystem);
  this.diagnosticCollection = vscode.languages.createDiagnosticCollection('python');
  }
 
- public lintOpenPythonFiles(): void {
- this.documents.textDocuments.forEach(async document => {
+ public async lintOpenPythonFiles(): Promise<vscode.DiagnosticCollection> {
+ this.documents.textDocuments.forEach(document => {
  if (document.languageId === PythonLanguage.language) {
- await this.lintDocument(document, 'auto');
+ this.lintDocument(document, 'auto').ignoreErrors();
  }
  });
+ return this.diagnosticCollection;
  }
 
- public async lintDocument(document: vscode.TextDocument, trigger: LinterTrigger): Promise<void> {
+ public async lintDocument(document: vscode.TextDocument, trigger: LinterTrigger): Promise<vscode.DiagnosticCollection> {
  // Check if we need to lint this document
- const workspaceFolder = this.workspace.getWorkspaceFolder(document.uri);
- const workspaceRootPath = (workspaceFolder && typeof workspaceFolder.uri.fsPath === 'string') ? workspaceFolder.uri.fsPath : undefined;
- const relativeFileName = typeof workspaceRootPath === 'string' ? path.relative(workspaceRootPath, document.fileName) : document.fileName;
- const settings = this.configurationService.getSettings(document.uri);
- if (document.languageId !== PythonLanguage.language) {
- return;
- }
- if (!this.linterManager.isLintingEnabled(document.uri)) {
- this.diagnosticCollection.set(document.uri, []);
- }
- const ignoreMinmatches = settings.linting.ignorePatterns.map(pattern => {
- return new Minimatch(pattern);
- });
-
- if (ignoreMinmatches.some(matcher => matcher.match(document.fileName) || matcher.match(relativeFileName))) {
- return;
+ if (!await this.shouldLintDocument(document)) {
+ return this.diagnosticCollection;
  }
 
  if (this.pendingLintings.has(document.uri.fsPath)) {
@@ -107,14 +97,14 @@ export class LintingEngine implements ILintingEngine {
  // linters will resolve asynchronously - keep a track of all
  // diagnostics reported as them come in.
  let diagnostics: vscode.Diagnostic[] = [];
+ const settings = this.configurationService.getSettings(document.uri);
 
  for (const p of promises) {
  const msgs = await p;
  if (cancelToken.token.isCancellationRequested) {
  break;
  }
 
- diagnostics = [];
  if (this.isDocumentOpen(document.uri)) {
  // Build the message and suffix the message with the name of the linter used.
  for (const m of msgs) {
@@ -123,17 +113,17 @@ export class LintingEngine implements ILintingEngine {
  (m.code === LinterErrors.pylint.InvalidSyntax ||
  m.code === LinterErrors.prospector.InvalidSyntax ||
  m.code === LinterErrors.flake8.InvalidSyntax)) {
- return;
+ continue;
  }
  diagnostics.push(this.createDiagnostics(m, document));
  }
-
  // Limit the number of messages to the max value.
  diagnostics = diagnostics.filter((value, index) => index <= settings.linting.maxNumberOfProblems);
  }
- // Set all diagnostics found in this pass, as this method always clears existing diagnostics.
- this.diagnosticCollection.set(document.uri, diagnostics);
  }
+ // Set all diagnostics found in this pass, as this method always clears existing diagnostics.
+ this.diagnosticCollection.set(document.uri, diagnostics);
+ return this.diagnosticCollection;
  }
 
  // tslint:disable-next-line:no-any
@@ -175,4 +165,29 @@ export class LintingEngine implements ILintingEngine {
  diagnostic.source = message.provider;
  return diagnostic;
  }
+
+ private async shouldLintDocument(document: vscode.TextDocument): Promise<boolean> {
+ if (!this.linterManager.isLintingEnabled(document.uri)) {
+ this.diagnosticCollection.set(document.uri, []);
+ return false;
+ }
+
+ if (document.languageId !== PYTHON.language) {
+ return false;
+ }
+
+ const workspaceFolder = this.workspace.getWorkspaceFolder(document.uri);
+ const workspaceRootPath = (workspaceFolder && typeof workspaceFolder.uri.fsPath === 'string') ? workspaceFolder.uri.fsPath : undefined;
+ const relativeFileName = typeof workspaceRootPath === 'string' ? path.relative(workspaceRootPath, document.fileName) : document.fileName;
+
+ const settings = this.configurationService.getSettings(document.uri);
+ const ignoreMinmatches = settings.linting.ignorePatterns.map(pattern => new Minimatch(pattern));
+ if (ignoreMinmatches.some(matcher => matcher.match(document.fileName) || matcher.match(relativeFileName))) {
+ return false;
+ }
+ if (document.uri.scheme !== 'file' || !document.uri.fsPath) {
+ return false;
+ }
+ return await this.fileSystem.fileExistsAsync(document.uri.fsPath);
+ }
 }

src/client/linters/types.ts

@@ -61,8 +61,8 @@ export enum LintMessageSeverity {
 
 export const ILintingEngine = Symbol('ILintingEngine');
 export interface ILintingEngine {
- lintOpenPythonFiles(): void;
- lintDocument(document: vscode.TextDocument, trigger: LinterTrigger): Promise<void>;
+ lintOpenPythonFiles(): Promise<vscode.DiagnosticCollection>;
+ lintDocument(document: vscode.TextDocument, trigger: LinterTrigger): Promise<vscode.DiagnosticCollection>;
  // tslint:disable-next-line:no-any
  linkJupiterExtension(jupiter: vscode.Extension<any> | undefined): Promise<void>;
 }

src/client/providers/linterProvider.ts

@@ -6,14 +6,13 @@ import * as vscode from 'vscode';
 import { ConfigurationTarget, Uri, workspace } from 'vscode';
 import { IDocumentManager } from '../common/application/types';
 import { ConfigSettingMonitor } from '../common/configSettingMonitor';
+import { isTestExecution } from '../common/constants';
 import { IFileSystem } from '../common/platform/types';
 import { IConfigurationService } from '../common/types';
 import { IInterpreterService } from '../interpreter/contracts';
 import { IServiceContainer } from '../ioc/types';
 import { ILinterManager, ILintingEngine } from '../linters/types';
 
-const uriSchemesToIgnore = ['git', 'showModifications', 'svn'];
-
 export class LinterProvider implements vscode.Disposable {
  private diagnosticCollection: vscode.DiagnosticCollection;
  private context: vscode.ExtensionContext;
@@ -46,6 +45,12 @@ export class LinterProvider implements vscode.Disposable {
 
  this.configMonitor = new ConfigSettingMonitor('linting');
  this.configMonitor.on('change', this.lintSettingsChangedHandler.bind(this));
+
+ // On workspace reopen we don't get `onDocumentOpened` since it is first opened
+ // and then the extension is activated. So schedule linting pass now.
+ if (!isTestExecution) {
+ setTimeout(() => this.engine.lintOpenPythonFiles().ignoreErrors(), 2000);
+ }
  }
 
  public get diagnostics(): vscode.DiagnosticCollection {
@@ -63,35 +68,23 @@ export class LinterProvider implements vscode.Disposable {
 
  private lintSettingsChangedHandler(configTarget: ConfigurationTarget, wkspaceOrFolder: Uri) {
  if (configTarget === ConfigurationTarget.Workspace) {
- this.engine.lintOpenPythonFiles();
+ this.engine.lintOpenPythonFiles().ignoreErrors();
  return;
  }
  // Look for python files that belong to the specified workspace folder.
  workspace.textDocuments.forEach(async document => {
  const wkspaceFolder = workspace.getWorkspaceFolder(document.uri);
  if (wkspaceFolder && wkspaceFolder.uri.fsPath === wkspaceOrFolder.fsPath) {
- await this.engine.lintDocument(document, 'auto');
+ this.engine.lintDocument(document, 'auto').ignoreErrors();
  }
  });
  }
 
- private async onDocumentOpened(document: vscode.TextDocument): Promise<void> {
- const settings = this.configuration.getSettings(document.uri);
- if (document.languageId !== 'python' || !settings.linting.enabled) {
- return;
- }
- // Exclude files opened by vscode when showing a diff view.
- if (uriSchemesToIgnore.indexOf(document.uri.scheme) >= 0) {
- return;
- }
- if (!document.uri.path ||
- (path.basename(document.uri.path) === document.uri.path && !await this.fs.fileExistsAsync(document.uri.path))) {
- return;
- }
+ private onDocumentOpened(document: vscode.TextDocument): void {
  this.engine.lintDocument(document, 'auto').ignoreErrors();
  }
 
- private onDocumentSaved(document: vscode.TextDocument) {
+ private onDocumentSaved(document: vscode.TextDocument): void {
  const settings = this.configuration.getSettings(document.uri);
  if (document.languageId === 'python' && settings.linting.enabled && settings.linting.lintOnSave) {
  this.engine.lintDocument(document, 'save').ignoreErrors();

src/test/interpreters/venv.test.ts

@@ -49,7 +49,7 @@ suite('Virtual environments', () => {
 
  let paths = pathProvider.getSearchPaths();
  let expected = folders.map(item => path.join(homedir, item));
- expected.push(path.join('.pyenv', 'versions'));
+ expected.push(path.join(homedir, '.pyenv', 'versions'));
 
  expect(paths).to.deep.equal(expected, 'Global search folder list is incorrect.');
 

src/test/linters/lint.test.ts

@@ -8,7 +8,7 @@ import { Product } from '../../client/common/installer/productInstaller';
 import { IConfigurationService, IOutputChannel } from '../../client/common/types';
 import { LinterManager } from '../../client/linters/linterManager';
 import { ILinterManager, ILintMessage, LintMessageSeverity } from '../../client/linters/types';
-import { deleteFile, PythonSettingKeys, rootWorkspaceUri } from '../common';
+import { deleteFile, PythonSettingKeys, rootWorkspaceUri, sleep } from '../common';
 import { closeActiveWindows, initialize, initializeTest, IS_MULTI_ROOT_TEST } from '../initialize';
 import { MockOutputChannel } from '../mockClasses';
 import { UnitTestIocContainer } from '../unittests/serviceRegistry';
@@ -99,7 +99,7 @@ suite('Linting', () => {
 
  suiteSetup(initialize);
  setup(async () => {
- initializeDI();
+ await initializeDI();
  await initializeTest();
  await resetSettings();
  });
@@ -112,7 +112,7 @@ suite('Linting', () => {
  await deleteFile(path.join(workspaceUri.fsPath, '.pydocstyle'));
  });
 
- function initializeDI() {
+ async function initializeDI() {
  ioc = new UnitTestIocContainer();
  ioc.registerCommonTypes(false);
  ioc.registerProcessTypes();
@@ -122,6 +122,7 @@ suite('Linting', () => {
 
  linterManager = new LinterManager(ioc.serviceContainer);
  configService = ioc.serviceContainer.get<IConfigurationService>(IConfigurationService);
+ await linterManager.enableLintingAsync(true);
  }
 
  async function resetSettings() {
@@ -250,4 +251,29 @@ suite('Linting', () => {
  await configService.updateSettingAsync('linting.pylintUseMinimalCheckers', false, workspaceUri);
  await testEnablingDisablingOfLinter(Product.pylint, true, file);
  });
+ test('Multiple linters', async () => {
+ await linterManager.setActiveLintersAsync([Product.pylint, Product.flake8]);
+
+ const document = await vscode.workspace.openTextDocument(path.join(pythoFilesPath, 'print.py'));
+ const collection = await vscode.commands.executeCommand('python.runLinting') as vscode.DiagnosticCollection;
+ assert.notEqual(collection, undefined, 'python.runLinting did not return valid diagnostics collection.');
+
+ const ready = await waitForCondition(() => collection!.has(document.uri) && collection!.get(document.uri)!.length >= 3);
+ assert.equal(ready, true, 'Timeout expired but linting results are not available still.');
+
+ const messages = collection!.get(document.uri);
+ assert.notEqual(messages!.filter(x => x.source === 'pylint').length, 0, 'No pylint messages.');
+ assert.notEqual(messages!.filter(x => x.source === 'flake8').length, 0, 'No flake8 messages.');
+ });
+
+ async function waitForCondition(predicate: () => boolean, interval = 1000, maxAttempts = 15): Promise<boolean> {
+ return new Promise<boolean>(async (resolve) => {
+ let retries = 0;
+ while (!predicate() && retries < maxAttempts) {
+ await sleep(1000);
+ retries += 1;
+ }
+ resolve(retries < maxAttempts);
+ });
+ }
 });