Skip to content

Commit

Permalink
includes 3 changes.
Browse files Browse the repository at this point in the history 
1. added a hook to ImportResolver for custom importing logics.
2. added an ability to add library code directly in fourslash test file
3. added an ability to mount different folder as typeshed folder in virtual file system in fourslash test
4. added an ability to mount multiple folders in virtual file system
  • Loading branch information
@heejaechang
heejaechang committed Feb 13, 2020
1 parent de41104 commit 00db9d6
Show file tree
Hide file tree
Showing 20 changed files with 333 additions and 139 deletions.
4 changes: 2 additions & 2 deletions server/.vscode/launch.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@
"windows": {
"program": "${workspaceFolder}/node_modules/jest/bin/jest",
}
}
},
{
"type": "node",
"name": "fourslash current file",
"request": "launch",
"args": [
"fourslashrunner.test.ts",
"fourSlashRunner.test.ts",
"-t ${fileBasenameNoExtension}",
"--config",
"jest.config.js"
Expand Down
38 changes: 29 additions & 9 deletions server/src/analyzer/importResolver.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ import {
} from '../common/pathUtils';
import { versionToString } from '../common/pythonVersion';
import * as StringUtils from '../common/stringUtils';
import { VirtualFileSystem } from '../common/vfs';
import { ImplicitImport, ImportResult, ImportType } from './importResult';
import * as PythonPathUtils from './pythonPathUtils';
import { isDunderName } from './symbolNameUtils';
import { VirtualFileSystem } from '../common/vfs';

export interface ImportedModuleDescriptor {
leadingDots: number;
Expand Down Expand Up @@ -95,7 +95,7 @@ export class ImportResolver {
// Look for it in the root directory of the execution environment.
importFailureInfo.push(`Looking in root directory of execution environment ` +
`'${ execEnv.root }'`);
let localImport = this._resolveAbsoluteImport(
let localImport = this.resolveAbsoluteImport(
execEnv.root, moduleDescriptor, importName, importFailureInfo);
if (localImport && localImport.isImportFound) {
return this._addResultsToCache(execEnv, importName, localImport,
Expand All @@ -105,7 +105,7 @@ export class ImportResolver {

for (const extraPath of execEnv.extraPaths) {
importFailureInfo.push(`Looking in extraPath '${ extraPath }'`);
localImport = this._resolveAbsoluteImport(extraPath, moduleDescriptor,
localImport = this.resolveAbsoluteImport(extraPath, moduleDescriptor,
importName, importFailureInfo);
if (localImport && localImport.isImportFound) {
return this._addResultsToCache(execEnv, importName, localImport,
Expand All @@ -121,7 +121,7 @@ export class ImportResolver {
// Check for a typings file.
if (this._configOptions.typingsPath) {
importFailureInfo.push(`Looking in typingsPath '${ this._configOptions.typingsPath }'`);
const typingsImport = this._resolveAbsoluteImport(
const typingsImport = this.resolveAbsoluteImport(
this._configOptions.typingsPath, moduleDescriptor, importName, importFailureInfo);
if (typingsImport && typingsImport.isImportFound) {
// We will treat typings files as "local" rather than "third party".
Expand Down Expand Up @@ -149,17 +149,20 @@ export class ImportResolver {
// Allow partial resolution because some third-party packages
// use tricks to populate their package namespaces.
importFailureInfo.push(`Looking in python search path '${ searchPath }'`);
const thirdPartyImport = this._resolveAbsoluteImport(
const thirdPartyImport = this.resolveAbsoluteImport(
searchPath, moduleDescriptor, importName, importFailureInfo,
true, true, true);
if (thirdPartyImport) {
thirdPartyImport.importType = ImportType.ThirdParty;

if (thirdPartyImport.isImportFound) {
if (thirdPartyImport.isImportFound && thirdPartyImport.isStubFile) {
return this._addResultsToCache(execEnv, importName,
thirdPartyImport, moduleDescriptor.importedSymbols);
}

// We did not find it, or we did and it's not from a
// stub, so give chance for resolveImportEx to find
// one from a stub.
if (bestResultSoFar === undefined ||
thirdPartyImport.resolvedPaths.length > bestResultSoFar.resolvedPaths.length) {
bestResultSoFar = thirdPartyImport;
Expand All @@ -170,6 +173,12 @@ export class ImportResolver {
importFailureInfo.push('No python interpreter search path');
}

const extraResults = this.resolveImportEx(sourceFilePath, execEnv, moduleDescriptor, importName, importFailureInfo);
if (extraResults !== undefined) {
return this._addResultsToCache(execEnv, importName, extraResults,
moduleDescriptor.importedSymbols);
}

// We weren't able to find an exact match, so return the best
// partial match.
if (bestResultSoFar) {
Expand All @@ -193,6 +202,15 @@ export class ImportResolver {
return this._addResultsToCache(execEnv, importName, notFoundResult, undefined);
}

// Intended to be overridden by subclasses to provide additional stub
// resolving capabilities. Return undefined if no stubs were found for
// this import.
protected resolveImportEx(sourceFilePath: string, execEnv: ExecutionEnvironment,
moduleDescriptor: ImportedModuleDescriptor, importName: string,
importFailureInfo: string[] = []): ImportResult | undefined {
return undefined;
}

getCompletionSuggestions(sourceFilePath: string, execEnv: ExecutionEnvironment,
moduleDescriptor: ImportedModuleDescriptor, similarityLimit: number): string[] {

Expand Down Expand Up @@ -414,7 +432,7 @@ export class ImportResolver {
minorVersion === 0 ? '3' : '2and3';
const testPath = combinePaths(typeshedPath, pythonVersionString);
if (this.fileSystem.existsSync(testPath)) {
const importInfo = this._resolveAbsoluteImport(testPath, moduleDescriptor,
const importInfo = this.resolveAbsoluteImport(testPath, moduleDescriptor,
importName, importFailureInfo);
if (importInfo && importInfo.isImportFound) {
importInfo.importType = isStdLib ? ImportType.BuiltIn : ImportType.ThirdParty;
Expand Down Expand Up @@ -536,7 +554,7 @@ export class ImportResolver {
}

// Now try to match the module parts from the current directory location.
const absImport = this._resolveAbsoluteImport(curDir, moduleDescriptor,
const absImport = this.resolveAbsoluteImport(curDir, moduleDescriptor,
importName, importFailureInfo);
if (!absImport) {
return undefined;
Expand Down Expand Up @@ -565,7 +583,7 @@ export class ImportResolver {

// Follows import resolution algorithm defined in PEP-420:
// https://www.python.org/dev/peps/pep-0420/
private _resolveAbsoluteImport(rootPath: string, moduleDescriptor: ImportedModuleDescriptor,
protected resolveAbsoluteImport(rootPath: string, moduleDescriptor: ImportedModuleDescriptor,
importName: string, importFailureInfo: string[], allowPartial = false,
allowPydFile = false, allowStubsFolder = false): ImportResult | undefined {

Expand Down Expand Up @@ -893,3 +911,5 @@ export class ImportResolver {
return name + moduleDescriptor.nameParts.map(part => part).join('.');
}
}

export type ImportResolverFactory = (fs: VirtualFileSystem, options: ConfigOptions) => ImportResolver;
3 changes: 1 addition & 2 deletions server/src/analyzer/program.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
* An object that tracks all of the source files being analyzed
* and all of their recursive imports.
*/

import * as assert from 'assert';
import { CompletionItem, CompletionList, DocumentSymbol, SymbolInformation } from 'vscode-languageserver';

Expand All @@ -20,6 +19,7 @@ import {
combinePaths, getDirectoryPath, getRelativePath, makeDirectories,
normalizePath, stripFileExtension
} from '../common/pathUtils';
import { DocumentRange, doRangesOverlap, Position, Range } from '../common/textRange';
import { Duration, timingStats } from '../common/timing';
import { ModuleSymbolMap } from '../languageService/completionProvider';
import { HoverResults } from '../languageService/hoverProvider';
Expand All @@ -34,7 +34,6 @@ import { SourceFile } from './sourceFile';
import { SymbolTable } from './symbol';
import { createTypeEvaluator, TypeEvaluator } from './typeEvaluator';
import { TypeStubWriter } from './typeStubWriter';
import { Position, Range, DocumentRange, doRangesOverlap } from '../common/textRange';

const _maxImportDepth = 256;

Expand Down
17 changes: 9 additions & 8 deletions server/src/analyzer/pythonPathUtils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import * as child_process from 'child_process';
import { ConfigOptions } from '../common/configOptions';
import * as consts from '../common/consts';
import {
combinePaths, ensureTrailingDirectorySeparator, getDirectoryPath,
getFileSystemEntries, isDirectory, normalizePath
Expand All @@ -22,7 +23,7 @@ export function getTypeShedFallbackPath(moduleDirectory?: string) {
moduleDirectory = normalizePath(moduleDirectory);
return combinePaths(getDirectoryPath(
ensureTrailingDirectorySeparator(moduleDirectory)),
'typeshed-fallback');
consts.TYPESHED_FALLBACK);
}

return undefined;
Expand All @@ -49,22 +50,22 @@ export function findPythonSearchPaths(fs: VirtualFileSystem, configOptions: Conf
}

if (venvPath) {
let libPath = combinePaths(venvPath, 'lib');
let libPath = combinePaths(venvPath, consts.LIB);
if (fs.existsSync(libPath)) {
importFailureInfo.push(`Found path '${ libPath }'; looking for site-packages`);
importFailureInfo.push(`Found path '${ libPath }'; looking for ${ consts.SITE_PACKAGES }`);
} else {
importFailureInfo.push(`Did not find '${ libPath }'; trying 'Lib' instead`);
libPath = combinePaths(venvPath, 'Lib');
if (fs.existsSync(libPath)) {
importFailureInfo.push(`Found path '${ libPath }'; looking for site-packages`);
importFailureInfo.push(`Found path '${ libPath }'; looking for ${ consts.SITE_PACKAGES }`);
} else {
importFailureInfo.push(`Did not find '${ libPath }'`);
libPath = '';
}
}

if (libPath) {
const sitePackagesPath = combinePaths(libPath, 'site-packages');
const sitePackagesPath = combinePaths(libPath, consts.SITE_PACKAGES);
if (fs.existsSync(sitePackagesPath)) {
importFailureInfo.push(`Found path '${ sitePackagesPath }'`);
return [sitePackagesPath];
Expand All @@ -74,11 +75,11 @@ export function findPythonSearchPaths(fs: VirtualFileSystem, configOptions: Conf

// We didn't find a site-packages directory directly in the lib
// directory. Scan for a "python*" directory instead.
const entries = getFileSystemEntries(this._fs, libPath);
const entries = getFileSystemEntries(fs, libPath);
for (let i = 0; i < entries.directories.length; i++) {
const dirName = entries.directories[i];
if (dirName.startsWith('python')) {
const dirPath = combinePaths(libPath, dirName, 'site-packages');
const dirPath = combinePaths(libPath, dirName, consts.SITE_PACKAGES);
if (fs.existsSync(dirPath)) {
importFailureInfo.push(`Found path '${ dirPath }'`);
return [dirPath];
Expand All @@ -89,7 +90,7 @@ export function findPythonSearchPaths(fs: VirtualFileSystem, configOptions: Conf
}
}

importFailureInfo.push(`Did not find site-packages. Falling back on python interpreter.`);
importFailureInfo.push(`Did not find '${ consts.SITE_PACKAGES }'. Falling back on python interpreter.`);
}

// Fall back on the python interpreter.
Expand Down
14 changes: 10 additions & 4 deletions server/src/analyzer/service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import { Duration, timingStats } from '../common/timing';
import { FileWatcher, VirtualFileSystem } from '../common/vfs';
import { HoverResults } from '../languageService/hoverProvider';
import { SignatureHelpResults } from '../languageService/signatureHelpProvider';
import { ImportedModuleDescriptor, ImportResolver } from './importResolver';
import { ImportedModuleDescriptor, ImportResolver, ImportResolverFactory } from './importResolver';
import { MaxAnalysisTime, Program } from './program';
import * as PythonPathUtils from './pythonPathUtils';

Expand All @@ -50,6 +50,7 @@ export class AnalyzerService {
private _instanceName: string;
private _program: Program;
private _configOptions: ConfigOptions;
private _importResolverFactory: ImportResolverFactory;
private _importResolver: ImportResolver;
private _executionRootPath: string;
private _typeStubTargetImportName: string | undefined;
Expand All @@ -68,11 +69,12 @@ export class AnalyzerService {
private _requireTrackedFileUpdate = true;
private _lastUserInteractionTime = Date.now();

constructor(instanceName: string, fs: VirtualFileSystem, console?: ConsoleInterface, configOptions?: ConfigOptions) {
constructor(instanceName: string, fs: VirtualFileSystem, console?: ConsoleInterface, importResolverFactory?: ImportResolverFactory, configOptions?: ConfigOptions) {
this._instanceName = instanceName;
this._console = console || new StandardConsole();
this._configOptions = configOptions ?? new ConfigOptions(process.cwd());
this._importResolver = new ImportResolver(fs, this._configOptions);
this._importResolverFactory = importResolverFactory || AnalyzerService.createImportResolver;
this._importResolver = this._importResolverFactory(fs, this._configOptions);
this._program = new Program(this._importResolver, this._configOptions, this._console);
this._executionRootPath = '';
this._typeStubTargetImportName = undefined;
Expand All @@ -85,6 +87,10 @@ export class AnalyzerService {
this._clearReanalysisTimer();
}

static createImportResolver(fs: VirtualFileSystem, options: ConfigOptions): ImportResolver {
return new ImportResolver(fs, options);
}

setCompletionCallback(callback: AnalysisCompleteCallback | undefined): void {
this._onCompletionCallback = callback;
}
Expand Down Expand Up @@ -815,7 +821,7 @@ export class AnalyzerService {
private _applyConfigOptions() {
// Allocate a new import resolver because the old one has information
// cached based on the previous config options.
this._importResolver = new ImportResolver(this._fs, this._configOptions);
this._importResolver = this._importResolverFactory(this._fs, this._configOptions);
this._program.setImportResolver(this._importResolver);

this._updateSourceFileWatchers();
Expand Down
11 changes: 11 additions & 0 deletions server/src/common/consts.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/*
* consts.ts
* Copyright (c) Microsoft Corporation.
* Licensed under the MIT license.
*
* Defines well known consts and names
*/

export const TYPESHED_FALLBACK = 'typeshed-fallback';
export const LIB = 'lib';
export const SITE_PACKAGES = 'site-packages';
13 changes: 13 additions & 0 deletions server/src/common/core.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,16 @@ export interface MapLike<T> {
export function hasProperty(map: MapLike<any>, key: string): boolean {
return hasOwnProperty.call(map, key);
}

/**
* Convert the given value to boolean
* @param trueOrFalse string value 'true' or 'false'
*/
export function toBoolean(trueOrFalse: string): boolean {
const normalized = trueOrFalse?.trim().toUpperCase();
if (normalized === 'TRUE') {
return true;
}

return false;
}
Loading

0 comments on commit 00db9d6

Please sign in to comment.