diff --git a/build/.mocha.functional.perf.opts b/build/.mocha.functional.perf.opts new file mode 100644 index 000000000000..decfc3ece6e6 --- /dev/null +++ b/build/.mocha.functional.perf.opts @@ -0,0 +1,9 @@ +./out/test/**/*.functional.test.js +--require=out/test/unittests.js +--exclude=out/**/*.jsx +--ui=tdd +--recursive +--colors +--exit +--timeout=180000 +--reporter spec diff --git a/build/ci/vscode-python-nightly-flake-ci.yaml b/build/ci/vscode-python-nightly-flake-ci.yaml new file mode 100644 index 000000000000..191e3503a3ea --- /dev/null +++ b/build/ci/vscode-python-nightly-flake-ci.yaml @@ -0,0 +1,137 @@ +# Nightly build + +name: '$(Year:yyyy).$(Month).0.$(BuildID)-nightly-flake' + +# Not the CI build, see `vscode-python-nightly-flake-ci.yaml`. +trigger: none + +# Not the PR build for merges to master and release. +pr: none + +schedules: +- cron: "0 8 * * 1-5" + # Daily midnight PST build, runs Monday - Friday always + displayName: Nightly Flake build + branches: + include: + - master + - release* + always: true + +# Variables that are available for the entire pipeline. +variables: +- template: templates/globals.yml + +stages: +- stage: Build + jobs: + - template: templates/jobs/build_compile.yml + +# Each item in each matrix has a number of possible values it may +# define. They are detailed in templates/test_phases.yml. The only +# required value is "TestsToRun". + +- stage: Linux + dependsOn: + - Build + jobs: + - job: 'Py3x' + dependsOn: [] + timeoutInMinutes: 120 + strategy: + matrix: + 'Functional': + TestsToRun: 'testfunctional' + NeedsPythonTestReqs: true + NeedsPythonFunctionalReqs: true + VSCODE_PYTHON_ROLLING: true + pool: + vmImage: 'ubuntu-16.04' + steps: + - template: templates/test_phases.yml + + - job: 'Py36' + dependsOn: [] + timeoutInMinutes: 120 + strategy: + matrix: + 'Functional': + PythonVersion: '3.6' + TestsToRun: 'testfunctional' + NeedsPythonTestReqs: true + NeedsPythonFunctionalReqs: true + VSCODE_PYTHON_ROLLING: true + pool: + vmImage: 'ubuntu-16.04' + steps: + - template: templates/test_phases.yml + +- stage: Mac + dependsOn: + - Build + jobs: + - job: 'Py3x' + dependsOn: [] + timeoutInMinutes: 120 + strategy: + matrix: + 'Functional': + TestsToRun: 'testfunctional' + NeedsPythonTestReqs: true + NeedsPythonFunctionalReqs: true + VSCODE_PYTHON_ROLLING: true + pool: + vmImage: 'macos-10.13' + steps: + - template: templates/test_phases.yml + + - job: 'Py36' + dependsOn: [] + timeoutInMinutes: 120 + strategy: + matrix: + 'Functional': + PythonVersion: '3.6' + TestsToRun: 'testfunctional' + NeedsPythonTestReqs: true + NeedsPythonFunctionalReqs: true + VSCODE_PYTHON_ROLLING: true + pool: + vmImage: 'macos-10.13' + steps: + - template: templates/test_phases.yml + +- stage: Windows + dependsOn: + - Build + jobs: + - job: 'Py3x' + dependsOn: [] + timeoutInMinutes: 120 + strategy: + matrix: + 'Functional': + TestsToRun: 'testfunctional' + NeedsPythonTestReqs: true + NeedsPythonFunctionalReqs: true + VSCODE_PYTHON_ROLLING: true + pool: + vmImage: 'vs2017-win2016' + steps: + - template: templates/test_phases.yml + + - job: 'Py36' + dependsOn: [] + timeoutInMinutes: 120 + strategy: + matrix: + 'Functional': + PythonVersion: '3.6' + TestsToRun: 'testfunctional' + NeedsPythonTestReqs: true + NeedsPythonFunctionalReqs: true + VSCODE_PYTHON_ROLLING: true + pool: + vmImage: 'vs2017-win2016' + steps: + - template: templates/test_phases.yml \ No newline at end of file diff --git a/news/3 Code Health/7997.md b/news/3 Code Health/7997.md new file mode 100644 index 000000000000..cd92de44dd9d --- /dev/null +++ b/news/3 Code Health/7997.md @@ -0,0 +1 @@ +Functional tests using real jupyter can take 30-90 seconds each. Most of this time is searching for interpreters. Cache the interpreter search. \ No newline at end of file diff --git a/package.json b/package.json index c4b7fbff2891..ee5942e7c1a2 100644 --- a/package.json +++ b/package.json @@ -2844,6 +2844,7 @@ "test:unittests": "mocha --opts ./build/.mocha.unittests.js.opts", "test:unittests:cover": "nyc --no-clean --nycrc-path build/.nycrc mocha --opts ./build/.mocha.unittests.ts.opts", "test:functional": "mocha --require source-map-support/register --opts ./build/.mocha.functional.opts", + "test:functional:perf": "node --inspect-brk ./node_modules/mocha/bin/_mocha --require source-map-support/register --opts ./build/.mocha.functional.perf.opts", "test:functional:cover": "npm run test:functional", "test:cover:report": "nyc --nycrc-path build/.nycrc report --reporter=text --reporter=html --reporter=text-summary --reporter=cobertura", "testDebugger": "node ./out/test/testBootstrap.js ./out/test/debuggerTest.js", diff --git a/src/client/interpreter/locators/services/cacheableLocatorService.ts b/src/client/interpreter/locators/services/cacheableLocatorService.ts index 8a5dd240324c..f2608fed6a34 100644 --- a/src/client/interpreter/locators/services/cacheableLocatorService.ts +++ b/src/client/interpreter/locators/services/cacheableLocatorService.ts @@ -17,10 +17,42 @@ import { sendTelemetryEvent } from '../../../telemetry'; import { EventName } from '../../../telemetry/constants'; import { IInterpreterLocatorService, IInterpreterWatcher, PythonInterpreter } from '../../contracts'; +export class CacheableLocatorPromiseCache { + private static useStatic = false; + private static staticMap = new Map>(); + private normalMap = new Map>(); + + public static forceUseStatic() { + CacheableLocatorPromiseCache.useStatic = true; + } + public get(key: string): Deferred | undefined { + if (CacheableLocatorPromiseCache.useStatic) { + return CacheableLocatorPromiseCache.staticMap.get(key); + } + return this.normalMap.get(key); + } + + public set(key: string, value: Deferred) { + if (CacheableLocatorPromiseCache.useStatic) { + CacheableLocatorPromiseCache.staticMap.set(key, value); + } else { + this.normalMap.set(key, value); + } + } + + public delete(key: string) { + if (CacheableLocatorPromiseCache.useStatic) { + CacheableLocatorPromiseCache.staticMap.delete(key); + } else { + this.normalMap.delete(key); + } + } +} + @injectable() export abstract class CacheableLocatorService implements IInterpreterLocatorService { protected readonly _hasInterpreters: Deferred; - private readonly promisesPerResource = new Map>(); + private readonly promisesPerResource = new CacheableLocatorPromiseCache(); private readonly handlersAddedToResource = new Set(); private readonly cacheKeyPrefix: string; private readonly locating = new EventEmitter>(); @@ -32,6 +64,7 @@ export abstract class CacheableLocatorService implements IInterpreterLocatorServ this._hasInterpreters = createDeferred(); this.cacheKeyPrefix = `INTERPRETERS_CACHE_v3_${name}`; } + public get onLocating(): Event> { return this.locating.event; } @@ -43,7 +76,6 @@ export abstract class CacheableLocatorService implements IInterpreterLocatorServ public async getInterpreters(resource?: Uri, ignoreCache?: boolean): Promise { const cacheKey = this.getCacheKey(resource); let deferred = this.promisesPerResource.get(cacheKey); - if (!deferred || ignoreCache) { deferred = createDeferred(); this.promisesPerResource.set(cacheKey, deferred); diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index f214ba127a3f..0fe4b63bea82 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -16,6 +16,7 @@ import { Event, EventEmitter, FileSystemWatcher, + Memento, Uri, WorkspaceConfiguration, WorkspaceFolder, @@ -99,7 +100,6 @@ import { } from '../../client/common/installer/productPath'; import { ProductService } from '../../client/common/installer/productService'; import { IInstallationChannelManager, IProductPathService, IProductService } from '../../client/common/installer/types'; -import { PersistentStateFactory } from '../../client/common/persistentState'; import { IS_WINDOWS } from '../../client/common/platform/constants'; import { PathUtils } from '../../client/common/platform/pathUtils'; import { RegistryImplementation } from '../../client/common/platform/registry'; @@ -130,6 +130,7 @@ import { } from '../../client/common/terminal/types'; import { BANNER_NAME_LS_SURVEY, + GLOBAL_MEMENTO, IAsyncDisposableRegistry, IConfigurationService, ICryptoUtils, @@ -139,12 +140,14 @@ import { IExtensionContext, IExtensions, IInstaller, + IMemento, IOutputChannel, IPathUtils, IPersistentStateFactory, IPythonExtensionBanner, IsWindows, - ProductType + ProductType, + WORKSPACE_MEMENTO } from '../../client/common/types'; import { Deferred, sleep } from '../../client/common/utils/async'; import { noop } from '../../client/common/utils/misc'; @@ -283,6 +286,7 @@ import { InterpreterService } from '../../client/interpreter/interpreterService' import { InterpreterVersionService } from '../../client/interpreter/interpreterVersion'; import { PythonInterpreterLocatorService } from '../../client/interpreter/locators'; import { InterpreterLocatorHelper } from '../../client/interpreter/locators/helpers'; +import { CacheableLocatorPromiseCache } from '../../client/interpreter/locators/services/cacheableLocatorService'; import { CondaEnvFileService } from '../../client/interpreter/locators/services/condaEnvFileService'; import { CondaEnvService } from '../../client/interpreter/locators/services/condaEnvService'; import { @@ -333,6 +337,7 @@ import { MockWorkspaceConfiguration } from './mockWorkspaceConfig'; import { blurWindow, createMessageEvent } from './reactHelpers'; import { TestInteractiveWindowProvider } from './testInteractiveWindowProvider'; import { TestNativeEditorProvider } from './testNativeEditorProvider'; +import { TestPersistentStateFactory } from './testPersistentStateFactory'; export class DataScienceIocContainer extends UnitTestIocContainer { public webPanelListener: IWebPanelMessageListener | undefined; @@ -914,7 +919,19 @@ export class DataScienceIocContainer extends UnitTestIocContainer { IInterpreterVersionService, InterpreterVersionService ); - this.serviceManager.addSingleton(IPersistentStateFactory, PersistentStateFactory); + + const globalStorage = this.serviceManager.get(IMemento, GLOBAL_MEMENTO); + const localStorage = this.serviceManager.get(IMemento, WORKSPACE_MEMENTO); + + // Create a custom persistent state factory that remembers specific things between tests + this.serviceManager.addSingletonInstance( + IPersistentStateFactory, + new TestPersistentStateFactory(globalStorage, localStorage) + ); + + // Inform the cacheable locator service to use a static map so that it stays in memory in between tests + CacheableLocatorPromiseCache.forceUseStatic(); + this.serviceManager.addSingletonInstance(IInterpreterDisplay, interpreterDisplay.object); this.serviceManager.addSingleton( diff --git a/src/test/datascience/testPersistentStateFactory.ts b/src/test/datascience/testPersistentStateFactory.ts new file mode 100644 index 000000000000..53743a202de1 --- /dev/null +++ b/src/test/datascience/testPersistentStateFactory.ts @@ -0,0 +1,53 @@ +import { Memento } from 'vscode'; +import { PersistentStateFactory } from '../../client/common/persistentState'; +import { IPersistentState, IPersistentStateFactory } from '../../client/common/types'; + +const PrefixesToStore = ['INTERPRETERS_CACHE']; + +// tslint:disable-next-line: no-any +const persistedState = new Map(); + +class TestPersistentState implements IPersistentState { + constructor(private key: string, defaultValue?: T | undefined) { + if (defaultValue) { + persistedState.set(key, defaultValue); + } + } + public get value(): T { + return persistedState.get(this.key); + } + public async updateValue(value: T): Promise { + persistedState.set(this.key, value); + } +} + +// This class is used to make certain values persist across tests. +export class TestPersistentStateFactory implements IPersistentStateFactory { + private realStateFactory: PersistentStateFactory; + constructor(globalState: Memento, localState: Memento) { + this.realStateFactory = new PersistentStateFactory(globalState, localState); + } + + public createGlobalPersistentState( + key: string, + defaultValue?: T | undefined, + expiryDurationMs?: number | undefined + ): IPersistentState { + if (PrefixesToStore.find(p => key.startsWith(p))) { + return new TestPersistentState(key, defaultValue); + } + + return this.realStateFactory.createGlobalPersistentState(key, defaultValue, expiryDurationMs); + } + public createWorkspacePersistentState( + key: string, + defaultValue?: T | undefined, + expiryDurationMs?: number | undefined + ): IPersistentState { + if (PrefixesToStore.find(p => key.startsWith(p))) { + return new TestPersistentState(key, defaultValue); + } + + return this.realStateFactory.createWorkspacePersistentState(key, defaultValue, expiryDurationMs); + } +}