From efdf65e7cd2eb18616242c211b56190938a367fe Mon Sep 17 00:00:00 2001 From: Eleanor Boyd Date: Tue, 20 Jun 2023 10:41:42 -0700 Subject: [PATCH] Workspace & resolver tests (#21441) This PR - moves populateTestTree to utils - adds tests for execution adapters (pytest and unittest) - resultResolver tests - workspaceTestAdapater tests --- .../testController/common/resultResolver.ts | 98 +- .../testing/testController/common/utils.ts | 70 ++ .../testController/workspaceTestAdapter.ts | 13 +- src/test/mocks/vsc/index.ts | 112 +++ .../pytestExecutionAdapter.unit.test.ts | 23 +- .../resultResolver.unit.test.ts | 368 ++++++++ .../testExecutionAdapter.unit.test.ts | 5 +- .../workspaceTestAdapter.unit.test.ts | 883 ++++++++++++------ src/test/vscode-mock.ts | 2 + 9 files changed, 1200 insertions(+), 374 deletions(-) create mode 100644 src/test/testing/testController/resultResolver.unit.test.ts diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index 8f8cb859b9d5..49243390ad0f 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -1,41 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { - CancellationToken, - Position, - TestController, - TestItem, - Uri, - Range, - TestMessage, - Location, - TestRun, -} from 'vscode'; +import { CancellationToken, TestController, TestItem, Uri, TestMessage, Location, TestRun } from 'vscode'; import * as util from 'util'; -import * as path from 'path'; -import { - DiscoveredTestItem, - DiscoveredTestNode, - DiscoveredTestPayload, - ExecutionTestPayload, - ITestResultResolver, -} from './types'; +import { DiscoveredTestPayload, ExecutionTestPayload, ITestResultResolver } from './types'; import { TestProvider } from '../../types'; import { traceError, traceLog } from '../../../logging'; import { Testing } from '../../../common/utils/localize'; -import { - DebugTestTag, - ErrorTestItemOptions, - RunTestTag, - clearAllChildren, - createErrorTestItem, - getTestCaseNodes, -} from './testItemUtilities'; +import { clearAllChildren, createErrorTestItem, getTestCaseNodes } from './testItemUtilities'; import { sendTelemetryEvent } from '../../../telemetry'; import { EventName } from '../../../telemetry/constants'; import { splitLines } from '../../../common/stringUtils'; -import { fixLogLines } from './utils'; +import { buildErrorNodeOptions, fixLogLines, populateTestTree } from './utils'; export class PythonResultResolver implements ITestResultResolver { testController: TestController; @@ -253,69 +229,5 @@ export class PythonResultResolver implements ITestResultResolver { return Promise.resolve(); } } -// had to switch the order of the original parameter since required param cannot follow optional. -function populateTestTree( - testController: TestController, - testTreeData: DiscoveredTestNode, - testRoot: TestItem | undefined, - resultResolver: ITestResultResolver, - token?: CancellationToken, -): void { - // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. - if (!testRoot) { - testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); - - testRoot.canResolveChildren = true; - testRoot.tags = [RunTestTag, DebugTestTag]; - - testController.items.add(testRoot); - } - - // Recursively populate the tree with test data. - testTreeData.children.forEach((child) => { - if (!token?.isCancellationRequested) { - if (isTestItem(child)) { - const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - testItem.tags = [RunTestTag, DebugTestTag]; - - const range = new Range( - new Position(Number(child.lineno) - 1, 0), - new Position(Number(child.lineno), 0), - ); - testItem.canResolveChildren = false; - testItem.range = range; - testItem.tags = [RunTestTag, DebugTestTag]; - - testRoot!.children.add(testItem); - // add to our map - resultResolver.runIdToTestItem.set(child.runID, testItem); - resultResolver.runIdToVSid.set(child.runID, child.id_); - resultResolver.vsIdToRunId.set(child.id_, child.runID); - } else { - let node = testController.items.get(child.path); - - if (!node) { - node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); - node.canResolveChildren = true; - node.tags = [RunTestTag, DebugTestTag]; - testRoot!.children.add(node); - } - populateTestTree(testController, child, node, resultResolver, token); - } - } - }); -} - -function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { - return test.type_ === 'test'; -} - -export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { - const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; - return { - id: `DiscoveryError:${uri.fsPath}`, - label: `${labelText} [${path.basename(uri.fsPath)}]`, - error: message, - }; -} +// had to switch the order of the original parameter since required param cannot follow optional. diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 232aaccd5341..e92db22747c6 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -1,11 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import * as net from 'net'; +import * as path from 'path'; +import { CancellationToken, Position, TestController, TestItem, Uri, Range } from 'vscode'; import { traceError, traceLog, traceVerbose } from '../../../logging'; import { EnableTestAdapterRewrite } from '../../../common/experiments/groups'; import { IExperimentService } from '../../../common/types'; import { IServiceContainer } from '../../../ioc/types'; +import { DebugTestTag, ErrorTestItemOptions, RunTestTag } from './testItemUtilities'; +import { DiscoveredTestItem, DiscoveredTestNode, ITestResultResolver } from './types'; export function fixLogLines(content: string): string { const lines = content.split(/\r?\n/g); @@ -111,3 +115,69 @@ export async function startTestIdServer(testIds: string[]): Promise { }); return 0; } + +export function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { + const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; + return { + id: `DiscoveryError:${uri.fsPath}`, + label: `${labelText} [${path.basename(uri.fsPath)}]`, + error: message, + }; +} + +export function populateTestTree( + testController: TestController, + testTreeData: DiscoveredTestNode, + testRoot: TestItem | undefined, + resultResolver: ITestResultResolver, + token?: CancellationToken, +): void { + // If testRoot is undefined, use the info of the root item of testTreeData to create a test item, and append it to the test controller. + if (!testRoot) { + testRoot = testController.createTestItem(testTreeData.path, testTreeData.name, Uri.file(testTreeData.path)); + + testRoot.canResolveChildren = true; + testRoot.tags = [RunTestTag, DebugTestTag]; + + testController.items.add(testRoot); + } + + // Recursively populate the tree with test data. + testTreeData.children.forEach((child) => { + if (!token?.isCancellationRequested) { + if (isTestItem(child)) { + const testItem = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + testItem.tags = [RunTestTag, DebugTestTag]; + + const range = new Range( + new Position(Number(child.lineno) - 1, 0), + new Position(Number(child.lineno), 0), + ); + testItem.canResolveChildren = false; + testItem.range = range; + testItem.tags = [RunTestTag, DebugTestTag]; + + testRoot!.children.add(testItem); + // add to our map + resultResolver.runIdToTestItem.set(child.runID, testItem); + resultResolver.runIdToVSid.set(child.runID, child.id_); + resultResolver.vsIdToRunId.set(child.id_, child.runID); + } else { + let node = testController.items.get(child.path); + + if (!node) { + node = testController.createTestItem(child.id_, child.name, Uri.file(child.path)); + + node.canResolveChildren = true; + node.tags = [RunTestTag, DebugTestTag]; + testRoot!.children.add(node); + } + populateTestTree(testController, child, node, resultResolver, token); + } + } + }); +} + +function isTestItem(test: DiscoveredTestNode | DiscoveredTestItem): test is DiscoveredTestItem { + return test.type_ === 'test'; +} diff --git a/src/client/testing/testController/workspaceTestAdapter.ts b/src/client/testing/testController/workspaceTestAdapter.ts index 5c7ee9cfe520..f3ea0b9f6193 100644 --- a/src/client/testing/testController/workspaceTestAdapter.ts +++ b/src/client/testing/testController/workspaceTestAdapter.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import * as path from 'path'; import * as util from 'util'; import { CancellationToken, TestController, TestItem, TestRun, Uri } from 'vscode'; import { createDeferred, Deferred } from '../../common/utils/async'; @@ -10,10 +9,11 @@ import { traceError } from '../../logging'; import { sendTelemetryEvent } from '../../telemetry'; import { EventName } from '../../telemetry/constants'; import { TestProvider } from '../types'; -import { createErrorTestItem, ErrorTestItemOptions, getTestCaseNodes } from './common/testItemUtilities'; +import { createErrorTestItem, getTestCaseNodes } from './common/testItemUtilities'; import { ITestDiscoveryAdapter, ITestExecutionAdapter, ITestResultResolver } from './common/types'; import { IPythonExecutionFactory } from '../../common/process/types'; import { ITestDebugLauncher } from '../common/types'; +import { buildErrorNodeOptions } from './common/utils'; /** * This class exposes a test-provider-agnostic way of discovering tests. @@ -162,12 +162,3 @@ export class WorkspaceTestAdapter { return Promise.resolve(); } } - -function buildErrorNodeOptions(uri: Uri, message: string, testType: string): ErrorTestItemOptions { - const labelText = testType === 'pytest' ? 'Pytest Discovery Error' : 'Unittest Discovery Error'; - return { - id: `DiscoveryError:${uri.fsPath}`, - label: `${labelText} [${path.basename(uri.fsPath)}]`, - error: message, - }; -} diff --git a/src/test/mocks/vsc/index.ts b/src/test/mocks/vsc/index.ts index 89f4ab1a2d07..092fc67da6c6 100644 --- a/src/test/mocks/vsc/index.ts +++ b/src/test/mocks/vsc/index.ts @@ -6,6 +6,7 @@ import { EventEmitter as NodeEventEmitter } from 'events'; import * as vscode from 'vscode'; + // export * from './range'; // export * from './position'; // export * from './selection'; @@ -443,3 +444,114 @@ export enum LogLevel { */ Error = 5, } + +export class TestMessage { + /** + * Human-readable message text to display. + */ + message: string | MarkdownString; + + /** + * Expected test output. If given with {@link TestMessage.actualOutput actualOutput }, a diff view will be shown. + */ + expectedOutput?: string; + + /** + * Actual test output. If given with {@link TestMessage.expectedOutput expectedOutput }, a diff view will be shown. + */ + actualOutput?: string; + + /** + * Associated file location. + */ + location?: vscode.Location; + + /** + * Creates a new TestMessage that will present as a diff in the editor. + * @param message Message to display to the user. + * @param expected Expected output. + * @param actual Actual output. + */ + static diff(message: string | MarkdownString, expected: string, actual: string): TestMessage { + const testMessage = new TestMessage(message); + testMessage.expectedOutput = expected; + testMessage.actualOutput = actual; + return testMessage; + } + + /** + * Creates a new TestMessage instance. + * @param message The message to show to the user. + */ + constructor(message: string | MarkdownString) { + this.message = message; + } +} + +export interface TestItemCollection extends Iterable<[string, vscode.TestItem]> { + /** + * Gets the number of items in the collection. + */ + readonly size: number; + + /** + * Replaces the items stored by the collection. + * @param items Items to store. + */ + replace(items: readonly vscode.TestItem[]): void; + + /** + * Iterate over each entry in this collection. + * + * @param callback Function to execute for each entry. + * @param thisArg The `this` context used when invoking the handler function. + */ + forEach(callback: (item: vscode.TestItem, collection: TestItemCollection) => unknown, thisArg?: unknown): void; + + /** + * Adds the test item to the children. If an item with the same ID already + * exists, it'll be replaced. + * @param item Item to add. + */ + add(item: vscode.TestItem): void; + + /** + * Removes a single test item from the collection. + * @param itemId Item ID to delete. + */ + delete(itemId: string): void; + + /** + * Efficiently gets a test item by ID, if it exists, in the children. + * @param itemId Item ID to get. + * @returns The found item or undefined if it does not exist. + */ + get(itemId: string): vscode.TestItem | undefined; +} + +/** + * Represents a location inside a resource, such as a line + * inside a text file. + */ +export class Location { + /** + * The resource identifier of this location. + */ + uri: vscode.Uri; + + /** + * The document range of this location. + */ + range: vscode.Range; + + /** + * Creates a new location object. + * + * @param uri The resource identifier. + * @param rangeOrPosition The range or position. Positions will be converted to an empty range. + */ + constructor(uri: vscode.Uri, rangeOrPosition: vscode.Range) { + this.uri = uri; + this.range = rangeOrPosition; + } +} diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index f60429d38e5b..9f4c41ba8309 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -29,6 +29,7 @@ suite('pytest test execution adapter', () => { let debugLauncher: typeMoq.IMock; (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; let myTestPath: string; + let startTestIdServerStub: sinon.SinonStub; setup(() => { testServer = typeMoq.Mock.ofType(); @@ -65,7 +66,7 @@ suite('pytest test execution adapter', () => { deferred.resolve(); return Promise.resolve(); }); - sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); + startTestIdServerStub = sinon.stub(util, 'startTestIdServer').returns(Promise.resolve(54321)); execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); @@ -75,6 +76,26 @@ suite('pytest test execution adapter', () => { teardown(() => { sinon.restore(); }); + test('startTestIdServer called with correct testIds', async () => { + const uri = Uri.file(myTestPath); + const uuid = 'uuid123'; + testServer + .setup((t) => t.onDiscoveryDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => uuid); + const outputChannel = typeMoq.Mock.ofType(); + const testRun = typeMoq.Mock.ofType(); + adapter = new PytestTestExecutionAdapter(testServer.object, configService, outputChannel.object); + + const testIds = ['test1id', 'test2id']; + await adapter.runTests(uri, testIds, false, testRun.object, execFactory.object); + + sinon.assert.calledWithExactly(startTestIdServerStub, testIds); + }); test('pytest execution called with correct args', async () => { const uri = Uri.file(myTestPath); const uuid = 'uuid123'; diff --git a/src/test/testing/testController/resultResolver.unit.test.ts b/src/test/testing/testController/resultResolver.unit.test.ts new file mode 100644 index 000000000000..57b321c2e36c --- /dev/null +++ b/src/test/testing/testController/resultResolver.unit.test.ts @@ -0,0 +1,368 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { TestController, Uri, TestItem, CancellationToken, TestRun, TestItemCollection, Range } from 'vscode'; +import * as typemoq from 'typemoq'; +import * as sinon from 'sinon'; +import { TestProvider } from '../../../client/testing/types'; +import { + DiscoveredTestNode, + DiscoveredTestPayload, + ExecutionTestPayload, +} from '../../../client/testing/testController/common/types'; +import * as testItemUtilities from '../../../client/testing/testController/common/testItemUtilities'; +import * as ResultResolver from '../../../client/testing/testController/common/resultResolver'; +import * as util from '../../../client/testing/testController/common/utils'; + +suite('Result Resolver tests', () => { + suite('Test discovery', () => { + let resultResolver: ResultResolver.PythonResultResolver; + let testController: TestController; + const log: string[] = []; + let workspaceUri: Uri; + let testProvider: TestProvider; + let defaultErrorMessage: string; + let blankTestItem: TestItem; + let cancelationToken: CancellationToken; + + setup(() => { + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + + dispose: () => { + // empty + }, + } as unknown) as TestController; + defaultErrorMessage = 'pytest test discovery error (see Output > Python)'; + blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + cancelationToken = ({ + isCancellationRequested: false, + } as unknown) as CancellationToken; + }); + teardown(() => { + sinon.restore(); + }); + + test('resolveDiscovery calls populate test tree correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver(testController, testProvider, workspaceUri); + const tests: DiscoveredTestNode = { + path: 'path', + name: 'name', + type_: 'folder', + id_: 'id', + children: [], + }; + const payload: DiscoveredTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + tests, + }; + + // stub out functionality of populateTestTreeStub which is called in resolveDiscovery + const populateTestTreeStub = sinon.stub(util, 'populateTestTree').returns(); + + // call resolve discovery + resultResolver.resolveDiscovery(payload, cancelationToken); + + // assert the stub functions were called with the correct parameters + + // header of populateTestTree is (testController: TestController, testTreeData: DiscoveredTestNode, testRoot: TestItem | undefined, resultResolver: ITestResultResolver, token?: CancellationToken) + sinon.assert.calledWithMatch( + populateTestTreeStub, + testController, // testController + tests, // testTreeData + undefined, // testRoot + resultResolver, // resultResolver + cancelationToken, // token + ); + }); + // what about if the error node already exists: this.testController.items.get(`DiscoveryError:${workspacePath}`); + test('resolveDiscovery should create error node on error with correct params', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver(testController, testProvider, workspaceUri); + const errorMessage = 'error msg A'; + const expectedErrorMessage = `${defaultErrorMessage}\r\n ${errorMessage}`; + + // stub out return values of functions called in resolveDiscovery + const payload: DiscoveredTestPayload = { + cwd: workspaceUri.fsPath, + status: 'error', + errors: [errorMessage], + }; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + + // stub out functionality of buildErrorNodeOptions and createErrorTestItem which are called in resolveDiscovery + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + + // call resolve discovery + resultResolver.resolveDiscovery(payload); + + // assert the stub functions were called with the correct parameters + + // header of buildErrorNodeOptions is (uri: Uri, message: string, testType: string) + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, workspaceUri, expectedErrorMessage, testProvider); + // header of createErrorTestItem is (options: ErrorTestItemOptions, testController: TestController, uri: Uri) + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + }); + }); + suite('Test execution result resolver', () => { + let resultResolver: ResultResolver.PythonResultResolver; + const log: string[] = []; + let workspaceUri: Uri; + let testProvider: TestProvider; + let cancelationToken: CancellationToken; + let runInstance: typemoq.IMock; + let testControllerMock: typemoq.IMock; + let mockTestItem1: TestItem; + let mockTestItem2: TestItem; + + setup(() => { + // create mock test items + mockTestItem1 = createMockTestItem('mockTestItem1'); + mockTestItem2 = createMockTestItem('mockTestItem2'); + + // create mock testItems to pass into a iterable + const mockTestItems: [string, TestItem][] = [ + ['1', mockTestItem1], + ['2', mockTestItem2], + ]; + const iterableMock = mockTestItems[Symbol.iterator](); + + // create mock testItemCollection + const testItemCollectionMock = typemoq.Mock.ofType(); + testItemCollectionMock + .setup((x) => x.forEach(typemoq.It.isAny())) + .callback((callback) => { + let result = iterableMock.next(); + while (!result.done) { + callback(result.value[1]); + result = iterableMock.next(); + } + }) + .returns(() => mockTestItem1); + + // create mock testController + testControllerMock = typemoq.Mock.ofType(); + testControllerMock.setup((t) => t.items).returns(() => testItemCollectionMock.object); + + cancelationToken = ({ + isCancellationRequested: false, + } as unknown) as CancellationToken; + + // define functions within runInstance + runInstance = typemoq.Mock.ofType(); + runInstance.setup((r) => r.name).returns(() => 'name'); + runInstance.setup((r) => r.token).returns(() => cancelationToken); + runInstance.setup((r) => r.isPersisted).returns(() => true); + runInstance + .setup((r) => r.enqueued(typemoq.It.isAny())) + .returns(() => { + // empty + log.push('enqueue'); + return undefined; + }); + runInstance + .setup((r) => r.started(typemoq.It.isAny())) + .returns(() => { + // empty + log.push('start'); + }); + + // mock getTestCaseNodes to just return the given testNode added + sinon.stub(testItemUtilities, 'getTestCaseNodes').callsFake((testNode: TestItem) => [testNode]); + }); + teardown(() => { + sinon.restore(); + }); + test('resolveExecution handles failed tests correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + // add a mock test item to the map of known VSCode ids to run ids + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2'); + + // add this mock test to the map of known test items + resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1); + resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2); + + // create a successful payload with a single test called mockTestItem1 + const successPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + result: { + mockTestItem1: { + test: 'test', + outcome: 'failure', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess + message: 'message', + traceback: 'traceback', + subtest: 'subtest', + }, + }, + error: '', + }; + + // call resolveExecution + resultResolver.resolveExecution(successPayload, runInstance.object); + + // verify that the passed function was called for the single test item + runInstance.verify((r) => r.failed(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.once()); + }); + test('resolveExecution handles skipped correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + // add a mock test item to the map of known VSCode ids to run ids + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2'); + + // add this mock test to the map of known test items + resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1); + resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2); + + // create a successful payload with a single test called mockTestItem1 + const successPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + result: { + mockTestItem1: { + test: 'test', + outcome: 'skipped', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess + message: 'message', + traceback: 'traceback', + subtest: 'subtest', + }, + }, + error: '', + }; + + // call resolveExecution + resultResolver.resolveExecution(successPayload, runInstance.object); + + // verify that the passed function was called for the single test item + runInstance.verify((r) => r.skipped(typemoq.It.isAny()), typemoq.Times.once()); + }); + test('resolveExecution handles success correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + // add a mock test item to the map of known VSCode ids to run ids + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2'); + + // add this mock test to the map of known test items + resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1); + resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2); + + // create a successful payload with a single test called mockTestItem1 + const successPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'success', + result: { + mockTestItem1: { + test: 'test', + outcome: 'success', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess + message: 'message', + traceback: 'traceback', + subtest: 'subtest', + }, + }, + error: '', + }; + + // call resolveExecution + resultResolver.resolveExecution(successPayload, runInstance.object); + + // verify that the passed function was called for the single test item + runInstance.verify((r) => r.passed(typemoq.It.isAny()), typemoq.Times.once()); + }); + test('resolveExecution handles error correctly', async () => { + // test specific constants used expected values + testProvider = 'pytest'; + workspaceUri = Uri.file('/foo/bar'); + resultResolver = new ResultResolver.PythonResultResolver( + testControllerMock.object, + testProvider, + workspaceUri, + ); + + const errorPayload: ExecutionTestPayload = { + cwd: workspaceUri.fsPath, + status: 'error', + error: 'error', + }; + + resultResolver.resolveExecution(errorPayload, runInstance.object); + + // verify that none of these functions are called + + runInstance.verify((r) => r.passed(typemoq.It.isAny()), typemoq.Times.never()); + runInstance.verify((r) => r.failed(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.never()); + runInstance.verify((r) => r.skipped(typemoq.It.isAny()), typemoq.Times.never()); + }); + }); +}); + +function createMockTestItem(id: string): TestItem { + const range = new Range(0, 0, 0, 0); + const mockTestItem = ({ + id, + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + range, + uri: Uri.file('/foo/bar'), + } as unknown) as TestItem; + + return mockTestItem; +} diff --git a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts index 7db287a3355c..88126225a177 100644 --- a/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/unittest/testExecutionAdapter.unit.test.ts @@ -49,14 +49,15 @@ suite('Unittest test execution adapter', () => { const script = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'unittestadapter', 'execution.py'); const adapter = new UnittestTestExecutionAdapter(stubTestServer, stubConfigSettings, outputChannel.object); - adapter.runTests(uri, [], false).then(() => { + const testIds = ['test1id', 'test2id']; + adapter.runTests(uri, testIds, false).then(() => { const expectedOptions: TestCommandOptions = { workspaceFolder: uri, command: { script, args: ['--udiscovery', '-v', '-s', '.', '-p', 'test*'] }, cwd: uri.fsPath, uuid: '123456789', debugBool: false, - testIds: [], + testIds, }; assert.deepStrictEqual(options, expectedOptions); }); diff --git a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts index 42e38d200546..5a2e48130746 100644 --- a/src/test/testing/testController/workspaceTestAdapter.unit.test.ts +++ b/src/test/testing/testController/workspaceTestAdapter.unit.test.ts @@ -1,267 +1,616 @@ -// // Copyright (c) Microsoft Corporation. All rights reserved. -// // Licensed under the MIT License. - -// import * as assert from 'assert'; -// import * as sinon from 'sinon'; -// import * as typemoq from 'typemoq'; - -// import { TestController, TestItem, Uri } from 'vscode'; -// import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; -// import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; -// import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 -// import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; -// import * as Telemetry from '../../../client/telemetry'; -// import { EventName } from '../../../client/telemetry/constants'; -// import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; - -// suite('Workspace test adapter', () => { -// suite('Test discovery', () => { -// let stubTestServer: ITestServer; -// let stubConfigSettings: IConfigurationService; -// let stubResultResolver: ITestResultResolver; - -// let discoverTestsStub: sinon.SinonStub; -// let sendTelemetryStub: sinon.SinonStub; -// let outputChannel: typemoq.IMock; - -// let telemetryEvent: { eventName: EventName; properties: Record }[] = []; - -// // Stubbed test controller (see comment around L.40) -// let testController: TestController; -// let log: string[] = []; - -// const sandbox = sinon.createSandbox(); - -// setup(() => { -// stubConfigSettings = ({ -// getSettings: () => ({ -// testing: { unittestArgs: ['--foo'] }, -// }), -// } as unknown) as IConfigurationService; - -// stubTestServer = ({ -// sendCommand(): Promise { -// return Promise.resolve(); -// }, -// onDataReceived: () => { -// // no body -// }, -// } as unknown) as ITestServer; - -// stubResultResolver = ({ -// resolveDiscovery: () => { -// // no body -// }, -// resolveExecution: () => { -// // no body -// }, -// vsIdToRunId: { -// get: sinon.stub().returns('expectedRunId'), -// }, -// } as unknown) as ITestResultResolver; - -// // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); -// // const expectedRunId = 'expectedRunId'; -// // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); - -// // For some reason the 'tests' namespace in vscode returns undefined. -// // While I figure out how to expose to the tests, they will run -// // against a stub test controller and stub test items. -// const testItem = ({ -// canResolveChildren: false, -// tags: [], -// children: { -// add: () => { -// // empty -// }, -// }, -// } as unknown) as TestItem; - -// testController = ({ -// items: { -// get: () => { -// log.push('get'); -// }, -// add: () => { -// log.push('add'); -// }, -// replace: () => { -// log.push('replace'); -// }, -// delete: () => { -// log.push('delete'); -// }, -// }, -// createTestItem: () => { -// log.push('createTestItem'); -// return testItem; -// }, -// dispose: () => { -// // empty -// }, -// } as unknown) as TestController; - -// // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); - -// const mockSendTelemetryEvent = ( -// eventName: EventName, -// _: number | Record | undefined, -// properties: unknown, -// ) => { -// telemetryEvent.push({ -// eventName, -// properties: properties as Record, -// }); -// }; - -// discoverTestsStub = sandbox.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); -// sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); -// outputChannel = typemoq.Mock.ofType(); -// }); - -// teardown(() => { -// telemetryEvent = []; -// log = []; -// testController.dispose(); -// sandbox.restore(); -// }); - -// test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { -// discoverTestsStub.resolves(); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); - -// sinon.assert.calledOnce(discoverTestsStub); -// }); - -// test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { -// discoverTestsStub.callsFake( -// async () => -// new Promise((resolve) => { -// setTimeout(() => { -// // Simulate time taken by discovery. -// resolve(); -// }, 2000); -// }), -// ); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// // Try running discovery twice -// const one = workspaceTestAdapter.discoverTests(testController); -// const two = workspaceTestAdapter.discoverTests(testController); - -// Promise.all([one, two]); - -// sinon.assert.calledOnce(discoverTestsStub); -// }); - -// test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { -// discoverTestsStub.resolves({ status: 'success' }); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); - -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); - -// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); -// assert.strictEqual(telemetryEvent.length, 2); - -// const lastEvent = telemetryEvent[1]; -// assert.strictEqual(lastEvent.properties.failed, false); -// }); - -// test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { -// discoverTestsStub.rejects(new Error('foo')); - -// const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); -// const testExecutionAdapter = new UnittestTestExecutionAdapter( -// stubTestServer, -// stubConfigSettings, -// outputChannel.object, -// ); - -// const workspaceTestAdapter = new WorkspaceTestAdapter( -// 'unittest', -// testDiscoveryAdapter, -// testExecutionAdapter, -// Uri.parse('foo'), -// stubResultResolver, -// ); - -// await workspaceTestAdapter.discoverTests(testController); - -// sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); -// assert.strictEqual(telemetryEvent.length, 2); - -// const lastEvent = telemetryEvent[1]; -// assert.ok(lastEvent.properties.failed); - -// assert.deepStrictEqual(log, ['createTestItem', 'add']); -// }); - -// /** -// * TODO To test: -// * - successful discovery but no data: delete everything from the test controller -// * - successful discovery with error status: add error node to tree -// * - single root: populate tree if there's no root node -// * - single root: update tree if there's a root node -// * - single root: delete tree if there are no tests in the test data -// * - multiroot: update the correct folders -// */ -// }); -// }); +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import * as typemoq from 'typemoq'; + +import { TestController, TestItem, TestItemCollection, TestRun, Uri } from 'vscode'; +import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; +import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; // 7/7 +import { WorkspaceTestAdapter } from '../../../client/testing/testController/workspaceTestAdapter'; +import * as Telemetry from '../../../client/telemetry'; +import { EventName } from '../../../client/telemetry/constants'; +import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; +import * as testItemUtilities from '../../../client/testing/testController/common/testItemUtilities'; +import * as util from '../../../client/testing/testController/common/utils'; +import * as ResultResolver from '../../../client/testing/testController/common/resultResolver'; + +suite('Workspace test adapter', () => { + suite('Test discovery', () => { + let stubTestServer: ITestServer; + let stubConfigSettings: IConfigurationService; + let stubResultResolver: ITestResultResolver; + + let discoverTestsStub: sinon.SinonStub; + let sendTelemetryStub: sinon.SinonStub; + let outputChannel: typemoq.IMock; + + let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + + // Stubbed test controller (see comment around L.40) + let testController: TestController; + let log: string[] = []; + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['--foo'] }, + }), + } as unknown) as IConfigurationService; + + stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + stubResultResolver = ({ + resolveDiscovery: () => { + // no body + }, + resolveExecution: () => { + // no body + }, + } as unknown) as ITestResultResolver; + + // const vsIdToRunIdGetStub = sinon.stub(stubResultResolver.vsIdToRunId, 'get'); + // const expectedRunId = 'expectedRunId'; + // vsIdToRunIdGetStub.withArgs(sinon.match.any).returns(expectedRunId); + + // For some reason the 'tests' namespace in vscode returns undefined. + // While I figure out how to expose to the tests, they will run + // against a stub test controller and stub test items. + const testItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + createTestItem: () => { + log.push('createTestItem'); + return testItem; + }, + dispose: () => { + // empty + }, + } as unknown) as TestController; + + // testController = tests.createTestController('mock-python-tests', 'Mock Python Tests'); + + const mockSendTelemetryEvent = ( + eventName: EventName, + _: number | Record | undefined, + properties: unknown, + ) => { + telemetryEvent.push({ + eventName, + properties: properties as Record, + }); + }; + + discoverTestsStub = sinon.stub(UnittestTestDiscoveryAdapter.prototype, 'discoverTests'); + sendTelemetryStub = sinon.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); + outputChannel = typemoq.Mock.ofType(); + }); + + teardown(() => { + telemetryEvent = []; + log = []; + testController.dispose(); + sinon.restore(); + }); + + test('If discovery failed correctly create error node', async () => { + discoverTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + const blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const testProvider = 'unittest'; + + const abc = await workspaceTestAdapter.discoverTests(testController); + console.log(abc); + + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); + }); + + test("When discovering tests, the workspace test adapter should call the test discovery adapter's discoverTest method", async () => { + discoverTestsStub.resolves(); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledOnce(discoverTestsStub); + }); + + test('If discovery is already running, do not call discoveryAdapter.discoverTests again', async () => { + discoverTestsStub.callsFake( + async () => + new Promise((resolve) => { + setTimeout(() => { + // Simulate time taken by discovery. + resolve(); + }, 2000); + }), + ); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + // Try running discovery twice + const one = workspaceTestAdapter.discoverTests(testController); + const two = workspaceTestAdapter.discoverTests(testController); + + Promise.all([one, two]); + + sinon.assert.calledOnce(discoverTestsStub); + }); + + test('If discovery succeeds, send a telemetry event with the "failed" key set to false', async () => { + discoverTestsStub.resolves({ status: 'success' }); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + + const lastEvent = telemetryEvent[1]; + assert.strictEqual(lastEvent.properties.failed, false); + }); + + test('If discovery failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { + discoverTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.discoverTests(testController); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_DISCOVERY_DONE); + assert.strictEqual(telemetryEvent.length, 2); + + const lastEvent = telemetryEvent[1]; + assert.ok(lastEvent.properties.failed); + }); + }); + suite('Test execution workspace test adapter', () => { + let stubTestServer: ITestServer; + let stubConfigSettings: IConfigurationService; + let stubResultResolver: ITestResultResolver; + let executionTestsStub: sinon.SinonStub; + let sendTelemetryStub: sinon.SinonStub; + let outputChannel: typemoq.IMock; + let runInstance: typemoq.IMock; + let testControllerMock: typemoq.IMock; + let telemetryEvent: { eventName: EventName; properties: Record }[] = []; + let resultResolver: ResultResolver.PythonResultResolver; + + // Stubbed test controller (see comment around L.40) + let testController: TestController; + let log: string[] = []; + + const sandbox = sinon.createSandbox(); + + setup(() => { + stubConfigSettings = ({ + getSettings: () => ({ + testing: { unittestArgs: ['--foo'] }, + }), + } as unknown) as IConfigurationService; + + stubTestServer = ({ + sendCommand(): Promise { + return Promise.resolve(); + }, + onDataReceived: () => { + // no body + }, + } as unknown) as ITestServer; + + stubResultResolver = ({ + resolveDiscovery: () => { + // no body + }, + resolveExecution: () => { + // no body + }, + vsIdToRunId: { + get: sinon.stub().returns('expectedRunId'), + }, + } as unknown) as ITestResultResolver; + const testItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + + testController = ({ + items: { + get: () => { + log.push('get'); + }, + add: () => { + log.push('add'); + }, + replace: () => { + log.push('replace'); + }, + delete: () => { + log.push('delete'); + }, + }, + createTestItem: () => { + log.push('createTestItem'); + return testItem; + }, + dispose: () => { + // empty + }, + } as unknown) as TestController; + + const mockSendTelemetryEvent = ( + eventName: EventName, + _: number | Record | undefined, + properties: unknown, + ) => { + telemetryEvent.push({ + eventName, + properties: properties as Record, + }); + }; + + executionTestsStub = sandbox.stub(UnittestTestExecutionAdapter.prototype, 'runTests'); + sendTelemetryStub = sandbox.stub(Telemetry, 'sendTelemetryEvent').callsFake(mockSendTelemetryEvent); + outputChannel = typemoq.Mock.ofType(); + runInstance = typemoq.Mock.ofType(); + + const testProvider = 'pytest'; + const workspaceUri = Uri.file('foo'); + resultResolver = new ResultResolver.PythonResultResolver(testController, testProvider, workspaceUri); + }); + + teardown(() => { + telemetryEvent = []; + log = []; + testController.dispose(); + sandbox.restore(); + }); + test('When executing tests, the right tests should be sent to be executed', async () => { + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + resultResolver, + ); + resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1'); + + sinon.stub(testItemUtilities, 'getTestCaseNodes').callsFake((testNode: TestItem) => + // Custom implementation logic here based on the provided testNode and collection + + // Example implementation: returning a predefined array of TestItem objects + [testNode], + ); + + const mockTestItem1 = createMockTestItem('mockTestItem1'); + const mockTestItem2 = createMockTestItem('mockTestItem2'); + const mockTestItems: [string, TestItem][] = [ + ['1', mockTestItem1], + ['2', mockTestItem2], + // Add as many mock TestItems as needed + ]; + const iterableMock = mockTestItems[Symbol.iterator](); + + const testItemCollectionMock = typemoq.Mock.ofType(); + + testItemCollectionMock + .setup((x) => x.forEach(typemoq.It.isAny())) + .callback((callback) => { + let result = iterableMock.next(); + while (!result.done) { + callback(result.value[1]); + result = iterableMock.next(); + } + }) + .returns(() => mockTestItem1); + testControllerMock = typemoq.Mock.ofType(); + testControllerMock.setup((t) => t.items).returns(() => testItemCollectionMock.object); + + await workspaceTestAdapter.executeTests(testController, runInstance.object, [mockTestItem1, mockTestItem2]); + + runInstance.verify((r) => r.started(typemoq.It.isAny()), typemoq.Times.exactly(2)); + }); + + test("When executing tests, the workspace test adapter should call the test execute adapter's executionTest method", async () => { + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + sinon.assert.calledOnce(executionTestsStub); + }); + + test('If execution is already running, do not call executionAdapter.runTests again', async () => { + executionTestsStub.callsFake( + async () => + new Promise((resolve) => { + setTimeout(() => { + // Simulate time taken by discovery. + resolve(); + }, 2000); + }), + ); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + // Try running discovery twice + const one = workspaceTestAdapter.executeTests(testController, runInstance.object, []); + const two = workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + Promise.all([one, two]); + + sinon.assert.calledOnce(executionTestsStub); + }); + + test('If execution failed correctly create error node', async () => { + executionTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + const blankTestItem = ({ + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + } as unknown) as TestItem; + const errorTestItemOptions: testItemUtilities.ErrorTestItemOptions = { + id: 'id', + label: 'label', + error: 'error', + }; + const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); + const buildErrorNodeOptionsStub = sinon.stub(util, 'buildErrorNodeOptions').returns(errorTestItemOptions); + const testProvider = 'unittest'; + + await workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + sinon.assert.calledWithMatch(createErrorTestItemStub, sinon.match.any, sinon.match.any); + sinon.assert.calledWithMatch(buildErrorNodeOptionsStub, Uri.parse('foo'), sinon.match.any, testProvider); + }); + + test('If execution failed, send a telemetry event with the "failed" key set to true, and add an error node to the test controller', async () => { + executionTestsStub.rejects(new Error('foo')); + + const testDiscoveryAdapter = new UnittestTestDiscoveryAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + const testExecutionAdapter = new UnittestTestExecutionAdapter( + stubTestServer, + stubConfigSettings, + outputChannel.object, + ); + + const workspaceTestAdapter = new WorkspaceTestAdapter( + 'unittest', + testDiscoveryAdapter, + testExecutionAdapter, + Uri.parse('foo'), + stubResultResolver, + ); + + await workspaceTestAdapter.executeTests(testController, runInstance.object, []); + + sinon.assert.calledWith(sendTelemetryStub, EventName.UNITTEST_RUN_ALL_FAILED); + assert.strictEqual(telemetryEvent.length, 1); + }); + }); +}); + +function createMockTestItem(id: string): TestItem { + const range = typemoq.Mock.ofType(); + const mockTestItem = ({ + id, + canResolveChildren: false, + tags: [], + children: { + add: () => { + // empty + }, + }, + range, + uri: Uri.file('/foo/bar'), + } as unknown) as TestItem; + + return mockTestItem; +} diff --git a/src/test/vscode-mock.ts b/src/test/vscode-mock.ts index ebbe7ca59e72..44518e7575a7 100644 --- a/src/test/vscode-mock.ts +++ b/src/test/vscode-mock.ts @@ -70,6 +70,8 @@ mockedVSCode.Hover = vscodeMocks.Hover; mockedVSCode.Disposable = vscodeMocks.Disposable as any; mockedVSCode.ExtensionKind = vscodeMocks.ExtensionKind; mockedVSCode.CodeAction = vscodeMocks.CodeAction; +mockedVSCode.TestMessage = vscodeMocks.TestMessage; +mockedVSCode.Location = vscodeMocks.Location; mockedVSCode.EventEmitter = vscodeMocks.EventEmitter; mockedVSCode.CancellationTokenSource = vscodeMocks.CancellationTokenSource; mockedVSCode.CompletionItemKind = vscodeMocks.CompletionItemKind;