From f0416c4c94e1c33cff96fe3aa712e943b4268c74 Mon Sep 17 00:00:00 2001 From: Zack Grannan Date: Fri, 19 Jan 2024 19:38:46 -0800 Subject: [PATCH 1/3] Update vscode/test-electron to 2.3.8, fixing CI issues --- client/package-lock.json | 20 ++++++++++---------- client/package.json | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 223ed417..c6f9459c 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1,12 +1,12 @@ { "name": "viper", - "version": "4.1.1", + "version": "4.2.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "viper", - "version": "4.1.1", + "version": "4.2.2", "license": "SEE LICENSE IN LICENSE.txt", "devDependencies": { "@types/glob": "^8.1.0", @@ -19,7 +19,7 @@ "@typescript-eslint/eslint-plugin": "^5.54.0", "@typescript-eslint/parser": "^5.54.0", "@viperproject/locate-java-home": "^1.1.11", - "@vscode/test-electron": "^2.3.0", + "@vscode/test-electron": "^2.3.8", "@vscode/vsce": "^2.18.0", "eslint": "^8.35.0", "eslint-webpack-plugin": "^4.0.0", @@ -877,15 +877,15 @@ } }, "node_modules/@vscode/test-electron": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@vscode/test-electron/-/test-electron-2.3.0.tgz", - "integrity": "sha512-fwzA9RtazH1GT/sckYlbxu6t5e4VaMXwCVtyLv4UAG0hP6NTfnMaaG25XCfWqlVwFhBMcQXHBCy5dmz2eLUnkw==", + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/@vscode/test-electron/-/test-electron-2.3.8.tgz", + "integrity": "sha512-b4aZZsBKtMGdDljAsOPObnAi7+VWIaYl3ylCz1jTs+oV6BZ4TNHcVNC3xUn0azPeszBmwSBDQYfFESIaUQnrOg==", "dev": true, "dependencies": { "http-proxy-agent": "^4.0.1", "https-proxy-agent": "^5.0.0", "jszip": "^3.10.1", - "semver": "^7.3.8" + "semver": "^7.5.2" }, "engines": { "node": ">=16" @@ -5429,9 +5429,9 @@ "dev": true }, "node_modules/semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" diff --git a/client/package.json b/client/package.json index f8d349ee..8bfdfffa 100644 --- a/client/package.json +++ b/client/package.json @@ -44,7 +44,7 @@ "@typescript-eslint/eslint-plugin": "^5.54.0", "@typescript-eslint/parser": "^5.54.0", "@viperproject/locate-java-home": "^1.1.11", - "@vscode/test-electron": "^2.3.0", + "@vscode/test-electron": "^2.3.8", "@vscode/vsce": "^2.18.0", "eslint": "^8.35.0", "eslint-webpack-plugin": "^4.0.0", From bd7dfac973dcd14249b07132302b1f0a581eeb5c Mon Sep 17 00:00:00 2001 From: Zack Grannan Date: Fri, 19 Jan 2024 20:32:28 -0800 Subject: [PATCH 2/3] test for CI --- client/src/test/1_ide.test.ts | 154 +++++++++++++------------- client/src/test/2_stress.test.ts | 89 --------------- client/src/test/3_updateTools.test.ts | 83 -------------- client/src/test/4_workspace.test.ts | 25 ----- 4 files changed, 77 insertions(+), 274 deletions(-) delete mode 100644 client/src/test/2_stress.test.ts delete mode 100644 client/src/test/3_updateTools.test.ts delete mode 100644 client/src/test/4_workspace.test.ts diff --git a/client/src/test/1_ide.test.ts b/client/src/test/1_ide.test.ts index 820ba272..7181b58e 100644 --- a/client/src/test/1_ide.test.ts +++ b/client/src/test/1_ide.test.ts @@ -15,7 +15,7 @@ suite('ViperIDE Tests', () => { suiteTeardown(async function() { await TestHelper.teardown(); }); - + test("Test abort", async function() { this.timeout(30000); @@ -36,82 +36,82 @@ suite('ViperIDE Tests', () => { assert (!TestHelper.hasObservedInternalError()); }); - test("Test closing files", async function() { - this.timeout(30000); - TestHelper.resetErrors(); - - await TestHelper.openAndVerify(LONG); - await TestHelper.wait(500); - await TestHelper.closeFile(); - await TestHelper.openFile(SIMPLE); - await TestHelper.wait(200); - await TestHelper.stopVerification(); - await TestHelper.closeFile(); - await TestHelper.openFile(LONG); - await TestHelper.verify(); // otherwise, `LONG` might not get verified because it has been verified in the past - await TestHelper.waitForVerification(LONG); - assert (!TestHelper.hasObservedInternalError()); - }); - - test("Test not verifying verified files", async function() { - this.timeout(40000); - - await TestHelper.openAndVerify(SIMPLE); - // simulate context switch by opening non-viper file - await TestHelper.openFile(EMPTY_TXT); - const verificationStart = TestHelper.waitForVerificationStart(SIMPLE); - await TestHelper.openFile(SIMPLE); - // wait 1000ms for verification start - it should not start - const timeoutHit = TestHelper.waitForTimeout(1000, verificationStart); - assert(timeoutHit, "unwanted reverification of verified file after switching context"); - }); - - test("Test zooming", async function() { - this.timeout(20000); - - const activated = TestHelper.waitForExtensionActivation(); - await TestHelper.executeCommand("workbench.action.zoomIn") - await TestHelper.wait(5000); - await TestHelper.executeCommand("workbench.action.zoomOut"); - const timeoutHit = await TestHelper.waitForTimeout(9000, activated); - assert(timeoutHit, "Viper IDE was activated, but it should not be"); - }); - - test("Test autoVerify", async function() { - this.timeout(3000); - - // disable auto verify: - await TestHelper.executeCommand("viper.toggleAutoVerify"); - const started = TestHelper.waitForVerificationStart(LONG); - await TestHelper.openFile(LONG); - await TestHelper.openFile(SIMPLE); - const timeoutHit = await TestHelper.waitForTimeout(1000, started); - assert(timeoutHit, "verification was started even if autoVerify is disabled"); - // turn auto verify back on: - await TestHelper.executeCommand("viper.toggleAutoVerify"); - }); - - test("Test Helper Methods", async function() { - this.timeout(2000); - - await TestHelper.openFile(SIMPLE); - checkAssert(path.basename(Common.uriToString(Helper.getActiveFileUri())), SIMPLE, "active file"); - - checkAssert(Helper.formatProgress(12.9), "13%", "formatProgress"); - checkAssert(Helper.formatSeconds(12.99), "13.0 seconds", "formatSeconds"); - checkAssert(Helper.isViperSourceFile("/folder/file.vpr"), true, "isViperSourceFile unix path"); - checkAssert(Helper.isViperSourceFile("..\\.\\folder\\file.sil"), true, "isViperSourceFile relavive windows path"); - checkAssert(!Helper.isViperSourceFile("C:\\absolute\\path\\file.ts"), true, "isViperSourceFile absolute windows path"); - }); - - test("Test opening logFile", async function() { - this.timeout(2000); - - const opened = TestHelper.waitForLogFile(); - await TestHelper.executeCommand('viper.openLogFile'); - await opened; - await TestHelper.closeFile(); - }); + // test("Test closing files", async function() { + // this.timeout(30000); + // TestHelper.resetErrors(); + + // await TestHelper.openAndVerify(LONG); + // await TestHelper.wait(500); + // await TestHelper.closeFile(); + // await TestHelper.openFile(SIMPLE); + // await TestHelper.wait(200); + // await TestHelper.stopVerification(); + // await TestHelper.closeFile(); + // await TestHelper.openFile(LONG); + // await TestHelper.verify(); // otherwise, `LONG` might not get verified because it has been verified in the past + // await TestHelper.waitForVerification(LONG); + // assert (!TestHelper.hasObservedInternalError()); + // }); + + // test("Test not verifying verified files", async function() { + // this.timeout(40000); + + // await TestHelper.openAndVerify(SIMPLE); + // // simulate context switch by opening non-viper file + // await TestHelper.openFile(EMPTY_TXT); + // const verificationStart = TestHelper.waitForVerificationStart(SIMPLE); + // await TestHelper.openFile(SIMPLE); + // // wait 1000ms for verification start - it should not start + // const timeoutHit = TestHelper.waitForTimeout(1000, verificationStart); + // assert(timeoutHit, "unwanted reverification of verified file after switching context"); + // }); + + // test("Test zooming", async function() { + // this.timeout(20000); + + // const activated = TestHelper.waitForExtensionActivation(); + // await TestHelper.executeCommand("workbench.action.zoomIn") + // await TestHelper.wait(5000); + // await TestHelper.executeCommand("workbench.action.zoomOut"); + // const timeoutHit = await TestHelper.waitForTimeout(9000, activated); + // assert(timeoutHit, "Viper IDE was activated, but it should not be"); + // }); + + // test("Test autoVerify", async function() { + // this.timeout(3000); + + // // disable auto verify: + // await TestHelper.executeCommand("viper.toggleAutoVerify"); + // const started = TestHelper.waitForVerificationStart(LONG); + // await TestHelper.openFile(LONG); + // await TestHelper.openFile(SIMPLE); + // const timeoutHit = await TestHelper.waitForTimeout(1000, started); + // assert(timeoutHit, "verification was started even if autoVerify is disabled"); + // // turn auto verify back on: + // await TestHelper.executeCommand("viper.toggleAutoVerify"); + // }); + + // test("Test Helper Methods", async function() { + // this.timeout(2000); + + // await TestHelper.openFile(SIMPLE); + // checkAssert(path.basename(Common.uriToString(Helper.getActiveFileUri())), SIMPLE, "active file"); + + // checkAssert(Helper.formatProgress(12.9), "13%", "formatProgress"); + // checkAssert(Helper.formatSeconds(12.99), "13.0 seconds", "formatSeconds"); + // checkAssert(Helper.isViperSourceFile("/folder/file.vpr"), true, "isViperSourceFile unix path"); + // checkAssert(Helper.isViperSourceFile("..\\.\\folder\\file.sil"), true, "isViperSourceFile relavive windows path"); + // checkAssert(!Helper.isViperSourceFile("C:\\absolute\\path\\file.ts"), true, "isViperSourceFile absolute windows path"); + // }); + + // test("Test opening logFile", async function() { + // this.timeout(2000); + + // const opened = TestHelper.waitForLogFile(); + // await TestHelper.executeCommand('viper.openLogFile'); + // await opened; + // await TestHelper.closeFile(); + // }); }); function checkAssert(seen: T, expected: T, message: string): void { diff --git a/client/src/test/2_stress.test.ts b/client/src/test/2_stress.test.ts deleted file mode 100644 index 02d7f2a1..00000000 --- a/client/src/test/2_stress.test.ts +++ /dev/null @@ -1,89 +0,0 @@ -import assert from 'assert'; -import TestHelper, { CARBON, SETUP_TIMEOUT, SILICON, SIMPLE } from './TestHelper'; - -suite('ViperIDE Stress Tests', () => { - - suiteSetup(async function() { - this.timeout(SETUP_TIMEOUT); - await TestHelper.setup(); - }); - - suiteTeardown(async function() { - await TestHelper.teardown(); - }); - - test("1. multiple fast verification requests", async function() { - this.timeout(15000); - - TestHelper.resetErrors(); - // 1 verification is expected, there should be no subsequent ones - const verified = TestHelper.waitForVerification(SIMPLE) - .then(() => TestHelper.waitForTimeout(5000, TestHelper.waitForVerification(SIMPLE))); - await TestHelper.openFile(SIMPLE); - //submit 10 verification requests - for (let i = 0; i < 10; i++) { - await TestHelper.verify(); - } - const timeout = await verified; - assert(timeout, "multiple verifications seen"); - }); - - test("2. quickly change backends", async function() { - this.timeout(50000); - - TestHelper.resetErrors(); - - await TestHelper.selectBackend(CARBON); - await TestHelper.openFile(SIMPLE); - //submit 10 verification requests - for (let i = 0; i < 10; i++) { - await TestHelper.selectBackend(SILICON); - await TestHelper.selectBackend(CARBON); - } - - await TestHelper.wait(500); - await TestHelper.selectBackend(SILICON); - await TestHelper.waitForVerification(SIMPLE, SILICON); - assert(!TestHelper.hasObservedInternalError()); - }); - - test("3. quickly start, stop, and restart verification", async function() { - this.timeout(15000); - - TestHelper.resetErrors(); - - await TestHelper.openFile(SIMPLE); - await TestHelper.verify(); - await TestHelper.stopVerification(); - const verified = TestHelper.waitForVerification(SIMPLE); - await TestHelper.verify(); - await verified; - assert(!TestHelper.hasObservedInternalError()); - }); - - test("4. closing all files right after starting verificaiton", async function() { - this.timeout(6000); - - TestHelper.resetErrors(); - - const end = new Promise(resolve => { - setTimeout(resolve, 5000); - }); - await TestHelper.verify(); - await TestHelper.executeCommand('workbench.action.closeAllEditors'); - await end; - assert(!TestHelper.hasObservedInternalError()); - }); - - test("Test simple verification with carbon", async function() { - this.timeout(35000); - - await TestHelper.openFile(SIMPLE); - const carbonVerified = TestHelper.waitForVerification(SIMPLE, CARBON); - await TestHelper.selectBackend(CARBON); - await carbonVerified; - const siliconVerified = TestHelper.waitForVerification(SIMPLE, SILICON); - await TestHelper.selectBackend(SILICON); - await siliconVerified; - }); -}); diff --git a/client/src/test/3_updateTools.test.ts b/client/src/test/3_updateTools.test.ts deleted file mode 100644 index 804d1154..00000000 --- a/client/src/test/3_updateTools.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import assert from 'assert'; -import { Log } from '../Log'; -import TestHelper, { LONG, SETUP_TIMEOUT, SIMPLE, VIPER_TOOLS_TIMEOUT } from './TestHelper'; - -suite('Viper Tools Update Test', () => { - - suiteSetup(async function() { - this.timeout(SETUP_TIMEOUT); - await TestHelper.setup(); - }); - - suiteTeardown(async function() { - await TestHelper.teardown(); - }); - - test("Viper Tools Update Test & test abort of first verification", async function() { - this.timeout(VIPER_TOOLS_TIMEOUT); - TestHelper.resetErrors(); - - const updateDone = TestHelper.waitForViperToolsUpdate(); - const activated = TestHelper.waitForExtensionActivation(); - const restarted = TestHelper.waitForExtensionRestart(); - await TestHelper.startViperToolsUpdate(); - - await updateDone; - TestHelper.log("Viper Tools Update done"); - await activated; - await restarted; - TestHelper.log("Extension has been restarted after performing Viper Tools update"); - - // open LONG such that it will be verified as soon as backend has started - // note that we open the file only after awaiting the extension's restart as - // the command to open the file otherwise seems to get lost - const aborted = TestHelper.waitForAbort(); - /* - await TestHelper.openFile(LONG); - - - TestHelper.log("extension has activated"); - await TestHelper.verify(); - */ - await TestHelper.openFile(LONG); - - // stop the verification after 1s - setTimeout(() => { - TestHelper.log("timeout triggered: stopping verification"); - TestHelper.stopVerification() - .catch(err => Log.error(`error while stopping verification: ${err}`)); - }, 1000); - - // wait until verification is aborted: - await aborted; - TestHelper.log("verification has been aborted"); - await TestHelper.checkForRunningProcesses(false, false, true); - - //reverify - await TestHelper.openAndVerify(LONG); - assert (!TestHelper.hasObservedInternalError()); - - await TestHelper.closeAllFiles(); - }); - - test("Viper Tools Update Test", async function() { - this.timeout(VIPER_TOOLS_TIMEOUT); - TestHelper.resetErrors(); - - const updateDone = TestHelper.waitForViperToolsUpdate(); - const activated = TestHelper.waitForExtensionActivation(); - await TestHelper.startViperToolsUpdate(); - - await updateDone; - TestHelper.log("Viper Tools Update done"); - await activated; - TestHelper.log("extension has activated"); - - assert (!TestHelper.hasObservedInternalError()); - }); - - test("Test verification is possible after viper tools update", async function() { - this.timeout(40000); - await TestHelper.openAndVerify(SIMPLE); - }); -}); diff --git a/client/src/test/4_workspace.test.ts b/client/src/test/4_workspace.test.ts deleted file mode 100644 index 93347c1f..00000000 --- a/client/src/test/4_workspace.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import assert from 'assert'; -import { Common } from '../ViperProtocol'; -import TestHelper, { DATA_ROOT, SETUP_TIMEOUT } from './TestHelper'; - -suite('Workspace Tests', () => { - - suiteSetup(async function() { - this.timeout(SETUP_TIMEOUT); - await TestHelper.setup(); - }); - - suiteTeardown(async function() { - await TestHelper.teardown(); - }); - - test("Test Verification of all files in folder", async function() { - this.timeout(200000); - - await TestHelper.closeAllFiles(); - await TestHelper.waitForIdle(); - await TestHelper.executeCommand('viper.verifyAllFilesInWorkspace', Common.uriToString(DATA_ROOT)); - const result = await TestHelper.waitForVerificationOfAllFilesInWorkspace(); - assert(result.verified == result.total, `partially verified workspace: (${result.verified}/${result.total})`); - }); -}); From da4681a8925a18ff41a6298dbb886e76f1e371e3 Mon Sep 17 00:00:00 2001 From: Zack Grannan Date: Fri, 19 Jan 2024 22:06:33 -0800 Subject: [PATCH 3/3] Revert "test for CI" This reverts commit bd7dfac973dcd14249b07132302b1f0a581eeb5c. --- client/src/test/1_ide.test.ts | 154 +++++++++++++------------- client/src/test/2_stress.test.ts | 89 +++++++++++++++ client/src/test/3_updateTools.test.ts | 83 ++++++++++++++ client/src/test/4_workspace.test.ts | 25 +++++ 4 files changed, 274 insertions(+), 77 deletions(-) create mode 100644 client/src/test/2_stress.test.ts create mode 100644 client/src/test/3_updateTools.test.ts create mode 100644 client/src/test/4_workspace.test.ts diff --git a/client/src/test/1_ide.test.ts b/client/src/test/1_ide.test.ts index 7181b58e..820ba272 100644 --- a/client/src/test/1_ide.test.ts +++ b/client/src/test/1_ide.test.ts @@ -15,7 +15,7 @@ suite('ViperIDE Tests', () => { suiteTeardown(async function() { await TestHelper.teardown(); }); - + test("Test abort", async function() { this.timeout(30000); @@ -36,82 +36,82 @@ suite('ViperIDE Tests', () => { assert (!TestHelper.hasObservedInternalError()); }); - // test("Test closing files", async function() { - // this.timeout(30000); - // TestHelper.resetErrors(); - - // await TestHelper.openAndVerify(LONG); - // await TestHelper.wait(500); - // await TestHelper.closeFile(); - // await TestHelper.openFile(SIMPLE); - // await TestHelper.wait(200); - // await TestHelper.stopVerification(); - // await TestHelper.closeFile(); - // await TestHelper.openFile(LONG); - // await TestHelper.verify(); // otherwise, `LONG` might not get verified because it has been verified in the past - // await TestHelper.waitForVerification(LONG); - // assert (!TestHelper.hasObservedInternalError()); - // }); - - // test("Test not verifying verified files", async function() { - // this.timeout(40000); - - // await TestHelper.openAndVerify(SIMPLE); - // // simulate context switch by opening non-viper file - // await TestHelper.openFile(EMPTY_TXT); - // const verificationStart = TestHelper.waitForVerificationStart(SIMPLE); - // await TestHelper.openFile(SIMPLE); - // // wait 1000ms for verification start - it should not start - // const timeoutHit = TestHelper.waitForTimeout(1000, verificationStart); - // assert(timeoutHit, "unwanted reverification of verified file after switching context"); - // }); - - // test("Test zooming", async function() { - // this.timeout(20000); - - // const activated = TestHelper.waitForExtensionActivation(); - // await TestHelper.executeCommand("workbench.action.zoomIn") - // await TestHelper.wait(5000); - // await TestHelper.executeCommand("workbench.action.zoomOut"); - // const timeoutHit = await TestHelper.waitForTimeout(9000, activated); - // assert(timeoutHit, "Viper IDE was activated, but it should not be"); - // }); - - // test("Test autoVerify", async function() { - // this.timeout(3000); - - // // disable auto verify: - // await TestHelper.executeCommand("viper.toggleAutoVerify"); - // const started = TestHelper.waitForVerificationStart(LONG); - // await TestHelper.openFile(LONG); - // await TestHelper.openFile(SIMPLE); - // const timeoutHit = await TestHelper.waitForTimeout(1000, started); - // assert(timeoutHit, "verification was started even if autoVerify is disabled"); - // // turn auto verify back on: - // await TestHelper.executeCommand("viper.toggleAutoVerify"); - // }); - - // test("Test Helper Methods", async function() { - // this.timeout(2000); - - // await TestHelper.openFile(SIMPLE); - // checkAssert(path.basename(Common.uriToString(Helper.getActiveFileUri())), SIMPLE, "active file"); - - // checkAssert(Helper.formatProgress(12.9), "13%", "formatProgress"); - // checkAssert(Helper.formatSeconds(12.99), "13.0 seconds", "formatSeconds"); - // checkAssert(Helper.isViperSourceFile("/folder/file.vpr"), true, "isViperSourceFile unix path"); - // checkAssert(Helper.isViperSourceFile("..\\.\\folder\\file.sil"), true, "isViperSourceFile relavive windows path"); - // checkAssert(!Helper.isViperSourceFile("C:\\absolute\\path\\file.ts"), true, "isViperSourceFile absolute windows path"); - // }); - - // test("Test opening logFile", async function() { - // this.timeout(2000); - - // const opened = TestHelper.waitForLogFile(); - // await TestHelper.executeCommand('viper.openLogFile'); - // await opened; - // await TestHelper.closeFile(); - // }); + test("Test closing files", async function() { + this.timeout(30000); + TestHelper.resetErrors(); + + await TestHelper.openAndVerify(LONG); + await TestHelper.wait(500); + await TestHelper.closeFile(); + await TestHelper.openFile(SIMPLE); + await TestHelper.wait(200); + await TestHelper.stopVerification(); + await TestHelper.closeFile(); + await TestHelper.openFile(LONG); + await TestHelper.verify(); // otherwise, `LONG` might not get verified because it has been verified in the past + await TestHelper.waitForVerification(LONG); + assert (!TestHelper.hasObservedInternalError()); + }); + + test("Test not verifying verified files", async function() { + this.timeout(40000); + + await TestHelper.openAndVerify(SIMPLE); + // simulate context switch by opening non-viper file + await TestHelper.openFile(EMPTY_TXT); + const verificationStart = TestHelper.waitForVerificationStart(SIMPLE); + await TestHelper.openFile(SIMPLE); + // wait 1000ms for verification start - it should not start + const timeoutHit = TestHelper.waitForTimeout(1000, verificationStart); + assert(timeoutHit, "unwanted reverification of verified file after switching context"); + }); + + test("Test zooming", async function() { + this.timeout(20000); + + const activated = TestHelper.waitForExtensionActivation(); + await TestHelper.executeCommand("workbench.action.zoomIn") + await TestHelper.wait(5000); + await TestHelper.executeCommand("workbench.action.zoomOut"); + const timeoutHit = await TestHelper.waitForTimeout(9000, activated); + assert(timeoutHit, "Viper IDE was activated, but it should not be"); + }); + + test("Test autoVerify", async function() { + this.timeout(3000); + + // disable auto verify: + await TestHelper.executeCommand("viper.toggleAutoVerify"); + const started = TestHelper.waitForVerificationStart(LONG); + await TestHelper.openFile(LONG); + await TestHelper.openFile(SIMPLE); + const timeoutHit = await TestHelper.waitForTimeout(1000, started); + assert(timeoutHit, "verification was started even if autoVerify is disabled"); + // turn auto verify back on: + await TestHelper.executeCommand("viper.toggleAutoVerify"); + }); + + test("Test Helper Methods", async function() { + this.timeout(2000); + + await TestHelper.openFile(SIMPLE); + checkAssert(path.basename(Common.uriToString(Helper.getActiveFileUri())), SIMPLE, "active file"); + + checkAssert(Helper.formatProgress(12.9), "13%", "formatProgress"); + checkAssert(Helper.formatSeconds(12.99), "13.0 seconds", "formatSeconds"); + checkAssert(Helper.isViperSourceFile("/folder/file.vpr"), true, "isViperSourceFile unix path"); + checkAssert(Helper.isViperSourceFile("..\\.\\folder\\file.sil"), true, "isViperSourceFile relavive windows path"); + checkAssert(!Helper.isViperSourceFile("C:\\absolute\\path\\file.ts"), true, "isViperSourceFile absolute windows path"); + }); + + test("Test opening logFile", async function() { + this.timeout(2000); + + const opened = TestHelper.waitForLogFile(); + await TestHelper.executeCommand('viper.openLogFile'); + await opened; + await TestHelper.closeFile(); + }); }); function checkAssert(seen: T, expected: T, message: string): void { diff --git a/client/src/test/2_stress.test.ts b/client/src/test/2_stress.test.ts new file mode 100644 index 00000000..02d7f2a1 --- /dev/null +++ b/client/src/test/2_stress.test.ts @@ -0,0 +1,89 @@ +import assert from 'assert'; +import TestHelper, { CARBON, SETUP_TIMEOUT, SILICON, SIMPLE } from './TestHelper'; + +suite('ViperIDE Stress Tests', () => { + + suiteSetup(async function() { + this.timeout(SETUP_TIMEOUT); + await TestHelper.setup(); + }); + + suiteTeardown(async function() { + await TestHelper.teardown(); + }); + + test("1. multiple fast verification requests", async function() { + this.timeout(15000); + + TestHelper.resetErrors(); + // 1 verification is expected, there should be no subsequent ones + const verified = TestHelper.waitForVerification(SIMPLE) + .then(() => TestHelper.waitForTimeout(5000, TestHelper.waitForVerification(SIMPLE))); + await TestHelper.openFile(SIMPLE); + //submit 10 verification requests + for (let i = 0; i < 10; i++) { + await TestHelper.verify(); + } + const timeout = await verified; + assert(timeout, "multiple verifications seen"); + }); + + test("2. quickly change backends", async function() { + this.timeout(50000); + + TestHelper.resetErrors(); + + await TestHelper.selectBackend(CARBON); + await TestHelper.openFile(SIMPLE); + //submit 10 verification requests + for (let i = 0; i < 10; i++) { + await TestHelper.selectBackend(SILICON); + await TestHelper.selectBackend(CARBON); + } + + await TestHelper.wait(500); + await TestHelper.selectBackend(SILICON); + await TestHelper.waitForVerification(SIMPLE, SILICON); + assert(!TestHelper.hasObservedInternalError()); + }); + + test("3. quickly start, stop, and restart verification", async function() { + this.timeout(15000); + + TestHelper.resetErrors(); + + await TestHelper.openFile(SIMPLE); + await TestHelper.verify(); + await TestHelper.stopVerification(); + const verified = TestHelper.waitForVerification(SIMPLE); + await TestHelper.verify(); + await verified; + assert(!TestHelper.hasObservedInternalError()); + }); + + test("4. closing all files right after starting verificaiton", async function() { + this.timeout(6000); + + TestHelper.resetErrors(); + + const end = new Promise(resolve => { + setTimeout(resolve, 5000); + }); + await TestHelper.verify(); + await TestHelper.executeCommand('workbench.action.closeAllEditors'); + await end; + assert(!TestHelper.hasObservedInternalError()); + }); + + test("Test simple verification with carbon", async function() { + this.timeout(35000); + + await TestHelper.openFile(SIMPLE); + const carbonVerified = TestHelper.waitForVerification(SIMPLE, CARBON); + await TestHelper.selectBackend(CARBON); + await carbonVerified; + const siliconVerified = TestHelper.waitForVerification(SIMPLE, SILICON); + await TestHelper.selectBackend(SILICON); + await siliconVerified; + }); +}); diff --git a/client/src/test/3_updateTools.test.ts b/client/src/test/3_updateTools.test.ts new file mode 100644 index 00000000..804d1154 --- /dev/null +++ b/client/src/test/3_updateTools.test.ts @@ -0,0 +1,83 @@ +import assert from 'assert'; +import { Log } from '../Log'; +import TestHelper, { LONG, SETUP_TIMEOUT, SIMPLE, VIPER_TOOLS_TIMEOUT } from './TestHelper'; + +suite('Viper Tools Update Test', () => { + + suiteSetup(async function() { + this.timeout(SETUP_TIMEOUT); + await TestHelper.setup(); + }); + + suiteTeardown(async function() { + await TestHelper.teardown(); + }); + + test("Viper Tools Update Test & test abort of first verification", async function() { + this.timeout(VIPER_TOOLS_TIMEOUT); + TestHelper.resetErrors(); + + const updateDone = TestHelper.waitForViperToolsUpdate(); + const activated = TestHelper.waitForExtensionActivation(); + const restarted = TestHelper.waitForExtensionRestart(); + await TestHelper.startViperToolsUpdate(); + + await updateDone; + TestHelper.log("Viper Tools Update done"); + await activated; + await restarted; + TestHelper.log("Extension has been restarted after performing Viper Tools update"); + + // open LONG such that it will be verified as soon as backend has started + // note that we open the file only after awaiting the extension's restart as + // the command to open the file otherwise seems to get lost + const aborted = TestHelper.waitForAbort(); + /* + await TestHelper.openFile(LONG); + + + TestHelper.log("extension has activated"); + await TestHelper.verify(); + */ + await TestHelper.openFile(LONG); + + // stop the verification after 1s + setTimeout(() => { + TestHelper.log("timeout triggered: stopping verification"); + TestHelper.stopVerification() + .catch(err => Log.error(`error while stopping verification: ${err}`)); + }, 1000); + + // wait until verification is aborted: + await aborted; + TestHelper.log("verification has been aborted"); + await TestHelper.checkForRunningProcesses(false, false, true); + + //reverify + await TestHelper.openAndVerify(LONG); + assert (!TestHelper.hasObservedInternalError()); + + await TestHelper.closeAllFiles(); + }); + + test("Viper Tools Update Test", async function() { + this.timeout(VIPER_TOOLS_TIMEOUT); + TestHelper.resetErrors(); + + const updateDone = TestHelper.waitForViperToolsUpdate(); + const activated = TestHelper.waitForExtensionActivation(); + await TestHelper.startViperToolsUpdate(); + + await updateDone; + TestHelper.log("Viper Tools Update done"); + await activated; + TestHelper.log("extension has activated"); + + assert (!TestHelper.hasObservedInternalError()); + }); + + test("Test verification is possible after viper tools update", async function() { + this.timeout(40000); + await TestHelper.openAndVerify(SIMPLE); + }); +}); diff --git a/client/src/test/4_workspace.test.ts b/client/src/test/4_workspace.test.ts new file mode 100644 index 00000000..93347c1f --- /dev/null +++ b/client/src/test/4_workspace.test.ts @@ -0,0 +1,25 @@ +import assert from 'assert'; +import { Common } from '../ViperProtocol'; +import TestHelper, { DATA_ROOT, SETUP_TIMEOUT } from './TestHelper'; + +suite('Workspace Tests', () => { + + suiteSetup(async function() { + this.timeout(SETUP_TIMEOUT); + await TestHelper.setup(); + }); + + suiteTeardown(async function() { + await TestHelper.teardown(); + }); + + test("Test Verification of all files in folder", async function() { + this.timeout(200000); + + await TestHelper.closeAllFiles(); + await TestHelper.waitForIdle(); + await TestHelper.executeCommand('viper.verifyAllFilesInWorkspace', Common.uriToString(DATA_ROOT)); + const result = await TestHelper.waitForVerificationOfAllFilesInWorkspace(); + assert(result.verified == result.total, `partially verified workspace: (${result.verified}/${result.total})`); + }); +});