Fix #11684: Improve AfterTest() for Jasmine (#11957)

* dynamically collect test results in global var

* improve testFnWrapper for Jasmine

* create error message for 'after-hook' variable

* fix

* Update packages/wdio-utils/src/test-framework/testFnWrapper.ts

* Update packages/wdio-utils/src/test-framework/testFnWrapper.ts

* Update packages/wdio-utils/src/test-framework/testFnWrapper.ts

* add files with expected results

* add tests

* add config and write test results to files in afterTest()

* fix: assign test results to 'result' to avoid 'undefined'

* add smoke test and optimize assertions counting dynamic values

* fix

---------

Co-authored-by: Christian Bromann <git@bromann.dev>

packages/wdio-jasmine-framework/src/index.ts

@@ -201,6 +201,8 @@ class JasmineAdapter {
             self._lastTest = this.result
             // @ts-ignore overwrite existing type
             self._lastTest.start = new Date().getTime()
+            globalThis._wdioDynamicJasmineResultErrorList = this.result.failedExpectations
+            globalThis._jasmineTestResult = this.result
             executeMock.apply(this, args)
         }
 

packages/wdio-utils/src/constants.ts

@@ -115,4 +115,4 @@ export const HOOK_DEFINITION = {
             throw new Error('expected hook to be type of function')
         }
     }
-}
+}

packages/wdio-utils/src/test-framework/testFnWrapper.ts

@@ -8,11 +8,20 @@ import type {
     AfterHookParam
 } from './types.js'
 
+declare global {
+    // Firstly variable '_wdioDynamicJasmineResultErrorList' gets reference to test result in packages/wdio-jasmine-framework/src/index.ts and then used here in wdio-utils/ as workaround for Jasmine
+    // eslint-disable-next-line no-var
+    var _wdioDynamicJasmineResultErrorList: any | undefined
+    // eslint-disable-next-line no-var
+    var _jasmineTestResult: any | undefined
+}
+
 const STACKTRACE_FILTER = [
     'node_modules/webdriver/',
     'node_modules/webdriverio/',
     'node_modules/@wdio/',
     '(internal/process/task',
+    '(node:internal/process/task'
 ]
 
 /**
@@ -83,6 +92,16 @@ export const testFrameworkFnWrapper = async function (
     const testStart = Date.now()
     try {
         result = await executeAsync.call(this, specFn, retries, specFnArgs, timeout)
+        if (globalThis._jasmineTestResult !== undefined) {
+            result = globalThis._jasmineTestResult
+            globalThis._jasmineTestResult = undefined
+        }
+
+        if (globalThis._wdioDynamicJasmineResultErrorList?.length > 0) {
+            globalThis._wdioDynamicJasmineResultErrorList[0].stack = filterStackTrace(globalThis._wdioDynamicJasmineResultErrorList[0].stack)
+            error = globalThis._wdioDynamicJasmineResultErrorList[0]
+            globalThis._wdioDynamicJasmineResultErrorList = undefined
+        }
     } catch (err: any) {
         if (err.stack) {
             err.stack = filterStackTrace(err.stack)

tests/helpers/jasmine-after-hook-validation/expected-results/expectedTestFailed.json

@@ -0,0 +1,59 @@
+{
+    "test": {
+        "id": "spec1",
+        "description": "should failed",
+        "fullName": "Jasmine afterTest() hook validation should failed",
+        "parentSuiteId": "suite1",
+        "failedExpectations": [
+            {
+                "matcherName": "toBe",
+                "message": "Expected false to be true.",
+                "passed": false,
+                "expected": true,
+                "actual": false
+            }
+        ],
+        "passedExpectations": [],
+        "deprecationWarnings": [],
+        "pendingReason": "",
+        "duration": null,
+        "properties": null,
+        "debugLogs": null
+    },
+    "context": {
+        "wdioRetries": 0
+    },
+    "error": {
+        "matcherName": "toBe",
+        "message": "Expected false to be true.",
+        "passed": false,
+        "expected": true,
+        "actual": false
+    },
+    "result": {
+        "id": "spec1",
+        "description": "should failed",
+        "fullName": "Jasmine afterTest() hook validation should failed",
+        "parentSuiteId": "suite1",
+        "failedExpectations": [
+            {
+                "matcherName": "toBe",
+                "message": "Expected false to be true.",
+                "passed": false,
+                "expected": true,
+                "actual": false
+            }
+        ],
+        "passedExpectations": [],
+        "deprecationWarnings": [],
+        "pendingReason": "",
+        "duration": null,
+        "properties": null,
+        "debugLogs": null
+    },
+    "passed": false,
+    "retries": {
+        "attempts": 0,
+        "limit": 0
+    }
+}

tests/helpers/jasmine-after-hook-validation/expected-results/expectedTestPassed.json

@@ -0,0 +1,50 @@
+{
+    "test": {
+        "id": "spec0",
+        "description": "should pass",
+        "fullName": "Jasmine afterTest() hook validation should pass",
+        "parentSuiteId": "suite1",
+        "failedExpectations": [],
+        "passedExpectations": [
+            {
+                "matcherName": "toBe",
+                "message": "Passed.",
+                "stack": "",
+                "passed": true
+            }
+        ],
+        "deprecationWarnings": [],
+        "pendingReason": "",
+        "duration": null,
+        "properties": null,
+        "debugLogs": null
+    },
+    "context": {
+        "wdioRetries": 0
+    },
+    "result": {
+        "id": "spec0",
+        "description": "should pass",
+        "fullName": "Jasmine afterTest() hook validation should pass",
+        "parentSuiteId": "suite1",
+        "failedExpectations": [],
+        "passedExpectations": [
+            {
+                "matcherName": "toBe",
+                "message": "Passed.",
+                "stack": "",
+                "passed": true
+            }
+        ],
+        "deprecationWarnings": [],
+        "pendingReason": "",
+        "duration": null,
+        "properties": null,
+        "debugLogs": null
+    },
+    "passed": true,
+    "retries": {
+        "attempts": 0,
+        "limit": 0
+    }
+}

tests/helpers/jasmine.after-hook-validation.conf.js

@@ -0,0 +1,41 @@
+import path from 'node:path'
+import url from 'node:url'
+import fs from 'node:fs/promises'
+
+import { config as baseConfig } from './config.js'
+
+const __dirname = path.dirname(url.fileURLToPath(import.meta.url))
+
+export const config = {
+    ...baseConfig,
+    reporters: [
+        ['spec', {
+            addConsoleLogs: true
+        }]
+    ],
+    framework: 'jasmine',
+    jasmineOpts: {
+        ...baseConfig.jasmineOpts,
+    },
+    afterTest: async function (
+        test,
+        context,
+        { error, result, duration, passed, retries }
+    ) {
+        const actualTestResult = {
+            test: test,
+            context: context,
+            error: error,
+            result: result,
+            duration: duration,
+            passed: passed,
+            retries: retries
+        }
+
+        if (test.description.includes('pass')) {
+            fs.writeFile(path.resolve(__dirname, 'actualResultsPassed.log'), JSON.stringify(actualTestResult))
+        } else {
+            fs.writeFile(path.resolve(__dirname, 'actualResultsFailed.log'), JSON.stringify(actualTestResult))
+        }
+    }
+}

tests/jasmine/test.after-hook-validation.ts

@@ -0,0 +1,9 @@
+describe('Jasmine afterTest() hook validation', () => {
+    it('should pass', () => {
+        expect(true).toBe(true)
+    })
+
+    it('should failed', () => {
+        expect(false).toBe(true)
+    })
+})

tests/smoke.runner.js

@@ -742,6 +742,9 @@ const runSpecsWithFlagNoArg = async () => {
 }
 // *** END - tests for CLI --spec ***
 
+// *************************
+// *** Tests for Jasmine ***
+// *************************
 const jasmineHooksTestrunner = async () => {
     const logFile = path.join(__dirname, 'jasmineHooksTestrunner.spec.log')
     await launch('jasmineHooksTestrunner',
@@ -766,6 +769,59 @@ const jasmineHooksTestrunner = async () => {
     )
 }
 
+const jasmineAfterHookArgsValidation = async () => {
+    const expectedPassedTestResultPath = path.join(__dirname, 'helpers', 'jasmine-after-hook-validation', 'expected-results', 'expectedTestPassed.json')
+    const expectedFailedTestResultPath = path.join(__dirname, 'helpers', 'jasmine-after-hook-validation', 'expected-results', 'expectedTestFailed.json')
+
+    // Actual test results are written to files in tests\helpers\jasmine.after-hook-validation.conf.js - afterTest()
+    const actualPassedTestResultPath = path.join(__dirname, 'helpers', 'actualResultsPassed.log')
+    const actualFailedTestResultPath = path.join(__dirname, 'helpers', 'actualResultsFailed.log')
+
+    await launch('jasmineAfterHookArgsValidation',
+        path.resolve(__dirname, 'helpers', 'jasmine.after-hook-validation.conf.js'),
+        {
+            autoCompileOpts: { autoCompile: false },
+            specs: [
+                path.resolve(__dirname, 'jasmine', 'test.after-hook-validation.ts')
+            ]
+        }).catch((err) => err) // error expected
+
+    const actualPassedTestLogs = JSON.parse((await fs.readFile(actualPassedTestResultPath)))
+    const actualFailedTestLogs = JSON.parse((await fs.readFile(actualFailedTestResultPath)))
+    const expectedPassedTestLogs = JSON.parse((await fs.readFile(expectedPassedTestResultPath)))
+    const expectedFailedTestLogs = JSON.parse((await fs.readFile(expectedFailedTestResultPath)))
+
+    // Check before removing
+    assert.equal(typeof actualPassedTestLogs.test.start, 'number')
+    assert.equal(typeof actualPassedTestLogs.result.start, 'number')
+    assert.equal(typeof actualPassedTestLogs.duration, 'number')
+    assert.equal(typeof actualFailedTestLogs.test.start, 'number')
+    assert.equal(typeof actualFailedTestLogs.duration, 'number')
+    assert.equal(typeof actualFailedTestLogs.test.failedExpectations[0].stack, 'string')
+    assert.equal(typeof actualFailedTestLogs.result.failedExpectations[0].stack, 'string')
+    assert.equal(typeof actualFailedTestLogs.result.start, 'number')
+    assert.equal(typeof actualFailedTestLogs.error.stack, 'string')
+
+    // Remove dynamic values that will be different every time you run tests, e.g. start time or filepaths
+    delete actualPassedTestLogs.test.start
+    delete actualPassedTestLogs.test.filename
+    delete actualPassedTestLogs.result.start
+    delete actualPassedTestLogs.result.filename
+    delete actualPassedTestLogs.duration
+    delete actualFailedTestLogs.test.start
+    delete actualFailedTestLogs.test.filename
+    delete actualFailedTestLogs.test.failedExpectations[0].stack
+    delete actualFailedTestLogs.error.stack
+    delete actualFailedTestLogs.result.start
+    delete actualFailedTestLogs.result.filename
+    delete actualFailedTestLogs.result.failedExpectations[0].stack
+    delete actualFailedTestLogs.duration
+
+    assert.deepStrictEqual(actualPassedTestLogs, expectedPassedTestLogs)
+    assert.deepStrictEqual(actualFailedTestLogs, expectedFailedTestLogs)
+}
+// *** END - Tests for Jasmine ***
+
 (async () => {
     const smokeTests = [
         mochaTestrunner,
@@ -801,7 +857,8 @@ const jasmineHooksTestrunner = async () => {
         runSpecsWithFlagSeveralPassed,
         runSpecsWithFlagDirectPath,
         runSpecsWithFlagNoArg,
-        jasmineHooksTestrunner
+        jasmineHooksTestrunner,
+        jasmineAfterHookArgsValidation
     ]
 
     console.log('\nRunning smoke tests...\n')