From 0072751e338012d77768a6acdbb949903a481af0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 23 Aug 2025 04:51:46 +0000 Subject: [PATCH 1/2] Initial plan From 9fc5836775830f2eebb5c4811c3e9b15cbd6cdce Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 23 Aug 2025 05:07:07 +0000 Subject: [PATCH 2/2] Add comprehensive test coverage and finalize retry fix implementation --- lib/codecept.js | 1 + lib/helper/Mochawesome.js | 26 +++++- lib/listener/retryEnhancer.js | 85 +++++++++++++++++ test/unit/mocha/mochawesome_retry_test.js | 98 +++++++++++++++++++ test/unit/mocha/retry_integration_test.js | 109 ++++++++++++++++++++++ test/unit/mocha/test_clone_test.js | 96 +++++++++++++++++++ 6 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 lib/listener/retryEnhancer.js create mode 100644 test/unit/mocha/mochawesome_retry_test.js create mode 100644 test/unit/mocha/retry_integration_test.js diff --git a/lib/codecept.js b/lib/codecept.js index 06752f593..c9f9aa9b8 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -111,6 +111,7 @@ class Codecept { runHook(require('./listener/helpers')) runHook(require('./listener/globalTimeout')) runHook(require('./listener/globalRetry')) + runHook(require('./listener/retryEnhancer')) runHook(require('./listener/exit')) runHook(require('./listener/emptyRun')) diff --git a/lib/helper/Mochawesome.js b/lib/helper/Mochawesome.js index 0f45ff723..181ba414e 100644 --- a/lib/helper/Mochawesome.js +++ b/lib/helper/Mochawesome.js @@ -37,7 +37,20 @@ class Mochawesome extends Helper { } _test(test) { - currentTest = { test } + // If this is a retried test, we want to add context to the retried test + // but also potentially preserve context from the original test + const originalTest = test.retriedTest && test.retriedTest() + if (originalTest) { + // This is a retried test - use the retried test for context + currentTest = { test } + + // Optionally copy context from original test if it exists + // Note: mochawesome context is stored in test.ctx, but we need to be careful + // not to break the mocha context structure + } else { + // Normal test (not a retry) + currentTest = { test } + } } _failed(test) { @@ -64,7 +77,16 @@ class Mochawesome extends Helper { addMochawesomeContext(context) { if (currentTest === '') currentTest = { test: currentSuite.ctx.test } - return this._addContext(currentTest, context) + + // For retried tests, make sure we're adding context to the current (retried) test + // not the original test + let targetTest = currentTest + if (currentTest.test && currentTest.test.retriedTest && currentTest.test.retriedTest()) { + // This test has been retried, make sure we're using the current test for context + targetTest = { test: currentTest.test } + } + + return this._addContext(targetTest, context) } } diff --git a/lib/listener/retryEnhancer.js b/lib/listener/retryEnhancer.js new file mode 100644 index 000000000..d53effca8 --- /dev/null +++ b/lib/listener/retryEnhancer.js @@ -0,0 +1,85 @@ +const event = require('../event') +const { enhanceMochaTest } = require('../mocha/test') + +/** + * Enhance retried tests by copying CodeceptJS-specific properties from the original test + * This fixes the issue where Mocha's shallow clone during retries loses CodeceptJS properties + */ +module.exports = function () { + event.dispatcher.on(event.test.before, test => { + // Check if this test is a retry (has a reference to the original test) + const originalTest = test.retriedTest && test.retriedTest() + + if (originalTest) { + // This is a retried test - copy CodeceptJS-specific properties from the original + copyCodeceptJSProperties(originalTest, test) + + // Ensure the test is enhanced with CodeceptJS functionality + enhanceMochaTest(test) + } + }) +} + +/** + * Copy CodeceptJS-specific properties from the original test to the retried test + * @param {CodeceptJS.Test} originalTest - The original test object + * @param {CodeceptJS.Test} retriedTest - The retried test object + */ +function copyCodeceptJSProperties(originalTest, retriedTest) { + // Copy CodeceptJS-specific properties + if (originalTest.opts !== undefined) { + retriedTest.opts = originalTest.opts ? { ...originalTest.opts } : {} + } + + if (originalTest.tags !== undefined) { + retriedTest.tags = originalTest.tags ? [...originalTest.tags] : [] + } + + if (originalTest.notes !== undefined) { + retriedTest.notes = originalTest.notes ? [...originalTest.notes] : [] + } + + if (originalTest.meta !== undefined) { + retriedTest.meta = originalTest.meta ? { ...originalTest.meta } : {} + } + + if (originalTest.artifacts !== undefined) { + retriedTest.artifacts = originalTest.artifacts ? [...originalTest.artifacts] : [] + } + + if (originalTest.steps !== undefined) { + retriedTest.steps = originalTest.steps ? [...originalTest.steps] : [] + } + + if (originalTest.config !== undefined) { + retriedTest.config = originalTest.config ? { ...originalTest.config } : {} + } + + if (originalTest.inject !== undefined) { + retriedTest.inject = originalTest.inject ? { ...originalTest.inject } : {} + } + + // Copy methods that might be missing + if (originalTest.addNote && !retriedTest.addNote) { + retriedTest.addNote = function (type, note) { + this.notes = this.notes || [] + this.notes.push({ type, text: note }) + } + } + + if (originalTest.applyOptions && !retriedTest.applyOptions) { + retriedTest.applyOptions = originalTest.applyOptions.bind(retriedTest) + } + + if (originalTest.simplify && !retriedTest.simplify) { + retriedTest.simplify = originalTest.simplify.bind(retriedTest) + } + + // Preserve the uid if it exists + if (originalTest.uid !== undefined) { + retriedTest.uid = originalTest.uid + } + + // Mark as enhanced + retriedTest.codeceptjs = true +} diff --git a/test/unit/mocha/mochawesome_retry_test.js b/test/unit/mocha/mochawesome_retry_test.js new file mode 100644 index 000000000..9af8616f5 --- /dev/null +++ b/test/unit/mocha/mochawesome_retry_test.js @@ -0,0 +1,98 @@ +const { expect } = require('chai') +const { createTest } = require('../../../lib/mocha/test') +const { createSuite } = require('../../../lib/mocha/suite') +const MochaSuite = require('mocha/lib/suite') +const Test = require('mocha/lib/test') +const Mochawesome = require('../../../lib/helper/Mochawesome') +const retryEnhancer = require('../../../lib/listener/retryEnhancer') +const event = require('../../../lib/event') + +describe('MochawesomeHelper with retries', function () { + let helper + + beforeEach(function () { + helper = new Mochawesome({}) + // Setup the retryEnhancer + retryEnhancer() + }) + + it('should add context to the correct test object when test is retried', function () { + // Create a CodeceptJS enhanced test + const originalTest = createTest('Test with mochawesome context', () => {}) + + // Create a mock suite and set up context + const rootSuite = new MochaSuite('', null, true) + const suite = createSuite(rootSuite, 'Test Suite') + originalTest.addToSuite(suite) + + // Set some CodeceptJS-specific properties + originalTest.opts = { timeout: 5000 } + originalTest.meta = { feature: 'reporting' } + + // Simulate what happens during mocha retries - using mocha's native clone method + const retriedTest = Test.prototype.clone.call(originalTest) + + // Trigger the retryEnhancer to copy properties + event.emit(event.test.before, retriedTest) + + // Verify that properties were copied + expect(retriedTest.opts).to.deep.equal({ timeout: 5000 }) + expect(retriedTest.meta).to.deep.equal({ feature: 'reporting' }) + + // Now simulate the test lifecycle hooks + helper._beforeSuite(suite) + helper._test(retriedTest) // This should set currentTest to the retried test + + // Add some context using the helper + const contextData = { screenshot: 'test.png', url: 'http://example.com' } + + // Mock the _addContext method to capture what test object is passed + let contextAddedToTest = null + helper._addContext = function (testWrapper, context) { + contextAddedToTest = testWrapper.test + return Promise.resolve() + } + + // Add context + helper.addMochawesomeContext(contextData) + + // The context should be added to the retried test, not the original + expect(contextAddedToTest).to.equal(retriedTest) + expect(contextAddedToTest).to.not.equal(originalTest) + + // Verify the retried test has the enhanced properties + expect(contextAddedToTest.opts).to.deep.equal({ timeout: 5000 }) + expect(contextAddedToTest.meta).to.deep.equal({ feature: 'reporting' }) + }) + + it('should add context to normal test when not retried', function () { + // Create a normal (non-retried) CodeceptJS enhanced test + const normalTest = createTest('Normal test', () => {}) + + // Create a mock suite + const rootSuite = new MochaSuite('', null, true) + const suite = createSuite(rootSuite, 'Test Suite') + normalTest.addToSuite(suite) + + // Simulate the test lifecycle hooks + helper._beforeSuite(suite) + helper._test(normalTest) + + // Mock the _addContext method to capture what test object is passed + let contextAddedToTest = null + helper._addContext = function (testWrapper, context) { + contextAddedToTest = testWrapper.test + return Promise.resolve() + } + + // Add some context using the helper + const contextData = { screenshot: 'normal.png' } + helper.addMochawesomeContext(contextData) + + // The context should be added to the normal test + expect(contextAddedToTest).to.equal(normalTest) + + // Verify this is not a retried test + expect(normalTest.retriedTest()).to.be.undefined + }) +}) diff --git a/test/unit/mocha/retry_integration_test.js b/test/unit/mocha/retry_integration_test.js new file mode 100644 index 000000000..357f1a4fe --- /dev/null +++ b/test/unit/mocha/retry_integration_test.js @@ -0,0 +1,109 @@ +const { expect } = require('chai') +const { createTest } = require('../../../lib/mocha/test') +const { createSuite } = require('../../../lib/mocha/suite') +const MochaSuite = require('mocha/lib/suite') +const retryEnhancer = require('../../../lib/listener/retryEnhancer') +const event = require('../../../lib/event') + +describe('Integration test: Retries with CodeceptJS properties', function () { + beforeEach(function () { + // Setup the retryEnhancer - this simulates what happens in CodeceptJS init + retryEnhancer() + }) + + it('should preserve all CodeceptJS properties during real retry scenario', function () { + // Create a test with retries like: Scenario().retries(2) + const originalTest = createTest('Test that might fail', () => { + throw new Error('Simulated failure') + }) + + // Set up test with various CodeceptJS properties that might be used in real scenarios + originalTest.opts = { + timeout: 30000, + metadata: 'important-test', + retries: 2, + feature: 'login', + } + originalTest.tags = ['@critical', '@smoke', '@login'] + originalTest.notes = [ + { type: 'info', text: 'This test validates user login' }, + { type: 'warning', text: 'May be flaky due to external service' }, + ] + originalTest.meta = { + feature: 'authentication', + story: 'user-login', + priority: 'high', + team: 'qa', + } + originalTest.artifacts = ['login-screenshot.png', 'network-log.json'] + originalTest.uid = 'auth-test-001' + originalTest.config = { helper: 'playwright', baseUrl: 'http://test.com' } + originalTest.inject = { userData: { email: 'test@example.com' } } + + // Add some steps to simulate CodeceptJS test steps + originalTest.steps = [ + { title: 'I am on page "/login"', status: 'success' }, + { title: 'I fill field "email", "test@example.com"', status: 'success' }, + { title: 'I fill field "password", "secretpassword"', status: 'success' }, + { title: 'I click "Login"', status: 'failed' }, + ] + + // Enable retries + originalTest.retries(2) + + // Now simulate what happens during mocha retry + const retriedTest = originalTest.clone() + + // Verify that the retried test has reference to original + expect(retriedTest.retriedTest()).to.equal(originalTest) + + // Before our fix, these properties would be lost + expect(retriedTest.opts || {}).to.deep.equal({}) + expect(retriedTest.tags || []).to.deep.equal([]) + + // Now trigger our retryEnhancer (this happens automatically in CodeceptJS) + event.emit(event.test.before, retriedTest) + + // After our fix, all properties should be preserved + expect(retriedTest.opts).to.deep.equal({ + timeout: 30000, + metadata: 'important-test', + retries: 2, + feature: 'login', + }) + expect(retriedTest.tags).to.deep.equal(['@critical', '@smoke', '@login']) + expect(retriedTest.notes).to.deep.equal([ + { type: 'info', text: 'This test validates user login' }, + { type: 'warning', text: 'May be flaky due to external service' }, + ]) + expect(retriedTest.meta).to.deep.equal({ + feature: 'authentication', + story: 'user-login', + priority: 'high', + team: 'qa', + }) + expect(retriedTest.artifacts).to.deep.equal(['login-screenshot.png', 'network-log.json']) + expect(retriedTest.uid).to.equal('auth-test-001') + expect(retriedTest.config).to.deep.equal({ helper: 'playwright', baseUrl: 'http://test.com' }) + expect(retriedTest.inject).to.deep.equal({ userData: { email: 'test@example.com' } }) + expect(retriedTest.steps).to.deep.equal([ + { title: 'I am on page "/login"', status: 'success' }, + { title: 'I fill field "email", "test@example.com"', status: 'success' }, + { title: 'I fill field "password", "secretpassword"', status: 'success' }, + { title: 'I click "Login"', status: 'failed' }, + ]) + + // Verify that enhanced methods are available + expect(retriedTest.addNote).to.be.a('function') + expect(retriedTest.applyOptions).to.be.a('function') + expect(retriedTest.simplify).to.be.a('function') + + // Test that we can use the methods + retriedTest.addNote('retry', 'Attempt #2') + expect(retriedTest.notes).to.have.length(3) + expect(retriedTest.notes[2]).to.deep.equal({ type: 'retry', text: 'Attempt #2' }) + + // Verify the test is enhanced with CodeceptJS functionality + expect(retriedTest.codeceptjs).to.be.true + }) +}) diff --git a/test/unit/mocha/test_clone_test.js b/test/unit/mocha/test_clone_test.js index dc5a1b1ba..0cbe310ed 100644 --- a/test/unit/mocha/test_clone_test.js +++ b/test/unit/mocha/test_clone_test.js @@ -2,6 +2,9 @@ const { expect } = require('chai') const { createTest, cloneTest } = require('../../../lib/mocha/test') const { createSuite } = require('../../../lib/mocha/suite') const MochaSuite = require('mocha/lib/suite') +const Test = require('mocha/lib/test') +const retryEnhancer = require('../../../lib/listener/retryEnhancer') +const event = require('../../../lib/event') describe('Test cloning for retries', function () { it('should maintain consistent fullTitle format after cloning', function () { @@ -41,4 +44,97 @@ describe('Test cloning for retries', function () { expect(clonedTest.parent.title).to.equal('Feature Suite') expect(clonedTest.fullTitle()).to.equal('Feature Suite: Scenario Test') }) + + it('should demonstrate the problem: mocha native clone does not preserve CodeceptJS properties', function () { + // This test demonstrates the issue - it's expected to fail + // Create a CodeceptJS enhanced test + const test = createTest('Test with options', () => {}) + + // Set some CodeceptJS-specific properties + test.opts = { timeout: 5000, metadata: 'test-data' } + test.tags = ['@smoke', '@regression'] + test.notes = [{ type: 'info', text: 'Test note' }] + test.meta = { feature: 'login', story: 'user-auth' } + test.artifacts = ['screenshot.png'] + + // Simulate what happens during mocha retries - using mocha's native clone method + const mochaClonedTest = Test.prototype.clone.call(test) + + // These properties are lost due to mocha's shallow clone - this demonstrates the problem + expect(mochaClonedTest.opts || {}).to.deep.equal({}) // opts are lost + expect(mochaClonedTest.tags || []).to.deep.equal([]) // tags are lost + expect(mochaClonedTest.notes || []).to.deep.equal([]) // notes are lost + expect(mochaClonedTest.meta || {}).to.deep.equal({}) // meta is lost + expect(mochaClonedTest.artifacts || []).to.deep.equal([]) // artifacts are lost + + // But the retried test should have access to the original + expect(mochaClonedTest.retriedTest()).to.equal(test) + }) + + it('should preserve CodeceptJS-specific properties when a retried test can access original', function () { + // Create a CodeceptJS enhanced test + const originalTest = createTest('Test with options', () => {}) + + // Set some CodeceptJS-specific properties + originalTest.opts = { timeout: 5000, metadata: 'test-data' } + originalTest.tags = ['@smoke', '@regression'] + originalTest.notes = [{ type: 'info', text: 'Test note' }] + originalTest.meta = { feature: 'login', story: 'user-auth' } + originalTest.artifacts = ['screenshot.png'] + + // Simulate what happens during mocha retries - using mocha's native clone method + const retriedTest = Test.prototype.clone.call(originalTest) + + // The retried test should have a reference to the original + expect(retriedTest.retriedTest()).to.equal(originalTest) + + // We should be able to copy properties from the original test + const originalProps = originalTest.retriedTest() || originalTest + expect(originalProps.opts).to.deep.equal({ timeout: 5000, metadata: 'test-data' }) + expect(originalProps.tags).to.deep.equal(['@smoke', '@regression']) + expect(originalProps.notes).to.deep.equal([{ type: 'info', text: 'Test note' }]) + expect(originalProps.meta).to.deep.equal({ feature: 'login', story: 'user-auth' }) + expect(originalProps.artifacts).to.deep.equal(['screenshot.png']) + }) + + it('should preserve CodeceptJS-specific properties after retryEnhancer processing', function () { + // Setup the retryEnhancer listener + retryEnhancer() + + // Create a CodeceptJS enhanced test + const originalTest = createTest('Test with options', () => {}) + + // Set some CodeceptJS-specific properties + originalTest.opts = { timeout: 5000, metadata: 'test-data' } + originalTest.tags = ['@smoke', '@regression'] + originalTest.notes = [{ type: 'info', text: 'Test note' }] + originalTest.meta = { feature: 'login', story: 'user-auth' } + originalTest.artifacts = ['screenshot.png'] + originalTest.uid = 'test-123' + + // Simulate what happens during mocha retries - using mocha's native clone method + const retriedTest = Test.prototype.clone.call(originalTest) + + // The retried test should have a reference to the original + expect(retriedTest.retriedTest()).to.equal(originalTest) + + // Before the retryEnhancer, properties should be missing + expect(retriedTest.opts || {}).to.deep.equal({}) + + // Now trigger the retryEnhancer by emitting the test.before event + event.emit(event.test.before, retriedTest) + + // After the retryEnhancer processes it, properties should be copied + expect(retriedTest.opts).to.deep.equal({ timeout: 5000, metadata: 'test-data' }) + expect(retriedTest.tags).to.deep.equal(['@smoke', '@regression']) + expect(retriedTest.notes).to.deep.equal([{ type: 'info', text: 'Test note' }]) + expect(retriedTest.meta).to.deep.equal({ feature: 'login', story: 'user-auth' }) + expect(retriedTest.artifacts).to.deep.equal(['screenshot.png']) + expect(retriedTest.uid).to.equal('test-123') + + // Verify that methods are also copied + expect(retriedTest.addNote).to.be.a('function') + expect(retriedTest.applyOptions).to.be.a('function') + expect(retriedTest.simplify).to.be.a('function') + }) })