diff --git a/lib/cli.js b/lib/cli.js index 4312a5078..269430287 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -186,20 +186,24 @@ class Cli extends Base { } } - let stack = err.stack ? err.stack.split('\n') : []; - if (stack[0] && stack[0].includes(err.message)) { - stack.shift(); - } + try { + let stack = err.stack ? err.stack.split('\n') : []; + if (stack[0] && stack[0].includes(err.message)) { + stack.shift(); + } - if (output.level() < 3) { - stack = stack.slice(0, 3); - } + if (output.level() < 3) { + stack = stack.slice(0, 3); + } - err.stack = `${stack.join('\n')}\n\n${output.colors.blue(log)}`; + err.stack = `${stack.join('\n')}\n\n${output.colors.blue(log)}`; - // clone err object so stack trace adjustments won't affect test other reports - test.err = err; - return test; + // clone err object so stack trace adjustments won't affect test other reports + test.err = err; + return test; + } catch (e) { + throw Error(e); + } }); const originalLog = Base.consoleLog; diff --git a/lib/command/gherkin/init.js b/lib/command/gherkin/init.js index a909c27bb..bcb418dc9 100644 --- a/lib/command/gherkin/init.js +++ b/lib/command/gherkin/init.js @@ -70,7 +70,7 @@ module.exports = function (genPath) { } config.gherkin = { - features: './features/*.feature', + features: "./features/*.feature", steps: [`./step_definitions/steps.${extension}`], }; diff --git a/lib/plugin/retryTo.js b/lib/plugin/retryTo.js index 9929944f1..4a2940176 100644 --- a/lib/plugin/retryTo.js +++ b/lib/plugin/retryTo.js @@ -1,5 +1,4 @@ const recorder = require('../recorder'); -const store = require('../store'); const { debug } = require('../output'); const defaultConfig = { @@ -73,49 +72,58 @@ const defaultConfig = { * const retryTo = codeceptjs.container.plugins('retryTo'); * ``` * -*/ + */ module.exports = function (config) { config = Object.assign(defaultConfig, config); + function retryTo(callback, maxTries, pollInterval = config.pollInterval) { + return new Promise((done, reject) => { + let tries = 1; - if (config.registerGlobal) { - global.retryTo = retryTo; - } - return retryTo; + function handleRetryException(err) { + recorder.throw(err); + reject(err); + } - function retryTo(callback, maxTries, pollInterval = undefined) { - let tries = 1; - if (!pollInterval) pollInterval = config.pollInterval; - - let err = null; - - return new Promise((done) => { const tryBlock = async () => { + tries++; recorder.session.start(`retryTo ${tries}`); - await callback(tries); + try { + await callback(tries); + } catch (err) { + handleRetryException(err); + } + + // Call done if no errors recorder.add(() => { recorder.session.restore(`retryTo ${tries}`); done(null); }); - recorder.session.catch((e) => { - err = e; + + // Catch errors and retry + recorder.session.catch((err) => { recorder.session.restore(`retryTo ${tries}`); - tries++; if (tries <= maxTries) { debug(`Error ${err}... Retrying`); - err = null; - - recorder.add(`retryTo ${tries}`, () => setTimeout(tryBlock, pollInterval)); + recorder.add(`retryTo ${tries}`, () => + setTimeout(tryBlock, pollInterval) + ); } else { - done(null); + // if maxTries reached + handleRetryException(err); } }); }; - recorder.add('retryTo', async () => { - tryBlock(); + recorder.add('retryTo', tryBlock).catch(err => { + console.error('An error occurred:', err); + done(null); }); - }).then(() => { - if (err) recorder.throw(err); }); } + + if (config.registerGlobal) { + global.retryTo = retryTo; + } + + return retryTo; }; diff --git a/lib/scenario.js b/lib/scenario.js index 40f5759a1..ab816a1ab 100644 --- a/lib/scenario.js +++ b/lib/scenario.js @@ -18,6 +18,16 @@ const injectHook = function (inject, suite) { return recorder.promise(); }; +function makeDoneCallableOnce(done) { + let called = false; + return function (err) { + if (called) { + return; + } + called = true; + return done(err); + }; +} /** * Wraps test function, injects support objects from container, * starts promise chain with recorder, performs before/after hooks @@ -34,15 +44,17 @@ module.exports.test = (test) => { test.async = true; test.fn = function (done) { + const doneFn = makeDoneCallableOnce(done); recorder.errHandler((err) => { recorder.session.start('teardown'); recorder.cleanAsyncErr(); - if (test.throws) { // check that test should actually fail + if (test.throws) { + // check that test should actually fail try { assertThrown(err, test.throws); event.emit(event.test.passed, test); event.emit(event.test.finished, test); - recorder.add(() => done()); + recorder.add(doneFn); return; } catch (newErr) { err = newErr; @@ -50,40 +62,26 @@ module.exports.test = (test) => { } event.emit(event.test.failed, test, err); event.emit(event.test.finished, test); - recorder.add(() => done(err)); + recorder.add(() => doneFn(err)); }); if (isAsyncFunction(testFn)) { event.emit(event.test.started, test); - - const catchError = e => { - recorder.throw(e); - recorder.catch((e) => { - const err = (recorder.getAsyncErr() === null) ? e : recorder.getAsyncErr(); - recorder.session.start('teardown'); - recorder.cleanAsyncErr(); - event.emit(event.test.failed, test, err); - event.emit(event.test.finished, test); - recorder.add(() => done(err)); + testFn + .call(test, getInjectedArguments(testFn, test)) + .then(() => { + recorder.add('fire test.passed', () => { + event.emit(event.test.passed, test); + event.emit(event.test.finished, test); + }); + recorder.add('finish test', doneFn); + }) + .catch((err) => { + recorder.throw(err); + }) + .finally(() => { + recorder.catch(); }); - }; - - let injectedArguments; - try { - injectedArguments = getInjectedArguments(testFn, test); - } catch (e) { - catchError(e); - return; - } - - testFn.call(test, injectedArguments).then(() => { - recorder.add('fire test.passed', () => { - event.emit(event.test.passed, test); - event.emit(event.test.finished, test); - }); - recorder.add('finish test', () => done()); - recorder.catch(); - }).catch(catchError); return; } @@ -97,7 +95,7 @@ module.exports.test = (test) => { event.emit(event.test.passed, test); event.emit(event.test.finished, test); }); - recorder.add('finish test', () => done()); + recorder.add('finish test', doneFn); recorder.catch(); } }; @@ -109,13 +107,14 @@ module.exports.test = (test) => { */ module.exports.injected = function (fn, suite, hookName) { return function (done) { + const doneFn = makeDoneCallableOnce(done); const errHandler = (err) => { recorder.session.start('teardown'); recorder.cleanAsyncErr(); event.emit(event.test.failed, suite, err); if (hookName === 'after') event.emit(event.test.after, suite); if (hookName === 'afterSuite') event.emit(event.suite.after, suite); - recorder.add(() => done(err)); + recorder.add(() => doneFn(err)); }; recorder.errHandler((err) => { @@ -137,28 +136,32 @@ module.exports.injected = function (fn, suite, hookName) { const opts = suite.opts || {}; const retries = opts[`retry${ucfirst(hookName)}`] || 0; - promiseRetry(async (retry, number) => { - try { - recorder.startUnlessRunning(); - await fn.call(this, getInjectedArguments(fn)); - await recorder.promise().catch(err => retry(err)); - } catch (err) { - retry(err); - } finally { - if (number < retries) { - recorder.stop(); - recorder.start(); + promiseRetry( + async (retry, number) => { + try { + recorder.startUnlessRunning(); + await fn.call(this, getInjectedArguments(fn)); + await recorder.promise().catch((err) => retry(err)); + } catch (err) { + retry(err); + } finally { + if (number < retries) { + recorder.stop(); + recorder.start(); + } } - } - }, { retries }) + }, + { retries }, + ) .then(() => { recorder.add('fire hook.passed', () => event.emit(event.hook.passed, suite)); - recorder.add(`finish ${hookName} hook`, () => done()); + recorder.add(`finish ${hookName} hook`, doneFn); recorder.catch(); - }).catch((e) => { + }) + .catch((e) => { recorder.throw(e); recorder.catch((e) => { - const err = (recorder.getAsyncErr() === null) ? e : recorder.getAsyncErr(); + const err = recorder.getAsyncErr() === null ? e : recorder.getAsyncErr(); errHandler(err); }); recorder.add('fire hook.failed', () => event.emit(event.hook.failed, suite, e)); diff --git a/test/acceptance/retryTo_test.js b/test/acceptance/retryTo_test.js index b568e8607..c3bfe5a74 100644 --- a/test/acceptance/retryTo_test.js +++ b/test/acceptance/retryTo_test.js @@ -14,3 +14,23 @@ Scenario('retryTo works with non await steps @plugin', async () => { if (tryNum < 3) I.waitForVisible('.nothing', 1); }, 4); }); + +Scenario('Should be succeed', async ({ I }) => { + I.amOnPage('http://example.org'); + I.waitForVisible('.nothing', 1); // should fail here but it won't terminate + await retryTo((tryNum) => { + I.see('.doesNotMatter'); + }, 10); +}); + +Scenario('Should fail after reached max retries', async () => { + await retryTo(() => { + throw new Error('Custom pluginRetryTo Error'); + }, 3); +}); + +Scenario('Should succeed at the third attempt @plugin', async () => { + await retryTo(async (tryNum) => { + if (tryNum < 2) throw new Error('Custom pluginRetryTo Error'); + }, 3); +}); \ No newline at end of file diff --git a/test/data/sandbox/codecept.scenario-stale.js b/test/data/sandbox/codecept.scenario-stale.js new file mode 100644 index 000000000..f07399b38 --- /dev/null +++ b/test/data/sandbox/codecept.scenario-stale.js @@ -0,0 +1,10 @@ +exports.config = { + tests: './test.scenario-stale.js', + timeout: 10000, + retry: 2, + output: './output', + include: {}, + bootstrap: false, + mocha: {}, + name: 'sandbox', +}; diff --git a/test/data/sandbox/test.scenario-stale.js b/test/data/sandbox/test.scenario-stale.js new file mode 100644 index 000000000..30ec3c9b0 --- /dev/null +++ b/test/data/sandbox/test.scenario-stale.js @@ -0,0 +1,22 @@ +Feature('Scenario should not be staling'); + +const SHOULD_NOT_STALE = 'should not stale scenario error'; + +Scenario('Rejected promise should not stale the process', async () => { + await new Promise((_resolve, reject) => setTimeout(reject(new Error(SHOULD_NOT_STALE)), 500)); +}); + +Scenario('Should handle throw inside synchronous and terminate gracefully', () => { + throw new Error(SHOULD_NOT_STALE); +}); +Scenario('Should handle throw inside async and terminate gracefully', async () => { + throw new Error(SHOULD_NOT_STALE); +}); + +Scenario('Should throw, retry and keep failing', async () => { + setTimeout(() => { + throw new Error(SHOULD_NOT_STALE); + }, 500); + await new Promise((resolve) => setTimeout(resolve, 300)); + throw new Error(SHOULD_NOT_STALE); +}).retry(2); diff --git a/test/plugin/plugin_test.js b/test/plugin/plugin_test.js index 714da72cd..0f5e6f0a7 100644 --- a/test/plugin/plugin_test.js +++ b/test/plugin/plugin_test.js @@ -32,6 +32,15 @@ describe('CodeceptJS plugin', function () { }); }); + it('should failed before the retryTo instruction', (done) => { + exec(`${config_run_config('codecept.Playwright.retryTo.js', 'Should be succeed')} --verbose`, (err, stdout) => { + expect(stdout).toContain('locator.waitFor: Timeout 1000ms exceeded.'), + expect(stdout).toContain('[1] Error | Error: element (.nothing) still not visible after 1 sec'), + expect(err).toBeTruthy(); + done(); + }); + }); + it('should generate the coverage report', (done) => { exec(`${config_run_config('codecept.Playwright.coverage.js', '@coverage')} --debug`, (err, stdout) => { const lines = stdout.split('\n'); @@ -61,4 +70,17 @@ describe('CodeceptJS plugin', function () { done(); }); }); + + it('should retry to failure', (done) => { + exec( + `${config_run_config('codecept.Playwright.retryTo.js', 'Should fail after reached max retries')} --verbose`, (err, stdout) => { + const lines = stdout.split('\n'); + expect(lines).toEqual( + expect.arrayContaining([expect.stringContaining('Custom pluginRetryTo Error')]) + ); + expect(err).toBeTruthy(); + done(); + } + ); + }); }); diff --git a/test/runner/run_workers_test.js b/test/runner/run_workers_test.js index 0cd0f3b46..ccd1afe99 100644 --- a/test/runner/run_workers_test.js +++ b/test/runner/run_workers_test.js @@ -19,7 +19,6 @@ describe('CodeceptJS Workers Runner', function () { if (!semver.satisfies(process.version, '>=11.7.0')) this.skip('not for node version'); console.log(`${codecept_run} 3 --debug`); exec(`${codecept_run} 3 --debug`, (err, stdout) => { - console.log('aaaaaaaaaaaaa', stdout); expect(stdout).toContain('CodeceptJS'); // feature expect(stdout).toContain('glob current dir'); expect(stdout).toContain('From worker @1_grep print message 1'); diff --git a/test/runner/scenario_stale_test.js b/test/runner/scenario_stale_test.js new file mode 100644 index 000000000..927d92acd --- /dev/null +++ b/test/runner/scenario_stale_test.js @@ -0,0 +1,22 @@ +const { expect } = require('expect'); +const path = require('path'); +const { exec } = require('child_process'); + +const runner = path.join(__dirname, '/../../bin/codecept.js'); +const codecept_dir = path.join(__dirname, '/../data/sandbox'); +const codecept_run = `${runner} run`; +const config_run_config = config => `${codecept_run} --config ${codecept_dir}/${config}`; + +describe('Scenario termination check', () => { + before(() => { + process.chdir(codecept_dir); + }); + + it('Should always fail and terminate', (done) => { + exec(config_run_config('codecept.scenario-stale.js'), (err, stdout) => { + expect(stdout).toContain('should not stale scenario error'); // feature + expect(err).toBeTruthy(); + done(); + }); + }); +}); diff --git a/test/runner/timeout_test.js b/test/runner/timeout_test.js index 79d74f054..8e14efc09 100644 --- a/test/runner/timeout_test.js +++ b/test/runner/timeout_test.js @@ -11,7 +11,7 @@ describe('CodeceptJS Timeouts', function () { it('should stop test when timeout exceeded', (done) => { exec(config_run_config('codecept.conf.js', 'timed out'), (err, stdout) => { - console.log(stdout); + debug_this_test && console.log(stdout); expect(stdout).toContain('Timeout 2s exceeded'); expect(stdout).toContain('Timeout 1s exceeded'); expect(err).toBeTruthy(); diff --git a/test/unit/plugin/retryto_test.js b/test/unit/plugin/retryto_test.js index 28187e02c..293adf1d7 100644 --- a/test/unit/plugin/retryto_test.js +++ b/test/unit/plugin/retryto_test.js @@ -20,11 +20,11 @@ describe('retryTo plugin', () => { it('should execute few times command on fail', async () => { let counter = 0; let errorCaught = false; - await retryTo(() => { - recorder.add(() => counter++); - recorder.add(() => { throw new Error('Ups'); }); - }, 5, 10); try { + await retryTo(() => { + recorder.add(() => counter++); + recorder.add(() => { throw new Error('Ups'); }); + }, 5, 10); await recorder.promise(); } catch (err) { errorCaught = true;