Skip to content
This repository has been archived by the owner on Jul 29, 2024. It is now read-only.

Commit

Permalink
chore(test): error tests fixed
Browse files Browse the repository at this point in the history
- Fix a missing await on createNextTaskRunner where the recursive call
should be awaited and if there are no new tasks to still resolve the
promise. Things were passing previously probably because we were
running tasks out of sync.
- Add in errorTest portion of the test suite
- Turn on the angular2 and unit tests.
  • Loading branch information
cnishina committed Nov 30, 2018
1 parent e9622f4 commit b64a8df
Show file tree
Hide file tree
Showing 23 changed files with 157 additions and 153 deletions.
4 changes: 3 additions & 1 deletion lib/launcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ let initFn = async function(configFile: string, additionalConfig: Config) {
}
taskResults_.add(result);
task.done();
createNextTaskRunner();
await createNextTaskRunner();
// If all tasks are finished
if (scheduler.numTasksOutstanding() === 0) {
resolve();
Expand All @@ -232,6 +232,8 @@ let initFn = async function(configFile: string, additionalConfig: Config) {
logger.error('Error:', (err as any).stack || err.message || err);
await cleanUpAndExit(errorCode ? errorCode : RUNNERS_FAILED_EXIT_CODE);
}
} else {
resolve();
}
});
};
Expand Down
172 changes: 86 additions & 86 deletions scripts/test.js
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@ const passingTests = [
'node built/cli.js spec/controlLockConf.js',
'node built/cli.js spec/customFramework.js',
'node built/cli.js spec/noGlobalsConf.js',
// 'node built/cli.js spec/angular2Conf.js',
'node built/cli.js spec/angular2Conf.js',
'node built/cli.js spec/hybridConf.js',
'node built/cli.js spec/built/noCFBasicConf.js',
'node built/cli.js spec/built/noCFBasicConf.js --useBlockingProxy',
'node built/cli.js spec/built/noCFPluginConf.js',
'node scripts/driverProviderAttachSession.js',
// 'node scripts/errorTest.js',
// // Unit tests
// 'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/unit_test.json',
'node scripts/errorTest.js',
// Unit tests
'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/unit_test.json',
// Dependency tests
'node node_modules/jasmine/bin/jasmine.js JASMINE_CONFIG_PATH=scripts/dependency_test.json',
// Typings tests
Expand All @@ -65,88 +65,88 @@ passingTests.forEach((passing_test) => {
*************************/

// assert stacktrace shows line of failure
// executor.addCommandlineTest('node built/cli.js spec/errorTest/singleFailureConf.js')
// .expectExitCode(1)
// .expectErrors({
// stackTrace: 'single_failure_spec1.js:5:32'
// });

// // assert timeout works
// executor.addCommandlineTest('node built/cli.js spec/errorTest/timeoutConf.js')
// .expectExitCode(1)
// .expectErrors({
// message: 'Timeout - Async callback was not invoked within timeout ' +
// 'specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.'
// })
// .expectTestDuration(0, 1000);

// executor.addCommandlineTest('node built/cli.js spec/errorTest/afterLaunchChangesExitCodeConf.js')
// .expectExitCode(11)
// .expectErrors({
// message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.'
// });

// executor.addCommandlineTest('node built/cli.js spec/errorTest/multiFailureConf.js')
// .expectExitCode(1)
// .expectErrors([{
// message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
// stacktrace: 'single_failure_spec1.js:5:32'
// }, {
// message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
// stacktrace: 'single_failure_spec2.js:5:32'
// }]);

// executor.addCommandlineTest('node built/cli.js spec/errorTest/shardedFailureConf.js')
// .expectExitCode(1)
// .expectErrors([{
// message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
// stacktrace: 'single_failure_spec1.js:5:32'
// }, {
// message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
// stacktrace: 'single_failure_spec2.js:5:32'
// }]);

// executor.addCommandlineTest('node built/cli.js spec/errorTest/mochaFailureConf.js')
// .expectExitCode(1)
// .expectErrors([{
// message: 'expected \'My AngularJS App\' to equal \'INTENTIONALLY INCORRECT\'',
// stacktrace: 'mocha_failure_spec.js:11:20'
// }]);

// executor.addCommandlineTest('node built/cli.js spec/errorTest/pluginsFailingConf.js')
// .expectExitCode(1)
// .expectErrors([
// {message: 'Expected true to be false'},
// {message: 'from setup'},
// {message: 'from postTest passing'},
// {message: 'from postTest failing'},
// {message: 'from teardown'}
// ]);

// executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js')
// .expectExitCode(1)
// .expectErrors([
// {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
// {message: 'The following tasks were pending:[\\s\\S]*' +
// '- \\$timeout: function\\(\\) {[\\s\\S]*' +
// '\\$scope\\.slowAngularTimeoutStatus = \'done\';[\\s\\S]' +
// '*}'}
// ]);

// executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js ' +
// '--untrackOutstandingTimeouts true')
// .expectExitCode(1)
// .expectErrors([
// {message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
// {message: 'While waiting for element with locator - ' +
// 'Locator: by.binding\\(\\"slowAngularTimeoutStatus\\"\\)$'}
// ]);

// executor.addCommandlineTest('node built/cli.js spec/angular2TimeoutConf.js')
// .expectExitCode(1)
// .expectErrors([
// {message: 'Timed out waiting for asynchronous Angular tasks to finish'},
// ]);
executor.addCommandlineTest('node built/cli.js spec/errorTest/singleFailureConf.js')
.expectExitCode(1)
.expectErrors({
stackTrace: 'single_failure_spec1.js:5:38'
});

// assert timeout works
executor.addCommandlineTest('node built/cli.js spec/errorTest/timeoutConf.js')
.expectExitCode(1)
.expectErrors({
message: 'Timeout - Async callback was not invoked within timeout ' +
'specified by jasmine.DEFAULT_TIMEOUT_INTERVAL.'
})
.expectTestDuration(0, 1000);

executor.addCommandlineTest('node built/cli.js spec/errorTest/afterLaunchChangesExitCodeConf.js')
.expectExitCode(11)
.expectErrors({
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.'
});

executor.addCommandlineTest('node built/cli.js spec/errorTest/multiFailureConf.js')
.expectExitCode(1)
.expectErrors([{
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
stacktrace: 'single_failure_spec1.js:5:38'
}, {
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
stacktrace: 'single_failure_spec2.js:5:38'
}]);

executor.addCommandlineTest('node built/cli.js spec/errorTest/shardedFailureConf.js')
.expectExitCode(1)
.expectErrors([{
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
stacktrace: 'single_failure_spec1.js:5:38'
}, {
message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
stacktrace: 'single_failure_spec2.js:5:38'
}]);

executor.addCommandlineTest('node built/cli.js spec/errorTest/mochaFailureConf.js')
.expectExitCode(1)
.expectErrors([{
message: 'expected \'My AngularJS App\' to equal \'INTENTIONALLY INCORRECT\'',
stacktrace: 'mocha_failure_spec.js:11:41'
}]);

executor.addCommandlineTest('node built/cli.js spec/errorTest/pluginsFailingConf.js')
.expectExitCode(1)
.expectErrors([
{message: 'Expected true to be false'},
{message: 'from setup'},
{message: 'from postTest passing'},
{message: 'from postTest failing'},
{message: 'from teardown'}
]);

executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js')
.expectExitCode(1)
.expectErrors([
{message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
{message: 'The following tasks were pending:[\\s\\S]*' +
'- \\$timeout: function\\(\\) {[\\s\\S]*' +
'\\$scope\\.slowAngularTimeoutStatus = \'done\';[\\s\\S]' +
'*}'}
]);

executor.addCommandlineTest('node built/cli.js spec/errorTest/slowHttpAndTimeoutConf.js ' +
'--untrackOutstandingTimeouts true')
.expectExitCode(1)
.expectErrors([
{message: 'The following tasks were pending[\\s\\S]*\\$http: slowcall'},
{message: 'While waiting for element with locator - ' +
'Locator: by.binding\\(\\"slowAngularTimeoutStatus\\"\\)$'}
]);

executor.addCommandlineTest('node built/cli.js spec/angular2TimeoutConf.js')
.expectExitCode(1)
.expectErrors([
{message: 'Timed out waiting for asynchronous Angular tasks to finish'},
]);

// If we're running on CircleCI, save stdout and stderr from the test run to a log file.
if (process.env['CIRCLE_ARTIFACTS']) {
Expand Down
12 changes: 1 addition & 11 deletions spec/ciFullConf.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,8 @@ exports.config = {
framework: 'jasmine',

// Spec patterns are relative to this directory.
// TODO(selenium4): revert back to basic/*_spec.js
specs: [
'basic/lib_spec.js',
'basic/locators_spec.js'
// 'basic/elements_spec.js',
// 'basic/expected_conditions_spec.js',
// 'basic/handling_spec.js'
// 'basic/mockmodule_spec.js',
// 'basic/navigation_spec.js',
// 'basic/polling_spec.js',
// 'basic/restart_spec.js',
// 'basic/synchronize_spec.js',
'basic/*_spec.js',
],

// Exclude patterns are relative to this directory.
Expand Down
1 change: 1 addition & 0 deletions spec/errorTest/afterLaunchChangesExitCodeConf.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ var env = require('../environment.js');

exports.config = {
seleniumAddress: env.seleniumAddress,
SELENIUM_PROMISE_MANAGER: false,

framework: 'jasmine',

Expand Down
6 changes: 3 additions & 3 deletions spec/errorTest/baseCase/error_spec.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
describe('finding an element that does not exist', function() {
it('should throw an error', function() {
browser.get('index.html');
describe('finding an element that does not exist', () => {
it('should throw an error', async () => {
await browser.get('index.html');
element(by.binding('INVALID')); // greeting
});
});
14 changes: 7 additions & 7 deletions spec/errorTest/baseCase/mocha_failure_spec.js
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
// Use the external Chai As Promised to deal with resolving promises in
// expectations.
var chai = require('chai');
var chaiAsPromised = require('chai-as-promised');
const chai = require('chai');
const chaiAsPromised = require('chai-as-promised');
chai.use(chaiAsPromised);
var expect = chai.expect;
const expect = chai.expect;

describe('protractor library', function() {
it('should fail', function() {
browser.get('index.html');
expect(browser.getTitle()).to.eventually.equal('INTENTIONALLY INCORRECT');
describe('protractor library', () => {
it('should fail', async () => {
await browser.get('index.html');
expect(await browser.getTitle()).to.equal('INTENTIONALLY INCORRECT');
});
});
10 changes: 5 additions & 5 deletions spec/errorTest/baseCase/single_failure_spec1.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
describe('single failure spec1', function() {
it('should fail expectation', function() {
browser.get('index.html');
var greeting = element(by.binding('greeting'));
expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
describe('single failure spec1', () => {
it('should fail expectation', async () => {
await browser.get('index.html');
const greeting = element(by.binding('greeting'));
expect(await greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
});
});
10 changes: 5 additions & 5 deletions spec/errorTest/baseCase/single_failure_spec2.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
describe('single failure spec2', function() {
it('should fail expectation', function() {
browser.get('index.html');
var greeting = element(by.binding('greeting'));
expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
describe('single failure spec2', () => {
it('should fail expectation', async () => {
await browser.get('index.html');
const greeting = element(by.binding('greeting'));
expect(await greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
});
});
30 changes: 15 additions & 15 deletions spec/errorTest/baseCase/slow_http_and_timeout_spec.js
Original file line number Diff line number Diff line change
@@ -1,27 +1,27 @@
describe('slow asynchronous events', function() {
beforeEach(function() {
browser.get('index.html#/async');
describe('slow asynchronous events', () => {
beforeEach(async () => {
await browser.get('index.html#/async');
});

it('waits for http calls', function() {
var status = element(by.binding('slowHttpStatus'));
var button = element(by.css('[ng-click="slowHttp()"]'));
it('waits for http calls', async () => {
const status = element(by.binding('slowHttpStatus'));
const button = element(by.css('[ng-click="slowHttp()"]'));

expect(status.getText()).toEqual('not started');
expect(await status.getText()).toEqual('not started');

button.click();
await button.click();

expect(status.getText()).toEqual('done');
expect(await status.getText()).toEqual('done');
});

it('waits for $timeout', function() {
var status = element(by.binding('slowAngularTimeoutStatus'));
var button = element(by.css('[ng-click="slowAngularTimeout()"]'));
it('waits for $timeout', async () => {
const status = element(by.binding('slowAngularTimeoutStatus'));
const button = element(by.css('[ng-click="slowAngularTimeout()"]'));

expect(status.getText()).toEqual('not started');
expect(await status.getText()).toEqual('not started');

button.click();
await button.click();

expect(status.getText()).toEqual('done');
expect(await status.getText()).toEqual('done');
});
});
10 changes: 5 additions & 5 deletions spec/errorTest/baseCase/success_spec.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
describe('success spec', function() {
it('should pass', function() {
browser.get('index.html');
var greeting = element(by.binding('greeting'));
expect(greeting.getText()).toEqual('Hiya');
describe('success spec', () => {
it('should pass', async () => {
await browser.get('index.html');
const greeting = element(by.binding('greeting'));
expect(await greeting.getText()).toEqual('Hiya');
});
});
6 changes: 3 additions & 3 deletions spec/errorTest/baseCase/timeout_spec.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
describe('timeout spec', function() {
it('should timeout due to jasmine spec limit', function() {
browser.get('index.html#/form');
describe('timeout spec', () => {
it('should timeout due to jasmine spec limit', async () => {
await browser.get('index.html#/form');
}, 1);
});
3 changes: 2 additions & 1 deletion spec/errorTest/browserStackAuthentication.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
var env = require('../environment.js');
const env = require('../environment.js');

exports.config = {
browserstackUser: 'foobar',
browserstackKey: 'foobar',
SELENIUM_PROMISE_MANAGER: false,

framework: 'jasmine',

Expand Down
3 changes: 2 additions & 1 deletion spec/errorTest/debugMultiCapabilities.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
var env = require('../environment.js');
const env = require('../environment.js');

exports.config = {
seleniumAddress: env.seleniumAddress,
SELENIUM_PROMISE_MANAGER: false,
framework: 'jasmine',
debug: true,
specs: [
Expand Down
3 changes: 2 additions & 1 deletion spec/errorTest/getMultiCapabilitiesConf.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
var env = require('../environment.js');
const env = require('../environment.js');

exports.config = {
seleniumAddress: env.seleniumAddress,
SELENIUM_PROMISE_MANAGER: false,

// Spec patterns are relative to this directory.
specs: [
Expand Down
3 changes: 2 additions & 1 deletion spec/errorTest/mochaFailureConf.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
var env = require('../environment.js');
const env = require('../environment.js');

exports.config = {
seleniumAddress: env.seleniumAddress,
SELENIUM_PROMISE_MANAGER: false,

specs: [
'baseCase/mocha_failure_spec.js'
Expand Down
Loading

0 comments on commit b64a8df

Please sign in to comment.