diff --git a/lib/plugins.js b/lib/plugins.js index 32b3f1fa0..6af59b35a 100644 --- a/lib/plugins.js +++ b/lib/plugins.js @@ -15,12 +15,9 @@ var log_ = function() { }; /** - * The plugin API for Protractor. Note that this API is extremely unstable - * and current consists of only two functions: - * .setup - called before tests - * .teardown - called after tests - * .postResults - called after test results have been processed - * More information on plugins coming in the future + * The plugin API for Protractor. Note that this API is unstable. See + * plugins/README.md for more information. + * * @constructor * @param {Object} config parsed from the config file */ @@ -58,8 +55,10 @@ function pluginFunFactory(funName) { var pluginConf = this.pluginConfs[name]; var pluginObj = this.pluginObjs[name]; names.push(name); - promises.push((pluginObj[funName] || noop)(pluginConf)); + promises.push( + (pluginObj[funName] || noop)(pluginConf, [].slice.call(arguments))); } + return q.all(promises).then(function(results) { // Join the results into a single object and output any test results var ret = {failedCount: 0}; @@ -131,4 +130,12 @@ Plugins.prototype.teardown = pluginFunFactory('teardown'); */ Plugins.prototype.postResults = pluginFunFactory('postResults'); +/** + * Called after each test block completes. + * + * @return {q.Promise} A promise which resolves when the plugins have all been + * torn down. + */ +Plugins.prototype.postTest = pluginFunFactory('postTest'); + module.exports = Plugins; diff --git a/lib/runner.js b/lib/runner.js index f4e3c264c..e6144ecd8 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -311,9 +311,27 @@ Runner.prototype.run = function() { self.on('testFail', restartDriver); } + // We need to save these promises to make sure they're run, but we don't + // want to delay starting the next test (because we can't, it's just + // an event emitter). + var pluginPostTestPromises = []; + + self.on('testPass', function() { + pluginPostTestPromises.push(plugins.postTest(true)); + }); + self.on('testFail', function() { + pluginPostTestPromises.push(plugins.postTest(false)); + }); + return require(frameworkPath).run(self, self.config_.specs). then(function(testResults) { - return helper.joinTestLogs(pluginSetupResults, testResults); + return q.all(pluginPostTestPromises).then(function(postTestResultList) { + var results = helper.joinTestLogs(pluginSetupResults, testResults); + postTestResultList.forEach(function(postTestResult) { + results = helper.joinTestLogs(results, postTestResult); + }); + return results; + }); }); // 5) Teardown plugins }).then(function(testResults) { diff --git a/plugins/README.md b/plugins/README.md index ffb3a2c32..55bfba004 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -36,6 +36,10 @@ exports.config = { Writing Plugins --------------- +Plugins are designed to work with any test framework (Jasmine, Mocha, etc), +so they use generic hooks which Protractor provides. Plugins may change +the output of Protractor by returning a results object. + Plugins are node modules which export an object with the following API: ```js @@ -72,6 +76,18 @@ exports.teardown = function(config) {}; * @return Return values are ignored. */ exports.postResults = function(config) {}; + +/** + * Called after each test block (in Jasmine, this means an `it` block) + * completes. + * + * @param {Object} config The plugin configuration object. + * @param {boolean} passed True if the test passed. + * + * @return Object If an object is returned, it is merged with the Protractor + * result object. May return a promise. + */ +exports.postTest = function(config, passed) {}; ``` The protractor results object follows the format specified in diff --git a/scripts/test.js b/scripts/test.js index 98b5f3d81..d856b14dc 100755 --- a/scripts/test.js +++ b/scripts/test.js @@ -97,6 +97,15 @@ executor.addCommandlineTest('node lib/cli.js spec/errorTest/mochaFailureConf.js' stacktrace: 'mocha_failure_spec.js:11:20' }]); +executor.addCommandlineTest('node lib/cli.js spec/errorTest/pluginsFailingConf.js') + .expectExitCode(1) + .expectErrors([ + {message: 'from setup'}, + {message: 'from postTest'}, + {message: 'from postTest'}, + {message: 'from teardown'} + ]); + // Check ngHint plugin executor.addCommandlineTest( diff --git a/spec/errorTest/pluginsFailingConf.js b/spec/errorTest/pluginsFailingConf.js new file mode 100644 index 000000000..db27161e3 --- /dev/null +++ b/spec/errorTest/pluginsFailingConf.js @@ -0,0 +1,30 @@ +var env = require('../environment.js'); + +// A small suite to make sure the full functionality of plugins work +exports.config = { + // seleniumAddress: env.seleniumAddress, + mockSelenium: true, + + framework: 'jasmine2', + + // Spec patterns are relative to this directory. + specs: [ + '../plugins/basic_spec.js' + ], + + capabilities: env.capabilities, + + baseUrl: env.baseUrl, + + jasmineNodeOpts: { + isVerbose: true, + realtimeFailure: true + }, + + // Plugin patterns are relative to this directory. + plugins: [{ + path: '../plugins/basic_plugin.js' + }, { + path: '../plugins/failing_plugin.js' + }] +}; diff --git a/spec/plugins/basic_spec.js b/spec/plugins/basic_spec.js index d7cc1d1d0..71d1d928d 100644 --- a/spec/plugins/basic_spec.js +++ b/spec/plugins/basic_spec.js @@ -2,4 +2,8 @@ describe('check if plugin setup ran', function() { it('should have set protractor.__BASIC_PLUGIN_RAN', function() { expect(protractor.__BASIC_PLUGIN_RAN).toBe(true); }); + + it('should run multiple tests', function() { + expect(true).toBe(true); + }); }); diff --git a/spec/plugins/failing_plugin.js b/spec/plugins/failing_plugin.js new file mode 100644 index 000000000..71af6f4ea --- /dev/null +++ b/spec/plugins/failing_plugin.js @@ -0,0 +1,39 @@ +var q = require('q'); + +var failingResult = function(message) { + return { + failedCount: 1, + specResults: [{ + description: 'plugin test which fails', + assertions: [{ + passed: false, + errorMsg: message, + }], + duration: 4 + }] + }; +}; + +module.exports = { + setup: function() { + return q.delay(100).then(function() { + return failingResult('from setup'); + }); + }, + + teardown: function() { + return q.delay(100).then(function() { + return failingResult('from teardown'); + }); + }, + + postResults: function() { + // This function should cause no failures. + }, + + postTest: function(config, passed) { + return q.delay(100).then(function() { + return failingResult('from postTest'); + }); + } +}; diff --git a/spec/plugins/test_plugin.js b/spec/plugins/test_plugin.js index 054e95fd3..a1604cbe8 100644 --- a/spec/plugins/test_plugin.js +++ b/spec/plugins/test_plugin.js @@ -1,12 +1,34 @@ +var q = require('q'); + +var passingResult = { + failedCount: 0, + specResults: [{ + description: 'plugin test which passes', + assertions: [], + duration: 4 + }] +}; + module.exports = { + setup: function() { + return q.delay(100).then(function() { + return passingResult; + }); + }, + teardown: function() { - return { - failedCount: 0, - specResults: [{ - description: 'This succeeds', - assertions: [], - duration: 1 - }] - }; + return q.delay(100).then(function() { + return passingResult; + }); + }, + + postResults: function() { + // This function should cause no failures. + }, + + postTest: function() { + return q.delay(100).then(function() { + return passingResult; + }); } }; diff --git a/spec/pluginsBasicConf.js b/spec/pluginsBasicConf.js index e10e2f99b..313b4a722 100644 --- a/spec/pluginsBasicConf.js +++ b/spec/pluginsBasicConf.js @@ -3,7 +3,7 @@ var env = require('./environment.js'); // A small suite to make sure the basic functionality of plugins work // Tests the (potential) edge case of exactly one plugin being used exports.config = { - seleniumAddress: env.seleniumAddress, + mockSelenium: true, framework: 'jasmine2', diff --git a/spec/pluginsFullConf.js b/spec/pluginsFullConf.js index 2ae7b8d57..15be7b831 100644 --- a/spec/pluginsFullConf.js +++ b/spec/pluginsFullConf.js @@ -2,7 +2,8 @@ var env = require('./environment.js'); // A small suite to make sure the full functionality of plugins work exports.config = { - seleniumAddress: env.seleniumAddress, + // seleniumAddress: env.seleniumAddress, + mockSelenium: true, framework: 'jasmine2',