feat(plugins): add postTest hook for plugins

Additionally, add some tests to make sure that plugins can fail properly.

Closes angular#1842

lib/plugins.js

@@ -15,12 +15,9 @@ var log_ = function() {
 };
 
 /**
- * The plugin API for Protractor.  Note that this API is extremely unstable
- * and current consists of only two functions:
- *    <plugin>.setup - called before tests
- *    <plugin>.teardown - called after tests
- *    <plugin>.postResults - called after test results have been processed
- * More information on plugins coming in the future
+ * The plugin API for Protractor.  Note that this API is unstable. See
+ * plugins/README.md for more information.
+ *
  * @constructor
  * @param {Object} config parsed from the config file
  */
@@ -58,8 +55,12 @@ function pluginFunFactory(funName) {
       var pluginConf = this.pluginConfs[name];
       var pluginObj = this.pluginObjs[name];
       names.push(name);
-      promises.push((pluginObj[funName] || noop)(pluginConf));
+      promises.push(
+          (pluginObj[funName] || noop).apply(
+              pluginObj[funName],
+              [pluginConf].concat([].slice.call(arguments))));
     }
+
     return q.all(promises).then(function(results) {
       // Join the results into a single object and output any test results
       var ret = {failedCount: 0};
@@ -131,4 +132,12 @@ Plugins.prototype.teardown = pluginFunFactory('teardown');
  */
 Plugins.prototype.postResults = pluginFunFactory('postResults');
 
+/**
+ * Called after each test block completes.
+ *
+ * @return {q.Promise} A promise which resolves when the plugins have all been
+ *     torn down.
+ */
+Plugins.prototype.postTest = pluginFunFactory('postTest');
+
 module.exports = Plugins;

lib/runner.js

@@ -311,9 +311,27 @@ Runner.prototype.run = function() {
       self.on('testFail', restartDriver);
     }
 
+    // We need to save these promises to make sure they're run, but we don't
+    // want to delay starting the next test (because we can't, it's just
+    // an event emitter).
+    var pluginPostTestPromises = [];
+
+    self.on('testPass', function() {
+      pluginPostTestPromises.push(plugins.postTest(true));
+    });
+    self.on('testFail', function() {
+      pluginPostTestPromises.push(plugins.postTest(false));
+    });
+
     return require(frameworkPath).run(self, self.config_.specs).
         then(function(testResults) {
-          return helper.joinTestLogs(pluginSetupResults, testResults);
+          return q.all(pluginPostTestPromises).then(function(postTestResultList) {
+            var results = helper.joinTestLogs(pluginSetupResults, testResults);
+            postTestResultList.forEach(function(postTestResult) {
+              results = helper.joinTestLogs(results, postTestResult);
+            });
+            return results;
+          });
         });
   // 5) Teardown plugins
   }).then(function(testResults) {

plugins/README.md

@@ -36,6 +36,10 @@ exports.config = {
 Writing Plugins
 ---------------
 
+Plugins are designed to work with any test framework (Jasmine, Mocha, etc),
+so they use generic hooks which Protractor provides. Plugins may change
+the output of Protractor by returning a results object.
+
 Plugins are node modules which export an object with the following API:
 
 ```js
@@ -72,6 +76,18 @@ exports.teardown = function(config) {};
  * @return Return values are ignored.
  */
 exports.postResults = function(config) {};
+
+/**
+ * Called after each test block (in Jasmine, this means an `it` block)
+ * completes.
+ *
+ * @param {Object} config The plugin configuration object.
+ * @param {boolean} passed True if the test passed.
+ *
+ * @return Object If an object is returned, it is merged with the Protractor
+ *     result object. May return a promise.
+ */
+exports.postTest = function(config, passed) {};
 ```
 
 The protractor results object follows the format specified in

scripts/test.js

@@ -97,6 +97,16 @@ executor.addCommandlineTest('node lib/cli.js spec/errorTest/mochaFailureConf.js'
       stacktrace: 'mocha_failure_spec.js:11:20'
     }]);
 
+executor.addCommandlineTest('node lib/cli.js spec/errorTest/pluginsFailingConf.js')
+    .expectExitCode(1)
+    .expectErrors([
+      {message: 'Expected true to be false'},
+      {message: 'from setup'},
+      {message: 'from postTest passing'},
+      {message: 'from postTest failing'},
+      {message: 'from teardown'}
+    ]);
+
 // Check ngHint plugin
 
 executor.addCommandlineTest(

spec/errorTest/pluginsFailingConf.js

@@ -0,0 +1,30 @@
+var env = require('../environment.js');
+
+// A small suite to make sure the full functionality of plugins work
+exports.config = {
+  // seleniumAddress: env.seleniumAddress,
+  mockSelenium: true,
+
+  framework: 'jasmine2',
+
+  // Spec patterns are relative to this directory.
+  specs: [
+    '../plugins/fail_spec.js'
+  ],
+
+  capabilities: env.capabilities,
+
+  baseUrl: env.baseUrl,
+
+  jasmineNodeOpts: {
+    isVerbose: true,
+    realtimeFailure: true
+  },
+
+  // Plugin patterns are relative to this directory.
+  plugins: [{
+    path: '../plugins/basic_plugin.js'
+  }, {
+    path: '../plugins/failing_plugin.js'
+  }]
+};

spec/plugins/basic_spec.js

@@ -2,4 +2,8 @@ describe('check if plugin setup ran', function() {
   it('should have set protractor.__BASIC_PLUGIN_RAN', function() {
     expect(protractor.__BASIC_PLUGIN_RAN).toBe(true);
   });
+
+  it('should run multiple tests', function() {
+    expect(true).toBe(true);
+  });
 });

spec/plugins/fail_spec.js

@@ -0,0 +1,9 @@
+describe('check if plugin setup ran', function() {
+  it('should have set protractor.__BASIC_PLUGIN_RAN', function() {
+    expect(protractor.__BASIC_PLUGIN_RAN).toBe(true);
+  });
+
+  it('should run multiple tests which fail', function() {
+    expect(true).toBe(false);
+  });
+});

spec/plugins/failing_plugin.js

@@ -0,0 +1,45 @@
+var q = require('q');
+
+var failingResult = function(message) {
+  return {
+    failedCount: 1,
+    specResults: [{
+      description: 'plugin test which fails',
+      assertions: [{
+        passed: false,
+        errorMsg: message,
+      }],
+      duration: 4
+    }]
+  };
+};
+
+module.exports = {
+  setup: function() {
+    return q.delay(100).then(function() {
+      return failingResult('from setup');
+    });
+  },
+
+  teardown: function() {
+    return q.delay(100).then(function() {
+      return failingResult('from teardown');
+    });
+  },
+
+  postResults: function() {
+    // This function should cause no failures.
+  },
+
+  postTest: function(config, passed) {
+    if (passed) {
+      return q.delay(100).then(function() {
+        return failingResult('from postTest passing');
+      });
+    } else {
+      return q.delay(100).then(function() {
+        return failingResult('from postTest failing');
+      });
+    }
+  }
+};

spec/plugins/test_plugin.js

@@ -1,12 +1,34 @@
+var q = require('q');
+
+var passingResult = {
+  failedCount: 0,
+  specResults: [{
+    description: 'plugin test which passes',
+    assertions: [],
+    duration: 4
+  }]
+};
+
 module.exports = {
+  setup: function() {
+    return q.delay(100).then(function() {
+      return passingResult;
+    });
+  },
+
   teardown: function() {
-    return {
-      failedCount: 0,
-      specResults: [{
-        description: 'This succeeds',
-        assertions: [],
-        duration: 1
-      }]
-    };
+    return q.delay(100).then(function() {
+      return passingResult;
+    });
+  },
+
+  postResults: function() {
+    // This function should cause no failures.
+  },
+
+  postTest: function() {
+    return q.delay(100).then(function() {
+      return passingResult;
+    });
   }
 };

spec/pluginsBasicConf.js

@@ -3,7 +3,7 @@ var env = require('./environment.js');
 // A small suite to make sure the basic functionality of plugins work
 // Tests the (potential) edge case of exactly one plugin being used
 exports.config = {
-  seleniumAddress: env.seleniumAddress,
+  mockSelenium: true,
 
   framework: 'jasmine2',
 

spec/pluginsFullConf.js

@@ -2,7 +2,8 @@ var env = require('./environment.js');
 
 // A small suite to make sure the full functionality of plugins work
 exports.config = {
-  seleniumAddress: env.seleniumAddress,
+  // seleniumAddress: env.seleniumAddress,
+  mockSelenium: true,
 
   framework: 'jasmine2',