Merge pull request #86 from localmed/test-errors-skips

Catches errors in hooks, allows skipping tests, fixes small bug

package.json

@@ -1,6 +1,6 @@
 {
   "name": "dredd",
-  "version": "0.3.8",
+  "version": "0.3.9",
   "description": "API Blueprint testing tool",
   "main": "lib/dredd.js",
   "bin": {

src/add-hooks.coffee

@@ -8,7 +8,8 @@ async = require 'async'
 hooks = require './hooks'
 logger = require './logger'
 
-addHooks = (runner, transactions) ->
+addHooks = (runner, transactions, emitter) ->
+    @emitter = emitter
 
     for transaction in transactions
       hooks.transactions[transaction.name] = transaction
@@ -44,7 +45,11 @@ addHooks = (runner, transactions) ->
         logger.debug 'Running hooks...'
 
         runHookWithTransaction = (hook, callback) ->
-          runHook hook, transaction, callback
+          try
+            runHook hook, transaction, callback
+          catch error
+            emitError(transaction, error)
+            callback()
 
         async.eachSeries hooksForTransaction, runHookWithTransaction, () ->
           callback()
@@ -62,4 +67,12 @@ addHooks = (runner, transactions) ->
       hook transaction, () =>
         callback()
 
+  emitError = (transaction, error) ->
+    test =
+      status: ''
+      title: transaction.id
+      message: transaction.name
+      origin: transaction.origin
+    @emitter.emit 'test error', error, test if error
+
 module.exports = addHooks

src/reporters/x-unit-reporter.coffee

@@ -21,6 +21,7 @@ class XUnitReporter extends EventEmitter
     filePath = if path? then file.path.abspath(path) else file.path.abspath("./report.xml")
     if fs.existsSync(filePath)
       logger.info "File exists at #{filePath}, will be overwritten..."
+      fs.unlinkSync(filePath)
     filePath
 
   configureEmitter: (emitter) =>
@@ -35,7 +36,7 @@ class XUnitReporter extends EventEmitter
             , time: @stats.duration / 1000
           }, false)
       callback()
-    
+
     emitter.on 'end', (callback) =>
       updateSuiteStats @path, @stats, callback
 

src/transaction-runner.coffee

@@ -22,15 +22,15 @@ String::startsWith = (str) ->
 class TransactionRunner
   constructor: (@configuration) ->
     advisable.async.call TransactionRunner.prototype
-    addHooks @, {}
+    addHooks @, {}, @configuration.emitter
 
   run: (transactions, callback) ->
     transactions = if @configuration.options['sorted'] then sortTransactions(transactions) else transactions
 
     async.mapSeries transactions, @configureTransaction, (err, results) ->
       transactions = results
 
-    addHooks {}, transactions
+    addHooks {}, transactions, @configuration.emitter
 
     async.eachSeries transactions, @executeTransaction, () ->
       callback()
@@ -101,6 +101,7 @@ class TransactionRunner
       origin: origin
       fullPath: fullPath
       protocol: parsedUrl.protocol
+      skip: false
 
     return callback(null, configuredTransaction)
 
@@ -132,6 +133,10 @@ class TransactionRunner
     else if configuration.options.method.length > 0 and not (transaction.request.method in configuration.options.method)
       configuration.emitter.emit 'test skip', test
       return callback()
+    else if transaction.skip
+      # manually set to skip a test
+      configuration.emitter.emit 'test skip', test
+      return callback()
     else
       buffer = ""
 
@@ -170,7 +175,7 @@ class TransactionRunner
                 for entity, data of result
                   for entityResult in data['results']
                     message += entity + ": " + entityResult['message'] + "\n"
-                
+
                 test.status = "fail"
                 test.title = transaction.id
                 test.message = message
@@ -183,9 +188,12 @@ class TransactionRunner
                 return callback()
 
       transport = if transaction.protocol is 'https:' then https else http
-      req = transport.request requestOptions, handleRequest
-
-      req.write transaction.request['body'] if transaction.request['body'] != ''
-      req.end()
+      try
+        req = transport.request requestOptions, handleRequest
+        req.write transaction.request['body'] if transaction.request['body'] != ''
+        req.end()
+      catch error
+        configuration.emitter.emit 'test error', error, test if error
+        return callback()
 
 module.exports = TransactionRunner

test/unit/add-hooks-test.coffee

@@ -210,6 +210,23 @@ describe 'addHooks(runner, transaction)', () ->
             assert.ok loggerStub.info.calledWith "second"
             done()
 
+      describe 'with hook that throws an error', () ->
+        beforeEach () ->
+          hooksStub.beforeHooks =
+            'Group Machine > Machine > Delete Message > Bogus example name' : [
+              (transaction) ->
+                JSON.parse '<<<>>>!@#!@#!@#4234234'
+            ]
+          sinon.stub configuration.emitter, 'emit'
+
+        after () ->
+          configuration.emitter.emit.restore()
+
+        it 'should report an error with the test', (done) ->
+          runner.executeTransaction transaction, () ->
+            assert.ok emitter.emit.calledWith "test error"
+            done()
+
       describe 'without hooks', () ->
         beforeEach () ->
           hooksStub.beforeHooks = []

test/unit/reporters/x-unit-reporter-test.coffee

@@ -27,10 +27,13 @@ describe 'XUnitReporter', () ->
       before () ->
         sinon.stub fsStub, 'existsSync', (path) ->
           return true
+        sinon.stub fsStub, 'unlinkSync', (path) ->
+          return true
         sinon.stub loggerStub, 'info'
 
       after () ->
         fsStub.existsSync.restore()
+        fsStub.unlinkSync.restore()
         loggerStub.info.restore()
 
       it 'should inform about the existing file', () ->

test/unit/transaction-runner-test.coffee

@@ -201,6 +201,21 @@ describe 'TransactionRunner', ()->
           assert.ok configuration.emitter.emit.calledWith 'test skip'
           done()
 
+    describe 'when a test has been manually set to skip in a hook', () ->
+
+      beforeEach () ->
+        sinon.stub configuration.emitter, 'emit'
+        runner = new Runner(configuration)
+
+      afterEach () ->
+        configuration.emitter.emit.restore()
+
+      it 'should skip the test', (done) ->
+        transaction.skip = true
+        runner.executeTransaction transaction, () ->
+          assert.ok configuration.emitter.emit.calledWith 'test skip'
+          done()
+
     describe 'when server uses https', () ->
 
       beforeEach () ->