diff --git a/README.md b/README.md index 47061df..3f6e0cf 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,13 @@ This is the pelias fuzzy tester library, used for running our What are fuzzy tests? See the original [problem statement](https://github.com/pelias/acceptance-tests/issues/109) that lead to the creation of this library. +Most importantly, fuzzy tests deliver more than just a single bit of pass or fail for each test: +they specify a total number of points (a score) for the test, and return how many points out of the +maximum were achieved. The weighting of individual parts of the test can be adjusted. + **Note:** fuzzy-tester requires NPM version 2 or greater. The NPM team [recommends](http://blog.npmjs.org/post/85484771375/how-to-install-npm) you update NPM using NPM itself with `sudo npm install -g npm`. - ## Usage ``` @@ -32,6 +35,8 @@ properties: + `priorityThresh` indicates the expected result must be found within the top N locations. This can be set for the entire suite as well as overwritten in individual test cases. + `tests` is an array of test cases that make up the suite. + `endpoint` the API endpoint (`search`, `reverse`, `suggest`) to target. Defaults to `search`. + + `weights` (optional) test suite wide weighting for scores of the individual expectations. See the + weights section below `tests` consists of objects with the following properties: + `id` is a unique identifier within the test suite (this could be unnecessary, let's discuss) @@ -55,6 +60,8 @@ properties: + `unexpected` is analogous to `expected`, except that you *cannot* specify a `priorityThresh` and the `properties` array does *not* support strings. + + `weights` (optional) test case specific weighting for scores of the individual expectations. See the + weights section below ## output generators The acceptance-tests support multiple different output generators, like an email and terminal output. See `node test @@ -89,3 +96,22 @@ override the default aliases and define your own in `pelias-config`: } } ``` + +## Weights + +Weights are used to influence the score each individual expectation contributes to the total score +for a test. By default, all fields in expected properties, passing the priority threshold, and the +absence of any unexpected properties each contribute one point. + +Any score for any individual property can be changed by specifying an object `weights` in a test +suite, or in an individual test case. For example, to more heavily weight the `name` property by +giving it a weight of 10 points, set weights to the following: +```javascript +{ + "properties": { + "name": 10 + } +} +``` + +Weights can be nested and are completely optional, in which case the defaults will be in effect. diff --git a/lib/eval_test.js b/lib/eval_test.js index 61df986..66d69e4 100644 --- a/lib/eval_test.js +++ b/lib/eval_test.js @@ -1,15 +1,46 @@ 'use strict'; -var equalProperties = require( '../lib/equal_properties' ); +var scoreTest = require( '../lib/scoreTest' ); +var sanitiseTestCase = require( '../lib/sanitiseTestCase' ); var isObject = require( 'is-object' ); -var util = require( 'util' ); -var path = require( 'path' ); - -var locations; -try { - locations = require( path.resolve(process.cwd() + '/locations.json') ); -} catch (e) { - locations = []; + +function formatTestErrors(score) { + var message = 'score ' + score.score + ' out of ' + score.max_score; + + if (score.score < score.max_score) { + message += '\ndiff: ' + JSON.stringify(score.diff, null, 2); + } + + return message; +} + +/** + * Ensure the weights object is valid by filling in any missing + * default values. + */ +function setDefaultWeights(weights) { + weights = weights || {}; + weights.properties = weights.properties || {}; + weights.priorityThresh = weights.priorityThresh || 1; + weights.unexpected = weights.unexpected || 1; + + return weights; +} + +/** + * Combine a context passed in from a test suite with properties + * from one individual test case to create the final context for this + * test case. It handles locations, weights, and priorityThresh + */ +function makeTestContext( testCase, context ) { + context.locations = context.locations || {}; + context.weights = setDefaultWeights(context.weights); + + if( 'expected' in testCase && 'priorityThresh' in testCase.expected ){ + context.priorityThresh = testCase.expected.priorityThresh; + } + + return context; } /** @@ -17,12 +48,16 @@ try { * priority-threshold to find the results in, return an object indicating the * status of this test (whether it passed, failed, is a placeholder, etc.) */ -function evalTest( priorityThresh, testCase, apiResults ){ - if( (!( 'expected' in testCase ) || testCase.expected.properties === null) && - !( 'unexpected' in testCase ) ){ +function evalTest( testCase, apiResults, context ){ + context = makeTestContext( testCase, context ); + + testCase = sanitiseTestCase(testCase, context.locations); + + // on error, sanitiseTestCase returns an error message string + if (typeof testCase === 'string') { return { result: 'placeholder', - msg: 'Placeholder test, no `expected` specified.' + msg: testCase }; } @@ -33,77 +68,14 @@ function evalTest( priorityThresh, testCase, apiResults ){ }; } - var ind; - var expected = []; - if( 'expected' in testCase ){ - for( ind = 0; ind < testCase.expected.properties.length; ind++ ){ - var testCaseProps = testCase.expected.properties[ ind ]; - if( typeof testCaseProps === 'string' ){ - if( testCaseProps in locations ){ - expected.push(locations[ testCaseProps ]); - } - else { - return { - result: 'placeholder', - msg: 'Placeholder test, no `out` object matches in `locations.json`.' - }; - } - } - else { - expected.push( testCaseProps ); - } - } - - if( 'priorityThresh' in testCase.expected ){ - priorityThresh = testCase.expected.priorityThresh; - } - } - - var unexpected = ( testCase.hasOwnProperty( 'unexpected' ) ) ? - testCase.unexpected.properties : []; - - var expectedResultsFound = []; - - for( ind = 0; ind < apiResults.length; ind++ ){ - var result = apiResults[ ind ]; - for( var expectedInd = 0; expectedInd < expected.length; expectedInd++ ){ - if( expectedResultsFound.indexOf( expectedInd ) === -1 && - equalProperties( expected[ expectedInd ], result.properties ) ){ - var success = ( ind + 1 ) <= priorityThresh; - if( !success ){ - return { - result: 'fail', - msg: util.format( 'Result found, but not in top %s. (%s)', priorityThresh, ind+1 ) - }; - } - else { - expectedResultsFound.push( expectedInd ); - } - } - } - - for( var unexpectedInd = 0; unexpectedInd < unexpected.length; unexpectedInd++ ){ - if( equalProperties( unexpected[ unexpectedInd ], result.properties ) ){ - return { - result: 'fail', - msg: util.format( 'Unexpected result found.' ) - }; - } - } - } - - if ( expectedResultsFound.length === expected.length ) { - return { result: 'pass' }; - } - - if ( expected.length === 0 && unexpected.length > 0 ) { - return {result: 'pass'}; - } + var score = scoreTest(testCase, apiResults, context); return { - result: 'fail', - msg: 'No matching result found.' - }; + result: (score.score < score.max_score) ? 'fail' : 'pass', + score: score.score, + max_score: score.max_score, + msg: formatTestErrors(score) + }; } module.exports = evalTest; diff --git a/lib/exec_test_suite.js b/lib/exec_test_suite.js index d57ce6d..2d690d1 100644 --- a/lib/exec_test_suite.js +++ b/lib/exec_test_suite.js @@ -7,10 +7,18 @@ var evalTest = require( '../lib/eval_test' ); var ExponentialBackoff = require( '../lib/ExponentialBackoff'); var request = require( 'request' ); +var path = require( 'path' ); var util = require( 'util' ); var validTestStatuses = [ 'pass', 'fail', undefined ]; +var locations; +try { + locations = require( path.resolve(process.cwd() + '/locations.json') ); +} catch (e) { + locations = {}; +} + function validateTestSuite(testSuite) { testSuite.tests.forEach( function ( testCase ){ if( validTestStatuses.indexOf( testCase.status ) === -1 ){ @@ -136,8 +144,14 @@ function execTestSuite( apiUrl, testSuite, stats, cb ){ var results; if( res.statusCode === 200 ){ + var context = { + priorityThresh: testSuite.priorityThresh, + locations: locations, + weights: testSuite.weights + }; + test_interval.decreaseBackoff(); - results = evalTest( testSuite.priorityThresh, testCase, res.body.features ); + results = evalTest( testCase, res.body.features, context ); } else { // unexpected (non 200 or retry) status code test_interval.increaseBackoff(); printRequestErrorMessage(testCase, res); diff --git a/lib/sanitiseTestCase.js b/lib/sanitiseTestCase.js new file mode 100644 index 0000000..b84664f --- /dev/null +++ b/lib/sanitiseTestCase.js @@ -0,0 +1,66 @@ +'use strict'; + +/** + * Given the properties of a test case, + * construct the actual expected object. + * This simply acounts for pre-defiend locations + */ +function constructExpectedOutput(properties, locations) { + return properties.map(function(property) { + if ( typeof property === 'string' && property in locations ) { + return locations[property]; + // this intentionally leaves unmatched location strings as strings + // that way it is possible to go back and look for them later + } else { + return property; + } + }); +} + +/** + * Find unmatched location strings left from running constructExpectedOutput + */ +function findPlaceholders(expected) { + return expected.filter(function(item) { + return typeof item === 'string'; + }); +} + +/** + * some tests don't have a properties array, if the properties + * object is just a single string, turn it into a one element + * array + */ +function normalizeProperties(properties) { + if (typeof properties === 'string') { + properties = [properties]; + } + return properties; +} + +function sanitiseTestCase(testCase, locations) +{ + locations = locations || {}; + + if (!testCase.expected && !testCase.unexpected) { + return 'Placeholder test: no `expected` or `unexpected` given'; + } + + if (testCase.expected) { + if (!testCase.expected.properties) { + return 'Placeholder test: `expected` block is empty'; + } + + testCase.expected.properties = normalizeProperties(testCase.expected.properties); + testCase.expected.properties = constructExpectedOutput(testCase.expected.properties, locations); + + var unmatched_placeholders = findPlaceholders(testCase.expected.properties); + if (unmatched_placeholders.length > 0) { + return 'Placeholder: no matches for ' + unmatched_placeholders.join(', ') + ' in `locations.json`.'; + } + } + + return testCase; +} + +module.exports = sanitiseTestCase; diff --git a/lib/scoreHelpers.js b/lib/scoreHelpers.js new file mode 100644 index 0000000..912ef03 --- /dev/null +++ b/lib/scoreHelpers.js @@ -0,0 +1,70 @@ +var deepDiff = require( 'deep-diff' ); + +var initial_score = {score: 0, max_score: 0, diff: []}; + +/** + * Use the deep-diff library to create an (almost too) detailed description + * of the differences between the expected and actual properties. Some massaging + * of the data so only the parts we care about are shown is done. + */ +function createDiff(expectation, result) { + var diff = deepDiff.diff(expectation, result); + + // objects with no differences have an undefined diff + if (diff === undefined) { + return ''; // return an empty string for less confusing output later + } + + // filter out diff elements corresponding to a new element on the right side + // these are ignored by our tests and would just be noise + return diff.filter(function(diff_part) { + return diff_part.kind !== 'N'; + }); +} + +function filterDiffs(diff) { + if( diff === '' || (Array.isArray(diff) && diff.length === 0)) { + return false; + } + return true; +} + + +/** + * function to be used with Array.reduce to combine several subscores for + * the same object and build one score that combines all of them. + * It totals up the actual and maximum score, and concatenates + * multiple diff or error outputs into an array + */ +function combineScores(total_score, this_score) { + var new_diff = total_score.diff; + if (this_score.diff) { + new_diff = total_score.diff.concat(this_score.diff); + new_diff = new_diff.filter(filterDiffs); + } + + return { + score: total_score.score + this_score.score, + max_score: total_score.max_score + this_score.max_score, + diff: new_diff + }; +} + +/** + * Small helper function to determine if a given apiResult is high + * enough in a set of results to pass a priority threshold. + * Caveat: the result object must be the exact same javascript object + * as the one taken from the apiResults, not just another object with + * identical properties + */ +function inPriorityThresh(apiResults, result, priorityThresh) { + var index = apiResults.indexOf(result); + return index !== -1 && index <= priorityThresh - 1; +} + +module.exports = { + initial_score: initial_score, + createDiff: createDiff, + combineScores: combineScores, + inPriorityThresh: inPriorityThresh +}; diff --git a/lib/scoreProperties.js b/lib/scoreProperties.js new file mode 100644 index 0000000..c73f665 --- /dev/null +++ b/lib/scoreProperties.js @@ -0,0 +1,36 @@ +'use strict'; + +var isObject = require( 'is-object' ); +var helpers = require( '../lib/scoreHelpers' ); + +/** + * Helper function to score javascript primitives (i.e. not objects) + */ +function scorePrimitiveProperty(expectation, result, weight) { + weight = weight || 1; + + return { + score: expectation === result ? weight : 0, + max_score: weight + }; +} + +/** + * Calculate the score of an api result against given expectations by iterating + * through all the keys, calculating the subscores, and aggregating them + */ +function scoreProperties(expectation, result, weight) { + if (isObject(expectation) && isObject(result)) { + weight = weight || {}; + var subscores = Object.keys(expectation).map(function(property){ + return scoreProperties(expectation[property], result[property], weight[property]); + }); + + var diff = [helpers.createDiff(expectation, result)]; + return subscores.reduce(helpers.combineScores, { score: 0, max_score: 0, diff: diff}); + } else { + return scorePrimitiveProperty(expectation, result, weight); + } +} + +module.exports = scoreProperties; diff --git a/lib/scoreTest.js b/lib/scoreTest.js new file mode 100644 index 0000000..b0a69ed --- /dev/null +++ b/lib/scoreTest.js @@ -0,0 +1,99 @@ +var helpers = require( '../lib/scoreHelpers' ); +var equalProperties = require( '../lib/equal_properties' ); +var scoreProperties = require( '../lib/scoreProperties' ); + +/** + * Calculate the score component for a results object's position in the returned + * results. If it is later in the results than the threshold, no points. + * Otherwise, full points. + */ +function scorePriorityThresh(priorityThresh, result, apiResults, weight) { + var index = apiResults.indexOf(result); + var diff = ''; + var score = weight; + var success = helpers.inPriorityThresh(apiResults, result, priorityThresh); + if (!success) { + score = 0; + diff = 'priorityThresh is ' + priorityThresh + ' but found at position ' + (index + 1); + } + + return { + score: score, + max_score: weight, + diff: diff + }; +} + +/** + * Calculate the score component for the absence of unexpected properties in + * API results. Full points are awarded if all of the unexpected properties + * are completely absent from all API results. Otherwise no points. + */ +function scoreUnexpected(unexpected, apiResults, weight) { + // unexpected result should be absent from every returned result + var pass = apiResults.every(function(result) { + // if every unexpected input is not found, the tests pass + return unexpected.properties.every(function(property) { + return !equalProperties(property, result.properties); + }); + }); + + return { + score: pass ? weight : 0, + max_score: weight, + diff: pass ? '' : 'unexpected result found' + }; +} + +/** + * Score one result from the set of api results by combining the score + * for its properties and whether or not it satisfies the priorityThresh + */ +function scoreOneResult(expected, apiResult, apiResults, context) { + var subscores = [ + scoreProperties(expected, apiResult.properties, context.weights.properties), + scorePriorityThresh(context.priorityThresh, apiResult, apiResults, context.weights.priorityThresh) + ]; + + return subscores.reduce(helpers.combineScores, helpers.initial_score); +} + +/** + * Helper method to sort scores in descending order + */ +function sortScores(score_a, score_b) { + return score_b.score - score_a.score; +} + +/** + * Score one (out of potentially many) expectations in a test suite by finding + * the highest score of any api result when scored against this expectation. + */ +function scoreOneExpectation(expected, apiResults, context) { + return apiResults.map(function(result) { + var score = scoreOneResult(expected, result, apiResults, context); + return score; + }).sort(sortScores)[0]; +} + +/** + * Score an entire test by combining the score for each expectation, and the + * score for any unexpected properties. + */ +function scoreTest(testCase, apiResults, context) { + var scores = []; + + if (testCase.expected) { + scores = scores.concat(testCase.expected.properties.map(function(expected) { + return scoreOneExpectation(expected, apiResults, context); + })); + } + + if (testCase.unexpected) { + scores = scores.concat(scoreUnexpected(testCase.unexpected, apiResults, context.weights.unexpected)); + } + + return scores.reduce(helpers.combineScores, helpers.initial_score); +} + +module.exports = scoreTest; diff --git a/package.json b/package.json index 166e75f..77d6c63 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "dependencies": { "colors": "^1.x.x", "commander": "2.7.0", + "deep-diff": "0.3.2", "fs-extra": "^0.22.1", "handlebars": "^3.x.x", "is-object": "^1.0.1", diff --git a/test/eval_test.js b/test/eval_test.js index 44b1fd9..591a9bb 100644 --- a/test/eval_test.js +++ b/test/eval_test.js @@ -50,6 +50,24 @@ tape( 'evalTest() evaluates all edge cases correctly', function ( test ){ testCase: {}, expected: 'placeholder' }, + { + description: 'Test case with string in properties array matching location uses that location', + priorityThresh: 2, + locations: { + 'my home': { + 'name': 'la casa' + } + }, + apiResults: [{ properties: {} }, { properties: { name: 'la casa' }}], + testCase: { + expected: { + properties: [ + 'my home' + ] + } + }, + expected: 'pass' + }, { description: 'Test case with no `locations.json` props identified as a placeholder.', priorityThresh: -1, @@ -125,6 +143,24 @@ tape( 'evalTest() evaluates all edge cases correctly', function ( test ){ }, expected: 'pass' }, + { + description: 'Expected blocks need not be specified in the order they appear in the api results', + priorityThresh: 3, + apiResults: [ + { properties: {a:1, b:2} }, + { properties: {a:3, b:5} }, + { properties: {a:4, b:6} } + ], + testCase: { + expected: { + properties: [ + {a:4, b:6}, + {a:1, b:2} + ] + } + }, + expected: 'pass' + }, { description: 'Only one of multiple expected blocks found should fail', priorityThresh: 3, @@ -159,14 +195,47 @@ tape( 'evalTest() evaluates all edge cases correctly', function ( test ){ } }, expected: 'fail' + }, + { + description: 'Weights can be set at the testSuite level', + priorityThresh: 3, + weights: { + properties: { + a: 50 + } + }, + apiResults: [ + { properties: {a:1, b:2} }, + { properties: {a:4, b:6} } + ], + testCase: { + expected: { + properties: [ + {a:1, b:2}, + {a:4, b:6} + ] + } + }, + expected: 'pass', + expected_score: 104 // 2x 50 for a, 2x1 for b, 2x1 for priorityThresh } ]; - tests.forEach( function ( testCase ){ + tests.forEach( function ( one_test ){ + var context = { + priorityThresh: one_test.priorityThresh, + locations: one_test.locations, + weights: one_test.weights + }; + var result = evalTest( - testCase.priorityThresh, testCase.testCase, testCase.apiResults + one_test.testCase, one_test.apiResults, context ); - test.equal( result.result, testCase.expected, testCase.description ); + test.equal( result.result, one_test.expected, one_test.description ); + + if (one_test.expected_score) { + test.equal( result.score, one_test.expected_score, 'score should be as expected'); + } }); test.end(); diff --git a/test/sanitiseTestCase.js b/test/sanitiseTestCase.js new file mode 100644 index 0000000..409b6d9 --- /dev/null +++ b/test/sanitiseTestCase.js @@ -0,0 +1,86 @@ +'use strict'; + +var tape = require( 'tape' ); + +var sanitiseTestCase = require( '../lib/sanitiseTestCase' ); + +tape( 'testCaseSanitiser', function ( test ){ + test.test( 'test case with no expected or unexpected block is placeholder', function( t ) { + var testCase = {}; + var expected_message = 'Placeholder test: no `expected` or `unexpected` given'; + + t.equal(expected_message, sanitiseTestCase(testCase), 'return message specifies placeholder'); + t.end(); + }); + + test.test( 'test case with empty expected block is placeholder', function( t ) { + var testCase = { expected: {}}; + var expected_message = 'Placeholder test: `expected` block is empty'; + + t.equal(expected_message, sanitiseTestCase(testCase), 'return message specifies placeholder'); + t.end(); + }); + + test.test( 'test case with expected block is not placeholder', function( t ) { + var testCase = {expected: {properties:[]}}; + + t.equal(testCase, sanitiseTestCase(testCase), 'test case returned unaltered'); + t.end(); + }); + + test.test( 'test case with unexpected block is not placeholder', function( t ) { + var testCase = {unexpected: {}}; + + t.equal(testCase, sanitiseTestCase(testCase), 'test case returned unaltered'); + t.end(); + }); + + test.test( 'test case with string in properties array and no locations marked as placeholder', function( t ) { + var testCase = { + expected: { + properties: [ + 'a place' + ] + } + }; + var expected_message = 'Placeholder: no matches for a place in `locations.json`.'; + + t.equal(expected_message, sanitiseTestCase(testCase), 'return message specifies placeholder due to location'); + t.end(); + }); + + test.test( 'test case with string in properties array and matching location replaces string', function( t ) { + var testCase = { + expected: { + properties: [ + 'a real place' + ] + } + }; + var locations = { + 'a real place': { + 'name': 'a real place', + 'admin0': 'USA', + 'admin1': 'New York' + } + }; + + var expected_testCase = { + expected: { + properties: [ + { + 'name': 'a real place', + 'admin0': 'USA', + 'admin1': 'New York' + } + ] + } + }; + + var result = sanitiseTestCase(testCase, locations); + t.deepEqual(result, expected_testCase, 'test case has location filled in'); + t.end(); + }); + + test.end(); +}); diff --git a/test/scoreProperties.js b/test/scoreProperties.js new file mode 100644 index 0000000..7ac9347 --- /dev/null +++ b/test/scoreProperties.js @@ -0,0 +1,151 @@ +'use strict'; + +var tape = require( 'tape' ); + +var scoreProperties = require( '../lib/scoreProperties' ); + +tape( 'scoreProperties basics', function ( test ){ + + test.test('single matching primitive', function ( t ){ + var expectation = 'a string'; + var result = 'a string'; + + var score_result = scoreProperties(expectation, result); + + t.equal(score_result.max_score, 1, 'max score equals 1'); + t.equal(score_result.score, 1, 'score equals 1'); + + t.end(); + }); + + test.test('single unmatching primitive', function ( t ){ + var expectation = 'a string'; + var result = 'a different string'; + + var score_result = scoreProperties(expectation, result); + + t.equal(score_result.max_score, 1, 'max score equals 1'); + t.equal(score_result.score, 0, 'score equals 0'); + + t.end(); + }); + + test.test('object with single matching primitive property', function ( t ){ + var expectation = { foo: 'a string' }; + var result = { foo: 'a string' }; + + var score_result = scoreProperties(expectation, result); + + t.equal(score_result.max_score, 1, 'max score equals 1'); + t.equal(score_result.score, 1, 'score equals 1'); + + t.end(); + }); + + test.test('object with single mismatching primitive property', function ( t ){ + var expectation = { foo: 'a string' }; + var result = { foo:'a different string' }; + + var score_result = scoreProperties(expectation, result); + + t.equal(score_result.max_score, 1, 'max score equals 1'); + t.equal(score_result.score, 0, 'score equals 0'); + + t.end(); + }); + + test.test('object with matching and mismatching properties', function ( t ) { + var expectation = { foo: 'a string', bar: 'another string' }; + var result = { foo:'a different string', bar: 'another string' }; + + var score_result = scoreProperties(expectation, result); + + t.equal(score_result.max_score, 2, 'max score equals 2'); + t.equal(score_result.score, 1, 'score equals 1'); + + t.end(); + }); + + test.test('object with nested object and matching properties', function ( t ) { + var expectation = { foo: 'a string', bar: { baz: 'another string' } }; + var result = { foo:'a string', bar: { baz: 'another string' } }; + + var score_result = scoreProperties(expectation, result); + + t.equal(score_result.max_score, 2, 'max score equals 2'); + t.equal(score_result.score, 2, 'score equals 2'); + + t.end(); + }); + + test.test('object with nested object and matching and mismatching properties', function ( t ) { + var expectation = { foo: 'a string', bar: { baz: 'another string', wrong: 'expected value' } }; + var result = { foo:'a string', bar: { baz: 'another string', wrong: 'unexpected value' } }; + + var score_result = scoreProperties(expectation, result); + + t.equal(score_result.max_score, 3, 'max score equals 3'); + t.equal(score_result.score, 2, 'score equals 2'); + + t.end(); + }); + + test.end(); +}); + + +tape( 'scoreProperties with weights', function ( test ){ + test.test('single matching weighted primitive', function ( t ){ + var expectation = 'a string'; + var result = 'a string'; + var weight = 5; + + var score_result = scoreProperties(expectation, result, weight); + + t.equal(score_result.max_score, 5, 'max score equals 5'); + t.equal(score_result.score, 5, 'score equals 5'); + + t.end(); + }); + + test.test('single unmatching weighted primitive', function ( t ){ + var expectation = 'a string'; + var result = 'a different string'; + var weight = 5; + + var score_result = scoreProperties(expectation, result, weight); + + t.equal(score_result.max_score, 5, 'max score equals 5'); + t.equal(score_result.score, 0, 'score equals 0'); + + t.end(); + }); + + test.test('object with nested object and matching properties with weight', function ( t ) { + var expectation = { foo: 'a string', bar: { baz: 'another string' } }; + var result = { foo:'a string', bar: { baz: 'another string' } }; + var weight = { foo: 5, bar: { baz: 3}}; + + var score_result = scoreProperties(expectation, result, weight); + + t.equal(score_result.max_score, 8, 'max score equals 8'); + t.equal(score_result.score, 8, 'score equals 8'); + + t.end(); + }); + + test.test('object with nested object and matching and mismatching properties', function ( t ) { + var expectation = { foo: 'a string', bar: { baz: 'another string', wrong: 'expected value' } }; + var result = { foo:'a string', bar: { baz: 'another string', wrong: 'unexpected value' } }; + var weight = { foo: 5, bar: { baz: 3, wrong: 2}}; + + var score_result = scoreProperties(expectation, result, weight); + + t.equal(score_result.max_score, 10, 'max score equals 10'); + t.equal(score_result.score, 8, 'score equals 8'); + + t.end(); + }); + + test.end(); +}); diff --git a/test/test.js b/test/test.js index ad0f0bc..0772cce 100644 --- a/test/test.js +++ b/test/test.js @@ -8,3 +8,5 @@ require('./exec_test_suite'); require('./ExponentialBackoff'); require('./eval_test'); require('./equal_properties'); +require('./sanitiseTestCase'); +require('./scoreProperties');