diff --git a/manifest.php b/manifest.php index 1540caae..b52b5bf6 100755 --- a/manifest.php +++ b/manifest.php @@ -34,7 +34,7 @@ 'label' => 'Test core extension', 'description' => 'TAO Tests extension contains the abstraction of the test-runners, but requires an implementation in order to be able to run tests', 'license' => 'GPL-2.0', - 'version' => '2.9', + 'version' => '2.10', 'author' => 'Open Assessment Technologies, CRP Henri Tudor', 'requires' => array( 'taoItems' => '>=2.6', diff --git a/scripts/update/Updater.php b/scripts/update/Updater.php index 4d42f4e7..398f04f6 100644 --- a/scripts/update/Updater.php +++ b/scripts/update/Updater.php @@ -45,9 +45,8 @@ public function update($initialVersion) $this->setVersion('2.7.1'); } - if ($this->isBetween('2.7.1','2.9')){ - $this->setVersion('2.9'); - } + $this->skip('2.7.1','2.10'); + return null; } } diff --git a/views/js/runner/pluginLoader.js b/views/js/runner/pluginLoader.js new file mode 100644 index 00000000..cbe82289 --- /dev/null +++ b/views/js/runner/pluginLoader.js @@ -0,0 +1,175 @@ +/** + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; under version 2 + * of the License (non-upgradable). + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * Copyright (c) 2016 (original work) Open Assessment Technologies SA ; + */ + +/** + * Loads the test runner plugins. + * It provides 2 distinct way of loading plugins : + * 1. The "required" plugins that are necessary to the runner, they are provided as plugins. + * 2. The "dynamic" plugins that are loaded on demand, they are provided as AMD modules. The module is loaded using the load method. + * + * @author Bertrand Chevrier + */ +define([ + 'lodash', + 'core/promise' +], function(_, Promise) { + 'use strict'; + + /** + * Creates a loader with the list of required plugins + * @param {String: Function[]} requiredPlugins - where the key is the category and the value are an array of plugins + * @returns {loader} the plugin loader + * @throws TypeError if something is not well formated + */ + return function pluginLoader(requiredPlugins) { + + var plugins = {}; + var modules = {}; + + /** + * The plugin loader + * @typedef {loader} + */ + var loader = { + + /** + * Add a new dynamic plugin + * @param {String} module - AMD module name of the plugin + * @param {String} category - the plugin category + * @param {String|Number} [position = 'append'] - append, prepend or plugin position within the category + * @returns {loader} chains + * @throws {TypeError} misuse + */ + add: function add(module, category, position) { + if(!_.isString(module)){ + throw new TypeError('An AMD module must be defined'); + } + if(!_.isString(category)){ + throw new TypeError('Plugins must belong to a category'); + } + + modules[category] = modules[category] || []; + + if(_.isNumber(position)){ + modules[category][position] = module; + } + else if(position === 'prepend' || position === 'before'){ + modules[category].unshift(module); + } else { + modules[category].push(module); + } + + return this; + }, + + /** + * Append a new dynamic plugin to a category + * @param {String} module - AMD module name of the plugin + * @param {String} category - the plugin category + * @returns {loader} chains + * @throws {TypeError} misuse + */ + append: function append(module, category) { + return this.add(module, category); + }, + + /** + * Prepend a new dynamic plugin to a category + * @param {String} module - AMD module name of the plugin + * @param {String} category - the plugin category + * @returns {loader} chains + * @throws {TypeError} misuse + */ + prepend: function prepend(module, category) { + return this.add(module, category, 'before'); + }, + + /** + * Loads the dynamic plugins : trigger the dependency resolution + * @returns {Promise} + */ + load: function load() { + return new Promise(function(resolve, reject){ + + var dependencies = _(modules).values().flatten().uniq().value(); + + if(dependencies.length){ + + require(dependencies, function(){ + var loadedModules = [].slice.call(arguments); + _.forEach(dependencies, function(dependency, index){ + var plugin = loadedModules[index]; + var category = _.findKey(modules, function(val){ + return _.contains(val, dependency); + }); + if(_.isFunction(plugin) && _.isString(category)){ + plugins[category] = plugins[category] || []; + plugins[category].push(plugin); + } + }); + resolve(); + + }, reject); + return; + } + resolve(); + }); + }, + + /** + * Get the resolved plugin list. + * Load needs to be called before to have the dynamic plugins. + * @param {String} [category] - to get the plugins for a given category, if not set, we get everything + * @returns {Function[]} the plugins + */ + getPlugins: function getPlugins(category) { + if(_.isString(category)){ + return plugins[category] || []; + } + return _(plugins).values().flatten().uniq().value(); + }, + + /** + * Get the plugin categories + * @returns {String[]} the categories + */ + getCategories: function getCategories(){ + return _.keys(plugins); + } + }; + + //verify and add the required plugins + _.forEach(requiredPlugins, function(pluginList, category){ + if(!_.isString(category)){ + throw new TypeError('Plugins must belong to a category'); + } + + if(!_.isArray(pluginList) || !_.all(pluginList, _.isFunction)){ + throw new TypeError('A plugin must be an array of function'); + } + + if(plugins[category]){ + plugins[category] = plugins[category].concat(pluginList); + } else { + plugins[category] = pluginList; + } + }); + + return loader; + }; +}); diff --git a/views/js/runner/probeOverseer.js b/views/js/runner/probeOverseer.js new file mode 100644 index 00000000..b31eda29 --- /dev/null +++ b/views/js/runner/probeOverseer.js @@ -0,0 +1,316 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; under version 2 + * of the License (non-upgradable). + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * Copyright (c) 2016 (original work) Open Assessment Technlogies SA + * + */ + +/** + * The probeOverseer let's you define probes that will listen for events and record logs + * + * @author Bertrand Chevrier + */ +define([ + 'lodash', + 'core/promise', + 'moment', + 'lib/uuid', + 'lib/localforage', + 'lib/moment-timezone.min' +], function (_, Promise, moment, uuid, localforage){ + 'use strict'; + + var timeZone = moment.tz.guess(); + + /** + * Create the overseer intance + * @param {String} testIdentifier - a unique id for a test execution + * @param {runner} runner - a insance of a test runner + * @returns {probeOverseer} the new probe overseer + * @throws TypeError if something goes wrong + */ + return function probeOverseerFactory(testIdentifier, runner){ + + // the created instance + var overseer; + + // the list of registered probes + var probes = []; + + //the data store instance + var store; + + //temp queue + var queue = []; + + //current write promise + var writing; + + //is the overseer started + var started = false; + + /** + * Register the collection event of a probe against a runner + * @param {Object} probe - a valid probe + */ + var collectEvent = function collectEvent(probe){ + + var eventNs = '.probe-' + probe.name; + + //event handler registered to collect data + var probeHandler = function probeHandler(){ + var now = moment(); + var last; + var data = { + id : uuid(8, 16), + type : probe.name, + timestamp : now.format('x') / 1000, + timezone : now.tz(timeZone).format('Z') + }; + if(typeof probe.capture === 'function'){ + data.context = probe.capture(runner); + } + overseer.push(data); + }; + + //fallback + if(probe.latency){ + return collectLatencyEvent(probe); + } + + _.forEach(probe.events, function(eventName){ + var listen = eventName.indexOf('.') > 0 ? eventName : eventName + eventNs; + runner.on(listen, probeHandler); + }); + }; + + var collectLatencyEvent = function collectLatencyEvent(probe){ + + var eventNs = '.probe-' + probe.name; + + //start event handler registered to collect data + var startHandler = function startHandler(){ + var now = moment(); + var data = { + id: uuid(8, 16), + marker: 'start', + type : probe.name, + timestamp : now.format('x') / 1000, + timezone : now.tz(timeZone).format('Z') + }; + + if(typeof probe.capture === 'function'){ + data.context = probe.capture(runner); + } + overseer.push(data); + }; + + //stop event handler registered to collect data + var stopHandler = function stopHandler(){ + var now = moment(); + var last; + var data = { + type : probe.name, + timestamp : now.format('x') / 1000, + timezone : now.tz(timeZone).format('Z') + }; + overseer.getQueue().then(function(queue){ + last = _.findLast(queue, { type : probe.name, marker : 'start' }); + if(last && !_.findLast(queue, { type : probe.name, marker : 'stop', id : last.id })){ + data.id = last.id; + data.marker = 'end'; + if(typeof probe.capture === 'function'){ + data.context = probe.capture(runner); + } + overseer.push(data); + } + }); + }; + + //fallback + if(!probe.latency){ + return collectEvent(probe); + } + + _.forEach(probe.startEvents, function(eventName){ + var listen = eventName.indexOf('.') > 0 ? eventName : eventName + eventNs; + runner.on(listen, startHandler); + }); + _.forEach(probe.stopEvents, function(eventName){ + var listen = eventName.indexOf('.') > 0 ? eventName : eventName + eventNs; + runner.on(listen, stopHandler); + }); + }; + + //argument validation + if(_.isEmpty(testIdentifier)){ + throw new TypeError('Please set a test identifier'); + } + if(!_.isPlainObject(runner) || !_.isFunction(runner.init) || !_.isFunction(runner.on)){ + throw new TypeError('Please set a test runner'); + } + + //create a unique instance in the offline storage + store = localforage.createInstance({ + name: 'test-probe-' + testIdentifier + }); + + /** + * @typedef {probeOverseer} + */ + overseer = { + + /** + * Add a new probe + * @param {Object} probe + * @param {String} probe.name - the probe name + * @param {Boolean} [probe.latency = false] - simple or latency mode + * @param {String[]} [probe.events] - the list of events to listen (simple mode) + * @param {String[]} [probe.startEvents] - the list of events to mark the start (lantency mode) + * @param {String[]} [probe.stopEvents] - the list of events to mark the end (latency mode) + * @param {Function} [probe.capture] - lambda fn to define the data context, it receive the test runner and the event parameters + * @returns {probeOverseer} chains + * @throws TypeError if the probe is not well formatted + */ + add: function add(probe){ + + // probe structure strict validation + + if(!_.isPlainObject(probe)){ + throw new TypeError('A probe is a plain object'); + } + if(!_.isString(probe.name) || _.isEmpty(probe.name)){ + throw new TypeError('A probe must have a name'); + } + if(_.where(probes, {name : probe.name }).length > 0){ + throw new TypeError('A probe with this name is already regsitered'); + } + + if(probe.latency){ + if(_.isString(probe.startEvents) && !_.isEmpty(probe.startEvents)){ + probe.startEvents = [probe.startEvents]; + } + if(_.isString(probe.stopEvents) && !_.isEmpty(probe.stopEvents)){ + probe.stopEvents = [probe.stopEvents]; + } + if(!probe.startEvents.length || !probe.stopEvents.length){ + throw new TypeError('Latency based probes must have startEvents and stopEvents defined'); + } + + //if already started we register the events on addition + if(started){ + collectLatencyEvent(probe); + } + } else { + if(_.isString(probe.events) && !_.isEmpty(probe.events)){ + probe.events = [probe.events]; + } + if(!_.isArray(probe.events) || probe.events.length === 0){ + throw new TypeError('A probe must define events'); + } + + //if already started we register the events on addition + if(started){ + collectEvent(probe); + } + } + + probes.push(probe); + + return this; + }, + + + /** + * Get the time entries queue + * @returns {Promise} with the data in parameterj + */ + getQueue : function getQueue(){ + return store.getItem('queue'); + }, + + /** + * Get the list of defined probes + * @returns {Object[]} the probes collection + */ + getProbes : function getProbes(){ + return probes; + }, + + /** + * Push an time entry to the queue + * @param {Object} entry - the time entry + */ + push : function push(entry){ + queue.push(entry); + + //ensure the queue is pushed to the store consistently and atomically + if(writing){ + writing.then(function(){ + return store.setItem('queue', queue); + }); + } else { + writing = store.setItem('queue', queue); + } + }, + + /** + * Flush the queue and get the entries + * @returns {Promise} with the data in parameter + */ + flush: function flush(){ + + return new Promise(function(resolve){ + store.getItem('queue').then(function(flushed){ + queue = []; + store.setItem('queue', queue); + resolve(flushed); + }); + }); + }, + + /** + * Start the probes + */ + start : function start(){ + _.forEach(probes, collectEvent); + started = true; + }, + + + /** + * Stop the probes + * Be carefull, stop will also clear the store and the queue + */ + stop : function stop(){ + started = false; + _.forEach(probes, function(probe){ + var eventNs = '.probe-' + probe.name; + var removeHandler = function removeHandler(eventName){ + runner.off(eventName + eventNs); + }; + + _.forEach(probe.startEvents, removeHandler); + _.forEach(probe.stopEvents, removeHandler); + _.forEach(probe.events, removeHandler); + }); + + queue = []; + store.clear(); + } + }; + return overseer; + }; +}); diff --git a/views/js/runner/runner.js b/views/js/runner/runner.js index 86711276..458931a1 100644 --- a/views/js/runner/runner.js +++ b/views/js/runner/runner.js @@ -56,6 +56,11 @@ define([ */ var testContext = {}; + /** + * @type {Object} contextual test map (the map of accessible items) + */ + var testMap = {}; + /** * @type {Object} the registered plugins */ @@ -72,6 +77,11 @@ define([ 'destroy': false }; + /** + * @type {Object} keeps the states of the items + */ + var itemStates = {}; + /** * The selected test runner provider */ @@ -89,6 +99,13 @@ define([ */ var proxy; + + /** + * Keep the instance of the probes overseer + * @see taoTests/runner/probeOverseer + */ + var probeOverseer; + /** * Run a method of the provider (by delegation) * @@ -205,8 +222,9 @@ define([ var self = this; providerRun('loadItem', itemRef).then(function(itemData){ - self.trigger('loaditem', itemRef) - .renderItem(itemData); + self.setItemState(itemRef, 'loaded', true) + .trigger('loaditem', itemRef) + .renderItem(itemRef, itemData); }).catch(reportError); return this; }, @@ -219,11 +237,12 @@ define([ * @fires runner#renderitem * @returns {runner} chains */ - renderItem : function renderItem(itemData){ + renderItem : function renderItem(itemRef, itemData){ var self = this; - providerRun('renderItem', itemData).then(function(){ - self.trigger('renderitem', itemData); + providerRun('renderItem', itemRef, itemData).then(function(){ + self.setItemState(itemRef, 'ready', true) + .trigger('renderitem', itemRef, itemData); }).catch(reportError); return this; }, @@ -240,11 +259,51 @@ define([ var self = this; providerRun('unloadItem', itemRef).then(function(){ + itemStates = _.omit(itemStates, itemRef); self.trigger('unloaditem', itemRef); }).catch(reportError); return this; }, + /** + * Disable an item + * - provider disableItem + * @param {*} itemRef - something that let you identify the item + * @fires runner#disableitem + * @returns {runner} chains + */ + disableItem : function disableItem(itemRef){ + var self = this; + + if(!this.getItemState(itemRef, 'disabled')){ + + providerRun('disableItem', itemRef).then(function(){ + self.setItemState(itemRef, 'disabled', true) + .trigger('disableitem', itemRef); + }).catch(reportError); + } + return this; + }, + + /** + * Enable an item + * - provider enableItem + * @param {*} itemRef - something that let you identify the item + * @fires runner#disableitem + * @returns {runner} chains + */ + enableItem : function enableItem(itemRef){ + var self = this; + + if(this.getItemState(itemRef, 'disabled')){ + providerRun('enableItem', itemRef).then(function(){ + self.setItemState(itemRef, 'disabled', false) + .trigger('enableitem', itemRef); + }).catch(reportError); + } + return this; + }, + /** * When the test is terminated * - provider finish @@ -339,6 +398,19 @@ define([ return proxy; }, + /** + * Get the probeOverseer, and load it if not present + * + * @returns {probeOverseer} the probe overseer + */ + getProbeOverseer : function getProbeOverseer(){ + if(!probeOverseer && _.isFunction(provider.loadProbeOverseer)){ + probeOverseer = provider.loadProbeOverseer.call(this); + } + + return probeOverseer; + }, + /** * Check a runner state * @@ -366,6 +438,47 @@ define([ return this; }, + /** + * Check an item state + * + * @param {*} itemRef - something that let you identify the item + * @param {String} name - the state name + * @returns {Boolean} if active, false if not set + * + * @throws {TypeError} if there is no itemRef nor name + */ + getItemState : function getItemState(itemRef, name){ + if( _.isEmpty(itemRef) || _.isEmpty(name)){ + throw new TypeError('The state is identified by an itemRef and a name'); + } + return !!(itemStates[itemRef] && itemStates[itemRef][name]); + }, + + /** + * Check an item state + * + * @param {*} itemRef - something that let you identify the item + * @param {String} name - the state name + * @param {Boolean} active - is the state active + * @returns {runner} chains + * + * @throws {TypeError} if there is no itemRef nor name + */ + setItemState : function setItemState(itemRef, name, active){ + if( _.isEmpty(itemRef) || _.isEmpty(name)){ + throw new TypeError('The state is identified by an itemRef and a name'); + } + itemStates[itemRef] = itemStates[itemRef] || { + 'loaded' : false, + 'ready' : false, + 'disabled': false + }; + + itemStates[itemRef][name] = !!active; + + return this; + }, + /** * Get the test data/definition * @returns {Object} the test data @@ -405,6 +518,26 @@ define([ return this; }, + /** + * Get the test items map + * @returns {Object} the test map + */ + getTestMap : function getTestMap(){ + return testMap; + }, + + /** + * Set the test items map + * @param {Object} map - the map to set + * @returns {runner} chains + */ + setTestMap : function setTestMap(map){ + if(_.isPlainObject(map)){ + testMap = map; + } + return this; + }, + /** * Move next alias * @param {String|*} [scope] - the movement scope @@ -435,7 +568,7 @@ define([ * @returns {runner} chains */ jump : function jump(position, scope){ - this.trigger('move', 'jump', position, scope); + this.trigger('move', 'jump', scope, position); return this; }, @@ -485,6 +618,16 @@ define([ .trigger('resume'); } return this; + }, + + /** + * Notify a test timeout + * @fires runner#timeout + * @returns {runner} chains + */ + timeout : function timeout(){ + this.trigger('timeout'); + return this; } }); diff --git a/views/js/test/runner/integration/test.js b/views/js/test/runner/integration/test.js index ba158335..35b7959e 100644 --- a/views/js/test/runner/integration/test.js +++ b/views/js/test/runner/integration/test.js @@ -52,6 +52,9 @@ define([ url : '/taoTests/views/js/test/runner/sample/minimalisticTest.json', renderTo : $container }) + .on('error', function(err){ + throw err; + }) .on('ready', function(){ assert.equal($('.test-runner', $container).length, 1, 'The test runner container is attached'); @@ -97,6 +100,9 @@ define([ url : '/taoTests/views/js/test/runner/sample/minimalisticTest.json', renderTo : $container }) + .on('error', function(err){ + throw err; + }) .on('ready', function(){ assert.ok(true, 'the test is ready'); diff --git a/views/js/test/runner/pluginLoader/mockPlugin.js b/views/js/test/runner/pluginLoader/mockPlugin.js new file mode 100644 index 00000000..f8691beb --- /dev/null +++ b/views/js/test/runner/pluginLoader/mockPlugin.js @@ -0,0 +1,12 @@ +define([], function(){ + 'use strict'; + + return function(){ + return { + name : 'mock', + init : function(){ + + } + }; + }; +}); diff --git a/views/js/test/runner/pluginLoader/test.html b/views/js/test/runner/pluginLoader/test.html new file mode 100644 index 00000000..b1e01480 --- /dev/null +++ b/views/js/test/runner/pluginLoader/test.html @@ -0,0 +1,34 @@ + + + + + Test Runner - Plugins Loader + + + + + + + + + + +
+
+ + diff --git a/views/js/test/runner/pluginLoader/test.js b/views/js/test/runner/pluginLoader/test.js new file mode 100644 index 00000000..8fe4682e --- /dev/null +++ b/views/js/test/runner/pluginLoader/test.js @@ -0,0 +1,146 @@ +/** + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; under version 2 + * of the License (non-upgradable). + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * Copyright (c) 2015 (original work) Open Assessment Technologies SA ; + */ + +/** + * Test the test plugin + * @author Sam + * @author Bertrand Chevrier + */ +define([ + 'lodash', + 'taoTests/runner/pluginLoader', + 'core/promise', +], function (_, pluginLoader, Promise){ + 'use strict'; + + + QUnit.module('API'); + + QUnit.test('module', function (assert){ + QUnit.expect(3); + + assert.equal(typeof pluginLoader, 'function', "The plugin loader exposes a function"); + assert.equal(typeof pluginLoader(), 'object', "The plugin loader produces an object"); + assert.notStrictEqual(pluginLoader(), pluginLoader(), "The plugin loader provides a different object on each call"); + }); + + QUnit.test('loader methods', function (assert){ + QUnit.expect(6); + var loader = pluginLoader(); + + assert.equal(typeof loader, 'object', "The loader is an object"); + assert.equal(typeof loader.add, 'function', "The loader exposes the add method"); + assert.equal(typeof loader.append, 'function', "The loader exposes the append method"); + assert.equal(typeof loader.prepend, 'function', "The loader exposes the prepend method"); + assert.equal(typeof loader.load, 'function', "The loader exposes the load method"); + assert.equal(typeof loader.getPlugins, 'function', "The loader exposes the getPlugins method"); + + }); + + + QUnit.module('required'); + + QUnit.test('required plugin format', function (assert){ + QUnit.expect(4); + + assert.throws(function(){ + pluginLoader({ 12 : _.noop }); + }, TypeError, 'Wrong category format'); + + assert.throws(function(){ + pluginLoader({ 'foo' : true }); + }, TypeError, 'The plugin list must be an array'); + + assert.throws(function(){ + pluginLoader({ 'foo' : [true] }); + }, TypeError, 'The plugin list must be an array of function'); + + assert.throws(function(){ + pluginLoader({ 'foo' : ['true', _.noop] }); + }, TypeError, 'The plugin list must be an array with only functions'); + + var loader = pluginLoader({ + foo : [_.noop], + bar : [_.noop, _.noop] + }); + }); + + QUnit.test('required plugin loading', function (assert){ + QUnit.expect(5); + + var a = function a (){ return 'a'; }; + var b = function b (){ return 'b'; }; + var c = function c (){ return 'c'; }; + var plugins = { + foo : [a], + bar : [b, c] + }; + + var loader = pluginLoader(plugins); + + assert.equal(typeof loader, 'object', "The loader is an object"); + assert.deepEqual(loader.getCategories(), ['foo', 'bar'], "The plugins categories are correct"); + assert.deepEqual(loader.getPlugins(), [a, b, c], "The plugins have been registered"); + assert.deepEqual(loader.getPlugins('foo'), plugins.foo, "The plugins are registered under the right category"); + assert.deepEqual(loader.getPlugins('bar'), plugins.bar, "The plugins are registered under the right category"); + }); + + + QUnit.module('dynamic'); + + QUnit.test('add plugin module format', function (assert){ + QUnit.expect(3); + + var loader = pluginLoader(); + + assert.equal(typeof loader, 'object', "The loader is an object"); + + assert.throws(function(){ + loader.add(12); + }, TypeError, 'A module is a string'); + + assert.throws(function(){ + loader.add('12', true); + }, TypeError, 'A category is a string'); + + loader.add('foo', 'foo'); + }); + + QUnit.asyncTest('load a plugin', function (assert){ + QUnit.expect(5); + + var loader = pluginLoader(); + + assert.equal(typeof loader, 'object', "The loader is an object"); + + assert.deepEqual(loader.append('taoTests/test/runner/pluginLoader/mockPlugin', 'mock'), loader, 'The loader chains'); + + var p = loader.load(); + + assert.ok(p instanceof Promise, "The load method returns a Promise"); + assert.deepEqual(loader.getPlugins('mock'), [], 'The loader mock category is empty'); + + p.then(function(){ + assert.equal(loader.getPlugins('mock').length, 1, 'The mock category contains now a plugin'); + QUnit.start(); + }).catch(function(e){ + assert.ok(false, e); + QUnit.start(); + }); + }); +}); diff --git a/views/js/test/runner/probeOverseer/test.html b/views/js/test/runner/probeOverseer/test.html new file mode 100644 index 00000000..bd8f15b4 --- /dev/null +++ b/views/js/test/runner/probeOverseer/test.html @@ -0,0 +1,34 @@ + + + + + Test Runner - ProbeOverseer + + + + + + + + + + +
+
+ + diff --git a/views/js/test/runner/probeOverseer/test.js b/views/js/test/runner/probeOverseer/test.js new file mode 100644 index 00000000..49ecf634 --- /dev/null +++ b/views/js/test/runner/probeOverseer/test.js @@ -0,0 +1,442 @@ +/** + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; under version 2 + * of the License (non-upgradable). + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * Copyright (c) 2015 (original work) Open Assessment Technologies SA ; + */ +/** + * @author Jean-Sébastien Conan + * @author Sam + */ +define([ + 'jquery', + 'lodash', + 'taoTests/runner/runner', + 'taoTests/runner/probeOverseer', +], function($, _, runnerFactory, probeOverseer) { + 'use strict'; + + var testId = 'test-123'; + var mockRunner = { + init: _.noop, + on: _.noop + }; + var mockProvider = { + init: _.noop, + loadAreaBroker: _.noop + }; + + + QUnit.module('API'); + + QUnit.test('module factory', function(assert) { + QUnit.expect(6); + assert.equal(typeof probeOverseer, 'function', "The module exposes a function"); + + assert.throws(function() { + probeOverseer(); + }, TypeError, "The factory needs a test id"); + + assert.throws(function() { + probeOverseer(testId); + }, TypeError, "The factory needs a runner"); + + assert.throws(function() { + probeOverseer(testId, {}); + }, TypeError, "The factory needs a valid runner"); + + assert.equal(typeof probeOverseer(testId, mockRunner), 'object', "The module is a factory"); + assert.notEqual(probeOverseer(testId, mockRunner), probeOverseer(testId, mockRunner), "The factory creates different instances"); + }); + + + QUnit.test('own api', function(assert) { + QUnit.expect(7); + var probes = probeOverseer(testId, mockRunner); + assert.equal(typeof probes.add, 'function', "The module as the add method"); + assert.equal(typeof probes.getProbes, 'function', "The module as the getProbes method"); + assert.equal(typeof probes.getQueue, 'function', "The module as the getQueue method"); + assert.equal(typeof probes.push, 'function', "The module as the push method"); + assert.equal(typeof probes.flush, 'function', "The module as the flush method"); + assert.equal(typeof probes.start, 'function', "The module as the start method"); + assert.equal(typeof probes.stop, 'function', "The module as the stop method"); + }); + + QUnit.module('probes'); + + QUnit.test('normal validation', function(assert) { + QUnit.expect(5); + + var probes = probeOverseer(testId, mockRunner); + + assert.throws(function() { + probes.add(); + }, TypeError, "A probe is an object"); + + assert.throws(function() { + probes.add({}); + }, TypeError, "A probe is an object with a predefined format"); + + assert.throws(function() { + probes.add({ + name: true + }); + }, TypeError, "A probe is an object with a valid name"); + + assert.throws(function() { + probes.add({ + name: 'foo' + }); + }, TypeError, "A probe is an object with events"); + + probes.add({ + name: 'foo', + events: ['bar'] + }); + + assert.throws(function() { + probes.add({ + name: 'foo', + events: ['bar'] + }); + }, TypeError, "A probe cannot be added twice"); + + }); + + QUnit.test('latency validation', function(assert) { + QUnit.expect(4); + + var probes = probeOverseer(testId, mockRunner); + + assert.throws(function() { + probes.add({}); + }, TypeError, "A probe is an object with a predefined format"); + + assert.throws(function() { + probes.add({ + name: 'foo', + latency: true + }); + }, TypeError, "A latency probe must have events"); + + assert.throws(function() { + probes.add({ + name: 'foo', + latency: true, + events: ['bar'] + }); + }, TypeError, "A latency probe must have start and stop events"); + + assert.throws(function() { + probes.add({ + name: 'foo', + latency: true, + startEvents: [], + stopEvents: [] + }); + }, TypeError, "A latency probe must have start and stop events defined"); + + probes.add({ + name: 'foo', + latency: true, + startEvents: ['init'], + stopEvents: ['finish'] + }); + }); + + QUnit.test('add and get', function(assert) { + QUnit.expect(3); + + var probes = probeOverseer(testId, mockRunner); + var p1 = { + name: 'foo', + latency: true, + startEvents: ['init'], + stopEvents: ['finish'] + }; + var p2 = { + name: 'bar', + events: ['ready'] + }; + + assert.deepEqual(probes.add(p1), probes, 'The add method chains'); + assert.deepEqual(probes.add(p2), probes, 'The add method chains'); + assert.deepEqual(probes.getProbes(), [p1, p2], 'The probes are added correclty'); + }); + + QUnit.test('reformat events', function(assert) { + QUnit.expect(3); + + var probes = probeOverseer(testId, mockRunner); + var p1 = { + name: 'foo', + latency: true, + startEvents: 'init', + stopEvents: 'finish' + }; + var p2 = { + name: 'bar', + events: 'ready' + }; + + probes.add(p1) + .add(p2); + + assert.deepEqual(probes.getProbes()[0].startEvents, ['init'], 'Events are wrapped in an array'); + assert.deepEqual(probes.getProbes()[0].stopEvents, ['finish'], 'Events are wrapped in an array'); + assert.deepEqual(probes.getProbes()[1].events, ['ready'], 'Events are wrapped in an array'); + }); + + QUnit.module('collection', { + setup: function() { + runnerFactory.clearProviders(); + } + }); + + QUnit.asyncTest('simple', function(assert) { + QUnit.expect(12); + + runnerFactory.registerProvider('foo', { + loadAreaBroker: _.noop, + init: _.noop + }); + + var runner = runnerFactory('foo'); + + var probes = probeOverseer(testId, runner); + + probes.add({ + name: 'test-ready', + events: 'ready', + capture: function(testRunner) { + assert.equal(typeof testRunner, 'object', 'The runner is given in parameter'); + assert.deepEqual(testRunner, runner, 'The runner instance is given in parameter'); + return { + 'foo': 'bar' + }; + } + }); + probes.start(); + + var creation = Date.now() / 1000; + var init; + runner + .on('init', function() { + init = Date.now() / 1000; + }) + .after('ready', function() { + setTimeout(function() { + probes.getQueue().then(function(queue) { + + assert.equal(queue.length, 1, 'The queue contains an entry'); + assert.equal(typeof queue[0], 'object', 'The queue entry is an object'); + assert.equal(typeof queue[0].id, 'string', 'The queue entry contains an id'); + assert.equal(typeof queue[0].timestamp, 'number', 'The queue entry contains a timestamp'); + assert.ok(queue[0].timestamp >= creation && creation > 0, 'The timestamp is superior to the test creation'); + assert.ok(queue[0].timestamp >= init && init > 0, 'The timestamp is superior or equal to the test init'); + assert.equal(typeof queue[0].timezone, 'string', 'The queue entry contains a timezone'); + assert.ok(/^[\+\-]{1}[0-9]{2}:[0-9]{2}$/.test(queue[0].timezone), 'The timezone is formatted correclty'); + assert.equal(queue[0].type, 'test-ready', 'The entry type is correct'); + assert.deepEqual(queue[0].context, { + foo: 'bar' + }, 'The entry context is correct'); + + probes.stop(); + + QUnit.start(); + + }).catch(function(err) { + assert.ok(false, err); + }); + }, 50); //time to write in the db + }) + .init(); + }); + + QUnit.asyncTest('latency', function(assert) { + QUnit.expect(16); + + runnerFactory.registerProvider('foo', { + loadAreaBroker: _.noop, + init: _.noop + }); + + var runner = runnerFactory('foo'); + + var probes = probeOverseer(testId, runner); + + probes.add({ + name: 'test-latency', + latency: true, + startEvents: ['ready'], + stopEvents: ['finish'], + capture: function(testRunner) { + assert.equal(typeof testRunner, 'object', 'The runner is given in parameter'); + assert.deepEqual(testRunner, runner, 'The runner instance is given in parameter'); + return { + 'foo': 'bar' + }; + } + }); + probes.start(); + + + var creation = Date.now() / 1000; + var init; + runner + .on('init', function() { + init = Date.now() / 1000; + }) + .after('ready', function() { + setTimeout(function() { + runner.finish(); + }, 50); + }) + .after('finish', function() { + + setTimeout(function() { + probes.getQueue().then(function(queue) { + + assert.equal(queue.length, 2, 'The queue contains the two entries'); + var startEntry = queue[0]; + var stopEntry = queue[1]; + + assert.equal(typeof startEntry, 'object', 'The start entry is an object'); + assert.equal(typeof startEntry.id, 'string', 'The start entry contains an id'); + assert.equal(typeof startEntry.timestamp, 'number', 'The start entry contains a timestamp'); + assert.equal(startEntry.type, 'test-latency', 'The entry type is correct'); + assert.deepEqual(startEntry.context, { + foo: 'bar' + }, 'The entry context is correct'); + assert.ok(startEntry.timestamp >= creation && creation > 0, 'The timestamp is superior to the test creation'); + assert.ok(startEntry.timestamp >= init && init > 0, 'The timestamp is superior or equal to the test init'); + + assert.equal(typeof queue[0].timezone, 'string', 'The queue entry contains a timezone'); + assert.ok(/^[\+\-]{1}[0-9]{2}:[0-9]{2}$/.test(queue[0].timezone), 'The timezone is formatted correclty'); + + assert.equal(typeof stopEntry, 'object', 'The stop entry is an object'); + assert.equal(stopEntry.id, startEntry.id, 'string', 'The stop entry id is the same than the start entry'); + + probes.stop(); + + QUnit.start(); + + }).catch(function(err) { + assert.ok(false, err); + }); + }, 50); //time to write in the db + }) + .init(); + }); + + + QUnit.asyncTest('flush', function(assert) { + QUnit.expect(3); + + runnerFactory.registerProvider('foo', { + loadAreaBroker: _.noop, + init: _.noop + }); + + var runner = runnerFactory('foo'); + + var probes = probeOverseer(testId, runner); + + probes.add({ + name: 'foo', + events: 'foo' + }); + probes.start(); + + runner + .on('ready', function() { + runner.trigger('foo') + .trigger('foo') + .trigger('foo'); + }) + .after('ready', function() { + + setTimeout(function() { + + probes.getQueue() + .then(function(queue) { + assert.equal(queue.length, 3, 'The queue contains 3 entries'); + }) + .then(function() { + return probes.flush().then(function(flushed) { + assert.equal(flushed.length, 3, 'The queue contains 3 entries'); + }); + }) + .then(function() { + return probes.getQueue().then(function(queue) { + assert.equal(queue.length, 0, 'The queue is empty now'); + }); + }) + .then(function() { + probes.stop(); + QUnit.start(); + }); + }, 150); + }) + .init(); + }); + + QUnit.asyncTest('stop', function(assert) { + QUnit.expect(2); + + runnerFactory.registerProvider('foo', { + loadAreaBroker: _.noop, + init: _.noop + }); + + var runner = runnerFactory('foo'); + + var probes = probeOverseer(testId, runner); + + probes.add({ + name: 'foo', + events: 'foo' + }); + probes.start(); + + runner + .on('ready', function() { + runner.trigger('foo') + .trigger('foo') + .trigger('foo'); + }) + .after('ready', function() { + setTimeout(function() { + + probes.getQueue() + .then(function(queue) { + assert.equal(queue.length, 3, 'The queue contains 3 entries'); + }) + .then(function() { + probes.stop(); + runner.trigger('foo'); + }) + .then(function() { + setTimeout(function() { + probes.getQueue().then(function(queue) { + assert.equal(queue, null, 'The queue is not there'); + QUnit.start(); + }); + }, 150); + }); + }, 150); + }) + .init(); + }); +}); diff --git a/views/js/test/runner/runner/test.js b/views/js/test/runner/runner/test.js index c1f937de..f7516e93 100644 --- a/views/js/test/runner/runner/test.js +++ b/views/js/test/runner/runner/test.js @@ -60,18 +60,23 @@ define([ {name : 'loadItem', title : 'loadItem'}, {name : 'renderItem', title : 'renderItem'}, {name : 'unloadItem', title : 'unloadItem'}, + {name : 'disableItem', title : 'disableItem'}, + {name : 'enableItem', title : 'enableItem'}, {name : 'getPlugins', title : 'getPlugins'}, {name : 'getPlugin', title : 'getPlugin'}, {name : 'getConfig', title : 'getConfig'}, {name : 'getState', title : 'getState'}, {name : 'setState', title : 'setState'}, + {name : 'getItemState', title : 'getItemState'}, + {name : 'setItemState', title : 'setItemState'}, {name : 'getTestData', title : 'getTestData'}, {name : 'setTestData', title : 'setTestData'}, {name : 'getTestContext', title : 'getTestContext'}, {name : 'setTestContext', title : 'setTestContext'}, {name : 'getAreaBroker', title : 'getAreaBroker'}, {name : 'getProxy', title : 'getProxy'}, + {name : 'getProbeOverseer', title : 'getProbeOverseer'}, {name : 'next', title : 'next'}, {name : 'previous', title : 'previous'}, @@ -80,6 +85,7 @@ define([ {name : 'exit', title : 'exit'}, {name : 'pause', title : 'pause'}, {name : 'resume', title : 'resume'}, + {name : 'timeout', title : 'timeout'}, {name : 'trigger', title : 'trigger'}, {name : 'before', title : 'before'}, @@ -226,7 +232,7 @@ define([ }); QUnit.asyncTest('load and render item', function(assert){ - QUnit.expect(1); + QUnit.expect(2); var items = { 'aaa' : 'AAA', @@ -241,7 +247,8 @@ define([ loadItem : function(itemRef){ return items[itemRef]; }, - renderItem : function(itemData){ + renderItem : function(itemRef, itemData){ + assert.equal(itemRef, 'zzz', 'The rendered item is correct'); assert.equal(itemData, 'ZZZ', 'The rendered item is correct'); QUnit.start(); } @@ -256,7 +263,7 @@ define([ }); QUnit.asyncTest('load async and render item', function(assert){ - QUnit.expect(3); + QUnit.expect(4); var resolved = false; var items = { @@ -279,9 +286,10 @@ define([ assert.equal(resolved, false, 'Item loading is not yet resolved'); return p; }, - renderItem : function(itemData){ + renderItem : function(itemRef, itemData){ - assert.equal(resolved, true, 'Item loading is resolved'); + assert.equal(resolved, true, 'Item loading is resolved'); + assert.equal(itemRef, 'zzz', 'The rendered item is correct'); assert.equal(itemData, 'ZZZ', 'The rendered item is correct'); QUnit.start(); } @@ -335,6 +343,118 @@ define([ .init(); }); + QUnit.asyncTest('item state', function(assert){ + QUnit.expect(15); + + var items = { + 'aaa' : 'AAA', + 'zzz' : 'ZZZ' + }; + + runnerFactory.registerProvider('foo', { + loadAreaBroker : function(){ + return {}; + }, + init : _.noop, + loadItem : function(itemRef){ + return items[itemRef]; + } + }); + + var runner = runnerFactory('foo'); + runner + .on('init', function(){ + + assert.throws(function(){ + this.getItemState(); + }, TypeError, 'The item state should have an itemRef'); + + assert.throws(function(){ + this.getItemState('zzz'); + }, TypeError, 'The item state should have an itemRef and a name'); + + assert.throws(function(){ + this.setItemState(); + }, TypeError, 'The item state should have an itemRef'); + + assert.throws(function(){ + this.setItemState('zzz'); + }, TypeError, 'The item state should have an itemRef and a name'); + + assert.equal(this.getItemState('zzz', 'loaded'), false, 'The item is not loaded'); + assert.equal(this.getItemState('zzz', 'ready'), false, 'The item is not ready'); + assert.equal(this.getItemState('zzz', 'foo'), false, 'The item is not foo'); + }) + .on('ready', function(){ + this.loadItem('zzz'); + }) + .on('loaditem', function(itemRef){ + assert.equal(itemRef, 'zzz', 'The loaded item is correct'); + assert.equal(this.getItemState('zzz', 'loaded'), true, 'The item is loaded'); + assert.equal(this.getItemState('zzz', 'ready'), false, 'The item is not ready'); + + this.setItemState('zzz', 'foo', true); + assert.equal(this.getItemState('zzz', 'foo'), true, 'The item is foo'); + }) + .on('renderitem', function(itemRef, itemData){ + + assert.equal(itemRef, 'zzz', 'The rendered item is correct'); + assert.equal(this.getItemState('zzz', 'loaded'), true, 'The item is loaded'); + assert.equal(this.getItemState('zzz', 'ready'), true, 'The item is ready'); + assert.equal(this.getItemState('zzz', 'foo'), true, 'The item is foo'); + + QUnit.start(); + }) + .init(); + }); + + QUnit.asyncTest('disable items', function(assert){ + QUnit.expect(6); + + var items = { + 'aaa' : 'AAA', + 'zzz' : 'ZZZ' + }; + + runnerFactory.registerProvider('foo', { + loadAreaBroker : function(){ + return {}; + }, + init : _.noop, + loadItem : function(itemRef){ + return items[itemRef]; + }, + renderItem : function(itemRef){ + var self = this; + this.disableItem(itemRef); + setTimeout(function(){ + self.enableItem(itemRef); + }, 50); + } + }); + + var runner = runnerFactory('foo'); + runner + .on('ready', function(){ + this.loadItem('zzz'); + }) + .on('loaditem', function(itemRef){ + assert.equal(itemRef, 'zzz', 'The provider is called with the correct reference'); + assert.equal(this.getItemState('zzz', 'disabled'), false, 'The item is not disabled'); + }) + .on('disableitem', function(itemRef){ + assert.equal(itemRef, 'zzz', 'The provider is called with the correct reference'); + assert.equal(this.getItemState('zzz', 'disabled'), true, 'The item is now disabled'); + }) + .on('enableitem', function(itemRef){ + assert.equal(itemRef, 'zzz', 'The provider is called with the correct reference'); + assert.equal(this.getItemState('zzz', 'disabled'), false, 'The item is not disabled anymore'); + + QUnit.start(); + }) + .init(); + }); + QUnit.asyncTest('init error', function(assert){ QUnit.expect(2); @@ -457,6 +577,30 @@ define([ .init(); }); + QUnit.asyncTest('timeout', function(assert){ + QUnit.expect(2); + + runnerFactory.registerProvider('foo', { + loadAreaBroker : function(){ + return {}; + }, + init : function init(){ + + this.on('init', function(){ + assert.ok(true, 'we can listen for init in providers init'); + }) + .on('timeout', function(){ + assert.ok(true, 'The timeout event has been triggered'); + QUnit.start(); + }); + } + }); + + runnerFactory('foo') + .init() + .timeout(); + }); + QUnit.module('plugins', { setup: function(){ runnerFactory.clearProviders(); diff --git a/views/js/test/runner/sample/minimalisticProvider.js b/views/js/test/runner/sample/minimalisticProvider.js index 1ab92f6e..8232ef91 100644 --- a/views/js/test/runner/sample/minimalisticProvider.js +++ b/views/js/test/runner/sample/minimalisticProvider.js @@ -39,7 +39,7 @@ define([ //install event based behavior this.on('ready', function(){ - this.loadItem(0); + this.loadItem('item-0'); }) .on('move', function(type){ @@ -48,8 +48,8 @@ define([ if(type === 'next'){ if(test.items[test.current + 1]){ - self.unloadItem(test.current); - self.loadItem(test.current + 1); + self.unloadItem('item-' +test.current); + self.loadItem('item-' + (test.current + 1)); } else { self.finish(); } @@ -57,10 +57,10 @@ define([ else if(type === 'previous'){ if(test.items[test.current - 1]){ - self.unloadItem(test.current); - self.loadItem(test.current - 1); + self.unloadItem('item-' +test.current); + self.loadItem('item-' + (test.current - 1)); } else { - self.loadItem(0); + self.loadItem('item-0'); } } }); @@ -106,16 +106,17 @@ define([ return new Promise(function(resolve, reject){ setTimeout(function(){ - test.current = itemIndex; + + test.current = parseInt(itemIndex.replace('item-',''), 10); self.setTestContext(test); - resolve(test.items[itemIndex]); + resolve(test.items[test.current]); }, 500); }); }, - renderItem : function renderItem(item){ + renderItem : function renderItem(itemIndex, item){ var broker = this.getAreaBroker(); var $content = broker.getContentArea();