diff --git a/docs/implementation.md b/docs/implementation.md index fa7bdeac7..5f2bc9875 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -36,6 +36,8 @@ There are two primary entry points for middleware: ## Persistence +### Brain + Hubot has a memory exposed as the `robot.brain` object that can be used to store and retrieve data. Furthermore, Hubot scripts exist to enable persistence across Hubot restarts. `hubot-redis-brain` is such a script and uses a backend Redis server. @@ -44,3 +46,12 @@ By default, the brain contains a list of all users seen by Hubot. Therefore, without persistence across restarts, the brain will contain the list of users encountered so far, during the current run of Hubot. On the other hand, with persistence across restarts, the brain will contain all users encountered by Hubot during all of its runs. This list of users can be accessed through `hubot.brain.users()` and other utility methods. + +### Datastore + +Hubot's optional datastore, exposed as the `robot.datastore` object, provides a more robust persistence model. Compared to the brain, the datastore: + +1. Is always (instead of optionally) backed by a database +2. Fetches data from the database and stores data in the database on every request, instead of periodically persisting the entire in-memory brain. + +The datastore is useful in cases where there's a need for greater reassurances of data integrity or in cases where multiple Hubot instances need to access the same database. diff --git a/docs/scripting.md b/docs/scripting.md index e5e13afb6..964ce77db 100644 --- a/docs/scripting.md +++ b/docs/scripting.md @@ -565,8 +565,10 @@ The other sections are more relevant to developers of the bot, particularly depe ## Persistence -Hubot has an in-memory key-value store exposed as `robot.brain` that can be -used to store and retrieve data by scripts. +Hubot has two persistence methods available that can be +used to store and retrieve data by scripts: an in-memory key-value store exposed as `robot.brain`, and an optional persistent database-backed key-value store expsoed as `robot.datastore` + +### Brain ```coffeescript robot.respond /have a soda/i, (res) -> @@ -600,6 +602,42 @@ module.exports = (robot) -> res.send "#{name} is user - #{user}" ``` +### Datastore + +Unlike the brain, the datastore's getter and setter methods are asynchronous and don't resolve until the call to the underlying database has resolved. This requires a slightly different approach to accessing data: + +```coffeescript +robot.respond /have a soda/i, (res) -> + # Get number of sodas had (coerced to a number). + robot.datastore.get('totalSodas').then (value) -> + sodasHad = value * 1 or 0 + + if sodasHad > 4 + res.reply "I'm too fizzy.." + else + res.reply 'Sure!' + robot.brain.set 'totalSodas', sodasHad + 1 + +robot.respond /sleep it off/i, (res) -> + robot.datastore.set('totalSodas', 0).then () -> + res.reply 'zzzzz' +``` + +The datastore also allows setting and getting values which are scoped to individual users: + +```coffeescript +module.exports = (robot) -> + + robot.respond /who is @?([\w .\-]+)\?*$/i, (res) -> + name = res.match[1].trim() + + users = robot.brain.usersForFuzzyName(name) + if users.length is 1 + user = users[0] + user.get('roles').then (roles) -> + res.send "#{name} is #{roles.join(', ')}" +``` + ## Script Loading There are three main sources to load scripts from: diff --git a/es2015.js b/es2015.js index af706962c..7f523b366 100644 --- a/es2015.js +++ b/es2015.js @@ -7,6 +7,7 @@ const Adapter = require('./src/adapter') const Response = require('./src/response') const Listener = require('./src/listener') const Message = require('./src/message') +const DataStore = require('./src/datastore') module.exports = { User, @@ -22,6 +23,8 @@ module.exports = { LeaveMessage: Message.LeaveMessage, TopicMessage: Message.TopicMessage, CatchAllMessage: Message.CatchAllMessage, + DataStore: DataStore.DataStore, + DataStoreUnavailable: DataStore.DataStoreUnavailable, loadBot (adapterPath, adapterName, enableHttpd, botName, botAlias) { return new module.exports.Robot(adapterPath, adapterName, enableHttpd, botName, botAlias) diff --git a/src/brain.js b/src/brain.js index 7e7b14363..b0694a975 100644 --- a/src/brain.js +++ b/src/brain.js @@ -10,7 +10,7 @@ const User = require('./user') // 2. If the original object was a User object, the original object // 3. If the original object was a plain JavaScript object, return // a User object with all of the original object's properties. -let reconstructUserIfNecessary = function (user) { +let reconstructUserIfNecessary = function (user, robot) { if (!user) { return null } @@ -20,6 +20,9 @@ let reconstructUserIfNecessary = function (user) { delete user.id // Use the old user as the "options" object, // populating the new user with its values. + // Also add the `robot` field so it gets a reference. + user.robot = robot + return new User(id, user) } else { return user @@ -36,6 +39,7 @@ class Brain extends EventEmitter { users: {}, _private: {} } + this.robot = robot this.autoSave = true @@ -142,7 +146,7 @@ class Brain extends EventEmitter { if (data && data.users) { for (let k in data.users) { let user = this.data.users[k] - this.data.users[k] = reconstructUserIfNecessary(user) + this.data.users[k] = reconstructUserIfNecessary(user, this.robot) } } @@ -161,6 +165,10 @@ class Brain extends EventEmitter { // Returns a User instance of the specified user. userForId (id, options) { let user = this.data.users[id] + if (!options) { + options = {} + } + options.robot = this.robot if (!user) { user = new User(id, options) diff --git a/src/datastore.js b/src/datastore.js new file mode 100644 index 000000000..7f211c213 --- /dev/null +++ b/src/datastore.js @@ -0,0 +1,94 @@ +'use strict' + +class DataStore { + // Represents a persistent, database-backed storage for the robot. Extend this. + // + // Returns a new Datastore with no storage. + constructor (robot) { + this.robot = robot + } + + // Public: Set value for key in the database. Overwrites existing + // values if present. Returns a promise which resolves when the + // write has completed. + // + // Value can be any JSON-serializable type. + set (key, value) { + return this._set(key, value, 'global') + } + + // Public: Assuming `key` represents an object in the database, + // sets its `objectKey` to `value`. If `key` isn't already + // present, it's instantiated as an empty object. + setObject (key, objectKey, value) { + return this.get(key).then((object) => { + let target = object || {} + target[objectKey] = value + return this.set(key, target) + }) + } + + // Public: Adds the supplied value(s) to the end of the existing + // array in the database marked by `key`. If `key` isn't already + // present, it's instantiated as an empty array. + setArray (key, value) { + return this.get(key).then((object) => { + let target = object || [] + // Extend the array if the value is also an array, otherwise + // push the single value on the end. + if (Array.isArray(value)) { + return this.set(key, target.push.apply(target, value)) + } else { + return this.set(key, target.concat(value)) + } + }) + } + + // Public: Get value by key if in the database or return `undefined` + // if not found. Returns a promise which resolves to the + // requested value. + get (key) { + return this._get(key, 'global') + } + + // Public: Digs inside the object at `key` for a key named + // `objectKey`. If `key` isn't already present, or if it doesn't + // contain an `objectKey`, returns `undefined`. + getObject (key, objectKey) { + return this.get(key).then((object) => { + let target = object || {} + return target[objectKey] + }) + } + + // Private: Implements the underlying `set` logic for the datastore. + // This will be called by the public methods. This is one of two + // methods that must be implemented by subclasses of this class. + // `table` represents a unique namespace for this key, such as a + // table in a SQL database. + // + // This returns a resolved promise when the `set` operation is + // successful, and a rejected promise if the operation fails. + _set (key, value, table) { + return Promise.reject(new DataStoreUnavailable('Setter called on the abstract class.')) + } + + // Private: Implements the underlying `get` logic for the datastore. + // This will be called by the public methods. This is one of two + // methods that must be implemented by subclasses of this class. + // `table` represents a unique namespace for this key, such as a + // table in a SQL database. + // + // This returns a resolved promise containing the fetched value on + // success, and a rejected promise if the operation fails. + _get (key, table) { + return Promise.reject(new DataStoreUnavailable('Getter called on the abstract class.')) + } +} + +class DataStoreUnavailable extends Error {} + +module.exports = { + DataStore, + DataStoreUnavailable +} diff --git a/src/datastores/memory.js b/src/datastores/memory.js new file mode 100644 index 000000000..506989911 --- /dev/null +++ b/src/datastores/memory.js @@ -0,0 +1,23 @@ +'use strict' + +const DataStore = require('../datastore').DataStore + +class InMemoryDataStore extends DataStore { + constructor (robot) { + super(robot) + this.data = { + global: {}, + users: {} + } + } + + _get (key, table) { + return Promise.resolve(this.data[table][key]) + } + + _set (key, value, table) { + return Promise.resolve(this.data[table][key] = value) + } +} + +module.exports = InMemoryDataStore diff --git a/src/robot.js b/src/robot.js index ae5029020..74046d4ca 100644 --- a/src/robot.js +++ b/src/robot.js @@ -40,6 +40,7 @@ class Robot { this.brain = new Brain(this) this.alias = alias this.adapter = null + this.datastore = null this.Response = Response this.commands = [] this.listeners = [] diff --git a/src/user.js b/src/user.js index d071f2dd7..877fd8ed4 100644 --- a/src/user.js +++ b/src/user.js @@ -1,5 +1,7 @@ 'use strict' +const DataStoreUnavailable = require('./datastore').DataStoreUnavailable + class User { // Represents a participating user in the chat. // @@ -12,6 +14,17 @@ class User { options = {} } + // Define a getter method so we don't actually store the + // robot itself on the user object, preventing it from + // being serialized into the brain. + if (options.robot) { + let robot = options.robot + delete options.robot + this._getRobot = function () { return robot } + } else { + this._getRobot = function () { } + } + Object.keys(options).forEach((key) => { this[key] = options[key] }) @@ -20,6 +33,33 @@ class User { this.name = this.id.toString() } } + + set (key, value) { + this._checkDatastoreAvailable() + return this._getDatastore()._set(this._constructKey(key), value, 'users') + } + + get (key) { + this._checkDatastoreAvailable() + return this._getDatastore()._get(this._constructKey(key), 'users') + } + + _constructKey (key) { + return `${this.id}+${key}` + } + + _checkDatastoreAvailable () { + if (!this._getDatastore()) { + throw new DataStoreUnavailable('datastore is not initialized') + } + } + + _getDatastore () { + let robot = this._getRobot() + if (robot) { + return robot.datastore + } + } } module.exports = User diff --git a/test/datastore_test.js b/test/datastore_test.js new file mode 100644 index 000000000..036fb8626 --- /dev/null +++ b/test/datastore_test.js @@ -0,0 +1,150 @@ +'use strict' + +/* global describe, beforeEach, it */ + +const chai = require('chai') +const sinon = require('sinon') +chai.use(require('sinon-chai')) + +const expect = chai.expect + +const Brain = require('../src/brain') +const InMemoryDataStore = require('../src/datastores/memory') + +describe('Datastore', function () { + beforeEach(function () { + this.clock = sinon.useFakeTimers() + this.robot = { + emit () {}, + on () {}, + receive: sinon.spy() + } + + // This *should* be callsArgAsync to match the 'on' API, but that makes + // the tests more complicated and seems irrelevant. + sinon.stub(this.robot, 'on').withArgs('running').callsArg(1) + + this.robot.brain = new Brain(this.robot) + this.robot.datastore = new InMemoryDataStore(this.robot) + this.robot.brain.userForId('1', {name: 'User One'}) + this.robot.brain.userForId('2', {name: 'User Two'}) + }) + + describe('global scope', function () { + it('returns undefined for values not in the datastore', function () { + return this.robot.datastore.get('blah').then(function (value) { + expect(value).to.be.an('undefined') + }) + }) + + it('can store simple values', function () { + return this.robot.datastore.set('key', 'value').then(() => { + return this.robot.datastore.get('key').then((value) => { + expect(value).to.equal('value') + }) + }) + }) + + it('can store arbitrary JavaScript values', function () { + let object = { + 'name': 'test', + 'data': [1, 2, 3] + } + return this.robot.datastore.set('key', object).then(() => { + return this.robot.datastore.get('key').then((value) => { + expect(value.name).to.equal('test') + expect(value.data).to.deep.equal([1, 2, 3]) + }) + }) + }) + + it('can dig inside objects for values', function () { + let object = { + 'a': 'one', + 'b': 'two' + } + return this.robot.datastore.set('key', object).then(() => { + return this.robot.datastore.getObject('key', 'a').then((value) => { + expect(value).to.equal('one') + }) + }) + }) + + it('can set individual keys inside objects', function () { + let object = { + 'a': 'one', + 'b': 'two' + } + return this.robot.datastore.set('object', object).then(() => { + return this.robot.datastore.setObject('object', 'c', 'three').then(() => { + return this.robot.datastore.get('object').then((value) => { + expect(value.a).to.equal('one') + expect(value.b).to.equal('two') + expect(value.c).to.equal('three') + }) + }) + }) + }) + + it('creates an object from scratch when none exists', function () { + return this.robot.datastore.setObject('object', 'key', 'value').then(() => { + return this.robot.datastore.get('object').then((value) => { + let expected = {'key': 'value'} + expect(value).to.deep.equal(expected) + }) + }) + }) + + it('can append to an existing array', function () { + return this.robot.datastore.set('array', [1, 2, 3]).then(() => { + return this.robot.datastore.setArray('array', 4).then(() => { + return this.robot.datastore.get('array').then((value) => { + expect(value).to.deep.equal([1, 2, 3, 4]) + }) + }) + }) + }) + + it('creates an array from scratch when none exists', function () { + return this.robot.datastore.setArray('array', 4).then(() => { + return this.robot.datastore.get('array').then((value) => { + expect(value).to.deep.equal([4]) + }) + }) + }) + }) + + describe('User scope', function () { + it('has access to the robot object', function () { + let user = this.robot.brain.userForId('1') + expect(user._getRobot()).to.equal(this.robot) + }) + + it('can store user data which is separate from global data', function () { + let user = this.robot.brain.userForId('1') + return user.set('blah', 'blah').then(() => { + return user.get('blah').then((userBlah) => { + return this.robot.datastore.get('blah').then((datastoreBlah) => { + expect(userBlah).to.not.equal(datastoreBlah) + expect(userBlah).to.equal('blah') + expect(datastoreBlah).to.be.an('undefined') + }) + }) + }) + }) + + it('stores user data separate per-user', function () { + let userOne = this.robot.brain.userForId('1') + let userTwo = this.robot.brain.userForId('2') + return userOne.set('blah', 'blah').then(() => { + return userOne.get('blah').then((valueOne) => { + return userTwo.get('blah').then((valueTwo) => { + expect(valueOne).to.not.equal(valueTwo) + expect(valueOne).to.equal('blah') + expect(valueTwo).to.be.an('undefined') + }) + }) + }) + }) + }) +})