From 07f76abff108af16e2229cf1769741d9dc8472f6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 9 Jan 2019 17:13:45 +0100 Subject: [PATCH] feat: allow using proprietary data models decoded by application server (#33) --- README.md | 1 + docs/users_manual.md | 2 +- lib/applicationServers/abstractAppService.js | 30 ++- .../loraserverioAppService.js | 26 +- lib/applicationServers/ttnAppService.js | 18 +- lib/dataModels/cayenneLpp.js | 43 +--- lib/dataModels/cbor.js | 38 +-- lib/dataTranslationService.js | 51 +++- lib/iotagent-lora.js | 2 + .../provisionDeviceApplicationServer1TTN.json | 53 +++++ test/unit/applicationServerDecoding.js | 222 ++++++++++++++++++ test/unit/cayenneLppDecoding.js | 5 +- 12 files changed, 394 insertions(+), 97 deletions(-) create mode 100644 test/deviceProvisioning/provisionDeviceApplicationServer1TTN.json create mode 100644 test/unit/applicationServerDecoding.js diff --git a/README.md b/README.md index cc4ab47..d18e6f4 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ devices. - [CayenneLpp](https://www.thethingsnetwork.org/docs/devices/arduino/api/cayennelpp.html) - [CBOR](https://tools.ietf.org/html/rfc7049) +- Proprietary format decoded by LoRaWAN application server. ## Install diff --git a/docs/users_manual.md b/docs/users_manual.md index 3e2aa96..8b0e045 100644 --- a/docs/users_manual.md +++ b/docs/users_manual.md @@ -101,7 +101,7 @@ EOF ``` - provider: Identifies the LoRaWAN stack. **Current possible value is TTN.** -- data_model: Identifies the data model used by the device to report new observations. **Current possible values are cayennelpp and cbor.** +- data_model: Identifies the data model used by the device to report new observations. **Current possible values are cayennelpp,cbor and application_server. The last one can be used in case the payload format decoding is done by the application server.** The IoTa will automatically subscribe to new observation notifications from the device. Whenever a new update is received, it will be translated to NGSI and forwarded to the Orion Context Broker. diff --git a/lib/applicationServers/abstractAppService.js b/lib/applicationServers/abstractAppService.js index a1d196a..646230a 100644 --- a/lib/applicationServers/abstractAppService.js +++ b/lib/applicationServers/abstractAppService.js @@ -35,9 +35,10 @@ class AbstractAppService { * @param {String} applicationId The application identifier * @param {String} applicationKey The application key * @param {Function} messageHandler The message handler + * @param {String} dataModel The data model * @param {Object} iotaConfiguration The IOTA configuration associated to this Application Server. */ - constructor (applicationServer, appEui, applicationId, applicationKey, messageHandler, iotaConfiguration) { + constructor (applicationServer, appEui, applicationId, applicationKey, messageHandler, dataModel, iotaConfiguration) { if (this.constructor === AbstractAppService) { throw new TypeError('Abstract class "AbstractAppService" cannot be instantiated directly.'); } @@ -65,6 +66,7 @@ class AbstractAppService { this.messageHandler = messageHandler; this.devices = {}; this.iotaConfiguration = iotaConfiguration; + this.dataModel = dataModel; } /** @@ -154,6 +156,32 @@ class AbstractAppService { } } + /** + * Get data model for the device + * @param {String} devId Device's ID + * @param {String} devEui Device's EUI + */ + getDataModel (devId, devEui) { + var device = {}; + var dataModel = {}; + if (!devId && !devEui) { + winston.error('Device ID or device EUI must be provided'); + throw new Error('Device ID or device EUI must be provided'); + } else if (devId) { + device = this.getDevice(devId); + } else { + device = this.getDeviceByEui(devEui); + } + + if (device && device.internalAttributes && device.internalAttributes.lorawan) { + dataModel = device.internalAttributes.lorawan.data_model; + } else { + dataModel = this.dataModel; + } + + return dataModel; + } + /** * Gets the device by the EUI * diff --git a/lib/applicationServers/loraserverioAppService.js b/lib/applicationServers/loraserverioAppService.js index 7aa7539..ddf067b 100644 --- a/lib/applicationServers/loraserverioAppService.js +++ b/lib/applicationServers/loraserverioAppService.js @@ -37,14 +37,15 @@ class LoraserverIoService extends appService.AbstractAppService { * @param {String} applicationId The application identifier * @param {String} applicationKey The application key * @param {Function} messageHandler The message handler + * @param {String} dataModel The data model * @param {Object} iotaConfiguration The IOTA configuration associated to this Application Server. */ - constructor (applicationServer, appEui, applicationId, applicationKey, messageHandler, iotaConfiguration) { + constructor (applicationServer, appEui, applicationId, applicationKey, messageHandler, dataModel, iotaConfiguration) { if (!applicationId) { throw new Error('applicationId is mandatory for LoRaServer'); } - super(applicationServer, appEui, applicationId, applicationKey, messageHandler, iotaConfiguration); + super(applicationServer, appEui, applicationId, applicationKey, messageHandler, dataModel, iotaConfiguration); } /** @@ -98,16 +99,21 @@ class LoraserverIoService extends appService.AbstractAppService { return; } + var dataModel = this.getDataModel(null, message['devEUI']); + + var deviceId; if (device) { - if (message && message['data']) { - this.messageHandler(this, device.id, message['devEUI'], message['data']); - } else { - this.messageHandler(this, device.id, message['devEUI'], null); + deviceId = device.id; + } else if (message && message['deviceName']) { + deviceId = message['deviceName']; + } + + if (message) { + if (dataModel === 'application_server' && message.object) { + this.messageHandler(this, deviceId, message['devEUI'], message['object']); + } else if (dataModel !== 'application_server' && message.data) { + this.messageHandler(this, deviceId, message['devEUI'], message['data']); } - } else if (message && message['data']) { - this.messageHandler(this, message['deviceName'], message['devEUI'], message['data']); - } else { - this.messageHandler(this, message['deviceName'], message['devEUI'], null); } } } diff --git a/lib/applicationServers/ttnAppService.js b/lib/applicationServers/ttnAppService.js index 985b3b0..a908e3c 100644 --- a/lib/applicationServers/ttnAppService.js +++ b/lib/applicationServers/ttnAppService.js @@ -37,14 +37,15 @@ class TtnAppService extends appService.AbstractAppService { * @param {String} applicationId The application identifier * @param {String} applicationKey The application key * @param {Function} messageHandler The message handler + * @param {String} dataModel The data model * @param {Object} iotaConfiguration The IOTA configuration associated to this Application Server. */ - constructor (applicationServer, appEui, applicationId, applicationKey, messageHandler, iotaConfiguration) { + constructor (applicationServer, appEui, applicationId, applicationKey, messageHandler, dataModel, iotaConfiguration) { if (!applicationId) { throw new Error('applicationId is mandatory for TTN'); } - super(applicationServer, appEui, applicationId, applicationKey, messageHandler, iotaConfiguration); + super(applicationServer, appEui, applicationId, applicationKey, messageHandler, dataModel, iotaConfiguration); } /** @@ -94,8 +95,17 @@ class TtnAppService extends appService.AbstractAppService { message = null; return; } - if (message && message['payload_raw']) { - this.messageHandler(this, deviceId, null, message['payload_raw']); + + var dataModel = this.getDataModel(deviceId, null); + + if (message) { + if (dataModel === 'application_server' && message['payload_fields']) { + this.messageHandler(this, deviceId, null, message['payload_fields']); + } else if (message['payload_raw']) { + this.messageHandler(this, deviceId, null, message['payload_raw']); + } else { + this.messageHandler(this, deviceId, null, null); + } } else { this.messageHandler(this, deviceId, null, null); } diff --git a/lib/dataModels/cayenneLpp.js b/lib/dataModels/cayenneLpp.js index f8e43b8..7e3c767 100644 --- a/lib/dataModels/cayenneLpp.js +++ b/lib/dataModels/cayenneLpp.js @@ -66,13 +66,11 @@ const LPP_GYROMETER_SIZE = 6; // 2 bytes per axis, 0.01 °/s const LPP_GPS_SIZE = 9; // 3 byte lon/lat 0.0001 °, 3 bytes alt 0.01 meter /** - * Converts a Cayenne LPP payload to NGSI + * Decodes a Cayenne LPP payload * * @param {String} payload Cayenne LPP payload - * @param {Objects} device IOTA Device object */ -function toNgsi (payload, device) { - var ngsiAtts = []; +function decodePayload (payload) { var decodedObject = {}; winston.info('Decoding CaynneLPP message:' + payload); try { @@ -81,40 +79,7 @@ function toNgsi (payload, device) { winston.error('Error decoding CaynneLPP message:' + e); return; } - - if (device.active && device.active.length > 0) { - if (decodedObject) { - for (var field in decodedObject) { - var value = decodedObject[field]; - for (var i = 0; i < device.active.length; i++) { - if (device.active[i].type === 'geo:point' && value.latitude && value.longitude) { - value = value.latitude + ',' + value.longitude; - } - - if (field === device.active[i].name) { - ngsiAtts.push( - { - 'name': field, - 'type': device.active[i].type, - 'value': value - } - ); - } else if (device.active[i].object_id && device.active[i].object_id === field) { - ngsiAtts.push( - { - 'name': field, - 'type': device.active[i].type, - 'value': value - } - ); - } - } - } - } - } else { - winston.debug('Device provisioned without active attributes'); - } - return ngsiAtts; + return decodedObject; } function decodeCayenneLpp (bufferBase64) { @@ -322,5 +287,5 @@ function readInt24BE (buf, offset) { return buf.readIntBE(offset, 3); } -exports.toNgsi = toNgsi; +exports.decodePayload = decodePayload; exports.decodeCayenneLpp = decodeCayenneLpp; diff --git a/lib/dataModels/cbor.js b/lib/dataModels/cbor.js index 4568ff0..9a612b4 100644 --- a/lib/dataModels/cbor.js +++ b/lib/dataModels/cbor.js @@ -25,13 +25,11 @@ var winston = require('winston'); var CBOR = require('cbor-sync'); /** - * Converts a CBOR payload to NGSI + * Decodes a CBOR payload * * @param {String} payload Cayenne LPP payload - * @param {Objects} device IOTA Device object */ -function toNgsi (payload, device) { - var ngsiAtts = []; +function decodePayload (payload) { var decodedObject; winston.info('Decoding CBOR message:' + payload); try { @@ -42,35 +40,7 @@ function toNgsi (payload, device) { return; } - if (device.active && device.active.length > 0) { - if (decodedObject) { - for (var field in decodedObject) { - for (var i = 0; i < device.active.length; i++) { - if (field === device.active[i].name) { - ngsiAtts.push( - { - 'name': field, - 'type': device.active[i].type, - 'value': decodedObject[field] - } - ); - } else if (device.active[i].object_id && device.active[i].object_id === field) { - ngsiAtts.push( - { - 'name': field, - 'type': device.active[i].type, - 'value': decodedObject[field] - } - ); - } - } - } - } - } else { - winston.debug('Device provisioned without active attributes'); - } - - return ngsiAtts; + return decodedObject; } -exports.toNgsi = toNgsi; +exports.decodePayload = decodePayload; diff --git a/lib/dataTranslationService.js b/lib/dataTranslationService.js index 84aebf2..bb385f0 100644 --- a/lib/dataTranslationService.js +++ b/lib/dataTranslationService.js @@ -23,6 +23,7 @@ var cayenneLpp = require('./dataModels/cayenneLpp'); var cbor = require('./dataModels/cbor'); +var winston = require('winston'); /** * It converts a message received from a LoRaWAN application server to NGSI @@ -32,7 +33,8 @@ var cbor = require('./dataModels/cbor'); * @return {Object} {NGSI message} */ function toNgsi (payload, device) { - var ngsiMessage = {}; + var ngsiAtts = []; + var decodedPayload = {}; if (payload && device) { if (device.internalAttributes) { var lorawanConf = {}; @@ -48,18 +50,55 @@ function toNgsi (payload, device) { } if (lorawanConf) { - if (lorawanConf.data_model === 'cbor') { - ngsiMessage = cbor.toNgsi(payload, device); + if (lorawanConf.data_model === 'application_server') { + decodedPayload = payload; + } else if (lorawanConf.data_model === 'cbor') { + decodedPayload = cbor.decodePayload(payload); } else { - ngsiMessage = cayenneLpp.toNgsi(payload, device); + decodedPayload = cayenneLpp.decodePayload(payload); } } } else { - ngsiMessage = cayenneLpp.toNgsi(payload, device); + decodedPayload = cayenneLpp.decodePayload(payload); + } + + if (decodedPayload) { + if (device.active && device.active.length > 0) { + if (decodedPayload) { + for (var field in decodedPayload) { + var value = decodedPayload[field]; + for (i = 0; i < device.active.length; i++) { + if (device.active[i].type === 'geo:point' && value.latitude && value.longitude) { + value = value.latitude + ',' + value.longitude; + } + + if (field === device.active[i].name) { + ngsiAtts.push( + { + 'name': field, + 'type': device.active[i].type, + 'value': value + } + ); + } else if (device.active[i].object_id && device.active[i].object_id === field) { + ngsiAtts.push( + { + 'name': field, + 'type': device.active[i].type, + 'value': value + } + ); + } + } + } + } + } else { + winston.debug('Device provisioned without active attributes'); + } } } - return ngsiMessage; + return ngsiAtts; }; exports.toNgsi = toNgsi; diff --git a/lib/iotagent-lora.js b/lib/iotagent-lora.js index 8ee14d2..6e1c197 100644 --- a/lib/iotagent-lora.js +++ b/lib/iotagent-lora.js @@ -207,6 +207,7 @@ function registerApplicationServer (appServerConf, iotaConfiguration, callback) appServerConf.lorawan.application_id, appServerConf.lorawan.application_key, messageHandler, + appServerConf.lorawan.data_model, iotaConfiguration ); } else if (appServerConf.lorawan.application_server.provider === 'loraserver.io') { @@ -216,6 +217,7 @@ function registerApplicationServer (appServerConf, iotaConfiguration, callback) appServerConf.lorawan.application_id, appServerConf.lorawan.application_key, messageHandler, + appServerConf.lorawan.data_model, iotaConfiguration ); } else { diff --git a/test/deviceProvisioning/provisionDeviceApplicationServer1TTN.json b/test/deviceProvisioning/provisionDeviceApplicationServer1TTN.json new file mode 100644 index 0000000..6c77f66 --- /dev/null +++ b/test/deviceProvisioning/provisionDeviceApplicationServer1TTN.json @@ -0,0 +1,53 @@ +{ + "devices": [ + { + "protocol": "GENERIC_PROTO", + "device_id": "lora_n_003", + "entity_name": "LORA-N-003", + "entity_type": "LoraDevice", + "timezone": "America/Santiago", + "attributes": [ + { + "object_id": "bp0", + "name": "barometric_pressure_0", + "type": "Number" + }, + { + "object_id": "di3", + "name": "digital_in_3", + "type": "Number" + }, + { + "object_id": "do4", + "name": "digital_out_4", + "type": "Number" + }, + { + "object_id": "rh2", + "name": "relative_humidity_2", + "type": "Number" + }, + { + "object_id": "t1", + "name": "temperature_1", + "type": "Number" + } + ], + "internal_attributes": { + "lorawan": { + "application_server": { + "host": "localhost", + "username": "ari_ioe_app_demo1", + "password": "ttn-account-v2.UitfM5cPazqW52_zbtgUS6wM5vp1MeLC9Yu-Cozjfp0", + "provider": "TTN" + }, + "dev_eui": "3339343752356A14", + "app_eui": "70B3D57ED000985F", + "application_id": "ari_ioe_app_demo1", + "application_key": "9BE6B8EF16415B5F6ED4FBEAFE695C49", + "data_model": "application_server" + } + } + } + ] +} \ No newline at end of file diff --git a/test/unit/applicationServerDecoding.js b/test/unit/applicationServerDecoding.js new file mode 100644 index 0000000..69e835f --- /dev/null +++ b/test/unit/applicationServerDecoding.js @@ -0,0 +1,222 @@ +/* + * Copyright 2019 Atos Spain S.A + * + * This file is part of iotagent-lora + * + * iotagent-lora is free software: you can redistribute it and/or + * modify it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the License, + * or (at your option) any later version. + * + * iotagent-lora is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with iotagent-lora. + * If not, seehttp://www.gnu.org/licenses/. + * + */ + +'use strict'; + +var request = require('request'); +var async = require('async'); +var iotAgentConfig = require('../config-test.js'); +var utils = require('../utils'); +var iotagentLora = require('../../'); +var iotAgentLib = require('iotagent-node-lib'); +var mqtt = require('mqtt'); +var should = require('chai').should(); + +describe('CBOR Attributes', function () { + var testMosquittoHost = 'localhost'; + var orionHost = iotAgentConfig.iota.contextBroker.host; + var orionPort = iotAgentConfig.iota.contextBroker.port; + var orionServer = orionHost + ':' + orionPort; + var service = 'smartgondor'; + var subservice = '/gardens'; + + function readEnvVariables () { + if (process.env.TEST_MOSQUITTO_HOST) { + testMosquittoHost = process.env.TEST_MOSQUITTO_HOST; + } + + if (process.env.IOTA_CB_HOST) { + orionHost = process.env.IOTA_CB_HOST; + iotAgentConfig.iota.contextBroker.host = orionHost; + } + + if (process.env.IOTA_CB_PORT) { + orionPort = process.env.IOTA_CB_PORT; + iotAgentConfig.iota.contextBroker.port = orionPort; + } + + orionServer = orionHost + ':' + orionPort; + + if (process.env.TEST_MONGODB_HOST) { + iotAgentConfig.iota.mongodb.host = process.env.TEST_MONGODB_HOST; + } + } + + before(function (done) { + readEnvVariables(); + async.series([ + async.apply(utils.deleteEntityCB, iotAgentConfig.iota.contextBroker, service, subservice, 'LORA-N-003'), + async.apply(iotagentLora.start, iotAgentConfig) + ], done); + }); + + after(function (done) { + async.series([ + iotAgentLib.clearAll, + iotagentLora.stop, + async.apply(utils.deleteEntityCB, iotAgentConfig.iota.contextBroker, service, subservice, 'LORA-N-003') + ], done); + }); + + describe('When a device provisioning request with all the required data arrives to the IoT Agent. Proprietary decoding at application server', function () { + var options = { + url: 'http://localhost:' + iotAgentConfig.iota.server.port + '/iot/devices', + method: 'POST', + json: utils.readExampleFile('./test/deviceProvisioning/provisionDeviceApplicationServer1TTN.json'), + headers: { + 'fiware-service': service, + 'fiware-servicepath': subservice + } + }; + var optionsGetDevice = { + url: 'http://localhost:' + iotAgentConfig.iota.server.port + '/iot/devices', + method: 'GET', + json: true, + headers: { + 'fiware-service': service, + 'fiware-servicepath': subservice + } + }; + + it('should add the device to the devices list', function (done) { + if (testMosquittoHost) { + options.json.devices[0]['internal_attributes']['lorawan']['application_server']['host'] = testMosquittoHost; + } + + request(options, function (error, response, body) { + should.not.exist(error); + response.should.be.an('object'); + response.should.have.property('statusCode', 201); + setTimeout(function () { + request(optionsGetDevice, function (error, response, body) { + should.not.exist(error); + response.should.have.property('statusCode', 200); + body.should.have.property('count', 1); + body.should.have.property('devices'); + body.devices.should.be.an('array'); + body.devices.should.have.length(1); + body.devices[0].should.have.property('device_id', options.json.devices[0]['device_id']); + done(); + }); + }, 500); + }); + }); + + it('should register the entity in the CB', function (done) { + var optionsCB = { + url: 'http://' + orionServer + '/v2/entities/' + options.json.devices[0]['entity_name'], + method: 'GET', + json: true, + headers: { + 'fiware-service': service, + 'fiware-servicepath': subservice + } + }; + + request(optionsCB, function (error, response, body) { + should.not.exist(error); + response.should.have.property('statusCode', 200); + body.should.have.property('id', options.json.devices[0]['entity_name']); + done(); + }); + }); + + it('Should process correctly active attributes represented in CBOR model', function (done) { + var rawJSONPayload = { + barometric_pressure_0: 0, + digital_in_3: 100, + digital_out_4: 0, + relative_humidity_2: 0, + temperature_1: 27.2 + }; + + var optionsCB = { + url: 'http://' + orionServer + '/v2/entities/' + options.json.devices[0]['entity_name'], + method: 'GET', + json: true, + headers: { + 'fiware-service': service, + 'fiware-servicepath': subservice + } + }; + + var attributesExample = utils.readExampleFile('./test/activeAttributes/emptyCbor.json'); + attributesExample['payload_raw'] = null; + attributesExample['payload_fields'] = rawJSONPayload; + var client = mqtt.connect('mqtt://' + testMosquittoHost); + client.on('connect', function () { + client.publish(options.json.devices[0]['internal_attributes']['lorawan']['application_id'] + '/devices/' + options.json.devices[0]['device_id'] + '/up', JSON.stringify(attributesExample)); + setTimeout(function () { + request(optionsCB, function (error, response, body) { + should.not.exist(error); + response.should.have.property('statusCode', 200); + body.should.have.property('id', options.json.devices[0]['entity_name']); + body.should.have.property('temperature_1'); + body.temperature_1.should.have.property('type', 'Number'); + body.temperature_1.should.have.property('value', 27.2); + client.end(); + done(); + }); + }, 500); + }); + }); + }); + + describe('Active attributes are reported using attributes alias', function () { + it('Should process correctly active attributes', function (done) { + var optionsCB = { + url: 'http://' + orionServer + '/v2/entities/LORA-N-003', + method: 'GET', + json: true, + headers: { + 'fiware-service': service, + 'fiware-servicepath': subservice + } + }; + var rawJSONPayload = { + bp0: 0, + dg3: 100, + do4: 0, + rh2: 0, + t1: 27.2 + }; + + var attributesExample = utils.readExampleFile('./test/activeAttributes/emptyCbor.json'); + attributesExample['payload_fields'] = rawJSONPayload; + var client = mqtt.connect('mqtt://' + testMosquittoHost); + client.on('connect', function () { + client.publish('ari_ioe_app_demo1/devices/lora_n_003/up', JSON.stringify(attributesExample)); + setTimeout(function () { + request(optionsCB, function (error, response, body) { + should.not.exist(error); + response.should.have.property('statusCode', 200); + body.should.have.property('id', 'LORA-N-003'); + body.should.have.property('temperature_1'); + body.temperature_1.should.have.property('type', 'Number'); + body.temperature_1.should.have.property('value', 27.2); + client.end(); + done(); + }); + }, 500); + }); + }); + }); +}); diff --git a/test/unit/cayenneLppDecoding.js b/test/unit/cayenneLppDecoding.js index a19908a..4cd874c 100644 --- a/test/unit/cayenneLppDecoding.js +++ b/test/unit/cayenneLppDecoding.js @@ -22,6 +22,7 @@ 'use strict'; var decoder = require('../../lib/dataModels/cayenneLpp'); +var translator = require('../../lib/dataTranslationService'); require('chai').should(); describe('CayenneLpp decoding', function () { @@ -114,14 +115,14 @@ describe('NGSI translation', function (done) { it('Should translate a CayenneLpp payload to NGSI', function (done) { var cayenneLppMessageBase64 = 'AHMAAAFnARACaAADAGQEAQA='; - var decodedMessage = decoder.toNgsi(cayenneLppMessageBase64, device); + var decodedMessage = translator.toNgsi(cayenneLppMessageBase64, device); decodedMessage.should.be.an('array'); return done(); }); it('Should translate a CayenneLpp payload including GPS to NGSI', function (done) { var cayenneLppMessageBase64 = 'AYgGdl/ylgoAA+g='; - var decodedMessage = decoder.toNgsi(cayenneLppMessageBase64, deviceGps); + var decodedMessage = translator.toNgsi(cayenneLppMessageBase64, deviceGps); decodedMessage.should.be.an('array'); decodedMessage.should.have.length(1); decodedMessage[0].should.be.an('object');