From 2d78b3305a3c55f03bb365676c326d44cdb81329 Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 30 May 2024 11:26:36 +1200 Subject: [PATCH 01/27] jellyfish updates based on reuse of legacy _id for migration --- lib/jellyfishService.js | 1 + lib/schema/schemaEnv.js | 3 +++ lib/streamDAO.js | 30 +++++++++++++++++++++++++++-- package-lock.json | 2 +- test/api/basal/output.json | 27 +++++++++++++++++--------- test/api/bolusAndWizard/output.json | 12 ++++++++---- test/api/cbg/output.json | 12 ++++++++---- test/api/cgmSettings/output.json | 3 ++- test/api/upload/output.json | 6 +++++- test/streamDAOTest.js | 20 ++++++++++++------- 10 files changed, 87 insertions(+), 29 deletions(-) diff --git a/lib/jellyfishService.js b/lib/jellyfishService.js index 7292239d..348eaf29 100644 --- a/lib/jellyfishService.js +++ b/lib/jellyfishService.js @@ -163,6 +163,7 @@ function createServer(serverConfig, mongoClient, seagullClient, userApiClient, g }, versions: { uploaderMinimum : schemaEnv.minimumUploaderVersion, + uploaderDestination: schemaEnv.uploadDestination, loop: { minimumSupported: loop.minimumVersion, criticalUpdateNeeded: loop.criticalUpdateVersions, diff --git a/lib/schema/schemaEnv.js b/lib/schema/schemaEnv.js index 752e0701..ad2b859f 100644 --- a/lib/schema/schemaEnv.js +++ b/lib/schema/schemaEnv.js @@ -30,5 +30,8 @@ module.exports = (function () { schemaEnv.authRealm = config.fromEnvironment('KEYCLOAK_AUTH_REALM', null); schemaEnv.authUrl = config.fromEnvironment('KEYCLOAK_AUTH_URL', null); + // Used during migration of data + schemaEnv.uploadDestination = 'jellyfish'; + return schemaEnv; })(); diff --git a/lib/streamDAO.js b/lib/streamDAO.js index 53a5da00..adb5fa37 100644 --- a/lib/streamDAO.js +++ b/lib/streamDAO.js @@ -73,6 +73,27 @@ function convertDatesToLegacy(entry) { } } +// as part of migration process we attach the platform style _deduplicator using the 'legacy' hash +function setDeduplicator(datum) { + if (isUploadType(datum)){ + datum._deduplicator = { + name: 'org.tidepool.deduplicator.device.deactivate.legacy.hash', + version: '0.0.0', + }; + } else { + datum._deduplicator = { + hash: datum._id, + }; + } + return datum; +} + +function isUploadType(datum) { + return ( + typeof datum?.type === 'string' && datum.type.toLowerCase() === 'upload' + ); +} + function filterDatumForMongo(datum) { // Filters Datum for bad characteristics. This is a fix for problems with data that was accepted by earlier version of mongo // but will cause problems for versions of mongo > 3.4. Currently - the only problem fixed is truncating the @@ -104,13 +125,14 @@ function filterDatumForMongo(datum) { module.exports = function(mongoClient){ function getCollectionName(datum) { - return typeof datum?.type === 'string' && datum.type.toLowerCase() === 'upload' ? 'deviceDataSets' : 'deviceData'; + return isUploadType(datum) ? 'deviceDataSets' : 'deviceData'; } async function updateDatumInternal(session, collection, datum) { var clone = _.clone(datum); clone.modifiedTime = new Date(); clone = ensureId(clone); + clone = setDeduplicator(clone); let previous = await collection.findOne({_id: clone._id}, {session}); if (previous == null) { @@ -126,9 +148,11 @@ module.exports = function(mongoClient){ previous = _.assign(previous, { _id: previous._id + '_' + previous._version, _archivedTime: new Date(), - _active: false + _active: false, }); + previous = setDeduplicator(previous); + await collection.insertOne(previous, modificationOpts); clone.createdTime = previous.createdTime; @@ -199,6 +223,7 @@ module.exports = function(mongoClient){ datum._active = true; datum = ensureId(datum); + datum = setDeduplicator(datum); const errHandler = (err) => { if (err != null) { @@ -219,6 +244,7 @@ module.exports = function(mongoClient){ pre.hasProperty(datum, '_userId'); pre.hasProperty(datum, '_groupId'); datum.modifiedTime = new Date(); + datum = setDeduplicator(datum) var filteredDatum = filterDatumForMongo(datum); let firstResult = null; diff --git a/package-lock.json b/package-lock.json index a627eff8..d264ad7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2841,7 +2841,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true + "extraneous": true } } } diff --git a/test/api/basal/output.json b/test/api/basal/output.json index 7b480e10..977c9ba5 100644 --- a/test/api/basal/output.json +++ b/test/api/basal/output.json @@ -15,7 +15,8 @@ "_groupId": "1234", "id": "kmm427pfbrc6rugtmbuli8j4q61u17uk", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T06:00:00", @@ -33,7 +34,8 @@ "_groupId": "1234", "id": "cjou7vscvp8ogv34d6vejootulqfn3jd", "_version": 0, - "_active": false + "_active": false, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T06:00:00", @@ -52,7 +54,8 @@ "id": "cjou7vscvp8ogv34d6vejootulqfn3jd", "_version": 1, "_active": true, - "expectedDuration": 21600000 + "expectedDuration": 21600000, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T09:00:00", @@ -83,7 +86,8 @@ "rate": 1, "id": "tn33bjb0241j9qh4jg9vdnf1g6k1g9r8", "_version": 0, - "_active": false + "_active": false, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T09:00:00", @@ -115,7 +119,8 @@ "id": "tn33bjb0241j9qh4jg9vdnf1g6k1g9r8", "_version": 1, "_active": true, - "expectedDuration": 21600000 + "expectedDuration": 21600000, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T12:00:00", @@ -146,7 +151,8 @@ "rate": 0.5, "id": "ga58m84ggkscldj30lehrg74n313skcj", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T15:00:00", @@ -164,7 +170,8 @@ "_groupId": "1234", "id": "lsqqotp92j1d25d3d3ac53dsh6rgjg94", "_version": 0, - "_active": false + "_active": false, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T15:00:00", @@ -189,7 +196,8 @@ "code": "basal/mismatched-series", "nextId": "kftn188l8rjuvma3qkd3iqg34t0plajp" } - ] + ], + "_deduplicator": {} }, { "deviceTime": "2014-06-11T19:00:00", @@ -206,6 +214,7 @@ "_groupId": "1234", "id": "kftn188l8rjuvma3qkd3iqg34t0plajp", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} } ] diff --git a/test/api/bolusAndWizard/output.json b/test/api/bolusAndWizard/output.json index 25dc1f41..4d0b25a6 100644 --- a/test/api/bolusAndWizard/output.json +++ b/test/api/bolusAndWizard/output.json @@ -13,7 +13,8 @@ "_groupId": "1234", "id": "oi4effqrtdsk8g353dcstbeg06pip5kp", "_version": 0, - "_active": false + "_active": false, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T11:20:00", @@ -30,7 +31,8 @@ "id": "oi4effqrtdsk8g353dcstbeg06pip5kp", "_version": 1, "_active": true, - "expectedNormal": 2.5 + "expectedNormal": 2.5, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T18:00:00", @@ -46,7 +48,8 @@ "_groupId": "1234", "id": "hvnk1ngku6sqcpgav3t08f0m8lmhk6hv", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T18:00:00", @@ -73,6 +76,7 @@ "_groupId": "1234", "id": "vnputuqq0jtfhvi8tm7jinum5s24kkdq", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} } ] diff --git a/test/api/cbg/output.json b/test/api/cbg/output.json index 0bff5980..2e448ff6 100644 --- a/test/api/cbg/output.json +++ b/test/api/cbg/output.json @@ -13,7 +13,8 @@ "_groupId": "1234", "id": "eb12p6h892pmd0hhccpt2r17muc407o0", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} }, { "deviceTime": "2014-06-11T17:57:01", @@ -29,7 +30,8 @@ "_groupId": "1234", "id": "ha2ogn1kenqqhseed504sqnanhnclg5s", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} }, { "deviceTime": "2014-06-12T11:12:43", @@ -46,7 +48,8 @@ "_groupId": "1234", "id": "i922lobl3kron3t81pjap31anopkspvb", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} }, { "time": "2015-12-21T11:23:08Z", @@ -59,6 +62,7 @@ "_groupId": "1234", "id": "nsikjhfaprplpq78hc7di2lu5qpt1e3k", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} } ] diff --git a/test/api/cgmSettings/output.json b/test/api/cgmSettings/output.json index 75ecb3e1..42b57193 100644 --- a/test/api/cgmSettings/output.json +++ b/test/api/cgmSettings/output.json @@ -37,6 +37,7 @@ "_groupId": "1234", "id": "grop2783hkosd5lbv4sr9047a74vggch", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} } ] diff --git a/test/api/upload/output.json b/test/api/upload/output.json index 0175f27b..d8960077 100644 --- a/test/api/upload/output.json +++ b/test/api/upload/output.json @@ -24,6 +24,10 @@ "_groupId": "1234", "id": "7h3jc5jj5nl3bcu1gvs4j4k442pup1ef", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": { + "name": "org.tidepool.deduplicator.device.deactivate.legacy.hash", + "version": "0.0.0" + } } ] diff --git a/test/streamDAOTest.js b/test/streamDAOTest.js index 84f16161..e2987892 100644 --- a/test/streamDAOTest.js +++ b/test/streamDAOTest.js @@ -80,8 +80,8 @@ describe('streamDAO', function(){ expect(new Date(datum.createdTime).valueOf()).that.is.within(now, new Date()); expect(new Date(datum.modifiedTime).valueOf()).that.is.within(now, new Date()); expect(new Date(datum.modifiedTime).valueOf()).to.equal(new Date(datum.createdTime).valueOf()); - expect(_.omit(datum, 'createdTime', 'modifiedTime', '_id')).to.deep.equals( - { id: 'abcd', v: 1, _userId: 'u', _groupId: 'g', _version: 0, _active: true } + expect(_.omit(datum, 'createdTime', 'modifiedTime')).to.deep.equals( + { _id: '7jedtb7fq114l6n7nu3vkhrcdimdc07m', id: 'abcd', v: 1, _userId: 'u', _groupId: 'g', _version: 0, _active: true, _deduplicator: { hash: '7jedtb7fq114l6n7nu3vkhrcdimdc07m' } } ); done(err); @@ -125,22 +125,28 @@ describe('streamDAO', function(){ } streamDAO.getDatum('abcd', 'g', function(err, datum){ + if (err != null) { + return done(err); + } expect(datum).to.exist; expect(new Date(datum.modifiedTime).valueOf()).that.is.within(now, new Date()); - expect(_.omit(datum, 'modifiedTime', '_archivedTime', '_id')).to.deep.equals( - { id: 'abcd', f: 'a', v: 2828, _userId: 'u', _groupId: 'g', createdTime: createdTime, _version: 1, _active: true } + expect(_.omit(datum, 'modifiedTime', '_archivedTime')).to.deep.equals( + { _id: '7jedtb7fq114l6n7nu3vkhrcdimdc07m', id: 'abcd', f: 'a', v: 2828, _userId: 'u', _groupId: 'g', createdTime: createdTime, _version: 1, _active: true, _deduplicator: { hash: '7jedtb7fq114l6n7nu3vkhrcdimdc07m' }} ); var overwrittenId = datum._id + '_0'; mongoClient.withCollection('deviceData', done, function(coll, done){ + coll.find({_id: overwrittenId}).toArray(function(err, elements){ + if (err != null) { + return done(err); + } expect(elements).to.have.length(1); expect(elements[0]._archivedTime).that.is.within(now, Date.now()); expect(_.omit(elements[0], 'modifiedTime', '_archivedTime')).to.deep.equals( - { _id: overwrittenId, id: 'abcd', f: 'a', _userId: 'u', _groupId: 'g', v: 1, createdTime: createdTime, _version: 0, _active: false } + { _id: overwrittenId, id: 'abcd', f: 'a', _userId: 'u', _groupId: 'g', v: 1, createdTime: createdTime, _version: 0, _active: false, _deduplicator: { hash: overwrittenId }} ); - - done(err); + done(); }); }); }); From 12940572868818b3cefa7c1a92d5628212140005 Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 30 May 2024 11:36:04 +1200 Subject: [PATCH 02/27] remove unneeded change --- lib/streamDAO.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/streamDAO.js b/lib/streamDAO.js index adb5fa37..5b1eef9b 100644 --- a/lib/streamDAO.js +++ b/lib/streamDAO.js @@ -244,7 +244,6 @@ module.exports = function(mongoClient){ pre.hasProperty(datum, '_userId'); pre.hasProperty(datum, '_groupId'); datum.modifiedTime = new Date(); - datum = setDeduplicator(datum) var filteredDatum = filterDatumForMongo(datum); let firstResult = null; From be453d013bcf03c73a5329c90e726f2cd831b8a9 Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 30 May 2024 11:42:19 +1200 Subject: [PATCH 03/27] add _deduplicator to output --- test/api/deviceEvent/output.json | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/test/api/deviceEvent/output.json b/test/api/deviceEvent/output.json index 8c125edc..7055d308 100644 --- a/test/api/deviceEvent/output.json +++ b/test/api/deviceEvent/output.json @@ -14,7 +14,8 @@ "_groupId": "1234", "id": "64h7169livcae35c10d32ai721ugse79", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} }, { "type": "deviceEvent", @@ -30,7 +31,8 @@ "_groupId": "1234", "id": "f4t0qjh45tgjep3vjsm2cn0isqg5u6ie", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} }, { "type": "deviceEvent", @@ -54,7 +56,8 @@ ], "id": "qdmhpmdd2v5v8uuv6otmqp7sjqlmb1cr", "_version": 0, - "_active": false + "_active": false, + "_deduplicator": {} }, { "type": "deviceEvent", @@ -75,7 +78,8 @@ "id": "qdmhpmdd2v5v8uuv6otmqp7sjqlmb1cr", "_version": 1, "_active": true, - "duration": 14000 + "duration": 14000, + "_deduplicator": {} }, { "type": "deviceEvent", @@ -92,7 +96,8 @@ "_groupId": "1234", "id": "ep1kqd57okpgke6c0g7al5j1f80cqg5m", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} }, { "type": "deviceEvent", @@ -116,6 +121,7 @@ "_groupId": "1234", "id": "j80h9aqiioe8d8e41sh4ujf7ek28rppi", "_version": 0, - "_active": true + "_active": true, + "_deduplicator": {} } ] From ef45bc2d8da0feaf371be456b550bb83e334f54c Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 1 Aug 2024 11:33:26 +1200 Subject: [PATCH 04/27] change hash name --- lib/streamDAO.js | 2 +- test/api/upload/output.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/streamDAO.js b/lib/streamDAO.js index 5b1eef9b..5e3623bd 100644 --- a/lib/streamDAO.js +++ b/lib/streamDAO.js @@ -77,7 +77,7 @@ function convertDatesToLegacy(entry) { function setDeduplicator(datum) { if (isUploadType(datum)){ datum._deduplicator = { - name: 'org.tidepool.deduplicator.device.deactivate.legacy.hash', + name: 'org.tidepool.deduplicator.device.deactivate.hash', version: '0.0.0', }; } else { diff --git a/test/api/upload/output.json b/test/api/upload/output.json index d8960077..eef13b39 100644 --- a/test/api/upload/output.json +++ b/test/api/upload/output.json @@ -26,7 +26,7 @@ "_version": 0, "_active": true, "_deduplicator": { - "name": "org.tidepool.deduplicator.device.deactivate.legacy.hash", + "name": "org.tidepool.deduplicator.device.deactivate.hash", "version": "0.0.0" } } From 4cb22622ebcb4f6a51b322b4fd88157da988e171 Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 5 Aug 2024 15:07:23 +1200 Subject: [PATCH 05/27] debug for smbg Id --- lib/schema/schema.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/schema/schema.js b/lib/schema/schema.js index f084fb22..98a20259 100644 --- a/lib/schema/schema.js +++ b/lib/schema/schema.js @@ -496,6 +496,12 @@ exports.generateId = function(datum, fields) { } } + if (datum['type'] == 'smbg') { + console.log('smbg fields: ', fields); + console.log('smbg vals: ', vals); + console.log('smbg ID : ', misc.generateId(vals)); + } + return misc.generateId(vals); }; @@ -511,5 +517,7 @@ exports.makeId = function(datum) { throw except.IAE('No known idFields for type[%s]', type); } + + return exports.generateId(datum, idFields); }; From da7def91ba7b905a84eadd5bda06874f65eac11f Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 5 Aug 2024 15:14:00 +1200 Subject: [PATCH 06/27] fix lint --- lib/schema/schema.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/schema/schema.js b/lib/schema/schema.js index 98a20259..a0b7bac4 100644 --- a/lib/schema/schema.js +++ b/lib/schema/schema.js @@ -496,7 +496,7 @@ exports.generateId = function(datum, fields) { } } - if (datum['type'] == 'smbg') { + if (datum.type == 'smbg') { console.log('smbg fields: ', fields); console.log('smbg vals: ', vals); console.log('smbg ID : ', misc.generateId(vals)); From af8971d5e391a4b85107755a7c963fd83d472d33 Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 5 Aug 2024 15:45:33 +1200 Subject: [PATCH 07/27] debug --- lib/schema/schema.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/schema/schema.js b/lib/schema/schema.js index a0b7bac4..04ea6cfd 100644 --- a/lib/schema/schema.js +++ b/lib/schema/schema.js @@ -23,6 +23,8 @@ var semver = require('semver'); var except = amoeba.except; var util = require('util'); +var log = require('../log.js')('schema.js'); + var misc = require('../misc.js'); exports.validDeviceTime = function(val) { @@ -497,8 +499,12 @@ exports.generateId = function(datum, fields) { } if (datum.type == 'smbg') { - console.log('smbg fields: ', fields); - console.log('smbg vals: ', vals); + log.info('smbg vals ', JSON.stringify(vals)); + log.info('smbg vals ', JSON.stringify(datum)); + log.info('smbg ID ', misc.generateId(vals)); + + console.log('smbg vals: ', JSON.stringify(vals)); + console.log('smbg datum: ', JSON.stringify(datum)); console.log('smbg ID : ', misc.generateId(vals)); } From f75e54b6ebbcd96113a0fc2ccfa21efc0506f4f3 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 6 Aug 2024 12:38:29 +1200 Subject: [PATCH 08/27] debug --- lib/misc.js | 13 +++++++++++-- lib/schema/schema.js | 7 +------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/misc.js b/lib/misc.js index 5b5ecf4e..fbb8e1a7 100644 --- a/lib/misc.js +++ b/lib/misc.js @@ -45,23 +45,32 @@ var except = amoeba.except; * @param fields an array of values to be concatenated together into a unique string * @returns {string} the base32 encoded hash of the delimited-concatenation of the provided fields (also known as a "unique" id) */ -exports.generateId = function(fields) { +exports.generateId = function(fields, showHashStr=false) { var hasher = crypto.createHash('sha1'); + var hashStr = '' + for (var i = 0; i < fields.length; ++i) { var val = fields[i]; if (val == null) { throw except.IAE('null value in fields[%s]', fields); } + hasher.update(String(val)); + hashStr += String(val); hasher.update('_'); + hashStr += '_'; } // adding an additional string to the hash data for BtUTC // to ensure different IDs generated when uploading data // that has been uploaded before hasher.update(String('bootstrap')); + hashStr += 'bootstrap'; hasher.update('_'); - + hashStr += '_'; + if (showHashStr){ + console.log('Jellyfish SMBG hash string ',hashStr); + } return base32hex.encodeBuffer(hasher.digest(), { paddingChar: '-' }); }; diff --git a/lib/schema/schema.js b/lib/schema/schema.js index 04ea6cfd..f38b10bd 100644 --- a/lib/schema/schema.js +++ b/lib/schema/schema.js @@ -500,12 +500,7 @@ exports.generateId = function(datum, fields) { if (datum.type == 'smbg') { log.info('smbg vals ', JSON.stringify(vals)); - log.info('smbg vals ', JSON.stringify(datum)); - log.info('smbg ID ', misc.generateId(vals)); - - console.log('smbg vals: ', JSON.stringify(vals)); - console.log('smbg datum: ', JSON.stringify(datum)); - console.log('smbg ID : ', misc.generateId(vals)); + log.info('smbg ID ', misc.generateId(vals, true)); } return misc.generateId(vals); From f58a8e102d265c63ecad919fd495cbe0b58b0a44 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 6 Aug 2024 13:20:00 +1200 Subject: [PATCH 09/27] fix lint --- lib/misc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/misc.js b/lib/misc.js index fbb8e1a7..83b653d0 100644 --- a/lib/misc.js +++ b/lib/misc.js @@ -48,7 +48,7 @@ var except = amoeba.except; exports.generateId = function(fields, showHashStr=false) { var hasher = crypto.createHash('sha1'); - var hashStr = '' + var hashStr = ''; for (var i = 0; i < fields.length; ++i) { var val = fields[i]; From 66aeb7db9657e6dad04fb236fc0cad901bdc1e24 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 6 Aug 2024 16:44:54 +1200 Subject: [PATCH 10/27] more debug --- lib/schema/schema.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/schema/schema.js b/lib/schema/schema.js index f38b10bd..edbcb4a5 100644 --- a/lib/schema/schema.js +++ b/lib/schema/schema.js @@ -381,6 +381,9 @@ exports.makeHandler = function(key, spec) { var datum = data[i]; if (datum.id == null) { datum.id = exports.makeId(datum); + if (datum.type == 'smbg') { + log.info('attachId ID ', datum.id); + } } } @@ -499,8 +502,8 @@ exports.generateId = function(datum, fields) { } if (datum.type == 'smbg') { - log.info('smbg vals ', JSON.stringify(vals)); - log.info('smbg ID ', misc.generateId(vals, true)); + log.info('generateId vals ',vals); + log.info('generateId ID for vals ',misc.generateId(vals, true)); } return misc.generateId(vals); From 280ff667b34ca3a3df8a614d1189bdeb42cebf0a Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 6 Aug 2024 16:59:12 +1200 Subject: [PATCH 11/27] debug comment --- lib/schema/schema.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/schema/schema.js b/lib/schema/schema.js index edbcb4a5..0fdf6af9 100644 --- a/lib/schema/schema.js +++ b/lib/schema/schema.js @@ -382,7 +382,7 @@ exports.makeHandler = function(key, spec) { if (datum.id == null) { datum.id = exports.makeId(datum); if (datum.type == 'smbg') { - log.info('attachId ID ', datum.id); + log.info('Jellyfish SMBG attachId ID ', datum.id); } } } @@ -502,8 +502,8 @@ exports.generateId = function(datum, fields) { } if (datum.type == 'smbg') { - log.info('generateId vals ',vals); - log.info('generateId ID for vals ',misc.generateId(vals, true)); + log.info('Jellyfish SMBG generateId vals ',vals); + log.info('Jellyfish SMBG generateId ID for vals ',misc.generateId(vals, true)); } return misc.generateId(vals); From 4ba426655e0bced7424cf5312775b42a746eb54f Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 15 Aug 2024 11:06:58 +1200 Subject: [PATCH 12/27] remove debug --- lib/misc.js | 3 --- lib/schema/schema.js | 12 ------------ 2 files changed, 15 deletions(-) diff --git a/lib/misc.js b/lib/misc.js index 83b653d0..801ea390 100644 --- a/lib/misc.js +++ b/lib/misc.js @@ -68,9 +68,6 @@ exports.generateId = function(fields, showHashStr=false) { hashStr += 'bootstrap'; hasher.update('_'); hashStr += '_'; - if (showHashStr){ - console.log('Jellyfish SMBG hash string ',hashStr); - } return base32hex.encodeBuffer(hasher.digest(), { paddingChar: '-' }); }; diff --git a/lib/schema/schema.js b/lib/schema/schema.js index 0fdf6af9..645e60fb 100644 --- a/lib/schema/schema.js +++ b/lib/schema/schema.js @@ -22,9 +22,6 @@ var amoeba = require('amoeba'); var semver = require('semver'); var except = amoeba.except; var util = require('util'); - -var log = require('../log.js')('schema.js'); - var misc = require('../misc.js'); exports.validDeviceTime = function(val) { @@ -381,9 +378,6 @@ exports.makeHandler = function(key, spec) { var datum = data[i]; if (datum.id == null) { datum.id = exports.makeId(datum); - if (datum.type == 'smbg') { - log.info('Jellyfish SMBG attachId ID ', datum.id); - } } } @@ -500,12 +494,6 @@ exports.generateId = function(datum, fields) { throw except.IAE('Can\'t generate id, field[%s] didn\'t exist on datum of type[%s]', fields[i], datum.type); } } - - if (datum.type == 'smbg') { - log.info('Jellyfish SMBG generateId vals ',vals); - log.info('Jellyfish SMBG generateId ID for vals ',misc.generateId(vals, true)); - } - return misc.generateId(vals); }; From f518ad278c376d3e5716f00debbdd250a4d47347 Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 15 Aug 2024 12:19:54 +1200 Subject: [PATCH 13/27] minimal implementation of service switch based on userid --- env.js | 42 ++++++-- lib/jellyfishService.js | 225 +++++++++++++++++++++++----------------- lib/schema/schemaEnv.js | 3 - 3 files changed, 163 insertions(+), 107 deletions(-) diff --git a/env.js b/env.js index bc031ed5..0df1e104 100644 --- a/env.js +++ b/env.js @@ -48,33 +48,59 @@ module.exports = (function () { maybeReplaceWithContentsOfFile(env.httpsConfig, 'pfx'); } if (env.httpsPort != null && env.httpsConfig == null) { - throw new Error('No https config provided, please set HTTPS_CONFIG with at least the certificate to use.'); + throw new Error( + 'No https config provided, please set HTTPS_CONFIG with at least the certificate to use.' + ); } if (env.httpPort == null && env.httpsPort == null) { - throw new Error('Must specify either PORT or HTTPS_PORT in your environment.'); + throw new Error( + 'Must specify either PORT or HTTPS_PORT in your environment.' + ); } env.userApi = { - service: config.fromEnvironment('TIDEPOOL_AUTH_CLIENT_ADDRESS', 'shoreline:9107'), + service: config.fromEnvironment( + 'TIDEPOOL_AUTH_CLIENT_ADDRESS', + 'shoreline:9107' + ), // Name of this server to pass to user-api when getting a server token - serverName: config.fromEnvironment("SERVER_NAME", "jellyfish:default"), + serverName: config.fromEnvironment('SERVER_NAME', 'jellyfish:default'), // The secret to use when getting a server token from user-api - serverSecret: config.fromEnvironment("TIDEPOOL_SERVER_SECRET") + serverSecret: config.fromEnvironment('TIDEPOOL_SERVER_SECRET'), }; env.gatekeeper = { - service: config.fromEnvironment('TIDEPOOL_PERMISSION_CLIENT_ADDRESS', 'gatekeeper:9123') + service: config.fromEnvironment( + 'TIDEPOOL_PERMISSION_CLIENT_ADDRESS', + 'gatekeeper:9123' + ), }; env.seagull = { - service: config.fromEnvironment('TIDEPOOL_SEAGULL_CLIENT_ADDRESS', 'seagull:9120') + service: config.fromEnvironment( + 'TIDEPOOL_SEAGULL_CLIENT_ADDRESS', + 'seagull:9120' + ), }; env.mongo = { - connectionString: cs('data') + connectionString: cs('data'), + }; + + // users who are going to upload via platform during migration + var platformUsers; + try { + usersFile = fs.readFileSync(__dirname + '/platform_users.json'); + platformUsers = JSON.parse(usersFile); + } catch (err) { + platformUsers = []; + } + + env.uploader = { + platformUsers, }; return env; diff --git a/lib/jellyfishService.js b/lib/jellyfishService.js index 348eaf29..079f9549 100644 --- a/lib/jellyfishService.js +++ b/lib/jellyfishService.js @@ -18,12 +18,9 @@ const { createTerminus, HealthCheckError } = require('@godaddy/terminus'); -var fs = require('fs'); - var log = require('./log.js')('jellyfishService.js'); var async = require('async'); -var env = require('../env.js'); var schemaEnv = require('./schema/schemaEnv.js'); var loop = require('./schema/loop.js'); var express = require('express'); @@ -32,13 +29,19 @@ var bodyparser = require('body-parser'); var util = require('util'); var _ = require('lodash'); -var jellyfishService = function(envConfig, mongoClient, seagullClient, userApiClient, gatekeeperClient) { +var jellyfishService = function ( + envConfig, + mongoClient, + seagullClient, + userApiClient, + gatekeeperClient +) { var app, servicePort; //create the server depending on the type if (envConfig.httpPort != null) { servicePort = envConfig.httpPort; app = createServer( - _.extend({ name: 'TidepoolJellyfishHttp'}, envConfig), + _.extend({ name: 'TidepoolJellyfishHttp' }, envConfig), mongoClient, seagullClient, userApiClient, @@ -47,7 +50,7 @@ var jellyfishService = function(envConfig, mongoClient, seagullClient, userApiCl } else if (envConfig.httpsPort != null) { servicePort = envConfig.httpsPort; app = createServer( - _.extend({ name: 'TidepoolJellyfishHttps'}, envConfig), + _.extend({ name: 'TidepoolJellyfishHttps' }, envConfig), mongoClient, seagullClient, userApiClient, @@ -58,12 +61,14 @@ var jellyfishService = function(envConfig, mongoClient, seagullClient, userApiCl function beforeShutdown() { // avoid running into any race conditions // https://github.com/godaddy/terminus#how-to-set-terminus-up-with-kubernetes - return new Promise(resolve => setTimeout(resolve, 5000)); + return new Promise((resolve) => setTimeout(resolve, 5000)); } async function healthCheck() { if (!mongoClient.healthCheck()) { - throw new HealthCheckError('Database Error', ['Failed to connect to MongoDB']); + throw new HealthCheckError('Database Error', [ + 'Failed to connect to MongoDB', + ]); } } @@ -82,42 +87,48 @@ var jellyfishService = function(envConfig, mongoClient, seagullClient, userApiCl this.theServer = http.createServer(app).listen(servicePort, cb); } else if (envConfig.httpsPort != null) { var https = require('https'); - this.theServer = https.createServer(envConfig.httpsConfig, app).listen(servicePort, cb); + this.theServer = https + .createServer(envConfig.httpsConfig, app) + .listen(servicePort, cb); } - if(this.theServer) { + if (this.theServer) { this.theServer.keepAliveTimeout = 151 * 1000; this.theServer.headersTimeout = 155 * 1000; // This should be bigger than `keepAliveTimeout + your server's expected response time` = 61 * 1000; createTerminus(this.theServer, { healthChecks: { - '/status': healthCheck + '/status': healthCheck, }, beforeShutdown, }); } - } + }, }; return { - close : serviceManager.stopService.bind(serviceManager, app), - start : serviceManager.startService.bind(serviceManager, app, servicePort) + close: serviceManager.stopService.bind(serviceManager, app), + start: serviceManager.startService.bind(serviceManager, app, servicePort), }; }; -var jsonp = function(response) { - return function(error, data) { - if(error) { +var jsonp = function (response) { + return function (error, data) { + if (error) { log.warn(error, 'an error occurred!?'); - response.status(500).jsonp({error: error}); + response.status(500).jsonp({ error: error }); return; } response.jsonp(data); }; }; - -function createServer(serverConfig, mongoClient, seagullClient, userApiClient, gatekeeperClient){ - +function createServer( + serverConfig, + mongoClient, + seagullClient, + userApiClient, + gatekeeperClient +) { // this is a little weird because we get a client and we also require it, but // in this case we're getting a sibling of the client we're passed in var middleware = require('user-api-client').middleware; @@ -129,14 +140,14 @@ function createServer(serverConfig, mongoClient, seagullClient, userApiClient, g function getPrivatePair(userid, callback) { async.waterfall( [ - function(cb) { + function (cb) { userApiClient.withServerToken(cb); }, - function(token, cb) { + function (token, cb) { seagullClient.getPrivatePair(userid, 'uploads', token, cb); - } + }, ], - function(err, privatePair) { + function (err, privatePair) { if (err != null) { return callback(err); } @@ -153,61 +164,71 @@ function createServer(serverConfig, mongoClient, seagullClient, userApiClient, g app.use(bodyparser.json({ limit: '4mb' })); app.use(errorHandler({ dumpExceptions: true, showStack: true })); - app.get('/info', function(request, response) { + app.get('/info/?:userId?', function (request, response) { log.info('Handling versions request'); + var uploaderDestination = 'jellyfish'; + if (request.params.userId) { + log.info('using user id to detrimine upload destination'); + if (serverConfig.uploader.platformUsers.includes(request.params.userId)) { + uploaderDestination = 'platform'; + log.info(`upload to ${uploaderDestination}`); + } + } + var body = { auth: { realm: schemaEnv.authRealm, url: schemaEnv.authUrl, }, versions: { - uploaderMinimum : schemaEnv.minimumUploaderVersion, - uploaderDestination: schemaEnv.uploadDestination, + uploaderMinimum: schemaEnv.minimumUploaderVersion, + uploaderDestination, loop: { minimumSupported: loop.minimumVersion, criticalUpdateNeeded: loop.criticalUpdateVersions, - } - } + }, + }, }; - response.status(200).send(body); }); /* send the actual ingested data to the platform */ - app.post( - '/data/?:groupId?', - checkToken, - function(request, response) { - var userid = request._tokendata.userid; + app.post('/data/?:groupId?', checkToken, function (request, response) { + var userid = request._tokendata.userid; - var array = request.body; + var array = request.body; - if (typeof(array) !== 'object') { - return response.status(400).send(util.format('Expected an object body, got[%s]', typeof(array))); - } + if (typeof array !== 'object') { + return response + .status(400) + .send(util.format('Expected an object body, got[%s]', typeof array)); + } - if (!Array.isArray(array)) { - array = [array]; - } + if (!Array.isArray(array)) { + array = [array]; + } - var count = 0; - var duplicates = []; + var count = 0; + var duplicates = []; - let datasetUserId; + let datasetUserId; - async.waterfall( - [ - function(cb) { - // if no groupId was specified, just continue to upload for the - // connected user - if (!request.params.groupId) { - return cb(null, userid); - } + async.waterfall( + [ + function (cb) { + // if no groupId was specified, just continue to upload for the + // connected user + if (!request.params.groupId) { + return cb(null, userid); + } - gatekeeperClient.userInGroup(userid, request.params.groupId, function(err, perms) { + gatekeeperClient.userInGroup( + userid, + request.params.groupId, + function (err, perms) { if (err && err.statusCode !== 404) { return cb(err); } @@ -218,58 +239,70 @@ function createServer(serverConfig, mongoClient, seagullClient, userApiClient, g cb({ statusCode: 403, - message: 'You don\'t have rights to upload to that account.' + message: "You don't have rights to upload to that account.", }); - }); - }, - function(userId, cb) { - getPrivatePair(userId, function(err, privatePair) { - cb(err, userId, privatePair ? privatePair.id : null); - }); - }, - function(userId, groupId, cb) { - datasetUserId = userId; - - async.mapSeries( - array, - function(obj, cb) { - obj._userId = userId; - obj._groupId = groupId; - dataBroker.addDatum(obj,function(err){ - if (err != null) { - if (err.errorCode === 'duplicate') { - duplicates.push(count); - err = null; - } else { - err.dataIndex = count; - } + } + ); + }, + function (userId, cb) { + getPrivatePair(userId, function (err, privatePair) { + cb(err, userId, privatePair ? privatePair.id : null); + }); + }, + function (userId, groupId, cb) { + datasetUserId = userId; + + async.mapSeries( + array, + function (obj, cb) { + obj._userId = userId; + obj._groupId = groupId; + dataBroker.addDatum(obj, function (err) { + if (err != null) { + if (err.errorCode === 'duplicate') { + duplicates.push(count); + err = null; + } else { + err.dataIndex = count; } - ++count; - cb(err); - }); - }, - cb - ); - } - ], - function(err) { - dataBroker.setSummariesOutdated(datasetUserId, array, count, function() { + } + ++count; + cb(err); + }); + }, + cb + ); + }, + ], + function (err) { + dataBroker.setSummariesOutdated( + datasetUserId, + array, + count, + function () { if (err != null) { if (err.statusCode != null) { response.status(err.statusCode).send(err); } else { - var groupMessage = request.params.groupId ? ('To group[' + request.params.groupId + ']') : ''; - log.warn(err, 'Problem uploading for user[%s]. %s', userid, groupMessage); + var groupMessage = request.params.groupId + ? 'To group[' + request.params.groupId + ']' + : ''; + log.warn( + err, + 'Problem uploading for user[%s]. %s', + userid, + groupMessage + ); response.status(500); } } else { response.status(200).send(duplicates); } - }); - } - ); - } - ); + } + ); + } + ); + }); return app; } diff --git a/lib/schema/schemaEnv.js b/lib/schema/schemaEnv.js index ad2b859f..752e0701 100644 --- a/lib/schema/schemaEnv.js +++ b/lib/schema/schemaEnv.js @@ -30,8 +30,5 @@ module.exports = (function () { schemaEnv.authRealm = config.fromEnvironment('KEYCLOAK_AUTH_REALM', null); schemaEnv.authUrl = config.fromEnvironment('KEYCLOAK_AUTH_URL', null); - // Used during migration of data - schemaEnv.uploadDestination = 'jellyfish'; - return schemaEnv; })(); From b7b8c213b3b832201c0c0460c4a6a2538d8bc36d Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 22 Aug 2024 12:46:47 +1200 Subject: [PATCH 14/27] place holder list --- platform_api_users.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 platform_api_users.json diff --git a/platform_api_users.json b/platform_api_users.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/platform_api_users.json @@ -0,0 +1 @@ +[] From 42fa67a82c88358c962395508f65d419909f081c Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 6 Sep 2024 15:34:17 +1200 Subject: [PATCH 15/27] fix lint --- lib/jellyfishService.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/jellyfishService.js b/lib/jellyfishService.js index 079f9549..7e3e8017 100644 --- a/lib/jellyfishService.js +++ b/lib/jellyfishService.js @@ -284,9 +284,7 @@ function createServer( if (err.statusCode != null) { response.status(err.statusCode).send(err); } else { - var groupMessage = request.params.groupId - ? 'To group[' + request.params.groupId + ']' - : ''; + var groupMessage = request.params.groupId ? 'To group[' + request.params.groupId + ']' : ''; log.warn( err, 'Problem uploading for user[%s]. %s', From 837ab5863c4eea18ebb77391c0c4f44d8179a3a2 Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 6 Sep 2024 15:41:04 +1200 Subject: [PATCH 16/27] remove debug --- lib/misc.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/misc.js b/lib/misc.js index 801ea390..ac1864c6 100644 --- a/lib/misc.js +++ b/lib/misc.js @@ -1,15 +1,15 @@ /* * == BSD2 LICENSE == * Copyright (c) 2014, Tidepool Project - * + * * This program is free software; you can redistribute it and/or modify it under * the terms of the associated License, which is identical to the BSD 2-Clause * License as published by the Open Source Initiative at opensource.org. - * + * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the License for more details. - * + * * You should have received a copy of the License along with this program; if * not, you can obtain one from Tidepool Project at tidepool.org. * == BSD2 LICENSE == @@ -45,7 +45,7 @@ var except = amoeba.except; * @param fields an array of values to be concatenated together into a unique string * @returns {string} the base32 encoded hash of the delimited-concatenation of the provided fields (also known as a "unique" id) */ -exports.generateId = function(fields, showHashStr=false) { +exports.generateId = function (fields) { var hasher = crypto.createHash('sha1'); var hashStr = ''; @@ -55,7 +55,7 @@ exports.generateId = function(fields, showHashStr=false) { if (val == null) { throw except.IAE('null value in fields[%s]', fields); } - + hasher.update(String(val)); hashStr += String(val); hasher.update('_'); @@ -70,4 +70,3 @@ exports.generateId = function(fields, showHashStr=false) { hashStr += '_'; return base32hex.encodeBuffer(hasher.digest(), { paddingChar: '-' }); }; - From b323d5385733ba0eeebe3a4449a7f0774b1f7ddc Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 27 Sep 2024 14:26:59 +1200 Subject: [PATCH 17/27] updates from review --- env.js | 16 ++++++++-------- lib/jellyfishService.js | 11 ++++++----- lib/misc.js | 17 ++++++----------- lib/schema/schema.js | 4 ++-- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/env.js b/env.js index 0df1e104..dc707c7d 100644 --- a/env.js +++ b/env.js @@ -91,16 +91,16 @@ module.exports = (function () { }; // users who are going to upload via platform during migration - var platformUsers; - try { - usersFile = fs.readFileSync(__dirname + '/platform_users.json'); - platformUsers = JSON.parse(usersFile); - } catch (err) { - platformUsers = []; + var usersFile = fs.readFileSync(__dirname + '/platform_users.json'); + var platformUsers = JSON.parse(usersFile); + if (!Array.isArray(platformUsers)){ + throw new Error( + 'Must contain an array in platform_users.json' + ); } - + env.uploader = { - platformUsers, + platformUsers: platformUsers.map(item => item.toLowerCase().trim()), }; return env; diff --git a/lib/jellyfishService.js b/lib/jellyfishService.js index 7e3e8017..ccffeec3 100644 --- a/lib/jellyfishService.js +++ b/lib/jellyfishService.js @@ -165,15 +165,16 @@ function createServer( app.use(errorHandler({ dumpExceptions: true, showStack: true })); app.get('/info/?:userId?', function (request, response) { - log.info('Handling versions request'); + + var uploaderDestination = null; - var uploaderDestination = 'jellyfish'; if (request.params.userId) { - log.info('using user id to detrimine upload destination'); - if (serverConfig.uploader.platformUsers.includes(request.params.userId)) { + if (serverConfig.uploader.platformUsers.includes(request.params.userId.trim().toLowerCase())) { uploaderDestination = 'platform'; - log.info(`upload to ${uploaderDestination}`); + }else{ + uploaderDestination = 'jellyfish'; } + log.info(`uploader destination for user with id '${request.params.userId}' is '${uploaderDestination}'`); } var body = { diff --git a/lib/misc.js b/lib/misc.js index ac1864c6..5b5ecf4e 100644 --- a/lib/misc.js +++ b/lib/misc.js @@ -1,15 +1,15 @@ /* * == BSD2 LICENSE == * Copyright (c) 2014, Tidepool Project - * + * * This program is free software; you can redistribute it and/or modify it under * the terms of the associated License, which is identical to the BSD 2-Clause * License as published by the Open Source Initiative at opensource.org. - * + * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the License for more details. - * + * * You should have received a copy of the License along with this program; if * not, you can obtain one from Tidepool Project at tidepool.org. * == BSD2 LICENSE == @@ -45,28 +45,23 @@ var except = amoeba.except; * @param fields an array of values to be concatenated together into a unique string * @returns {string} the base32 encoded hash of the delimited-concatenation of the provided fields (also known as a "unique" id) */ -exports.generateId = function (fields) { +exports.generateId = function(fields) { var hasher = crypto.createHash('sha1'); - var hashStr = ''; - for (var i = 0; i < fields.length; ++i) { var val = fields[i]; if (val == null) { throw except.IAE('null value in fields[%s]', fields); } - hasher.update(String(val)); - hashStr += String(val); hasher.update('_'); - hashStr += '_'; } // adding an additional string to the hash data for BtUTC // to ensure different IDs generated when uploading data // that has been uploaded before hasher.update(String('bootstrap')); - hashStr += 'bootstrap'; hasher.update('_'); - hashStr += '_'; + return base32hex.encodeBuffer(hasher.digest(), { paddingChar: '-' }); }; + diff --git a/lib/schema/schema.js b/lib/schema/schema.js index 645e60fb..f084fb22 100644 --- a/lib/schema/schema.js +++ b/lib/schema/schema.js @@ -22,6 +22,7 @@ var amoeba = require('amoeba'); var semver = require('semver'); var except = amoeba.except; var util = require('util'); + var misc = require('../misc.js'); exports.validDeviceTime = function(val) { @@ -494,6 +495,7 @@ exports.generateId = function(datum, fields) { throw except.IAE('Can\'t generate id, field[%s] didn\'t exist on datum of type[%s]', fields[i], datum.type); } } + return misc.generateId(vals); }; @@ -509,7 +511,5 @@ exports.makeId = function(datum) { throw except.IAE('No known idFields for type[%s]', type); } - - return exports.generateId(datum, idFields); }; From b96544184a6688f3e91c8887518a43350b249429 Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 3 Oct 2024 13:12:35 +1300 Subject: [PATCH 18/27] move config val to top-level --- lib/jellyfishService.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/jellyfishService.js b/lib/jellyfishService.js index ccffeec3..090b5074 100644 --- a/lib/jellyfishService.js +++ b/lib/jellyfishService.js @@ -184,12 +184,12 @@ function createServer( }, versions: { uploaderMinimum: schemaEnv.minimumUploaderVersion, - uploaderDestination, loop: { minimumSupported: loop.minimumVersion, criticalUpdateNeeded: loop.criticalUpdateVersions, }, }, + uploaderDestination, }; response.status(200).send(body); }); From ae0560744053bb98d4f0226f94125b0235e4fee9 Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 9 Oct 2024 13:29:50 +1300 Subject: [PATCH 19/27] pull users ids from config --- env.js | 13 ------------- lib/jellyfishService.js | 2 +- lib/schema/schemaEnv.js | 13 ++++++++++++- platform_api_users.json | 1 - 4 files changed, 13 insertions(+), 16 deletions(-) delete mode 100644 platform_api_users.json diff --git a/env.js b/env.js index dc707c7d..99734ecd 100644 --- a/env.js +++ b/env.js @@ -89,19 +89,6 @@ module.exports = (function () { env.mongo = { connectionString: cs('data'), }; - - // users who are going to upload via platform during migration - var usersFile = fs.readFileSync(__dirname + '/platform_users.json'); - var platformUsers = JSON.parse(usersFile); - if (!Array.isArray(platformUsers)){ - throw new Error( - 'Must contain an array in platform_users.json' - ); - } - env.uploader = { - platformUsers: platformUsers.map(item => item.toLowerCase().trim()), - }; - return env; })(); diff --git a/lib/jellyfishService.js b/lib/jellyfishService.js index 090b5074..f13c4cf0 100644 --- a/lib/jellyfishService.js +++ b/lib/jellyfishService.js @@ -169,7 +169,7 @@ function createServer( var uploaderDestination = null; if (request.params.userId) { - if (serverConfig.uploader.platformUsers.includes(request.params.userId.trim().toLowerCase())) { + if (schemaEnv.platformUserIds.includes(request.params.userId.trim().toLowerCase())) { uploaderDestination = 'platform'; }else{ uploaderDestination = 'jellyfish'; diff --git a/lib/schema/schemaEnv.js b/lib/schema/schemaEnv.js index 752e0701..a90539f0 100644 --- a/lib/schema/schemaEnv.js +++ b/lib/schema/schemaEnv.js @@ -25,10 +25,21 @@ module.exports = (function () { // The version that we will accept upload requests from, will be in format of 0.200.0 // NOTE: This will probably be eliminated in favor of using schemaVersion above for the same purpose - schemaEnv.minimumUploaderVersion = config.fromEnvironment('MINIMUM_UPLOADER_VERSION', '2.53.0'); + schemaEnv.minimumUploaderVersion = config.fromEnvironment( + 'MINIMUM_UPLOADER_VERSION', + '2.53.0' + ); schemaEnv.authRealm = config.fromEnvironment('KEYCLOAK_AUTH_REALM', null); schemaEnv.authUrl = config.fromEnvironment('KEYCLOAK_AUTH_URL', null); + var usersStr = config.fromEnvironment('PLATFORM_UPLOAD_USERS', null); + var platformUsers = []; + if (usersStr) { + platformUsers = usersStr + .split(',') + .map((item) => item.toLowerCase().trim()); + } + schemaEnv.platformUserIds = platformUsers; return schemaEnv; })(); diff --git a/platform_api_users.json b/platform_api_users.json deleted file mode 100644 index fe51488c..00000000 --- a/platform_api_users.json +++ /dev/null @@ -1 +0,0 @@ -[] From 5342fe99ce22e682e3fd31ab00d78d44d9d4c5af Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 16 Oct 2024 09:04:49 +1300 Subject: [PATCH 20/27] update to match config changes --- lib/schema/schemaEnv.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/schema/schemaEnv.js b/lib/schema/schemaEnv.js index a90539f0..671999c6 100644 --- a/lib/schema/schemaEnv.js +++ b/lib/schema/schemaEnv.js @@ -32,7 +32,7 @@ module.exports = (function () { schemaEnv.authRealm = config.fromEnvironment('KEYCLOAK_AUTH_REALM', null); schemaEnv.authUrl = config.fromEnvironment('KEYCLOAK_AUTH_URL', null); - var usersStr = config.fromEnvironment('PLATFORM_UPLOAD_USERS', null); + var usersStr = config.fromEnvironment('UPLOADER_PLATFORM_USER_IDS', null); var platformUsers = []; if (usersStr) { platformUsers = usersStr From 4b9d7ea6c94aba49af05a2c47b9b049c0e410fb4 Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 8 Nov 2024 08:00:18 +1300 Subject: [PATCH 21/27] read platform user ids from encrypted file --- lib/misc.js | 42 ++++++++++++++++++++++--- lib/schema/qa3_user_ids.json | 1 + lib/schema/schemaEnv.js | 24 +++++++++------ test/schema/schemaEnvTest.js | 59 ++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 15 deletions(-) create mode 100644 lib/schema/qa3_user_ids.json create mode 100644 test/schema/schemaEnvTest.js diff --git a/lib/misc.js b/lib/misc.js index 5b5ecf4e..9c74d51b 100644 --- a/lib/misc.js +++ b/lib/misc.js @@ -1,15 +1,15 @@ /* * == BSD2 LICENSE == * Copyright (c) 2014, Tidepool Project - * + * * This program is free software; you can redistribute it and/or modify it under * the terms of the associated License, which is identical to the BSD 2-Clause * License as published by the Open Source Initiative at opensource.org. - * + * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the License for more details. - * + * * You should have received a copy of the License along with this program; if * not, you can obtain one from Tidepool Project at tidepool.org. * == BSD2 LICENSE == @@ -18,7 +18,7 @@ 'use strict'; var crypto = require('crypto'); - +var fs = require('fs'); var amoeba = require('amoeba'); var base32hex = amoeba.base32hex; var except = amoeba.except; @@ -45,7 +45,7 @@ var except = amoeba.except; * @param fields an array of values to be concatenated together into a unique string * @returns {string} the base32 encoded hash of the delimited-concatenation of the provided fields (also known as a "unique" id) */ -exports.generateId = function(fields) { +exports.generateId = function (fields) { var hasher = crypto.createHash('sha1'); for (var i = 0; i < fields.length; ++i) { @@ -65,3 +65,35 @@ exports.generateId = function(fields) { return base32hex.encodeBuffer(hasher.digest(), { paddingChar: '-' }); }; +exports.encryptArrayToFile = function (dataArray, filePath, env, serverSecret) { + var iv = `${env}-environment`.substring(0, 16); + var key = serverSecret.substring(0, 32); + var algorithm = 'aes-256-cbc'; + + var encryptedArray = dataArray.map((item) => { + var cipher = crypto.createCipheriv(algorithm, key, iv); + let encrypted = cipher.update(item, 'utf8', 'hex'); + encrypted += cipher.final('hex'); + return encrypted; + }); + fs.writeFileSync(filePath, JSON.stringify(encryptedArray)); +}; + +exports.decryptArrayFromFile = function (filePath, env, serverSecret) { + if (!fs.existsSync(filePath)) { + throw new Error(`Missing required file ${filePath}`); + } + var iv = `${env}-environment`.substring(0, 16); + var key = serverSecret.substring(0, 32); + var algorithm = 'aes-256-cbc'; + var encryptedArray = JSON.parse(fs.readFileSync(filePath, 'utf8')); + + var decryptedArray = encryptedArray.map((item) => { + var decipher = crypto.createDecipheriv(algorithm, key, iv); + let decrypted = decipher.update(item, 'hex', 'utf8'); + decrypted += decipher.final('utf8'); + return decrypted; + }); + + return decryptedArray; +}; diff --git a/lib/schema/qa3_user_ids.json b/lib/schema/qa3_user_ids.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/lib/schema/qa3_user_ids.json @@ -0,0 +1 @@ +[] diff --git a/lib/schema/schemaEnv.js b/lib/schema/schemaEnv.js index 671999c6..37475ac1 100644 --- a/lib/schema/schemaEnv.js +++ b/lib/schema/schemaEnv.js @@ -18,7 +18,19 @@ 'use strict'; var config = require('amoeba').config; -var schema = require('./schema.js'); +var misc = require('../misc.js'); + +function loadEnvironmentUserIds() { + const environment = config.fromEnvironment('POD_NAMESPACE', 'local'); + const serverSecret = config.fromEnvironment('TIDEPOOL_SERVER_SECRET'); + const environmentUserIdsFile = `${__dirname}/../${environment}_user_ids.json`; + + return misc.decryptArrayFromFile( + environmentUserIdsFile, + environment, + serverSecret + ); +} module.exports = (function () { var schemaEnv = {}; @@ -32,14 +44,6 @@ module.exports = (function () { schemaEnv.authRealm = config.fromEnvironment('KEYCLOAK_AUTH_REALM', null); schemaEnv.authUrl = config.fromEnvironment('KEYCLOAK_AUTH_URL', null); - var usersStr = config.fromEnvironment('UPLOADER_PLATFORM_USER_IDS', null); - var platformUsers = []; - if (usersStr) { - platformUsers = usersStr - .split(',') - .map((item) => item.toLowerCase().trim()); - } - schemaEnv.platformUserIds = platformUsers; - + schemaEnv.platformUserIds = loadEnvironmentUserIds(); return schemaEnv; })(); diff --git a/test/schema/schemaEnvTest.js b/test/schema/schemaEnvTest.js new file mode 100644 index 00000000..3f5c3fef --- /dev/null +++ b/test/schema/schemaEnvTest.js @@ -0,0 +1,59 @@ +/* + * == BSD2 LICENSE == + * Copyright (c) 2024, Tidepool Project + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the associated License, which is identical to the BSD 2-Clause + * License as published by the Open Source Initiative at opensource.org. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the License for more details. + * + * You should have received a copy of the License along with this program; if + * not, you can obtain one from Tidepool Project at tidepool.org. + * == BSD2 LICENSE == + */ + +/* global describe, before, beforeEach, it, after */ + +'use strict'; + +var expect = require('salinity').expect; +var misc = require('../../lib/misc.js'); +var fs = require('fs'); + +describe('schema/schemaEnv.js', function () { + const env = Object.assign({}, process.env); + const POD_NAMESPACE = 'test'; + const TIDEPOOL_SERVER_SECRET = 'some kinda secret goes here I guess'; + const USER_IDS = ['123', '456', 'ddbs', 'blahblah']; + const filePath = `${__dirname}/../../lib/${POD_NAMESPACE}_user_ids.json`; + + function setupFile(path, data, env, secret) { + misc.encryptArrayToFile(data, path, env, secret); + } + + function tearDownFile(path) { + fs.unlinkSync(path); + } + + before((done) => { + process.env.POD_NAMESPACE = POD_NAMESPACE; + process.env.TIDEPOOL_SERVER_SECRET = TIDEPOOL_SERVER_SECRET; + setupFile(filePath, USER_IDS, POD_NAMESPACE, TIDEPOOL_SERVER_SECRET); + done(); + }); + + after((done) => { + process.env = env; + tearDownFile(filePath); + done(); + }); + + it('file present gets stored platform users', function (done) { + var schemaEnv = require('../../lib/schema/schemaEnv.js'); + expect(schemaEnv.platformUserIds).to.deep.equal(USER_IDS); + done(); + }); +}); From ff284ffc67c14f69072e19ff6fdd27e5618df4a4 Mon Sep 17 00:00:00 2001 From: Darin Krauss Date: Thu, 7 Nov 2024 11:27:57 -0800 Subject: [PATCH 22/27] Resolve latest vulnerabilities (#205) --- package-lock.json | 434 +++++++++++++++++++++++++--------------------- package.json | 16 +- 2 files changed, 241 insertions(+), 209 deletions(-) diff --git a/package-lock.json b/package-lock.json index d264ad7b..6608786e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,30 +10,30 @@ "license": "BSD-2-Clause", "dependencies": { "@godaddy/terminus": "^4.5.0", - "amoeba": "1.2.3", + "amoeba": "1.2.4", "async": "3.2.4", "aws-sdk": "^2.814.0", - "body-parser": "1.20.2", + "body-parser": "1.20.3", "bunyan": "1.8.15", "compression": "1.7.4", "errorhandler": "^1.5.1", - "express": "^4.18.2", + "express": "^4.20.0", "lodash": "4.17.21", "moment": "^2.29.4", "mongodb": "^5.0.1", "mongodb-legacy": "^5.0.0", "rx": "4.1.0", "semver": "7.5.4", - "sundial": "1.7.4", - "tidepool-gatekeeper": "0.2.7", - "tidepool-seagull-client": "0.1.11", - "user-api-client": "0.5.2" + "sundial": "1.7.5", + "tidepool-gatekeeper": "0.2.8", + "tidepool-seagull-client": "0.1.13", + "user-api-client": "0.5.3" }, "devDependencies": { "jshint": "^2.12.0", "jshint-stylish": "2.2.1", "mocha": "^10.3.0", - "salinity": "0.0.10" + "salinity": "0.0.11" } }, "node_modules/@godaddy/terminus": { @@ -85,9 +85,9 @@ } }, "node_modules/amoeba": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/amoeba/-/amoeba-1.2.3.tgz", - "integrity": "sha512-C0TR6grWgasKG1ZsJkWRCY2cmvYTYoqP357YLAOXbxTlzWuyypT3EpAJyhHjLRYZZ1qVoK/Rij7/rl+5qFGswA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/amoeba/-/amoeba-1.2.4.tgz", + "integrity": "sha512-oV2q43T/VZ5RiT98qR3SxcOd0pyWyADgrjjXO1WzfjBS2x4Tyu2GQLYz/MrAMUD3QLIVGljN+k5Ai6fa0TVlZA==", "dependencies": { "axios": "^1.6.8", "bunyan": "1.8.15", @@ -184,9 +184,9 @@ } }, "node_modules/axios": { - "version": "1.6.8", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", - "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -228,9 +228,9 @@ } }, "node_modules/body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", "dependencies": { "bytes": "3.1.2", "content-type": "~1.0.5", @@ -240,7 +240,7 @@ "http-errors": "2.0.0", "iconv-lite": "0.4.24", "on-finished": "2.4.1", - "qs": "6.11.0", + "qs": "6.13.0", "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" @@ -260,6 +260,18 @@ "concat-map": "0.0.1" } }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", @@ -310,13 +322,18 @@ } }, "node_modules/call-bind": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", - "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", + "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==", "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.1", - "set-function-length": "^1.1.1" + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -501,9 +518,9 @@ } }, "node_modules/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", "engines": { "node": ">= 0.6" } @@ -546,16 +563,19 @@ } }, "node_modules/define-data-property": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", - "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", "dependencies": { - "get-intrinsic": "^1.2.1", - "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.0" + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/delayed-stream": { @@ -673,9 +693,9 @@ "dev": true }, "node_modules/encodeurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", - "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "engines": { "node": ">= 0.8" } @@ -698,6 +718,25 @@ "node": ">= 0.8" } }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -747,36 +786,36 @@ } }, "node_modules/express": { - "version": "4.19.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", - "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.2", + "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.2.0", + "finalhandler": "1.3.1", "fresh": "0.5.2", "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", + "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "2.4.1", "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", + "path-to-regexp": "0.1.10", "proxy-addr": "~2.0.7", - "qs": "6.11.0", + "qs": "6.13.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", + "send": "0.19.0", + "serve-static": "1.16.2", "setprototypeof": "1.2.0", "statuses": "2.0.1", "type-is": "~1.6.18", @@ -787,21 +826,25 @@ "node": ">= 0.10.0" } }, - "node_modules/express/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, "engines": { - "node": ">= 0.8" + "node": ">=8" } }, "node_modules/finalhandler": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", - "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", "dependencies": { "debug": "2.6.9", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "on-finished": "2.4.1", "parseurl": "~1.3.3", @@ -812,14 +855,6 @@ "node": ">= 0.8" } }, - "node_modules/finalhandler/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/flat": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", @@ -910,15 +945,19 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", - "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", "dependencies": { + "es-errors": "^1.3.0", "function-bind": "^1.1.2", "has-proto": "^1.0.1", "has-symbols": "^1.0.3", "hasown": "^2.0.0" }, + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -972,11 +1011,11 @@ } }, "node_modules/has-property-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", - "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", "dependencies": { - "get-intrinsic": "^1.2.2" + "es-define-property": "^1.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -1090,14 +1129,6 @@ "node": ">= 0.8" } }, - "node_modules/http-errors/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -1129,10 +1160,17 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, - "node_modules/ip": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.1.tgz", - "integrity": "sha512-lJUL9imLTNi1ZfXT+DU6rBBdbiKGBuay9B6xGSPVjUeQwaH1RIGqef8RZkUtHioLmSNpPR5M4HVKJGm1j8FWVQ==" + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } }, "node_modules/ipaddr.js": { "version": "1.9.1", @@ -1212,6 +1250,15 @@ "node": ">=0.10.0" } }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/is-plain-obj": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", @@ -1272,6 +1319,11 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + }, "node_modules/jshint": { "version": "2.13.6", "resolved": "https://registry.npmjs.org/jshint/-/jshint-2.13.6.tgz", @@ -1379,9 +1431,12 @@ "optional": true }, "node_modules/merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/methods": { "version": "1.1.2", @@ -1541,18 +1596,6 @@ "node": ">=8" } }, - "node_modules/mocha/node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, - "dependencies": { - "fill-range": "^7.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/mocha/node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -1654,18 +1697,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/mocha/node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/mocha/node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -1757,15 +1788,6 @@ "node": ">=8" } }, - "node_modules/mocha/node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, "node_modules/mocha/node_modules/log-symbols": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", @@ -1892,18 +1914,6 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, - "node_modules/mocha/node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, "node_modules/mocha/node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -2079,9 +2089,12 @@ } }, "node_modules/object-inspect": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", - "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", + "engines": { + "node": ">= 0.4" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -2171,9 +2184,9 @@ } }, "node_modules/path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz", + "integrity": "sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==" }, "node_modules/picomatch": { "version": "2.3.1", @@ -2225,11 +2238,11 @@ } }, "node_modules/qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "dependencies": { - "side-channel": "^1.0.4" + "side-channel": "^1.0.6" }, "engines": { "node": ">=0.6" @@ -2335,9 +2348,9 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/salinity": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/salinity/-/salinity-0.0.10.tgz", - "integrity": "sha512-22J7k3+N0ctICwyjm9gkFptl95lBD67YMm4rZBHx8Y/Mj4bsEy9WYg0fpBpOfj8opFkGbUiAleDm3KZR2N6Dvw==", + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/salinity/-/salinity-0.0.11.tgz", + "integrity": "sha512-oB1HYaDz8rfqdy4aHoyZK/xgWoXlQJ8iMNIFBbVsxi0legAn61cT1M4oFgBkJc16bjSiu/lBBI4fYzUSZeQ8Ag==", "dev": true, "dependencies": { "chai": "2.1.0", @@ -2372,9 +2385,9 @@ } }, "node_modules/send": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", "dependencies": { "debug": "2.6.9", "depd": "2.0.0", @@ -2394,19 +2407,19 @@ "node": ">= 0.8.0" } }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/send/node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, - "node_modules/send/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/serialize-javascript": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", @@ -2417,28 +2430,30 @@ } }, "node_modules/serve-static": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", - "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", "dependencies": { - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.18.0" + "send": "0.19.0" }, "engines": { "node": ">= 0.8.0" } }, "node_modules/set-function-length": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.1.1.tgz", - "integrity": "sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", "dependencies": { - "define-data-property": "^1.1.1", - "get-intrinsic": "^1.2.1", + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.0" + "has-property-descriptors": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -2450,13 +2465,17 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, "node_modules/side-channel": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", - "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", "dependencies": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -2497,15 +2516,15 @@ } }, "node_modules/socks": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", - "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", "dependencies": { - "ip": "^2.0.0", + "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" }, "engines": { - "node": ">= 10.13.0", + "node": ">= 10.0.0", "npm": ">= 3.0.0" } }, @@ -2518,6 +2537,19 @@ "memory-pager": "^1.0.2" } }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/stoppable": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", @@ -2564,9 +2596,9 @@ } }, "node_modules/sundial": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/sundial/-/sundial-1.7.4.tgz", - "integrity": "sha512-k6UBVVzJ+lARSy3vnE6VxYP/J86Cu4thwoAKGG1MTBg0o8NQsw8YZ+Zg3irw91kQF7wjysnK77/ie5vzYedSpA==", + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/sundial/-/sundial-1.7.5.tgz", + "integrity": "sha512-mgahMcr+r1acFkwO+yCa8CyQWEgidLDY59lJPbx52an89vndzjRePknFl9k2mr0kRRXMOj+jNt3XWaOwS/S7Og==", "dependencies": { "moment-timezone": "0.5.43" } @@ -2587,31 +2619,31 @@ "dev": true }, "node_modules/tidepool-gatekeeper": { - "version": "0.2.7", - "resolved": "https://registry.npmjs.org/tidepool-gatekeeper/-/tidepool-gatekeeper-0.2.7.tgz", - "integrity": "sha512-OazeCa+fGRTx6Y4m2QeleboNGi3+dT3iXrlHe9kjyg84tWl+I3fUvfcHT3EwISIdRw0n3Gix6BDLtq7JhdxNlQ==", + "version": "0.2.8", + "resolved": "https://registry.npmjs.org/tidepool-gatekeeper/-/tidepool-gatekeeper-0.2.8.tgz", + "integrity": "sha512-5NnlOhXyY/x0VgD/74TR5el00PpyMT5zb3P5HwK0dNvdR/bpD7mSyBamwGQnHtibx8uYjeDc8hJ9mRZzoshIzw==", "dependencies": { - "amoeba": "1.2.3" + "amoeba": "1.2.4" } }, "node_modules/tidepool-seagull-client": { - "version": "0.1.11", - "resolved": "https://registry.npmjs.org/tidepool-seagull-client/-/tidepool-seagull-client-0.1.11.tgz", - "integrity": "sha512-TdIYWhxHYVTRWcxhY9myK08bGj4c9LIyMj4Dmzcd6YT0dkVGniiZgcVz6xFhr3EfNN1Yyf7occ6NoiTsb+w2uw==", + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/tidepool-seagull-client/-/tidepool-seagull-client-0.1.13.tgz", + "integrity": "sha512-ecJMR0GfbUpEjQgfLxCRoHTAI6JD4Dqo4WJJqXRgRrlFafNSef0spWscBubak3qHtV82635koyfGyxlSF62YiQ==", "dependencies": { - "amoeba": "1.2.2" + "amoeba": "1.2.4" } }, - "node_modules/tidepool-seagull-client/node_modules/amoeba": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/amoeba/-/amoeba-1.2.2.tgz", - "integrity": "sha512-ki/Gt7yOzbRvWQkwEgqmc0/e+Znb4yjrSrPH8RFl84l4TTuEyZa2z4Q5PRRct23kCeMTSmLRFDun2t+2M6hdEw==", + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, "dependencies": { - "axios": "^1.6.8", - "bunyan": "1.8.15", - "kafkajs": "^1.14.0", - "lodash": "4.17.21", - "uuid": "^8.3.1" + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" } }, "node_modules/toidentifier": { @@ -2677,11 +2709,11 @@ "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" }, "node_modules/user-api-client": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/user-api-client/-/user-api-client-0.5.2.tgz", - "integrity": "sha512-oMvkbYk2d9votifBF0aosyIeyZhyM/1SKk/yxEsJyuUhp0/pVkHqFz53EqoKd7NfxnNn55xLB0Oh5EaG1b/NyQ==", + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/user-api-client/-/user-api-client-0.5.3.tgz", + "integrity": "sha512-xgayyOsoCtcMhbb70D+4qJT7pFR79oCxXwbFfdK7IJ33PHPRJ8Pdh4qg3zTwAQUBjXwUftxuP3dV+c1T/jj2Sg==", "dependencies": { - "amoeba": "1.2.3", + "amoeba": "1.2.4", "axios": "^1.6.8", "bunyan": "1.8.15", "lodash": "4.17.21" diff --git a/package.json b/package.json index e9688a1e..e2e794e2 100644 --- a/package.json +++ b/package.json @@ -23,30 +23,30 @@ "license": "BSD-2-Clause", "dependencies": { "@godaddy/terminus": "^4.5.0", - "amoeba": "1.2.3", + "amoeba": "1.2.4", "async": "3.2.4", "aws-sdk": "^2.814.0", - "body-parser": "1.20.2", + "body-parser": "1.20.3", "bunyan": "1.8.15", "compression": "1.7.4", "errorhandler": "^1.5.1", - "express": "^4.18.2", + "express": "^4.20.0", "lodash": "4.17.21", "moment": "^2.29.4", "mongodb": "^5.0.1", "mongodb-legacy": "^5.0.0", "rx": "4.1.0", "semver": "7.5.4", - "sundial": "1.7.4", - "tidepool-gatekeeper": "0.2.7", - "tidepool-seagull-client": "0.1.11", - "user-api-client": "0.5.2" + "sundial": "1.7.5", + "tidepool-gatekeeper": "0.2.8", + "tidepool-seagull-client": "0.1.13", + "user-api-client": "0.5.3" }, "devDependencies": { "jshint": "^2.12.0", "jshint-stylish": "2.2.1", "mocha": "^10.3.0", - "salinity": "0.0.10" + "salinity": "0.0.11" }, "overrides": { "serialize-javascript@>=6.0.0 <6.0.2": "6.0.2" From 3ef21b0fdf83e4e453f0a7d1c0a6013f966c74f5 Mon Sep 17 00:00:00 2001 From: Jamie Bate Date: Mon, 25 Nov 2024 07:48:26 +1300 Subject: [PATCH 23/27] userid hash for platform users (#206) * userid hash for platform users --- .gitignore | 3 ++ hash_platform_user_ids.sh | 13 +++++++ lib/jellyfishService.js | 3 +- lib/misc.js | 64 +++++++++++++++++++++------------- lib/schema/qa3_user_ids.json | 1 - lib/schema/schemaEnv.js | 16 ++++----- test/schema/schemaEnvTest.js | 50 ++++++++++++++++++-------- test/schema/user_ids/test.json | 1 + 8 files changed, 101 insertions(+), 50 deletions(-) create mode 100755 hash_platform_user_ids.sh delete mode 100644 lib/schema/qa3_user_ids.json create mode 100644 test/schema/user_ids/test.json diff --git a/.gitignore b/.gitignore index a9958daa..c18776a3 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,6 @@ artifact_docker.sh .vscode/ *.envrc *.env +/lib/platform_users/* +!/lib/platform_users/*_hashed.json +lib/platform_users/test_hashed.json diff --git a/hash_platform_user_ids.sh b/hash_platform_user_ids.sh new file mode 100755 index 00000000..62e684ff --- /dev/null +++ b/hash_platform_user_ids.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh -euo pipefail + +## e.g. qa3 +ENV=$1 +USER_IDS_FILE=$2 +## e.g. "qa3 hash" +SALT_ITEM_NAME=$3 + +SALT=$(op item get "$SALT_ITEM_NAME" --account tidepool.1password.com --fields label=credential --format json | jq -r '.value') + +USER_IDS=$(jq -r '.[]' "$USER_IDS_FILE") + +node -e "console.log(require('./lib/misc.js').addUserIdsToHashedEnvironmentFile('$ENV', '$SALT', '$USER_IDS'))" diff --git a/lib/jellyfishService.js b/lib/jellyfishService.js index f13c4cf0..8747caa8 100644 --- a/lib/jellyfishService.js +++ b/lib/jellyfishService.js @@ -27,6 +27,7 @@ var express = require('express'); var compression = require('compression'); var bodyparser = require('body-parser'); var util = require('util'); +var misc = require('./misc.js'); var _ = require('lodash'); var jellyfishService = function ( @@ -169,7 +170,7 @@ function createServer( var uploaderDestination = null; if (request.params.userId) { - if (schemaEnv.platformUserIds.includes(request.params.userId.trim().toLowerCase())) { + if (schemaEnv.isPlatformUserId(request.params.userId)) { uploaderDestination = 'platform'; }else{ uploaderDestination = 'jellyfish'; diff --git a/lib/misc.js b/lib/misc.js index 9c74d51b..2485987e 100644 --- a/lib/misc.js +++ b/lib/misc.js @@ -65,35 +65,51 @@ exports.generateId = function (fields) { return base32hex.encodeBuffer(hasher.digest(), { paddingChar: '-' }); }; -exports.encryptArrayToFile = function (dataArray, filePath, env, serverSecret) { - var iv = `${env}-environment`.substring(0, 16); - var key = serverSecret.substring(0, 32); - var algorithm = 'aes-256-cbc'; +var userIdsPath = function (env) { + return { + hashedPath: `${__dirname}/platform_users/${env}_hashed.json`, + }; +}; - var encryptedArray = dataArray.map((item) => { - var cipher = crypto.createCipheriv(algorithm, key, iv); - let encrypted = cipher.update(item, 'utf8', 'hex'); - encrypted += cipher.final('hex'); - return encrypted; - }); - fs.writeFileSync(filePath, JSON.stringify(encryptedArray)); +var hashItem = function (item, salt) { + if (!item) { + throw new Error('Missing required userid'); + } + if (!salt) { + throw new Error('Missing required salt'); + } + return crypto.createHmac('sha256', salt).update(item).digest('hex'); }; -exports.decryptArrayFromFile = function (filePath, env, serverSecret) { - if (!fs.existsSync(filePath)) { - throw new Error(`Missing required file ${filePath}`); +exports.hashUserId = hashItem; + +// Function to encrypt the contents of a JSON file +exports.addUserIdsToHashedEnvironmentFile = function (env, salt, unhashedUserIds) { + const filePaths = userIdsPath(env); + + let hashedArray = []; + if (fs.existsSync(filePaths.hashedPath)) { + const data = fs.readFileSync(filePaths.hashedPath, 'utf8'); + hashedArray = JSON.parse(data); } - var iv = `${env}-environment`.substring(0, 16); - var key = serverSecret.substring(0, 32); - var algorithm = 'aes-256-cbc'; - var encryptedArray = JSON.parse(fs.readFileSync(filePath, 'utf8')); - var decryptedArray = encryptedArray.map((item) => { - var decipher = crypto.createDecipheriv(algorithm, key, iv); - let decrypted = decipher.update(item, 'hex', 'utf8'); - decrypted += decipher.final('utf8'); - return decrypted; + unhashedUserIds.forEach((userid) => { + const hashedId = hashItem(userid, salt); + if (!hashedArray.includes(hashedId)) { + hashedArray.push(hashItem(userid, salt)); + } }); - return decryptedArray; + fs.writeFileSync(filePaths.hashedPath, JSON.stringify(hashedArray)); + console.log(`Hashed user IDs saved as ${filePaths.hashedPath}`); +}; + +exports.loadHashedPlatformUserId = function (env) { + const filePaths = userIdsPath(env); + if (!fs.existsSync(filePaths.hashedPath)) { + throw new Error(`Missing required file ${filePaths.hashedPath}`); + } + const data = fs.readFileSync(filePaths.hashedPath, 'utf8'); + const jsonArray = JSON.parse(data); + return jsonArray; }; diff --git a/lib/schema/qa3_user_ids.json b/lib/schema/qa3_user_ids.json deleted file mode 100644 index fe51488c..00000000 --- a/lib/schema/qa3_user_ids.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/lib/schema/schemaEnv.js b/lib/schema/schemaEnv.js index 37475ac1..f2b7a96a 100644 --- a/lib/schema/schemaEnv.js +++ b/lib/schema/schemaEnv.js @@ -20,16 +20,11 @@ var config = require('amoeba').config; var misc = require('../misc.js'); -function loadEnvironmentUserIds() { +function usePlatform(userId) { const environment = config.fromEnvironment('POD_NAMESPACE', 'local'); - const serverSecret = config.fromEnvironment('TIDEPOOL_SERVER_SECRET'); - const environmentUserIdsFile = `${__dirname}/../${environment}_user_ids.json`; - - return misc.decryptArrayFromFile( - environmentUserIdsFile, - environment, - serverSecret - ); + const salt = config.fromEnvironment('USER_ID_SALT', null); + const platformUsers = misc.loadHashedPlatformUserId(environment); + return platformUsers.includes(misc.hashUserId(userId, salt)); } module.exports = (function () { @@ -44,6 +39,7 @@ module.exports = (function () { schemaEnv.authRealm = config.fromEnvironment('KEYCLOAK_AUTH_REALM', null); schemaEnv.authUrl = config.fromEnvironment('KEYCLOAK_AUTH_URL', null); - schemaEnv.platformUserIds = loadEnvironmentUserIds(); + schemaEnv.isPlatformUserId = usePlatform; + return schemaEnv; })(); diff --git a/test/schema/schemaEnvTest.js b/test/schema/schemaEnvTest.js index 3f5c3fef..43dbb9aa 100644 --- a/test/schema/schemaEnvTest.js +++ b/test/schema/schemaEnvTest.js @@ -26,34 +26,56 @@ var fs = require('fs'); describe('schema/schemaEnv.js', function () { const env = Object.assign({}, process.env); const POD_NAMESPACE = 'test'; - const TIDEPOOL_SERVER_SECRET = 'some kinda secret goes here I guess'; + const USER_ID_SALT = 'some kinda salt goes here I guess'; const USER_IDS = ['123', '456', 'ddbs', 'blahblah']; - const filePath = `${__dirname}/../../lib/${POD_NAMESPACE}_user_ids.json`; + const envJSONHashedFile = `${__dirname}/../../lib/platform_users/${POD_NAMESPACE}_hashed.json`; - function setupFile(path, data, env, secret) { - misc.encryptArrayToFile(data, path, env, secret); + function setupFile(env, salt) { + misc.addUserIdsToHashedEnvironmentFile(env, salt, USER_IDS); } - function tearDownFile(path) { - fs.unlinkSync(path); + function tearDownFile() { + fs.unlinkSync(envJSONHashedFile); + process.env = env; } - before((done) => { + it('will throw an error if the environment specific user_ids file is not present', function (done) { + process.env.POD_NAMESPACE = 'not_test'; + process.env.USER_ID_SALT = USER_ID_SALT; + var schemaEnv = require('../../lib/schema/schemaEnv.js'); + expect(schemaEnv.isPlatformUserId).to.throw; + done(); + }); + + it('will throw an error if the salt is not set', function (done) { process.env.POD_NAMESPACE = POD_NAMESPACE; - process.env.TIDEPOOL_SERVER_SECRET = TIDEPOOL_SERVER_SECRET; - setupFile(filePath, USER_IDS, POD_NAMESPACE, TIDEPOOL_SERVER_SECRET); + process.env.USER_ID_SALT = null; + setupFile(POD_NAMESPACE, USER_ID_SALT); + + var schemaEnv = require('../../lib/schema/schemaEnv.js'); + expect(schemaEnv.isPlatformUserId('not-a-user')).to.throw; + tearDownFile(); done(); }); - after((done) => { - process.env = env; - tearDownFile(filePath); + it('will return false when the user is not a platform user', function (done) { + process.env.POD_NAMESPACE = POD_NAMESPACE; + process.env.USER_ID_SALT = USER_ID_SALT; + setupFile(POD_NAMESPACE, USER_ID_SALT); + var schemaEnv = require('../../lib/schema/schemaEnv.js'); + expect(schemaEnv.isPlatformUserId('not-a-user')).to.be.false; + tearDownFile(); done(); }); - it('file present gets stored platform users', function (done) { + it('will return true when the user is a platform user', function (done) { + process.env.POD_NAMESPACE = POD_NAMESPACE; + process.env.USER_ID_SALT = USER_ID_SALT; + setupFile(POD_NAMESPACE, USER_ID_SALT); var schemaEnv = require('../../lib/schema/schemaEnv.js'); - expect(schemaEnv.platformUserIds).to.deep.equal(USER_IDS); + const randomUserId = Math.floor(Math.random() * USER_IDS.length); + expect(schemaEnv.isPlatformUserId(USER_IDS[randomUserId])).to.be.true; + tearDownFile(); done(); }); }); diff --git a/test/schema/user_ids/test.json b/test/schema/user_ids/test.json new file mode 100644 index 00000000..83ab174b --- /dev/null +++ b/test/schema/user_ids/test.json @@ -0,0 +1 @@ +["123", "456", "ddbs", "blahblah"] From df3c5a40051e7f287a635d0d48480f690d33c259 Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 18 Dec 2024 16:49:23 +1300 Subject: [PATCH 24/27] initial load of QA3 test users --- hash_platform_user_ids.sh | 6 +++--- lib/platform_users/qa3_hashed.json | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 lib/platform_users/qa3_hashed.json diff --git a/hash_platform_user_ids.sh b/hash_platform_user_ids.sh index 62e684ff..e7818458 100755 --- a/hash_platform_user_ids.sh +++ b/hash_platform_user_ids.sh @@ -6,8 +6,8 @@ USER_IDS_FILE=$2 ## e.g. "qa3 hash" SALT_ITEM_NAME=$3 -SALT=$(op item get "$SALT_ITEM_NAME" --account tidepool.1password.com --fields label=credential --format json | jq -r '.value') +SALT=$(op item get "$SALT_ITEM_NAME" --account tidepool.1password.com --fields label=password --format json | jq -r '.value') -USER_IDS=$(jq -r '.[]' "$USER_IDS_FILE") +USER_IDS=$(jq -r '.' "$USER_IDS_FILE") -node -e "console.log(require('./lib/misc.js').addUserIdsToHashedEnvironmentFile('$ENV', '$SALT', '$USER_IDS'))" +node -e "console.log(require('./lib/misc.js').addUserIdsToHashedEnvironmentFile('$ENV', '$SALT', $USER_IDS))" diff --git a/lib/platform_users/qa3_hashed.json b/lib/platform_users/qa3_hashed.json new file mode 100644 index 00000000..46c51af4 --- /dev/null +++ b/lib/platform_users/qa3_hashed.json @@ -0,0 +1 @@ +["9ff7fc1b8d4ae42c4ce7bbbb80dbc8a29b7e47e93715d22a2178c07593c29f0c","5019ca9b437d9acac34a8f386deae4fed5441ef6bb908dc126c4a107cdf5a019","d3acd70503f572078b87ad910ba81f1e26caefb31dccc8a2a6ac9eb24efc8267","ad076f24012a0acb856fee36c6caa8245547cbd259195a19353a2c5ce9efbbe2","f08d6c73d42676c4ec4d7fdb2298a3a18f19e789a6c61eabb2399c7d6b3bfe3b","a8dd3017610c684814610f0e8139bb5681097796650c65ee92b13594ccccccc6","e2cb15731fe324e1b8204d1e0d7335fe94b14d42287ecc6165cc1aa1a8369b1a","65938c5376b3670b36b9df2957b397df81df089c5a119e812e0d75e3970ec751","69b66943189f860aec93f7305a807315b9de3e5f6dda5c98f474d0cc2d27fd80","0424ea4ced21967f72416e1e838469444f370c1ab348b59d9e9b477c357c20e6","b6b99c78eddd70c27c189d3b60f87c2d85bfa211a5e2d4ba168c9b0aad111d4e","ed556c0a41d503c2fcfd85f7be89360a3af7d889e4bfa597c767b7896c9347af","b29fe0fa299971da2d3b404ca705173b889fde0adf9e801d7444d4e2aa2f76ba","8c23f82b044d5d7fb5e8612b882b90c6604522da31c574808eb130120b3a5cf2","e967a53fd0553ce179e62ba18e1051ad0b269457ed8dba723e006e74e95b7f5f","600c0413f064a0a21f6c93aea6ccd6a379ee9413c3e81d8b0fe2e713b2b0e62e","c554559f0b0d47c291720a8fcd4422d2f746dae5202e96fe2590ac6d42f373fa","479dcce8acfcfa67297d3e6a3d7db9ab1d2b932a5127217eb51cdad543b779b4","bc4524ce2b0ccd2c46dcf02db8def969c4f7e9bdbe43d24809d23ac16ca9a4da"] \ No newline at end of file From 05de0342a28764dc66b5a0803f7e63962c5bde5b Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 18 Dec 2024 17:04:16 +1300 Subject: [PATCH 25/27] empty user files per environment --- lib/platform_users/dev1_hashed.json | 1 + lib/platform_users/external_hashed.json | 1 + lib/platform_users/qa1_hashed.json | 1 + lib/platform_users/qa2_hashed.json | 1 + lib/platform_users/qa4_hashed.json | 1 + lib/platform_users/qa5_hashed.json | 1 + lib/platform_users/tidepool-prod_hashed.json | 1 + 7 files changed, 7 insertions(+) create mode 100644 lib/platform_users/dev1_hashed.json create mode 100644 lib/platform_users/external_hashed.json create mode 100644 lib/platform_users/qa1_hashed.json create mode 100644 lib/platform_users/qa2_hashed.json create mode 100644 lib/platform_users/qa4_hashed.json create mode 100644 lib/platform_users/qa5_hashed.json create mode 100644 lib/platform_users/tidepool-prod_hashed.json diff --git a/lib/platform_users/dev1_hashed.json b/lib/platform_users/dev1_hashed.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/lib/platform_users/dev1_hashed.json @@ -0,0 +1 @@ +[] diff --git a/lib/platform_users/external_hashed.json b/lib/platform_users/external_hashed.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/lib/platform_users/external_hashed.json @@ -0,0 +1 @@ +[] diff --git a/lib/platform_users/qa1_hashed.json b/lib/platform_users/qa1_hashed.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/lib/platform_users/qa1_hashed.json @@ -0,0 +1 @@ +[] diff --git a/lib/platform_users/qa2_hashed.json b/lib/platform_users/qa2_hashed.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/lib/platform_users/qa2_hashed.json @@ -0,0 +1 @@ +[] diff --git a/lib/platform_users/qa4_hashed.json b/lib/platform_users/qa4_hashed.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/lib/platform_users/qa4_hashed.json @@ -0,0 +1 @@ +[] diff --git a/lib/platform_users/qa5_hashed.json b/lib/platform_users/qa5_hashed.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/lib/platform_users/qa5_hashed.json @@ -0,0 +1 @@ +[] diff --git a/lib/platform_users/tidepool-prod_hashed.json b/lib/platform_users/tidepool-prod_hashed.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/lib/platform_users/tidepool-prod_hashed.json @@ -0,0 +1 @@ +[] From 5aef248ff8c03567045b2aba785e4093123b0032 Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 20 Dec 2024 16:30:12 +1300 Subject: [PATCH 26/27] add personal qa user id --- lib/platform_users/qa3_hashed.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/platform_users/qa3_hashed.json b/lib/platform_users/qa3_hashed.json index 46c51af4..316af005 100644 --- a/lib/platform_users/qa3_hashed.json +++ b/lib/platform_users/qa3_hashed.json @@ -1 +1 @@ -["9ff7fc1b8d4ae42c4ce7bbbb80dbc8a29b7e47e93715d22a2178c07593c29f0c","5019ca9b437d9acac34a8f386deae4fed5441ef6bb908dc126c4a107cdf5a019","d3acd70503f572078b87ad910ba81f1e26caefb31dccc8a2a6ac9eb24efc8267","ad076f24012a0acb856fee36c6caa8245547cbd259195a19353a2c5ce9efbbe2","f08d6c73d42676c4ec4d7fdb2298a3a18f19e789a6c61eabb2399c7d6b3bfe3b","a8dd3017610c684814610f0e8139bb5681097796650c65ee92b13594ccccccc6","e2cb15731fe324e1b8204d1e0d7335fe94b14d42287ecc6165cc1aa1a8369b1a","65938c5376b3670b36b9df2957b397df81df089c5a119e812e0d75e3970ec751","69b66943189f860aec93f7305a807315b9de3e5f6dda5c98f474d0cc2d27fd80","0424ea4ced21967f72416e1e838469444f370c1ab348b59d9e9b477c357c20e6","b6b99c78eddd70c27c189d3b60f87c2d85bfa211a5e2d4ba168c9b0aad111d4e","ed556c0a41d503c2fcfd85f7be89360a3af7d889e4bfa597c767b7896c9347af","b29fe0fa299971da2d3b404ca705173b889fde0adf9e801d7444d4e2aa2f76ba","8c23f82b044d5d7fb5e8612b882b90c6604522da31c574808eb130120b3a5cf2","e967a53fd0553ce179e62ba18e1051ad0b269457ed8dba723e006e74e95b7f5f","600c0413f064a0a21f6c93aea6ccd6a379ee9413c3e81d8b0fe2e713b2b0e62e","c554559f0b0d47c291720a8fcd4422d2f746dae5202e96fe2590ac6d42f373fa","479dcce8acfcfa67297d3e6a3d7db9ab1d2b932a5127217eb51cdad543b779b4","bc4524ce2b0ccd2c46dcf02db8def969c4f7e9bdbe43d24809d23ac16ca9a4da"] \ No newline at end of file +["9ff7fc1b8d4ae42c4ce7bbbb80dbc8a29b7e47e93715d22a2178c07593c29f0c","5019ca9b437d9acac34a8f386deae4fed5441ef6bb908dc126c4a107cdf5a019","d3acd70503f572078b87ad910ba81f1e26caefb31dccc8a2a6ac9eb24efc8267","ad076f24012a0acb856fee36c6caa8245547cbd259195a19353a2c5ce9efbbe2","f08d6c73d42676c4ec4d7fdb2298a3a18f19e789a6c61eabb2399c7d6b3bfe3b","a8dd3017610c684814610f0e8139bb5681097796650c65ee92b13594ccccccc6","e2cb15731fe324e1b8204d1e0d7335fe94b14d42287ecc6165cc1aa1a8369b1a","65938c5376b3670b36b9df2957b397df81df089c5a119e812e0d75e3970ec751","69b66943189f860aec93f7305a807315b9de3e5f6dda5c98f474d0cc2d27fd80","0424ea4ced21967f72416e1e838469444f370c1ab348b59d9e9b477c357c20e6","b6b99c78eddd70c27c189d3b60f87c2d85bfa211a5e2d4ba168c9b0aad111d4e","ed556c0a41d503c2fcfd85f7be89360a3af7d889e4bfa597c767b7896c9347af","b29fe0fa299971da2d3b404ca705173b889fde0adf9e801d7444d4e2aa2f76ba","8c23f82b044d5d7fb5e8612b882b90c6604522da31c574808eb130120b3a5cf2","e967a53fd0553ce179e62ba18e1051ad0b269457ed8dba723e006e74e95b7f5f","600c0413f064a0a21f6c93aea6ccd6a379ee9413c3e81d8b0fe2e713b2b0e62e","c554559f0b0d47c291720a8fcd4422d2f746dae5202e96fe2590ac6d42f373fa","479dcce8acfcfa67297d3e6a3d7db9ab1d2b932a5127217eb51cdad543b779b4","bc4524ce2b0ccd2c46dcf02db8def969c4f7e9bdbe43d24809d23ac16ca9a4da","ddb15d41da017ca4aeaa1a8d91241f0120f3e7e08e1301326278feb9b9bcd7ba"] \ No newline at end of file From d893ea381e99ca5d160344f5fd30a69b9ed1f77f Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 20 Dec 2024 16:33:32 +1300 Subject: [PATCH 27/27] format list --- lib/platform_users/qa3_hashed.json | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/platform_users/qa3_hashed.json b/lib/platform_users/qa3_hashed.json index 316af005..7033ce72 100644 --- a/lib/platform_users/qa3_hashed.json +++ b/lib/platform_users/qa3_hashed.json @@ -1 +1,22 @@ -["9ff7fc1b8d4ae42c4ce7bbbb80dbc8a29b7e47e93715d22a2178c07593c29f0c","5019ca9b437d9acac34a8f386deae4fed5441ef6bb908dc126c4a107cdf5a019","d3acd70503f572078b87ad910ba81f1e26caefb31dccc8a2a6ac9eb24efc8267","ad076f24012a0acb856fee36c6caa8245547cbd259195a19353a2c5ce9efbbe2","f08d6c73d42676c4ec4d7fdb2298a3a18f19e789a6c61eabb2399c7d6b3bfe3b","a8dd3017610c684814610f0e8139bb5681097796650c65ee92b13594ccccccc6","e2cb15731fe324e1b8204d1e0d7335fe94b14d42287ecc6165cc1aa1a8369b1a","65938c5376b3670b36b9df2957b397df81df089c5a119e812e0d75e3970ec751","69b66943189f860aec93f7305a807315b9de3e5f6dda5c98f474d0cc2d27fd80","0424ea4ced21967f72416e1e838469444f370c1ab348b59d9e9b477c357c20e6","b6b99c78eddd70c27c189d3b60f87c2d85bfa211a5e2d4ba168c9b0aad111d4e","ed556c0a41d503c2fcfd85f7be89360a3af7d889e4bfa597c767b7896c9347af","b29fe0fa299971da2d3b404ca705173b889fde0adf9e801d7444d4e2aa2f76ba","8c23f82b044d5d7fb5e8612b882b90c6604522da31c574808eb130120b3a5cf2","e967a53fd0553ce179e62ba18e1051ad0b269457ed8dba723e006e74e95b7f5f","600c0413f064a0a21f6c93aea6ccd6a379ee9413c3e81d8b0fe2e713b2b0e62e","c554559f0b0d47c291720a8fcd4422d2f746dae5202e96fe2590ac6d42f373fa","479dcce8acfcfa67297d3e6a3d7db9ab1d2b932a5127217eb51cdad543b779b4","bc4524ce2b0ccd2c46dcf02db8def969c4f7e9bdbe43d24809d23ac16ca9a4da","ddb15d41da017ca4aeaa1a8d91241f0120f3e7e08e1301326278feb9b9bcd7ba"] \ No newline at end of file +[ + "9ff7fc1b8d4ae42c4ce7bbbb80dbc8a29b7e47e93715d22a2178c07593c29f0c", + "5019ca9b437d9acac34a8f386deae4fed5441ef6bb908dc126c4a107cdf5a019", + "d3acd70503f572078b87ad910ba81f1e26caefb31dccc8a2a6ac9eb24efc8267", + "ad076f24012a0acb856fee36c6caa8245547cbd259195a19353a2c5ce9efbbe2", + "f08d6c73d42676c4ec4d7fdb2298a3a18f19e789a6c61eabb2399c7d6b3bfe3b", + "a8dd3017610c684814610f0e8139bb5681097796650c65ee92b13594ccccccc6", + "e2cb15731fe324e1b8204d1e0d7335fe94b14d42287ecc6165cc1aa1a8369b1a", + "65938c5376b3670b36b9df2957b397df81df089c5a119e812e0d75e3970ec751", + "69b66943189f860aec93f7305a807315b9de3e5f6dda5c98f474d0cc2d27fd80", + "0424ea4ced21967f72416e1e838469444f370c1ab348b59d9e9b477c357c20e6", + "b6b99c78eddd70c27c189d3b60f87c2d85bfa211a5e2d4ba168c9b0aad111d4e", + "ed556c0a41d503c2fcfd85f7be89360a3af7d889e4bfa597c767b7896c9347af", + "b29fe0fa299971da2d3b404ca705173b889fde0adf9e801d7444d4e2aa2f76ba", + "8c23f82b044d5d7fb5e8612b882b90c6604522da31c574808eb130120b3a5cf2", + "e967a53fd0553ce179e62ba18e1051ad0b269457ed8dba723e006e74e95b7f5f", + "600c0413f064a0a21f6c93aea6ccd6a379ee9413c3e81d8b0fe2e713b2b0e62e", + "c554559f0b0d47c291720a8fcd4422d2f746dae5202e96fe2590ac6d42f373fa", + "479dcce8acfcfa67297d3e6a3d7db9ab1d2b932a5127217eb51cdad543b779b4", + "bc4524ce2b0ccd2c46dcf02db8def969c4f7e9bdbe43d24809d23ac16ca9a4da", + "ddb15d41da017ca4aeaa1a8d91241f0120f3e7e08e1301326278feb9b9bcd7ba" +]