Merge pull request #414 from bitfinexcom/staging

Release version to master

package.json

@@ -1,6 +1,6 @@
 {
   "name": "bfx-reports-framework",
-  "version": "4.19.0",
+  "version": "4.20.0",
   "description": "Bitfinex reports framework",
   "main": "worker.js",
   "license": "Apache-2.0",

test/test-cases/api-sync-mode-sqlite-test-cases.js

@@ -612,6 +612,26 @@ module.exports = (
     assert.isOk(Number.isInteger(res.body.result.lastSyncMts))
   })
 
+  it('it should be successfully performed by the getLastFinishedSyncMts method, lastSyncMts is integer', async function () {
+    this.timeout(5000)
+
+    const res = await agent
+      .post(`${basePath}/json-rpc`)
+      .type('json')
+      .send({
+        auth,
+        method: 'getLastFinishedSyncMts',
+        id: 5
+      })
+      .expect('Content-Type', /json/)
+      .expect(200)
+
+    assert.isObject(res.body)
+    assert.propertyVal(res.body, 'id', 5)
+    assert.isObject(res.body.result)
+    assert.isOk(Number.isInteger(res.body.result.lastSyncMts))
+  })
+
   it('it should be successfully performed by the haveCollsBeenSyncedAtLeastOnce method, returns true', async function () {
     this.timeout(60000)
 

workers/loc.api/service.report.framework.js

@@ -104,6 +104,17 @@ class FrameworkReportService extends ReportService {
     }, 'isStagingBfxApi', args, cb)
   }
 
+  getLastFinishedSyncMts (space, args, cb) {
+    return this._privResponder(async () => {
+      const lastFinishedSyncQueueJob = await this._dao
+        .getLastFinishedSyncQueueJob(args?.auth?._id)
+
+      return {
+        lastSyncMts: lastFinishedSyncQueueJob?.updatedAt ?? null
+      }
+    }, 'getLastFinishedSyncMts', args, cb)
+  }
+
   signUp (space, args, cb) {
     return this._responder(() => {
       return this._authenticator.signUp(args)

workers/loc.api/sync/dao/dao.better.sqlite.js

@@ -40,12 +40,7 @@ const {
   RemoveElemsLeaveLastNRecordsError
 } = require('../../errors')
 
-const {
-  CONSTR_FIELD_NAME,
-  TRIGGER_FIELD_NAME,
-  INDEX_FIELD_NAME,
-  UNIQUE_INDEX_FIELD_NAME
-} = require('../schema/models/model/db.service.field.names')
+const Model = require('../schema/models/model')
 const ALLOWED_COLLS = require('../schema/allowed.colls')
 const SYNC_QUEUE_STATES = require('../sync.queue/sync.queue.states')
 const DB_WORKER_ACTIONS = require(
@@ -221,14 +216,7 @@ class BetterSqliteDAO extends DAO {
   }
 
   _createTablesIfNotExists (opts = {}) {
-    const models = this._getModelsMap({
-      models: opts?.models,
-      omittedFields: [
-        TRIGGER_FIELD_NAME,
-        INDEX_FIELD_NAME,
-        UNIQUE_INDEX_FIELD_NAME
-      ]
-    })
+    const models = opts?.models ?? this._getModelsMap()
     const sql = getTableCreationQuery(models, opts)
 
     return this.query({
@@ -239,10 +227,7 @@ class BetterSqliteDAO extends DAO {
   }
 
   _createTriggerIfNotExists (opts = {}) {
-    const models = this._getModelsMap({
-      models: opts?.models,
-      omittedFields: []
-    })
+    const models = opts?.models ?? this._getModelsMap()
     const sql = getTriggerCreationQuery(models, opts)
 
     return this.query({
@@ -253,10 +238,7 @@ class BetterSqliteDAO extends DAO {
   }
 
   _createIndexisIfNotExists (opts = {}) {
-    const models = this._getModelsMap({
-      models: opts?.models,
-      omittedFields: []
-    })
+    const models = opts?.models ?? this._getModelsMap()
     const sql = getIndexCreationQuery(models, opts)
 
     return this.query({
@@ -488,16 +470,6 @@ class BetterSqliteDAO extends DAO {
       return false
     }
 
-    const modelsMap = this._getModelsMap({
-      omittedFields: [
-        '_id',
-        CONSTR_FIELD_NAME,
-        TRIGGER_FIELD_NAME,
-        INDEX_FIELD_NAME,
-        UNIQUE_INDEX_FIELD_NAME
-      ]
-    })
-
     await this._beginTrans(async () => {
       const tableNames = await this.getTablesNames({ doNotQueueQuery })
       const filteredTempTableNames = tableNames.filter((name) => (
@@ -508,12 +480,16 @@ class BetterSqliteDAO extends DAO {
 
       for (const tempName of filteredTempTableNames) {
         const name = tempName.replace(namePrefix, '')
-        const model = modelsMap.get(name)
-        const projection = Object.keys(model).join(', ')
+        const model = this._getModelOf(name)
 
         if (!model) {
           continue
         }
+
+        const projection = model.getModelFieldKeys()
+          .filter((fieldName) => fieldName !== Model.UID_FIELD_NAME)
+          .join(', ')
+
         if (isStrictEqual) {
           sqlArr.push(`DELETE FROM ${name}`)
         }

workers/loc.api/sync/dao/dao.js

@@ -25,8 +25,12 @@ class DAO {
     this.processMessageManagerFactory = processMessageManagerFactory
   }
 
-  _getModelsMap (params) {
-    return this.syncSchema.getModelsMap(params)
+  _getModelsMap () {
+    return this.syncSchema.getModelsMap()
+  }
+
+  _getModelOf (tableName) {
+    return this.syncSchema.getModelOf(tableName)
   }
 
   _getMethodCollMap (params) {

workers/loc.api/sync/dao/helpers/find-in-coll-by/get-query.js

@@ -25,7 +25,10 @@ module.exports = (args, methodColl, opts) => {
     filter
   } = getFilterParams(args, methodColl, { isPublic })
 
-  const _model = { ...model, ...additionalModel }
+  const modelFields = {
+    ...model.getModelFields(),
+    ...additionalModel
+  }
   const exclude = isPublic ? ['_id', 'user_id'] : ['_id']
 
   const {
@@ -46,7 +49,7 @@ module.exports = (args, methodColl, opts) => {
   } = getGroupQuery(methodColl)
   const { subQuery, subQueryValues } = getSubQuery(methodColl)
   const projection = getProjectionQuery(
-    _model,
+    modelFields,
     exclude,
     isExcludePrivate
   )

workers/loc.api/sync/dao/helpers/get-index-creation-query.js

@@ -1,10 +1,5 @@
 'use strict'
 
-const {
-  INDEX_FIELD_NAME,
-  UNIQUE_INDEX_FIELD_NAME
-} = require('../../schema/models/model/db.service.field.names')
-
 const _getIndexQuery = (
   name,
   fields = [],
@@ -67,18 +62,20 @@ const _getIndexQueryFromModel = (
     shouldNotAddIfNotExistsStm
   } = opts ?? {}
 
+  const modelUniqueIndexies = model.getUniqueIndexies()
+  const modelIndexies = model.getIndexies()
   const uniqueIndexFields = (
-    model[UNIQUE_INDEX_FIELD_NAME] &&
-    typeof model[UNIQUE_INDEX_FIELD_NAME] === 'string'
+    modelUniqueIndexies &&
+    typeof modelUniqueIndexies === 'string'
   )
-    ? model[UNIQUE_INDEX_FIELD_NAME].split(' ')
-    : model[UNIQUE_INDEX_FIELD_NAME]
+    ? modelUniqueIndexies.split(' ')
+    : modelUniqueIndexies
   const indexFields = (
-    model[INDEX_FIELD_NAME] &&
-    typeof model[INDEX_FIELD_NAME] === 'string'
+    modelIndexies &&
+    typeof modelIndexies === 'string'
   )
-    ? model[INDEX_FIELD_NAME].split(' ')
-    : model[INDEX_FIELD_NAME]
+    ? modelIndexies.split(' ')
+    : modelIndexies
 
   const uniqueIndexiesArr = _getIndexQuery(
     name,

workers/loc.api/sync/dao/helpers/get-insertable-array-objects-filter.js

@@ -16,10 +16,7 @@ const getFieldsFilters = (
     const _params = { ...params }
     const field = _params[fieldName]
 
-    if (
-      Object.keys(model)
-        .some(key => key === modelFieldName)
-    ) {
+    if (model.hasModelFieldName(modelFieldName)) {
       if (typeof field === 'boolean') {
         return {
           ...accum,

workers/loc.api/sync/dao/helpers/get-projection-query.js

@@ -1,9 +1,14 @@
 'use strict'
 
+const Model = require('../../schema/models/model')
+
 const _getProjArr = (model) => {
   if (Array.isArray(model)) {
     return model
   }
+  if (model instanceof Model) {
+    return model.getModelFieldKeys()
+  }
   if (
     model &&
     typeof model === 'object'

workers/loc.api/sync/dao/helpers/get-table-creation-query.js

@@ -1,13 +1,10 @@
 'use strict'
 
-const {
-  CONSTR_FIELD_NAME
-} = require('../../schema/models/model/db.service.field.names')
-
 const _getConstraintsQuery = (name, model) => {
-  const constraintsArr = Array.isArray(model[CONSTR_FIELD_NAME])
-    ? model[CONSTR_FIELD_NAME]
-    : [model[CONSTR_FIELD_NAME]]
+  const modelConstraints = model.getConstraints()
+  const constraintsArr = Array.isArray(modelConstraints)
+    ? modelConstraints
+    : [modelConstraints]
 
   return constraintsArr.reduce((accum, item) => {
     const _constraints = (
@@ -43,8 +40,7 @@ module.exports = (models = [], opts = {}) => {
     const _name = `${prefix}${name}`
     const constraints = _getConstraintsQuery(_name, model)
 
-    const keys = Object.keys(model)
-      .filter((field) => field !== CONSTR_FIELD_NAME)
+    const keys = model.getModelFieldKeys()
     const columnDefs = keys.reduce((accum, field, i, arr) => {
       const isLast = arr.length === (i + 1)
       const type = model[field]

workers/loc.api/sync/dao/helpers/get-trigger-creation-query.js

@@ -1,9 +1,5 @@
 'use strict'
 
-const {
-  TRIGGER_FIELD_NAME
-} = require('../../schema/models/model/db.service.field.names')
-
 const _getTriggersQuery = (
   name,
   model,
@@ -13,9 +9,10 @@ const _getTriggersQuery = (
     shouldNotAddIfNotExistsStm
   } = opts ?? {}
 
-  const triggersArr = Array.isArray(model[TRIGGER_FIELD_NAME])
-    ? model[TRIGGER_FIELD_NAME]
-    : [model[TRIGGER_FIELD_NAME]]
+  const modelTriggers = model.getTriggers()
+  const triggersArr = Array.isArray(modelTriggers)
+    ? modelTriggers
+    : [modelTriggers]
 
   return triggersArr.reduce((accum, item) => {
     if (

workers/loc.api/sync/data.inserter/helpers/utils.js

@@ -16,7 +16,7 @@ const normalizeApiData = (
     return data
   }
 
-  const modelKeys = Object.keys(model)
+  const modelKeys = model.getModelFieldKeys()
 
   if (modelKeys.length === 0) {
     return data

workers/loc.api/sync/data.inserter/index.js

@@ -539,11 +539,7 @@ class DataInserter extends EventEmitter {
     _args.params.notThrowError = true
     const currIterationArgs = cloneDeep(_args)
 
-    const { subUserId } = model ?? {}
-    const hasNotSubUserField = (
-      !subUserId ||
-      typeof subUserId !== 'string'
-    )
+    const hasNotSubUserField = !model.hasModelFieldName('subUserId')
     const { auth: _auth } = _args ?? {}
     const { session } = _auth ?? {}
     const sessionAuth = isPublic || hasNotSubUserField

workers/loc.api/sync/data.inserter/sync.user.step.manager/index.js

@@ -213,24 +213,22 @@ class SyncUserStepManager {
     const hasUserIdField = (
       Number.isInteger(userId)
     )
-    const hasUserIdFieldInModel = (
-      typeof model?.user_id === 'string'
-    )
+    const hasUserIdFieldInModel = model
+      .hasModelFieldName('user_id')
     const hasSubUserIdField = (
       Number.isInteger(subUserId)
     )
-    const hasSubUserIdFieldInModel = (
-      typeof model?.subUserId === 'string'
-    )
+    const hasSubUserIdFieldInModel = model
+      .hasModelFieldName('subUserId')
     const hasSymbolField = (
       symbolFieldName &&
-      typeof model?.[symbolFieldName] === 'string' &&
+      model.hasModelFieldName(symbolFieldName) &&
       symbol &&
       typeof symbol === 'string'
     )
     const hasTimeframeField = (
       timeframeFieldName &&
-      typeof model?.[timeframeFieldName] === 'string' &&
+      model.hasModelFieldName(timeframeFieldName) &&
       timeframe &&
       typeof timeframe === 'string'
     )

workers/loc.api/sync/movements/index.js

@@ -25,10 +25,12 @@ class Movements {
     this.ALLOWED_COLLS = ALLOWED_COLLS
     this.authenticator = authenticator
 
-    this.movementsModel = this.syncSchema.getModelsMap()
-      .get(this.ALLOWED_COLLS.MOVEMENTS)
-    this.ledgersModel = this.syncSchema.getModelsMap()
-      .get(this.ALLOWED_COLLS.LEDGERS)
+    this.movementsModelFields = this.syncSchema
+      .getModelOf(this.ALLOWED_COLLS.MOVEMENTS)
+      .getModelFields()
+    this.ledgersModelFields = this.syncSchema
+      .getModelOf(this.ALLOWED_COLLS.LEDGERS)
+      .getModelFields()
   }
 
   async getMovements (params = {}) {
@@ -177,7 +179,7 @@ class Movements {
       end = Date.now(),
       filter: _filter,
       sort = [['mts', -1], ['id', -1]],
-      projection = this.ledgersModel,
+      projection = this.ledgersModelFields,
       exclude = ['user_id'],
       isExcludePrivate = true,
       isWithdrawals = false,
@@ -227,7 +229,7 @@ class Movements {
       end = Date.now(),
       filter: _filter,
       sort = [['mts', -1], ['id', -1]],
-      projection = this.ledgersModel,
+      projection = this.ledgersModelFields,
       exclude = ['user_id'],
       isExcludePrivate = true,
       isWithdrawals = false,