feat(mojaloop/#3565): database and cache optimizations (#318)

* knex upgrade yike

* fix(mojaloop/#3564): move get queries outside of transaction

* chore(snapshot): 15.2.3-snapshot.0

* chore: move hash closer

* chore(snapshot): 15.2.3-snapshot.1

* chore: cache fix

* chore(snapshot): 15.2.3-snapshot.2

* chore: changes

* cache

* chore: fix open handle

* imrpove coverage

* chore(snapshot): 15.2.3-snapshot.3

config/default.json

@@ -68,5 +68,9 @@
         }
       }
     }
+  },
+  "CACHE": {
+    "ENUM_DATA_EXPIRES_IN_MS": 4170000,
+    "PARTICIPANT_DATA_EXPIRES_IN_MS": 60000
   }
 }

package.json

@@ -2,7 +2,7 @@
   "name": "quoting-service",
   "description": "Quoting Service hosted by a scheme",
   "license": "Apache-2.0",
-  "version": "15.2.2",
+  "version": "15.2.3-snapshot.3",
   "author": "ModusBox",
   "contributors": [
     "Georgi Georgiev <georgi.georgiev@modusbox.com>",
@@ -55,7 +55,7 @@
     "lint": "npx standard",
     "lint:fix": "npx standard --fix",
     "test": "npm run test:unit",
-    "test:unit": "jest --runInBand --testMatch '**/test/unit/**/*.test.js'",
+    "test:unit": "jest --runInBand --testMatch  '**/test/unit/**/*.test.js' ",
     "test:coverage": "jest --runInBand --coverage --coverageThreshold='{}' --testMatch '**/test/unit/**/*.test.js'",
     "test:coverage-check": "jest --runInBand --coverage --testMatch '**/test/unit/**/*.test.js'",
     "test:junit": "jest --runInBand --reporters=default --reporters=jest-junit --testMatch '**/test/unit/**/*.test.js'",
@@ -99,7 +99,7 @@
     "good-squeeze": "5.1.0",
     "joi": "17.11.0",
     "json-rules-engine": "5.0.2",
-    "knex": "2.5.1",
+    "knex": "3.0.1",
     "memory-cache": "0.2.0",
     "minimist": "1.2.8",
     "mysql": "2.18.1",
@@ -113,7 +113,7 @@
     "eslint-config-standard": "17.1.0",
     "jest": "29.7.0",
     "jest-junit": "16.0.0",
-    "npm-check-updates": "16.14.5",
+    "npm-check-updates": "16.14.6",
     "nyc": "15.1.0",
     "pre-commit": "1.2.2",
     "proxyquire": "2.1.3",

src/data/cachedDatabase.js

@@ -38,8 +38,6 @@ const Metrics = require('@mojaloop/central-services-metrics')
 
 const { getStackOrInspect } = require('../lib/util')
 
-const DEFAULT_TTL_SECONDS = 60
-
 /**
  * An extension of the Database class that caches enum values in memory
  */
@@ -83,10 +81,13 @@ class CachedDatabase extends Database {
     return this.getCacheValue('getPartyIdentifierType', [partyIdentifierType])
   }
 
-  // This has been commented out as the participant data should not be cached. This is mainly due to the scenario when the participant is made inactive vs active. Ref: https://github.com/mojaloop/project/issues/933
-  // async getParticipant (participantName) {
-  //   return this.getCacheValue('getParticipant', [participantName])
-  // }
+  async getParticipant (participantName, participantType, currencyId, ledgerAccountTypeId) {
+    return this.getCacheValue('getParticipant', [participantName, participantType, currencyId, ledgerAccountTypeId])
+  }
+
+  async getParticipantByName (participantName, participantType) {
+    return this.getCacheValue('getParticipantByName', [participantName, participantType])
+  }
 
   async getTransferParticipantRoleType (name) {
     return this.getCacheValue('getTransferParticipantRoleType', [name])
@@ -96,9 +97,9 @@ class CachedDatabase extends Database {
     return this.getCacheValue('getLedgerEntryType', [name])
   }
 
-  // async getParticipantEndpoint (participantName, endpointType) {
-  //  return this.getCacheValue('getParticipantEndpoint', [participantName, endpointType])
-  // }
+  async getParticipantEndpoint (participantName, endpointType) {
+    return this.getCacheValue('getParticipantEndpoint', [participantName, endpointType])
+  }
 
   async getCacheValue (type, params) {
     const histTimer = Metrics.getHistogram(
@@ -113,7 +114,16 @@ class CachedDatabase extends Database {
         // we need to get the value from the db and cache it
         this.writeLog(`Cache miss for ${type}: ${util.inspect(params)}`)
         value = await super[type].apply(this, params)
-        this.cachePut(type, params, value)
+        // cache participant with a shorter TTL than enums (participant data is more likely to change)
+        if (
+          type === 'getParticipant' ||
+          type === 'getParticipantByName' ||
+          type === 'getParticipantEndpoint'
+        ) {
+          this.cachePut(type, params, value, this.config.participantDataCacheExpiresInMs)
+        } else {
+          this.cachePut(type, params, value, this.config.enumDataCacheExpiresInMs)
+        }
         histTimer({ success: true, queryName: type, hit: false })
       } else {
         this.writeLog(`Cache hit for ${type} ${util.inspect(params)}: ${value}`)
@@ -133,9 +143,9 @@ class CachedDatabase extends Database {
      *
      * @returns {undefined}
      */
-  cachePut (type, params, value) {
+  cachePut (type, params, value, ttl) {
     const key = this.getCacheKey(type, params)
-    this.cache.put(key, value, (this.config.database.cacheTtlSeconds || DEFAULT_TTL_SECONDS) * 1000)
+    this.cache.put(key, value, ttl)
   }
 
   /**
@@ -156,6 +166,10 @@ class CachedDatabase extends Database {
   getCacheKey (type, params) {
     return `${type}_${params.join('__')}`
   }
+
+  invalidateCache () {
+    this.cache.clear()
+  }
 }
 
 module.exports = CachedDatabase