Skip to content

Commit

Permalink
feat(mojaloop/#3427): add oracle endpoint db caching and oracle reque…
Browse files Browse the repository at this point in the history
…st caching (#467)

* chore: fix logger statements

* chore: fix logger statements

* chore(snapshot): 15.0.1-snapshot.0

* chore: log

* chore: revert

* chore: statement

* test

* chore(snapshot): 15.0.1-snapshot.1

* dep

* chore(snapshot): 15.0.1-snapshot.2

* chore: change

* chore: tests

* feat: add oracle endpoint db caching

* add oracle request caching

* tests

* coverage

* chore(snapshot): 15.2.0-snapshot.0

* chore: header
  • Loading branch information
kleyow committed Nov 28, 2023
1 parent 0e503fa commit 42e93d0
Show file tree
Hide file tree
Showing 40 changed files with 779 additions and 102 deletions.
9 changes: 7 additions & 2 deletions config/default.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,21 @@
},
"DISPLAY_ROUTES": true,
"RUN_MIGRATIONS": true,
"ENDPOINT_CACHE_CONFIG": {
"CENTRAL_SHARED_ENDPOINT_CACHE_CONFIG": {
"expiresIn": 180000,
"generateTimeout": 30000,
"getDecoratedValue": true
},
"PARTICIPANT_CACHE_CONFIG": {
"CENTRAL_SHARED_PARTICIPANT_CACHE_CONFIG": {
"expiresIn": 61000,
"generateTimeout": 30000,
"getDecoratedValue": true
},
"ALS_GENERAL_CACHE_CONFIG": {
"CACHE_ENABLED": false,
"MAX_BYTE_SIZE": 10000000,
"EXPIRES_IN_MS": 61000
},
"ERROR_HANDLING": {
"includeCauseExtension": false,
"truncateExtensions": true
Expand Down
2 changes: 1 addition & 1 deletion docker/account-lookup-service/default.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
},
"DISPLAY_ROUTES": true,
"RUN_MIGRATIONS": true,
"ENDPOINT_CACHE_CONFIG": {
"CENTRAL_SHARED_ENDPOINT_CACHE_CONFIG": {
"expiresIn": 180000,
"generateTimeout": 30000
},
Expand Down
5 changes: 3 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "account-lookup-service",
"description": "Account Lookup Service is used to validate Party and Participant lookups.",
"version": "15.1.0",
"version": "15.2.0-snapshot.0",
"license": "Apache-2.0",
"author": "ModusBox",
"contributors": [
Expand Down Expand Up @@ -77,17 +77,18 @@
"dependencies": {
"@hapi/basic": "7.0.2",
"@hapi/boom": "10.0.1",
"@hapi/catbox-memory": "^6.0.1",
"@hapi/good": "9.0.1",
"@hapi/hapi": "21.3.2",
"@hapi/inert": "7.1.0",
"@hapi/joi": "17.1.1",
"@hapi/vision": "7.0.3",
"@mojaloop/database-lib": "11.0.3",
"@mojaloop/central-services-error-handling": "12.0.7",
"@mojaloop/central-services-health": "14.0.2",
"@mojaloop/central-services-logger": "11.2.2",
"@mojaloop/central-services-metrics": "12.0.8",
"@mojaloop/central-services-shared": "18.2.0",
"@mojaloop/database-lib": "11.0.3",
"@mojaloop/event-sdk": "14.0.0",
"@mojaloop/sdk-standard-components": "17.1.3",
"@now-ims/hapi-now-auth": "2.1.0",
Expand Down
7 changes: 4 additions & 3 deletions src/domain/oracle/oracle.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const currency = require('../../models/currency')
const ErrorHandler = require('@mojaloop/central-services-error-handling')
const Config = require('../../lib/config')
const Metrics = require('@mojaloop/central-services-metrics')
const cachedOracleEndpoint = require('../../models/oracle/oracleEndpointCached')

/**
* @function createOracle
Expand Down Expand Up @@ -109,11 +110,11 @@ exports.getOracle = async (query) => {
isType = true
}
if (isCurrency && isType) {
oracleEndpointModelList = await oracleEndpoint.getOracleEndpointByTypeAndCurrency(query.type, query.currency)
oracleEndpointModelList = await cachedOracleEndpoint.getOracleEndpointByTypeAndCurrency(query.type, query.currency)
} else if (isCurrency && !isType) {
oracleEndpointModelList = await oracleEndpoint.getOracleEndpointByCurrency(query.currency)
oracleEndpointModelList = await cachedOracleEndpoint.getOracleEndpointByCurrency(query.currency)
} else if (isType && !isCurrency) {
oracleEndpointModelList = await oracleEndpoint.getOracleEndpointByType(query.type)
oracleEndpointModelList = await cachedOracleEndpoint.getOracleEndpointByType(query.type)
} else {
oracleEndpointModelList = await oracleEndpoint.getAllOracleEndpoint()
}
Expand Down
4 changes: 2 additions & 2 deletions src/domain/parties/parties.js
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ const Config = require('../../lib/config')
* @param {object} query - uri query parameters of the http request
* @param {object} span
*/
const getPartiesByTypeAndID = async (headers, params, method, query, span = undefined) => {
const getPartiesByTypeAndID = async (headers, params, method, query, span = undefined, cache) => {
const histTimerEnd = Metrics.getHistogram(
'getPartiesByTypeAndID',
'Get party by Type and Id',
Expand Down Expand Up @@ -95,7 +95,7 @@ const getPartiesByTypeAndID = async (headers, params, method, query, span = unde
return
}

const response = await oracle.oracleRequest(headers, method, params, query)
const response = await oracle.oracleRequest(headers, method, params, query, cache)

if (response && response.data && Array.isArray(response.data.partyList) && response.data.partyList.length > 0) {
// Oracle's API is a standard rest-style end-point Thus a GET /party on the oracle will return all participant-party records. We must filter the results based on the callbackEndpointType to make sure we remove records containing partySubIdOrType when we are in FSPIOP_CALLBACK_URL_PARTIES_GET mode:
Expand Down
2 changes: 1 addition & 1 deletion src/handlers/endpointcache.js
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ module.exports = {
}, EventSdk.AuditEventAction.start)
try {
await ParticipantEndpointCache.stopCache()
await ParticipantEndpointCache.initializeCache(Config.ENDPOINT_CACHE_CONFIG)
await ParticipantEndpointCache.initializeCache(Config.CENTRAL_SHARED_ENDPOINT_CACHE_CONFIG)
histTimerEnd({ success: true })
} catch (err) {
histTimerEnd({ success: false })
Expand Down
59 changes: 17 additions & 42 deletions src/handlers/participants/{Type}/{ID}.js
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@
'use strict'

const Enum = require('@mojaloop/central-services-shared').Enum
const ErrorHandler = require('@mojaloop/central-services-error-handling')
const EventSdk = require('@mojaloop/event-sdk')
const Logger = require('@mojaloop/central-services-logger')
const Metrics = require('@mojaloop/central-services-metrics')
const LibUtil = require('../../../lib/util')
const participants = require('../../../domain/participants')
Expand Down Expand Up @@ -58,16 +56,11 @@ module.exports = {
payload: request.payload
}, EventSdk.AuditEventAction.start)
const metadata = `${request.method} ${request.path}`
try {
participants.getParticipantsByTypeAndID(request.headers, request.params, request.method, request.query, span).catch(err => {
request.server.log(['error'], `ERROR - getParticipantsByTypeAndID:${metadata}: ${LibUtil.getStackOrInspect(err)}`)
})
histTimerEnd({ success: true })
} catch (err) {
Logger.isErrorEnabled && Logger.error(`ERROR - ${metadata}: ${err.stack}`)
histTimerEnd({ success: false })
throw ErrorHandler.Factory.reformatFSPIOPError(err)
}
participants.getParticipantsByTypeAndID(request.headers, request.params, request.method, request.query, span).catch(err => {
request.server.log(['error'], `ERROR - getParticipantsByTypeAndID:${metadata}: ${LibUtil.getStackOrInspect(err)}`)
})
histTimerEnd({ success: true })

return h.response().code(202)
},
/**
Expand All @@ -84,16 +77,10 @@ module.exports = {
['success']
).startTimer()
const metadata = `${request.method} ${request.path}`
try {
participants.putParticipantsByTypeAndID(request.headers, request.params, request.method, request.payload).catch(err => {
request.server.log(['error'], `ERROR - putParticipantsByTypeAndID:${metadata}: ${LibUtil.getStackOrInspect(err)}`)
})
histTimerEnd({ success: true })
} catch (err) {
Logger.isErrorEnabled && Logger.error(`ERROR - ${metadata}: ${err.stack}`)
histTimerEnd({ success: false })
throw ErrorHandler.Factory.reformatFSPIOPError(err)
}
participants.putParticipantsByTypeAndID(request.headers, request.params, request.method, request.payload).catch(err => {
request.server.log(['error'], `ERROR - putParticipantsByTypeAndID:${metadata}: ${LibUtil.getStackOrInspect(err)}`)
})
histTimerEnd({ success: true })
return h.response().code(200)
},
/**
Expand All @@ -117,16 +104,10 @@ module.exports = {
payload: request.payload
}, EventSdk.AuditEventAction.start)
const metadata = `${request.method} ${request.path}`
try {
participants.postParticipants(request.headers, request.method, request.params, request.payload, span).catch(err => {
request.server.log(['error'], `ERROR - postParticipants:${metadata}: ${LibUtil.getStackOrInspect(err)}`)
})
histTimerEnd({ success: true })
} catch (err) {
Logger.isErrorEnabled && Logger.error(`ERROR - ${metadata}: ${err.stack}`)
histTimerEnd({ success: false })
throw ErrorHandler.Factory.reformatFSPIOPError(err)
}
participants.postParticipants(request.headers, request.method, request.params, request.payload, span).catch(err => {
request.server.log(['error'], `ERROR - postParticipants:${metadata}: ${LibUtil.getStackOrInspect(err)}`)
})
histTimerEnd({ success: true })
return h.response().code(202)
},
/**
Expand All @@ -143,16 +124,10 @@ module.exports = {
['success']
).startTimer()
const metadata = `${request.method} ${request.path}`
try {
participants.deleteParticipants(request.headers, request.params, request.method, request.query).catch(err => {
request.server.log(['error'], `ERROR - deleteParticipants:${metadata}: ${LibUtil.getStackOrInspect(err)}`)
})
histTimerEnd({ success: true })
} catch (err) {
Logger.isErrorEnabled && Logger.error(`ERROR - ${metadata}: ${err.stack}`)
histTimerEnd({ success: false })
throw ErrorHandler.Factory.reformatFSPIOPError(err)
}
participants.deleteParticipants(request.headers, request.params, request.method, request.query).catch(err => {
request.server.log(['error'], `ERROR - deleteParticipants:${metadata}: ${LibUtil.getStackOrInspect(err)}`)
})
histTimerEnd({ success: true })
return h.response().code(202)
}
}
2 changes: 1 addition & 1 deletion src/handlers/parties/{Type}/{ID}.js
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ module.exports = {
}, EventSdk.AuditEventAction.start)
// Here we call an async function- but as we send an immediate sync response, _all_ errors
// _must_ be handled by getPartiesByTypeAndID.
parties.getPartiesByTypeAndID(request.headers, request.params, request.method, request.query, span).catch(err => {
parties.getPartiesByTypeAndID(request.headers, request.params, request.method, request.query, span, request.server.app.cache).catch(err => {
request.server.log(['error'], `ERROR - getPartiesByTypeAndID: ${LibUtil.getStackOrInspect(err)}`)
})
histTimerEnd({ success: true })
Expand Down
126 changes: 126 additions & 0 deletions src/lib/cache.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
/*****
License
--------------
Copyright © 2017 Bill & Melinda Gates Foundation
The Mojaloop files are made available by the Bill & Melinda Gates Foundation under the Apache License, Version 2.0 (the "License") and you may not use these files except in compliance with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, the Mojaloop files are distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
Contributors
--------------
This is the official list of the Mojaloop project contributors for this file.
Names of the original copyright holders (individuals or organizations)
should be listed with a '*' in the first column. People who have
contributed from an organization can be listed under the organization
that actually holds the copyright for their contributions (see the
Gates Foundation organization for an example). Those individuals should have
their names indented and be marked with a '-'. Email address can be added
optionally within square brackets <email>.
* Gates Foundation
* Name Surname <name.surname@gatesfoundation.com>
* Kevin Leyow <kevin.leyow@infitx.com>
--------------
******/

'use strict'

const CatboxMemory = require('@hapi/catbox-memory')
const Config = require('../lib/config')

let enabled = true
let ttl
let catboxMemoryClient = null

class CacheClient {
constructor (meta) {
this.meta = meta
}

getMeta () {
return this.meta
}

createKey (id) {
return {
segment: this.meta.id,
id
}
}

get (key) {
if (enabled) {
return catboxMemoryClient.get(key)
}
return null
}

set (key, value) {
catboxMemoryClient.set(key, value, parseInt(ttl))
}

drop (key) {
catboxMemoryClient.drop(key)
}
}

/*
Each client should register itself during module load.
The client meta should be:
{
id [MANDATORY]
preloadCache() [OPTIONAL]
this will be called to preload data
}
*/
let cacheClients = {}

const registerCacheClient = (clientMeta) => {
const newClient = new CacheClient(clientMeta)
cacheClients[clientMeta.id] = newClient
return newClient
}

const initCache = async function () {
// Read config
ttl = Config.ALS_GENERAL_CACHE_CONFIG.EXPIRES_IN_MS
enabled = Config.ALS_GENERAL_CACHE_CONFIG.CACHE_ENABLED

// Init catbox.
catboxMemoryClient = new CatboxMemory.Engine({
maxByteSize: Config.ALS_GENERAL_CACHE_CONFIG.MAX_BYTE_SIZE
})
catboxMemoryClient.start()

for (const clientId in cacheClients) {
const clientMeta = cacheClients[clientId].getMeta()
await clientMeta.preloadCache()
}
}

const destroyCache = async function () {
catboxMemoryClient.stop()
catboxMemoryClient = null
}

const dropClients = function () {
cacheClients = {}
}

const isCacheEnabled = function () {
return enabled
}

module.exports = {
// Clients registration
registerCacheClient,

// Init & destroy the cache
initCache,
destroyCache,
isCacheEnabled,

// exposed for tests
CatboxMemory,
dropClients
}
5 changes: 3 additions & 2 deletions src/lib/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,9 @@ const config = {
DISPLAY_ROUTES: RC.DISPLAY_ROUTES,
RUN_MIGRATIONS: RC.RUN_MIGRATIONS,
ADMIN_PORT: RC.ADMIN_PORT,
ENDPOINT_CACHE_CONFIG: RC.ENDPOINT_CACHE_CONFIG,
PARTICIPANT_CACHE_CONFIG: RC.PARTICIPANT_CACHE_CONFIG,
CENTRAL_SHARED_ENDPOINT_CACHE_CONFIG: RC.CENTRAL_SHARED_ENDPOINT_CACHE_CONFIG,
CENTRAL_SHARED_PARTICIPANT_CACHE_CONFIG: RC.CENTRAL_SHARED_PARTICIPANT_CACHE_CONFIG,
ALS_GENERAL_CACHE_CONFIG: RC.ALS_GENERAL_CACHE_CONFIG,
ERROR_HANDLING: RC.ERROR_HANDLING,
SWITCH_ENDPOINT: RC.SWITCH_ENDPOINT,
INSTRUMENTATION_METRICS_DISABLED: RC.INSTRUMENTATION.METRICS.DISABLED,
Expand Down
Loading

0 comments on commit 42e93d0

Please sign in to comment.