From d294fe78738e365b7265004036a796a747753ec4 Mon Sep 17 00:00:00 2001 From: tholulomo Date: Mon, 19 Feb 2024 10:07:52 -0500 Subject: [PATCH] feat: Refactoring ES error handling --- .../metamineNU/VisualizationLayout.spec.js | 43 +- .../src/controllers/kgWrapperController.js | 7 +- .../src/middlewares/knowledge-cache.js | 2 +- resfulservice/src/utils/elasticSearch.js | 504 +++++++++++------- 4 files changed, 333 insertions(+), 223 deletions(-) diff --git a/app/tests/unit/components/metamineNU/VisualizationLayout.spec.js b/app/tests/unit/components/metamineNU/VisualizationLayout.spec.js index f7da2bb9..82b69e83 100644 --- a/app/tests/unit/components/metamineNU/VisualizationLayout.spec.js +++ b/app/tests/unit/components/metamineNU/VisualizationLayout.spec.js @@ -14,18 +14,35 @@ describe('VisualizationLayout.vue', () => { link: { to: '/mm', text: 'Pairwise' }, dense: false } - const dispatch = jest.spyOn(store, 'dispatch') + const originalFunction = store.dispatch + const dispatch = jest + .spyOn(store, 'dispatch') + .mockImplementationOnce( + async () => await originalFunction('metamineNU/fetchMetamineDataset') + ) + .mockImplementationOnce(() => Promise.resolve({ val: 'reload' })) + .mockImplementationOnce(() => Promise.resolve({ val: 'force-cache' })) + .mockImplementationOnce(() => Promise.resolve({ val: 'reload' })) const commitSpy = jest.spyOn(store, 'commit') global.fetch = jest .fn() .mockImplementationOnce(() => - Promise.resolve({ json: () => Promise.resolve(mockValues) }) + Promise.resolve({ + json: () => Promise.resolve(mockValues), + url: 'windows_url/metamine/file_1.csv' + }) ) .mockImplementationOnce(() => - Promise.resolve({ json: () => Promise.resolve(mockValues2) }) + Promise.resolve({ + json: () => Promise.resolve(mockValues2), + url: 'windows_url/metamine/file_2.csv' + }) ) .mockImplementationOnce(() => - Promise.resolve({ json: () => Promise.resolve(mockValues2) }) + Promise.resolve({ + json: () => Promise.resolve(mockValues2), + url: 'windows_url/metamine/file_3.csv' + }) ) .mockImplementation(() => {}) @@ -39,8 +56,8 @@ describe('VisualizationLayout.vue', () => { await jest.resetAllMocks() }) - it.skip('makes a fetch call when mounted ', () => { - expect.assertions(12) + it('makes a fetch call when mounted ', async () => { + expect.assertions(10) expect(wrapper.exists()).toBe(true) expect(dispatch).toHaveBeenCalledTimes(2) expect(commitSpy).toHaveBeenCalledTimes(5) @@ -73,16 +90,14 @@ describe('VisualizationLayout.vue', () => { [], undefined ) + expect(dispatch).toHaveBeenNthCalledWith(2, 'fetchWrapper', { + url: '/api/files/metamine' + }) - expect(fetch).toHaveBeenCalledTimes(1 + mockValues.fetchedNames.length) // check initial fetch request - expect(fetch).toHaveBeenNthCalledWith(1, '/api/files/metamine') - for (let i = 0; i < mockValues.fetchedNames.length; i++) { - expect(fetch).toHaveBeenNthCalledWith( - i + 2, - `/api/files/metamine/${mockValues.fetchedNames[i].name}` - ) - } + expect(global.fetch).toHaveBeenNthCalledWith(1, '/api/files/metamine', { + cache: 'reload' + }) }) it('renders layout properly ', () => { diff --git a/resfulservice/src/controllers/kgWrapperController.js b/resfulservice/src/controllers/kgWrapperController.js index 31214792..9d2669d3 100644 --- a/resfulservice/src/controllers/kgWrapperController.js +++ b/resfulservice/src/controllers/kgWrapperController.js @@ -172,6 +172,8 @@ exports.getKnowledge = async (req, res, next) => { * @returns {*} response.data */ exports.getSparql = async (req, res, next) => { + const log = req.logger; + log.info('getSparql(): Function entry'); const whyisPath = req.query.whyisPath; try { if (!req.env.KNOWLEDGE_ADDRESS) { @@ -206,6 +208,7 @@ exports.getSparql = async (req, res, next) => { if (response.data && whyisPath !== 'pub') { req.knowledgeId = req.knowledgeId ?? uuidv4(); await elasticSearch.createKnowledgeGraphDoc( + log, req.knowledgeId, req.query.queryString, response?.data @@ -277,7 +280,9 @@ exports.getInstanceFromKnowledgeGraph = async (req, res, next) => { const view = req?.query?.view; let url; if (!view) url = `${req.env.KNOWLEDGE_ADDRESS}/about?uri=${req.query.uri}`; - else { url = `${req.env.KNOWLEDGE_ADDRESS}/about?uri=${req.query.uri}&view=${view}`; } + else { + url = `${req.env.KNOWLEDGE_ADDRESS}/about?uri=${req.query.uri}&view=${view}`; + } return axios .get(url, { responseType: 'arraybuffer' diff --git a/resfulservice/src/middlewares/knowledge-cache.js b/resfulservice/src/middlewares/knowledge-cache.js index d73f0615..e1821204 100644 --- a/resfulservice/src/middlewares/knowledge-cache.js +++ b/resfulservice/src/middlewares/knowledge-cache.js @@ -13,7 +13,7 @@ exports.isKnowledgeCached = async (req, res, next) => { req.logger.info('Middleware.isKnowledgeCached - Function entry'); const query = req.query.query ?? req.body?.query; - const cacheResult = await elasticSearch.searchKnowledgeGraph(query); + const cacheResult = await elasticSearch.searchKnowledgeGraph(req, query); if (cacheResult.length) { const { _id, diff --git a/resfulservice/src/utils/elasticSearch.js b/resfulservice/src/utils/elasticSearch.js index 7d60e335..c06a11de 100644 --- a/resfulservice/src/utils/elasticSearch.js +++ b/resfulservice/src/utils/elasticSearch.js @@ -16,19 +16,24 @@ class ElasticSearch { */ async ping (log, waitTime = 50000) { log.info('elasticsearch.ping(): Function entry'); - return new Promise((resolve, reject) => { - const timer = setTimeout(async () => { - const response = await this.client.ping(); - clearTimeout(timer); - if (!response) { - const error = new Error('Elastic Search Service Not Available'); - log.error(`elasticsearch.ping(): 500 - ${error}`); - reject(error); - } - log.debug(`elasticsearch.ping(): response ${response}`); - resolve(response); - }, waitTime); - }); + try { + return new Promise((resolve, reject) => { + const timer = setTimeout(async () => { + const response = await this.client.ping(); + clearTimeout(timer); + if (!response) { + const error = new Error('Elastic Search Service Not Available'); + log.error(`elasticsearch.ping(): 500 - ${error}`); + reject(error); + } + log.debug(`elasticsearch.ping(): response ${response}`); + resolve(response); + }, waitTime); + }); + } catch (err) { + log.error(`elasticsearch.ping(): ${err.status || 500} - ${err}`); + throw err; + } } /** @@ -36,16 +41,22 @@ class ElasticSearch { * @param {String} type * @returns {Object} response */ - async _createConfig (type) { - const configResponse = await axios({ - method: 'put', - url: `http://${env.ESADDRESS}/${type}`, - headers: { - 'Content-Type': 'application/json' - }, - data: JSON.stringify({ ...configPayload.config }) - }); - return configResponse; + async _createConfig (type, log) { + log.info('elasticsearch._createConfig(): Function entry'); + try { + const configResponse = await axios({ + method: 'put', + url: `http://${env.ESADDRESS}/${type}`, + headers: { + 'Content-Type': 'application/json' + }, + data: JSON.stringify({ ...configPayload.config }) + }); + return configResponse; + } catch (err) { + log.error(`elasticsearch._createConfig(): ${err.status || 500} - ${err}`); + throw err; + } } /** @@ -57,91 +68,117 @@ class ElasticSearch { async deleteIndexDocs (req, type) { const log = req.logger; log.info('elasticsearch.deleteIndexDocs(): Function entry'); - return this.client.deleteByQuery({ - index: type, - body: { - query: { - match_all: {} - } - }, - timeout: '5m' // Todo: Increase when data becomes larger - }); + try { + return this.client.deleteByQuery({ + index: type, + body: { + query: { + match_all: {} + } + }, + timeout: '5m' // Todo: Increase when data becomes larger + }); + } catch (err) { + log.error( + `elasticsearch.deleteIndexDocs(): ${err.status || 500} - ${err}` + ); + throw err; + } } async deleteSingleDoc (req, type, identifier) { const log = req.logger; log.info('elasticsearch.deleteSingleDoc(): Function entry'); - return this.client.deleteByQuery({ - index: type, - body: { - query: { - match_phrase: { - identifier + try { + return this.client.deleteByQuery({ + index: type, + body: { + query: { + match_phrase: { + identifier + } } } - } - }); + }); + } catch (err) { + log.error( + `elasticsearch.deleteSingleDoc(): ${err.status || 500} - ${err}` + ); + throw err; + } } - async _putMappings (type, schema) { - return await this.client.indices.putMapping({ - index: type, - // type: 'articles', - body: { - ...schema - } - }); + async _putMappings (type, schema, log) { + log.info('elasticsearch._putMappings(): Function entry'); + try { + return await this.client.indices.putMapping({ + index: type, + // type: 'articles', + body: { + ...schema + } + }); + } catch (err) { + log.error(`elasticsearch._putMappings(): ${err.status || 500} - ${err}`); + throw err; + } } - async _getExistingIndices () { - return await this.client.cat.indices({ format: 'json' }); + async _getExistingIndices (log) { + log.info('elasticsearch._getExistingIndices(): Function entry'); + try { + return await this.client.cat.indices({ format: 'json' }); + } catch (err) { + log.error( + `elasticsearch._getExistingIndices(): ${err.status || 500} - ${err}` + ); + throw err; + } } async initES (req) { const log = req.logger; log.info('elasticsearch.initES(): Function entry'); + try { + // Check and ignore existing indexes before create + const existingIndexes = await this._getExistingIndices(log); - // Check and ignore existing indexes before create - const existingIndexes = await this._getExistingIndices(); - - // Remove elastic search index config from list of keys - const preparedKeys = Object.keys(configPayload)?.filter( - (e) => e !== 'config' - ); + // Remove elastic search index config from list of keys + const preparedKeys = Object.keys(configPayload)?.filter( + (e) => e !== 'config' + ); - // Create a set of existing indices - const existingIndicesSet = new Set( - existingIndexes.map((index) => index.index) - ); - const nonExistingKeys = []; - // Check if all indices in indices exist in existingIndicesSet - const allIndicesExist = preparedKeys.every((index) => { - const exists = existingIndicesSet.has(index); - if (!exists) nonExistingKeys.push(index); - return exists; - }); + // Create a set of existing indices + const existingIndicesSet = new Set( + existingIndexes.map((index) => index.index) + ); + const nonExistingKeys = []; + // Check if all indices in indices exist in existingIndicesSet + const allIndicesExist = preparedKeys.every((index) => { + const exists = existingIndicesSet.has(index); + if (!exists) nonExistingKeys.push(index); + return exists; + }); - if (allIndicesExist) { - log.info('elasticsearch.initES(): All indexes exist in Elastic search'); - return; - } + if (allIndicesExist) { + log.info('elasticsearch.initES(): All indexes exist in Elastic search'); + return; + } - if (nonExistingKeys.length) { - log.info( - `elasticsearch.initES(): Adding the following missing index(es) ${nonExistingKeys.join( - ',' - )}` - ); - } + if (nonExistingKeys.length) { + log.info( + `elasticsearch.initES(): Adding the following missing index(es) ${nonExistingKeys.join( + ',' + )}` + ); + } - try { Object.entries(configPayload).forEach(async ([key, value]) => { if (nonExistingKeys.includes(key)) { try { - await this._createConfig(key); - await this._putMappings(key, value); + await this._createConfig(key, log); + await this._putMappings(key, value, log); } catch (error) { - console.log(error); log.error( `elasticsearch.initES(): ${error.status || 500} - ${error}` ); @@ -159,31 +196,43 @@ class ElasticSearch { } async indexDocument (req, type, doc) { - req.logger.info('elasticsearch.indexDocument(): Function entry'); const log = req.logger; + log.info('elasticsearch.indexDocument(): Function entry'); if (!type || !doc) { const error = new Error('Category type is missing'); error.statusCode = 400; log.error(`indexDocument(): ${error}`); throw error; } - return this.client.index({ - index: type, - refresh: true, - document: { ...doc } - }); + try { + return this.client.index({ + index: type, + refresh: true, + document: { ...doc } + }); + } catch (err) { + log.error(`elasticsearch.indexDocument(): ${err.status || 500} - ${err}`); + throw err; + } } async refreshIndices (req, type) { - req.logger.info('elasticsearch.refreshIndices(): Function entry'); const log = req.logger; + log.info('elasticsearch.refreshIndices(): Function entry'); if (!type) { const error = new Error('Category type is missing'); error.statusCode = 400; log.error(`refreshIndices(): ${error}`); throw error; } - return this.client.indices.refresh({ index: type }); + try { + return this.client.indices.refresh({ index: type }); + } catch (err) { + log.error( + `elasticsearch.refreshIndices(): ${err.status || 500} - ${err}` + ); + throw err; + } } searchSanitizer (search) { @@ -210,148 +259,189 @@ class ElasticSearch { } async searchType (req, searchPhrase, searchField, type, page = 1, size = 20) { - req.logger.info('elasticsearch.searchType(): Function entry'); - // TODO: use searchField to change which field is queried - const phrase = this.searchSanitizer(searchPhrase); - const url = `http://${env.ESADDRESS}/${type}/_search?size=${size}`; - const response = await axios({ - method: 'get', - url, - headers: { - 'Content-Type': 'application/json' - }, - data: JSON.stringify({ - from: (page - 1) * size, - query: { - bool: { - should: [ - { - match_phrase: { - label: phrase - } - }, - { - match_phrase: { - description: phrase + const log = req.logger; + log.info('elasticsearch.searchType(): Function entry'); + + try { + // TODO: use searchField to change which field is queried + const phrase = this.searchSanitizer(searchPhrase); + const url = `http://${env.ESADDRESS}/${type}/_search?size=${size}`; + const response = await axios({ + method: 'get', + url, + headers: { + 'Content-Type': 'application/json' + }, + data: JSON.stringify({ + from: (page - 1) * size, + query: { + bool: { + should: [ + { + match_phrase: { + label: phrase + } + }, + { + match_phrase: { + description: phrase + } } - } - ] + ] + } } - } - }) - }); - return response; + }) + }); + return response; + } catch (err) { + log.error(`elasticsearch.searchType(): ${err.status || 500} - ${err}`); + throw err; + } } async search (req, searchPhrase, autosuggest = false) { - req.logger.info('elasticsearch.search(): Function entry'); - const phrase = this.searchSanitizer(searchPhrase); - let url = `http://${env.ESADDRESS}/_all/_search?size=400`; - - if (autosuggest) { - url = `http://${env.ESADDRESS}/_all/_search?size=100&pretty=true`; - } + const log = req.logger; + log.info('elasticsearch.search(): Function entry'); + try { + const phrase = this.searchSanitizer(searchPhrase); + let url = `http://${env.ESADDRESS}/_all/_search?size=400`; - return axios({ - method: 'get', - url, - headers: { - 'Content-Type': 'application/json' - }, - data: JSON.stringify({ - query: { - bool: { - should: [ - { - match: { - label: phrase - } - }, - { - match: { - description: phrase + if (autosuggest) { + url = `http://${env.ESADDRESS}/_all/_search?size=100&pretty=true`; + } + return axios({ + method: 'get', + url, + headers: { + 'Content-Type': 'application/json' + }, + data: JSON.stringify({ + query: { + bool: { + should: [ + { + match: { + label: phrase + } + }, + { + match: { + description: phrase + } } - } - ] + ] + } } - } - }) - }); + }) + }); + } catch (err) { + log.error(`elasticsearch.search(): ${err.status || 500} - ${err}`); + throw err; + } } async loadAllCharts (req, page, size) { - req.logger.info('elasticsearch.loadAllCharts(): Function entry'); - const url = `http://${env.ESADDRESS}/charts/_search`; - return axios({ - method: 'get', - url, - headers: { - 'Content-Type': 'application/json' - }, - data: JSON.stringify({ - from: (page - 1) * size, - size, - query: { - match_all: {} - } - }) - }); + const log = req.logger; + log.info('elasticsearch.loadAllCharts(): Function entry'); + try { + const url = `http://${env.ESADDRESS}/charts/_search`; + return axios({ + method: 'get', + url, + headers: { + 'Content-Type': 'application/json' + }, + data: JSON.stringify({ + from: (page - 1) * size, + size, + query: { + match_all: {} + } + }) + }); + } catch (err) { + log.error(`elasticsearch.loadAllCharts(): ${err.status || 500} - ${err}`); + throw err; + } } async loadAllDatasets (req, page, size) { - req.logger.info('elasticsearch.loadAllDatasets(): Function entry'); + const log = req.logger; + log.info('elasticsearch.loadAllDatasets(): Function entry'); const url = `http://${env.ESADDRESS}/datasets/_search`; - return axios({ - method: 'get', - url, - headers: { - 'Content-Type': 'application/json' - }, - data: JSON.stringify({ - from: (page - 1) * size, - size, - query: { - match_all: {} - } - }) - }); + try { + return axios({ + method: 'get', + url, + headers: { + 'Content-Type': 'application/json' + }, + data: JSON.stringify({ + from: (page - 1) * size, + size, + query: { + match_all: {} + } + }) + }); + } catch (err) { + log.error( + `elasticsearch.loadAllDatasets(): ${err.status || 500} - ${err}` + ); + throw err; + } } async searchKnowledgeGraph (req, searchPhrase) { - req.logger.info('elasticsearch.searchKnowledgeGraph(): Function entry'); + const log = req.logger; + log.info('elasticsearch.searchKnowledgeGraph(): Function entry'); // search knowledge index for key - const result = await this.client.search({ - index: 'knowledge', - body: { - query: { - match_phrase: { - label: searchPhrase + try { + const result = await this.client.search({ + index: 'knowledge', + body: { + query: { + match_phrase: { + label: searchPhrase + } } } - } - }); - return result.hits.hits; + }); + return result.hits.hits; + } catch (err) { + log.error( + `elasticsearch.searchKnowledgeGraph(): ${err.status || 500} - ${err}` + ); + throw err; + } } - async createKnowledgeGraphDoc (req, _id, label, result) { - req.logger.info('elasticsearch.createKnowledgeGraphDoc(): Function entry'); + async createKnowledgeGraphDoc (log, _id, label, result) { + log.info('elasticsearch.createKnowledgeGraphDoc(): Function entry'); // create new doc under knowledge index const url = `http://${env.ESADDRESS}/knowledge/_update/${_id}`; - return await axios({ - method: 'post', - url, - headers: { - 'Content-Type': 'application/json' - }, - data: JSON.stringify({ - doc: { - label, - response: result, - date: new Date().toISOString().slice(0, 10) + try { + return await axios({ + method: 'post', + url, + headers: { + 'Content-Type': 'application/json' }, - doc_as_upsert: true - }) - }); + data: JSON.stringify({ + doc: { + label, + response: result, + date: new Date().toISOString().slice(0, 10) + }, + doc_as_upsert: true + }) + }); + } catch (err) { + log.error( + `elasticsearch.createKnowledgeGraphDoc(): ${err.status || 500} - ${err}` + ); + throw err; + } } }