Skip to content

Commit

Permalink
refactor logic for creating DSL from saved object using buildOpensear…
Browse files Browse the repository at this point in the history 
…chQuery() (#213)
  • Loading branch information
@zhongnansu
zhongnansu authored Nov 3, 2021
1 parent 30bd9cf commit b4ec424
Show file tree
Hide file tree
Showing 4 changed files with 96 additions and 137 deletions.
18 changes: 0 additions & 18 deletions dashboards-reports/.babelrc
View file

This file was deleted.

23 changes: 23 additions & 0 deletions dashboards-reports/babel.config.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

module.exports = {
presets: [
require('@babel/preset-env', {
targets: { node: '10' },
}),
require('@babel/preset-react'),
require('@babel/preset-typescript'),
],
plugins: [
require('@babel/plugin-proposal-class-properties'),
require('@babel/plugin-proposal-object-rest-spread'),
['@babel/plugin-transform-modules-commonjs', { allowTopLevelThis: true }],
[require('@babel/plugin-transform-runtime'), { regenerator: true }],
],
};
163 changes: 67 additions & 96 deletions dashboards-reports/server/routes/utils/dataReportHelpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ import converter from 'json-2-csv';
import _ from 'lodash';
import moment from 'moment';
import { DATA_REPORT_CONFIG } from './constants';
import {
buildOpenSearchQuery,
Filter,
Query,
} from '../../../../../src/plugins/data/common';

export var metaData = {
saved_search_id: <string>null,
Expand All @@ -42,7 +47,7 @@ export var metaData = {
fields_exist: <boolean>false,
selectedFields: <any>[],
paternName: <string>null,
filters: <any>[],
searchSourceJSON: <any>[],
dateFields: <any>[],
};

Expand All @@ -65,93 +70,20 @@ export const getSelectedFields = async (columns) => {

// Build the OpenSearch query from the meta data
// is_count is set to 1 if we building the count query but 0 if we building the fetch data query
export const buildQuery = (report, is_count) => {
let requestBody = esb.boolQuery();
const filters = report._source.filters;
for (let item of JSON.parse(filters).filter) {
if (item.meta.disabled === false) {
switch (item.meta.negate) {
case false:
switch (item.meta.type) {
case 'phrase':
requestBody.must(
esb.matchPhraseQuery(item.meta.key, item.meta.params.query)
);
break;
case 'exists':
requestBody.must(esb.existsQuery(item.meta.key));
break;
case 'phrases':
if (item.meta.value.indexOf(',') > -1) {
const valueSplit = item.meta.value.split(', ');
for (const [key, incr] of valueSplit.entries()) {
requestBody.should(esb.matchPhraseQuery(item.meta.key, incr));
}
} else {
requestBody.should(
esb.matchPhraseQuery(item.meta.key, item.meta.value)
);
}
requestBody.minimumShouldMatch(1);
break;
case 'range':
const builder = esb.rangeQuery(item.meta.key);
if (item.meta.params.gte) builder.gte(item.meta.params.gte);
if (item.meta.params.lte) builder.lte(item.meta.params.lte);
if (item.meta.params.gt) builder.gt(item.meta.params.gt);
if (item.meta.params.lt) builder.lt(item.meta.params.lt);
requestBody.must(builder);
break;
}
break;
case true:
switch (item.meta.type) {
case 'phrase':
requestBody.mustNot(
esb.matchPhraseQuery(item.meta.key, item.meta.params.query)
);
break;
case 'exists':
requestBody.mustNot(esb.existsQuery(item.meta.key));
break;
case 'phrases':
let negatedBody = esb.boolQuery();
if (item.meta.value.indexOf(',') > -1) {
const valueSplit = item.meta.value.split(', ');
for (const [key, incr] of valueSplit.entries()) {
negatedBody.should(esb.matchPhraseQuery(item.meta.key, incr));
}
} else {
negatedBody.should(
esb.matchPhraseQuery(item.meta.key, item.meta.value)
);
}
negatedBody.minimumShouldMatch(1);
requestBody.mustNot(negatedBody);
break;
case 'range':
const builder = esb.rangeQuery(item.meta.key);
if (item.meta.params.gte) builder.gte(item.meta.params.gte);
if (item.meta.params.lte) builder.lte(item.meta.params.lte);
if (item.meta.params.gt) builder.gt(item.meta.params.gt);
if (item.meta.params.lt) builder.lt(item.meta.params.lt);
requestBody.mustNot(builder);
break;
}
break;
}
}
}
//search part
let searchQuery = JSON.parse(filters)
.query.query.replace(/ and /g, ' AND ')
.replace(/ or /g, ' OR ')
.replace(/ not /g, ' NOT ');
if (searchQuery) {
requestBody.must(esb.queryStringQuery(searchQuery));
}
export const buildRequestBody = (report: any, is_count: number) => {
let esbBoolQuery = esb.boolQuery();
const searchSourceJSON = report._source.searchSourceJSON;

const savedObjectQuery: Query = JSON.parse(searchSourceJSON).query;
const savedObjectFilter: Filter = JSON.parse(searchSourceJSON).filter;
const QueryFromSavedObject = buildOpenSearchQuery(
undefined,
savedObjectQuery,
savedObjectFilter
);
// Add time range
if (report._source.timeFieldName && report._source.timeFieldName.length > 0) {
requestBody.must(
esbBoolQuery.must(
esb
.rangeQuery(report._source.timeFieldName)
.format('epoch_millis')
Expand All @@ -160,36 +92,58 @@ export const buildQuery = (report, is_count) => {
);
}
if (is_count) {
return esb.requestBodySearch().query(requestBody);
return esb.requestBodySearch().query(esbBoolQuery);
}

//Add the Sort to the query
let reqBody = esb.requestBodySearch().query(requestBody).version(true);
// Add sorting to the query
let esbSearchQuery = esb
.requestBodySearch()
.query(esbBoolQuery)
.version(true);

if (report._source.sorting.length > 0) {
const sortings: Sort[] = report._source.sorting.map((element: string[]) => {
return esb.sort(element[0], element[1]);
});
reqBody.sorts(sortings);
esbSearchQuery.sorts(sortings);
}

//get the selected fields only
// add selected fields to query
if (report._source.fields_exist) {
reqBody.source({ includes: report._source.selectedFields });
esbSearchQuery.source({ includes: report._source.selectedFields });
}
return reqBody;
// Add a customizer to merge queries to generate request body
let requestBody = _.mergeWith(
{ query: QueryFromSavedObject },
esbSearchQuery.toJSON(),
(objValue, srcValue) => {
if (_.isArray(objValue)) {
return objValue.concat(srcValue);
}
}
);

requestBody = addDocValueFields(report, requestBody);
return requestBody;
};

// Fetch the data from OpenSearch
export const getOpenSearchData = (arrayHits, report, params, dateFormat: string) => {
export const getOpenSearchData = (
arrayHits,
report,
params,
dateFormat: string
) => {
let hits: any = [];
for (let valueRes of arrayHits) {
for (let data of valueRes.hits) {
const fields = data.fields;
// get all the fields of type date and format them to excel format
for (let dateType of report._source.dateFields) {
if (data._source[dateType]) {
data._source[dateType] = moment(fields[dateType][0]).format(dateFormat);
data._source[dateType] = moment(fields[dateType][0]).format(
dateFormat
);
}
}
delete data['fields'];
Expand Down Expand Up @@ -274,3 +228,20 @@ function sanitize(doc: any) {
}
return doc;
}

const addDocValueFields = (report: any, requestBody: any) => {
const docValues = [];
for (const dateType of report._source.dateFields) {
docValues.push({
field: dateType,
format: 'date_hour_minute_second_fraction',
});
}
// elastic-builder doesn't provide function to build docvalue_fields with format,
// this is a workaround which appends docvalues field to the request body.
requestBody = {
...requestBody,
docvalue_fields: docValues,
};
return requestBody;
};
29 changes: 6 additions & 23 deletions dashboards-reports/server/routes/utils/savedSearchReportHelper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
*/

import {
buildQuery,
buildRequestBody,
convertToCSV,
getOpenSearchData,
getSelectedFields,
Expand Down Expand Up @@ -104,15 +104,15 @@ async function populateMetaData(

metaData.sorting = ssInfos._source.search.sort;
metaData.type = ssInfos._source.type;
metaData.filters =
metaData.searchSourceJSON =
ssInfos._source.search.kibanaSavedObjectMeta.searchSourceJSON;

// Get the list of selected columns in the saved search.Otherwise select all the fields under the _source
await getSelectedFields(ssInfos._source.search.columns);

// Get index name
for (const item of ssInfos._source.references) {
if (item.name === JSON.parse(metaData.filters).indexRefName) {
if (item.name === JSON.parse(metaData.searchSourceJSON).indexRefName) {
// Get index-pattern information
const indexPattern = await callCluster(
client,
Expand Down Expand Up @@ -163,7 +163,7 @@ async function generateReportData(
return '';
}

const reqBody = buildRequestBody(buildQuery(report, 0));
const reqBody = buildRequestBody(report, 0);
logger.info(
`[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}`
);
Expand Down Expand Up @@ -200,13 +200,13 @@ async function generateReportData(

// Build the OpenSearch Count query to count the size of result
async function getOpenSearchDataSize() {
const countReq = buildQuery(report, 1);
const countReq = buildRequestBody(report, 1);
return await callCluster(
client,
'count',
{
index: indexPattern,
body: countReq.toJSON(),
body: countReq,
},
isScheduledTask
);
Expand Down Expand Up @@ -273,23 +273,6 @@ async function generateReportData(
arrayHits.push(opensearchData.hits);
}

function buildRequestBody(query: esb.RequestBodySearch) {
const docvalues = [];
for (const dateType of report._source.dateFields) {
docvalues.push({
field: dateType,
format: 'date_hour_minute_second_fraction',
});
}

// elastic-builder doesn't provide function to build docvalue_fields with format,
// this is a workaround which appends docvalues field to the request body.
return {
...query.toJSON(),
docvalue_fields: docvalues,
};
}

// Parse OpenSearch data and convert to CSV
async function convertOpenSearchDataToCsv() {
const dataset: any = [];
Expand Down

0 comments on commit b4ec424

Please sign in to comment.