refactor logic for creating DSL from saved object using buildOpensear…

…chQuery() (opensearch-project#213)
zhongnansu · Nov 9, 2021 · 73c3767 · 73c3767
1 parent 1cae8f5
commit 73c3767
Show file tree

Hide file tree

Showing 4 changed files with 96 additions and 137 deletions.
diff --git a/dashboards-reports/.babelrc b/dashboards-reports/.babelrc
diff --git a/dashboards-reports/babel.config.js b/dashboards-reports/babel.config.js
@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+module.exports = {
+  presets: [
+    require('@babel/preset-env', {
+      targets: { node: '10' },
+    }),
+    require('@babel/preset-react'),
+    require('@babel/preset-typescript'),
+  ],
+  plugins: [
+    require('@babel/plugin-proposal-class-properties'),
+    require('@babel/plugin-proposal-object-rest-spread'),
+    ['@babel/plugin-transform-modules-commonjs', { allowTopLevelThis: true }],
+    [require('@babel/plugin-transform-runtime'), { regenerator: true }],
+  ],
+};
diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts
@@ -29,6 +29,11 @@ import converter from 'json-2-csv';
 import _ from 'lodash';
 import moment from 'moment';
 import { DATA_REPORT_CONFIG } from './constants';
+import {
+  buildOpenSearchQuery,
+  Filter,
+  Query,
+} from '../../../../../src/plugins/data/common';
 
 export var metaData = {
   saved_search_id: <string>null,
@@ -42,7 +47,7 @@ export var metaData = {
   fields_exist: <boolean>false,
   selectedFields: <any>[],
   paternName: <string>null,
-  filters: <any>[],
+  searchSourceJSON: <any>[],
   dateFields: <any>[],
 };
 
@@ -65,93 +70,20 @@ export const getSelectedFields = async (columns) => {
 
 // Build the OpenSearch query from the meta data
 // is_count is set to 1 if we building the count query but 0 if we building the fetch data query
-export const buildQuery = (report, is_count) => {
-  let requestBody = esb.boolQuery();
-  const filters = report._source.filters;
-  for (let item of JSON.parse(filters).filter) {
-    if (item.meta.disabled === false) {
-      switch (item.meta.negate) {
-        case false:
-          switch (item.meta.type) {
-            case 'phrase':
-              requestBody.must(
-                esb.matchPhraseQuery(item.meta.key, item.meta.params.query)
-              );
-              break;
-            case 'exists':
-              requestBody.must(esb.existsQuery(item.meta.key));
-              break;
-            case 'phrases':
-              if (item.meta.value.indexOf(',') > -1) {
-                const valueSplit = item.meta.value.split(', ');
-                for (const [key, incr] of valueSplit.entries()) {
-                  requestBody.should(esb.matchPhraseQuery(item.meta.key, incr));
-                }
-              } else {
-                requestBody.should(
-                  esb.matchPhraseQuery(item.meta.key, item.meta.value)
-                );
-              }
-              requestBody.minimumShouldMatch(1);
-              break;
-            case 'range':
-              const builder = esb.rangeQuery(item.meta.key);
-              if (item.meta.params.gte) builder.gte(item.meta.params.gte);
-              if (item.meta.params.lte) builder.lte(item.meta.params.lte);
-              if (item.meta.params.gt) builder.gt(item.meta.params.gt);
-              if (item.meta.params.lt) builder.lt(item.meta.params.lt);
-              requestBody.must(builder);
-              break;
-          }
-          break;
-        case true:
-          switch (item.meta.type) {
-            case 'phrase':
-              requestBody.mustNot(
-                esb.matchPhraseQuery(item.meta.key, item.meta.params.query)
-              );
-              break;
-            case 'exists':
-              requestBody.mustNot(esb.existsQuery(item.meta.key));
-              break;
-            case 'phrases':
-              let negatedBody = esb.boolQuery();
-              if (item.meta.value.indexOf(',') > -1) {
-                const valueSplit = item.meta.value.split(', ');
-                for (const [key, incr] of valueSplit.entries()) {
-                  negatedBody.should(esb.matchPhraseQuery(item.meta.key, incr));
-                }
-              } else {
-                negatedBody.should(
-                  esb.matchPhraseQuery(item.meta.key, item.meta.value)
-                );
-              }
-              negatedBody.minimumShouldMatch(1);
-              requestBody.mustNot(negatedBody);
-              break;
-            case 'range':
-              const builder = esb.rangeQuery(item.meta.key);
-              if (item.meta.params.gte) builder.gte(item.meta.params.gte);
-              if (item.meta.params.lte) builder.lte(item.meta.params.lte);
-              if (item.meta.params.gt) builder.gt(item.meta.params.gt);
-              if (item.meta.params.lt) builder.lt(item.meta.params.lt);
-              requestBody.mustNot(builder);
-              break;
-          }
-          break;
-      }
-    }
-  }
-  //search part
-  let searchQuery = JSON.parse(filters)
-    .query.query.replace(/ and /g, ' AND ')
-    .replace(/ or /g, ' OR ')
-    .replace(/ not /g, ' NOT ');
-  if (searchQuery) {
-    requestBody.must(esb.queryStringQuery(searchQuery));
-  }
+export const buildRequestBody = (report: any, is_count: number) => {
+  let esbBoolQuery = esb.boolQuery();
+  const searchSourceJSON = report._source.searchSourceJSON;
+
+  const savedObjectQuery: Query = JSON.parse(searchSourceJSON).query;
+  const savedObjectFilter: Filter = JSON.parse(searchSourceJSON).filter;
+  const QueryFromSavedObject = buildOpenSearchQuery(
+    undefined,
+    savedObjectQuery,
+    savedObjectFilter
+  );
+  // Add time range
   if (report._source.timeFieldName && report._source.timeFieldName.length > 0) {
-    requestBody.must(
+    esbBoolQuery.must(
       esb
         .rangeQuery(report._source.timeFieldName)
         .format('epoch_millis')
@@ -160,36 +92,58 @@ export const buildQuery = (report, is_count) => {
     );
   }
   if (is_count) {
-    return esb.requestBodySearch().query(requestBody);
+    return esb.requestBodySearch().query(esbBoolQuery);
   }
 
-  //Add the Sort to the query
-  let reqBody = esb.requestBodySearch().query(requestBody).version(true);
+  // Add sorting to the query
+  let esbSearchQuery = esb
+    .requestBodySearch()
+    .query(esbBoolQuery)
+    .version(true);
 
   if (report._source.sorting.length > 0) {
     const sortings: Sort[] = report._source.sorting.map((element: string[]) => {
       return esb.sort(element[0], element[1]);
     });
-    reqBody.sorts(sortings);
+    esbSearchQuery.sorts(sortings);
   }
 
-  //get the selected fields only
+  // add selected fields to query
   if (report._source.fields_exist) {
-    reqBody.source({ includes: report._source.selectedFields });
+    esbSearchQuery.source({ includes: report._source.selectedFields });
   }
-  return reqBody;
+  // Add a customizer to merge queries to generate request body
+  let requestBody = _.mergeWith(
+    { query: QueryFromSavedObject },
+    esbSearchQuery.toJSON(),
+    (objValue, srcValue) => {
+      if (_.isArray(objValue)) {
+        return objValue.concat(srcValue);
+      }
+    }
+  );
+
+  requestBody = addDocValueFields(report, requestBody);
+  return requestBody;
 };
 
 // Fetch the data from OpenSearch
-export const getOpenSearchData = (arrayHits, report, params, dateFormat: string) => {
+export const getOpenSearchData = (
+  arrayHits,
+  report,
+  params,
+  dateFormat: string
+) => {
   let hits: any = [];
   for (let valueRes of arrayHits) {
     for (let data of valueRes.hits) {
       const fields = data.fields;
       // get all the fields of type date and format them to excel format
       for (let dateType of report._source.dateFields) {
         if (data._source[dateType]) {
-          data._source[dateType] = moment(fields[dateType][0]).format(dateFormat);
+          data._source[dateType] = moment(fields[dateType][0]).format(
+            dateFormat
+          );
         }
       }
       delete data['fields'];
@@ -274,3 +228,20 @@ function sanitize(doc: any) {
   }
   return doc;
 }
+
+const addDocValueFields = (report: any, requestBody: any) => {
+  const docValues = [];
+  for (const dateType of report._source.dateFields) {
+    docValues.push({
+      field: dateType,
+      format: 'date_hour_minute_second_fraction',
+    });
+  }
+  // elastic-builder doesn't provide function to build docvalue_fields with format,
+  // this is a workaround which appends docvalues field to the request body.
+  requestBody = {
+    ...requestBody,
+    docvalue_fields: docValues,
+  };
+  return requestBody;
+};
diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts
@@ -25,7 +25,7 @@
  */
 
 import {
-  buildQuery,
+  buildRequestBody,
   convertToCSV,
   getOpenSearchData,
   getSelectedFields,
@@ -104,15 +104,15 @@ async function populateMetaData(
 
   metaData.sorting = ssInfos._source.search.sort;
   metaData.type = ssInfos._source.type;
-  metaData.filters =
+  metaData.searchSourceJSON =
     ssInfos._source.search.kibanaSavedObjectMeta.searchSourceJSON;
 
   // Get the list of selected columns in the saved search.Otherwise select all the fields under the _source
   await getSelectedFields(ssInfos._source.search.columns);
 
   // Get index name
   for (const item of ssInfos._source.references) {
-    if (item.name === JSON.parse(metaData.filters).indexRefName) {
+    if (item.name === JSON.parse(metaData.searchSourceJSON).indexRefName) {
       // Get index-pattern information
       const indexPattern = await callCluster(
         client,
@@ -163,7 +163,7 @@ async function generateReportData(
     return '';
   }
 
-  const reqBody = buildRequestBody(buildQuery(report, 0));
+  const reqBody = buildRequestBody(report, 0);
   logger.info(
     `[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}`
   );
@@ -200,13 +200,13 @@ async function generateReportData(
 
   // Build the OpenSearch Count query to count the size of result
   async function getOpenSearchDataSize() {
-    const countReq = buildQuery(report, 1);
+    const countReq = buildRequestBody(report, 1);
     return await callCluster(
       client,
       'count',
       {
         index: indexPattern,
-        body: countReq.toJSON(),
+        body: countReq,
       },
       isScheduledTask
     );
@@ -273,23 +273,6 @@ async function generateReportData(
     arrayHits.push(opensearchData.hits);
   }
 
-  function buildRequestBody(query: esb.RequestBodySearch) {
-    const docvalues = [];
-    for (const dateType of report._source.dateFields) {
-      docvalues.push({
-        field: dateType,
-        format: 'date_hour_minute_second_fraction',
-      });
-    }
-
-    // elastic-builder doesn't provide function to build docvalue_fields with format,
-    // this is a workaround which appends docvalues field to the request body.
-    return {
-      ...query.toJSON(),
-      docvalue_fields: docvalues,
-    };
-  }
-
   // Parse OpenSearch data and convert to CSV
   async function convertOpenSearchDataToCsv() {
     const dataset: any = [];