[Reporting] Fix ability to export CSV on searched data with frozen indices

x-pack/plugins/reporting/common/constants.ts

@@ -45,6 +45,7 @@ export const KBN_SCREENSHOT_HEADER_BLOCK_LIST = [
 
 export const KBN_SCREENSHOT_HEADER_BLOCK_LIST_STARTS_WITH_PATTERN = ['proxy-'];
 
+export const UI_SETTINGS_SEARCH_INCLUDE_FROZEN = 'search:includeFrozen';
 export const UI_SETTINGS_CUSTOM_PDF_LOGO = 'xpackReporting:customPdfLogo';
 export const UI_SETTINGS_CSV_SEPARATOR = 'csv:separator';
 export const UI_SETTINGS_CSV_QUOTE_VALUES = 'csv:quoteValues';

x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts

@@ -198,7 +198,7 @@ it('calculates the bytes of the content', async () => {
  Rx.of({
  rawResponse: {
  hits: {
- hits: range(0, HITS_TOTAL).map((hit, i) => ({
+ hits: range(0, HITS_TOTAL).map(() => ({
  fields: {
  message: ['this is a great message'],
  },
@@ -248,7 +248,7 @@ it('warns if max size was reached', async () => {
  Rx.of({
  rawResponse: {
  hits: {
- hits: range(0, HITS_TOTAL).map((hit, i) => ({
+ hits: range(0, HITS_TOTAL).map(() => ({
  fields: {
  date: ['2020-12-31T00:14:28.000Z'],
  ip: ['110.135.176.89'],
@@ -289,7 +289,7 @@ it('uses the scrollId to page all the data', async () => {
  rawResponse: {
  _scroll_id: 'awesome-scroll-hero',
  hits: {
- hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
+ hits: range(0, HITS_TOTAL / 10).map(() => ({
  fields: {
  date: ['2020-12-31T00:14:28.000Z'],
  ip: ['110.135.176.89'],
@@ -304,7 +304,7 @@ it('uses the scrollId to page all the data', async () => {
  mockEsClient.asCurrentUser.scroll = jest.fn().mockResolvedValue({
  body: {
  hits: {
- hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
+ hits: range(0, HITS_TOTAL / 10).map(() => ({
  fields: {
  date: ['2020-12-31T00:14:28.000Z'],
  ip: ['110.135.176.89'],
@@ -337,7 +337,7 @@ it('uses the scrollId to page all the data', async () => {
 
  expect(mockDataClient.search).toHaveBeenCalledTimes(1);
  expect(mockDataClient.search).toBeCalledWith(
- { params: { scroll: '30s', size: 500 } },
+ { params: { ignore_throttled: true, scroll: '30s', size: 500 } },
  { strategy: 'es' }
  );
 

x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts

@@ -83,8 +83,9 @@ export class CsvGenerator {
  private async scan(
  index: IndexPattern,
  searchSource: ISearchSource,
- scrollSettings: CsvExportSettings['scroll']
+ settings: CsvExportSettings
  ) {
+ const { scroll: scrollSettings, includeFrozen } = settings;
  const searchBody = searchSource.getSearchRequestBody();
  this.logger.debug(`executing search request`);
  const searchParams = {
@@ -93,8 +94,10 @@ export class CsvGenerator {
  index: index.title,
  scroll: scrollSettings.duration,
  size: scrollSettings.size,
+ ignore_throttled: !includeFrozen,
  },
  };
+
  const results = (
  await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise()
  ).rawResponse as estypes.SearchResponse<unknown>;
@@ -326,7 +329,7 @@ export class CsvGenerator {
  let results: estypes.SearchResponse<unknown> | undefined;
  if (scrollId == null) {
  // open a scroll cursor in Elasticsearch
- results = await this.scan(index, searchSource, scrollSettings);
+ results = await this.scan(index, searchSource, settings);
  scrollId = results?._scroll_id;
  if (results.hits?.total != null) {
  totalRecords = results.hits.total as number;

x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.test.ts

@@ -9,6 +9,7 @@ import {
  UI_SETTINGS_DATEFORMAT_TZ,
  UI_SETTINGS_CSV_QUOTE_VALUES,
  UI_SETTINGS_CSV_SEPARATOR,
+ UI_SETTINGS_SEARCH_INCLUDE_FROZEN,
 } from '../../../../common/constants';
 import { IUiSettingsClient } from 'kibana/server';
 import { savedObjectsClientMock, uiSettingsServiceMock } from 'src/core/server/mocks';
@@ -36,6 +37,8 @@ describe('getExportSettings', () => {
  return ',';
  case UI_SETTINGS_DATEFORMAT_TZ:
  return 'Browser';
+ case UI_SETTINGS_SEARCH_INCLUDE_FROZEN:
+ return false;
  }
 
  return 'helo world';
@@ -49,6 +52,7 @@ describe('getExportSettings', () => {
  "checkForFormulas": undefined,
  "escapeFormulaValues": undefined,
  "escapeValue": [Function],
+ "includeFrozen": false,
  "maxSizeBytes": undefined,
  "scroll": Object {
  "duration": undefined,

x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/get_export_settings.ts

@@ -12,9 +12,10 @@ import { createEscapeValue } from '../../../../../../../src/plugins/data/common'
 import { ReportingConfig } from '../../../';
 import {
  CSV_BOM_CHARS,
- UI_SETTINGS_DATEFORMAT_TZ,
  UI_SETTINGS_CSV_QUOTE_VALUES,
  UI_SETTINGS_CSV_SEPARATOR,
+ UI_SETTINGS_DATEFORMAT_TZ,
+ UI_SETTINGS_SEARCH_INCLUDE_FROZEN,
 } from '../../../../common/constants';
 import { LevelLogger } from '../../../lib';
 
@@ -30,6 +31,7 @@ export interface CsvExportSettings {
  checkForFormulas: boolean;
  escapeFormulaValues: boolean;
  escapeValue: (value: string) => string;
+ includeFrozen: boolean;
 }
 
 export const getExportSettings = async (
@@ -38,9 +40,7 @@ export const getExportSettings = async (
  timezone: string | undefined,
  logger: LevelLogger
 ): Promise<CsvExportSettings> => {
- // Timezone
  let setTimezone: string;
- // timezone in job params?
  if (timezone) {
  setTimezone = timezone;
  } else {
@@ -60,7 +60,8 @@ export const getExportSettings = async (
  }
 
  // Separator, QuoteValues
- const [separator, quoteValues] = await Promise.all([
+ const [includeFrozen, separator, quoteValues] = await Promise.all([
+ client.get(UI_SETTINGS_SEARCH_INCLUDE_FROZEN),
  client.get(UI_SETTINGS_CSV_SEPARATOR),
  client.get(UI_SETTINGS_CSV_QUOTE_VALUES),
  ]);
@@ -76,6 +77,7 @@ export const getExportSettings = async (
  duration: config.get('csv', 'scroll', 'duration'),
  },
  bom,
+ includeFrozen,
  separator,
  maxSizeBytes: config.get('csv', 'maxSizeBytes'),
  checkForFormulas: config.get('csv', 'checkForFormulas'),

x-pack/test/reporting_api_integration/reporting_and_security/index.ts

@@ -29,5 +29,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) {
  loadTestFile(require.resolve('./spaces'));
  loadTestFile(require.resolve('./usage'));
  loadTestFile(require.resolve('./ilm_migration_apis'));
+ loadTestFile(require.resolve('./search_frozen_indices'));
  });
 }

x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts

@@ -0,0 +1,146 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { Filter, FilterMeta } from '@kbn/es-query';
+import { SortDirectionFormat } from 'src/plugins/data/common';
+import { JobParamsDownloadCSV } from '../../../plugins/reporting/server/export_types/csv_searchsource_immediate/types';
+import { FtrProviderContext } from '../ftr_provider_context';
+
+// eslint-disable-next-line import/no-default-export
+export default function ({ getService }: FtrProviderContext) {
+ const kibanaServer = getService('kibanaServer');
+ const supertestSvc = getService('supertest');
+ const esSupertest = getService('esSupertest');
+
+ const generateAPI = {
+ getCSVFromSearchSource: async (job: JobParamsDownloadCSV) => {
+ return await supertestSvc
+ .post(`/api/reporting/v1/generate/immediate/csv_searchsource`)
+ .set('kbn-xsrf', 'xxx')
+ .send(job);
+ },
+ };
+
+ describe('Frozen indices search', () => {
+ const indexPatternId = 'cool-test-index-pattern';
+ const reset = async () => {
+ await esSupertest.delete('/test1,test2,test3');
+ await kibanaServer.savedObjects.delete({ type: 'index-pattern', id: indexPatternId });
+ await kibanaServer.uiSettings.update({
+ 'csv:quoteValues': true,
+ 'search:includeFrozen': false,
+ });
+ };
+
+ before(reset);
+ after(reset);
+
+ it('Search includes frozen indices based on Advanced Setting', async () => {
+ // setup: settings and clear data from previous testing
+ await kibanaServer.uiSettings.update({ 'csv:quoteValues': true });
+ await esSupertest.delete('/test2,test2,test3');
+
+ // setup: add mwhdflaksjefhultiple indices of test data
+ await Promise.all([
+ esSupertest
+ .post('/test1/_doc')
+ .send({ '@timestamp': '2021-08-24T21:36:40Z', ip: '43.98.8.183', utilization: 18725 }),
+ esSupertest
+ .post('/test2/_doc')
+ .send({ '@timestamp': '2021-08-21T09:36:40Z', ip: '63.91.103.79', utilization: 8480 }),
+ esSupertest
+ .post('/test3/_doc')
+ .send({ '@timestamp': '2021-08-17T21:36:40Z', ip: '139.108.162.171', utilization: 3078 }),
+ ]);
+ await esSupertest.post('/test*/_refresh');
+
+ // setup: create index pattern
+ const indexPatternCreateResponse = await kibanaServer.savedObjects.create({
+ type: 'index-pattern',
+ id: indexPatternId,
+ overwrite: true,
+ attributes: { title: 'test*', timeFieldName: '@timestamp' },
+ });
+
+ expect(indexPatternCreateResponse.id).to.be(indexPatternId);
+
+ const callExportAPI = async () => {
+ return await generateAPI.getCSVFromSearchSource({
+ browserTimezone: 'UTC',
+ columns: ['@timestamp', 'ip', 'utilization'],
+ searchSource: {
+ fields: [{ field: '*', include_unmapped: 'true' }],
+ filter: [
+ {
+ meta: {
+ field: '@timestamp',
+ index: indexPatternId,
+ params: {},
+ } as FilterMeta,
+ range: {
+ '@timestamp': {
+ format: 'strict_date_optional_time',
+ gte: '2020-08-24T00:00:00.000Z',
+ lte: '2022-08-24T21:40:48.346Z',
+ },
+ },
+ } as Filter,
+ ],
+ // @ts-expect-error Type 'string' is not assignable to type 'IndexPattern | undefined'.
+ index: indexPatternId,
+ parent: {
+ filter: [],
+ // @ts-expect-error Type 'string' is not assignable to type 'IndexPattern | undefined'.
+ index: indexPatternId,
+ query: { language: 'kuery', query: '' },
+ },
+ sort: ([{ '@timestamp': 'desc' }] as unknown) as Record<string, SortDirectionFormat>,
+ trackTotalHits: true,
+ },
+ title: 'Test search',
+ });
+ };
+
+ // 1. check the initial data with a CSV export
+ const initialSearch = await callExportAPI();
+ expectSnapshot(initialSearch.text).toMatchInline(`
+ "\\"@timestamp\\",ip,utilization
+ \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
+ \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
+ \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\"
+ "
+ `);
+
+ // 2. freeze an index in the pattern
+ await esSupertest.post('/test3/_freeze').expect(200);
+ await esSupertest.post('/test*/_refresh').expect(200);
+
+ // 3. recheck the search results
+ const afterFreezeSearch = await callExportAPI();
+ expectSnapshot(afterFreezeSearch.text).toMatchInline(`
+ "\\"@timestamp\\",ip,utilization
+ \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
+ \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
+ "
+ `);
+
+ // 4. update setting to allow searching frozen data
+ await kibanaServer.uiSettings.update({ 'search:includeFrozen': true });
+
+ // 5. recheck the search results
+ const afterAllowSearch = await callExportAPI();
+ expectSnapshot(afterAllowSearch.text).toMatchInline(`
+ "\\"@timestamp\\",ip,utilization
+ \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
+ \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
+ \\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\"
+ "
+ `);
+ });
+ });
+}