From 63c1aeb2062c2a8452e535af795e03e343e393ef Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 24 Aug 2021 16:33:09 -0700 Subject: [PATCH] add fixes --- .../generate_csv/generate_csv.test.ts | 10 ++--- .../generate_csv/generate_csv.ts | 9 ++-- .../search_frozen_indices.ts | 41 ++++++++++--------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts index 3e4e663733e2ce3..cb5670f6eafd960 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts @@ -198,7 +198,7 @@ it('calculates the bytes of the content', async () => { Rx.of({ rawResponse: { hits: { - hits: range(0, HITS_TOTAL).map((hit, i) => ({ + hits: range(0, HITS_TOTAL).map(() => ({ fields: { message: ['this is a great message'], }, @@ -248,7 +248,7 @@ it('warns if max size was reached', async () => { Rx.of({ rawResponse: { hits: { - hits: range(0, HITS_TOTAL).map((hit, i) => ({ + hits: range(0, HITS_TOTAL).map(() => ({ fields: { date: ['2020-12-31T00:14:28.000Z'], ip: ['110.135.176.89'], @@ -289,7 +289,7 @@ it('uses the scrollId to page all the data', async () => { rawResponse: { _scroll_id: 'awesome-scroll-hero', hits: { - hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({ + hits: range(0, HITS_TOTAL / 10).map(() => ({ fields: { date: ['2020-12-31T00:14:28.000Z'], ip: ['110.135.176.89'], @@ -304,7 +304,7 @@ it('uses the scrollId to page all the data', async () => { mockEsClient.asCurrentUser.scroll = jest.fn().mockResolvedValue({ body: { hits: { - hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({ + hits: range(0, HITS_TOTAL / 10).map(() => ({ fields: { date: ['2020-12-31T00:14:28.000Z'], ip: ['110.135.176.89'], @@ -337,7 +337,7 @@ it('uses the scrollId to page all the data', async () => { expect(mockDataClient.search).toHaveBeenCalledTimes(1); expect(mockDataClient.search).toBeCalledWith( - { params: { scroll: '30s', size: 500 } }, + { params: { ignore_throttled: true, scroll: '30s', size: 500 } }, { strategy: 'es' } ); diff --git a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts index 85fc531320247bf..7f01f9fcb297cb8 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts @@ -66,7 +66,6 @@ function isPlainStringArray( export class CsvGenerator { private _columns?: string[]; private _formatters?: Record; - private includeFrozen: boolean; private csvContainsFormulas = false; private maxSizeReached = false; private csvRowCount = 0; @@ -84,8 +83,9 @@ export class CsvGenerator { private async scan( index: IndexPattern, searchSource: ISearchSource, - scrollSettings: CsvExportSettings['scroll'] + settings: CsvExportSettings ) { + const { scroll: scrollSettings, includeFrozen } = settings; const searchBody = searchSource.getSearchRequestBody(); this.logger.debug(`executing search request`); const searchParams = { @@ -94,9 +94,10 @@ export class CsvGenerator { index: index.title, scroll: scrollSettings.duration, size: scrollSettings.size, - ignore_throttled: this.includeFrozen, + ignore_throttled: !includeFrozen, }, }; + const results = ( await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise() ).rawResponse as estypes.SearchResponse; @@ -328,7 +329,7 @@ export class CsvGenerator { let results: estypes.SearchResponse | undefined; if (scrollId == null) { // open a scroll cursor in Elasticsearch - results = await this.scan(index, searchSource, scrollSettings); + results = await this.scan(index, searchSource, settings); scrollId = results?._scroll_id; if (results.hits?.total != null) { totalRecords = results.hits.total as number; diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts index 12efd84ae282673..13a80e87e0f56be 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts @@ -5,6 +5,7 @@ * 2.0. */ +import expect from '@kbn/expect'; import { Filter, FilterMeta } from '@kbn/es-query'; import { SortDirectionFormat } from 'src/plugins/data/common'; import { JobParamsDownloadCSV } from '../../../plugins/reporting/server/export_types/csv_searchsource_immediate/types'; @@ -15,7 +16,6 @@ export default function ({ getService }: FtrProviderContext) { const kibanaServer = getService('kibanaServer'); const supertestSvc = getService('supertest'); const esSupertest = getService('esSupertest'); - const log = getService('log'); const generateAPI = { getCSVFromSearchSource: async (job: JobParamsDownloadCSV) => { @@ -46,15 +46,16 @@ export default function ({ getService }: FtrProviderContext) { // setup: add mwhdflaksjefhultiple indices of test data await Promise.all([ esSupertest - .post('/test1/_doc?refresh=true') + .post('/test1/_doc') .send({ '@timestamp': '2021-08-24T21:36:40Z', ip: '43.98.8.183', utilization: 18725 }), esSupertest - .post('/test2/_doc?refresh=true') + .post('/test2/_doc') .send({ '@timestamp': '2021-08-21T09:36:40Z', ip: '63.91.103.79', utilization: 8480 }), esSupertest - .post('/test3/_doc?refresh=true') + .post('/test3/_doc') .send({ '@timestamp': '2021-08-17T21:36:40Z', ip: '139.108.162.171', utilization: 3078 }), ]); + await esSupertest.post('/test*/_refresh'); // setup: create index pattern const indexPatternId = 'cool-test-index-pattern'; @@ -65,9 +66,9 @@ export default function ({ getService }: FtrProviderContext) { attributes: { title: 'test*', timeFieldName: '@timestamp' }, }); - log.info(`index pattern create response: ${JSON.stringify(indexPatternCreateResponse)}`); + expect(indexPatternCreateResponse.id).to.be(indexPatternId); - const callExportApi = async () => { + const callExportAPI = async () => { return await generateAPI.getCSVFromSearchSource({ browserTimezone: 'UTC', columns: ['@timestamp', 'ip', 'utilization'], @@ -106,9 +107,9 @@ export default function ({ getService }: FtrProviderContext) { // testing: call the API to export the search data as CSV try { - // check the initial data with a CSV export - const initialResult = await callExportApi(); - expectSnapshot(initialResult.text).toMatchInline(` + // 1. check the initial data with a CSV export + const initialSearch = await callExportAPI(); + expectSnapshot(initialSearch.text).toMatchInline(` "\\"@timestamp\\",ip,utilization \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" @@ -116,23 +117,25 @@ export default function ({ getService }: FtrProviderContext) { " `); - // freeze an index in the pattern - // recheck the search results - const freezeResult = await esSupertest.post('/test3/_freeze'); - log.info(`freeze index response: ${JSON.stringify(freezeResult)}`); - const afterFreezeResult = await callExportApi(); - expectSnapshot(afterFreezeResult.text).toMatchInline(` + // 2. freeze an index in the pattern + await esSupertest.post('/test3/_freeze').expect(200); + await esSupertest.post('/test*/_refresh').expect(200); + + // 3. recheck the search results + const afterFreezeSearch = await callExportAPI(); + expectSnapshot(afterFreezeSearch.text).toMatchInline(` "\\"@timestamp\\",ip,utilization \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\" " `); - // allow searching frozen data - // recheck the search results + // 4. allow searching frozen data await kibanaServer.uiSettings.update({ 'search:includeFrozen': true }); - const afterAllowSearchResult = await callExportApi(); - expectSnapshot(afterAllowSearchResult.text).toMatchInline(` + + // 5. recheck the search results + const afterAllowSearch = await callExportAPI(); + expectSnapshot(afterAllowSearch.text).toMatchInline(` "\\"@timestamp\\",ip,utilization \\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\" \\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"