Skip to content

Commit

Permalink
add fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@tsullivan
tsullivan committed Aug 24, 2021
1 parent a7cfc82 commit 63c1aeb
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 28 deletions.
10 changes: 5 additions & 5 deletions .../plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ it('calculates the bytes of the content', async () => {
Rx.of({
rawResponse: {
hits: {
hits: range(0, HITS_TOTAL).map((hit, i) => ({
hits: range(0, HITS_TOTAL).map(() => ({
fields: {
message: ['this is a great message'],
},
Expand Down Expand Up @@ -248,7 +248,7 @@ it('warns if max size was reached', async () => {
Rx.of({
rawResponse: {
hits: {
hits: range(0, HITS_TOTAL).map((hit, i) => ({
hits: range(0, HITS_TOTAL).map(() => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
Expand Down Expand Up @@ -289,7 +289,7 @@ it('uses the scrollId to page all the data', async () => {
rawResponse: {
_scroll_id: 'awesome-scroll-hero',
hits: {
hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
hits: range(0, HITS_TOTAL / 10).map(() => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
Expand All @@ -304,7 +304,7 @@ it('uses the scrollId to page all the data', async () => {
mockEsClient.asCurrentUser.scroll = jest.fn().mockResolvedValue({
body: {
hits: {
hits: range(0, HITS_TOTAL / 10).map((hit, i) => ({
hits: range(0, HITS_TOTAL / 10).map(() => ({
fields: {
date: ['2020-12-31T00:14:28.000Z'],
ip: ['110.135.176.89'],
Expand Down Expand Up @@ -337,7 +337,7 @@ it('uses the scrollId to page all the data', async () => {

expect(mockDataClient.search).toHaveBeenCalledTimes(1);
expect(mockDataClient.search).toBeCalledWith(
{ params: { scroll: '30s', size: 500 } },
{ params: { ignore_throttled: true, scroll: '30s', size: 500 } },
{ strategy: 'es' }
);

Expand Down
9 changes: 5 additions & 4 deletions x-pack/plugins/reporting/server/export_types/csv_searchsource/generate_csv/generate_csv.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ function isPlainStringArray(
export class CsvGenerator {
private _columns?: string[];
private _formatters?: Record<string, FieldFormat>;
private includeFrozen: boolean;
private csvContainsFormulas = false;
private maxSizeReached = false;
private csvRowCount = 0;
Expand All @@ -84,8 +83,9 @@ export class CsvGenerator {
private async scan(
index: IndexPattern,
searchSource: ISearchSource,
scrollSettings: CsvExportSettings['scroll']
settings: CsvExportSettings
) {
const { scroll: scrollSettings, includeFrozen } = settings;
const searchBody = searchSource.getSearchRequestBody();
this.logger.debug(`executing search request`);
const searchParams = {
Expand All @@ -94,9 +94,10 @@ export class CsvGenerator {
index: index.title,
scroll: scrollSettings.duration,
size: scrollSettings.size,
ignore_throttled: this.includeFrozen,
ignore_throttled: !includeFrozen,
},
};

const results = (
await this.clients.data.search(searchParams, { strategy: ES_SEARCH_STRATEGY }).toPromise()
).rawResponse as estypes.SearchResponse<unknown>;
Expand Down Expand Up @@ -328,7 +329,7 @@ export class CsvGenerator {
let results: estypes.SearchResponse<unknown> | undefined;
if (scrollId == null) {
// open a scroll cursor in Elasticsearch
results = await this.scan(index, searchSource, scrollSettings);
results = await this.scan(index, searchSource, settings);
scrollId = results?._scroll_id;
if (results.hits?.total != null) {
totalRecords = results.hits.total as number;
Expand Down
41 changes: 22 additions & 19 deletions x-pack/test/reporting_api_integration/reporting_and_security/search_frozen_indices.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
* 2.0.
*/

import expect from '@kbn/expect';
import { Filter, FilterMeta } from '@kbn/es-query';
import { SortDirectionFormat } from 'src/plugins/data/common';
import { JobParamsDownloadCSV } from '../../../plugins/reporting/server/export_types/csv_searchsource_immediate/types';
Expand All @@ -15,7 +16,6 @@ export default function ({ getService }: FtrProviderContext) {
const kibanaServer = getService('kibanaServer');
const supertestSvc = getService('supertest');
const esSupertest = getService('esSupertest');
const log = getService('log');

const generateAPI = {
getCSVFromSearchSource: async (job: JobParamsDownloadCSV) => {
Expand Down Expand Up @@ -46,15 +46,16 @@ export default function ({ getService }: FtrProviderContext) {
// setup: add mwhdflaksjefhultiple indices of test data
await Promise.all([
esSupertest
.post('/test1/_doc?refresh=true')
.post('/test1/_doc')
.send({ '@timestamp': '2021-08-24T21:36:40Z', ip: '43.98.8.183', utilization: 18725 }),
esSupertest
.post('/test2/_doc?refresh=true')
.post('/test2/_doc')
.send({ '@timestamp': '2021-08-21T09:36:40Z', ip: '63.91.103.79', utilization: 8480 }),
esSupertest
.post('/test3/_doc?refresh=true')
.post('/test3/_doc')
.send({ '@timestamp': '2021-08-17T21:36:40Z', ip: '139.108.162.171', utilization: 3078 }),
]);
await esSupertest.post('/test*/_refresh');

// setup: create index pattern
const indexPatternId = 'cool-test-index-pattern';
Expand All @@ -65,9 +66,9 @@ export default function ({ getService }: FtrProviderContext) {
attributes: { title: 'test*', timeFieldName: '@timestamp' },
});

log.info(`index pattern create response: ${JSON.stringify(indexPatternCreateResponse)}`);
expect(indexPatternCreateResponse.id).to.be(indexPatternId);

const callExportApi = async () => {
const callExportAPI = async () => {
return await generateAPI.getCSVFromSearchSource({
browserTimezone: 'UTC',
columns: ['@timestamp', 'ip', 'utilization'],
Expand Down Expand Up @@ -106,33 +107,35 @@ export default function ({ getService }: FtrProviderContext) {

// testing: call the API to export the search data as CSV
try {
// check the initial data with a CSV export
const initialResult = await callExportApi();
expectSnapshot(initialResult.text).toMatchInline(`
// 1. check the initial data with a CSV export
const initialSearch = await callExportAPI();
expectSnapshot(initialSearch.text).toMatchInline(`
"\\"@timestamp\\",ip,utilization
\\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
\\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
\\"Aug 17, 2021 @ 21:36:40.000\\",\\"139.108.162.171\\",\\"3,078\\"
"
`);

// freeze an index in the pattern
// recheck the search results
const freezeResult = await esSupertest.post('/test3/_freeze');
log.info(`freeze index response: ${JSON.stringify(freezeResult)}`);
const afterFreezeResult = await callExportApi();
expectSnapshot(afterFreezeResult.text).toMatchInline(`
// 2. freeze an index in the pattern
await esSupertest.post('/test3/_freeze').expect(200);
await esSupertest.post('/test*/_refresh').expect(200);

// 3. recheck the search results
const afterFreezeSearch = await callExportAPI();
expectSnapshot(afterFreezeSearch.text).toMatchInline(`
"\\"@timestamp\\",ip,utilization
\\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
\\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
"
`);

// allow searching frozen data
// recheck the search results
// 4. allow searching frozen data
await kibanaServer.uiSettings.update({ 'search:includeFrozen': true });
const afterAllowSearchResult = await callExportApi();
expectSnapshot(afterAllowSearchResult.text).toMatchInline(`

// 5. recheck the search results
const afterAllowSearch = await callExportAPI();
expectSnapshot(afterAllowSearch.text).toMatchInline(`
"\\"@timestamp\\",ip,utilization
\\"Aug 24, 2021 @ 21:36:40.000\\",\\"43.98.8.183\\",\\"18,725\\"
\\"Aug 21, 2021 @ 09:36:40.000\\",\\"63.91.103.79\\",\\"8,480\\"
Expand Down

0 comments on commit 63c1aeb

Please sign in to comment.