diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.ts b/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.ts index 10bf569bdc72d..fda91c51ef0c2 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -// @ts-ignore + import url from 'url'; import { getAbsoluteUrlFactory } from '../../../common/get_absolute_url'; import { ConditionalHeaders, JobDocPayload, KbnServer } from '../../../types'; diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.ts b/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.ts index 23fe6e69f41f8..b41e4c8218614 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + import { UI_SETTINGS_CUSTOM_PDF_LOGO } from '../../../common/constants'; import { ConditionalHeaders, JobDocPayload, KbnServer } from '../../../types'; diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/index.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/index.ts new file mode 100644 index 0000000000000..6cc56cb5fa4f1 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './index.d'; + +/* + * These functions are exported to share with the API route handler that + * generates csv from saved object immediately on request. + */ +export { executeJobFactory } from './server/execute_job'; +export { createJobFactory } from './server/create_job'; diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job.ts index d6166f16dab77..3ba54473474ee 100644 --- a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job.ts +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job.ts @@ -7,9 +7,9 @@ import { notFound, notImplemented } from 'boom'; import { Request } from 'hapi'; import { get } from 'lodash'; -// @ts-ignore -import { createTaggedLogger, cryptoFactory, oncePerServer } from '../../../../server/lib'; -import { JobDocPayload, JobParams, KbnServer, Logger } from '../../../../types'; + +import { cryptoFactory, LevelLogger, oncePerServer } from '../../../../server/lib'; +import { JobDocPayload, JobParams, KbnServer } from '../../../../types'; import { SavedObject, SavedObjectServiceError, @@ -18,7 +18,6 @@ import { TimeRangeParams, VisObjectAttributesJSON, } from '../../'; -import { createGenerateCsv } from '../lib/generate_csv'; import { createJobSearch } from './create_job_search'; interface VisData { @@ -27,21 +26,18 @@ interface VisData { panel: SearchPanel; } -function createJobFn(server: KbnServer) { +type CreateJobFn = (jobParams: JobParams, headers: any, req: Request) => Promise; + +function createJobFn(server: KbnServer): CreateJobFn { const crypto = cryptoFactory(server); - const logger: Logger = { - debug: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'debug']), - warning: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'warning']), - error: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'error']), - }; - const generateCsv = createGenerateCsv(logger); + const logger = LevelLogger.createForServer(server, ['reporting', 'savedobject-csv']); return async function createJob( jobParams: JobParams, headers: any, req: Request ): Promise { - const { isImmediate, savedObjectType, savedObjectId } = jobParams; + const { savedObjectType, savedObjectId } = jobParams; const serializedEncryptedHeaders = await crypto.encrypt(headers); const client = req.getSavedObjectsClient(); @@ -86,28 +82,13 @@ function createJobFn(server: KbnServer) { throw new Error(`Unable to create a job from saved object data! Error: ${err}`); }); - let type: string = ''; - let result: any = null; - - if (isImmediate) { - try { - ({ type, result } = await generateCsv(req, server, visType, panel)); - } catch (err) { - if (err.stack) { - logger.error(err.stack); - } - logger.error(`Generate CSV Error! ${err}`); - throw err; - } - } - return { + basePath: req.getBasePath(), + headers: serializedEncryptedHeaders, jobParams: { ...jobParams, panel, visType }, + type: null, // resolved in executeJob + objects: null, // resolved in executeJob title, - type, - objects: result ? result.content : result, - headers: serializedEncryptedHeaders, - basePath: req.getBasePath(), }; }; } diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/execute_job.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/execute_job.ts index 4c0168e442ae0..1b44154424f6b 100644 --- a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/execute_job.ts +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/execute_job.ts @@ -4,45 +4,46 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Request } from 'hapi'; import { i18n } from '@kbn/i18n'; + +import { cryptoFactory, LevelLogger, oncePerServer } from '../../../server/lib'; +import { JobDocOutputExecuted, JobDocPayload, KbnServer } from '../../../types'; import { CONTENT_TYPE_CSV } from '../../../common/constants'; -// @ts-ignore -import { createTaggedLogger, cryptoFactory, oncePerServer } from '../../../server/lib'; -import { JobDocPayload, KbnServer, Logger } from '../../../types'; import { createGenerateCsv } from './lib/generate_csv'; -interface JobDocOutputPseudo { - content_type: 'text/csv'; - content: string | null | undefined; -} - interface FakeRequest { headers: any; getBasePath: (opts: any) => string; server: KbnServer; } -/* - * @return {Object}: pseudo-JobDocOutput. See interface JobDocOutput - */ -function executeJobFn(server: KbnServer) { +type ExecuteJobFn = (job: JobDocPayload, realRequest?: Request) => Promise; + +function executeJobFn(server: KbnServer): ExecuteJobFn { const crypto = cryptoFactory(server); const config = server.config(); const serverBasePath = config.get('server.basePath'); - const logger: Logger = { - debug: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'debug']), - warning: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'warning']), - error: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'error']), - }; - + const logger = LevelLogger.createForServer(server, ['reporting', 'savedobject-csv']); const generateCsv = createGenerateCsv(logger); - return async function executeJob(job: JobDocPayload): Promise { - const { basePath, objects, headers: serializedEncryptedHeaders, jobParams } = job; // FIXME how to remove payload.objects for cleanup? + + return async function executeJob( + job: JobDocPayload, + realRequest?: Request + ): Promise { + const { basePath, jobParams } = job; const { isImmediate, panel, visType } = jobParams; - if (!isImmediate) { - logger.debug(`Execute job generating [${visType}] csv`); + logger.debug(`Execute job generating [${visType}] csv`); + + let requestObject: Request | FakeRequest; + if (isImmediate && realRequest) { + logger.debug(`executing job from immediate API`); + requestObject = realRequest; + } else { + logger.debug(`executing job async using encrypted headers`); let decryptedHeaders; + const serializedEncryptedHeaders = job.headers; try { decryptedHeaders = await crypto.decrypt(serializedEncryptedHeaders); } catch (err) { @@ -58,25 +59,33 @@ function executeJobFn(server: KbnServer) { ); } - const fakeRequest: FakeRequest = { + requestObject = { headers: decryptedHeaders, getBasePath: () => basePath || serverBasePath, server, }; - - const content = await generateCsv(fakeRequest, server, visType as string, panel); - return { - content_type: CONTENT_TYPE_CSV, - content: content.result ? content.result.content : null, - }; } - logger.debug(`Execute job using previously-generated [${visType}] csv`); + let content: string; + let maxSizeReached = false; + let size = 0; + try { + ({ + result: { content, maxSizeReached, size }, + } = await generateCsv(requestObject, server, visType as string, panel, jobParams)); + } catch (err) { + if (err.stack) { + logger.error(err.stack); + } + logger.error(`Generate CSV Error! ${err}`); + throw err; + } - // if job was created with "immediate", just return the data in the job doc return { content_type: CONTENT_TYPE_CSV, - content: objects, + content, + max_size_reached: maxSizeReached, + size, }; }; } diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv.ts index a5929e20847cf..cf2d621b7d201 100644 --- a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv.ts +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv.ts @@ -6,9 +6,7 @@ import { badRequest } from 'boom'; import { Request } from 'hapi'; -// @ts-ignore -import { createTaggedLogger } from '../../../../server/lib'; -import { KbnServer, Logger } from '../../../../types'; +import { KbnServer, Logger, JobParams } from '../../../../types'; import { SearchPanel, VisPanel } from '../../'; import { generateCsvSearch } from './generate_csv_search'; @@ -23,7 +21,8 @@ export function createGenerateCsv(logger: Logger) { request: Request | FakeRequest, server: KbnServer, visType: string, - panel: VisPanel | SearchPanel + panel: VisPanel | SearchPanel, + jobParams: JobParams ) { // This should support any vis type that is able to fetch // and model data on the server-side @@ -32,7 +31,13 @@ export function createGenerateCsv(logger: Logger) { // expression that we could run through the interpreter to get csv switch (visType) { case 'search': - return await generateCsvSearch(request as Request, server, logger, panel as SearchPanel); + return await generateCsvSearch( + request as Request, + server, + logger, + panel as SearchPanel, + jobParams + ); default: throw badRequest(`Unsupported or unrecognized saved object type: ${visType}`); } diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv_search.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv_search.ts index a17aa8d46b65b..fdc3f0fac6751 100644 --- a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv_search.ts +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv_search.ts @@ -7,7 +7,7 @@ // @ts-ignore no module definition import { buildEsQuery } from '@kbn/es-query'; import { Request } from 'hapi'; -import { KbnServer, Logger } from '../../../../types'; +import { KbnServer, Logger, JobParams } from '../../../../types'; // @ts-ignore no module definition import { createGenerateCsv } from '../../../csv/server/lib/generate_csv'; import { @@ -23,8 +23,8 @@ import { ESQueryConfig, GenerateCsvParams, Filter, - ReqPayload, IndexPatternField, + QueryFilter, } from './'; import { getDataSource } from './get_data_source'; import { getFilters } from './get_filters'; @@ -49,7 +49,8 @@ export async function generateCsvSearch( req: Request, server: KbnServer, logger: Logger, - searchPanel: SearchPanel + searchPanel: SearchPanel, + jobParams: JobParams ): Promise { const { savedObjects, uiSettingsServiceFactory } = server; const savedObjectsClient = savedObjects.getScopedSavedObjectsClient(req); @@ -77,9 +78,13 @@ export async function generateCsvSearch( fields: indexPatternFields, } = indexPatternSavedObject; - const { - state: { query: payloadQuery, sort: payloadSort = [] }, - } = req.payload as ReqPayload; + let payloadQuery: QueryFilter | undefined; + let payloadSort: any[] = []; + if (jobParams.post && jobParams.post.state) { + ({ + post: { state: { query: payloadQuery, sort: payloadSort = [] } }, + } = jobParams); + } const { includes, timezone, combinedFilter } = getFilters( indexPatternSavedObjectId, diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.d.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.d.ts index 7220cbc666083..350ea3fb2ab13 100644 --- a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.d.ts +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.d.ts @@ -4,13 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - SavedSearchObjectAttributes, - SearchPanel, - SearchRequest, - SearchSource, - TimeRangeParams, -} from '../../'; +import { SavedSearchObjectAttributes, SearchPanel, SearchRequest, SearchSource } from '../../'; export interface SavedSearchGeneratorResult { content: string; @@ -20,7 +14,7 @@ export interface SavedSearchGeneratorResult { export interface CsvResultFromSearch { type: string; - result: SavedSearchGeneratorResult | null; + result: SavedSearchGeneratorResult; } type EndpointCaller = (method: string, params: any) => Promise; @@ -34,18 +28,6 @@ type FormatsMap = Map< } >; -interface ReqPayload { - state: { - sort: Array<{ - [sortKey: string]: { - order: string; - }; - }>; - docvalue_fields: any; - query: any; - }; -} - export interface GenerateCsvParams { searchRequest: SearchRequest; callEndpoint: EndpointCaller; diff --git a/x-pack/plugins/reporting/server/lib/index.ts b/x-pack/plugins/reporting/server/lib/index.ts index 9021abb5374fd..026c444ca35d7 100644 --- a/x-pack/plugins/reporting/server/lib/index.ts +++ b/x-pack/plugins/reporting/server/lib/index.ts @@ -4,9 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -// @ts-ignore +// @ts-ignore untyped module export { createTaggedLogger } from './create_tagged_logger'; -// @ts-ignore +// @ts-ignore untyped module export { cryptoFactory } from './crypto'; -// @ts-ignore +// @ts-ignore untyped module export { oncePerServer } from './once_per_server'; + +export { LevelLogger } from './level_logger'; diff --git a/x-pack/plugins/reporting/server/routes/generate_from_savedobject.ts b/x-pack/plugins/reporting/server/routes/generate_from_savedobject.ts index 6c8b907d90d81..9339e33cac2c0 100644 --- a/x-pack/plugins/reporting/server/routes/generate_from_savedobject.ts +++ b/x-pack/plugins/reporting/server/routes/generate_from_savedobject.ts @@ -7,12 +7,23 @@ import { Request, ResponseObject, ResponseToolkit } from 'hapi'; import Joi from 'joi'; import { get } from 'lodash'; + +// @ts-ignore no module definition import { API_BASE_URL_V1, CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../common/constants'; -import { JobDoc, JobDocOutput, JobParams, KbnServer } from '../../types'; -// @ts-ignore +// @ts-ignore no module definition import { getDocumentPayloadFactory } from './lib/get_document_payload'; -import { getRouteConfigFactoryReportingPre } from './lib/route_config_factories'; + +import { createJobFactory, executeJobFactory } from '../../export_types/csv_from_savedobject'; +import { + JobDocPayload, + JobParamPostPayload, + JobDocOutputExecuted, + JobParams, + KbnServer, +} from '../../types'; +import { LevelLogger } from '../lib/level_logger'; import { HandlerErrorFunction, HandlerFunction, QueuedJobPayload } from './types'; +import { getRouteConfigFactoryReportingPre } from './lib/route_config_factories'; const BASE_GENERATE = `${API_BASE_URL_V1}/generate`; @@ -35,10 +46,13 @@ const getJobFromRouteHandler = async ( const { savedObjectType, savedObjectId } = request.params; let result: QueuedJobPayload; try { + const { timerange, state } = request.payload as JobParamPostPayload; + const post = timerange || state ? { timerange, state } : undefined; const jobParams: JobParams = { + isImmediate: options.isImmediate, savedObjectType, savedObjectId, - isImmediate: options.isImmediate, + post, }; result = await handleRoute(CSV_FROM_SAVEDOBJECT_JOB_TYPE, jobParams, request, h); } catch (err) { @@ -85,55 +99,52 @@ export function registerGenerateCsvFromSavedObject( }), }, }; - const getDocumentPayload = getDocumentPayloadFactory(server); - // csv: immediate download + /* + * CSV export with the `immediate` option does not queue a job with Reporting's ESQueue to run the job async. Instead, this does: + * - re-use the createJob function to build up es query config + * - re-use the executeJob function to run the scan and scroll queries and capture the entire CSV in a result object. + */ server.route({ path: `${BASE_GENERATE}/immediate/csv/saved-object/{savedObjectType}:{savedObjectId}`, method: 'POST', options: routeOptions, handler: async (request: Request, h: ResponseToolkit) => { - /* - * 1. Queue a job with getJobFromRouteHandler - * - `isImmediate: true` gets us the complete result data in payload.objects - * 2. Copy the completed data stashed in the job as output.content - * - Makes the content available for download - * 3. Return a response with CSV content - */ - const queuedJob: QueuedJobPayload = await getJobFromRouteHandler( - handleRoute, - handleRouteError, - request, - h, - { - isImmediate: true, - } - ); - - // FIXME this is REALLY ugly - const jobSource: JobDoc = get(queuedJob, 'source.job'); - const output: JobDocOutput = getDocumentPayload({ - _source: { - ...jobSource, - status: 'completed', - output: { - content: jobSource.payload.objects, - content_type: 'text/csv', - }, - }, - }); + const logger = LevelLogger.createForServer(server, ['reporting', 'savedobject-csv']); + const { savedObjectType, savedObjectId } = request.params; + const { timerange, state } = request.payload as JobParamPostPayload; + const post = timerange || state ? { timerange, state } : undefined; + const jobParams: JobParams = { + isImmediate: true, + savedObjectType, + savedObjectId, + post, + }; + const createJobFn = createJobFactory(server); + const executeJobFn = executeJobFactory(server, request); + const jobDocPayload: JobDocPayload = await createJobFn(jobParams, request.headers, request); + const { + content_type: jobOutputContentType, + content: jobOutputContent, + size: jobOutputSize, + }: JobDocOutputExecuted = await executeJobFn(jobDocPayload, request); - const response: ResponseObject = h - .response(output.content) - .type(output.contentType) - .code(output.statusCode); + logger.info(`job output size: ${jobOutputSize} bytes`); - if (output.headers) { - Object.keys(output.headers).forEach(key => { - response.header(key, output.headers[key]); - }); + /* + * ESQueue worker function defaults `content` to null, even if the + * executeJob returned undefined. + * + * This converts null to undefined so the value can be sent to h.response() + */ + if (jobOutputContent === null) { + logger.warn('CSV Job Execution created empty content result'); } + const response = h + .response(jobOutputContent ? jobOutputContent : undefined) + .type(jobOutputContentType); + // Set header for buffer download, not streaming const { isBoom } = response as KibanaResponse; if (isBoom == null) { response.header('accept-ranges', 'none'); diff --git a/x-pack/plugins/reporting/types.d.ts b/x-pack/plugins/reporting/types.d.ts index 4b54c2bf3748b..235bf04e58437 100644 --- a/x-pack/plugins/reporting/types.d.ts +++ b/x-pack/plugins/reporting/types.d.ts @@ -107,11 +107,31 @@ export interface CryptoFactory { decrypt: (headers?: Record) => string; } +export interface TimeRangeParams { + timezone: string; + min: Date | string | number; + max: Date | string | number; +} + +type PostPayloadState = Partial<{ + state: { + query: any; + sort: any[]; + columns: string[]; // TODO + }; +}>; + +// retain POST payload data, needed for async +interface JobParamPostPayload extends PostPayloadState { + timerange: TimeRangeParams; +} + // params that come into a request export interface JobParams { savedObjectType: string; savedObjectId: string; isImmediate: boolean; + post?: JobParamPostPayload; panel?: any; // has to be resolved by the request handler visType?: string; // has to be resolved by the request handler } @@ -121,20 +141,17 @@ export interface JobDocPayload { forceNow?: string; headers?: Record; jobParams: JobParams; - objects?: string | null; // string if completed job; null if incomplete job; relativeUrl?: string; timeRange?: any; title: string; - type: string; urls?: string[]; + type?: string | null; // string if completed job; null if incomplete job; + objects?: string | null; // string if completed job; null if incomplete job; } export interface JobDocOutput { content: string; // encoded content contentType: string; - headers: any; - size?: number; - statusCode: number; } export interface JobDoc { @@ -149,6 +166,25 @@ export interface JobSource { _source: JobDoc; } +/* + * A snake_cased field is the only significant difference in structure of + * JobDocOutputExecuted vs JobDocOutput. + * + * JobDocOutput is the structure of the object returned by getDocumentPayload + * + * data in the _source fields of the + * Reporting index. + * + * The ESQueueWorker internals have executed job objects returned with this + * structure. See `_formatOutput` in reporting/server/lib/esqueue/worker.js + */ +export interface JobDocOutputExecuted { + content_type: string; // vs `contentType` above + content: string | null; // defaultOutput is null + max_size_reached: boolean; + size: number; +} + export interface ESQueueWorker { on: (event: string, handler: any) => void; } diff --git a/x-pack/test/api_integration/config.js b/x-pack/test/api_integration/config.js index f4d7f462276e8..aea5aa73bc3fb 100644 --- a/x-pack/test/api_integration/config.js +++ b/x-pack/test/api_integration/config.js @@ -20,16 +20,11 @@ import { SpacesServiceProvider, } from '../common/services'; -export default async function ({ readConfigFile }) { - const kibanaAPITestsConfig = await readConfigFile( - require.resolve('../../../test/api_integration/config.js') - ); - const xPackFunctionalTestsConfig = await readConfigFile( - require.resolve('../functional/config.js') - ); - const kibanaCommonConfig = await readConfigFile( - require.resolve('../../../test/common/config.js') - ); +export async function getApiIntegrationConfig({ readConfigFile }) { + + const kibanaAPITestsConfig = await readConfigFile(require.resolve('../../../test/api_integration/config.js')); + const xPackFunctionalTestsConfig = await readConfigFile(require.resolve('../functional/config.js')); + const kibanaCommonConfig = await readConfigFile(require.resolve('../../../test/common/config.js')); return { testFiles: [require.resolve('./apis')], @@ -65,3 +60,5 @@ export default async function ({ readConfigFile }) { esTestCluster: xPackFunctionalTestsConfig.get('esTestCluster'), }; } + +export default getApiIntegrationConfig; diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index b698a26545de2..535464981c931 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -192,6 +192,7 @@ export default async function ({ readConfigFile }) { '--server.uuid=5b2de169-2785-441b-ae8c-186a1936b17d', '--xpack.xpack_main.telemetry.enabled=false', '--xpack.maps.showMapsInspectorAdapter=true', + '--xpack.reporting.queue.pollInterval=3000', // make it explicitly the default '--xpack.security.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', // server restarts should not invalidate active sessions '--xpack.code.security.enableGitCertCheck=false', // Disable git certificate check '--timelion.ui.enabled=true', diff --git a/x-pack/test/functional/es_archives/reporting/scripted/mappings.json b/x-pack/test/functional/es_archives/reporting/scripted/mappings.json index 3018d80352054..d36bbc72f4ffa 100644 --- a/x-pack/test/functional/es_archives/reporting/scripted/mappings.json +++ b/x-pack/test/functional/es_archives/reporting/scripted/mappings.json @@ -488,6 +488,9 @@ "_reserved": { "type": "boolean" }, + "disabledFeatures": { + "type": "keyword" + }, "color": { "type": "keyword" }, diff --git a/x-pack/test/reporting/api/generate/csv_saved_search.ts b/x-pack/test/reporting/api/generate/csv_saved_search.ts index 18cb791537bfa..488c93899f089 100644 --- a/x-pack/test/reporting/api/generate/csv_saved_search.ts +++ b/x-pack/test/reporting/api/generate/csv_saved_search.ts @@ -28,9 +28,13 @@ export default function({ getService }: { getService: any }) { const esArchiver = getService('esArchiver'); const supertestSvc = getService('supertest'); const generateAPI = { - getCsvFromSavedSearch: async (id: string, { timerange, state }: GenerateOpts) => { + getCsvFromSavedSearch: async ( + id: string, + { timerange, state }: GenerateOpts, + isImmediate = true + ) => { return await supertestSvc - .post(`/api/reporting/v1/generate/immediate/csv/saved-object/${id}`) + .post(`/api/reporting/v1/generate/${isImmediate ? 'immediate/' : ''}csv/saved-object/${id}`) .set('kbn-xsrf', 'xxx') .send({ timerange, state }); }, @@ -163,39 +167,22 @@ export default function({ getService }: { getService: any }) { // load test data that contains a saved search and documents await esArchiver.load('reporting/scripted'); + const params = { + searchId: 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24', + postPayload: { + timerange: { timezone: 'UTC', min: '1979-01-01T10:00:00Z', max: '1981-01-01T10:00:00Z' }, // prettier-ignore + state: { query: { bool: { filter: [ { bool: { filter: [ { bool: { minimum_should_match: 1, should: [{ query_string: { fields: ['name'], query: 'Fe*' } }] } } ] } } ] } } } // prettier-ignore + }, + isImmediate: true, + }; const { status: resStatus, text: resText, type: resType, } = (await generateAPI.getCsvFromSavedSearch( - 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24', - { - timerange: { - timezone: 'UTC', - min: '1979-01-01T10:00:00Z', - max: '1981-01-01T10:00:00Z', - }, - state: { - query: { - bool: { - filter: [ - { - bool: { - filter: [ - { - bool: { - minimum_should_match: 1, - should: [{ query_string: { fields: ['name'], query: 'Fe*' } }], - }, - }, - ], - }, - }, - ], - }, - }, - }, - } + params.searchId, + params.postPayload, + params.isImmediate )) as supertest.Response; expect(resStatus).to.eql(200); @@ -232,5 +219,100 @@ export default function({ getService }: { getService: any }) { await esArchiver.unload('reporting/scripted'); }); }); + + describe('Non-Immediate', () => { + it('using queries in job params', async () => { + // load test data that contains a saved search and documents + await esArchiver.load('reporting/scripted'); + + const params = { + searchId: 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24', + postPayload: { + timerange: { timezone: 'UTC', min: '1979-01-01T10:00:00Z', max: '1981-01-01T10:00:00Z' }, // prettier-ignore + state: { query: { bool: { filter: [ { bool: { filter: [ { bool: { minimum_should_match: 1, should: [{ query_string: { fields: ['name'], query: 'Fe*' } }] } } ] } } ] } } } // prettier-ignore + }, + isImmediate: false, + }; + const { + status: resStatus, + text: resText, + type: resType, + } = (await generateAPI.getCsvFromSavedSearch( + params.searchId, + params.postPayload, + params.isImmediate + )) as supertest.Response; + + expect(resStatus).to.eql(200); + expect(resType).to.eql('application/json'); + const { + path: jobDownloadPath, + job: { index: jobIndex, jobtype: jobType, created_by: jobCreatedBy, payload: jobPayload }, + } = JSON.parse(resText); + + expect(jobDownloadPath.slice(0, 29)).to.equal('/api/reporting/jobs/download/'); + expect(jobIndex.slice(0, 11)).to.equal('.reporting-'); + expect(jobType).to.be('csv_from_savedobject'); + expect(jobCreatedBy).to.be('elastic'); + + const { + title: payloadTitle, + objects: payloadObjects, + jobParams: payloadParams, + } = jobPayload; + expect(payloadTitle).to.be('EVERYBABY2'); + expect(payloadObjects).to.be(null); // value for non-immediate + expect(payloadParams.savedObjectType).to.be('search'); + expect(payloadParams.savedObjectId).to.be('f34bf440-5014-11e9-bce7-4dabcb8bef24'); + expect(payloadParams.isImmediate).to.be(false); + + const { state: postParamState, timerange: postParamTimerange } = payloadParams.post; + expect(postParamState).to.eql({ + query: { bool: { filter: [ { bool: { filter: [ { bool: { minimum_should_match: 1, should: [{ query_string: { fields: ['name'], query: 'Fe*' } }] } } ] } } ] } } // prettier-ignore + }); + expect(postParamTimerange).to.eql({ + max: '1981-01-01T10:00:00.000Z', + min: '1979-01-01T10:00:00.000Z', + timezone: 'UTC', + }); + + const { + indexPatternSavedObjectId: payloadPanelIndexPatternSavedObjectId, + timerange: payloadPanelTimerange, + } = payloadParams.panel; + expect(payloadPanelIndexPatternSavedObjectId).to.be('89655130-5013-11e9-bce7-4dabcb8bef24'); + expect(payloadPanelTimerange).to.eql({ + timezone: 'UTC', + min: '1979-01-01T10:00:00.000Z', + max: '1981-01-01T10:00:00.000Z', + }); + + expect(payloadParams.visType).to.be('search'); + + // check the resource at jobDownloadPath + const downloadFromPath = async (downloadPath: string) => { + const { status, text, type } = await supertestSvc + .get(downloadPath) + .set('kbn-xsrf', 'xxx'); + return { + status, + text, + type, + }; + }; + + await new Promise(resolve => { + setTimeout(async () => { + const { status, text, type } = await downloadFromPath(jobDownloadPath); + expect(status).to.eql(200); + expect(type).to.eql('text/csv'); + expect(text).to.eql(CSV_RESULT_SCRIPTED_REQUERY); + resolve(); + }, 5000); // x-pack/test/functional/config settings are inherited, uses 3 seconds for polling interval. + }); + + await esArchiver.unload('reporting/scripted'); + }); + }); }); }