Fix Non-Immediate CSV Generation / Test for async download

x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.ts

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-// @ts-ignore
+
 import url from 'url';
 import { getAbsoluteUrlFactory } from '../../../common/get_absolute_url';
 import { ConditionalHeaders, JobDocPayload, KbnServer } from '../../../types';

x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.ts

@@ -3,6 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+
 import { UI_SETTINGS_CUSTOM_PDF_LOGO } from '../../../common/constants';
 import { ConditionalHeaders, JobDocPayload, KbnServer } from '../../../types';
 

x-pack/plugins/reporting/export_types/csv_from_savedobject/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './index.d';
+
+/*
+ * These functions are exported to share with the API route handler that
+ * generates csv from saved object immediately on request.
+ */
+export { executeJobFactory } from './server/execute_job';
+export { createJobFactory } from './server/create_job';

x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job.ts

@@ -7,9 +7,9 @@
 import { notFound, notImplemented } from 'boom';
 import { Request } from 'hapi';
 import { get } from 'lodash';
-// @ts-ignore
-import { createTaggedLogger, cryptoFactory, oncePerServer } from '../../../../server/lib';
-import { JobDocPayload, JobParams, KbnServer, Logger } from '../../../../types';
+
+import { cryptoFactory, LevelLogger, oncePerServer } from '../../../../server/lib';
+import { JobDocPayload, JobParams, KbnServer } from '../../../../types';
 import {
   SavedObject,
   SavedObjectServiceError,
@@ -18,7 +18,6 @@ import {
   TimeRangeParams,
   VisObjectAttributesJSON,
 } from '../../';
-import { createGenerateCsv } from '../lib/generate_csv';
 import { createJobSearch } from './create_job_search';
 
 interface VisData {
@@ -27,21 +26,18 @@ interface VisData {
   panel: SearchPanel;
 }
 
-function createJobFn(server: KbnServer) {
+type CreateJobFn = (jobParams: JobParams, headers: any, req: Request) => Promise<JobDocPayload>;
+
+function createJobFn(server: KbnServer): CreateJobFn {
   const crypto = cryptoFactory(server);
-  const logger: Logger = {
-    debug: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'debug']),
-    warning: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'warning']),
-    error: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'error']),
-  };
-  const generateCsv = createGenerateCsv(logger);
+  const logger = LevelLogger.createForServer(server, ['reporting', 'savedobject-csv']);
 
   return async function createJob(
     jobParams: JobParams,
     headers: any,
     req: Request
   ): Promise<JobDocPayload> {
-    const { isImmediate, savedObjectType, savedObjectId } = jobParams;
+    const { savedObjectType, savedObjectId } = jobParams;
     const serializedEncryptedHeaders = await crypto.encrypt(headers);
     const client = req.getSavedObjectsClient();
 
@@ -86,28 +82,13 @@ function createJobFn(server: KbnServer) {
         throw new Error(`Unable to create a job from saved object data! Error: ${err}`);
       });
 
-    let type: string = '';
-    let result: any = null;
-
-    if (isImmediate) {
-      try {
-        ({ type, result } = await generateCsv(req, server, visType, panel));
-      } catch (err) {
-        if (err.stack) {
-          logger.error(err.stack);
-        }
-        logger.error(`Generate CSV Error! ${err}`);
-        throw err;
-      }
-    }
-
     return {
+      basePath: req.getBasePath(),
+      headers: serializedEncryptedHeaders,
       jobParams: { ...jobParams, panel, visType },
+      type: null, // resolved in executeJob
+      objects: null, // resolved in executeJob
       title,
-      type,
-      objects: result ? result.content : result,
-      headers: serializedEncryptedHeaders,
-      basePath: req.getBasePath(),
     };
   };
 }

x-pack/plugins/reporting/export_types/csv_from_savedobject/server/execute_job.ts

@@ -4,45 +4,46 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { Request } from 'hapi';
 import { i18n } from '@kbn/i18n';
+
+import { cryptoFactory, LevelLogger, oncePerServer } from '../../../server/lib';
+import { JobDocOutputExecuted, JobDocPayload, KbnServer } from '../../../types';
 import { CONTENT_TYPE_CSV } from '../../../common/constants';
-// @ts-ignore
-import { createTaggedLogger, cryptoFactory, oncePerServer } from '../../../server/lib';
-import { JobDocPayload, KbnServer, Logger } from '../../../types';
 import { createGenerateCsv } from './lib/generate_csv';
 
-interface JobDocOutputPseudo {
-  content_type: 'text/csv';
-  content: string | null | undefined;
-}
-
 interface FakeRequest {
   headers: any;
   getBasePath: (opts: any) => string;
   server: KbnServer;
 }
 
-/*
- * @return {Object}: pseudo-JobDocOutput. See interface JobDocOutput
- */
-function executeJobFn(server: KbnServer) {
+type ExecuteJobFn = (job: JobDocPayload, realRequest?: Request) => Promise<JobDocOutputExecuted>;
+
+function executeJobFn(server: KbnServer): ExecuteJobFn {
   const crypto = cryptoFactory(server);
   const config = server.config();
   const serverBasePath = config.get('server.basePath');
-  const logger: Logger = {
-    debug: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'debug']),
-    warning: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'warning']),
-    error: createTaggedLogger(server, ['reporting', 'savedobject-csv', 'error']),
-  };
-
+  const logger = LevelLogger.createForServer(server, ['reporting', 'savedobject-csv']);
   const generateCsv = createGenerateCsv(logger);
-  return async function executeJob(job: JobDocPayload): Promise<JobDocOutputPseudo> {
-    const { basePath, objects, headers: serializedEncryptedHeaders, jobParams } = job; // FIXME how to remove payload.objects for cleanup?
+
+  return async function executeJob(
+    job: JobDocPayload,
+    realRequest?: Request
+  ): Promise<JobDocOutputExecuted> {
+    const { basePath, jobParams } = job;
     const { isImmediate, panel, visType } = jobParams;
-    if (!isImmediate) {
-      logger.debug(`Execute job generating [${visType}] csv`);
 
+    logger.debug(`Execute job generating [${visType}] csv`);
+
+    let requestObject: Request | FakeRequest;
+    if (isImmediate && realRequest) {
+      logger.debug(`executing job from immediate API`);
+      requestObject = realRequest;
+    } else {
+      logger.debug(`executing job async using encrypted headers`);
       let decryptedHeaders;
+      const serializedEncryptedHeaders = job.headers;
       try {
         decryptedHeaders = await crypto.decrypt(serializedEncryptedHeaders);
       } catch (err) {
@@ -58,25 +59,33 @@ function executeJobFn(server: KbnServer) {
         );
       }
 
-      const fakeRequest: FakeRequest = {
+      requestObject = {
         headers: decryptedHeaders,
         getBasePath: () => basePath || serverBasePath,
         server,
       };
-
-      const content = await generateCsv(fakeRequest, server, visType as string, panel);
-      return {
-        content_type: CONTENT_TYPE_CSV,
-        content: content.result ? content.result.content : null,
-      };
     }
 
-    logger.debug(`Execute job using previously-generated [${visType}] csv`);
+    let content: string;
+    let maxSizeReached = false;
+    let size = 0;
+    try {
+      ({
+        result: { content, maxSizeReached, size },
+      } = await generateCsv(requestObject, server, visType as string, panel, jobParams));
+    } catch (err) {
+      if (err.stack) {
+        logger.error(err.stack);
+      }
+      logger.error(`Generate CSV Error! ${err}`);
+      throw err;
+    }
 
-    // if job was created with "immediate", just return the data in the job doc
     return {
       content_type: CONTENT_TYPE_CSV,
-      content: objects,
+      content,
+      max_size_reached: maxSizeReached,
+      size,
     };
   };
 }

x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv.ts

@@ -6,9 +6,7 @@
 
 import { badRequest } from 'boom';
 import { Request } from 'hapi';
-// @ts-ignore
-import { createTaggedLogger } from '../../../../server/lib';
-import { KbnServer, Logger } from '../../../../types';
+import { KbnServer, Logger, JobParams } from '../../../../types';
 import { SearchPanel, VisPanel } from '../../';
 import { generateCsvSearch } from './generate_csv_search';
 
@@ -23,7 +21,8 @@ export function createGenerateCsv(logger: Logger) {
     request: Request | FakeRequest,
     server: KbnServer,
     visType: string,
-    panel: VisPanel | SearchPanel
+    panel: VisPanel | SearchPanel,
+    jobParams: JobParams
   ) {
     // This should support any vis type that is able to fetch
     // and model data on the server-side
@@ -32,7 +31,13 @@ export function createGenerateCsv(logger: Logger) {
     // expression that we could run through the interpreter to get csv
     switch (visType) {
       case 'search':
-        return await generateCsvSearch(request as Request, server, logger, panel as SearchPanel);
+        return await generateCsvSearch(
+          request as Request,
+          server,
+          logger,
+          panel as SearchPanel,
+          jobParams
+        );
       default:
         throw badRequest(`Unsupported or unrecognized saved object type: ${visType}`);
     }

x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv_search.ts

@@ -7,7 +7,7 @@
 // @ts-ignore no module definition
 import { buildEsQuery } from '@kbn/es-query';
 import { Request } from 'hapi';
-import { KbnServer, Logger } from '../../../../types';
+import { KbnServer, Logger, JobParams } from '../../../../types';
 // @ts-ignore no module definition
 import { createGenerateCsv } from '../../../csv/server/lib/generate_csv';
 import {
@@ -23,8 +23,8 @@ import {
   ESQueryConfig,
   GenerateCsvParams,
   Filter,
-  ReqPayload,
   IndexPatternField,
+  QueryFilter,
 } from './';
 import { getDataSource } from './get_data_source';
 import { getFilters } from './get_filters';
@@ -49,7 +49,8 @@ export async function generateCsvSearch(
   req: Request,
   server: KbnServer,
   logger: Logger,
-  searchPanel: SearchPanel
+  searchPanel: SearchPanel,
+  jobParams: JobParams
 ): Promise<CsvResultFromSearch> {
   const { savedObjects, uiSettingsServiceFactory } = server;
   const savedObjectsClient = savedObjects.getScopedSavedObjectsClient(req);
@@ -77,9 +78,13 @@ export async function generateCsvSearch(
     fields: indexPatternFields,
   } = indexPatternSavedObject;
 
-  const {
-    state: { query: payloadQuery, sort: payloadSort = [] },
-  } = req.payload as ReqPayload;
+  let payloadQuery: QueryFilter | undefined;
+  let payloadSort: any[] = [];
+  if (jobParams.post && jobParams.post.state) {
+    ({
+      post: { state: { query: payloadQuery, sort: payloadSort = [] } },
+    } = jobParams);
+  }
 
   const { includes, timezone, combinedFilter } = getFilters(
     indexPatternSavedObjectId,

x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.d.ts

@@ -4,13 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import {
-  SavedSearchObjectAttributes,
-  SearchPanel,
-  SearchRequest,
-  SearchSource,
-  TimeRangeParams,
-} from '../../';
+import { SavedSearchObjectAttributes, SearchPanel, SearchRequest, SearchSource } from '../../';
 
 export interface SavedSearchGeneratorResult {
   content: string;
@@ -20,7 +14,7 @@ export interface SavedSearchGeneratorResult {
 
 export interface CsvResultFromSearch {
   type: string;
-  result: SavedSearchGeneratorResult | null;
+  result: SavedSearchGeneratorResult;
 }
 
 type EndpointCaller = (method: string, params: any) => Promise<any>;
@@ -34,18 +28,6 @@ type FormatsMap = Map<
   }
 >;
 
-interface ReqPayload {
-  state: {
-    sort: Array<{
-      [sortKey: string]: {
-        order: string;
-      };
-    }>;
-    docvalue_fields: any;
-    query: any;
-  };
-}
-
 export interface GenerateCsvParams {
   searchRequest: SearchRequest;
   callEndpoint: EndpointCaller;

x-pack/plugins/reporting/server/lib/index.ts

@@ -4,9 +4,11 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-// @ts-ignore
+// @ts-ignore untyped module
 export { createTaggedLogger } from './create_tagged_logger';
-// @ts-ignore
+// @ts-ignore untyped module
 export { cryptoFactory } from './crypto';
-// @ts-ignore
+// @ts-ignore untyped module
 export { oncePerServer } from './once_per_server';
+
+export { LevelLogger } from './level_logger';