From 9fef424bbafbe09ea7f237a23937d10474acf6bd Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Mon, 8 Mar 2021 17:33:19 -0700 Subject: [PATCH] Switch Reporting to Task Manager (#64853) * [Reporting] Task Manager * fix startup * use synchronous config for task registration * fix eslint * pr-90365 * --wip-- [skip ci] * set maxConcurrency to 0 if pollEnabled is false * add test for execute_report * remove unused test file * more tests * remove unused test files * remove priority * logging cleanups * fix for queue.pollEnabled: false * more logging fixes for less duplicated code * update jest snapshots * polish * remove unnecessary * Update mapping.ts * polish * fix bug if instance gets a monitoring task and pollEnabled is false * simplification * cosmetic * fix test * stop monitoring task sabotage * update api docs Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- api_docs/reporting.json | 305 ++++- docs/user/reporting/script-example.asciidoc | 3 +- .../reporting/common/schema_utils.test.ts | 16 + .../plugins/reporting/common/schema_utils.ts | 7 + x-pack/plugins/reporting/kibana.json | 1 + x-pack/plugins/reporting/server/core.ts | 51 +- .../server/export_types/csv/create_job.ts | 5 +- .../server/export_types/csv/execute_job.ts | 4 +- .../csv_from_savedobject/create_job.ts | 5 +- .../csv_from_savedobject/execute_job.ts | 6 +- .../export_types/png/create_job/index.ts | 4 +- .../printable_pdf/create_job/index.ts | 4 +- .../reporting/server/lib/create_queue.ts | 75 -- .../server/lib/create_worker.test.ts | 110 -- .../reporting/server/lib/create_worker.ts | 77 -- .../reporting/server/lib/enqueue_job.test.ts | 94 ++ .../reporting/server/lib/enqueue_job.ts | 50 +- .../server/lib/esqueue/__fixtures__/job.js | 24 - .../__fixtures__/legacy_elasticsearch.js | 112 -- .../server/lib/esqueue/__fixtures__/queue.js | 18 - .../server/lib/esqueue/__fixtures__/worker.js | 23 - .../lib/esqueue/constants/default_settings.js | 17 - .../server/lib/esqueue/constants/events.js | 21 - .../server/lib/esqueue/constants/index.js | 16 - .../lib/esqueue/create_tagged_logger.ts | 27 - .../lib/esqueue/helpers/create_index.js | 115 -- .../server/lib/esqueue/helpers/errors.js | 27 - .../server/lib/esqueue/helpers/errors.test.js | 57 - .../reporting/server/lib/esqueue/index.js | 55 - .../server/lib/esqueue/index.test.js | 160 --- .../reporting/server/lib/esqueue/worker.js | 444 ------- .../server/lib/esqueue/worker.test.js | 1120 ----------------- x-pack/plugins/reporting/server/lib/index.ts | 2 - .../reporting/server/lib/store/mapping.ts | 2 +- .../reporting/server/lib/store/report.test.ts | 17 + .../reporting/server/lib/store/report.ts | 38 +- .../reporting/server/lib/store/store.test.ts | 68 +- .../reporting/server/lib/store/store.ts | 159 ++- .../server/lib/tasks/error_logger.test.ts | 57 + .../server/lib/tasks/error_logger.ts | 44 + .../server/lib/tasks/execute_report.test.ts | 87 ++ .../server/lib/tasks/execute_report.ts | 400 ++++++ .../reporting/server/lib/tasks/index.ts | 33 +- .../server/lib/tasks/monitor_report.test.ts | 67 + .../server/lib/tasks/monitor_reports.ts | 166 +++ .../plugins/reporting/server/plugin.test.ts | 23 +- x-pack/plugins/reporting/server/plugin.ts | 12 +- .../reporting/server/routes/generation.ts | 2 +- .../create_mock_reportingplugin.ts | 9 +- x-pack/plugins/reporting/server/types.ts | 3 + .../reporting_without_security/job_apis.ts | 8 +- 51 files changed, 1598 insertions(+), 2652 deletions(-) create mode 100644 x-pack/plugins/reporting/common/schema_utils.test.ts delete mode 100644 x-pack/plugins/reporting/server/lib/create_queue.ts delete mode 100644 x-pack/plugins/reporting/server/lib/create_worker.test.ts delete mode 100644 x-pack/plugins/reporting/server/lib/create_worker.ts create mode 100644 x-pack/plugins/reporting/server/lib/enqueue_job.test.ts delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/job.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/legacy_elasticsearch.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/queue.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/worker.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/constants/default_settings.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/constants/events.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/constants/index.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/create_tagged_logger.ts delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/helpers/errors.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/helpers/errors.test.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/index.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/index.test.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/worker.js delete mode 100644 x-pack/plugins/reporting/server/lib/esqueue/worker.test.js create mode 100644 x-pack/plugins/reporting/server/lib/tasks/error_logger.test.ts create mode 100644 x-pack/plugins/reporting/server/lib/tasks/error_logger.ts create mode 100644 x-pack/plugins/reporting/server/lib/tasks/execute_report.test.ts create mode 100644 x-pack/plugins/reporting/server/lib/tasks/execute_report.ts create mode 100644 x-pack/plugins/reporting/server/lib/tasks/monitor_report.test.ts create mode 100644 x-pack/plugins/reporting/server/lib/tasks/monitor_reports.ts diff --git a/api_docs/reporting.json b/api_docs/reporting.json index 1296fb8926178..e07e3493a9d85 100644 --- a/api_docs/reporting.json +++ b/api_docs/reporting.json @@ -851,7 +851,29 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 58 + "lineNumber": 65 + } + }, + { + "type": "Object", + "label": "context", + "isRequired": true, + "signature": [ + { + "pluginId": "core", + "scope": "server", + "docId": "kibCorePluginApi", + "section": "def-server.PluginInitializerContext", + "text": "PluginInitializerContext" + }, + "; }>; autoDownload: boolean; }>; timeouts: Readonly<{} & { openUrl: number | moment.Duration; waitForElements: number | moment.Duration; renderComplete: number | moment.Duration; }>; networkPolicy: Readonly<{} & { enabled: boolean; rules: Readonly<{ host?: string | undefined; protocol?: string | undefined; } & { allow: boolean; }>[]; }>; zoom: number; viewport: Readonly<{} & { height: number; width: number; }>; loadDelay: number | moment.Duration; maxAttempts: number; }>; kibanaServer: Readonly<{ hostname?: string | undefined; port?: number | undefined; protocol?: string | undefined; } & {}>; queue: Readonly<{} & { timeout: number | moment.Duration; pollInterval: number | moment.Duration; indexInterval: string; pollEnabled: boolean; pollIntervalErrorMultiplier: number; }>; csv: Readonly<{} & { scroll: Readonly<{} & { size: number; duration: string; }>; checkForFormulas: boolean; escapeFormulaValues: boolean; enablePanelActionDownload: boolean; maxSizeBytes: number | ", + "ByteSizeValue", + "; useByteOrderMarkEncoding: boolean; }>; roles: Readonly<{} & { allow: string[]; }>; poll: Readonly<{} & { jobCompletionNotifier: Readonly<{} & { interval: number; intervalErrorMultiplier: number; }>; jobsRefresh: Readonly<{} & { interval: number; intervalErrorMultiplier: number; }>; }>; }>>" + ], + "description": [], + "source": { + "path": "x-pack/plugins/reporting/server/core.ts", + "lineNumber": 65 } } ], @@ -859,7 +881,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 58 + "lineNumber": 65 } }, { @@ -895,7 +917,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 63 + "lineNumber": 75 } } ], @@ -903,7 +925,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 63 + "lineNumber": 75 } }, { @@ -919,7 +941,7 @@ "section": "def-server.ReportingInternalStart", "text": "ReportingInternalStart" }, - ") => void" + ") => Promise" ], "description": [], "children": [ @@ -939,7 +961,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 71 + "lineNumber": 89 } } ], @@ -947,7 +969,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 71 + "lineNumber": 89 } }, { @@ -963,7 +985,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 79 + "lineNumber": 102 } }, { @@ -979,7 +1001,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 90 + "lineNumber": 113 } }, { @@ -995,7 +1017,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 97 + "lineNumber": 120 } }, { @@ -1031,7 +1053,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 104 + "lineNumber": 127 } } ], @@ -1039,7 +1061,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 104 + "lineNumber": 127 } }, { @@ -1057,7 +1079,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 112 + "lineNumber": 135 } }, { @@ -1080,7 +1102,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 132 + "lineNumber": 155 } }, { @@ -1104,7 +1126,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 142 + "lineNumber": 165 } }, { @@ -1127,21 +1149,90 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 150 + "lineNumber": 173 + } + }, + { + "id": "def-server.ReportingCore.scheduleTask", + "type": "Function", + "label": "scheduleTask", + "signature": [ + "(report: ", + { + "pluginId": "reporting", + "scope": "server", + "docId": "kibReportingPluginApi", + "section": "def-server.ReportTaskParams", + "text": "ReportTaskParams" + }, + "<", + { + "pluginId": "reporting", + "scope": "server", + "docId": "kibReportingPluginApi", + "section": "def-server.BasePayload", + "text": "BasePayload" + }, + ">) => Promise<", + { + "pluginId": "taskManager", + "scope": "server", + "docId": "kibTaskManagerPluginApi", + "section": "def-server.ConcreteTaskInstance", + "text": "ConcreteTaskInstance" + }, + ">" + ], + "description": [], + "children": [ + { + "type": "Object", + "label": "report", + "isRequired": true, + "signature": [ + { + "pluginId": "reporting", + "scope": "server", + "docId": "kibReportingPluginApi", + "section": "def-server.ReportTaskParams", + "text": "ReportTaskParams" + }, + "<", + { + "pluginId": "reporting", + "scope": "server", + "docId": "kibReportingPluginApi", + "section": "def-server.BasePayload", + "text": "BasePayload" + }, + ">" + ], + "description": [], + "source": { + "path": "x-pack/plugins/reporting/server/core.ts", + "lineNumber": 177 + } + } + ], + "tags": [], + "returnComment": [], + "source": { + "path": "x-pack/plugins/reporting/server/core.ts", + "lineNumber": 177 } }, { - "id": "def-server.ReportingCore.getEsqueue", + "id": "def-server.ReportingCore.getStore", "type": "Function", - "label": "getEsqueue", + "label": "getStore", "signature": [ "() => Promise<", { "pluginId": "reporting", "scope": "server", "docId": "kibReportingPluginApi", - "section": "def-server.ESQueueInstance", - "text": "ESQueueInstance" + "section": "def-server.ReportingStore", + "text": "ReportingStore" }, ">" ], @@ -1151,7 +1242,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 154 + "lineNumber": 181 } }, { @@ -1175,7 +1266,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 158 + "lineNumber": 185 } }, { @@ -1199,7 +1290,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 168 + "lineNumber": 195 } }, { @@ -1222,7 +1313,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 177 + "lineNumber": 204 } }, { @@ -1245,7 +1336,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 184 + "lineNumber": 211 } }, { @@ -1291,7 +1382,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 193 + "lineNumber": 220 } } ], @@ -1299,7 +1390,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 193 + "lineNumber": 220 } }, { @@ -1344,7 +1435,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 199 + "lineNumber": 226 } }, { @@ -1363,7 +1454,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 199 + "lineNumber": 226 } } ], @@ -1371,7 +1462,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 199 + "lineNumber": 226 } }, { @@ -1409,7 +1500,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 213 + "lineNumber": 240 } }, { @@ -1422,7 +1513,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 213 + "lineNumber": 240 } }, { @@ -1441,7 +1532,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 213 + "lineNumber": 240 } } ], @@ -1449,7 +1540,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 213 + "lineNumber": 240 } }, { @@ -1502,7 +1593,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 233 + "lineNumber": 260 } }, { @@ -1521,7 +1612,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 233 + "lineNumber": 260 } } ], @@ -1529,13 +1620,89 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 233 + "lineNumber": 260 + } + }, + { + "id": "def-server.ReportingCore.trackReport", + "type": "Function", + "label": "trackReport", + "signature": [ + "(reportId: string) => void" + ], + "description": [], + "children": [ + { + "type": "string", + "label": "reportId", + "isRequired": true, + "signature": [ + "string" + ], + "description": [], + "source": { + "path": "x-pack/plugins/reporting/server/core.ts", + "lineNumber": 270 + } + } + ], + "tags": [], + "returnComment": [], + "source": { + "path": "x-pack/plugins/reporting/server/core.ts", + "lineNumber": 270 + } + }, + { + "id": "def-server.ReportingCore.untrackReport", + "type": "Function", + "label": "untrackReport", + "signature": [ + "(reportId: string) => void" + ], + "description": [], + "children": [ + { + "type": "string", + "label": "reportId", + "isRequired": true, + "signature": [ + "string" + ], + "description": [], + "source": { + "path": "x-pack/plugins/reporting/server/core.ts", + "lineNumber": 274 + } + } + ], + "tags": [], + "returnComment": [], + "source": { + "path": "x-pack/plugins/reporting/server/core.ts", + "lineNumber": 274 + } + }, + { + "id": "def-server.ReportingCore.countConcurrentReports", + "type": "Function", + "label": "countConcurrentReports", + "signature": [ + "() => number" + ], + "description": [], + "children": [], + "tags": [], + "returnComment": [], + "source": { + "path": "x-pack/plugins/reporting/server/core.ts", + "lineNumber": 278 } } ], "source": { "path": "x-pack/plugins/reporting/server/core.ts", - "lineNumber": 50 + "lineNumber": 54 }, "initialIsOpen": false }, @@ -1732,7 +1899,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/plugin.ts", - "lineNumber": 105 + "lineNumber": 106 } }, { @@ -1751,7 +1918,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/plugin.ts", - "lineNumber": 105 + "lineNumber": 106 } } ], @@ -1759,7 +1926,7 @@ "returnComment": [], "source": { "path": "x-pack/plugins/reporting/server/plugin.ts", - "lineNumber": 105 + "lineNumber": 106 } } ], @@ -1827,7 +1994,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/types.ts", - "lineNumber": 28 + "lineNumber": 29 }, "signature": [ { @@ -1847,7 +2014,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/types.ts", - "lineNumber": 29 + "lineNumber": 30 }, "signature": [ { @@ -1867,7 +2034,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/types.ts", - "lineNumber": 30 + "lineNumber": 31 }, "signature": [ { @@ -1888,7 +2055,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/types.ts", - "lineNumber": 31 + "lineNumber": 32 }, "signature": [ { @@ -1901,6 +2068,26 @@ " | undefined" ] }, + { + "tags": [], + "id": "def-server.ReportingSetupDeps.taskManager", + "type": "CompoundType", + "label": "taskManager", + "description": [], + "source": { + "path": "x-pack/plugins/reporting/server/types.ts", + "lineNumber": 33 + }, + "signature": [ + { + "pluginId": "taskManager", + "scope": "server", + "docId": "kibTaskManagerPluginApi", + "section": "def-server.TaskManagerSetupContract", + "text": "TaskManagerSetupContract" + } + ] + }, { "tags": [], "id": "def-server.ReportingSetupDeps.usageCollection", @@ -1909,7 +2096,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/types.ts", - "lineNumber": 32 + "lineNumber": 34 }, "signature": [ "Pick<", @@ -1926,7 +2113,7 @@ ], "source": { "path": "x-pack/plugins/reporting/server/types.ts", - "lineNumber": 27 + "lineNumber": 28 }, "initialIsOpen": false }, @@ -1945,7 +2132,7 @@ "description": [], "source": { "path": "x-pack/plugins/reporting/server/types.ts", - "lineNumber": 36 + "lineNumber": 38 }, "signature": [ { @@ -1956,11 +2143,31 @@ "text": "DataPluginStart" } ] + }, + { + "tags": [], + "id": "def-server.ReportingStartDeps.taskManager", + "type": "CompoundType", + "label": "taskManager", + "description": [], + "source": { + "path": "x-pack/plugins/reporting/server/types.ts", + "lineNumber": 39 + }, + "signature": [ + { + "pluginId": "taskManager", + "scope": "server", + "docId": "kibTaskManagerPluginApi", + "section": "def-server.TaskManagerStartContract", + "text": "TaskManagerStartContract" + } + ] } ], "source": { "path": "x-pack/plugins/reporting/server/types.ts", - "lineNumber": 35 + "lineNumber": 37 }, "initialIsOpen": false } diff --git a/docs/user/reporting/script-example.asciidoc b/docs/user/reporting/script-example.asciidoc index 56721d20ea3c7..382d658a18dc9 100644 --- a/docs/user/reporting/script-example.asciidoc +++ b/docs/user/reporting/script-example.asciidoc @@ -38,8 +38,7 @@ Here is an example response for a successfully queued report: "created_by": "elastic", "payload": ..., <2> "timeout": 120000, - "max_attempts": 3, - "priority": 10 + "max_attempts": 3 } } --------------------------------------------------------- diff --git a/x-pack/plugins/reporting/common/schema_utils.test.ts b/x-pack/plugins/reporting/common/schema_utils.test.ts new file mode 100644 index 0000000000000..6e9bb2db75437 --- /dev/null +++ b/x-pack/plugins/reporting/common/schema_utils.test.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { numberToDuration } from './schema_utils'; + +describe('Schema Utils', () => { + it('numberToDuration converts a number/Duration into a Duration object', () => { + expect(numberToDuration(500)).toMatchInlineSnapshot(`"PT0.5S"`); + expect(numberToDuration(moment.duration(1, 'hour'))).toMatchInlineSnapshot(`"PT1H"`); + }); +}); diff --git a/x-pack/plugins/reporting/common/schema_utils.ts b/x-pack/plugins/reporting/common/schema_utils.ts index 6119a2f8582f1..798440bfbb69c 100644 --- a/x-pack/plugins/reporting/common/schema_utils.ts +++ b/x-pack/plugins/reporting/common/schema_utils.ts @@ -20,6 +20,13 @@ export const durationToNumber = (value: number | moment.Duration): number => { return value.asMilliseconds(); }; +export const numberToDuration = (value: number | moment.Duration): moment.Duration => { + if (typeof value === 'number') { + return moment.duration(value, 'milliseconds'); + } + return value; +}; + export const byteSizeValueToNumber = (value: number | ByteSizeValue) => { if (typeof value === 'number') { return value; diff --git a/x-pack/plugins/reporting/kibana.json b/x-pack/plugins/reporting/kibana.json index 93f914a78fe10..31f679a4ec8d0 100644 --- a/x-pack/plugins/reporting/kibana.json +++ b/x-pack/plugins/reporting/kibana.json @@ -14,6 +14,7 @@ "management", "licensing", "uiActions", + "taskManager", "embeddable", "share", "features" diff --git a/x-pack/plugins/reporting/server/core.ts b/x-pack/plugins/reporting/server/core.ts index d4d29f1f51089..4527547ef79b2 100644 --- a/x-pack/plugins/reporting/server/core.ts +++ b/x-pack/plugins/reporting/server/core.ts @@ -12,6 +12,7 @@ import { BasePath, ElasticsearchServiceSetup, KibanaRequest, + PluginInitializerContext, SavedObjectsClientContract, SavedObjectsServiceStart, UiSettingsServiceStart, @@ -21,12 +22,14 @@ import { LicensingPluginSetup } from '../../licensing/server'; import { SecurityPluginSetup } from '../../security/server'; import { DEFAULT_SPACE_ID } from '../../spaces/common/constants'; import { SpacesPluginSetup } from '../../spaces/server'; +import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; import { ReportingConfig } from './'; import { HeadlessChromiumDriverFactory } from './browsers/chromium/driver_factory'; +import { ReportingConfigType } from './config'; import { checkLicense, getExportTypesRegistry, LevelLogger } from './lib'; -import { ESQueueInstance } from './lib/create_queue'; import { screenshotsObservableFactory, ScreenshotsObservableFn } from './lib/screenshots'; import { ReportingStore } from './lib/store'; +import { ExecuteReportTask, MonitorReportsTask, ReportTaskParams } from './lib/tasks'; import { ReportingPluginRouter } from './types'; export interface ReportingInternalSetup { @@ -37,14 +40,15 @@ export interface ReportingInternalSetup { licensing: LicensingPluginSetup; security?: SecurityPluginSetup; spaces?: SpacesPluginSetup; + taskManager: TaskManagerSetupContract; } export interface ReportingInternalStart { browserDriverFactory: HeadlessChromiumDriverFactory; - esqueue: ESQueueInstance; store: ReportingStore; savedObjects: SavedObjectsServiceStart; uiSettings: UiSettingsServiceStart; + taskManager: TaskManagerStartContract; } export class ReportingCore { @@ -53,9 +57,17 @@ export class ReportingCore { private readonly pluginSetup$ = new Rx.ReplaySubject(); // observe async background setupDeps and config each are done private readonly pluginStart$ = new Rx.ReplaySubject(); // observe async background startDeps private exportTypesRegistry = getExportTypesRegistry(); + private executeTask: ExecuteReportTask; + private monitorTask: MonitorReportsTask; private config?: ReportingConfig; + private executing: Set; - constructor(private logger: LevelLogger) {} + constructor(private logger: LevelLogger, context: PluginInitializerContext) { + const config = context.config.get(); + this.executeTask = new ExecuteReportTask(this, config, this.logger); + this.monitorTask = new MonitorReportsTask(this, config, this.logger); + this.executing = new Set(); + } /* * Register setupDeps @@ -63,14 +75,25 @@ export class ReportingCore { public pluginSetup(setupDeps: ReportingInternalSetup) { this.pluginSetup$.next(true); // trigger the observer this.pluginSetupDeps = setupDeps; // cache + + const { executeTask, monitorTask } = this; + setupDeps.taskManager.registerTaskDefinitions({ + [executeTask.TYPE]: executeTask.getTaskDefinition(), + [monitorTask.TYPE]: monitorTask.getTaskDefinition(), + }); } /* * Register startDeps */ - public pluginStart(startDeps: ReportingInternalStart) { + public async pluginStart(startDeps: ReportingInternalStart) { this.pluginStart$.next(startDeps); // trigger the observer this.pluginStartDeps = startDeps; // cache + + const { taskManager } = startDeps; + const { executeTask, monitorTask } = this; + // enable this instance to generate reports and to monitor for pending reports + await Promise.all([executeTask.init(taskManager), monitorTask.init(taskManager)]); } /* @@ -151,8 +174,12 @@ export class ReportingCore { return this.exportTypesRegistry; } - public async getEsqueue() { - return (await this.getPluginStartDeps()).esqueue; + public async scheduleTask(report: ReportTaskParams) { + return await this.executeTask.scheduleTask(report); + } + + public async getStore() { + return (await this.getPluginStartDeps()).store; } public async getLicenseInfo() { @@ -239,4 +266,16 @@ export class ReportingCore { const savedObjectsClient = await this.getSavedObjectsClient(request); return await this.getUiSettingsServiceFactory(savedObjectsClient); } + + public trackReport(reportId: string) { + this.executing.add(reportId); + } + + public untrackReport(reportId: string) { + this.executing.delete(reportId); + } + + public countConcurrentReports(): number { + return this.executing.size; + } } diff --git a/x-pack/plugins/reporting/server/export_types/csv/create_job.ts b/x-pack/plugins/reporting/server/export_types/csv/create_job.ts index d0182d47e479d..876d190c9eee8 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/create_job.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/create_job.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { CSV_JOB_TYPE_DEPRECATED } from '../../../common/constants'; import { cryptoFactory } from '../../lib'; import { CreateJobFn, CreateJobFnFactory } from '../../types'; import { @@ -16,9 +15,7 @@ import { export const createJobFnFactory: CreateJobFnFactory< CreateJobFn -> = function createJobFactoryFn(reporting, parentLogger) { - const logger = parentLogger.clone([CSV_JOB_TYPE_DEPRECATED, 'create-job']); - +> = function createJobFactoryFn(reporting, logger) { const config = reporting.getConfig(); const crypto = cryptoFactory(config.get('encryptionKey')); diff --git a/x-pack/plugins/reporting/server/export_types/csv/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv/execute_job.ts index 86acd8f3c86ed..0e13a91649406 100644 --- a/x-pack/plugins/reporting/server/export_types/csv/execute_job.ts +++ b/x-pack/plugins/reporting/server/export_types/csv/execute_job.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { CONTENT_TYPE_CSV, CSV_JOB_TYPE_DEPRECATED } from '../../../common/constants'; +import { CONTENT_TYPE_CSV } from '../../../common/constants'; import { RunTaskFn, RunTaskFnFactory } from '../../types'; import { decryptJobHeaders } from '../common'; import { createGenerateCsv } from './generate_csv'; @@ -18,7 +18,7 @@ export const runTaskFnFactory: RunTaskFnFactory< return async function runTask(jobId, job, cancellationToken) { const elasticsearch = reporting.getElasticsearchService(); - const logger = parentLogger.clone([CSV_JOB_TYPE_DEPRECATED, 'execute-job', jobId]); + const logger = parentLogger.clone([jobId]); const generateCsv = createGenerateCsv(logger); const encryptionKey = config.get('encryptionKey'); diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/create_job.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/create_job.ts index b27c244aa11ae..75b07e5bca8c8 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/create_job.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/create_job.ts @@ -7,8 +7,8 @@ import { notFound, notImplemented } from '@hapi/boom'; import { get } from 'lodash'; -import { CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants'; import { CsvFromSavedObjectRequest } from '../../routes/generate_from_savedobject_immediate'; +import type { ReportingRequestHandlerContext } from '../../types'; import { CreateJobFnFactory } from '../../types'; import { JobParamsPanelCsv, @@ -18,7 +18,6 @@ import { SavedObjectServiceError, VisObjectAttributesJSON, } from './types'; -import type { ReportingRequestHandlerContext } from '../../types'; export type ImmediateCreateJobFn = ( jobParams: JobParamsPanelCsv, @@ -30,7 +29,7 @@ export const createJobFnFactory: CreateJobFnFactory = func reporting, parentLogger ) { - const logger = parentLogger.clone([CSV_FROM_SAVEDOBJECT_JOB_TYPE, 'create-job']); + const logger = parentLogger.clone(['create-job']); return async function createJob(jobParams, context, req) { const { savedObjectType, savedObjectId } = jobParams; diff --git a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/execute_job.ts b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/execute_job.ts index b037e72699dd6..b79bb063c26f8 100644 --- a/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/execute_job.ts +++ b/x-pack/plugins/reporting/server/export_types/csv_from_savedobject/execute_job.ts @@ -7,13 +7,13 @@ import { KibanaRequest } from 'src/core/server'; import { CancellationToken } from '../../../common'; -import { CONTENT_TYPE_CSV, CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants'; +import { CONTENT_TYPE_CSV } from '../../../common/constants'; import { TaskRunResult } from '../../lib/tasks'; +import type { ReportingRequestHandlerContext } from '../../types'; import { RunTaskFnFactory } from '../../types'; import { createGenerateCsv } from '../csv/generate_csv'; import { getGenerateCsvParams } from './lib/get_csv_job'; import { JobPayloadPanelCsv } from './types'; -import type { ReportingRequestHandlerContext } from '../../types'; /* * ImmediateExecuteFn receives the job doc payload because the payload was @@ -31,7 +31,7 @@ export const runTaskFnFactory: RunTaskFnFactory = function e parentLogger ) { const config = reporting.getConfig(); - const logger = parentLogger.clone([CSV_FROM_SAVEDOBJECT_JOB_TYPE, 'execute-job']); + const logger = parentLogger.clone(['execute-job']); return async function runTask(jobId, jobPayload, context, req) { const generateCsv = createGenerateCsv(logger); diff --git a/x-pack/plugins/reporting/server/export_types/png/create_job/index.ts b/x-pack/plugins/reporting/server/export_types/png/create_job/index.ts index 62af9a9b80120..488a339e3ef4b 100644 --- a/x-pack/plugins/reporting/server/export_types/png/create_job/index.ts +++ b/x-pack/plugins/reporting/server/export_types/png/create_job/index.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { PNG_JOB_TYPE } from '../../../../common/constants'; import { cryptoFactory } from '../../../lib'; import { CreateJobFn, CreateJobFnFactory } from '../../../types'; import { validateUrls } from '../../common'; @@ -13,8 +12,7 @@ import { JobParamsPNG, TaskPayloadPNG } from '../types'; export const createJobFnFactory: CreateJobFnFactory< CreateJobFn -> = function createJobFactoryFn(reporting, parentLogger) { - const logger = parentLogger.clone([PNG_JOB_TYPE, 'execute-job']); +> = function createJobFactoryFn(reporting, logger) { const config = reporting.getConfig(); const crypto = cryptoFactory(config.get('encryptionKey')); diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf/create_job/index.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf/create_job/index.ts index ed06d04855a3a..c0f30f96415f4 100644 --- a/x-pack/plugins/reporting/server/export_types/printable_pdf/create_job/index.ts +++ b/x-pack/plugins/reporting/server/export_types/printable_pdf/create_job/index.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { PDF_JOB_TYPE } from '../../../../common/constants'; import { cryptoFactory } from '../../../lib'; import { CreateJobFn, CreateJobFnFactory } from '../../../types'; import { validateUrls } from '../../common'; @@ -13,10 +12,9 @@ import { JobParamsPDF, TaskPayloadPDF } from '../types'; export const createJobFnFactory: CreateJobFnFactory< CreateJobFn -> = function createJobFactoryFn(reporting, parentLogger) { +> = function createJobFactoryFn(reporting, logger) { const config = reporting.getConfig(); const crypto = cryptoFactory(config.get('encryptionKey')); - const logger = parentLogger.clone([PDF_JOB_TYPE, 'create-job']); return async function createJob( { title, relativeUrls, browserTimezone, layout, objectType }, diff --git a/x-pack/plugins/reporting/server/lib/create_queue.ts b/x-pack/plugins/reporting/server/lib/create_queue.ts deleted file mode 100644 index afef241e0230a..0000000000000 --- a/x-pack/plugins/reporting/server/lib/create_queue.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ReportingCore } from '../core'; -import { createWorkerFactory } from './create_worker'; -// @ts-ignore -import { Esqueue } from './esqueue'; -import { createTaggedLogger } from './esqueue/create_tagged_logger'; -import { LevelLogger } from './level_logger'; -import { ReportDocument, ReportingStore } from './store'; -import { TaskRunResult } from './tasks'; - -interface ESQueueWorker { - on: (event: string, handler: any) => void; -} - -export interface ESQueueInstance { - registerWorker: ( - pluginId: string, - workerFn: GenericWorkerFn, - workerOptions: { - kibanaName: string; - kibanaId: string; - interval: number; - intervalErrorMultiplier: number; - } - ) => ESQueueWorker; -} - -// GenericWorkerFn is a generic for ImmediateExecuteFn | ESQueueWorkerExecuteFn, -type GenericWorkerFn = ( - jobSource: ReportDocument, - ...workerRestArgs: any[] -) => void | Promise; - -export async function createQueueFactory( - reporting: ReportingCore, - store: ReportingStore, - logger: LevelLogger -): Promise { - const config = reporting.getConfig(); - - // esqueue-related - const queueTimeout = config.get('queue', 'timeout'); - const isPollingEnabled = config.get('queue', 'pollEnabled'); - - const elasticsearch = reporting.getElasticsearchService(); - const queueOptions = { - timeout: queueTimeout, - client: elasticsearch.legacy.client, - logger: createTaggedLogger(logger, ['esqueue', 'queue-worker']), - }; - - const queue: ESQueueInstance = new Esqueue(store, queueOptions); - - if (isPollingEnabled) { - // create workers to poll the index for idle jobs waiting to be claimed and executed - const createWorker = createWorkerFactory(reporting, logger); - await createWorker(queue); - } else { - logger.info( - 'xpack.reporting.queue.pollEnabled is set to false. This Kibana instance ' + - 'will not poll for idle jobs to claim and execute. Make sure another ' + - 'Kibana instance with polling enabled is running in this cluster so ' + - 'reporting jobs can complete.', - ['create_queue'] - ); - } - - return queue; -} diff --git a/x-pack/plugins/reporting/server/lib/create_worker.test.ts b/x-pack/plugins/reporting/server/lib/create_worker.test.ts deleted file mode 100644 index 9a230d77e555a..0000000000000 --- a/x-pack/plugins/reporting/server/lib/create_worker.test.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as sinon from 'sinon'; -import { ReportingConfig, ReportingCore } from '../../server'; -import { - createMockConfig, - createMockConfigSchema, - createMockLevelLogger, - createMockReportingCore, -} from '../test_helpers'; -import { createWorkerFactory } from './create_worker'; -// @ts-ignore -import { Esqueue } from './esqueue'; -// @ts-ignore -import { ClientMock } from './esqueue/__fixtures__/legacy_elasticsearch'; -import { ExportTypesRegistry } from './export_types_registry'; - -const logger = createMockLevelLogger(); -const reportingConfig = { - queue: { pollInterval: 3300, pollIntervalErrorMultiplier: 10 }, - server: { name: 'test-server-123', uuid: 'g9ymiujthvy6v8yrh7567g6fwzgzftzfr' }, -}; - -const executeJobFactoryStub = sinon.stub(); - -const getMockExportTypesRegistry = ( - exportTypes: any[] = [{ runTaskFnFactory: executeJobFactoryStub }] -) => - ({ - getAll: () => exportTypes, - } as ExportTypesRegistry); - -describe('Create Worker', () => { - let mockReporting: ReportingCore; - let mockConfig: ReportingConfig; - let queue: Esqueue; - let client: ClientMock; - - beforeEach(async () => { - const mockSchema = createMockConfigSchema(reportingConfig); - mockConfig = createMockConfig(mockSchema); - mockReporting = await createMockReportingCore(mockConfig); - mockReporting.getExportTypesRegistry = () => getMockExportTypesRegistry(); - - client = new ClientMock(); - queue = new Esqueue('reporting-queue', { client }); - executeJobFactoryStub.reset(); - }); - - test('Creates a single Esqueue worker for Reporting', async () => { - const createWorker = createWorkerFactory(mockReporting, logger); - const registerWorkerSpy = sinon.spy(queue, 'registerWorker'); - - await createWorker(queue); - - sinon.assert.callCount(executeJobFactoryStub, 1); - sinon.assert.callCount(registerWorkerSpy, 1); - - const { firstCall } = registerWorkerSpy; - const [workerName, workerFn, workerOpts] = firstCall.args; - - expect(workerName).toBe('reporting'); - expect(workerFn).toMatchInlineSnapshot(`[Function]`); - expect(workerOpts).toMatchInlineSnapshot(` -Object { - "interval": 3300, - "intervalErrorMultiplier": 10, - "kibanaId": "g9ymiujthvy6v8yrh7567g6fwzgzftzfr", - "kibanaName": "test-server-123", -} -`); - }); - - test('Creates a single Esqueue worker for Reporting, even if there are multiple export types', async () => { - const exportTypesRegistry = getMockExportTypesRegistry([ - { runTaskFnFactory: executeJobFactoryStub }, - { runTaskFnFactory: executeJobFactoryStub }, - { runTaskFnFactory: executeJobFactoryStub }, - { runTaskFnFactory: executeJobFactoryStub }, - { runTaskFnFactory: executeJobFactoryStub }, - ]); - mockReporting.getExportTypesRegistry = () => exportTypesRegistry; - const createWorker = createWorkerFactory(mockReporting, logger); - const registerWorkerSpy = sinon.spy(queue, 'registerWorker'); - - await createWorker(queue); - - sinon.assert.callCount(executeJobFactoryStub, 5); - sinon.assert.callCount(registerWorkerSpy, 1); - - const { firstCall } = registerWorkerSpy; - const [workerName, workerFn, workerOpts] = firstCall.args; - - expect(workerName).toBe('reporting'); - expect(workerFn).toMatchInlineSnapshot(`[Function]`); - expect(workerOpts).toMatchInlineSnapshot(` -Object { - "interval": 3300, - "intervalErrorMultiplier": 10, - "kibanaId": "g9ymiujthvy6v8yrh7567g6fwzgzftzfr", - "kibanaName": "test-server-123", -} -`); - }); -}); diff --git a/x-pack/plugins/reporting/server/lib/create_worker.ts b/x-pack/plugins/reporting/server/lib/create_worker.ts deleted file mode 100644 index 6bbfd674604d9..0000000000000 --- a/x-pack/plugins/reporting/server/lib/create_worker.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { CancellationToken } from '../../common'; -import { PLUGIN_ID } from '../../common/constants'; -import { durationToNumber } from '../../common/schema_utils'; -import { ReportingCore } from '../../server'; -import { LevelLogger } from '../../server/lib'; -import { RunTaskFn } from '../../server/types'; -import { ESQueueInstance } from './create_queue'; -// @ts-ignore untyped dependency -import { events as esqueueEvents } from './esqueue'; -import { ReportDocument } from './store'; -import { ReportTaskParams } from './tasks'; - -export function createWorkerFactory(reporting: ReportingCore, logger: LevelLogger) { - const config = reporting.getConfig(); - const queueConfig = config.get('queue'); - const kibanaName = config.kbnConfig.get('server', 'name'); - const kibanaId = config.kbnConfig.get('server', 'uuid'); - - // Once more document types are added, this will need to be passed in - return async function createWorker(queue: ESQueueInstance) { - // export type / execute job map - const jobExecutors: Map = new Map(); - - for (const exportType of reporting.getExportTypesRegistry().getAll()) { - const jobExecutor = exportType.runTaskFnFactory(reporting, logger); - jobExecutors.set(exportType.jobType, jobExecutor); - } - - const workerFn = ( - jobSource: ReportDocument, - payload: ReportTaskParams['payload'], - cancellationToken: CancellationToken - ) => { - const { - _id: jobId, - _source: { jobtype: jobType }, - } = jobSource; - - if (!jobId) { - throw new Error(`Claimed job is missing an ID!: ${JSON.stringify(jobSource)}`); - } - - const jobTypeExecutor = jobExecutors.get(jobType); - if (!jobTypeExecutor) { - throw new Error(`Unable to find a job executor for the claimed job: [${jobId}]`); - } - - // pass the work to the jobExecutor - return jobTypeExecutor(jobId, payload, cancellationToken); - }; - - const workerOptions = { - kibanaName, - kibanaId, - interval: durationToNumber(queueConfig.pollInterval), - intervalErrorMultiplier: queueConfig.pollIntervalErrorMultiplier, - }; - const worker = queue.registerWorker(PLUGIN_ID, workerFn, workerOptions); - - worker.on(esqueueEvents.EVENT_WORKER_COMPLETE, (res: any) => { - logger.debug(`Worker completed: (${res.job.id})`); - }); - worker.on(esqueueEvents.EVENT_WORKER_JOB_EXECUTION_ERROR, (res: any) => { - logger.debug(`Worker error: (${res.job.id})`); - }); - worker.on(esqueueEvents.EVENT_WORKER_JOB_TIMEOUT, (res: any) => { - logger.debug(`Job timeout exceeded: (${res.job.id})`); - }); - }; -} diff --git a/x-pack/plugins/reporting/server/lib/enqueue_job.test.ts b/x-pack/plugins/reporting/server/lib/enqueue_job.test.ts new file mode 100644 index 0000000000000..8e5a61e46df91 --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/enqueue_job.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest } from 'src/core/server'; +import { ReportingCore } from '../'; +import { TaskManagerStartContract } from '../../../task_manager/server'; +import { ReportingInternalStart } from '../core'; +import { + createMockConfig, + createMockConfigSchema, + createMockLevelLogger, + createMockReportingCore, +} from '../test_helpers'; +import { BasePayload, ReportingRequestHandlerContext } from '../types'; +import { ExportTypesRegistry, ReportingStore } from './'; +import { enqueueJobFactory } from './enqueue_job'; +import { Report } from './store'; +import { TaskRunResult } from './tasks'; + +describe('Enqueue Job', () => { + const logger = createMockLevelLogger(); + const mockSchema = createMockConfigSchema(); + const mockConfig = createMockConfig(mockSchema); + let mockReporting: ReportingCore; + let mockExportTypesRegistry: ExportTypesRegistry; + + beforeAll(async () => { + mockExportTypesRegistry = new ExportTypesRegistry(); + mockExportTypesRegistry.register({ + id: 'printablePdf', + name: 'Printable PDFble', + jobType: 'printable_pdf', + jobContentEncoding: 'base64', + jobContentExtension: 'pdf', + validLicenses: ['turquoise'], + createJobFnFactory: () => async () => + (({ createJobTest: { test1: 'yes' } } as unknown) as BasePayload), + runTaskFnFactory: () => async () => + (({ runParamsTest: { test2: 'yes' } } as unknown) as TaskRunResult), + }); + mockReporting = await createMockReportingCore(mockConfig); + mockReporting.getExportTypesRegistry = () => mockExportTypesRegistry; + mockReporting.getStore = () => + Promise.resolve(({ + addReport: jest + .fn() + .mockImplementation( + (report) => new Report({ ...report, _index: '.reporting-foo-index-234' }) + ), + } as unknown) as ReportingStore); + + const scheduleMock = jest.fn().mockImplementation(() => ({ + id: '123-great-id', + })); + + await mockReporting.pluginStart(({ + taskManager: ({ + ensureScheduled: jest.fn(), + schedule: scheduleMock, + } as unknown) as TaskManagerStartContract, + } as unknown) as ReportingInternalStart); + }); + + it('returns a Report object', async () => { + const enqueueJob = enqueueJobFactory(mockReporting, logger); + const report = await enqueueJob( + 'printablePdf', + { + objectType: 'visualization', + title: 'cool-viz', + }, + false, + ({} as unknown) as ReportingRequestHandlerContext, + ({} as unknown) as KibanaRequest + ); + + expect(report).toMatchObject({ + _id: expect.any(String), + _index: '.reporting-foo-index-234', + attempts: 0, + created_by: false, + created_at: expect.any(String), + jobtype: 'printable_pdf', + meta: { objectType: 'visualization' }, + output: null, + payload: { createJobTest: { test1: 'yes' } }, + status: 'pending', + }); + }); +}); diff --git a/x-pack/plugins/reporting/server/lib/enqueue_job.ts b/x-pack/plugins/reporting/server/lib/enqueue_job.ts index 1c0eb8f4f5b77..5ac644298796d 100644 --- a/x-pack/plugins/reporting/server/lib/enqueue_job.ts +++ b/x-pack/plugins/reporting/server/lib/enqueue_job.ts @@ -7,7 +7,6 @@ import { KibanaRequest } from 'src/core/server'; import { ReportingCore } from '../'; -import { durationToNumber } from '../../common/schema_utils'; import { BaseParams, ReportingUser } from '../types'; import { LevelLogger } from './'; import { Report } from './store'; @@ -25,15 +24,7 @@ export function enqueueJobFactory( reporting: ReportingCore, parentLogger: LevelLogger ): EnqueueJobFn { - const logger = parentLogger.clone(['queue-job']); - const config = reporting.getConfig(); - const jobSettings = { - timeout: durationToNumber(config.get('queue', 'timeout')), - browser_type: config.get('capture', 'browser', 'type'), - max_attempts: config.get('capture', 'maxAttempts'), - priority: 10, // unused - }; - + const logger = parentLogger.clone(['createJob']); return async function enqueueJob( exportTypeId: string, jobParams: BaseParams, @@ -47,27 +38,34 @@ export function enqueueJobFactory( throw new Error(`Export type ${exportTypeId} does not exist in the registry!`); } - const [createJob, { store }] = await Promise.all([ - exportType.createJobFnFactory(reporting, logger), - reporting.getPluginStartDeps(), + const [createJob, store] = await Promise.all([ + exportType.createJobFnFactory(reporting, logger.clone([exportType.id])), + reporting.getStore(), ]); + const config = reporting.getConfig(); const job = await createJob(jobParams, context, request); - const pendingReport = new Report({ - jobtype: exportType.jobType, - created_by: user ? user.username : false, - payload: job, - meta: { - objectType: jobParams.objectType, - layout: jobParams.layout?.id, - }, - ...jobSettings, - }); - // store the pending report, puts it in the Reporting Management UI table - const report = await store.addReport(pendingReport); + // 1. Add the report to ReportingStore to show as pending + const report = await store.addReport( + new Report({ + jobtype: exportType.jobType, + created_by: user ? user.username : false, + max_attempts: config.get('capture', 'maxAttempts'), // NOTE: since max attempts is stored in the document, changing the capture.maxAttempts setting does not affect existing pending reports + payload: job, + meta: { + objectType: jobParams.objectType, + layout: jobParams.layout?.id, + }, + }) + ); + logger.debug(`Successfully stored pending job: ${report._index}/${report._id}`); - logger.info(`Scheduled ${exportType.name} report: ${report._id}`); + // 2. Schedule the report with Task Manager + const task = await reporting.scheduleTask(report.toReportTaskJSON()); + logger.info( + `Scheduled ${exportType.name} reporting task. Task ID: ${task.id}. Report ID: ${report._id}` + ); return report; }; diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/job.js b/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/job.js deleted file mode 100644 index 32f3d7dcaf706..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/job.js +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import events from 'events'; - -export class JobMock extends events.EventEmitter { - constructor(queue, index, type, payload, options = {}) { - super(); - - this.queue = queue; - this.index = index; - this.jobType = type; - this.payload = payload; - this.options = options; - } - - getProp(name) { - return this[name]; - } -} diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/legacy_elasticsearch.js b/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/legacy_elasticsearch.js deleted file mode 100644 index 8362431cacee5..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/legacy_elasticsearch.js +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { uniqueId, times, random } from 'lodash'; -import { errors as esErrors } from 'elasticsearch'; - -export function ClientMock() { - this.callAsInternalUser = (endpoint, params = {}, ...rest) => { - if (endpoint === 'indices.create') { - return Promise.resolve({ acknowledged: true }); - } - - if (endpoint === 'indices.exists') { - return Promise.resolve(false); - } - - if (endpoint === 'index') { - const shardCount = 2; - return Promise.resolve({ - _index: params.index || 'index', - _id: params.id || uniqueId('testDoc'), - _seq_no: 1, - _primary_term: 1, - _shards: { total: shardCount, successful: shardCount, failed: 0 }, - created: true, - }); - } - - if (endpoint === 'get') { - if (params === esErrors.NotFound) return esErrors.NotFound; - - const _source = { - jobtype: 'jobtype', - created_by: false, - - payload: { - id: 'sample-job-1', - now: 'Mon Apr 25 2016 14:13:04 GMT-0700 (MST)', - }, - - priority: 10, - timeout: 10000, - created_at: '2016-04-25T21:13:04.738Z', - attempts: 0, - max_attempts: 3, - status: 'pending', - ...(rest[0] || {}), - }; - - return Promise.resolve({ - _index: params.index || 'index', - _id: params.id || 'AVRPRLnlp7Ur1SZXfT-T', - _seq_no: params._seq_no || 1, - _primary_term: params._primary_term || 1, - found: true, - _source: _source, - }); - } - - if (endpoint === 'search') { - const [count = 5, source = {}] = rest; - const hits = times(count, () => { - return { - _index: params.index || 'index', - _id: uniqueId('documentId'), - _seq_no: random(1, 5), - _primar_term: random(1, 5), - _score: null, - _source: { - created_at: new Date().toString(), - number: random(0, count, true), - ...source, - }, - }; - }); - return Promise.resolve({ - took: random(0, 10), - timed_out: false, - _shards: { - total: 5, - successful: 5, - failed: 0, - }, - hits: { - total: count, - max_score: null, - hits: hits, - }, - }); - } - - if (endpoint === 'update') { - const shardCount = 2; - return Promise.resolve({ - _index: params.index || 'index', - _id: params.id || uniqueId('testDoc'), - _seq_no: params.if_seq_no + 1 || 2, - _primary_term: params.if_primary_term + 1 || 2, - _shards: { total: shardCount, successful: shardCount, failed: 0 }, - created: true, - }); - } - - return Promise.resolve(); - }; - - this.transport = {}; -} diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/queue.js b/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/queue.js deleted file mode 100644 index 765ee0f56cb8c..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/queue.js +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import events from 'events'; - -export class QueueMock extends events.EventEmitter { - constructor() { - super(); - } - - setClient(client) { - this.client = client; - } -} diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/worker.js deleted file mode 100644 index 2c9c547d60735..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/__fixtures__/worker.js +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import events from 'events'; - -export class WorkerMock extends events.EventEmitter { - constructor(queue, type, workerFn, opts = {}) { - super(); - - this.queue = queue; - this.type = type; - this.workerFn = workerFn; - this.options = opts; - } - - getProp(name) { - return this[name]; - } -} diff --git a/x-pack/plugins/reporting/server/lib/esqueue/constants/default_settings.js b/x-pack/plugins/reporting/server/lib/esqueue/constants/default_settings.js deleted file mode 100644 index 6446073562553..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/constants/default_settings.js +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const defaultSettings = { - DEFAULT_SETTING_TIMEOUT: 10000, - DEFAULT_SETTING_DATE_SEPARATOR: '-', - DEFAULT_SETTING_INTERVAL: 'week', - DEFAULT_SETTING_INDEX_SETTINGS: { - number_of_shards: 1, - auto_expand_replicas: '0-1', - }, - DEFAULT_WORKER_CHECK_SIZE: 1, -}; diff --git a/x-pack/plugins/reporting/server/lib/esqueue/constants/events.js b/x-pack/plugins/reporting/server/lib/esqueue/constants/events.js deleted file mode 100644 index 2c83408b0f88e..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/constants/events.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const events = { - EVENT_QUEUE_ERROR: 'queue:error', - EVENT_JOB_ERROR: 'job:error', - EVENT_JOB_CREATED: 'job:created', - EVENT_JOB_CREATE_ERROR: 'job:creation error', - EVENT_WORKER_COMPLETE: 'worker:job complete', - EVENT_WORKER_JOB_CLAIM_ERROR: 'worker:claim job error', - EVENT_WORKER_JOB_SEARCH_ERROR: 'worker:pending jobs error', - EVENT_WORKER_JOB_UPDATE_ERROR: 'worker:update job error', - EVENT_WORKER_JOB_FAIL: 'worker:job failed', - EVENT_WORKER_JOB_FAIL_ERROR: 'worker:failed job update error', - EVENT_WORKER_JOB_EXECUTION_ERROR: 'worker:job execution error', - EVENT_WORKER_JOB_TIMEOUT: 'worker:job timeout', -}; diff --git a/x-pack/plugins/reporting/server/lib/esqueue/constants/index.js b/x-pack/plugins/reporting/server/lib/esqueue/constants/index.js deleted file mode 100644 index 87ff1e354a7ad..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/constants/index.js +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { statuses } from '../../statuses'; -import { defaultSettings } from './default_settings'; -import { events } from './events'; - -export const constants = { - ...events, - ...statuses, - ...defaultSettings, -}; diff --git a/x-pack/plugins/reporting/server/lib/esqueue/create_tagged_logger.ts b/x-pack/plugins/reporting/server/lib/esqueue/create_tagged_logger.ts deleted file mode 100644 index 1bb30b4bc7cf0..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/create_tagged_logger.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { LevelLogger } from '../level_logger'; - -export function createTaggedLogger(logger: LevelLogger, tags: string[]) { - return (msg: string, additionalTags = []) => { - const allTags = [...tags, ...additionalTags]; - - if (allTags.includes('info')) { - const newTags = allTags.filter((t) => t !== 'info'); // Ensure 'info' is not included twice - logger.info(msg, newTags); - } else if (allTags.includes('debug')) { - const newTags = allTags.filter((t) => t !== 'debug'); - logger.debug(msg, newTags); - } else if (allTags.includes('warn') || allTags.includes('warning')) { - const newTags = allTags.filter((t) => t !== 'warn' && t !== 'warning'); - logger.warn(msg, newTags); - } else { - logger.error(msg, allTags); - } - }; -} diff --git a/x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js b/x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js deleted file mode 100644 index d3b859ea2cbd9..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/helpers/create_index.js +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { constants } from '../constants'; - -const schema = { - meta: { - // We are indexing these properties with both text and keyword fields because that's what will be auto generated - // when an index already exists. This schema is only used when a reporting index doesn't exist. This way existing - // reporting indexes and new reporting indexes will look the same and the data can be queried in the same - // manner. - properties: { - /** - * Type of object that is triggering this report. Should be either search, visualization or dashboard. - * Used for job listing and telemetry stats only. - */ - objectType: { - type: 'text', - fields: { - keyword: { - type: 'keyword', - ignore_above: 256, - }, - }, - }, - /** - * Can be either preserve_layout, print or none (in the case of csv export). - * Used for phone home stats only. - */ - layout: { - type: 'text', - fields: { - keyword: { - type: 'keyword', - ignore_above: 256, - }, - }, - }, - }, - }, - browser_type: { type: 'keyword' }, - jobtype: { type: 'keyword' }, - payload: { type: 'object', enabled: false }, - priority: { type: 'byte' }, - timeout: { type: 'long' }, - process_expiration: { type: 'date' }, - created_by: { type: 'keyword' }, - created_at: { type: 'date' }, - started_at: { type: 'date' }, - completed_at: { type: 'date' }, - attempts: { type: 'short' }, - max_attempts: { type: 'short' }, - kibana_name: { type: 'keyword' }, - kibana_id: { type: 'keyword' }, - status: { type: 'keyword' }, - output: { - type: 'object', - properties: { - content_type: { type: 'keyword' }, - size: { type: 'long' }, - content: { type: 'object', enabled: false }, - }, - }, -}; - -export function createIndex(client, indexName, indexSettings = {}) { - const body = { - settings: { - ...constants.DEFAULT_SETTING_INDEX_SETTINGS, - ...indexSettings, - }, - mappings: { - properties: schema, - }, - }; - - return client - .callAsInternalUser('indices.exists', { - index: indexName, - }) - .then((exists) => { - if (!exists) { - return client - .callAsInternalUser('indices.create', { - index: indexName, - body: body, - }) - .then(() => true) - .catch((err) => { - /* FIXME creating the index will fail if there were multiple jobs staged in parallel. - * Each staged job checks `client.indices.exists` and could each get `false` as a response. - * Only the first job in line can successfully create it though. - * The problem might only happen in automated tests, where the indices are deleted after each test run. - * This catch block is in place to not fail a job if the job runner hits this race condition. - * Unfortunately we don't have a logger in scope to log a warning. - */ - const isIndexExistsError = - err && - err.body && - err.body.error && - err.body.error.type === 'resource_already_exists_exception'; - if (isIndexExistsError) { - return true; - } - - throw err; - }); - } - return exists; - }); -} diff --git a/x-pack/plugins/reporting/server/lib/esqueue/helpers/errors.js b/x-pack/plugins/reporting/server/lib/esqueue/helpers/errors.js deleted file mode 100644 index ffe04839c42e5..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/helpers/errors.js +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export function WorkerTimeoutError(message, props = {}) { - this.name = 'WorkerTimeoutError'; - this.message = message; - this.timeout = props.timeout; - this.jobId = props.jobId; - - if ('captureStackTrace' in Error) Error.captureStackTrace(this, WorkerTimeoutError); - else this.stack = new Error().stack; -} -WorkerTimeoutError.prototype = Object.create(Error.prototype); - -export function UnspecifiedWorkerError(message, props = {}) { - this.name = 'UnspecifiedWorkerError'; - this.message = message; - this.jobId = props.jobId; - - if ('captureStackTrace' in Error) Error.captureStackTrace(this, UnspecifiedWorkerError); - else this.stack = new Error().stack; -} -UnspecifiedWorkerError.prototype = Object.create(Error.prototype); diff --git a/x-pack/plugins/reporting/server/lib/esqueue/helpers/errors.test.js b/x-pack/plugins/reporting/server/lib/esqueue/helpers/errors.test.js deleted file mode 100644 index 01e6430e671a0..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/helpers/errors.test.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { WorkerTimeoutError, UnspecifiedWorkerError } from './errors'; - -describe('custom errors', function () { - describe('WorkerTimeoutError', function () { - it('should be function', () => { - expect(typeof WorkerTimeoutError).toBe('function'); - }); - - it('should have a name', function () { - const err = new WorkerTimeoutError('timeout error'); - expect(err).toHaveProperty('name', 'WorkerTimeoutError'); - }); - - it('should take a jobId property', function () { - const err = new WorkerTimeoutError('timeout error', { jobId: 'il7hl34rqlo8ro' }); - expect(err).toHaveProperty('jobId', 'il7hl34rqlo8ro'); - }); - - it('should take a timeout property', function () { - const err = new WorkerTimeoutError('timeout error', { timeout: 15000 }); - expect(err).toHaveProperty('timeout', 15000); - }); - - it('should be stringifyable', function () { - const err = new WorkerTimeoutError('timeout error'); - expect(`${err}`).toEqual('WorkerTimeoutError: timeout error'); - }); - }); - - describe('UnspecifiedWorkerError', function () { - it('should be function', () => { - expect(typeof UnspecifiedWorkerError).toBe('function'); - }); - - it('should have a name', function () { - const err = new UnspecifiedWorkerError('unspecified error'); - expect(err).toHaveProperty('name', 'UnspecifiedWorkerError'); - }); - - it('should take a jobId property', function () { - const err = new UnspecifiedWorkerError('unspecified error', { jobId: 'il7hl34rqlo8ro' }); - expect(err).toHaveProperty('jobId', 'il7hl34rqlo8ro'); - }); - - it('should be stringifyable', function () { - const err = new UnspecifiedWorkerError('unspecified error'); - expect(`${err}`).toEqual('UnspecifiedWorkerError: unspecified error'); - }); - }); -}); diff --git a/x-pack/plugins/reporting/server/lib/esqueue/index.js b/x-pack/plugins/reporting/server/lib/esqueue/index.js deleted file mode 100644 index 299254f211fc7..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/index.js +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EventEmitter } from 'events'; -import { Worker } from './worker'; -import { constants } from './constants'; -import { omit } from 'lodash'; - -export { events } from './constants/events'; - -export class Esqueue extends EventEmitter { - constructor(store, options = {}) { - super(); - this.store = store; // for updating jobs in ES - this.index = this.store.indexPrefix; // for polling for pending jobs - this.settings = { - interval: constants.DEFAULT_SETTING_INTERVAL, - timeout: constants.DEFAULT_SETTING_TIMEOUT, - dateSeparator: constants.DEFAULT_SETTING_DATE_SEPARATOR, - ...omit(options, ['client']), - }; - this.client = options.client; - this._logger = options.logger || function () {}; - this._workers = []; - this._initTasks().catch((err) => this.emit(constants.EVENT_QUEUE_ERROR, err)); - } - - _initTasks() { - const initTasks = [this.client.callAsInternalUser('ping')]; - - return Promise.all(initTasks).catch((err) => { - this._logger(['initTasks', 'error'], err); - throw err; - }); - } - - registerWorker(type, workerFn, opts) { - const worker = new Worker(this, type, workerFn, { ...opts, logger: this._logger }); - this._workers.push(worker); - return worker; - } - - getWorkers() { - return this._workers.map((fn) => fn); - } - - destroy() { - const workers = this._workers.filter((worker) => worker.destroy()); - this._workers = workers; - } -} diff --git a/x-pack/plugins/reporting/server/lib/esqueue/index.test.js b/x-pack/plugins/reporting/server/lib/esqueue/index.test.js deleted file mode 100644 index d0bf4837e1666..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/index.test.js +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import events from 'events'; -import expect from '@kbn/expect'; -import sinon from 'sinon'; -import proxyquire from 'proxyquire'; -import { noop, times } from 'lodash'; -import { constants } from './constants'; -import { ClientMock } from './__fixtures__/legacy_elasticsearch'; -import { JobMock } from './__fixtures__/job'; -import { WorkerMock } from './__fixtures__/worker'; - -const { Esqueue } = proxyquire.noPreserveCache()('./index', { - './job': { Job: JobMock }, - './worker': { Worker: WorkerMock }, -}); - -// TODO: tests were not running and are not up to date -describe.skip('Esqueue class', function () { - let client; - - beforeEach(function () { - client = new ClientMock(); - }); - - it('should be an event emitter', function () { - const queue = new Esqueue('esqueue', { client }); - expect(queue).to.be.an(events.EventEmitter); - }); - - describe('Option validation', function () { - it('should throw without an index', function () { - const init = () => new Esqueue(); - expect(init).to.throwException(/must.+specify.+index/i); - }); - }); - - describe('Queue construction', function () { - it('should ping the ES server', function () { - const pingSpy = sinon.spy(client, 'callAsInternalUser').withArgs('ping'); - new Esqueue('esqueue', { client }); - sinon.assert.calledOnce(pingSpy); - }); - }); - - describe('Adding jobs', function () { - let indexName; - let jobType; - let payload; - let queue; - - beforeEach(function () { - indexName = 'esqueue-index'; - jobType = 'test-test'; - payload = { payload: true }; - queue = new Esqueue(indexName, { client }); - }); - - it('should throw with invalid dateSeparator setting', function () { - queue = new Esqueue(indexName, { client, dateSeparator: 'a' }); - const fn = () => queue.addJob(jobType, payload); - expect(fn).to.throwException(); - }); - - it('should pass queue instance, index name, type and payload', function () { - const job = queue.addJob(jobType, payload); - expect(job.getProp('queue')).to.equal(queue); - expect(job.getProp('index')).to.match(new RegExp(indexName)); - expect(job.getProp('jobType')).to.equal(jobType); - expect(job.getProp('payload')).to.equal(payload); - }); - - it('should pass default settings', function () { - const job = queue.addJob(jobType, payload); - const options = job.getProp('options'); - expect(options).to.have.property('timeout', constants.DEFAULT_SETTING_TIMEOUT); - }); - - it('should pass queue index settings', function () { - const indexSettings = { - index: { - number_of_shards: 1, - }, - }; - - queue = new Esqueue(indexName, { client, indexSettings }); - const job = queue.addJob(jobType, payload); - expect(job.getProp('options')).to.have.property('indexSettings', indexSettings); - }); - - it('should pass headers from options', function () { - const options = { - headers: { - authorization: 'Basic cXdlcnR5', - }, - }; - const job = queue.addJob(jobType, payload, options); - expect(job.getProp('options')).to.have.property('headers', options.headers); - }); - }); - - describe('Registering workers', function () { - let queue; - - beforeEach(function () { - queue = new Esqueue('esqueue', { client }); - }); - - it('should keep track of workers', function () { - expect(queue.getWorkers()).to.eql([]); - expect(queue.getWorkers()).to.have.length(0); - - queue.registerWorker('test', noop); - queue.registerWorker('test', noop); - queue.registerWorker('test2', noop); - expect(queue.getWorkers()).to.have.length(3); - }); - - it('should pass instance of queue, type, and worker function', function () { - const workerType = 'test-worker'; - const workerFn = () => true; - - const worker = queue.registerWorker(workerType, workerFn); - expect(worker.getProp('queue')).to.equal(queue); - expect(worker.getProp('type')).to.equal(workerType); - expect(worker.getProp('workerFn')).to.equal(workerFn); - }); - - it('should pass worker options', function () { - const workerOptions = { - size: 12, - }; - - queue = new Esqueue('esqueue', { client }); - const worker = queue.registerWorker('type', noop, workerOptions); - const options = worker.getProp('options'); - expect(options.size).to.equal(workerOptions.size); - }); - }); - - describe('Destroy', function () { - it('should destroy workers', function () { - const queue = new Esqueue('esqueue', { client }); - const stubs = times(3, () => { - return { destroy: sinon.stub() }; - }); - stubs.forEach((stub) => queue._workers.push(stub)); - expect(queue.getWorkers()).to.have.length(3); - - queue.destroy(); - stubs.forEach((stub) => sinon.assert.calledOnce(stub.destroy)); - expect(queue.getWorkers()).to.have.length(0); - }); - }); -}); diff --git a/x-pack/plugins/reporting/server/lib/esqueue/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/worker.js deleted file mode 100644 index ec42f5d2fc316..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/worker.js +++ /dev/null @@ -1,444 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import events from 'events'; -import moment from 'moment'; -import Puid from 'puid'; -import { CancellationToken, Poller } from '../../../common'; -import { constants } from './constants'; -import { UnspecifiedWorkerError, WorkerTimeoutError } from './helpers/errors'; - -const puid = new Puid(); - -export function formatJobObject(job) { - return { - index: job._index, - id: job._id, - }; -} - -export function getUpdatedDocPath(response) { - const { _index: ind, _id: id } = response; - return `/${ind}/${id}`; -} - -const MAX_PARTIAL_ERROR_LENGTH = 1000; // 1000 of beginning, 1000 of end -const ERROR_PARTIAL_SEPARATOR = '...'; -const MAX_ERROR_LENGTH = MAX_PARTIAL_ERROR_LENGTH * 2 + ERROR_PARTIAL_SEPARATOR.length; - -function getLogger(opts, id, logLevel) { - return (msg, err) => { - /* - * This does not get the logger instance from queue.registerWorker in the createWorker function. - * The logger instance in the Equeue lib comes from createTaggedLogger, so logLevel tags are passed differently - */ - const logger = opts.logger || function () {}; - const message = `${id} - ${msg}`; - const tags = [logLevel]; - - if (err) { - // The error message string could be very long if it contains the request - // body of a request that was too large for Elasticsearch. - // This takes a partial version of the error message without scanning - // every character of the string, which would block Node. - const errString = `${message}: ${err.stack ? err.stack : err}`; - const errLength = errString.length; - const subStr = String.prototype.substring.bind(errString); - if (errLength > MAX_ERROR_LENGTH) { - const partialError = - subStr(0, MAX_PARTIAL_ERROR_LENGTH) + - ERROR_PARTIAL_SEPARATOR + - subStr(errLength - MAX_PARTIAL_ERROR_LENGTH); - - logger(partialError, tags); - logger( - `A partial version of the entire error message was logged. ` + - `The entire error message length is: ${errLength} characters.`, - tags - ); - } else { - logger(errString, tags); - } - return; - } - - logger(message, tags); - }; -} - -export class Worker extends events.EventEmitter { - constructor(queue, type, workerFn, opts) { - if (typeof type !== 'string') throw new Error('type must be a string'); - if (typeof workerFn !== 'function') throw new Error('workerFn must be a function'); - if (typeof opts !== 'object') throw new Error('opts must be an object'); - if (typeof opts.interval !== 'number') throw new Error('opts.interval must be a number'); - if (typeof opts.intervalErrorMultiplier !== 'number') - throw new Error('opts.intervalErrorMultiplier must be a number'); - - super(); - - this.id = puid.generate(); - this.kibanaId = opts.kibanaId; - this.kibanaName = opts.kibanaName; - this.queue = queue; - this._client = this.queue.client; - this.jobtype = type; - this.workerFn = workerFn; - - this.debug = getLogger(opts, this.id, 'debug'); - this.warn = getLogger(opts, this.id, 'warning'); - this.error = getLogger(opts, this.id, 'error'); - this.info = getLogger(opts, this.id, 'info'); - - this._running = true; - this.debug(`Created worker for ${this.jobtype} jobs`); - - this._poller = new Poller({ - functionToPoll: () => { - return this._processPendingJobs(); - }, - pollFrequencyInMillis: opts.interval, - trailing: true, - continuePollingOnError: true, - pollFrequencyErrorMultiplier: opts.intervalErrorMultiplier, - }); - this._startJobPolling(); - } - - destroy() { - this._running = false; - this._stopJobPolling(); - } - - toJSON() { - return { - id: this.id, - index: this.queue.index, - jobType: this.jobType, - }; - } - - emit(name, ...args) { - super.emit(name, ...args); - this.queue.emit(name, ...args); - } - - _formatErrorParams(err, job) { - const response = { - error: err, - worker: this.toJSON(), - }; - - if (job) response.job = formatJobObject(job); - return response; - } - - _claimJob(job) { - const m = moment(); - const startTime = m.toISOString(); - const expirationTime = m.add(job._source.timeout).toISOString(); - const attempts = job._source.attempts + 1; - - if (attempts > job._source.max_attempts) { - const msg = !job._source.output - ? `Max attempts reached (${job._source.max_attempts})` - : false; - return this._failJob(job, msg).then(() => false); - } - - const doc = { - attempts: attempts, - started_at: startTime, - process_expiration: expirationTime, - status: constants.JOB_STATUS_PROCESSING, - kibana_id: this.kibanaId, - kibana_name: this.kibanaName, - }; - - return this.queue.store.setReportClaimed(job, doc).then((response) => { - this.info(`Job marked as claimed: ${getUpdatedDocPath(response)}`); - const updatedJob = { - ...job, - ...response, - }; - updatedJob._source = { - ...job._source, - ...doc, - }; - return updatedJob; - }); - } - - _failJob(job, output = false) { - this.warn(`Failing job ${job._id}`); - - const completedTime = moment().toISOString(); - const docOutput = this._formatOutput(output); - const doc = { - status: constants.JOB_STATUS_FAILED, - completed_at: completedTime, - output: docOutput, - }; - - this.emit(constants.EVENT_WORKER_JOB_FAIL, { - job: formatJobObject(job), - worker: this.toJSON(), - output: docOutput, - }); - - return this.queue.store - .setReportFailed(job, doc) - .then((response) => { - this.info(`Job marked as failed: ${getUpdatedDocPath(response)}`); - }) - .catch((err) => { - if (err.statusCode === 409) return true; - this.error(`_failJob failed to update job ${job._id}`, err); - this.emit(constants.EVENT_WORKER_FAIL_UPDATE_ERROR, this._formatErrorParams(err, job)); - return false; - }); - } - - _formatOutput(output) { - const unknownMime = false; - const defaultOutput = null; - const docOutput = {}; - - if (typeof output === 'object' && output.content) { - docOutput.content = output.content; - docOutput.content_type = output.content_type || unknownMime; - docOutput.max_size_reached = output.max_size_reached; - docOutput.csv_contains_formulas = output.csv_contains_formulas; - docOutput.size = output.size; - docOutput.warnings = - output.warnings && output.warnings.length > 0 ? output.warnings : undefined; - } else { - docOutput.content = output || defaultOutput; - docOutput.content_type = unknownMime; - } - - return docOutput; - } - - _performJob(job) { - this.info(`Starting job`); - - const workerOutput = new Promise((resolve, reject) => { - // run the worker's workerFn - let isResolved = false; - const cancellationToken = new CancellationToken(); - const jobSource = job._source; - - Promise.resolve(this.workerFn.call(null, job, jobSource.payload, cancellationToken)) - .then((res) => { - // job execution was successful - if (res && res.warnings && res.warnings.length > 0) { - this.warn(`Job execution completed with warnings`); - } else { - this.info(`Job execution completed successfully`); - } - - isResolved = true; - resolve(res); - }) - .catch((err) => { - isResolved = true; - reject(err); - }); - - // fail if workerFn doesn't finish before timeout - const { timeout } = jobSource; - setTimeout(() => { - if (isResolved) return; - - cancellationToken.cancel(); - this.warn(`Timeout processing job ${job._id}`); - reject( - new WorkerTimeoutError(`Worker timed out, timeout = ${timeout}`, { - jobId: job._id, - timeout, - }) - ); - }, timeout); - }); - - return workerOutput.then( - (output) => { - const completedTime = moment().toISOString(); - const docOutput = this._formatOutput(output); - - const status = - output && output.warnings && output.warnings.length > 0 - ? constants.JOB_STATUS_WARNINGS - : constants.JOB_STATUS_COMPLETED; - const doc = { - status, - completed_at: completedTime, - output: docOutput, - }; - - return this.queue.store - .setReportCompleted(job, doc) - .then((response) => { - const eventOutput = { - job: formatJobObject(job), - output: docOutput, - }; - this.emit(constants.EVENT_WORKER_COMPLETE, eventOutput); - - this.info(`Job data saved successfully: ${getUpdatedDocPath(response)}`); - }) - .catch((err) => { - if (err.statusCode === 409) return false; - this.error(`Failure saving job output ${job._id}`, err); - this.emit(constants.EVENT_WORKER_JOB_UPDATE_ERROR, this._formatErrorParams(err, job)); - return this._failJob(job, err.message ? err.message : false); - }); - }, - (jobErr) => { - if (!jobErr) { - jobErr = new UnspecifiedWorkerError('Unspecified worker error', { - jobId: job._id, - }); - } - - // job execution failed - if (jobErr.name === 'WorkerTimeoutError') { - this.warn(`Timeout on job ${job._id}`); - this.emit(constants.EVENT_WORKER_JOB_TIMEOUT, this._formatErrorParams(jobErr, job)); - return; - - // append the jobId to the error - } else { - try { - Object.assign(jobErr, { jobId: job._id }); - } catch (e) { - // do nothing if jobId can not be appended - } - } - - this.error(`Failure occurred on job ${job._id}`, jobErr); - this.emit(constants.EVENT_WORKER_JOB_EXECUTION_ERROR, this._formatErrorParams(jobErr, job)); - return this._failJob(job, jobErr.toString ? jobErr.toString() : false); - } - ); - } - - _startJobPolling() { - if (!this._running) { - return; - } - - this._poller.start(); - } - - _stopJobPolling() { - this._poller.stop(); - } - - _processPendingJobs() { - return this._getPendingJobs().then((jobs) => { - return this._claimPendingJobs(jobs); - }); - } - - _claimPendingJobs(jobs) { - if (!jobs || jobs.length === 0) return; - - let claimed = false; - - // claim a single job, stopping after first successful claim - return jobs - .reduce((chain, job) => { - return chain.then((claimedJob) => { - // short-circuit the promise chain if a job has been claimed - if (claimed) return claimedJob; - - return this._claimJob(job) - .then((claimResult) => { - claimed = true; - return claimResult; - }) - .catch((err) => { - if (err.statusCode === 409) { - this.warn( - `_claimPendingJobs encountered a version conflict on updating pending job ${job._id}`, - err - ); - return; // continue reducing and looking for a different job to claim - } - this.emit(constants.EVENT_WORKER_JOB_CLAIM_ERROR, this._formatErrorParams(err, job)); - return Promise.reject(err); - }); - }); - }, Promise.resolve()) - .then((claimedJob) => { - if (!claimedJob) { - this.debug(`Found no claimable jobs out of ${jobs.length} total`); - return; - } - return this._performJob(claimedJob); - }) - .catch((err) => { - this.error('Error claiming jobs', err); - return Promise.reject(err); - }); - } - - _getPendingJobs() { - const nowTime = moment().toISOString(); - const query = { - seq_no_primary_term: true, - _source: { - excludes: ['output.content'], - }, - query: { - bool: { - filter: { - bool: { - minimum_should_match: 1, - should: [ - { term: { status: 'pending' } }, - { - bool: { - must: [ - { term: { status: 'processing' } }, - { range: { process_expiration: { lte: nowTime } } }, - ], - }, - }, - ], - }, - }, - }, - }, - sort: [{ priority: { order: 'asc' } }, { created_at: { order: 'asc' } }], - size: constants.DEFAULT_WORKER_CHECK_SIZE, - }; - - return this._client - .callAsInternalUser('search', { - index: `${this.queue.index}-*`, - body: query, - }) - .then((results) => { - const jobs = results.hits.hits; - if (jobs.length > 0) { - this.debug(`${jobs.length} outstanding jobs returned`); - } - return jobs; - }) - .catch((err) => { - // ignore missing indices errors - if (err && err.status === 404) return []; - - this.error('job querying failed', err); - this.emit(constants.EVENT_WORKER_JOB_SEARCH_ERROR, this._formatErrorParams(err)); - throw err; - }); - } -} diff --git a/x-pack/plugins/reporting/server/lib/esqueue/worker.test.js b/x-pack/plugins/reporting/server/lib/esqueue/worker.test.js deleted file mode 100644 index 75a3a59e0f1c8..0000000000000 --- a/x-pack/plugins/reporting/server/lib/esqueue/worker.test.js +++ /dev/null @@ -1,1120 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import expect from '@kbn/expect'; -import sinon from 'sinon'; -import moment from 'moment'; -import { noop, random, get, find, identity } from 'lodash'; -import { ClientMock } from './__fixtures__/legacy_elasticsearch'; -import { QueueMock } from './__fixtures__/queue'; -import { formatJobObject, getUpdatedDocPath, Worker } from './worker'; -import { constants } from './constants'; - -const anchor = '2016-04-02T01:02:03.456'; // saturday -const defaults = { - timeout: 10000, - size: 1, - unknownMime: false, - contentBody: null, -}; - -const defaultWorkerOptions = { - interval: 3000, - intervalErrorMultiplier: 10, -}; - -// TODO: tests were not running and are not up to date -describe.skip('Worker class', function () { - // some of these tests might be a little slow, give them a little extra time - jest.setTimeout(10000); - - let anchorMoment; - let clock; - let client; - let mockQueue; - let worker; - let worker2; - - // Allowing the Poller to poll requires intimate knowledge of the inner workings of the Poller. - // We have to ensure that the Promises internal to the `_poll` method are resolved to queue up - // the next setTimeout before incrementing the clock. - const allowPoll = async (interval) => { - clock.tick(interval); - await Promise.resolve(); - await Promise.resolve(); - }; - - beforeEach(function () { - client = new ClientMock(); - mockQueue = new QueueMock(); - mockQueue.setClient(client); - }); - - afterEach(function () { - [worker, worker2].forEach((actualWorker) => { - if (actualWorker) { - actualWorker.destroy(); - } - }); - }); - - describe('invalid construction', function () { - it('should throw without a type', function () { - const init = () => new Worker(mockQueue); - expect(init).to.throwException(/type.+string/i); - }); - - it('should throw without an invalid type', function () { - const init = () => new Worker(mockQueue, { string: false }); - expect(init).to.throwException(/type.+string/i); - }); - - it('should throw without a workerFn', function () { - const init = () => new Worker(mockQueue, 'test'); - expect(init).to.throwException(/workerFn.+function/i); - }); - - it('should throw with an invalid workerFn', function () { - const init = () => new Worker(mockQueue, 'test', { function: false }); - expect(init).to.throwException(/workerFn.+function/i); - }); - - it('should throw without an opts', function () { - const init = () => new Worker(mockQueue, 'test', noop); - expect(init).to.throwException(/opts.+object/i); - }); - - it('should throw with an invalid opts.interval', function () { - const init = () => new Worker(mockQueue, 'test', noop, {}); - expect(init).to.throwException(/opts\.interval.+number/i); - }); - - it('should throw with an invalid opts.intervalErrorMultiplier', function () { - const init = () => new Worker(mockQueue, 'test', noop, { interval: 1 }); - expect(init).to.throwException(/opts\.intervalErrorMultiplier.+number/i); - }); - }); - - describe('construction', function () { - it('should assign internal properties', function () { - const jobtype = 'testjob'; - const workerFn = noop; - worker = new Worker(mockQueue, jobtype, workerFn, defaultWorkerOptions); - expect(worker).to.have.property('id'); - expect(worker).to.have.property('queue', mockQueue); - expect(worker).to.have.property('jobtype', jobtype); - expect(worker).to.have.property('workerFn', workerFn); - }); - - it('should have a unique ID', function () { - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - expect(worker.id).to.be.a('string'); - - worker2 = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - expect(worker2.id).to.be.a('string'); - - expect(worker.id).to.not.equal(worker2.id); - }); - }); - - describe('event emitting', function () { - beforeEach(function () { - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - }); - - it('should trigger events on the queue instance', function (done) { - const eventName = 'test event'; - const payload1 = { - test: true, - deep: { object: 'ok' }, - }; - const payload2 = 'two'; - const payload3 = new Error('test error'); - - mockQueue.on(eventName, (...args) => { - try { - expect(args[0]).to.equal(payload1); - expect(args[1]).to.equal(payload2); - expect(args[2]).to.equal(payload3); - done(); - } catch (e) { - done(e); - } - }); - - worker.emit(eventName, payload1, payload2, payload3); - }); - }); - - describe('output formatting', function () { - let f; - - beforeEach(function () { - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - f = (output) => worker._formatOutput(output); - }); - - it('should handle primitives', function () { - const primitives = ['test', true, 1234, { one: 1 }, [5, 6, 7, 8]]; - - primitives.forEach((val) => { - expect(f(val)).to.have.property('content_type', defaults.unknownMime); - expect(f(val)).to.have.property('content', val); - }); - }); - - it('should accept content object without type', function () { - const output = { - content: 'test output', - }; - - expect(f(output)).to.have.property('content_type', defaults.unknownMime); - expect(f(output)).to.have.property('content', output.content); - }); - - it('should accept a content type', function () { - const output = { - content_type: 'test type', - content: 'test output', - }; - - expect(f(output)).to.have.property('content_type', output.content_type); - expect(f(output)).to.have.property('content', output.content); - }); - - it('should work with no input', function () { - expect(f()).to.have.property('content_type', defaults.unknownMime); - expect(f()).to.have.property('content', defaults.contentBody); - }); - }); - - describe('polling for jobs', function () { - beforeEach(() => { - anchorMoment = moment(anchor); - clock = sinon.useFakeTimers(anchorMoment.valueOf()); - }); - - afterEach(() => { - clock.restore(); - }); - - it('should start polling for jobs after interval', async function () { - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - const processPendingJobsStub = sinon - .stub(worker, '_processPendingJobs') - .callsFake(() => Promise.resolve()); - sinon.assert.notCalled(processPendingJobsStub); - await allowPoll(defaultWorkerOptions.interval); - sinon.assert.calledOnce(processPendingJobsStub); - }); - - it('should use interval option to control polling', async function () { - const interval = 567; - worker = new Worker(mockQueue, 'test', noop, { ...defaultWorkerOptions, interval }); - const processPendingJobsStub = sinon - .stub(worker, '_processPendingJobs') - .callsFake(() => Promise.resolve()); - - sinon.assert.notCalled(processPendingJobsStub); - await allowPoll(interval); - sinon.assert.calledOnce(processPendingJobsStub); - }); - - it('should not poll once destroyed', async function () { - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - - const processPendingJobsStub = sinon - .stub(worker, '_processPendingJobs') - .callsFake(() => Promise.resolve()); - - // move the clock a couple times, test for searches each time - sinon.assert.notCalled(processPendingJobsStub); - await allowPoll(defaultWorkerOptions.interval); - sinon.assert.calledOnce(processPendingJobsStub); - await allowPoll(defaultWorkerOptions.interval); - sinon.assert.calledTwice(processPendingJobsStub); - - // destroy the worker, move the clock, make sure another search doesn't happen - worker.destroy(); - await allowPoll(defaultWorkerOptions.interval); - sinon.assert.calledTwice(processPendingJobsStub); - - // manually call job poller, move the clock, make sure another search doesn't happen - worker._startJobPolling(); - await allowPoll(defaultWorkerOptions.interval); - sinon.assert.calledTwice(processPendingJobsStub); - }); - - it('should use error multiplier when processPendingJobs rejects the Promise', async function () { - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - - const processPendingJobsStub = sinon - .stub(worker, '_processPendingJobs') - .rejects(new Error('test error')); - - await allowPoll(defaultWorkerOptions.interval); - expect(processPendingJobsStub.callCount).to.be(1); - await allowPoll(defaultWorkerOptions.interval); - expect(processPendingJobsStub.callCount).to.be(1); - await allowPoll(defaultWorkerOptions.interval * defaultWorkerOptions.intervalErrorMultiplier); - expect(processPendingJobsStub.callCount).to.be(2); - }); - - it('should not use error multiplier when processPendingJobs resolved the Promise', async function () { - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - - const processPendingJobsStub = sinon - .stub(worker, '_processPendingJobs') - .callsFake(() => Promise.resolve()); - - await allowPoll(defaultWorkerOptions.interval); - expect(processPendingJobsStub.callCount).to.be(1); - await allowPoll(defaultWorkerOptions.interval); - expect(processPendingJobsStub.callCount).to.be(2); - }); - }); - - describe('query for pending jobs', function () { - let searchStub; - - function getSearchParams(jobtype = 'test', params = {}) { - worker = new Worker(mockQueue, jobtype, noop, { ...defaultWorkerOptions, ...params }); - worker._getPendingJobs(); - return searchStub.firstCall.args[1]; - } - - describe('error handling', function () { - it('should pass search errors', function (done) { - searchStub = sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('search') - .callsFake(() => Promise.reject()); - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - worker - ._getPendingJobs() - .then(() => done(new Error('should not resolve'))) - .catch(() => { - done(); - }); - }); - - describe('missing index', function () { - it('should swallow error', function (done) { - searchStub = sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('search') - .callsFake(() => Promise.reject({ status: 404 })); - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - worker - ._getPendingJobs() - .then(() => { - done(); - }) - .catch(() => done(new Error('should not reject'))); - }); - - it('should return an empty array', function (done) { - searchStub = sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('search') - .callsFake(() => Promise.reject({ status: 404 })); - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - worker - ._getPendingJobs() - .then((res) => { - try { - expect(res).to.be.an(Array); - expect(res).to.have.length(0); - done(); - } catch (e) { - done(e); - } - }) - .catch(() => done(new Error('should not reject'))); - }); - }); - }); - - describe('query body', function () { - const conditionPath = 'query.bool.filter.bool'; - const jobtype = 'test_jobtype'; - - beforeEach(() => { - searchStub = sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('search') - .callsFake(() => Promise.resolve({ hits: { hits: [] } })); - anchorMoment = moment(anchor); - clock = sinon.useFakeTimers(anchorMoment.valueOf()); - }); - - afterEach(() => { - clock.restore(); - }); - - it('should query with seq_no_primary_term', function () { - const { body } = getSearchParams(jobtype); - expect(body).to.have.property('seq_no_primary_term', true); - }); - - it('should filter unwanted source data', function () { - const excludedFields = ['output.content']; - const { body } = getSearchParams(jobtype); - expect(body).to.have.property('_source'); - expect(body._source).to.eql({ excludes: excludedFields }); - }); - - it('should search for pending or expired jobs', function () { - const { body } = getSearchParams(jobtype); - const conditions = get(body, conditionPath); - expect(conditions).to.have.property('should'); - - // this works because we are stopping the clock, so all times match - const nowTime = moment().toISOString(); - const pending = { term: { status: 'pending' } }; - const expired = { - bool: { - must: [ - { term: { status: 'processing' } }, - { range: { process_expiration: { lte: nowTime } } }, - ], - }, - }; - - const pendingMatch = find(conditions.should, pending); - expect(pendingMatch).to.not.be(undefined); - - const expiredMatch = find(conditions.should, expired); - expect(expiredMatch).to.not.be(undefined); - }); - - it('specify that there should be at least one match', function () { - const { body } = getSearchParams(jobtype); - const conditions = get(body, conditionPath); - expect(conditions).to.have.property('minimum_should_match', 1); - }); - - it('should use default size', function () { - const { body } = getSearchParams(jobtype); - expect(body).to.have.property('size', defaults.size); - }); - }); - }); - - describe('claiming a job', function () { - let params; - let job; - let updateSpy; - - beforeEach(function () { - anchorMoment = moment(anchor); - clock = sinon.useFakeTimers(anchorMoment.valueOf()); - - params = { - index: 'myIndex', - type: 'test', - id: 12345, - }; - return mockQueue.client.callAsInternalUser('get', params).then((jobDoc) => { - job = jobDoc; - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - updateSpy = sinon.spy(mockQueue.client, 'callAsInternalUser').withArgs('update'); - }); - }); - - afterEach(() => { - clock.restore(); - }); - - it('should use seqNo and primaryTerm on update', function () { - worker._claimJob(job); - const query = updateSpy.firstCall.args[1]; - expect(query).to.have.property('index', job._index); - expect(query).to.have.property('id', job._id); - expect(query).to.have.property('if_seq_no', job._seq_no); - expect(query).to.have.property('if_primary_term', job._primary_term); - }); - - it('should increment the job attempts', function () { - worker._claimJob(job); - const doc = updateSpy.firstCall.args[1].body.doc; - expect(doc).to.have.property('attempts', job._source.attempts + 1); - }); - - it('should update the job status', function () { - worker._claimJob(job); - const doc = updateSpy.firstCall.args[1].body.doc; - expect(doc).to.have.property('status', constants.JOB_STATUS_PROCESSING); - }); - - it('should set job expiration time', function () { - worker._claimJob(job); - const doc = updateSpy.firstCall.args[1].body.doc; - const expiration = anchorMoment.add(defaults.timeout).toISOString(); - expect(doc).to.have.property('process_expiration', expiration); - }); - - it('should fail job if max_attempts are hit', function () { - const failSpy = sinon.spy(worker, '_failJob'); - job._source.attempts = job._source.max_attempts; - worker._claimJob(job); - sinon.assert.calledOnce(failSpy); - }); - - it('should append error message if no existing content', function () { - const failSpy = sinon.spy(worker, '_failJob'); - job._source.attempts = job._source.max_attempts; - expect(job._source.output).to.be(undefined); - worker._claimJob(job); - const msg = failSpy.firstCall.args[1]; - expect(msg).to.contain('Max attempts reached'); - expect(msg).to.contain(job._source.max_attempts); - }); - - it('should not append message if existing output', function () { - const failSpy = sinon.spy(worker, '_failJob'); - job._source.attempts = job._source.max_attempts; - job._source.output = 'i have some output'; - worker._claimJob(job); - const msg = failSpy.firstCall.args[1]; - expect(msg).to.equal(false); - }); - - it('should reject the promise on conflict errors', function () { - mockQueue.client.callAsInternalUser.restore(); - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('update') - .returns(Promise.reject({ statusCode: 409 })); - return worker._claimJob(job).catch((err) => { - expect(err).to.eql({ statusCode: 409 }); - }); - }); - - it('should reject the promise on other errors', function () { - mockQueue.client.callAsInternalUser.restore(); - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('update') - .returns(Promise.reject({ statusCode: 401 })); - return worker._claimJob(job).catch((err) => { - expect(err).to.eql({ statusCode: 401 }); - }); - }); - }); - - describe('find a pending job to claim', function () { - const getMockJobs = (status = 'pending') => [ - { - _index: 'myIndex', - _id: 12345, - _seq_no: 3, - _primary_term: 3, - found: true, - _source: { - jobtype: 'jobtype', - created_by: false, - payload: { id: 'sample-job-1', now: 'Mon Apr 25 2016 14:13:04 GMT-0700 (MST)' }, - priority: 10, - timeout: 10000, - created_at: '2016-04-25T21:13:04.738Z', - attempts: 0, - max_attempts: 3, - status, - }, - }, - ]; - - beforeEach(function () { - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - }); - - afterEach(() => { - mockQueue.client.callAsInternalUser.restore(); - }); - - it('should emit for errors from claiming job', function (done) { - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('update') - .rejects({ statusCode: 401 }); - - worker.once(constants.EVENT_WORKER_JOB_CLAIM_ERROR, function (err) { - try { - expect(err).to.have.property('error'); - expect(err).to.have.property('job'); - expect(err).to.have.property('worker'); - expect(err.error).to.have.property('statusCode', 401); - done(); - } catch (e) { - done(e); - } - }); - - worker._claimPendingJobs(getMockJobs()).catch(() => {}); - }); - - it('should reject the promise if an error claiming the job', function () { - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('update') - .rejects({ statusCode: 409 }); - return worker._claimPendingJobs(getMockJobs()).catch((err) => { - expect(err).to.eql({ statusCode: 409 }); - }); - }); - - it('should get the pending job', function () { - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('update') - .resolves({ test: 'cool' }); - sinon.stub(worker, '_performJob').callsFake(identity); - return worker._claimPendingJobs(getMockJobs()).then((claimedJob) => { - expect(claimedJob._index).to.be('myIndex'); - expect(claimedJob._source.jobtype).to.be('jobtype'); - expect(claimedJob._source.status).to.be('processing'); - expect(claimedJob.test).to.be('cool'); - worker._performJob.restore(); - }); - }); - }); - - describe('failing a job', function () { - let job; - let updateSpy; - - beforeEach(function () { - anchorMoment = moment(anchor); - clock = sinon.useFakeTimers(anchorMoment.valueOf()); - - return mockQueue.client.callAsInternalUser('get').then((jobDoc) => { - job = jobDoc; - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - updateSpy = sinon.spy(mockQueue.client, 'callAsInternalUser').withArgs('update'); - }); - }); - - afterEach(() => { - clock.restore(); - }); - - it('should use _seq_no and _primary_term on update', function () { - worker._failJob(job); - const query = updateSpy.firstCall.args[1]; - expect(query).to.have.property('index', job._index); - expect(query).to.have.property('id', job._id); - expect(query).to.have.property('if_seq_no', job._seq_no); - expect(query).to.have.property('if_primary_term', job._primary_term); - }); - - it('should set status to failed', function () { - worker._failJob(job); - const doc = updateSpy.firstCall.args[1].body.doc; - expect(doc).to.have.property('status', constants.JOB_STATUS_FAILED); - }); - - it('should append error message if supplied', function () { - const msg = 'test message'; - worker._failJob(job, msg); - const doc = updateSpy.firstCall.args[1].body.doc; - expect(doc).to.have.property('output'); - expect(doc.output).to.have.property('content', msg); - }); - - it('should return true on conflict errors', function () { - mockQueue.client.callAsInternalUser.restore(); - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('update') - .rejects({ statusCode: 409 }); - return worker._failJob(job).then((res) => expect(res).to.equal(true)); - }); - - it('should return false on other document update errors', function () { - mockQueue.client.callAsInternalUser.restore(); - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('update') - .rejects({ statusCode: 401 }); - return worker._failJob(job).then((res) => expect(res).to.equal(false)); - }); - - it('should set completed time and status to failure', function () { - const startTime = moment().valueOf(); - const msg = 'test message'; - clock.tick(100); - - worker._failJob(job, msg); - const doc = updateSpy.firstCall.args[1].body.doc; - expect(doc).to.have.property('output'); - expect(doc).to.have.property('status', constants.JOB_STATUS_FAILED); - expect(doc).to.have.property('completed_at'); - const completedTimestamp = moment(doc.completed_at).valueOf(); - expect(completedTimestamp).to.be.greaterThan(startTime); - }); - - it('should emit worker failure event', function (done) { - worker.on(constants.EVENT_WORKER_JOB_FAIL, (err) => { - try { - expect(err).to.have.property('output'); - expect(err).to.have.property('job'); - expect(err).to.have.property('worker'); - done(); - } catch (e) { - done(e); - } - }); - - return worker._failJob(job); - }); - - it('should emit on other document update errors', function (done) { - mockQueue.client.callAsInternalUser.restore(); - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('update') - .rejects({ statusCode: 401 }); - - worker.on(constants.EVENT_WORKER_FAIL_UPDATE_ERROR, function (err) { - try { - expect(err).to.have.property('error'); - expect(err).to.have.property('job'); - expect(err).to.have.property('worker'); - expect(err.error).to.have.property('statusCode', 401); - done(); - } catch (e) { - done(e); - } - }); - worker._failJob(job); - }); - }); - - describe('performing a job', function () { - let job; - let payload; - let updateSpy; - - beforeEach(function () { - payload = { - value: random(0, 100, true), - }; - - return mockQueue.client.callAsInternalUser('get', {}, { payload }).then((jobDoc) => { - job = jobDoc; - updateSpy = sinon.spy(mockQueue.client, 'callAsInternalUser').withArgs('update'); - }); - }); - - describe('worker success', function () { - it('should call the workerFn with the payload', function (done) { - const workerFn = function (jobPayload) { - expect(jobPayload).to.eql(payload); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - - worker._performJob(job).then(() => done()); - }); - - it('should update the job with the workerFn output', function () { - const workerFn = function (job, jobPayload) { - expect(jobPayload).to.eql(payload); - return payload; - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - - return worker._performJob(job).then(() => { - sinon.assert.calledOnce(updateSpy); - const query = updateSpy.firstCall.args[1]; - - expect(query).to.have.property('index', job._index); - expect(query).to.have.property('id', job._id); - expect(query).to.have.property('if_seq_no', job._seq_no); - expect(query).to.have.property('if_primary_term', job._primary_term); - expect(query.body.doc).to.have.property('output'); - expect(query.body.doc.output).to.have.property('content_type', false); - expect(query.body.doc.output).to.have.property('content', payload); - }); - }); - - it('should update the job status and completed time', function () { - const startTime = moment().valueOf(); - const workerFn = function (job, jobPayload) { - expect(jobPayload).to.eql(payload); - return new Promise(function (resolve) { - setTimeout(() => resolve(payload), 10); - }); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - - return worker._performJob(job).then(() => { - sinon.assert.calledOnce(updateSpy); - const doc = updateSpy.firstCall.args[1].body.doc; - expect(doc).to.have.property('status', constants.JOB_STATUS_COMPLETED); - expect(doc).to.have.property('completed_at'); - const completedTimestamp = moment(doc.completed_at).valueOf(); - expect(completedTimestamp).to.be.greaterThan(startTime); - }); - }); - - it('handle warnings in the output by reflecting a warning status', () => { - const workerFn = () => { - return Promise.resolve({ - ...payload, - warnings: [`Don't run with scissors!`], - }); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - - return worker - ._performJob({ - test: true, - ...job, - }) - .then(() => { - sinon.assert.calledOnce(updateSpy); - const doc = updateSpy.firstCall.args[1].body.doc; - expect(doc).to.have.property('status', constants.JOB_STATUS_WARNINGS); - }); - }); - - it('should emit completion event', function (done) { - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - - worker.once(constants.EVENT_WORKER_COMPLETE, (workerJob) => { - try { - expect(workerJob).to.not.have.property('_source'); - - expect(workerJob).to.have.property('job'); - expect(workerJob.job).to.have.property('id'); - expect(workerJob.job).to.have.property('index'); - - expect(workerJob).to.have.property('output'); - expect(workerJob.output).to.have.property('content'); - expect(workerJob.output).to.have.property('content_type'); - - done(); - } catch (e) { - done(e); - } - }); - - worker._performJob(job); - }); - }); - - describe('worker failure', function () { - it('should append error output to job', function () { - const workerFn = function () { - throw new Error('test error'); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - const failStub = sinon.stub(worker, '_failJob'); - - return worker._performJob(job).then(() => { - sinon.assert.calledOnce(failStub); - sinon.assert.calledWith(failStub, job, 'Error: test error'); - }); - }); - - it('should handle async errors', function () { - const workerFn = function () { - return new Promise((resolve, reject) => { - reject(new Error('test error')); - }); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - const failStub = sinon.stub(worker, '_failJob'); - - return worker._performJob(job).then(() => { - sinon.assert.calledOnce(failStub); - sinon.assert.calledWith(failStub, job, 'Error: test error'); - }); - }); - - it('should handle rejecting with strings', function () { - const errorMessage = 'this is a string error'; - const workerFn = function () { - return new Promise((resolve, reject) => { - reject(errorMessage); - }); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - const failStub = sinon.stub(worker, '_failJob'); - - return worker._performJob(job).then(() => { - sinon.assert.calledOnce(failStub); - sinon.assert.calledWith(failStub, job, errorMessage); - }); - }); - - it('should handle empty rejection', function (done) { - const workerFn = function () { - return new Promise((resolve, reject) => { - reject(); - }); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - - worker.once(constants.EVENT_WORKER_JOB_EXECUTION_ERROR, (err) => { - try { - expect(err).to.have.property('error'); - expect(err).to.have.property('job'); - expect(err).to.have.property('worker'); - expect(err.error).to.have.property('name', 'UnspecifiedWorkerError'); - done(); - } catch (e) { - done(e); - } - }); - - worker._performJob(job); - }); - }); - }); - - describe('job failures', function () { - function getFailStub(workerWithFailure) { - return sinon.stub(workerWithFailure, '_failJob').resolves(); - } - - describe('saving output failure', () => { - it('should mark the job as failed if saving to ES fails', async () => { - const job = { - _id: 'shouldSucced', - _source: { - timeout: 1000, - payload: 'test', - }, - }; - - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('update') - .rejects({ statusCode: 413 }); - - const workerFn = function (jobPayload) { - return new Promise(function (resolve) { - setTimeout(() => resolve(jobPayload), 10); - }); - }; - const worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - const failStub = getFailStub(worker); - - await worker._performJob(job); - worker.destroy(); - - sinon.assert.called(failStub); - }); - }); - - describe('search failure', function () { - it('causes _processPendingJobs to reject the Promise', function () { - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('search') - .rejects(new Error('test error')); - worker = new Worker(mockQueue, 'test', noop, defaultWorkerOptions); - return worker._processPendingJobs().then( - () => { - expect().fail('expected rejected Promise'); - }, - (err) => { - expect(err).to.be.an(Error); - } - ); - }); - }); - - describe('timeout', function () { - let failStub; - let job; - let cancellationCallback; - - beforeEach(function () { - const timeout = 20; - cancellationCallback = function () {}; - - const workerFn = function (job, payload, cancellationToken) { - cancellationToken.on(cancellationCallback); - return new Promise(function (resolve) { - setTimeout(() => { - resolve(); - }, timeout * 2); - }); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - failStub = getFailStub(worker); - - job = { - _id: 'testTimeoutJob', - _source: { - timeout: timeout, - payload: 'test', - }, - }; - }); - - it('should not fail job', function () { - // fire of the job worker - return worker._performJob(job).then(() => { - sinon.assert.notCalled(failStub); - }); - }); - - it('should emit timeout if not completed in time', function (done) { - worker.once(constants.EVENT_WORKER_JOB_TIMEOUT, (err) => { - try { - expect(err).to.have.property('error'); - expect(err).to.have.property('job'); - expect(err).to.have.property('worker'); - expect(err.error).to.have.property('name', 'WorkerTimeoutError'); - done(); - } catch (e) { - done(e); - } - }); - - // fire of the job worker - worker._performJob(job); - }); - - it('should call cancellation token callback if not completed in time', function (done) { - let called = false; - - cancellationCallback = () => { - called = true; - }; - - worker.once(constants.EVENT_WORKER_JOB_TIMEOUT, () => { - try { - expect(called).to.be(true); - done(); - } catch (err) { - done(err); - } - }); - - // fire of the job worker - worker._performJob(job); - }); - }); - - describe('worker failure', function () { - let failStub; - - const timeout = 20; - const job = { - _id: 'testTimeoutJob', - _source: { - timeout: timeout, - payload: 'test', - }, - }; - - beforeEach(function () { - sinon - .stub(mockQueue.client, 'callAsInternalUser') - .withArgs('search') - .callsFake(() => Promise.resolve({ hits: { hits: [] } })); - }); - - describe('workerFn rejects promise', function () { - beforeEach(function () { - const workerFn = function () { - return new Promise(function (resolve, reject) { - setTimeout(() => { - reject(); - }, timeout / 2); - }); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - failStub = getFailStub(worker); - }); - - it('should fail the job', function () { - return worker._performJob(job).then(() => { - sinon.assert.calledOnce(failStub); - }); - }); - - it('should emit worker execution error', function (done) { - worker.on(constants.EVENT_WORKER_JOB_EXECUTION_ERROR, (err) => { - try { - expect(err).to.have.property('error'); - expect(err).to.have.property('job'); - expect(err).to.have.property('worker'); - done(); - } catch (e) { - done(e); - } - }); - - // fire of the job worker - worker._performJob(job); - }); - }); - - describe('workerFn throws error', function () { - beforeEach(function () { - const workerFn = function () { - throw new Error('test throw'); - }; - worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); - - failStub = getFailStub(worker); - }); - - it('should fail the job', function () { - return worker._performJob(job).then(() => { - sinon.assert.calledOnce(failStub); - }); - }); - - it('should emit worker execution error', function (done) { - worker.on(constants.EVENT_WORKER_JOB_EXECUTION_ERROR, (err) => { - try { - expect(err).to.have.property('error'); - expect(err).to.have.property('job'); - expect(err).to.have.property('worker'); - done(); - } catch (e) { - done(e); - } - }); - - // fire of the job worker - worker._performJob(job); - }); - }); - }); - }); -}); - -describe('Format Job Object', () => { - it('pulls index and ID', function () { - const jobMock = { - _index: 'foo', - _id: 'booId', - }; - expect(formatJobObject(jobMock)).eql({ - index: 'foo', - id: 'booId', - }); - }); -}); - -describe('Get Doc Path from ES Response', () => { - it('returns a formatted string after response of an update', function () { - const responseMock = { - _index: 'foo', - _id: 'booId', - }; - expect(getUpdatedDocPath(responseMock)).equal('/foo/booId'); - }); -}); diff --git a/x-pack/plugins/reporting/server/lib/index.ts b/x-pack/plugins/reporting/server/lib/index.ts index cd7cdf3a8fd0b..e66f72f88f8ea 100644 --- a/x-pack/plugins/reporting/server/lib/index.ts +++ b/x-pack/plugins/reporting/server/lib/index.ts @@ -6,9 +6,7 @@ */ export { checkLicense } from './check_license'; -export { createQueueFactory } from './create_queue'; export { cryptoFactory } from './crypto'; -export { enqueueJobFactory } from './enqueue_job'; export { ExportTypesRegistry, getExportTypesRegistry } from './export_types_registry'; export { LevelLogger } from './level_logger'; export { statuses } from './statuses'; diff --git a/x-pack/plugins/reporting/server/lib/store/mapping.ts b/x-pack/plugins/reporting/server/lib/store/mapping.ts index e663c7e23b146..c35540f3869a0 100644 --- a/x-pack/plugins/reporting/server/lib/store/mapping.ts +++ b/x-pack/plugins/reporting/server/lib/store/mapping.ts @@ -43,7 +43,7 @@ export const mapping = { browser_type: { type: 'keyword' }, jobtype: { type: 'keyword' }, payload: { type: 'object', enabled: false }, - priority: { type: 'byte' }, + priority: { type: 'byte' }, // NOTE: this is unused, but older data may have a mapping for this field timeout: { type: 'long' }, process_expiration: { type: 'date' }, created_by: { type: 'keyword' }, // `null` if security is disabled diff --git a/x-pack/plugins/reporting/server/lib/store/report.test.ts b/x-pack/plugins/reporting/server/lib/store/report.test.ts index 4c5cd755f71c4..23d766f2190f6 100644 --- a/x-pack/plugins/reporting/server/lib/store/report.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/report.test.ts @@ -36,6 +36,14 @@ describe('Class Report', () => { timeout: 30000, }, }); + expect(report.toReportTaskJSON()).toMatchObject({ + attempts: 0, + created_by: 'created_by_test_string', + index: '.reporting-test-index-12345', + jobtype: 'test-report', + meta: { objectType: 'test' }, + payload: { headers: 'payload_test_field', objectType: 'testOt' }, + }); expect(report.toApiJSON()).toMatchObject({ attempts: 0, browser_type: 'browser_type_test_string', @@ -89,6 +97,15 @@ describe('Class Report', () => { timeout: 30000, }, }); + expect(report.toReportTaskJSON()).toMatchObject({ + attempts: 0, + created_by: 'created_by_test_string', + id: '12342p9o387549o2345', + index: '.reporting-test-update', + jobtype: 'test-report', + meta: { objectType: 'stange' }, + payload: { objectType: 'testOt' }, + }); expect(report.toApiJSON()).toMatchObject({ attempts: 0, browser_type: 'browser_type_test_string', diff --git a/x-pack/plugins/reporting/server/lib/store/report.ts b/x-pack/plugins/reporting/server/lib/store/report.ts index 735ba274322cd..817028cab1a39 100644 --- a/x-pack/plugins/reporting/server/lib/store/report.ts +++ b/x-pack/plugins/reporting/server/lib/store/report.ts @@ -9,7 +9,16 @@ import moment from 'moment'; // @ts-ignore no module definition import Puid from 'puid'; import { JOB_STATUSES } from '../../../common/constants'; -import { ReportApiJSON, ReportDocumentHead, ReportSource } from '../../../common/types'; +import { + ReportApiJSON, + ReportDocument, + ReportDocumentHead, + ReportSource, +} from '../../../common/types'; +import { ReportTaskParams } from '../tasks'; + +export { ReportDocument }; +export { ReportApiJSON, ReportSource }; const puid = new Puid(); @@ -35,9 +44,10 @@ export class Report implements Partial { public readonly output?: ReportSource['output']; public readonly started_at?: ReportSource['started_at']; public readonly completed_at?: ReportSource['completed_at']; - public readonly process_expiration?: ReportSource['process_expiration']; public readonly timeout?: ReportSource['timeout']; + public process_expiration?: ReportSource['process_expiration']; + /* * Create an unsaved report * Index string is required @@ -101,10 +111,32 @@ export class Report implements Partial { attempts: this.attempts, started_at: this.started_at, completed_at: this.completed_at, + process_expiration: this.process_expiration, }, }; } + /* + * Parameters to save in a task instance + */ + toReportTaskJSON(): ReportTaskParams { + if (!this._index) { + throw new Error(`Task is missing the _index field!`); + } + + return { + id: this._id, + index: this._index, + jobtype: this.jobtype, + created_at: this.created_at, + created_by: this.created_by, + payload: this.payload, + meta: this.meta, + attempts: this.attempts, + max_attempts: this.max_attempts, + }; + } + /* * Data structure for API responses */ @@ -129,5 +161,3 @@ export class Report implements Partial { }; } } - -export { ReportApiJSON, ReportSource }; diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index 4e8e113fb0698..01d91f8bc2ac2 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -7,14 +7,14 @@ import sinon from 'sinon'; import { ElasticsearchServiceSetup } from 'src/core/server'; -import { ReportingConfig, ReportingCore } from '../..'; +import { ReportingConfig, ReportingCore } from '../../'; import { createMockConfig, createMockConfigSchema, createMockLevelLogger, createMockReportingCore, } from '../../test_helpers'; -import { Report } from './report'; +import { Report, ReportDocument } from './report'; import { ReportingStore } from './store'; describe('ReportingStore', () => { @@ -187,6 +187,66 @@ describe('ReportingStore', () => { }); }); + it('findReport gets a report from ES and returns a Report object', async () => { + // setup + const mockReport: ReportDocument = { + _id: '1234-foo-78', + _index: '.reporting-test-17409', + _primary_term: 'primary_term string', + _seq_no: 'seq_no string', + _source: { + kibana_name: 'test', + kibana_id: 'test123', + created_at: 'some time', + created_by: 'some security person', + jobtype: 'csv', + status: 'pending', + meta: { testMeta: 'meta' } as any, + payload: { testPayload: 'payload' } as any, + browser_type: 'browser type string', + attempts: 0, + max_attempts: 1, + timeout: 30000, + output: null, + }, + }; + callClusterStub.withArgs('get').resolves(mockReport); + const store = new ReportingStore(mockCore, mockLogger); + const report = new Report({ + ...mockReport, + ...mockReport._source, + }); + + expect(await store.findReportFromTask(report.toReportTaskJSON())).toMatchInlineSnapshot(` + Report { + "_id": "1234-foo-78", + "_index": ".reporting-test-17409", + "_primary_term": "primary_term string", + "_seq_no": "seq_no string", + "attempts": 0, + "browser_type": "browser type string", + "completed_at": undefined, + "created_at": "some time", + "created_by": "some security person", + "jobtype": "csv", + "kibana_id": undefined, + "kibana_name": undefined, + "max_attempts": 1, + "meta": Object { + "testMeta": "meta", + }, + "output": null, + "payload": Object { + "testPayload": "payload", + }, + "process_expiration": undefined, + "started_at": undefined, + "status": "pending", + "timeout": 30000, + } + `); + }); + it('setReportClaimed sets the status of a record to processing', async () => { const store = new ReportingStore(mockCore, mockLogger); const report = new Report({ @@ -222,6 +282,7 @@ describe('ReportingStore', () => { "if_primary_term": undefined, "if_seq_no": undefined, "index": ".reporting-test-index-12345", + "refresh": true, }, ] `); @@ -262,6 +323,7 @@ describe('ReportingStore', () => { "if_primary_term": undefined, "if_seq_no": undefined, "index": ".reporting-test-index-12345", + "refresh": true, }, ] `); @@ -302,6 +364,7 @@ describe('ReportingStore', () => { "if_primary_term": undefined, "if_seq_no": undefined, "index": ".reporting-test-index-12345", + "refresh": true, }, ] `); @@ -352,6 +415,7 @@ describe('ReportingStore', () => { "if_primary_term": undefined, "if_seq_no": undefined, "index": ".reporting-test-index-12345", + "refresh": true, }, ] `); diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index 4cde4b9d6e0fc..fdac471c26cb0 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -5,12 +5,29 @@ * 2.0. */ +import { SearchParams } from 'elasticsearch'; import { ElasticsearchServiceSetup } from 'src/core/server'; import { LevelLogger, statuses } from '../'; import { ReportingCore } from '../../'; +import { numberToDuration } from '../../../common/schema_utils'; +import { JobStatus } from '../../../common/types'; +import { ReportTaskParams } from '../tasks'; import { indexTimestamp } from './index_timestamp'; import { mapping } from './mapping'; -import { Report } from './report'; +import { Report, ReportDocument } from './report'; + +/* + * When searching for long-pending reports, we get a subset of fields + */ +export interface ReportRecordTimeout { + _id: string; + _index: string; + _source: { + status: JobStatus; + process_expiration?: string; + created_at?: string; + }; +} const checkReportIsEditable = (report: Report) => { if (!report._id || !report._index) { @@ -25,8 +42,9 @@ const checkReportIsEditable = (report: Report) => { * - interface for downloading the report */ export class ReportingStore { - private readonly indexPrefix: string; - private readonly indexInterval: string; + private readonly indexPrefix: string; // config setting of index prefix in system index name + private readonly indexInterval: string; // config setting of index prefix: how often to poll for pending work + private readonly queueTimeoutMins: number; // config setting of queue timeout, rounded up to nearest minute private client: ElasticsearchServiceSetup['legacy']['client']; private logger: LevelLogger; @@ -37,7 +55,8 @@ export class ReportingStore { this.client = elasticsearch.legacy.client; this.indexPrefix = config.get('index'); this.indexInterval = config.get('queue', 'indexInterval'); - this.logger = logger; + this.logger = logger.clone(['store']); + this.queueTimeoutMins = Math.ceil(numberToDuration(config.get('queue', 'timeout')).asMinutes()); } private async createIndex(indexName: string) { @@ -119,7 +138,6 @@ export class ReportingStore { report.updateWithEsDoc(doc); await this.refreshIndex(index); - this.logger.debug(`Successfully stored pending job: ${report._index}/${report._id}`); return report; } catch (err) { @@ -129,7 +147,66 @@ export class ReportingStore { } } - public async setReportClaimed(report: Report, stats: Partial): Promise { + /* + * Search for a report from task data and return back the report + */ + public async findReportFromTask(taskJson: ReportTaskParams): Promise { + if (!taskJson.index) { + throw new Error('Task JSON is missing index field!'); + } + + try { + const document = await this.client.callAsInternalUser('get', { + index: taskJson.index, + id: taskJson.id, + }); + + return new Report({ + _id: document._id, + _index: document._index, + _seq_no: document._seq_no, + _primary_term: document._primary_term, + jobtype: document._source.jobtype, + attempts: document._source.attempts, + browser_type: document._source.browser_type, + created_at: document._source.created_at, + created_by: document._source.created_by, + max_attempts: document._source.max_attempts, + meta: document._source.meta, + payload: document._source.payload, + process_expiration: document._source.process_expiration, + status: document._source.status, + timeout: document._source.timeout, + }); + } catch (err) { + this.logger.error('Error in finding a report! ' + JSON.stringify({ report: taskJson })); + this.logger.error(err); + throw err; + } + } + + public async setReportPending(report: Report) { + const doc = { status: statuses.JOB_STATUS_PENDING }; + + try { + checkReportIsEditable(report); + + return await this.client.callAsInternalUser('update', { + id: report._id, + index: report._index, + if_seq_no: report._seq_no, + if_primary_term: report._primary_term, + refresh: true, + body: { doc }, + }); + } catch (err) { + this.logger.error('Error in setting report pending status!'); + this.logger.error(err); + throw err; + } + } + + public async setReportClaimed(report: Report, stats: Partial): Promise { const doc = { ...stats, status: statuses.JOB_STATUS_PROCESSING, @@ -143,6 +220,7 @@ export class ReportingStore { index: report._index, if_seq_no: report._seq_no, if_primary_term: report._primary_term, + refresh: true, body: { doc }, }); } catch (err) { @@ -152,7 +230,7 @@ export class ReportingStore { } } - public async setReportFailed(report: Report, stats: Partial): Promise { + public async setReportFailed(report: Report, stats: Partial): Promise { const doc = { ...stats, status: statuses.JOB_STATUS_FAILED, @@ -166,6 +244,7 @@ export class ReportingStore { index: report._index, if_seq_no: report._seq_no, if_primary_term: report._primary_term, + refresh: true, body: { doc }, }); } catch (err) { @@ -175,7 +254,7 @@ export class ReportingStore { } } - public async setReportCompleted(report: Report, stats: Partial): Promise { + public async setReportCompleted(report: Report, stats: Partial): Promise { try { const { output } = stats; const status = @@ -193,6 +272,7 @@ export class ReportingStore { index: report._index, if_seq_no: report._seq_no, if_primary_term: report._primary_term, + refresh: true, body: { doc }, }); } catch (err) { @@ -201,4 +281,67 @@ export class ReportingStore { throw err; } } + + public async clearExpiration(report: Report): Promise { + try { + checkReportIsEditable(report); + + const updateParams = { + id: report._id, + index: report._index, + if_seq_no: report._seq_no, + if_primary_term: report._primary_term, + refresh: true, + body: { doc: { process_expiration: null } }, + }; + + return await this.client.callAsInternalUser('update', updateParams); + } catch (err) { + this.logger.error('Error in clearing expiration!'); + this.logger.error(err); + throw err; + } + } + + /* + * A zombie report document is one that isn't completed or failed, isn't + * being executed, and isn't scheduled to run. They arise: + * - when the cluster has processing documents in ESQueue before upgrading to v7.13 when ESQueue was removed + * - if Kibana crashes while a report task is executing and it couldn't be rescheduled on its own + * + * Pending reports are not included in this search: they may be scheduled in TM just not run yet. + * TODO Should we get a list of the reports that are pending and scheduled in TM so we can exclude them from this query? + */ + public async findZombieReportDocuments( + logger = this.logger + ): Promise { + const searchParams: SearchParams = { + index: this.indexPrefix + '-*', + filterPath: 'hits.hits', + body: { + sort: { created_at: { order: 'desc' } }, + query: { + bool: { + filter: [ + { + bool: { + must: [ + { range: { process_expiration: { lt: `now-${this.queueTimeoutMins}m` } } }, + { terms: { status: [statuses.JOB_STATUS_PROCESSING] } }, + ], + }, + }, + ], + }, + }, + }, + }; + + const result = await this.client.callAsInternalUser( + 'search', + searchParams + ); + + return result.hits?.hits; + } } diff --git a/x-pack/plugins/reporting/server/lib/tasks/error_logger.test.ts b/x-pack/plugins/reporting/server/lib/tasks/error_logger.test.ts new file mode 100644 index 0000000000000..607c9c32538be --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/tasks/error_logger.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createMockLevelLogger } from '../../test_helpers'; +import { errorLogger } from './error_logger'; + +const logger = createMockLevelLogger(); + +describe('Execute Report Error Logger', () => { + const errorLogSpy = jest.spyOn(logger, 'error'); + + beforeEach(() => { + errorLogSpy.mockReset(); + }); + + it('cuts off the error message after 1000 characters, and includes the last 1000 characters', () => { + const longLogSet = new Array(2000); + for (let i = 0; i < longLogSet.length; i++) { + longLogSet[i] = `e`; // make a lot of e's + } + const longLog = longLogSet.join(''); + const longError = new Error(longLog); + + errorLogger(logger, 'Something went KABOOM!', longError); + + const { message, stack } = errorLogSpy.mock.calls[0][0] as Error; + expect(message).toMatch(/Something went KABOOM!: Error: e{969}\.\.\.e{1000}$/); + expect(stack).toEqual(longError.stack); + + const disclaimer = errorLogSpy.mock.calls[1][0] as string; + expect(disclaimer).toMatchInlineSnapshot( + `"A partial version of the entire error message was logged. The entire error message length is: 2031 characters."` + ); + }); + + it('does not cut off the error message when shorter than the max', () => { + const shortLogSet = new Array(100); + for (let i = 0; i < shortLogSet.length; i++) { + shortLogSet[i] = `e`; // make a lot of e's + } + const shortLog = shortLogSet.join(''); + const shortError = new Error(shortLog); + + errorLogger(logger, 'Something went KABOOM!', shortError); + + const { message, stack } = errorLogSpy.mock.calls[0][0] as Error; + expect(message).toMatch(/Something went KABOOM!: Error: e{100}$/); + expect(stack).toEqual(shortError.stack); + + const disclaimer = errorLogSpy.mock.calls[1]; + expect(disclaimer).toBeUndefined(); + }); +}); diff --git a/x-pack/plugins/reporting/server/lib/tasks/error_logger.ts b/x-pack/plugins/reporting/server/lib/tasks/error_logger.ts new file mode 100644 index 0000000000000..b4d4028230666 --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/tasks/error_logger.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LevelLogger } from '..'; + +const MAX_PARTIAL_ERROR_LENGTH = 1000; // 1000 of beginning, 1000 of end +const ERROR_PARTIAL_SEPARATOR = '...'; +const MAX_ERROR_LENGTH = MAX_PARTIAL_ERROR_LENGTH * 2 + ERROR_PARTIAL_SEPARATOR.length; + +/* + * An error message string could be very long, as it sometimes includes huge + * amount of base64 + */ +export const errorLogger = (logger: LevelLogger, message: string, err?: Error) => { + if (err) { + const errString = `${message}: ${err}`; + const errLength = errString.length; + if (errLength > MAX_ERROR_LENGTH) { + const subStr = String.prototype.substring.bind(errString); + const partialErrString = + subStr(0, MAX_PARTIAL_ERROR_LENGTH) + + ERROR_PARTIAL_SEPARATOR + + subStr(errLength - MAX_PARTIAL_ERROR_LENGTH); + + const partialError = new Error(partialErrString); + partialError.stack = err.stack; + logger.error(partialError); + logger.error( + `A partial version of the entire error message was logged. The entire error message length is: ${errLength} characters.` + ); + } else { + const combinedError = new Error(errString); + combinedError.stack = err.stack; + logger.error(combinedError); + } + return; + } + + logger.error(message); +}; diff --git a/x-pack/plugins/reporting/server/lib/tasks/execute_report.test.ts b/x-pack/plugins/reporting/server/lib/tasks/execute_report.test.ts new file mode 100644 index 0000000000000..5bd895360ef78 --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/tasks/execute_report.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ReportingCore } from '../..'; +import { RunContext } from '../../../../task_manager/server'; +import { taskManagerMock } from '../../../../task_manager/server/mocks'; +import { ReportingConfigType } from '../../config'; +import { + createMockConfig, + createMockConfigSchema, + createMockLevelLogger, + createMockReportingCore, +} from '../../test_helpers'; +import { ExecuteReportTask } from './'; + +const logger = createMockLevelLogger(); + +describe('Execute Report Task', () => { + let mockReporting: ReportingCore; + let configType: ReportingConfigType; + beforeAll(async () => { + configType = createMockConfigSchema(); + const mockConfig = createMockConfig(configType); + mockReporting = await createMockReportingCore(mockConfig); + }); + + it('Instance setup', () => { + const task = new ExecuteReportTask(mockReporting, configType, logger); + expect(task.getStatus()).toBe('uninitialized'); + expect(task.getTaskDefinition()).toMatchInlineSnapshot(` + Object { + "createTaskRunner": [Function], + "maxAttempts": 1, + "maxConcurrency": 1, + "timeout": "120s", + "title": "Reporting: execute job", + "type": "report:execute", + } + `); + }); + + it('Instance start', () => { + const mockTaskManager = taskManagerMock.createStart(); + const task = new ExecuteReportTask(mockReporting, configType, logger); + expect(task.init(mockTaskManager)); + expect(task.getStatus()).toBe('initialized'); + }); + + it('create task runner', async () => { + logger.info = jest.fn(); + logger.error = jest.fn(); + + const task = new ExecuteReportTask(mockReporting, configType, logger); + const taskDef = task.getTaskDefinition(); + const taskRunner = taskDef.createTaskRunner(({ + taskInstance: { + id: 'random-task-id', + params: { index: 'cool-reporting-index', id: 'cool-reporting-id' }, + }, + } as unknown) as RunContext); + expect(taskRunner).toHaveProperty('run'); + expect(taskRunner).toHaveProperty('cancel'); + }); + + it('Max Concurrency is 0 if pollEnabled is false', () => { + const queueConfig = ({ + queue: { pollEnabled: false, timeout: 55000 }, + } as unknown) as ReportingConfigType['queue']; + + const task = new ExecuteReportTask(mockReporting, { ...configType, ...queueConfig }, logger); + expect(task.getStatus()).toBe('uninitialized'); + expect(task.getTaskDefinition()).toMatchInlineSnapshot(` + Object { + "createTaskRunner": [Function], + "maxAttempts": 1, + "maxConcurrency": 0, + "timeout": "55s", + "title": "Reporting: execute job", + "type": "report:execute", + } + `); + }); +}); diff --git a/x-pack/plugins/reporting/server/lib/tasks/execute_report.ts b/x-pack/plugins/reporting/server/lib/tasks/execute_report.ts new file mode 100644 index 0000000000000..2960ce457b7ae --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/tasks/execute_report.ts @@ -0,0 +1,400 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import * as Rx from 'rxjs'; +import { timeout } from 'rxjs/operators'; +import { LevelLogger } from '../'; +import { ReportingCore } from '../../'; +import { + RunContext, + TaskManagerStartContract, + TaskRunCreatorFunction, +} from '../../../../task_manager/server'; +import { CancellationToken } from '../../../common'; +import { durationToNumber, numberToDuration } from '../../../common/schema_utils'; +import { ReportingConfigType } from '../../config'; +import { BasePayload, RunTaskFn } from '../../types'; +import { Report, ReportingStore } from '../store'; +import { + ReportingExecuteTaskInstance, + ReportingTask, + ReportingTaskStatus, + REPORTING_EXECUTE_TYPE, + ReportTaskParams, + TaskRunResult, +} from './'; +import { errorLogger } from './error_logger'; + +function isOutput(output: TaskRunResult | Error): output is TaskRunResult { + return typeof output === 'object' && (output as TaskRunResult).content != null; +} + +function reportFromTask(task: ReportTaskParams) { + return new Report({ ...task, _id: task.id, _index: task.index }); +} + +export class ExecuteReportTask implements ReportingTask { + public TYPE = REPORTING_EXECUTE_TYPE; + + private logger: LevelLogger; + private taskManagerStart?: TaskManagerStartContract; + private taskExecutors?: Map>; + private kibanaId?: string; + private kibanaName?: string; + private store?: ReportingStore; + + constructor( + private reporting: ReportingCore, + private config: ReportingConfigType, + logger: LevelLogger + ) { + this.logger = logger.clone(['runTask']); + } + + /* + * To be called from plugin start + */ + public async init(taskManager: TaskManagerStartContract) { + this.taskManagerStart = taskManager; + + const { reporting } = this; + + const exportTypesRegistry = reporting.getExportTypesRegistry(); + const executors = new Map>(); + for (const exportType of exportTypesRegistry.getAll()) { + const exportTypeLogger = this.logger.clone([exportType.id]); + const jobExecutor = exportType.runTaskFnFactory(reporting, exportTypeLogger); + // The task will run the function with the job type as a param. + // This allows us to retrieve the specific export type runFn when called to run an export + executors.set(exportType.jobType, jobExecutor); + } + + this.taskExecutors = executors; + + const config = reporting.getConfig(); + this.kibanaId = config.kbnConfig.get('server', 'uuid'); + this.kibanaName = config.kbnConfig.get('server', 'name'); + } + + /* + * Async get the ReportingStore: it is only available after PluginStart + */ + private async getStore(): Promise { + if (this.store) { + return this.store; + } + const { store } = await this.reporting.getPluginStartDeps(); + this.store = store; + return store; + } + + private getTaskManagerStart() { + if (!this.taskManagerStart) { + throw new Error('Reporting task runner has not been initialized!'); + } + return this.taskManagerStart; + } + + public async _claimJob(task: ReportTaskParams): Promise { + const store = await this.getStore(); + + let report: Report; + if (task.id && task.index) { + // if this is an ad-hoc report, there is a corresponding "pending" record in ReportingStore in need of updating + report = await store.findReportFromTask(task); // update seq_no + } else { + // if this is a scheduled report (not implemented), the report object needs to be instantiated + throw new Error('scheduled reports are not supported!'); + } + + // Check if this is a completed job. This may happen if the `reports:monitor` + // task detected it to be a zombie job and rescheduled it, but it + // eventually completed on its own. + if (report.status === 'completed') { + throw new Error(`Can not claim the report job: it is already completed!`); + } + + const m = moment(); + + // check if job has exceeded maxAttempts (stored in job params) and somehow hasn't been marked as failed yet + // NOTE: the max attempts value comes from the stored document, so changing the capture.maxAttempts config setting does not affect existing pending reports + const maxAttempts = task.max_attempts; + if (report.attempts >= maxAttempts) { + const err = new Error(`Max attempts reached (${maxAttempts}). Queue timeout reached.`); + await this._failJob(task, err); + throw err; + } + + const queueTimeout = durationToNumber(this.config.queue.timeout); + const startTime = m.toISOString(); + const expirationTime = m.add(queueTimeout).toISOString(); + + const stats = { + kibana_id: this.kibanaId, + kibana_name: this.kibanaName, + browser_type: this.config.capture.browser.type, + attempts: report.attempts + 1, + started_at: startTime, + timeout: queueTimeout, + process_expiration: expirationTime, + }; + + this.logger.debug(`Claiming ${report.jobtype} job ${report._id}`); + + const claimedReport = new Report({ + ...report, + ...stats, + }); + await store.setReportClaimed(claimedReport, stats); + + return claimedReport; + } + + private async _failJob(task: ReportTaskParams, error?: Error) { + const message = `Failing ${task.jobtype} job ${task.id}`; + + // log the error + let docOutput; + if (error) { + errorLogger(this.logger, message, error); + docOutput = this._formatOutput(error); + } else { + errorLogger(this.logger, message); + } + + // update the report in the store + const store = await this.getStore(); + const report = await store.findReportFromTask(task); + const completedTime = moment().toISOString(); + const doc = { + completed_at: completedTime, + output: docOutput, + }; + + return await store.setReportFailed(report, doc); + } + + private _formatOutput(output: TaskRunResult | Error) { + const docOutput = {} as TaskRunResult; + const unknownMime = null; + + if (isOutput(output)) { + docOutput.content = output.content; + docOutput.content_type = output.content_type || unknownMime; + docOutput.max_size_reached = output.max_size_reached; + docOutput.csv_contains_formulas = output.csv_contains_formulas; + docOutput.size = output.size; + docOutput.warnings = + output.warnings && output.warnings.length > 0 ? output.warnings : undefined; + } else { + const defaultOutput = null; + docOutput.content = output.toString() || defaultOutput; + docOutput.content_type = unknownMime; + docOutput.warnings = [output.toString()]; + } + + return docOutput; + } + + public async _performJob(task: ReportTaskParams, cancellationToken: CancellationToken) { + if (!this.taskExecutors) { + throw new Error(`Task run function factories have not been called yet!`); + } + + // get the run_task function + const runner = this.taskExecutors.get(task.jobtype); + if (!runner) { + throw new Error(`No defined task runner function for ${task.jobtype}!`); + } + + // run the report + // if workerFn doesn't finish before timeout, call the cancellationToken and throw an error + const queueTimeout = durationToNumber(this.config.queue.timeout); + return Rx.from(runner(task.id, task.payload, cancellationToken)) + .pipe(timeout(queueTimeout)) // throw an error if a value is not emitted before timeout + .toPromise(); + } + + public async _completeJob(task: ReportTaskParams, output: TaskRunResult) { + let docId = `/${task.index}/_doc/${task.id}`; + + this.logger.info(`Saving ${task.jobtype} job ${docId}.`); + + const completedTime = moment().toISOString(); + const docOutput = this._formatOutput(output); + + const store = await this.getStore(); + const doc = { + completed_at: completedTime, + output: docOutput, + }; + const report = await store.findReportFromTask(task); // update seq_no and primary_term + docId = `/${report._index}/_doc/${report._id}`; + + try { + await store.setReportCompleted(report, doc); + this.logger.debug(`Saved ${report.jobtype} job ${docId}`); + } catch (err) { + if (err.statusCode === 409) return false; + errorLogger(this.logger, `Failure saving completed job ${docId}!`); + } + } + + /* + * Provides a TaskRunner for Task Manager + */ + private getTaskRunner(): TaskRunCreatorFunction { + // Keep a separate local stack for each task run + return (context: RunContext) => { + let jobId: string | undefined; + const cancellationToken = new CancellationToken(); + + return { + /* + * Runs a reporting job + * Claim job: Finds the report in ReportingStore, updates it to "processing" + * Perform job: Gets the export type's runner, runs it with the job params + * Complete job: Updates the report in ReportStore with the output from the runner + * If any error happens, additional retry attempts may be picked up by a separate instance + */ + run: async () => { + let report: Report | undefined; + let attempts = 0; + + // find the job in the store and set status to processing + const task = context.taskInstance.params as ReportTaskParams; + jobId = task?.id; + + try { + if (!jobId) { + throw new Error('Invalid report data provided in scheduled task!'); + } + this.reporting.trackReport(jobId); + + // Update job status to claimed + report = await this._claimJob(task); + + const { jobtype: jobType, attempts: attempt, max_attempts: maxAttempts } = task; + this.logger.info( + `Starting ${jobType} report ${jobId}: attempt ${attempt + 1} of ${maxAttempts}.` + ); + this.logger.debug(`Reports running: ${this.reporting.countConcurrentReports()}.`); + } catch (failedToClaim) { + // error claiming report - log the error + // could be version conflict, or no longer connected to ES + errorLogger(this.logger, `Error in claiming report!`, failedToClaim); + } + + if (!report) { + errorLogger(this.logger, `Report could not be claimed. Exiting...`); + return; + } + + attempts = report.attempts; + + try { + const output = await this._performJob(task, cancellationToken); + if (output) { + await this._completeJob(task, output); + } + + // untrack the report for concurrency awareness + this.logger.debug(`Stopping ${jobId}.`); + this.reporting.untrackReport(jobId); + this.logger.debug(`Reports running: ${this.reporting.countConcurrentReports()}.`); + } catch (failedToExecuteErr) { + cancellationToken.cancel(); + + const maxAttempts = this.config.capture.maxAttempts; + if (attempts < maxAttempts) { + // attempts remain - reschedule + try { + // reschedule to retry + const remainingAttempts = maxAttempts - report.attempts; + errorLogger( + this.logger, + `Scheduling retry. Retries remaining: ${remainingAttempts}.`, + failedToExecuteErr + ); + + await this.rescheduleTask(reportFromTask(task).toReportTaskJSON(), this.logger); + } catch (rescheduleErr) { + // can not be rescheduled - log the error + errorLogger(this.logger, `Could not reschedule the errored job!`, rescheduleErr); + } + } else { + // 0 attempts remain - fail the job + try { + const maxAttemptsMsg = `Max attempts reached (${attempts}). Failed with: ${failedToExecuteErr}`; + await this._failJob(task, new Error(maxAttemptsMsg)); + } catch (failedToFailError) { + errorLogger(this.logger, `Could not fail the job!`, failedToFailError); + } + } + } + }, + + /* + * Called by Task Manager to stop the report execution process in case + * of timeout or server shutdown + */ + cancel: async () => { + if (jobId) { + this.logger.warn(`Cancelling job ${jobId}...`); + } + cancellationToken.cancel(); + }, + }; + }; + } + + public getTaskDefinition() { + // round up from ms to the nearest second + const queueTimeout = Math.ceil(numberToDuration(this.config.queue.timeout).asSeconds()) + 's'; + const maxConcurrency = this.config.queue.pollEnabled ? 1 : 0; + + return { + type: REPORTING_EXECUTE_TYPE, + title: 'Reporting: execute job', + createTaskRunner: this.getTaskRunner(), + maxAttempts: 1, // NOTE: not using Task Manager retries + timeout: queueTimeout, + maxConcurrency, + }; + } + + public async scheduleTask(report: ReportTaskParams) { + const taskInstance: ReportingExecuteTaskInstance = { + taskType: REPORTING_EXECUTE_TYPE, + state: {}, + params: report, + }; + return await this.getTaskManagerStart().schedule(taskInstance); + } + + private async rescheduleTask(task: ReportTaskParams, logger: LevelLogger) { + logger.info(`Rescheduling ${task.id} to retry after error.`); + + const oldTaskInstance: ReportingExecuteTaskInstance = { + taskType: REPORTING_EXECUTE_TYPE, + state: {}, + params: task, + }; + const newTask = await this.getTaskManagerStart().schedule(oldTaskInstance); + logger.debug(`Rescheduled ${task.id}`); + return newTask; + } + + public getStatus() { + if (this.taskManagerStart) { + return ReportingTaskStatus.INITIALIZED; + } + + return ReportingTaskStatus.UNINITIALIZED; + } +} diff --git a/x-pack/plugins/reporting/server/lib/tasks/index.ts b/x-pack/plugins/reporting/server/lib/tasks/index.ts index 81d834440152a..ec9e85e957d03 100644 --- a/x-pack/plugins/reporting/server/lib/tasks/index.ts +++ b/x-pack/plugins/reporting/server/lib/tasks/index.ts @@ -5,9 +5,17 @@ * 2.0. */ +import { TaskRunCreatorFunction } from '../../../../task_manager/server'; import { ReportSource, TaskRunResult } from '../../../common/types'; import { BasePayload } from '../../types'; +export const REPORTING_EXECUTE_TYPE = 'report:execute'; +export const REPORTING_MONITOR_TYPE = 'reports:monitor'; + +export { ExecuteReportTask } from './execute_report'; +export { MonitorReportsTask } from './monitor_reports'; +export { TaskRunResult }; + /* * The document created by Reporting to store as task parameters for Task * Manager to reference the report in .reporting @@ -19,8 +27,31 @@ export interface ReportTaskParams { created_at: ReportSource['created_at']; created_by: ReportSource['created_by']; jobtype: ReportSource['jobtype']; + max_attempts: ReportSource['max_attempts']; attempts: ReportSource['attempts']; meta: ReportSource['meta']; } -export { TaskRunResult }; +export interface ReportingExecuteTaskInstance /* extends TaskInstanceWithDeprecatedFields */ { + state: object; + taskType: string; + params: ReportTaskParams; + runAt?: Date; +} + +export enum ReportingTaskStatus { + UNINITIALIZED = 'uninitialized', + INITIALIZED = 'initialized', +} + +export interface ReportingTask { + getTaskDefinition: () => { + type: string; + title: string; + createTaskRunner: TaskRunCreatorFunction; + maxAttempts: number; + timeout: string; + }; + + getStatus: () => ReportingTaskStatus; +} diff --git a/x-pack/plugins/reporting/server/lib/tasks/monitor_report.test.ts b/x-pack/plugins/reporting/server/lib/tasks/monitor_report.test.ts new file mode 100644 index 0000000000000..65627dc86fa5a --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/tasks/monitor_report.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ReportingCore } from '../..'; +import { RunContext } from '../../../../task_manager/server'; +import { taskManagerMock } from '../../../../task_manager/server/mocks'; +import { ReportingConfigType } from '../../config'; +import { + createMockConfig, + createMockConfigSchema, + createMockLevelLogger, + createMockReportingCore, +} from '../../test_helpers'; +import { MonitorReportsTask } from './'; + +const logger = createMockLevelLogger(); + +describe('Execute Report Task', () => { + let mockReporting: ReportingCore; + let configType: ReportingConfigType; + beforeAll(async () => { + configType = createMockConfigSchema(); + const mockConfig = createMockConfig(configType); + mockReporting = await createMockReportingCore(mockConfig); + }); + + it('Instance setup', () => { + const task = new MonitorReportsTask(mockReporting, configType, logger); + expect(task.getStatus()).toBe('uninitialized'); + expect(task.getTaskDefinition()).toMatchInlineSnapshot(` + Object { + "createTaskRunner": [Function], + "maxAttempts": 1, + "timeout": "120s", + "title": "Reporting: monitor jobs", + "type": "reports:monitor", + } + `); + }); + + it('Instance start', () => { + const mockTaskManager = taskManagerMock.createStart(); + const task = new MonitorReportsTask(mockReporting, configType, logger); + expect(task.init(mockTaskManager)); + expect(task.getStatus()).toBe('initialized'); + }); + + it('create task runner', async () => { + logger.info = jest.fn(); + logger.error = jest.fn(); + + const task = new MonitorReportsTask(mockReporting, configType, logger); + const taskDef = task.getTaskDefinition(); + const taskRunner = taskDef.createTaskRunner(({ + taskInstance: { + id: 'random-task-id', + params: { index: 'cool-reporting-index', id: 'cool-reporting-id' }, + }, + } as unknown) as RunContext); + expect(taskRunner).toHaveProperty('run'); + expect(taskRunner).toHaveProperty('cancel'); + }); +}); diff --git a/x-pack/plugins/reporting/server/lib/tasks/monitor_reports.ts b/x-pack/plugins/reporting/server/lib/tasks/monitor_reports.ts new file mode 100644 index 0000000000000..36380f767e6d9 --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/tasks/monitor_reports.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import { LevelLogger, ReportingStore } from '../'; +import { ReportingCore } from '../../'; +import { TaskManagerStartContract, TaskRunCreatorFunction } from '../../../../task_manager/server'; +import { numberToDuration } from '../../../common/schema_utils'; +import { ReportingConfigType } from '../../config'; +import { Report } from '../store'; +import { + ReportingExecuteTaskInstance, + ReportingTask, + ReportingTaskStatus, + REPORTING_EXECUTE_TYPE, + REPORTING_MONITOR_TYPE, + ReportTaskParams, +} from './'; + +/* + * Task for finding the ReportingRecords left in the ReportingStore and stuck + * in pending or processing. It could happen if the server crashed while running + * a report and was cancelled. Normally a failure would mean scheduling a + * retry or failing the report, but the retry is not guaranteed to be scheduled. + */ +export class MonitorReportsTask implements ReportingTask { + public TYPE = REPORTING_MONITOR_TYPE; + + private logger: LevelLogger; + private taskManagerStart?: TaskManagerStartContract; + private store?: ReportingStore; + private timeout: moment.Duration; + + constructor( + private reporting: ReportingCore, + private config: ReportingConfigType, + parentLogger: LevelLogger + ) { + this.logger = parentLogger.clone([REPORTING_MONITOR_TYPE]); + this.timeout = numberToDuration(config.queue.timeout); + } + + private async getStore(): Promise { + if (this.store) { + return this.store; + } + const { store } = await this.reporting.getPluginStartDeps(); + this.store = store; + return store; + } + + public async init(taskManager: TaskManagerStartContract) { + this.taskManagerStart = taskManager; + + // Round the interval up to the nearest second since Task Manager doesn't + // support milliseconds + const scheduleInterval = + Math.ceil(numberToDuration(this.config.queue.pollInterval).asSeconds()) + 's'; + this.logger.debug(`Task to monitor for pending reports to run every ${scheduleInterval}.`); + await taskManager.ensureScheduled({ + id: this.TYPE, + taskType: this.TYPE, + schedule: { interval: scheduleInterval }, + state: {}, + params: {}, + }); + } + + private getTaskRunner(): TaskRunCreatorFunction { + return () => { + return { + run: async () => { + const reportingStore = await this.getStore(); + + try { + const results = await reportingStore.findZombieReportDocuments(); + if (results && results.length) { + this.logger.info( + `Found ${results.length} reports to reschedule: ${results + .map((pending) => pending._id) + .join(',')}` + ); + } else { + this.logger.debug(`Found 0 pending reports.`); + return; + } + + for (const pending of results) { + const { + _id: jobId, + _source: { process_expiration: processExpiration, status }, + } = pending; + const expirationTime = moment(processExpiration); // If it is the start of the Epoch, something went wrong + const timeWaitValue = moment().valueOf() - expirationTime.valueOf(); + const timeWaitTime = moment.duration(timeWaitValue); + this.logger.info( + `Task ${jobId} has ${status} status for ${timeWaitTime.humanize()}. The queue timeout is ${this.timeout.humanize()}.` + ); + + // clear process expiration and reschedule + const oldReport = new Report({ ...pending, ...pending._source }); + const reschedulingTask = oldReport.toReportTaskJSON(); + await reportingStore.clearExpiration(oldReport); + await this.rescheduleTask(reschedulingTask, this.logger); + } + } catch (err) { + this.logger.error(err); + } + + return; + }, + + cancel: async () => ({ state: {} }), + }; + }; + } + + public getTaskDefinition() { + return { + type: REPORTING_MONITOR_TYPE, + title: 'Reporting: monitor jobs', + createTaskRunner: this.getTaskRunner(), + maxAttempts: 1, + // round the timeout value up to the nearest second, since Task Manager + // doesn't support milliseconds + timeout: Math.ceil(this.timeout.asSeconds()) + 's', + }; + } + + // reschedule the task with TM and update the report document status to "Pending" + private async rescheduleTask(task: ReportTaskParams, logger: LevelLogger) { + if (!this.taskManagerStart) { + throw new Error('Reporting task runner has not been initialized!'); + } + logger.info(`Rescheduling ${task.id} to retry after timeout expiration.`); + + const store = await this.getStore(); + + const oldTaskInstance: ReportingExecuteTaskInstance = { + taskType: REPORTING_EXECUTE_TYPE, // schedule a task to EXECUTE + state: {}, + params: task, + }; + + const [report, newTask] = await Promise.all([ + await store.findReportFromTask(task), + await this.taskManagerStart.schedule(oldTaskInstance), + ]); + + await store.setReportPending(report); + + return newTask; + } + + public getStatus() { + if (this.taskManagerStart) { + return ReportingTaskStatus.INITIALIZED; + } + + return ReportingTaskStatus.UNINITIALIZED; + } +} diff --git a/x-pack/plugins/reporting/server/plugin.test.ts b/x-pack/plugins/reporting/server/plugin.test.ts index c21bc7376b0b3..ce3b8aabcaa8d 100644 --- a/x-pack/plugins/reporting/server/plugin.test.ts +++ b/x-pack/plugins/reporting/server/plugin.test.ts @@ -16,9 +16,10 @@ jest.mock('./browsers/install', () => ({ })); import { coreMock } from 'src/core/server/mocks'; +import { featuresPluginMock } from '../../features/server/mocks'; +import { TaskManagerSetupContract } from '../../task_manager/server'; import { ReportingPlugin } from './plugin'; import { createMockConfigSchema } from './test_helpers'; -import { featuresPluginMock } from '../../features/server/mocks'; const sleep = (time: number) => new Promise((r) => setTimeout(r, time)); @@ -42,6 +43,9 @@ describe('Reporting Plugin', () => { makeUsageCollector: jest.fn(), registerCollector: jest.fn(), }, + taskManager: ({ + registerTaskDefinitions: jest.fn(), + } as unknown) as TaskManagerSetupContract, security: { authc: { getCurrentUser: () => ({ @@ -65,23 +69,6 @@ describe('Reporting Plugin', () => { expect(plugin.setup(coreSetup, pluginSetup)).not.toHaveProperty('then'); }); - it('logs setup issues', async () => { - initContext.config = null; - const plugin = new ReportingPlugin(initContext); - // @ts-ignore overloading error logger - plugin.logger.error = jest.fn(); - plugin.setup(coreSetup, pluginSetup); - - await sleep(5); - - // @ts-ignore overloading error logger - expect(plugin.logger.error.mock.calls[0][0]).toMatch( - /Error in Reporting setup, reporting may not function properly/ - ); - // @ts-ignore overloading error logger - expect(plugin.logger.error).toHaveBeenCalledTimes(2); - }); - it('has a sync startup process', async () => { const plugin = new ReportingPlugin(initContext); plugin.setup(coreSetup, pluginSetup); diff --git a/x-pack/plugins/reporting/server/plugin.ts b/x-pack/plugins/reporting/server/plugin.ts index 7d30ae78e5c84..e910fecb76988 100644 --- a/x-pack/plugins/reporting/server/plugin.ts +++ b/x-pack/plugins/reporting/server/plugin.ts @@ -12,7 +12,7 @@ import { PLUGIN_ID, UI_SETTINGS_CUSTOM_PDF_LOGO } from '../common/constants'; import { ReportingCore } from './'; import { initializeBrowserDriverFactory } from './browsers'; import { buildConfig, ReportingConfigType } from './config'; -import { createQueueFactory, LevelLogger, ReportingStore } from './lib'; +import { LevelLogger, ReportingStore } from './lib'; import { registerRoutes } from './routes'; import { setFieldFormats } from './services'; import { ReportingSetup, ReportingSetupDeps, ReportingStart, ReportingStartDeps } from './types'; @@ -29,8 +29,8 @@ export class ReportingPlugin constructor(context: PluginInitializerContext) { this.logger = new LevelLogger(context.logger.get()); + this.reportingCore = new ReportingCore(this.logger, context); this.initializerContext = context; - this.reportingCore = new ReportingCore(this.logger); } public setup(core: CoreSetup, plugins: ReportingSetupDeps) { @@ -68,7 +68,7 @@ export class ReportingPlugin }); const { elasticsearch, http } = core; - const { features, licensing, security, spaces } = plugins; + const { features, licensing, security, spaces, taskManager } = plugins; const { initializerContext: initContext, reportingCore } = this; const router = http.createRouter(); @@ -82,6 +82,7 @@ export class ReportingPlugin router, security, spaces, + taskManager, }); registerReportingUsageCollector(reportingCore, plugins); @@ -115,14 +116,13 @@ export class ReportingPlugin const browserDriverFactory = await initializeBrowserDriverFactory(config, logger); const store = new ReportingStore(reportingCore, logger); - const esqueue = await createQueueFactory(reportingCore, store, logger); // starts polling for pending jobs - reportingCore.pluginStart({ + await reportingCore.pluginStart({ browserDriverFactory, savedObjects: core.savedObjects, uiSettings: core.uiSettings, - esqueue, store, + taskManager: plugins.taskManager, }); this.logger.debug('Start complete'); diff --git a/x-pack/plugins/reporting/server/routes/generation.ts b/x-pack/plugins/reporting/server/routes/generation.ts index c7ebd969fc01a..3edd898609f8c 100644 --- a/x-pack/plugins/reporting/server/routes/generation.ts +++ b/x-pack/plugins/reporting/server/routes/generation.ts @@ -48,7 +48,7 @@ export function registerJobGenerationRoutes(reporting: ReportingCore, logger: Lo const enqueueJob = enqueueJobFactory(reporting, logger); const report = await enqueueJob(exportTypeId, jobParams, user, context, req); - // return the queue's job information + // return task manager's task information and the download URL const downloadBaseUrl = getDownloadBaseUrl(reporting); return res.ok({ diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts index ea8480ef3493d..0700fbaff0fe3 100644 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -8,7 +8,6 @@ jest.mock('../routes'); jest.mock('../usage'); jest.mock('../browsers'); -jest.mock('../lib/create_queue'); import _ from 'lodash'; import * as Rx from 'rxjs'; @@ -39,6 +38,7 @@ export const createMockPluginSetup = (setupMock?: any): ReportingInternalSetup = router: setupMock.router, security: setupMock.security, licensing: { license$: Rx.of({ isAvailable: true, isActive: true, type: 'basic' }) } as any, + taskManager: { registerTaskDefinitions: jest.fn() } as any, ...setupMock, }; }; @@ -52,10 +52,13 @@ const createMockPluginStart = ( const store = new ReportingStore(mockReportingCore, logger); return { browserDriverFactory: startMock.browserDriverFactory, - esqueue: startMock.esqueue, savedObjects: startMock.savedObjects || { getScopedClient: jest.fn() }, uiSettings: startMock.uiSettings || { asScopedToClient: () => ({ get: jest.fn() }) }, store, + taskManager: { + schedule: jest.fn().mockImplementation(() => ({ id: 'taskId' })), + ensureScheduled: jest.fn(), + } as any, ...startMock, }; }; @@ -128,7 +131,7 @@ export const createMockReportingCore = async ( } const context = coreMock.createPluginInitializerContext(createMockConfigSchema()); - const core = new ReportingCore(logger); + const core = new ReportingCore(logger, context); core.setConfig(config); core.pluginSetup(setupDepsMock); diff --git a/x-pack/plugins/reporting/server/types.ts b/x-pack/plugins/reporting/server/types.ts index b395738ad4445..1b762c96079fa 100644 --- a/x-pack/plugins/reporting/server/types.ts +++ b/x-pack/plugins/reporting/server/types.ts @@ -13,6 +13,7 @@ import { PluginSetupContract as FeaturesPluginSetup } from '../../features/serve import { LicensingPluginSetup } from '../../licensing/server'; import { AuthenticatedUser, SecurityPluginSetup } from '../../security/server'; import { SpacesPluginSetup } from '../../spaces/server'; +import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; import { CancellationToken } from '../common'; import { BaseParams, TaskRunResult } from '../common/types'; import { ReportingConfigType } from './config'; @@ -29,11 +30,13 @@ export interface ReportingSetupDeps { features: FeaturesPluginSetup; security?: SecurityPluginSetup; spaces?: SpacesPluginSetup; + taskManager: TaskManagerSetupContract; usageCollection?: UsageCollectionSetup; } export interface ReportingStartDeps { data: DataPluginStart; + taskManager: TaskManagerStartContract; } export type ReportingStart = object; diff --git a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts index 99832e3f4b1c2..222a57ded1fe5 100644 --- a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts +++ b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts @@ -44,10 +44,7 @@ export default function ({ getService }: FtrProviderContext) { attempts: 0, created_by: false, jobtype: 'csv', - max_attempts: 1, status: 'pending', - timeout: 120000, - browser_type: 'chromium', // TODO: remove this field from the API response // TODO: remove the payload field from the api respones }; forOwn(expectedResJob, (value: any, key: string) => { @@ -78,12 +75,11 @@ export default function ({ getService }: FtrProviderContext) { .set('kbn-xsrf', 'xxx'); const listingJobs = JSON.parse(listText); + const expectedListJob: Record = { attempts: 0, created_by: false, jobtype: 'csv', - timeout: 120000, - browser_type: 'chromium', }; forOwn(expectedListJob, (value: any, key: string) => { expect(listingJobs[0]._source[key]).to.eql(value, key); @@ -112,8 +108,6 @@ export default function ({ getService }: FtrProviderContext) { attempts: 0, created_by: false, jobtype: 'csv', - timeout: 120000, - browser_type: 'chromium', }; forOwn(expectedListJob, (value: any, key: string) => { expect(listingJobs[0]._source[key]).to.eql(value, key);