From f17fe1bd9d2e12b6056d879637e594c8850c8ded Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Fri, 24 Mar 2023 15:43:06 -0400 Subject: [PATCH] [Cases] Adding file telemetry (#152968) This PR adds telemetry for the attachment framework and files attachments by plugin and for all cases. Issue: https://github.com/elastic/kibana/issues/151933 Notable changes: - We now send information for each registered attachment type - We send average file size - We leverage the files saved object to aggregate this information - We use an array type so that we don't have to hard code each attachment type, they are dynamically discovered instead - Stats for the attachments are done for all cases and broken down by plugin
Example telemetry payload ``` "cases": { "cases": { "all": { "total": 1, "daily": 1, "weekly": 1, "monthly": 1, "status": { "open": 1, "inProgress": 0, "closed": 0 }, "syncAlertsOn": 1, "syncAlertsOff": 0, "totalUsers": 1, "totalParticipants": 1, "totalTags": 1, "totalWithAlerts": 0, "totalWithConnectors": 0, "latestDates": { "createdAt": "2023-03-09T15:20:46.399Z", "updatedAt": "2023-03-09T15:21:02.399Z", "closedAt": null }, "assignees": { "total": 0, "totalWithZero": 1, "totalWithAtLeastOne": 0 }, "attachmentFramework": { "externalAttachments": [ { "type": ".files", "average": 4, "maxOnACase": 4, "total": 4 } ], "persistableAttachments": [], "files": { "averageSize": { "value": 3 }, "average": 4, "maxOnACase": 4, "total": 4 } } }, "sec": { "total": 0, "daily": 0, "weekly": 0, "monthly": 0, "attachmentFramework": { "externalAttachments": [], "persistableAttachments": [], "files": { "average": 0, "averageSize": 0, "maxOnACase": 0, "total": 0 } }, "assignees": { "total": 0, "totalWithZero": 0, "totalWithAtLeastOne": 0 } }, "obs": { "total": 0, "daily": 0, "weekly": 0, "monthly": 0, "attachmentFramework": { "externalAttachments": [], "persistableAttachments": [], "files": { "average": 0, "averageSize": 0, "maxOnACase": 0, "total": 0 } }, "assignees": { "total": 0, "totalWithZero": 0, "totalWithAtLeastOne": 0 } }, "main": { "total": 1, "daily": 1, "weekly": 1, "monthly": 1, "attachmentFramework": { "externalAttachments": [ { "type": ".files", "average": 4, "maxOnACase": 4, "total": 4 } ], "persistableAttachments": [], "files": { "averageSize": { "value": 3 }, "average": 4, "maxOnACase": 4, "total": 4 } }, "assignees": { "total": 0, "totalWithZero": 1, "totalWithAtLeastOne": 0 } } }, "userActions": { "all": { "total": 5, "daily": 5, "weekly": 5, "monthly": 5, "maxOnACase": 5 } }, "comments": { "all": { "total": 0, "daily": 0, "weekly": 0, "monthly": 0, "maxOnACase": 0 } }, "alerts": { "all": { "total": 0, "daily": 0, "weekly": 0, "monthly": 0, "maxOnACase": 0 } }, "connectors": { "all": { "all": { "totalAttached": 0 }, "itsm": { "totalAttached": 0 }, "sir": { "totalAttached": 0 }, "jira": { "totalAttached": 0 }, "resilient": { "totalAttached": 0 }, "swimlane": { "totalAttached": 0 }, "maxAttachedToACase": 0 } }, "pushes": { "all": { "total": 0, "maxOnACase": 0 } }, "configuration": { "all": { "closure": { "manually": 0, "automatic": 0 } } } }, ```
## Testing To test modify this file: https://github.com/elastic/kibana/blob/main/x-pack/plugins/cases/server/telemetry/schedule_telemetry_task.ts With: ``` export const scheduleCasesTelemetryTask = ( taskManager: TaskManagerStartContract, logger: Logger ) => { (async () => { await taskManager .ensureScheduled({ id: CASES_TELEMETRY_TASK_NAME, taskType: CASES_TELEMETRY_TASK_NAME, schedule: { interval: `${MINUTES_ON_HALF_DAY}m`, }, scope: ['cases'], params: {}, state: {}, }) .catch((err) => { logger.debug( `Error scheduling cases task with ID ${CASES_TELEMETRY_TASK_NAME} and type ${CASES_TELEMETRY_TASK_NAME}. Received ${err.message}` ); }); await taskManager.runSoon(CASES_TELEMETRY_TASK_NAME); })(); }; ``` This will cause the telemetry to be sent as soon as the server is restarted. To generate files and attachments to add stats to the telemetry I created this python script: https://github.com/elastic/cases-files-generator To retrieve the telemetry: ``` POST http://localhost:5601/api/telemetry/v2/clusters/_stats { "refreshCache": true, "unencrypted": true } ``` --- .../cases/server/telemetry/constants.ts | 13 - .../plugins/cases/server/telemetry/index.ts | 3 +- .../server/telemetry/queries/cases.test.ts | 547 +++++++++++++++++- .../cases/server/telemetry/queries/cases.ts | 302 +++++++--- .../server/telemetry/queries/utils.test.ts | 446 +++++++++++++- .../cases/server/telemetry/queries/utils.ts | 221 ++++--- .../plugins/cases/server/telemetry/schema.ts | 30 + .../plugins/cases/server/telemetry/types.ts | 86 ++- .../schema/xpack_plugins.json | 232 ++++++++ 9 files changed, 1642 insertions(+), 238 deletions(-) delete mode 100644 x-pack/plugins/cases/server/telemetry/constants.ts diff --git a/x-pack/plugins/cases/server/telemetry/constants.ts b/x-pack/plugins/cases/server/telemetry/constants.ts deleted file mode 100644 index 705321e3f1fa0..0000000000000 --- a/x-pack/plugins/cases/server/telemetry/constants.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { GENERAL_CASES_OWNER, OBSERVABILITY_OWNER, SECURITY_SOLUTION_OWNER } from '../../common'; - -/** - * This should only be used within telemetry - */ -export const OWNERS = [OBSERVABILITY_OWNER, SECURITY_SOLUTION_OWNER, GENERAL_CASES_OWNER] as const; diff --git a/x-pack/plugins/cases/server/telemetry/index.ts b/x-pack/plugins/cases/server/telemetry/index.ts index 6cd796eca2c18..7ef14541ced09 100644 --- a/x-pack/plugins/cases/server/telemetry/index.ts +++ b/x-pack/plugins/cases/server/telemetry/index.ts @@ -14,6 +14,7 @@ import type { import { SavedObjectsErrorHelpers } from '@kbn/core/server'; import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server'; import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; +import { FILE_SO_TYPE } from '@kbn/files-plugin/common'; import { collectTelemetryData } from './collect_telemetry_data'; import { CASE_TELEMETRY_SAVED_OBJECT, @@ -42,7 +43,7 @@ export const createCasesTelemetry = async ({ }: CreateCasesTelemetryArgs) => { const getInternalSavedObjectClient = async (): Promise => { const [coreStart] = await core.getStartServices(); - return coreStart.savedObjects.createInternalRepository(SAVED_OBJECT_TYPES); + return coreStart.savedObjects.createInternalRepository([...SAVED_OBJECT_TYPES, FILE_SO_TYPE]); }; taskManager.registerTaskDefinitions({ diff --git a/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts b/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts index 41cbfc2a12000..ca6049885b298 100644 --- a/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts +++ b/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts @@ -8,20 +8,25 @@ import type { SavedObjectsFindResponse } from '@kbn/core/server'; import { savedObjectsRepositoryMock, loggingSystemMock } from '@kbn/core/server/mocks'; import { ESCaseStatus } from '../../services/cases/types'; -import type { CaseAggregationResult } from '../types'; +import type { + AttachmentAggregationResult, + AttachmentFrameworkAggsResult, + CaseAggregationResult, + FileAttachmentAggregationResult, +} from '../types'; import { getCasesTelemetryData } from './cases'; +const MOCK_FIND_TOTAL = 5; +const SOLUTION_TOTAL = 1; + describe('getCasesTelemetryData', () => { describe('getCasesTelemetryData', () => { const logger = loggingSystemMock.createLogger(); const savedObjectsClient = savedObjectsRepositoryMock.create(); - const mockFind = ( - aggs: Record = {}, - so: SavedObjectsFindResponse['saved_objects'] = [] - ) => { + const mockFind = (aggs: object, so: SavedObjectsFindResponse['saved_objects'] = []) => { savedObjectsClient.find.mockResolvedValueOnce({ - total: 5, + total: MOCK_FIND_TOTAL, saved_objects: so, per_page: 1, page: 1, @@ -103,22 +108,100 @@ describe('getCasesTelemetryData', () => { buckets: [ { key: 'observability', - doc_count: 1, + doc_count: SOLUTION_TOTAL, }, { key: 'securitySolution', - doc_count: 1, + doc_count: SOLUTION_TOTAL, }, { key: 'cases', - doc_count: 1, + doc_count: SOLUTION_TOTAL, }, ], }, }; + const attachmentFramework: AttachmentFrameworkAggsResult = { + externalReferenceTypes: { + buckets: [ + { + doc_count: 5, + key: '.osquery', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + { + doc_count: 5, + key: '.files', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + ], + }, + persistableReferenceTypes: { + buckets: [ + { + doc_count: 5, + key: '.ml', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + { + doc_count: 5, + key: '.files', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + ], + }, + }; + + const attachmentAggsResult: AttachmentAggregationResult = { + securitySolution: { ...attachmentFramework }, + observability: { ...attachmentFramework }, + cases: { ...attachmentFramework }, + participants: { + value: 2, + }, + ...attachmentFramework, + }; + + const filesRes: FileAttachmentAggregationResult = { + securitySolution: { + averageSize: 500, + }, + observability: { + averageSize: 500, + }, + cases: { + averageSize: 500, + }, + averageSize: 500, + }; + mockFind(caseAggsResult); - mockFind({ participants: { value: 2 } }); + mockFind(attachmentAggsResult); mockFind({ references: { referenceType: { referenceAgg: { value: 3 } } } }); mockFind({ references: { referenceType: { referenceAgg: { value: 4 } } } }); mockSavedObjectResponse({ @@ -130,6 +213,7 @@ describe('getCasesTelemetryData', () => { mockSavedObjectResponse({ closed_at: '2022-03-08T12:24:11.429Z', }); + mockFind(filesRes); }; beforeEach(() => { @@ -139,10 +223,62 @@ describe('getCasesTelemetryData', () => { it('it returns the correct res', async () => { mockResponse(); + const attachmentFramework = (total: number, average: number) => { + return { + attachmentFramework: { + externalAttachments: [ + { + average, + maxOnACase: 10, + total, + type: '.osquery', + }, + { + average, + maxOnACase: 10, + total, + type: '.files', + }, + ], + persistableAttachments: [ + { + average, + maxOnACase: 10, + total, + type: '.ml', + }, + { + average, + maxOnACase: 10, + total, + type: '.files', + }, + ], + files: { + averageSize: 500, + average, + maxOnACase: 10, + total, + }, + }, + }; + }; + const res = await getCasesTelemetryData({ savedObjectsClient, logger }); + + const allAttachmentsTotal = 5; + const allAttachmentsAverage = allAttachmentsTotal / MOCK_FIND_TOTAL; + + const solutionAttachmentsTotal = 5; + const solutionAttachmentsAverage = solutionAttachmentsTotal / SOLUTION_TOTAL; + const solutionAttachmentFrameworkStats = attachmentFramework( + solutionAttachmentsTotal, + solutionAttachmentsAverage + ); + expect(res).toEqual({ all: { - total: 5, + total: MOCK_FIND_TOTAL, daily: 3, weekly: 2, monthly: 1, @@ -168,6 +304,7 @@ describe('getCasesTelemetryData', () => { totalWithZero: 100, totalWithAtLeastOne: 0, }, + ...attachmentFramework(allAttachmentsTotal, allAttachmentsAverage), }, main: { assignees: { @@ -175,6 +312,7 @@ describe('getCasesTelemetryData', () => { totalWithZero: 100, totalWithAtLeastOne: 0, }, + ...solutionAttachmentFrameworkStats, total: 1, daily: 3, weekly: 2, @@ -186,6 +324,7 @@ describe('getCasesTelemetryData', () => { totalWithZero: 100, totalWithAtLeastOne: 0, }, + ...solutionAttachmentFrameworkStats, total: 1, daily: 3, weekly: 2, @@ -197,6 +336,7 @@ describe('getCasesTelemetryData', () => { totalWithZero: 100, totalWithAtLeastOne: 0, }, + ...solutionAttachmentFrameworkStats, total: 1, daily: 3, weekly: 2, @@ -468,18 +608,311 @@ describe('getCasesTelemetryData', () => { } `); - expect(savedObjectsClient.find.mock.calls[1][0]).toEqual({ - aggs: { - participants: { - cardinality: { - field: 'cases-comments.attributes.created_by.username', + expect(savedObjectsClient.find.mock.calls[1][0]).toMatchInlineSnapshot(` + Object { + "aggs": Object { + "cases": Object { + "aggs": Object { + "externalReferenceTypes": Object { + "aggs": Object { + "references": Object { + "aggregations": Object { + "cases": Object { + "aggregations": Object { + "ids": Object { + "terms": Object { + "field": "cases-comments.references.id", + }, + }, + "max": Object { + "max_bucket": Object { + "buckets_path": "ids._count", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.references.type": "cases", + }, + }, + }, + }, + "nested": Object { + "path": "cases-comments.references", + }, + }, + }, + "terms": Object { + "field": "cases-comments.attributes.externalReferenceAttachmentTypeId", + }, + }, + "persistableReferenceTypes": Object { + "aggs": Object { + "references": Object { + "aggregations": Object { + "cases": Object { + "aggregations": Object { + "ids": Object { + "terms": Object { + "field": "cases-comments.references.id", + }, + }, + "max": Object { + "max_bucket": Object { + "buckets_path": "ids._count", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.references.type": "cases", + }, + }, + }, + }, + "nested": Object { + "path": "cases-comments.references", + }, + }, + }, + "terms": Object { + "field": "cases-comments.attributes.persistableStateAttachmentTypeId", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.attributes.owner": "cases", + }, + }, + }, + "externalReferenceTypes": Object { + "aggs": Object { + "references": Object { + "aggregations": Object { + "cases": Object { + "aggregations": Object { + "ids": Object { + "terms": Object { + "field": "cases-comments.references.id", + }, + }, + "max": Object { + "max_bucket": Object { + "buckets_path": "ids._count", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.references.type": "cases", + }, + }, + }, + }, + "nested": Object { + "path": "cases-comments.references", + }, + }, + }, + "terms": Object { + "field": "cases-comments.attributes.externalReferenceAttachmentTypeId", + }, + }, + "observability": Object { + "aggs": Object { + "externalReferenceTypes": Object { + "aggs": Object { + "references": Object { + "aggregations": Object { + "cases": Object { + "aggregations": Object { + "ids": Object { + "terms": Object { + "field": "cases-comments.references.id", + }, + }, + "max": Object { + "max_bucket": Object { + "buckets_path": "ids._count", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.references.type": "cases", + }, + }, + }, + }, + "nested": Object { + "path": "cases-comments.references", + }, + }, + }, + "terms": Object { + "field": "cases-comments.attributes.externalReferenceAttachmentTypeId", + }, + }, + "persistableReferenceTypes": Object { + "aggs": Object { + "references": Object { + "aggregations": Object { + "cases": Object { + "aggregations": Object { + "ids": Object { + "terms": Object { + "field": "cases-comments.references.id", + }, + }, + "max": Object { + "max_bucket": Object { + "buckets_path": "ids._count", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.references.type": "cases", + }, + }, + }, + }, + "nested": Object { + "path": "cases-comments.references", + }, + }, + }, + "terms": Object { + "field": "cases-comments.attributes.persistableStateAttachmentTypeId", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.attributes.owner": "observability", + }, + }, + }, + "participants": Object { + "cardinality": Object { + "field": "cases-comments.attributes.created_by.username", + }, + }, + "persistableReferenceTypes": Object { + "aggs": Object { + "references": Object { + "aggregations": Object { + "cases": Object { + "aggregations": Object { + "ids": Object { + "terms": Object { + "field": "cases-comments.references.id", + }, + }, + "max": Object { + "max_bucket": Object { + "buckets_path": "ids._count", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.references.type": "cases", + }, + }, + }, + }, + "nested": Object { + "path": "cases-comments.references", + }, + }, + }, + "terms": Object { + "field": "cases-comments.attributes.persistableStateAttachmentTypeId", + }, + }, + "securitySolution": Object { + "aggs": Object { + "externalReferenceTypes": Object { + "aggs": Object { + "references": Object { + "aggregations": Object { + "cases": Object { + "aggregations": Object { + "ids": Object { + "terms": Object { + "field": "cases-comments.references.id", + }, + }, + "max": Object { + "max_bucket": Object { + "buckets_path": "ids._count", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.references.type": "cases", + }, + }, + }, + }, + "nested": Object { + "path": "cases-comments.references", + }, + }, + }, + "terms": Object { + "field": "cases-comments.attributes.externalReferenceAttachmentTypeId", + }, + }, + "persistableReferenceTypes": Object { + "aggs": Object { + "references": Object { + "aggregations": Object { + "cases": Object { + "aggregations": Object { + "ids": Object { + "terms": Object { + "field": "cases-comments.references.id", + }, + }, + "max": Object { + "max_bucket": Object { + "buckets_path": "ids._count", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.references.type": "cases", + }, + }, + }, + }, + "nested": Object { + "path": "cases-comments.references", + }, + }, + }, + "terms": Object { + "field": "cases-comments.attributes.persistableStateAttachmentTypeId", + }, + }, + }, + "filter": Object { + "term": Object { + "cases-comments.attributes.owner": "securitySolution", + }, + }, }, }, - }, - page: 0, - perPage: 0, - type: 'cases-comments', - }); + "page": 0, + "perPage": 0, + "type": "cases-comments", + } + `); expect(savedObjectsClient.find.mock.calls[2][0]).toEqual({ aggs: { @@ -582,6 +1015,78 @@ describe('getCasesTelemetryData', () => { type: 'cases', }); } + + expect(savedObjectsClient.find.mock.calls[7][0]).toMatchInlineSnapshot(` + Object { + "aggs": Object { + "averageSize": Object { + "avg": Object { + "field": "file.attributes.size", + }, + }, + "cases": Object { + "aggs": Object { + "averageSize": Object { + "avg": Object { + "field": "file.attributes.size", + }, + }, + }, + "filter": Object { + "term": Object { + "file.attributes.Meta.owner": "cases", + }, + }, + }, + "observability": Object { + "aggs": Object { + "averageSize": Object { + "avg": Object { + "field": "file.attributes.size", + }, + }, + }, + "filter": Object { + "term": Object { + "file.attributes.Meta.owner": "observability", + }, + }, + }, + "securitySolution": Object { + "aggs": Object { + "averageSize": Object { + "avg": Object { + "field": "file.attributes.size", + }, + }, + }, + "filter": Object { + "term": Object { + "file.attributes.Meta.owner": "securitySolution", + }, + }, + }, + }, + "filter": Object { + "arguments": Array [ + Object { + "isQuoted": false, + "type": "literal", + "value": "file.attributes.Meta.caseId", + }, + Object { + "type": "wildcard", + "value": "@kuery-wildcard@", + }, + ], + "function": "is", + "type": "function", + }, + "page": 0, + "perPage": 0, + "type": "file", + } + `); }); }); }); diff --git a/x-pack/plugins/cases/server/telemetry/queries/cases.ts b/x-pack/plugins/cases/server/telemetry/queries/cases.ts index 128c0a09b5554..0e999721ae105 100644 --- a/x-pack/plugins/cases/server/telemetry/queries/cases.ts +++ b/x-pack/plugins/cases/server/telemetry/queries/cases.ts @@ -5,28 +5,33 @@ * 2.0. */ +import type { ISavedObjectsRepository, SavedObjectsFindResponse } from '@kbn/core/server'; +import { FILE_SO_TYPE } from '@kbn/files-plugin/common'; +import { fromKueryExpression } from '@kbn/es-query'; import { CASE_COMMENT_SAVED_OBJECT, CASE_SAVED_OBJECT, CASE_USER_ACTION_SAVED_OBJECT, + OWNERS, } from '../../../common/constants'; import { ESCaseStatus } from '../../services/cases/types'; import type { ESCaseAttributes } from '../../services/cases/types'; -import { OWNERS } from '../constants'; import type { CollectTelemetryDataParams, - Buckets, CasesTelemetry, - Cardinality, ReferencesAggregation, LatestDates, CaseAggregationResult, + AttachmentAggregationResult, + FileAttachmentAggregationResult, } from '../types'; import { findValueInBuckets, getAggregationsBuckets, + getAttachmentsFrameworkStats, getCountsAggregationQuery, getCountsFromBuckets, + getMaxBucketOnCaseAggregationQuery, getOnlyAlertsCommentsFilter, getOnlyConnectorsFilter, getReferencesAggregationQuery, @@ -52,9 +57,9 @@ export const getLatestCasesDates = async ({ ]); return { - createdAt: savedObjects?.[0].saved_objects?.[0].attributes?.created_at ?? null, - updatedAt: savedObjects?.[1].saved_objects?.[0].attributes?.updated_at ?? null, - closedAt: savedObjects?.[2].saved_objects?.[0].attributes?.closed_at ?? null, + createdAt: savedObjects?.[0]?.saved_objects?.[0]?.attributes?.created_at ?? '', + updatedAt: savedObjects?.[1]?.saved_objects?.[0]?.attributes?.updated_at ?? '', + closedAt: savedObjects?.[2]?.saved_objects?.[0]?.attributes?.closed_at ?? '', }; }; @@ -62,7 +67,84 @@ export const getCasesTelemetryData = async ({ savedObjectsClient, logger, }: CollectTelemetryDataParams): Promise => { - const byOwnerAggregationQuery = OWNERS.reduce( + try { + const [casesRes, commentsRes, totalAlertsRes, totalConnectorsRes, latestDates, filesRes] = + await Promise.all([ + getCasesSavedObjectTelemetry(savedObjectsClient), + getCommentsSavedObjectTelemetry(savedObjectsClient), + getAlertsTelemetry(savedObjectsClient), + getConnectorsTelemetry(savedObjectsClient), + getLatestCasesDates({ savedObjectsClient, logger }), + getFilesTelemetry(savedObjectsClient), + ]); + + const aggregationsBuckets = getAggregationsBuckets({ + aggs: casesRes.aggregations, + keys: ['counts', 'syncAlerts', 'status', 'users', 'totalAssignees'], + }); + + const allAttachmentFrameworkStats = getAttachmentsFrameworkStats({ + attachmentAggregations: commentsRes.aggregations, + totalCasesForOwner: casesRes.total, + filesAggregations: filesRes.aggregations, + }); + + return { + all: { + total: casesRes.total, + ...getCountsFromBuckets(aggregationsBuckets.counts), + status: { + open: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.OPEN), + inProgress: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.IN_PROGRESS), + closed: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.CLOSED), + }, + syncAlertsOn: findValueInBuckets(aggregationsBuckets.syncAlerts, 1), + syncAlertsOff: findValueInBuckets(aggregationsBuckets.syncAlerts, 0), + totalUsers: casesRes.aggregations?.users?.value ?? 0, + totalParticipants: commentsRes.aggregations?.participants?.value ?? 0, + totalTags: casesRes.aggregations?.tags?.value ?? 0, + totalWithAlerts: + totalAlertsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0, + totalWithConnectors: + totalConnectorsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0, + latestDates, + assignees: { + total: casesRes.aggregations?.totalAssignees.value ?? 0, + totalWithZero: casesRes.aggregations?.assigneeFilters.buckets.zero.doc_count ?? 0, + totalWithAtLeastOne: + casesRes.aggregations?.assigneeFilters.buckets.atLeastOne.doc_count ?? 0, + }, + ...allAttachmentFrameworkStats, + }, + sec: getSolutionValues({ + caseAggregations: casesRes.aggregations, + attachmentAggregations: commentsRes.aggregations, + filesAggregations: filesRes.aggregations, + owner: 'securitySolution', + }), + obs: getSolutionValues({ + caseAggregations: casesRes.aggregations, + attachmentAggregations: commentsRes.aggregations, + filesAggregations: filesRes.aggregations, + owner: 'observability', + }), + main: getSolutionValues({ + caseAggregations: casesRes.aggregations, + attachmentAggregations: commentsRes.aggregations, + filesAggregations: filesRes.aggregations, + owner: 'cases', + }), + }; + } catch (error) { + logger.error(`Cases telemetry failed with error: ${error}`); + throw error; + } +}; + +const getCasesSavedObjectTelemetry = async ( + savedObjectsClient: ISavedObjectsRepository +): Promise> => { + const caseByOwnerAggregationQuery = OWNERS.reduce( (aggQuery, owner) => ({ ...aggQuery, [owner]: { @@ -80,12 +162,12 @@ export const getCasesTelemetryData = async ({ {} ); - const casesRes = await savedObjectsClient.find({ + return savedObjectsClient.find({ page: 0, perPage: 0, type: CASE_SAVED_OBJECT, aggs: { - ...byOwnerAggregationQuery, + ...caseByOwnerAggregationQuery, ...getCountsAggregationQuery(CASE_SAVED_OBJECT), ...getAssigneesAggregations(), totalsByOwner: { @@ -111,17 +193,86 @@ export const getCasesTelemetryData = async ({ }, }, }); +}; + +const getAssigneesAggregations = () => ({ + totalAssignees: { + value_count: { + field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`, + }, + }, + assigneeFilters: { + filters: { + filters: { + zero: { + bool: { + must_not: { + exists: { + field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`, + }, + }, + }, + }, + atLeastOne: { + bool: { + filter: { + exists: { + field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`, + }, + }, + }, + }, + }, + }, + }, +}); + +const getCommentsSavedObjectTelemetry = async ( + savedObjectsClient: ISavedObjectsRepository +): Promise> => { + const attachmentRegistries = () => ({ + externalReferenceTypes: { + terms: { + field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.externalReferenceAttachmentTypeId`, + }, + aggs: { + ...getMaxBucketOnCaseAggregationQuery(CASE_COMMENT_SAVED_OBJECT), + }, + }, + persistableReferenceTypes: { + terms: { + field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.persistableStateAttachmentTypeId`, + }, + aggs: { + ...getMaxBucketOnCaseAggregationQuery(CASE_COMMENT_SAVED_OBJECT), + }, + }, + }); + + const attachmentsByOwnerAggregationQuery = OWNERS.reduce( + (aggQuery, owner) => ({ + ...aggQuery, + [owner]: { + filter: { + term: { + [`${CASE_COMMENT_SAVED_OBJECT}.attributes.owner`]: owner, + }, + }, + aggs: { + ...attachmentRegistries(), + }, + }, + }), + {} + ); - const commentsRes = await savedObjectsClient.find< - unknown, - Record & { - participants: Cardinality; - } & ReferencesAggregation - >({ + return savedObjectsClient.find({ page: 0, perPage: 0, type: CASE_COMMENT_SAVED_OBJECT, aggs: { + ...attachmentsByOwnerAggregationQuery, + ...attachmentRegistries(), participants: { cardinality: { field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.created_by.username`, @@ -129,8 +280,51 @@ export const getCasesTelemetryData = async ({ }, }, }); +}; + +const getFilesTelemetry = async ( + savedObjectsClient: ISavedObjectsRepository +): Promise> => { + const averageSize = () => ({ + averageSize: { + avg: { + field: `${FILE_SO_TYPE}.attributes.size`, + }, + }, + }); + + const filesByOwnerAggregationQuery = OWNERS.reduce( + (aggQuery, owner) => ({ + ...aggQuery, + [owner]: { + filter: { + term: { + [`${FILE_SO_TYPE}.attributes.Meta.owner`]: owner, + }, + }, + aggs: { + ...averageSize(), + }, + }, + }), + {} + ); + + const filterCaseIdExists = fromKueryExpression(`${FILE_SO_TYPE}.attributes.Meta.caseId: *`); - const totalAlertsRes = await savedObjectsClient.find({ + return savedObjectsClient.find({ + page: 0, + perPage: 0, + type: FILE_SO_TYPE, + filter: filterCaseIdExists, + aggs: { ...filesByOwnerAggregationQuery, ...averageSize() }, + }); +}; + +const getAlertsTelemetry = async ( + savedObjectsClient: ISavedObjectsRepository +): Promise> => { + return savedObjectsClient.find({ page: 0, perPage: 0, type: CASE_COMMENT_SAVED_OBJECT, @@ -143,8 +337,12 @@ export const getCasesTelemetryData = async ({ }), }, }); +}; - const totalConnectorsRes = await savedObjectsClient.find({ +const getConnectorsTelemetry = async ( + savedObjectsClient: ISavedObjectsRepository +): Promise> => { + return savedObjectsClient.find({ page: 0, perPage: 0, type: CASE_USER_ACTION_SAVED_OBJECT, @@ -157,74 +355,4 @@ export const getCasesTelemetryData = async ({ }), }, }); - - const latestDates = await getLatestCasesDates({ savedObjectsClient, logger }); - - const aggregationsBuckets = getAggregationsBuckets({ - aggs: casesRes.aggregations, - keys: ['counts', 'syncAlerts', 'status', 'users', 'totalAssignees'], - }); - - return { - all: { - total: casesRes.total, - ...getCountsFromBuckets(aggregationsBuckets.counts), - status: { - open: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.OPEN), - inProgress: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.IN_PROGRESS), - closed: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.CLOSED), - }, - syncAlertsOn: findValueInBuckets(aggregationsBuckets.syncAlerts, 1), - syncAlertsOff: findValueInBuckets(aggregationsBuckets.syncAlerts, 0), - totalUsers: casesRes.aggregations?.users?.value ?? 0, - totalParticipants: commentsRes.aggregations?.participants?.value ?? 0, - totalTags: casesRes.aggregations?.tags?.value ?? 0, - totalWithAlerts: - totalAlertsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0, - totalWithConnectors: - totalConnectorsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0, - latestDates, - assignees: { - total: casesRes.aggregations?.totalAssignees.value ?? 0, - totalWithZero: casesRes.aggregations?.assigneeFilters.buckets.zero.doc_count ?? 0, - totalWithAtLeastOne: - casesRes.aggregations?.assigneeFilters.buckets.atLeastOne.doc_count ?? 0, - }, - }, - sec: getSolutionValues(casesRes.aggregations, 'securitySolution'), - obs: getSolutionValues(casesRes.aggregations, 'observability'), - main: getSolutionValues(casesRes.aggregations, 'cases'), - }; }; - -const getAssigneesAggregations = () => ({ - totalAssignees: { - value_count: { - field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`, - }, - }, - assigneeFilters: { - filters: { - filters: { - zero: { - bool: { - must_not: { - exists: { - field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`, - }, - }, - }, - }, - atLeastOne: { - bool: { - filter: { - exists: { - field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`, - }, - }, - }, - }, - }, - }, - }, -}); diff --git a/x-pack/plugins/cases/server/telemetry/queries/utils.test.ts b/x-pack/plugins/cases/server/telemetry/queries/utils.test.ts index e466ba597108a..d7c6a0e9bf7b9 100644 --- a/x-pack/plugins/cases/server/telemetry/queries/utils.test.ts +++ b/x-pack/plugins/cases/server/telemetry/queries/utils.test.ts @@ -6,10 +6,16 @@ */ import { savedObjectsRepositoryMock } from '@kbn/core/server/mocks'; -import type { CaseAggregationResult } from '../types'; +import type { + AttachmentAggregationResult, + AttachmentFrameworkAggsResult, + CaseAggregationResult, + FileAttachmentAggregationResult, +} from '../types'; import { findValueInBuckets, getAggregationsBuckets, + getAttachmentsFrameworkStats, getBucketFromAggregation, getConnectorsCardinalityAggregationQuery, getCountsAggregationQuery, @@ -46,19 +52,19 @@ describe('utils', () => { totalAssignees: { value: 5 }, }; - const solutionValues = { + const caseSolutionValues = { counts, ...assignees, }; - const aggsResult: CaseAggregationResult = { + const caseAggsResult: CaseAggregationResult = { users: { value: 1 }, tags: { value: 2 }, ...assignees, counts, - securitySolution: { ...solutionValues }, - observability: { ...solutionValues }, - cases: { ...solutionValues }, + securitySolution: { ...caseSolutionValues }, + observability: { ...caseSolutionValues }, + cases: { ...caseSolutionValues }, syncAlerts: { buckets: [ { @@ -87,7 +93,7 @@ describe('utils', () => { }, { key: 'securitySolution', - doc_count: 1, + doc_count: 5, }, { key: 'cases', @@ -97,40 +103,247 @@ describe('utils', () => { }, }; + const attachmentFramework: AttachmentFrameworkAggsResult = { + externalReferenceTypes: { + buckets: [ + { + doc_count: 5, + key: '.osquery', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + { + doc_count: 5, + key: '.files', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + ], + }, + persistableReferenceTypes: { + buckets: [ + { + doc_count: 5, + key: '.ml', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + { + doc_count: 5, + key: '.files', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + ], + }, + }; + + const attachmentAggsResult: AttachmentAggregationResult = { + securitySolution: { ...attachmentFramework }, + observability: { ...attachmentFramework }, + cases: { ...attachmentFramework }, + participants: { + value: 5, + }, + ...attachmentFramework, + }; + + const filesRes: FileAttachmentAggregationResult = { + securitySolution: { + averageSize: 500, + }, + observability: { + averageSize: 500, + }, + cases: { + averageSize: 500, + }, + averageSize: 500, + }; + it('constructs the solution values correctly', () => { - expect(getSolutionValues(aggsResult, 'securitySolution')).toMatchInlineSnapshot(` + expect( + getSolutionValues({ + caseAggregations: caseAggsResult, + attachmentAggregations: attachmentAggsResult, + filesAggregations: filesRes, + owner: 'securitySolution', + }) + ).toMatchInlineSnapshot(` Object { "assignees": Object { "total": 5, "totalWithAtLeastOne": 0, "totalWithZero": 100, }, + "attachmentFramework": Object { + "externalAttachments": Array [ + Object { + "average": 1, + "maxOnACase": 10, + "total": 5, + "type": ".osquery", + }, + Object { + "average": 1, + "maxOnACase": 10, + "total": 5, + "type": ".files", + }, + ], + "files": Object { + "average": 1, + "averageSize": 500, + "maxOnACase": 10, + "total": 5, + }, + "persistableAttachments": Array [ + Object { + "average": 1, + "maxOnACase": 10, + "total": 5, + "type": ".ml", + }, + Object { + "average": 1, + "maxOnACase": 10, + "total": 5, + "type": ".files", + }, + ], + }, "daily": 3, "monthly": 1, - "total": 1, + "total": 5, "weekly": 2, } `); - expect(getSolutionValues(aggsResult, 'cases')).toMatchInlineSnapshot(` + expect( + getSolutionValues({ + caseAggregations: caseAggsResult, + attachmentAggregations: attachmentAggsResult, + filesAggregations: filesRes, + owner: 'cases', + }) + ).toMatchInlineSnapshot(` Object { "assignees": Object { "total": 5, "totalWithAtLeastOne": 0, "totalWithZero": 100, }, + "attachmentFramework": Object { + "externalAttachments": Array [ + Object { + "average": 5, + "maxOnACase": 10, + "total": 5, + "type": ".osquery", + }, + Object { + "average": 5, + "maxOnACase": 10, + "total": 5, + "type": ".files", + }, + ], + "files": Object { + "average": 5, + "averageSize": 500, + "maxOnACase": 10, + "total": 5, + }, + "persistableAttachments": Array [ + Object { + "average": 5, + "maxOnACase": 10, + "total": 5, + "type": ".ml", + }, + Object { + "average": 5, + "maxOnACase": 10, + "total": 5, + "type": ".files", + }, + ], + }, "daily": 3, "monthly": 1, "total": 1, "weekly": 2, } `); - expect(getSolutionValues(aggsResult, 'observability')).toMatchInlineSnapshot(` + expect( + getSolutionValues({ + caseAggregations: caseAggsResult, + attachmentAggregations: attachmentAggsResult, + filesAggregations: filesRes, + owner: 'observability', + }) + ).toMatchInlineSnapshot(` Object { "assignees": Object { "total": 5, "totalWithAtLeastOne": 0, "totalWithZero": 100, }, + "attachmentFramework": Object { + "externalAttachments": Array [ + Object { + "average": 5, + "maxOnACase": 10, + "total": 5, + "type": ".osquery", + }, + Object { + "average": 5, + "maxOnACase": 10, + "total": 5, + "type": ".files", + }, + ], + "files": Object { + "average": 5, + "averageSize": 500, + "maxOnACase": 10, + "total": 5, + }, + "persistableAttachments": Array [ + Object { + "average": 5, + "maxOnACase": 10, + "total": 5, + "type": ".ml", + }, + Object { + "average": 5, + "maxOnACase": 10, + "total": 5, + "type": ".files", + }, + ], + }, "daily": 3, "monthly": 1, "total": 1, @@ -140,6 +353,217 @@ describe('utils', () => { }); }); + describe('getAttachmentsFrameworkStats', () => { + it('returns empty stats if the aggregation is undefined', () => { + expect(getAttachmentsFrameworkStats({ totalCasesForOwner: 0 })).toMatchInlineSnapshot(` + Object { + "attachmentFramework": Object { + "externalAttachments": Array [], + "files": Object { + "average": 0, + "averageSize": 0, + "maxOnACase": 0, + "total": 0, + }, + "persistableAttachments": Array [], + }, + } + `); + }); + + describe('externalAttachments', () => { + const attachmentFramework: AttachmentFrameworkAggsResult = { + externalReferenceTypes: { + buckets: [ + { + doc_count: 5, + key: '.osquery', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + { + doc_count: 10, + key: '.files', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + ], + }, + persistableReferenceTypes: { + buckets: [], + }, + }; + + it('populates the externalAttachments array', () => { + const stats = getAttachmentsFrameworkStats({ + attachmentAggregations: attachmentFramework, + totalCasesForOwner: 5, + }); + + expect(stats.attachmentFramework.externalAttachments[0]).toEqual({ + // the average is 5 from the aggs result / 5 from the function parameter + average: 1, + maxOnACase: 10, + total: 5, + type: '.osquery', + }); + + expect(stats.attachmentFramework.externalAttachments[1]).toEqual({ + // the average is 10 from the aggs result / 5 from the function parameter + average: 2, + maxOnACase: 10, + total: 10, + type: '.files', + }); + }); + }); + + describe('persistableAttachments', () => { + const attachmentFramework: AttachmentFrameworkAggsResult = { + persistableReferenceTypes: { + buckets: [ + { + doc_count: 5, + key: '.osquery', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + { + doc_count: 10, + key: '.files', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + ], + }, + externalReferenceTypes: { + buckets: [], + }, + }; + + it('populates the externalAttachments array', () => { + const stats = getAttachmentsFrameworkStats({ + attachmentAggregations: attachmentFramework, + totalCasesForOwner: 5, + }); + + expect(stats.attachmentFramework.persistableAttachments[0]).toEqual({ + // the average is 5 from the aggs result / 5 from the function parameter + average: 1, + maxOnACase: 10, + total: 5, + type: '.osquery', + }); + + expect(stats.attachmentFramework.persistableAttachments[1]).toEqual({ + // the average is 10 from the aggs result / 5 from the function parameter + average: 2, + maxOnACase: 10, + total: 10, + type: '.files', + }); + }); + }); + + describe('files', () => { + it('sets the files stats to empty when it cannot find a files entry', () => { + const attachmentFramework: AttachmentFrameworkAggsResult = { + externalReferenceTypes: { + buckets: [ + { + doc_count: 5, + key: '.osquery', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + ], + }, + persistableReferenceTypes: { + buckets: [], + }, + }; + + expect( + getAttachmentsFrameworkStats({ + attachmentAggregations: attachmentFramework, + totalCasesForOwner: 5, + filesAggregations: { averageSize: 500 }, + }).attachmentFramework.files + ).toMatchInlineSnapshot(` + Object { + "average": 0, + "averageSize": 0, + "maxOnACase": 0, + "total": 0, + } + `); + }); + + it('sets the files stats when it finds a files entry', () => { + const attachmentFramework: AttachmentFrameworkAggsResult = { + externalReferenceTypes: { + buckets: [ + { + doc_count: 5, + key: '.files', + references: { + cases: { + max: { + value: 10, + }, + }, + }, + }, + ], + }, + persistableReferenceTypes: { + buckets: [], + }, + }; + + expect( + getAttachmentsFrameworkStats({ + attachmentAggregations: attachmentFramework, + filesAggregations: { averageSize: 500 }, + totalCasesForOwner: 5, + }).attachmentFramework.files + ).toMatchInlineSnapshot(` + Object { + "average": 1, + "averageSize": 500, + "maxOnACase": 10, + "total": 5, + } + `); + }); + }); + }); + describe('getCountsAggregationQuery', () => { it('returns the correct query', () => { expect(getCountsAggregationQuery('test')).toEqual({ diff --git a/x-pack/plugins/cases/server/telemetry/queries/utils.ts b/x-pack/plugins/cases/server/telemetry/queries/utils.ts index 82bdef3ebe825..0c0a4f7bbf87d 100644 --- a/x-pack/plugins/cases/server/telemetry/queries/utils.ts +++ b/x-pack/plugins/cases/server/telemetry/queries/utils.ts @@ -16,12 +16,20 @@ import { import type { CaseAggregationResult, Buckets, - CasesTelemetry, MaxBucketOnCaseAggregation, SolutionTelemetry, + AttachmentFramework, + AttachmentAggregationResult, + BucketsWithMaxOnCase, + AttachmentStats, + FileAttachmentStats, + FileAttachmentAggregationResult, + FileAttachmentAverageSize, + AttachmentFrameworkAggsResult, } from '../types'; import { buildFilter } from '../../client/utils'; -import type { OWNERS } from '../constants'; +import type { Owner } from '../../../common/constants/types'; +import { FILE_ATTACHMENT_TYPE } from '../../../common/api'; export const getCountsAggregationQuery = (savedObjectType: string) => ({ counts: { @@ -154,22 +162,39 @@ export const getBucketFromAggregation = ({ aggs?: Record; }): Buckets['buckets'] => (get(aggs, `${key}.buckets`) ?? []) as Buckets['buckets']; -export const getSolutionValues = ( - aggregations: CaseAggregationResult | undefined, - owner: typeof OWNERS[number] -): SolutionTelemetry => { +export const getSolutionValues = ({ + caseAggregations, + attachmentAggregations, + filesAggregations, + owner, +}: { + caseAggregations?: CaseAggregationResult; + attachmentAggregations?: AttachmentAggregationResult; + filesAggregations?: FileAttachmentAggregationResult; + owner: Owner; +}): SolutionTelemetry => { const aggregationsBuckets = getAggregationsBuckets({ - aggs: aggregations, + aggs: caseAggregations, keys: ['totalsByOwner', 'securitySolution.counts', 'observability.counts', 'cases.counts'], }); + const totalCasesForOwner = findValueInBuckets(aggregationsBuckets.totalsByOwner, owner); + const attachmentsAggsForOwner = attachmentAggregations?.[owner]; + const fileAttachmentsForOwner = filesAggregations?.[owner]; + return { - total: findValueInBuckets(aggregationsBuckets.totalsByOwner, owner), + total: totalCasesForOwner, ...getCountsFromBuckets(aggregationsBuckets[`${owner}.counts`]), + ...getAttachmentsFrameworkStats({ + attachmentAggregations: attachmentsAggsForOwner, + filesAggregations: fileAttachmentsForOwner, + totalCasesForOwner, + }), assignees: { - total: aggregations?.[owner].totalAssignees.value ?? 0, - totalWithZero: aggregations?.[owner].assigneeFilters.buckets.zero.doc_count ?? 0, - totalWithAtLeastOne: aggregations?.[owner].assigneeFilters.buckets.atLeastOne.doc_count ?? 0, + total: caseAggregations?.[owner].totalAssignees.value ?? 0, + totalWithZero: caseAggregations?.[owner].assigneeFilters.buckets.zero.doc_count ?? 0, + totalWithAtLeastOne: + caseAggregations?.[owner].assigneeFilters.buckets.atLeastOne.doc_count ?? 0, }, }; }; @@ -192,6 +217,92 @@ export const getAggregationsBuckets = ({ {} ); +export const getAttachmentsFrameworkStats = ({ + attachmentAggregations, + filesAggregations, + totalCasesForOwner, +}: { + attachmentAggregations?: AttachmentFrameworkAggsResult; + filesAggregations?: FileAttachmentAverageSize; + totalCasesForOwner: number; +}): AttachmentFramework => { + if (!attachmentAggregations) { + return emptyAttachmentFramework(); + } + const averageFileSize = filesAggregations?.averageSize; + + return { + attachmentFramework: { + externalAttachments: getAttachmentRegistryStats( + attachmentAggregations.externalReferenceTypes, + totalCasesForOwner + ), + persistableAttachments: getAttachmentRegistryStats( + attachmentAggregations.persistableReferenceTypes, + totalCasesForOwner + ), + files: getFileAttachmentStats({ + registryResults: attachmentAggregations.externalReferenceTypes, + averageFileSize, + totalCasesForOwner, + }), + }, + }; +}; + +const getAttachmentRegistryStats = ( + registryResults: BucketsWithMaxOnCase, + totalCasesForOwner: number +): AttachmentStats[] => { + const stats: AttachmentStats[] = []; + + for (const bucket of registryResults.buckets) { + const commonFields = { + average: calculateTypePerCaseAverage(bucket.doc_count, totalCasesForOwner), + maxOnACase: bucket.references.cases.max.value, + total: bucket.doc_count, + }; + + stats.push({ + type: bucket.key, + ...commonFields, + }); + } + + return stats; +}; + +const calculateTypePerCaseAverage = (typeDocCount: number, totalCases: number) => { + if (totalCases === 0) { + return 0; + } + + return Math.round(typeDocCount / totalCases); +}; + +const getFileAttachmentStats = ({ + registryResults, + averageFileSize, + totalCasesForOwner, +}: { + registryResults: BucketsWithMaxOnCase; + averageFileSize?: number; + totalCasesForOwner: number; +}): FileAttachmentStats => { + const fileBucket = registryResults.buckets.find((bucket) => bucket.key === FILE_ATTACHMENT_TYPE); + + if (!fileBucket || averageFileSize == null) { + return emptyFileAttachment(); + } + + return { + averageSize: averageFileSize, + average: calculateTypePerCaseAverage(fileBucket.doc_count, totalCasesForOwner), + maxOnACase: fileBucket.references.cases.max.value, + total: fileBucket.doc_count, + }; +}; + export const getOnlyAlertsCommentsFilter = () => buildFilter({ filters: ['alert'], @@ -208,81 +319,17 @@ export const getOnlyConnectorsFilter = () => type: CASE_USER_ACTION_SAVED_OBJECT, }); -export const getTelemetryDataEmptyState = (): CasesTelemetry => ({ - cases: { - all: { - assignees: { - total: 0, - totalWithZero: 0, - totalWithAtLeastOne: 0, - }, - total: 0, - monthly: 0, - weekly: 0, - daily: 0, - status: { - open: 0, - inProgress: 0, - closed: 0, - }, - syncAlertsOn: 0, - syncAlertsOff: 0, - totalUsers: 0, - totalParticipants: 0, - totalTags: 0, - totalWithAlerts: 0, - totalWithConnectors: 0, - latestDates: { - createdAt: null, - updatedAt: null, - closedAt: null, - }, - }, - sec: { - total: 0, - monthly: 0, - weekly: 0, - daily: 0, - assignees: { total: 0, totalWithAtLeastOne: 0, totalWithZero: 0 }, - }, - obs: { - total: 0, - monthly: 0, - weekly: 0, - daily: 0, - assignees: { total: 0, totalWithAtLeastOne: 0, totalWithZero: 0 }, - }, - main: { - total: 0, - monthly: 0, - weekly: 0, - daily: 0, - assignees: { total: 0, totalWithAtLeastOne: 0, totalWithZero: 0 }, - }, - }, - userActions: { all: { total: 0, monthly: 0, weekly: 0, daily: 0, maxOnACase: 0 } }, - comments: { all: { total: 0, monthly: 0, weekly: 0, daily: 0, maxOnACase: 0 } }, - alerts: { all: { total: 0, monthly: 0, weekly: 0, daily: 0, maxOnACase: 0 } }, - connectors: { - all: { - all: { totalAttached: 0 }, - itsm: { totalAttached: 0 }, - sir: { totalAttached: 0 }, - jira: { totalAttached: 0 }, - resilient: { totalAttached: 0 }, - swimlane: { totalAttached: 0 }, - maxAttachedToACase: 0, - }, - }, - pushes: { - all: { total: 0, maxOnACase: 0 }, - }, - configuration: { - all: { - closure: { - manually: 0, - automatic: 0, - }, - }, +const emptyAttachmentFramework = (): AttachmentFramework => ({ + attachmentFramework: { + persistableAttachments: [], + externalAttachments: [], + files: emptyFileAttachment(), }, }); + +const emptyFileAttachment = (): FileAttachmentStats => ({ + average: 0, + averageSize: 0, + maxOnACase: 0, + total: 0, +}); diff --git a/x-pack/plugins/cases/server/telemetry/schema.ts b/x-pack/plugins/cases/server/telemetry/schema.ts index 1f51ca134b577..59506c932a51f 100644 --- a/x-pack/plugins/cases/server/telemetry/schema.ts +++ b/x-pack/plugins/cases/server/telemetry/schema.ts @@ -14,6 +14,8 @@ import type { TypeString, SolutionTelemetrySchema, AssigneesSchema, + AttachmentFrameworkSchema, + AttachmentItemsSchema, } from './types'; const long: TypeLong = { type: 'long' }; @@ -26,6 +28,32 @@ const countSchema: CountSchema = { daily: long, }; +interface AttachmentRegistrySchema { + type: 'array'; + items: AttachmentItemsSchema; +} + +const attachmentRegistrySchema: AttachmentRegistrySchema = { + type: 'array', + items: { + average: long, + maxOnACase: long, + total: long, + type: string, + }, +}; + +const attachmentFrameworkSchema: AttachmentFrameworkSchema = { + persistableAttachments: attachmentRegistrySchema, + externalAttachments: attachmentRegistrySchema, + files: { + average: long, + averageSize: long, + maxOnACase: long, + total: long, + }, +}; + const assigneesSchema: AssigneesSchema = { total: long, totalWithZero: long, @@ -35,6 +63,7 @@ const assigneesSchema: AssigneesSchema = { const solutionTelemetry: SolutionTelemetrySchema = { ...countSchema, assignees: assigneesSchema, + attachmentFramework: attachmentFrameworkSchema, }; const statusSchema: StatusSchema = { @@ -53,6 +82,7 @@ export const casesSchema: CasesTelemetrySchema = { cases: { all: { ...countSchema, + attachmentFramework: attachmentFrameworkSchema, assignees: assigneesSchema, status: statusSchema, syncAlertsOn: long, diff --git a/x-pack/plugins/cases/server/telemetry/types.ts b/x-pack/plugins/cases/server/telemetry/types.ts index 095b967d1addf..e28c3abaf4d52 100644 --- a/x-pack/plugins/cases/server/telemetry/types.ts +++ b/x-pack/plugins/cases/server/telemetry/types.ts @@ -7,7 +7,7 @@ import type { ISavedObjectsRepository, Logger } from '@kbn/core/server'; import type { MakeSchemaFrom } from '@kbn/usage-collection-plugin/server'; -import type { OWNERS } from './constants'; +import type { Owner } from '../../common/constants/types'; export interface Buckets { buckets: Array<{ @@ -57,8 +57,33 @@ export interface AssigneesFilters { }; } +export interface FileAttachmentAverageSize { + averageSize: number; +} + +export type FileAttachmentAggregationResult = Record & + FileAttachmentAverageSize; + +export interface BucketsWithMaxOnCase { + buckets: Array< + { + doc_count: number; + key: string; + } & MaxBucketOnCaseAggregation + >; +} + +export interface AttachmentFrameworkAggsResult { + externalReferenceTypes: BucketsWithMaxOnCase; + persistableReferenceTypes: BucketsWithMaxOnCase; +} + +export type AttachmentAggregationResult = Record & { + participants: Cardinality; +} & AttachmentFrameworkAggsResult; + export type CaseAggregationResult = Record< - typeof OWNERS[number], + Owner, { counts: Buckets; totalAssignees: ValueCount; @@ -81,7 +106,29 @@ export interface Assignees { totalWithAtLeastOne: number; } -export interface SolutionTelemetry extends Count { +interface CommonAttachmentStats { + average: number; + maxOnACase: number; + total: number; +} + +export interface AttachmentStats extends CommonAttachmentStats { + type: string; +} + +export interface FileAttachmentStats extends CommonAttachmentStats { + averageSize: number; +} + +export interface AttachmentFramework { + attachmentFramework: { + externalAttachments: AttachmentStats[]; + persistableAttachments: AttachmentStats[]; + files: FileAttachmentStats; + }; +} + +export interface SolutionTelemetry extends Count, AttachmentFramework { assignees: Assignees; } @@ -92,25 +139,26 @@ export interface Status { } export interface LatestDates { - createdAt: string | null; - updatedAt: string | null; - closedAt: string | null; + createdAt: string; + updatedAt: string; + closedAt: string; } export interface CasesTelemetry { cases: { - all: Count & { - assignees: Assignees; - status: Status; - syncAlertsOn: number; - syncAlertsOff: number; - totalUsers: number; - totalParticipants: number; - totalTags: number; - totalWithAlerts: number; - totalWithConnectors: number; - latestDates: LatestDates; - }; + all: Count & + AttachmentFramework & { + assignees: Assignees; + status: Status; + syncAlertsOn: number; + syncAlertsOff: number; + totalUsers: number; + totalParticipants: number; + totalTags: number; + totalWithAlerts: number; + totalWithConnectors: number; + latestDates: LatestDates; + }; sec: SolutionTelemetry; obs: SolutionTelemetry; main: SolutionTelemetry; @@ -147,4 +195,6 @@ export type StatusSchema = MakeSchemaFrom; export type LatestDatesSchema = MakeSchemaFrom; export type CasesTelemetrySchema = MakeSchemaFrom; export type AssigneesSchema = MakeSchemaFrom; +export type AttachmentFrameworkSchema = MakeSchemaFrom; +export type AttachmentItemsSchema = MakeSchemaFrom; export type SolutionTelemetrySchema = MakeSchemaFrom; diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 99e466ebbb357..fc3f0ac44d7f3 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -4783,6 +4783,64 @@ "daily": { "type": "long" }, + "attachmentFramework": { + "properties": { + "persistableAttachments": { + "type": "array", + "items": { + "properties": { + "average": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + }, + "type": { + "type": "keyword" + } + } + } + }, + "externalAttachments": { + "type": "array", + "items": { + "properties": { + "average": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + }, + "type": { + "type": "keyword" + } + } + } + }, + "files": { + "properties": { + "average": { + "type": "long" + }, + "averageSize": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } + }, "assignees": { "properties": { "total": { @@ -4871,6 +4929,64 @@ "type": "long" } } + }, + "attachmentFramework": { + "properties": { + "persistableAttachments": { + "type": "array", + "items": { + "properties": { + "average": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + }, + "type": { + "type": "keyword" + } + } + } + }, + "externalAttachments": { + "type": "array", + "items": { + "properties": { + "average": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + }, + "type": { + "type": "keyword" + } + } + } + }, + "files": { + "properties": { + "average": { + "type": "long" + }, + "averageSize": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } } } }, @@ -4900,6 +5016,64 @@ "type": "long" } } + }, + "attachmentFramework": { + "properties": { + "persistableAttachments": { + "type": "array", + "items": { + "properties": { + "average": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + }, + "type": { + "type": "keyword" + } + } + } + }, + "externalAttachments": { + "type": "array", + "items": { + "properties": { + "average": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + }, + "type": { + "type": "keyword" + } + } + } + }, + "files": { + "properties": { + "average": { + "type": "long" + }, + "averageSize": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } } } }, @@ -4929,6 +5103,64 @@ "type": "long" } } + }, + "attachmentFramework": { + "properties": { + "persistableAttachments": { + "type": "array", + "items": { + "properties": { + "average": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + }, + "type": { + "type": "keyword" + } + } + } + }, + "externalAttachments": { + "type": "array", + "items": { + "properties": { + "average": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + }, + "type": { + "type": "keyword" + } + } + } + }, + "files": { + "properties": { + "average": { + "type": "long" + }, + "averageSize": { + "type": "long" + }, + "maxOnACase": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } } } }