From f17fe1bd9d2e12b6056d879637e594c8850c8ded Mon Sep 17 00:00:00 2001
From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com>
Date: Fri, 24 Mar 2023 15:43:06 -0400
Subject: [PATCH] [Cases] Adding file telemetry (#152968)
This PR adds telemetry for the attachment framework and files
attachments by plugin and for all cases.
Issue: https://github.com/elastic/kibana/issues/151933
Notable changes:
- We now send information for each registered attachment type
- We send average file size
- We leverage the files saved object to aggregate this information
- We use an array type so that we don't have to hard code each
attachment type, they are dynamically discovered instead
- Stats for the attachments are done for all cases and broken down by
plugin
Example telemetry payload
```
"cases": {
"cases": {
"all": {
"total": 1,
"daily": 1,
"weekly": 1,
"monthly": 1,
"status": {
"open": 1,
"inProgress": 0,
"closed": 0
},
"syncAlertsOn": 1,
"syncAlertsOff": 0,
"totalUsers": 1,
"totalParticipants": 1,
"totalTags": 1,
"totalWithAlerts": 0,
"totalWithConnectors": 0,
"latestDates": {
"createdAt": "2023-03-09T15:20:46.399Z",
"updatedAt": "2023-03-09T15:21:02.399Z",
"closedAt": null
},
"assignees": {
"total": 0,
"totalWithZero": 1,
"totalWithAtLeastOne": 0
},
"attachmentFramework": {
"externalAttachments": [
{
"type": ".files",
"average": 4,
"maxOnACase": 4,
"total": 4
}
],
"persistableAttachments": [],
"files": {
"averageSize": {
"value": 3
},
"average": 4,
"maxOnACase": 4,
"total": 4
}
}
},
"sec": {
"total": 0,
"daily": 0,
"weekly": 0,
"monthly": 0,
"attachmentFramework": {
"externalAttachments": [],
"persistableAttachments": [],
"files": {
"average": 0,
"averageSize": 0,
"maxOnACase": 0,
"total": 0
}
},
"assignees": {
"total": 0,
"totalWithZero": 0,
"totalWithAtLeastOne": 0
}
},
"obs": {
"total": 0,
"daily": 0,
"weekly": 0,
"monthly": 0,
"attachmentFramework": {
"externalAttachments": [],
"persistableAttachments": [],
"files": {
"average": 0,
"averageSize": 0,
"maxOnACase": 0,
"total": 0
}
},
"assignees": {
"total": 0,
"totalWithZero": 0,
"totalWithAtLeastOne": 0
}
},
"main": {
"total": 1,
"daily": 1,
"weekly": 1,
"monthly": 1,
"attachmentFramework": {
"externalAttachments": [
{
"type": ".files",
"average": 4,
"maxOnACase": 4,
"total": 4
}
],
"persistableAttachments": [],
"files": {
"averageSize": {
"value": 3
},
"average": 4,
"maxOnACase": 4,
"total": 4
}
},
"assignees": {
"total": 0,
"totalWithZero": 1,
"totalWithAtLeastOne": 0
}
}
},
"userActions": {
"all": {
"total": 5,
"daily": 5,
"weekly": 5,
"monthly": 5,
"maxOnACase": 5
}
},
"comments": {
"all": {
"total": 0,
"daily": 0,
"weekly": 0,
"monthly": 0,
"maxOnACase": 0
}
},
"alerts": {
"all": {
"total": 0,
"daily": 0,
"weekly": 0,
"monthly": 0,
"maxOnACase": 0
}
},
"connectors": {
"all": {
"all": {
"totalAttached": 0
},
"itsm": {
"totalAttached": 0
},
"sir": {
"totalAttached": 0
},
"jira": {
"totalAttached": 0
},
"resilient": {
"totalAttached": 0
},
"swimlane": {
"totalAttached": 0
},
"maxAttachedToACase": 0
}
},
"pushes": {
"all": {
"total": 0,
"maxOnACase": 0
}
},
"configuration": {
"all": {
"closure": {
"manually": 0,
"automatic": 0
}
}
}
},
```
## Testing
To test modify this file:
https://github.com/elastic/kibana/blob/main/x-pack/plugins/cases/server/telemetry/schedule_telemetry_task.ts
With:
```
export const scheduleCasesTelemetryTask = (
taskManager: TaskManagerStartContract,
logger: Logger
) => {
(async () => {
await taskManager
.ensureScheduled({
id: CASES_TELEMETRY_TASK_NAME,
taskType: CASES_TELEMETRY_TASK_NAME,
schedule: {
interval: `${MINUTES_ON_HALF_DAY}m`,
},
scope: ['cases'],
params: {},
state: {},
})
.catch((err) => {
logger.debug(
`Error scheduling cases task with ID ${CASES_TELEMETRY_TASK_NAME} and type ${CASES_TELEMETRY_TASK_NAME}. Received ${err.message}`
);
});
await taskManager.runSoon(CASES_TELEMETRY_TASK_NAME);
})();
};
```
This will cause the telemetry to be sent as soon as the server is
restarted.
To generate files and attachments to add stats to the telemetry I
created this python script:
https://github.com/elastic/cases-files-generator
To retrieve the telemetry:
```
POST http://localhost:5601/api/telemetry/v2/clusters/_stats
{
"refreshCache": true,
"unencrypted": true
}
```
---
.../cases/server/telemetry/constants.ts | 13 -
.../plugins/cases/server/telemetry/index.ts | 3 +-
.../server/telemetry/queries/cases.test.ts | 547 +++++++++++++++++-
.../cases/server/telemetry/queries/cases.ts | 302 +++++++---
.../server/telemetry/queries/utils.test.ts | 446 +++++++++++++-
.../cases/server/telemetry/queries/utils.ts | 221 ++++---
.../plugins/cases/server/telemetry/schema.ts | 30 +
.../plugins/cases/server/telemetry/types.ts | 86 ++-
.../schema/xpack_plugins.json | 232 ++++++++
9 files changed, 1642 insertions(+), 238 deletions(-)
delete mode 100644 x-pack/plugins/cases/server/telemetry/constants.ts
diff --git a/x-pack/plugins/cases/server/telemetry/constants.ts b/x-pack/plugins/cases/server/telemetry/constants.ts
deleted file mode 100644
index 705321e3f1fa0..0000000000000
--- a/x-pack/plugins/cases/server/telemetry/constants.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-import { GENERAL_CASES_OWNER, OBSERVABILITY_OWNER, SECURITY_SOLUTION_OWNER } from '../../common';
-
-/**
- * This should only be used within telemetry
- */
-export const OWNERS = [OBSERVABILITY_OWNER, SECURITY_SOLUTION_OWNER, GENERAL_CASES_OWNER] as const;
diff --git a/x-pack/plugins/cases/server/telemetry/index.ts b/x-pack/plugins/cases/server/telemetry/index.ts
index 6cd796eca2c18..7ef14541ced09 100644
--- a/x-pack/plugins/cases/server/telemetry/index.ts
+++ b/x-pack/plugins/cases/server/telemetry/index.ts
@@ -14,6 +14,7 @@ import type {
import { SavedObjectsErrorHelpers } from '@kbn/core/server';
import type { TaskManagerSetupContract } from '@kbn/task-manager-plugin/server';
import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
+import { FILE_SO_TYPE } from '@kbn/files-plugin/common';
import { collectTelemetryData } from './collect_telemetry_data';
import {
CASE_TELEMETRY_SAVED_OBJECT,
@@ -42,7 +43,7 @@ export const createCasesTelemetry = async ({
}: CreateCasesTelemetryArgs) => {
const getInternalSavedObjectClient = async (): Promise => {
const [coreStart] = await core.getStartServices();
- return coreStart.savedObjects.createInternalRepository(SAVED_OBJECT_TYPES);
+ return coreStart.savedObjects.createInternalRepository([...SAVED_OBJECT_TYPES, FILE_SO_TYPE]);
};
taskManager.registerTaskDefinitions({
diff --git a/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts b/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts
index 41cbfc2a12000..ca6049885b298 100644
--- a/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts
+++ b/x-pack/plugins/cases/server/telemetry/queries/cases.test.ts
@@ -8,20 +8,25 @@
import type { SavedObjectsFindResponse } from '@kbn/core/server';
import { savedObjectsRepositoryMock, loggingSystemMock } from '@kbn/core/server/mocks';
import { ESCaseStatus } from '../../services/cases/types';
-import type { CaseAggregationResult } from '../types';
+import type {
+ AttachmentAggregationResult,
+ AttachmentFrameworkAggsResult,
+ CaseAggregationResult,
+ FileAttachmentAggregationResult,
+} from '../types';
import { getCasesTelemetryData } from './cases';
+const MOCK_FIND_TOTAL = 5;
+const SOLUTION_TOTAL = 1;
+
describe('getCasesTelemetryData', () => {
describe('getCasesTelemetryData', () => {
const logger = loggingSystemMock.createLogger();
const savedObjectsClient = savedObjectsRepositoryMock.create();
- const mockFind = (
- aggs: Record = {},
- so: SavedObjectsFindResponse['saved_objects'] = []
- ) => {
+ const mockFind = (aggs: object, so: SavedObjectsFindResponse['saved_objects'] = []) => {
savedObjectsClient.find.mockResolvedValueOnce({
- total: 5,
+ total: MOCK_FIND_TOTAL,
saved_objects: so,
per_page: 1,
page: 1,
@@ -103,22 +108,100 @@ describe('getCasesTelemetryData', () => {
buckets: [
{
key: 'observability',
- doc_count: 1,
+ doc_count: SOLUTION_TOTAL,
},
{
key: 'securitySolution',
- doc_count: 1,
+ doc_count: SOLUTION_TOTAL,
},
{
key: 'cases',
- doc_count: 1,
+ doc_count: SOLUTION_TOTAL,
},
],
},
};
+ const attachmentFramework: AttachmentFrameworkAggsResult = {
+ externalReferenceTypes: {
+ buckets: [
+ {
+ doc_count: 5,
+ key: '.osquery',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ {
+ doc_count: 5,
+ key: '.files',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ ],
+ },
+ persistableReferenceTypes: {
+ buckets: [
+ {
+ doc_count: 5,
+ key: '.ml',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ {
+ doc_count: 5,
+ key: '.files',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ ],
+ },
+ };
+
+ const attachmentAggsResult: AttachmentAggregationResult = {
+ securitySolution: { ...attachmentFramework },
+ observability: { ...attachmentFramework },
+ cases: { ...attachmentFramework },
+ participants: {
+ value: 2,
+ },
+ ...attachmentFramework,
+ };
+
+ const filesRes: FileAttachmentAggregationResult = {
+ securitySolution: {
+ averageSize: 500,
+ },
+ observability: {
+ averageSize: 500,
+ },
+ cases: {
+ averageSize: 500,
+ },
+ averageSize: 500,
+ };
+
mockFind(caseAggsResult);
- mockFind({ participants: { value: 2 } });
+ mockFind(attachmentAggsResult);
mockFind({ references: { referenceType: { referenceAgg: { value: 3 } } } });
mockFind({ references: { referenceType: { referenceAgg: { value: 4 } } } });
mockSavedObjectResponse({
@@ -130,6 +213,7 @@ describe('getCasesTelemetryData', () => {
mockSavedObjectResponse({
closed_at: '2022-03-08T12:24:11.429Z',
});
+ mockFind(filesRes);
};
beforeEach(() => {
@@ -139,10 +223,62 @@ describe('getCasesTelemetryData', () => {
it('it returns the correct res', async () => {
mockResponse();
+ const attachmentFramework = (total: number, average: number) => {
+ return {
+ attachmentFramework: {
+ externalAttachments: [
+ {
+ average,
+ maxOnACase: 10,
+ total,
+ type: '.osquery',
+ },
+ {
+ average,
+ maxOnACase: 10,
+ total,
+ type: '.files',
+ },
+ ],
+ persistableAttachments: [
+ {
+ average,
+ maxOnACase: 10,
+ total,
+ type: '.ml',
+ },
+ {
+ average,
+ maxOnACase: 10,
+ total,
+ type: '.files',
+ },
+ ],
+ files: {
+ averageSize: 500,
+ average,
+ maxOnACase: 10,
+ total,
+ },
+ },
+ };
+ };
+
const res = await getCasesTelemetryData({ savedObjectsClient, logger });
+
+ const allAttachmentsTotal = 5;
+ const allAttachmentsAverage = allAttachmentsTotal / MOCK_FIND_TOTAL;
+
+ const solutionAttachmentsTotal = 5;
+ const solutionAttachmentsAverage = solutionAttachmentsTotal / SOLUTION_TOTAL;
+ const solutionAttachmentFrameworkStats = attachmentFramework(
+ solutionAttachmentsTotal,
+ solutionAttachmentsAverage
+ );
+
expect(res).toEqual({
all: {
- total: 5,
+ total: MOCK_FIND_TOTAL,
daily: 3,
weekly: 2,
monthly: 1,
@@ -168,6 +304,7 @@ describe('getCasesTelemetryData', () => {
totalWithZero: 100,
totalWithAtLeastOne: 0,
},
+ ...attachmentFramework(allAttachmentsTotal, allAttachmentsAverage),
},
main: {
assignees: {
@@ -175,6 +312,7 @@ describe('getCasesTelemetryData', () => {
totalWithZero: 100,
totalWithAtLeastOne: 0,
},
+ ...solutionAttachmentFrameworkStats,
total: 1,
daily: 3,
weekly: 2,
@@ -186,6 +324,7 @@ describe('getCasesTelemetryData', () => {
totalWithZero: 100,
totalWithAtLeastOne: 0,
},
+ ...solutionAttachmentFrameworkStats,
total: 1,
daily: 3,
weekly: 2,
@@ -197,6 +336,7 @@ describe('getCasesTelemetryData', () => {
totalWithZero: 100,
totalWithAtLeastOne: 0,
},
+ ...solutionAttachmentFrameworkStats,
total: 1,
daily: 3,
weekly: 2,
@@ -468,18 +608,311 @@ describe('getCasesTelemetryData', () => {
}
`);
- expect(savedObjectsClient.find.mock.calls[1][0]).toEqual({
- aggs: {
- participants: {
- cardinality: {
- field: 'cases-comments.attributes.created_by.username',
+ expect(savedObjectsClient.find.mock.calls[1][0]).toMatchInlineSnapshot(`
+ Object {
+ "aggs": Object {
+ "cases": Object {
+ "aggs": Object {
+ "externalReferenceTypes": Object {
+ "aggs": Object {
+ "references": Object {
+ "aggregations": Object {
+ "cases": Object {
+ "aggregations": Object {
+ "ids": Object {
+ "terms": Object {
+ "field": "cases-comments.references.id",
+ },
+ },
+ "max": Object {
+ "max_bucket": Object {
+ "buckets_path": "ids._count",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.references.type": "cases",
+ },
+ },
+ },
+ },
+ "nested": Object {
+ "path": "cases-comments.references",
+ },
+ },
+ },
+ "terms": Object {
+ "field": "cases-comments.attributes.externalReferenceAttachmentTypeId",
+ },
+ },
+ "persistableReferenceTypes": Object {
+ "aggs": Object {
+ "references": Object {
+ "aggregations": Object {
+ "cases": Object {
+ "aggregations": Object {
+ "ids": Object {
+ "terms": Object {
+ "field": "cases-comments.references.id",
+ },
+ },
+ "max": Object {
+ "max_bucket": Object {
+ "buckets_path": "ids._count",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.references.type": "cases",
+ },
+ },
+ },
+ },
+ "nested": Object {
+ "path": "cases-comments.references",
+ },
+ },
+ },
+ "terms": Object {
+ "field": "cases-comments.attributes.persistableStateAttachmentTypeId",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.attributes.owner": "cases",
+ },
+ },
+ },
+ "externalReferenceTypes": Object {
+ "aggs": Object {
+ "references": Object {
+ "aggregations": Object {
+ "cases": Object {
+ "aggregations": Object {
+ "ids": Object {
+ "terms": Object {
+ "field": "cases-comments.references.id",
+ },
+ },
+ "max": Object {
+ "max_bucket": Object {
+ "buckets_path": "ids._count",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.references.type": "cases",
+ },
+ },
+ },
+ },
+ "nested": Object {
+ "path": "cases-comments.references",
+ },
+ },
+ },
+ "terms": Object {
+ "field": "cases-comments.attributes.externalReferenceAttachmentTypeId",
+ },
+ },
+ "observability": Object {
+ "aggs": Object {
+ "externalReferenceTypes": Object {
+ "aggs": Object {
+ "references": Object {
+ "aggregations": Object {
+ "cases": Object {
+ "aggregations": Object {
+ "ids": Object {
+ "terms": Object {
+ "field": "cases-comments.references.id",
+ },
+ },
+ "max": Object {
+ "max_bucket": Object {
+ "buckets_path": "ids._count",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.references.type": "cases",
+ },
+ },
+ },
+ },
+ "nested": Object {
+ "path": "cases-comments.references",
+ },
+ },
+ },
+ "terms": Object {
+ "field": "cases-comments.attributes.externalReferenceAttachmentTypeId",
+ },
+ },
+ "persistableReferenceTypes": Object {
+ "aggs": Object {
+ "references": Object {
+ "aggregations": Object {
+ "cases": Object {
+ "aggregations": Object {
+ "ids": Object {
+ "terms": Object {
+ "field": "cases-comments.references.id",
+ },
+ },
+ "max": Object {
+ "max_bucket": Object {
+ "buckets_path": "ids._count",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.references.type": "cases",
+ },
+ },
+ },
+ },
+ "nested": Object {
+ "path": "cases-comments.references",
+ },
+ },
+ },
+ "terms": Object {
+ "field": "cases-comments.attributes.persistableStateAttachmentTypeId",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.attributes.owner": "observability",
+ },
+ },
+ },
+ "participants": Object {
+ "cardinality": Object {
+ "field": "cases-comments.attributes.created_by.username",
+ },
+ },
+ "persistableReferenceTypes": Object {
+ "aggs": Object {
+ "references": Object {
+ "aggregations": Object {
+ "cases": Object {
+ "aggregations": Object {
+ "ids": Object {
+ "terms": Object {
+ "field": "cases-comments.references.id",
+ },
+ },
+ "max": Object {
+ "max_bucket": Object {
+ "buckets_path": "ids._count",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.references.type": "cases",
+ },
+ },
+ },
+ },
+ "nested": Object {
+ "path": "cases-comments.references",
+ },
+ },
+ },
+ "terms": Object {
+ "field": "cases-comments.attributes.persistableStateAttachmentTypeId",
+ },
+ },
+ "securitySolution": Object {
+ "aggs": Object {
+ "externalReferenceTypes": Object {
+ "aggs": Object {
+ "references": Object {
+ "aggregations": Object {
+ "cases": Object {
+ "aggregations": Object {
+ "ids": Object {
+ "terms": Object {
+ "field": "cases-comments.references.id",
+ },
+ },
+ "max": Object {
+ "max_bucket": Object {
+ "buckets_path": "ids._count",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.references.type": "cases",
+ },
+ },
+ },
+ },
+ "nested": Object {
+ "path": "cases-comments.references",
+ },
+ },
+ },
+ "terms": Object {
+ "field": "cases-comments.attributes.externalReferenceAttachmentTypeId",
+ },
+ },
+ "persistableReferenceTypes": Object {
+ "aggs": Object {
+ "references": Object {
+ "aggregations": Object {
+ "cases": Object {
+ "aggregations": Object {
+ "ids": Object {
+ "terms": Object {
+ "field": "cases-comments.references.id",
+ },
+ },
+ "max": Object {
+ "max_bucket": Object {
+ "buckets_path": "ids._count",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.references.type": "cases",
+ },
+ },
+ },
+ },
+ "nested": Object {
+ "path": "cases-comments.references",
+ },
+ },
+ },
+ "terms": Object {
+ "field": "cases-comments.attributes.persistableStateAttachmentTypeId",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "cases-comments.attributes.owner": "securitySolution",
+ },
+ },
},
},
- },
- page: 0,
- perPage: 0,
- type: 'cases-comments',
- });
+ "page": 0,
+ "perPage": 0,
+ "type": "cases-comments",
+ }
+ `);
expect(savedObjectsClient.find.mock.calls[2][0]).toEqual({
aggs: {
@@ -582,6 +1015,78 @@ describe('getCasesTelemetryData', () => {
type: 'cases',
});
}
+
+ expect(savedObjectsClient.find.mock.calls[7][0]).toMatchInlineSnapshot(`
+ Object {
+ "aggs": Object {
+ "averageSize": Object {
+ "avg": Object {
+ "field": "file.attributes.size",
+ },
+ },
+ "cases": Object {
+ "aggs": Object {
+ "averageSize": Object {
+ "avg": Object {
+ "field": "file.attributes.size",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "file.attributes.Meta.owner": "cases",
+ },
+ },
+ },
+ "observability": Object {
+ "aggs": Object {
+ "averageSize": Object {
+ "avg": Object {
+ "field": "file.attributes.size",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "file.attributes.Meta.owner": "observability",
+ },
+ },
+ },
+ "securitySolution": Object {
+ "aggs": Object {
+ "averageSize": Object {
+ "avg": Object {
+ "field": "file.attributes.size",
+ },
+ },
+ },
+ "filter": Object {
+ "term": Object {
+ "file.attributes.Meta.owner": "securitySolution",
+ },
+ },
+ },
+ },
+ "filter": Object {
+ "arguments": Array [
+ Object {
+ "isQuoted": false,
+ "type": "literal",
+ "value": "file.attributes.Meta.caseId",
+ },
+ Object {
+ "type": "wildcard",
+ "value": "@kuery-wildcard@",
+ },
+ ],
+ "function": "is",
+ "type": "function",
+ },
+ "page": 0,
+ "perPage": 0,
+ "type": "file",
+ }
+ `);
});
});
});
diff --git a/x-pack/plugins/cases/server/telemetry/queries/cases.ts b/x-pack/plugins/cases/server/telemetry/queries/cases.ts
index 128c0a09b5554..0e999721ae105 100644
--- a/x-pack/plugins/cases/server/telemetry/queries/cases.ts
+++ b/x-pack/plugins/cases/server/telemetry/queries/cases.ts
@@ -5,28 +5,33 @@
* 2.0.
*/
+import type { ISavedObjectsRepository, SavedObjectsFindResponse } from '@kbn/core/server';
+import { FILE_SO_TYPE } from '@kbn/files-plugin/common';
+import { fromKueryExpression } from '@kbn/es-query';
import {
CASE_COMMENT_SAVED_OBJECT,
CASE_SAVED_OBJECT,
CASE_USER_ACTION_SAVED_OBJECT,
+ OWNERS,
} from '../../../common/constants';
import { ESCaseStatus } from '../../services/cases/types';
import type { ESCaseAttributes } from '../../services/cases/types';
-import { OWNERS } from '../constants';
import type {
CollectTelemetryDataParams,
- Buckets,
CasesTelemetry,
- Cardinality,
ReferencesAggregation,
LatestDates,
CaseAggregationResult,
+ AttachmentAggregationResult,
+ FileAttachmentAggregationResult,
} from '../types';
import {
findValueInBuckets,
getAggregationsBuckets,
+ getAttachmentsFrameworkStats,
getCountsAggregationQuery,
getCountsFromBuckets,
+ getMaxBucketOnCaseAggregationQuery,
getOnlyAlertsCommentsFilter,
getOnlyConnectorsFilter,
getReferencesAggregationQuery,
@@ -52,9 +57,9 @@ export const getLatestCasesDates = async ({
]);
return {
- createdAt: savedObjects?.[0].saved_objects?.[0].attributes?.created_at ?? null,
- updatedAt: savedObjects?.[1].saved_objects?.[0].attributes?.updated_at ?? null,
- closedAt: savedObjects?.[2].saved_objects?.[0].attributes?.closed_at ?? null,
+ createdAt: savedObjects?.[0]?.saved_objects?.[0]?.attributes?.created_at ?? '',
+ updatedAt: savedObjects?.[1]?.saved_objects?.[0]?.attributes?.updated_at ?? '',
+ closedAt: savedObjects?.[2]?.saved_objects?.[0]?.attributes?.closed_at ?? '',
};
};
@@ -62,7 +67,84 @@ export const getCasesTelemetryData = async ({
savedObjectsClient,
logger,
}: CollectTelemetryDataParams): Promise => {
- const byOwnerAggregationQuery = OWNERS.reduce(
+ try {
+ const [casesRes, commentsRes, totalAlertsRes, totalConnectorsRes, latestDates, filesRes] =
+ await Promise.all([
+ getCasesSavedObjectTelemetry(savedObjectsClient),
+ getCommentsSavedObjectTelemetry(savedObjectsClient),
+ getAlertsTelemetry(savedObjectsClient),
+ getConnectorsTelemetry(savedObjectsClient),
+ getLatestCasesDates({ savedObjectsClient, logger }),
+ getFilesTelemetry(savedObjectsClient),
+ ]);
+
+ const aggregationsBuckets = getAggregationsBuckets({
+ aggs: casesRes.aggregations,
+ keys: ['counts', 'syncAlerts', 'status', 'users', 'totalAssignees'],
+ });
+
+ const allAttachmentFrameworkStats = getAttachmentsFrameworkStats({
+ attachmentAggregations: commentsRes.aggregations,
+ totalCasesForOwner: casesRes.total,
+ filesAggregations: filesRes.aggregations,
+ });
+
+ return {
+ all: {
+ total: casesRes.total,
+ ...getCountsFromBuckets(aggregationsBuckets.counts),
+ status: {
+ open: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.OPEN),
+ inProgress: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.IN_PROGRESS),
+ closed: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.CLOSED),
+ },
+ syncAlertsOn: findValueInBuckets(aggregationsBuckets.syncAlerts, 1),
+ syncAlertsOff: findValueInBuckets(aggregationsBuckets.syncAlerts, 0),
+ totalUsers: casesRes.aggregations?.users?.value ?? 0,
+ totalParticipants: commentsRes.aggregations?.participants?.value ?? 0,
+ totalTags: casesRes.aggregations?.tags?.value ?? 0,
+ totalWithAlerts:
+ totalAlertsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0,
+ totalWithConnectors:
+ totalConnectorsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0,
+ latestDates,
+ assignees: {
+ total: casesRes.aggregations?.totalAssignees.value ?? 0,
+ totalWithZero: casesRes.aggregations?.assigneeFilters.buckets.zero.doc_count ?? 0,
+ totalWithAtLeastOne:
+ casesRes.aggregations?.assigneeFilters.buckets.atLeastOne.doc_count ?? 0,
+ },
+ ...allAttachmentFrameworkStats,
+ },
+ sec: getSolutionValues({
+ caseAggregations: casesRes.aggregations,
+ attachmentAggregations: commentsRes.aggregations,
+ filesAggregations: filesRes.aggregations,
+ owner: 'securitySolution',
+ }),
+ obs: getSolutionValues({
+ caseAggregations: casesRes.aggregations,
+ attachmentAggregations: commentsRes.aggregations,
+ filesAggregations: filesRes.aggregations,
+ owner: 'observability',
+ }),
+ main: getSolutionValues({
+ caseAggregations: casesRes.aggregations,
+ attachmentAggregations: commentsRes.aggregations,
+ filesAggregations: filesRes.aggregations,
+ owner: 'cases',
+ }),
+ };
+ } catch (error) {
+ logger.error(`Cases telemetry failed with error: ${error}`);
+ throw error;
+ }
+};
+
+const getCasesSavedObjectTelemetry = async (
+ savedObjectsClient: ISavedObjectsRepository
+): Promise> => {
+ const caseByOwnerAggregationQuery = OWNERS.reduce(
(aggQuery, owner) => ({
...aggQuery,
[owner]: {
@@ -80,12 +162,12 @@ export const getCasesTelemetryData = async ({
{}
);
- const casesRes = await savedObjectsClient.find({
+ return savedObjectsClient.find({
page: 0,
perPage: 0,
type: CASE_SAVED_OBJECT,
aggs: {
- ...byOwnerAggregationQuery,
+ ...caseByOwnerAggregationQuery,
...getCountsAggregationQuery(CASE_SAVED_OBJECT),
...getAssigneesAggregations(),
totalsByOwner: {
@@ -111,17 +193,86 @@ export const getCasesTelemetryData = async ({
},
},
});
+};
+
+const getAssigneesAggregations = () => ({
+ totalAssignees: {
+ value_count: {
+ field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
+ },
+ },
+ assigneeFilters: {
+ filters: {
+ filters: {
+ zero: {
+ bool: {
+ must_not: {
+ exists: {
+ field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
+ },
+ },
+ },
+ },
+ atLeastOne: {
+ bool: {
+ filter: {
+ exists: {
+ field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+});
+
+const getCommentsSavedObjectTelemetry = async (
+ savedObjectsClient: ISavedObjectsRepository
+): Promise> => {
+ const attachmentRegistries = () => ({
+ externalReferenceTypes: {
+ terms: {
+ field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.externalReferenceAttachmentTypeId`,
+ },
+ aggs: {
+ ...getMaxBucketOnCaseAggregationQuery(CASE_COMMENT_SAVED_OBJECT),
+ },
+ },
+ persistableReferenceTypes: {
+ terms: {
+ field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.persistableStateAttachmentTypeId`,
+ },
+ aggs: {
+ ...getMaxBucketOnCaseAggregationQuery(CASE_COMMENT_SAVED_OBJECT),
+ },
+ },
+ });
+
+ const attachmentsByOwnerAggregationQuery = OWNERS.reduce(
+ (aggQuery, owner) => ({
+ ...aggQuery,
+ [owner]: {
+ filter: {
+ term: {
+ [`${CASE_COMMENT_SAVED_OBJECT}.attributes.owner`]: owner,
+ },
+ },
+ aggs: {
+ ...attachmentRegistries(),
+ },
+ },
+ }),
+ {}
+ );
- const commentsRes = await savedObjectsClient.find<
- unknown,
- Record & {
- participants: Cardinality;
- } & ReferencesAggregation
- >({
+ return savedObjectsClient.find({
page: 0,
perPage: 0,
type: CASE_COMMENT_SAVED_OBJECT,
aggs: {
+ ...attachmentsByOwnerAggregationQuery,
+ ...attachmentRegistries(),
participants: {
cardinality: {
field: `${CASE_COMMENT_SAVED_OBJECT}.attributes.created_by.username`,
@@ -129,8 +280,51 @@ export const getCasesTelemetryData = async ({
},
},
});
+};
+
+const getFilesTelemetry = async (
+ savedObjectsClient: ISavedObjectsRepository
+): Promise> => {
+ const averageSize = () => ({
+ averageSize: {
+ avg: {
+ field: `${FILE_SO_TYPE}.attributes.size`,
+ },
+ },
+ });
+
+ const filesByOwnerAggregationQuery = OWNERS.reduce(
+ (aggQuery, owner) => ({
+ ...aggQuery,
+ [owner]: {
+ filter: {
+ term: {
+ [`${FILE_SO_TYPE}.attributes.Meta.owner`]: owner,
+ },
+ },
+ aggs: {
+ ...averageSize(),
+ },
+ },
+ }),
+ {}
+ );
+
+ const filterCaseIdExists = fromKueryExpression(`${FILE_SO_TYPE}.attributes.Meta.caseId: *`);
- const totalAlertsRes = await savedObjectsClient.find({
+ return savedObjectsClient.find({
+ page: 0,
+ perPage: 0,
+ type: FILE_SO_TYPE,
+ filter: filterCaseIdExists,
+ aggs: { ...filesByOwnerAggregationQuery, ...averageSize() },
+ });
+};
+
+const getAlertsTelemetry = async (
+ savedObjectsClient: ISavedObjectsRepository
+): Promise> => {
+ return savedObjectsClient.find({
page: 0,
perPage: 0,
type: CASE_COMMENT_SAVED_OBJECT,
@@ -143,8 +337,12 @@ export const getCasesTelemetryData = async ({
}),
},
});
+};
- const totalConnectorsRes = await savedObjectsClient.find({
+const getConnectorsTelemetry = async (
+ savedObjectsClient: ISavedObjectsRepository
+): Promise> => {
+ return savedObjectsClient.find({
page: 0,
perPage: 0,
type: CASE_USER_ACTION_SAVED_OBJECT,
@@ -157,74 +355,4 @@ export const getCasesTelemetryData = async ({
}),
},
});
-
- const latestDates = await getLatestCasesDates({ savedObjectsClient, logger });
-
- const aggregationsBuckets = getAggregationsBuckets({
- aggs: casesRes.aggregations,
- keys: ['counts', 'syncAlerts', 'status', 'users', 'totalAssignees'],
- });
-
- return {
- all: {
- total: casesRes.total,
- ...getCountsFromBuckets(aggregationsBuckets.counts),
- status: {
- open: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.OPEN),
- inProgress: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.IN_PROGRESS),
- closed: findValueInBuckets(aggregationsBuckets.status, ESCaseStatus.CLOSED),
- },
- syncAlertsOn: findValueInBuckets(aggregationsBuckets.syncAlerts, 1),
- syncAlertsOff: findValueInBuckets(aggregationsBuckets.syncAlerts, 0),
- totalUsers: casesRes.aggregations?.users?.value ?? 0,
- totalParticipants: commentsRes.aggregations?.participants?.value ?? 0,
- totalTags: casesRes.aggregations?.tags?.value ?? 0,
- totalWithAlerts:
- totalAlertsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0,
- totalWithConnectors:
- totalConnectorsRes.aggregations?.references?.referenceType?.referenceAgg?.value ?? 0,
- latestDates,
- assignees: {
- total: casesRes.aggregations?.totalAssignees.value ?? 0,
- totalWithZero: casesRes.aggregations?.assigneeFilters.buckets.zero.doc_count ?? 0,
- totalWithAtLeastOne:
- casesRes.aggregations?.assigneeFilters.buckets.atLeastOne.doc_count ?? 0,
- },
- },
- sec: getSolutionValues(casesRes.aggregations, 'securitySolution'),
- obs: getSolutionValues(casesRes.aggregations, 'observability'),
- main: getSolutionValues(casesRes.aggregations, 'cases'),
- };
};
-
-const getAssigneesAggregations = () => ({
- totalAssignees: {
- value_count: {
- field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
- },
- },
- assigneeFilters: {
- filters: {
- filters: {
- zero: {
- bool: {
- must_not: {
- exists: {
- field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
- },
- },
- },
- },
- atLeastOne: {
- bool: {
- filter: {
- exists: {
- field: `${CASE_SAVED_OBJECT}.attributes.assignees.uid`,
- },
- },
- },
- },
- },
- },
- },
-});
diff --git a/x-pack/plugins/cases/server/telemetry/queries/utils.test.ts b/x-pack/plugins/cases/server/telemetry/queries/utils.test.ts
index e466ba597108a..d7c6a0e9bf7b9 100644
--- a/x-pack/plugins/cases/server/telemetry/queries/utils.test.ts
+++ b/x-pack/plugins/cases/server/telemetry/queries/utils.test.ts
@@ -6,10 +6,16 @@
*/
import { savedObjectsRepositoryMock } from '@kbn/core/server/mocks';
-import type { CaseAggregationResult } from '../types';
+import type {
+ AttachmentAggregationResult,
+ AttachmentFrameworkAggsResult,
+ CaseAggregationResult,
+ FileAttachmentAggregationResult,
+} from '../types';
import {
findValueInBuckets,
getAggregationsBuckets,
+ getAttachmentsFrameworkStats,
getBucketFromAggregation,
getConnectorsCardinalityAggregationQuery,
getCountsAggregationQuery,
@@ -46,19 +52,19 @@ describe('utils', () => {
totalAssignees: { value: 5 },
};
- const solutionValues = {
+ const caseSolutionValues = {
counts,
...assignees,
};
- const aggsResult: CaseAggregationResult = {
+ const caseAggsResult: CaseAggregationResult = {
users: { value: 1 },
tags: { value: 2 },
...assignees,
counts,
- securitySolution: { ...solutionValues },
- observability: { ...solutionValues },
- cases: { ...solutionValues },
+ securitySolution: { ...caseSolutionValues },
+ observability: { ...caseSolutionValues },
+ cases: { ...caseSolutionValues },
syncAlerts: {
buckets: [
{
@@ -87,7 +93,7 @@ describe('utils', () => {
},
{
key: 'securitySolution',
- doc_count: 1,
+ doc_count: 5,
},
{
key: 'cases',
@@ -97,40 +103,247 @@ describe('utils', () => {
},
};
+ const attachmentFramework: AttachmentFrameworkAggsResult = {
+ externalReferenceTypes: {
+ buckets: [
+ {
+ doc_count: 5,
+ key: '.osquery',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ {
+ doc_count: 5,
+ key: '.files',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ ],
+ },
+ persistableReferenceTypes: {
+ buckets: [
+ {
+ doc_count: 5,
+ key: '.ml',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ {
+ doc_count: 5,
+ key: '.files',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ ],
+ },
+ };
+
+ const attachmentAggsResult: AttachmentAggregationResult = {
+ securitySolution: { ...attachmentFramework },
+ observability: { ...attachmentFramework },
+ cases: { ...attachmentFramework },
+ participants: {
+ value: 5,
+ },
+ ...attachmentFramework,
+ };
+
+ const filesRes: FileAttachmentAggregationResult = {
+ securitySolution: {
+ averageSize: 500,
+ },
+ observability: {
+ averageSize: 500,
+ },
+ cases: {
+ averageSize: 500,
+ },
+ averageSize: 500,
+ };
+
it('constructs the solution values correctly', () => {
- expect(getSolutionValues(aggsResult, 'securitySolution')).toMatchInlineSnapshot(`
+ expect(
+ getSolutionValues({
+ caseAggregations: caseAggsResult,
+ attachmentAggregations: attachmentAggsResult,
+ filesAggregations: filesRes,
+ owner: 'securitySolution',
+ })
+ ).toMatchInlineSnapshot(`
Object {
"assignees": Object {
"total": 5,
"totalWithAtLeastOne": 0,
"totalWithZero": 100,
},
+ "attachmentFramework": Object {
+ "externalAttachments": Array [
+ Object {
+ "average": 1,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".osquery",
+ },
+ Object {
+ "average": 1,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".files",
+ },
+ ],
+ "files": Object {
+ "average": 1,
+ "averageSize": 500,
+ "maxOnACase": 10,
+ "total": 5,
+ },
+ "persistableAttachments": Array [
+ Object {
+ "average": 1,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".ml",
+ },
+ Object {
+ "average": 1,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".files",
+ },
+ ],
+ },
"daily": 3,
"monthly": 1,
- "total": 1,
+ "total": 5,
"weekly": 2,
}
`);
- expect(getSolutionValues(aggsResult, 'cases')).toMatchInlineSnapshot(`
+ expect(
+ getSolutionValues({
+ caseAggregations: caseAggsResult,
+ attachmentAggregations: attachmentAggsResult,
+ filesAggregations: filesRes,
+ owner: 'cases',
+ })
+ ).toMatchInlineSnapshot(`
Object {
"assignees": Object {
"total": 5,
"totalWithAtLeastOne": 0,
"totalWithZero": 100,
},
+ "attachmentFramework": Object {
+ "externalAttachments": Array [
+ Object {
+ "average": 5,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".osquery",
+ },
+ Object {
+ "average": 5,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".files",
+ },
+ ],
+ "files": Object {
+ "average": 5,
+ "averageSize": 500,
+ "maxOnACase": 10,
+ "total": 5,
+ },
+ "persistableAttachments": Array [
+ Object {
+ "average": 5,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".ml",
+ },
+ Object {
+ "average": 5,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".files",
+ },
+ ],
+ },
"daily": 3,
"monthly": 1,
"total": 1,
"weekly": 2,
}
`);
- expect(getSolutionValues(aggsResult, 'observability')).toMatchInlineSnapshot(`
+ expect(
+ getSolutionValues({
+ caseAggregations: caseAggsResult,
+ attachmentAggregations: attachmentAggsResult,
+ filesAggregations: filesRes,
+ owner: 'observability',
+ })
+ ).toMatchInlineSnapshot(`
Object {
"assignees": Object {
"total": 5,
"totalWithAtLeastOne": 0,
"totalWithZero": 100,
},
+ "attachmentFramework": Object {
+ "externalAttachments": Array [
+ Object {
+ "average": 5,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".osquery",
+ },
+ Object {
+ "average": 5,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".files",
+ },
+ ],
+ "files": Object {
+ "average": 5,
+ "averageSize": 500,
+ "maxOnACase": 10,
+ "total": 5,
+ },
+ "persistableAttachments": Array [
+ Object {
+ "average": 5,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".ml",
+ },
+ Object {
+ "average": 5,
+ "maxOnACase": 10,
+ "total": 5,
+ "type": ".files",
+ },
+ ],
+ },
"daily": 3,
"monthly": 1,
"total": 1,
@@ -140,6 +353,217 @@ describe('utils', () => {
});
});
+ describe('getAttachmentsFrameworkStats', () => {
+ it('returns empty stats if the aggregation is undefined', () => {
+ expect(getAttachmentsFrameworkStats({ totalCasesForOwner: 0 })).toMatchInlineSnapshot(`
+ Object {
+ "attachmentFramework": Object {
+ "externalAttachments": Array [],
+ "files": Object {
+ "average": 0,
+ "averageSize": 0,
+ "maxOnACase": 0,
+ "total": 0,
+ },
+ "persistableAttachments": Array [],
+ },
+ }
+ `);
+ });
+
+ describe('externalAttachments', () => {
+ const attachmentFramework: AttachmentFrameworkAggsResult = {
+ externalReferenceTypes: {
+ buckets: [
+ {
+ doc_count: 5,
+ key: '.osquery',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ {
+ doc_count: 10,
+ key: '.files',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ ],
+ },
+ persistableReferenceTypes: {
+ buckets: [],
+ },
+ };
+
+ it('populates the externalAttachments array', () => {
+ const stats = getAttachmentsFrameworkStats({
+ attachmentAggregations: attachmentFramework,
+ totalCasesForOwner: 5,
+ });
+
+ expect(stats.attachmentFramework.externalAttachments[0]).toEqual({
+ // the average is 5 from the aggs result / 5 from the function parameter
+ average: 1,
+ maxOnACase: 10,
+ total: 5,
+ type: '.osquery',
+ });
+
+ expect(stats.attachmentFramework.externalAttachments[1]).toEqual({
+ // the average is 10 from the aggs result / 5 from the function parameter
+ average: 2,
+ maxOnACase: 10,
+ total: 10,
+ type: '.files',
+ });
+ });
+ });
+
+ describe('persistableAttachments', () => {
+ const attachmentFramework: AttachmentFrameworkAggsResult = {
+ persistableReferenceTypes: {
+ buckets: [
+ {
+ doc_count: 5,
+ key: '.osquery',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ {
+ doc_count: 10,
+ key: '.files',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ ],
+ },
+ externalReferenceTypes: {
+ buckets: [],
+ },
+ };
+
+ it('populates the externalAttachments array', () => {
+ const stats = getAttachmentsFrameworkStats({
+ attachmentAggregations: attachmentFramework,
+ totalCasesForOwner: 5,
+ });
+
+ expect(stats.attachmentFramework.persistableAttachments[0]).toEqual({
+ // the average is 5 from the aggs result / 5 from the function parameter
+ average: 1,
+ maxOnACase: 10,
+ total: 5,
+ type: '.osquery',
+ });
+
+ expect(stats.attachmentFramework.persistableAttachments[1]).toEqual({
+ // the average is 10 from the aggs result / 5 from the function parameter
+ average: 2,
+ maxOnACase: 10,
+ total: 10,
+ type: '.files',
+ });
+ });
+ });
+
+ describe('files', () => {
+ it('sets the files stats to empty when it cannot find a files entry', () => {
+ const attachmentFramework: AttachmentFrameworkAggsResult = {
+ externalReferenceTypes: {
+ buckets: [
+ {
+ doc_count: 5,
+ key: '.osquery',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ ],
+ },
+ persistableReferenceTypes: {
+ buckets: [],
+ },
+ };
+
+ expect(
+ getAttachmentsFrameworkStats({
+ attachmentAggregations: attachmentFramework,
+ totalCasesForOwner: 5,
+ filesAggregations: { averageSize: 500 },
+ }).attachmentFramework.files
+ ).toMatchInlineSnapshot(`
+ Object {
+ "average": 0,
+ "averageSize": 0,
+ "maxOnACase": 0,
+ "total": 0,
+ }
+ `);
+ });
+
+ it('sets the files stats when it finds a files entry', () => {
+ const attachmentFramework: AttachmentFrameworkAggsResult = {
+ externalReferenceTypes: {
+ buckets: [
+ {
+ doc_count: 5,
+ key: '.files',
+ references: {
+ cases: {
+ max: {
+ value: 10,
+ },
+ },
+ },
+ },
+ ],
+ },
+ persistableReferenceTypes: {
+ buckets: [],
+ },
+ };
+
+ expect(
+ getAttachmentsFrameworkStats({
+ attachmentAggregations: attachmentFramework,
+ filesAggregations: { averageSize: 500 },
+ totalCasesForOwner: 5,
+ }).attachmentFramework.files
+ ).toMatchInlineSnapshot(`
+ Object {
+ "average": 1,
+ "averageSize": 500,
+ "maxOnACase": 10,
+ "total": 5,
+ }
+ `);
+ });
+ });
+ });
+
describe('getCountsAggregationQuery', () => {
it('returns the correct query', () => {
expect(getCountsAggregationQuery('test')).toEqual({
diff --git a/x-pack/plugins/cases/server/telemetry/queries/utils.ts b/x-pack/plugins/cases/server/telemetry/queries/utils.ts
index 82bdef3ebe825..0c0a4f7bbf87d 100644
--- a/x-pack/plugins/cases/server/telemetry/queries/utils.ts
+++ b/x-pack/plugins/cases/server/telemetry/queries/utils.ts
@@ -16,12 +16,20 @@ import {
import type {
CaseAggregationResult,
Buckets,
- CasesTelemetry,
MaxBucketOnCaseAggregation,
SolutionTelemetry,
+ AttachmentFramework,
+ AttachmentAggregationResult,
+ BucketsWithMaxOnCase,
+ AttachmentStats,
+ FileAttachmentStats,
+ FileAttachmentAggregationResult,
+ FileAttachmentAverageSize,
+ AttachmentFrameworkAggsResult,
} from '../types';
import { buildFilter } from '../../client/utils';
-import type { OWNERS } from '../constants';
+import type { Owner } from '../../../common/constants/types';
+import { FILE_ATTACHMENT_TYPE } from '../../../common/api';
export const getCountsAggregationQuery = (savedObjectType: string) => ({
counts: {
@@ -154,22 +162,39 @@ export const getBucketFromAggregation = ({
aggs?: Record;
}): Buckets['buckets'] => (get(aggs, `${key}.buckets`) ?? []) as Buckets['buckets'];
-export const getSolutionValues = (
- aggregations: CaseAggregationResult | undefined,
- owner: typeof OWNERS[number]
-): SolutionTelemetry => {
+export const getSolutionValues = ({
+ caseAggregations,
+ attachmentAggregations,
+ filesAggregations,
+ owner,
+}: {
+ caseAggregations?: CaseAggregationResult;
+ attachmentAggregations?: AttachmentAggregationResult;
+ filesAggregations?: FileAttachmentAggregationResult;
+ owner: Owner;
+}): SolutionTelemetry => {
const aggregationsBuckets = getAggregationsBuckets({
- aggs: aggregations,
+ aggs: caseAggregations,
keys: ['totalsByOwner', 'securitySolution.counts', 'observability.counts', 'cases.counts'],
});
+ const totalCasesForOwner = findValueInBuckets(aggregationsBuckets.totalsByOwner, owner);
+ const attachmentsAggsForOwner = attachmentAggregations?.[owner];
+ const fileAttachmentsForOwner = filesAggregations?.[owner];
+
return {
- total: findValueInBuckets(aggregationsBuckets.totalsByOwner, owner),
+ total: totalCasesForOwner,
...getCountsFromBuckets(aggregationsBuckets[`${owner}.counts`]),
+ ...getAttachmentsFrameworkStats({
+ attachmentAggregations: attachmentsAggsForOwner,
+ filesAggregations: fileAttachmentsForOwner,
+ totalCasesForOwner,
+ }),
assignees: {
- total: aggregations?.[owner].totalAssignees.value ?? 0,
- totalWithZero: aggregations?.[owner].assigneeFilters.buckets.zero.doc_count ?? 0,
- totalWithAtLeastOne: aggregations?.[owner].assigneeFilters.buckets.atLeastOne.doc_count ?? 0,
+ total: caseAggregations?.[owner].totalAssignees.value ?? 0,
+ totalWithZero: caseAggregations?.[owner].assigneeFilters.buckets.zero.doc_count ?? 0,
+ totalWithAtLeastOne:
+ caseAggregations?.[owner].assigneeFilters.buckets.atLeastOne.doc_count ?? 0,
},
};
};
@@ -192,6 +217,92 @@ export const getAggregationsBuckets = ({
{}
);
+export const getAttachmentsFrameworkStats = ({
+ attachmentAggregations,
+ filesAggregations,
+ totalCasesForOwner,
+}: {
+ attachmentAggregations?: AttachmentFrameworkAggsResult;
+ filesAggregations?: FileAttachmentAverageSize;
+ totalCasesForOwner: number;
+}): AttachmentFramework => {
+ if (!attachmentAggregations) {
+ return emptyAttachmentFramework();
+ }
+ const averageFileSize = filesAggregations?.averageSize;
+
+ return {
+ attachmentFramework: {
+ externalAttachments: getAttachmentRegistryStats(
+ attachmentAggregations.externalReferenceTypes,
+ totalCasesForOwner
+ ),
+ persistableAttachments: getAttachmentRegistryStats(
+ attachmentAggregations.persistableReferenceTypes,
+ totalCasesForOwner
+ ),
+ files: getFileAttachmentStats({
+ registryResults: attachmentAggregations.externalReferenceTypes,
+ averageFileSize,
+ totalCasesForOwner,
+ }),
+ },
+ };
+};
+
+const getAttachmentRegistryStats = (
+ registryResults: BucketsWithMaxOnCase,
+ totalCasesForOwner: number
+): AttachmentStats[] => {
+ const stats: AttachmentStats[] = [];
+
+ for (const bucket of registryResults.buckets) {
+ const commonFields = {
+ average: calculateTypePerCaseAverage(bucket.doc_count, totalCasesForOwner),
+ maxOnACase: bucket.references.cases.max.value,
+ total: bucket.doc_count,
+ };
+
+ stats.push({
+ type: bucket.key,
+ ...commonFields,
+ });
+ }
+
+ return stats;
+};
+
+const calculateTypePerCaseAverage = (typeDocCount: number, totalCases: number) => {
+ if (totalCases === 0) {
+ return 0;
+ }
+
+ return Math.round(typeDocCount / totalCases);
+};
+
+const getFileAttachmentStats = ({
+ registryResults,
+ averageFileSize,
+ totalCasesForOwner,
+}: {
+ registryResults: BucketsWithMaxOnCase;
+ averageFileSize?: number;
+ totalCasesForOwner: number;
+}): FileAttachmentStats => {
+ const fileBucket = registryResults.buckets.find((bucket) => bucket.key === FILE_ATTACHMENT_TYPE);
+
+ if (!fileBucket || averageFileSize == null) {
+ return emptyFileAttachment();
+ }
+
+ return {
+ averageSize: averageFileSize,
+ average: calculateTypePerCaseAverage(fileBucket.doc_count, totalCasesForOwner),
+ maxOnACase: fileBucket.references.cases.max.value,
+ total: fileBucket.doc_count,
+ };
+};
+
export const getOnlyAlertsCommentsFilter = () =>
buildFilter({
filters: ['alert'],
@@ -208,81 +319,17 @@ export const getOnlyConnectorsFilter = () =>
type: CASE_USER_ACTION_SAVED_OBJECT,
});
-export const getTelemetryDataEmptyState = (): CasesTelemetry => ({
- cases: {
- all: {
- assignees: {
- total: 0,
- totalWithZero: 0,
- totalWithAtLeastOne: 0,
- },
- total: 0,
- monthly: 0,
- weekly: 0,
- daily: 0,
- status: {
- open: 0,
- inProgress: 0,
- closed: 0,
- },
- syncAlertsOn: 0,
- syncAlertsOff: 0,
- totalUsers: 0,
- totalParticipants: 0,
- totalTags: 0,
- totalWithAlerts: 0,
- totalWithConnectors: 0,
- latestDates: {
- createdAt: null,
- updatedAt: null,
- closedAt: null,
- },
- },
- sec: {
- total: 0,
- monthly: 0,
- weekly: 0,
- daily: 0,
- assignees: { total: 0, totalWithAtLeastOne: 0, totalWithZero: 0 },
- },
- obs: {
- total: 0,
- monthly: 0,
- weekly: 0,
- daily: 0,
- assignees: { total: 0, totalWithAtLeastOne: 0, totalWithZero: 0 },
- },
- main: {
- total: 0,
- monthly: 0,
- weekly: 0,
- daily: 0,
- assignees: { total: 0, totalWithAtLeastOne: 0, totalWithZero: 0 },
- },
- },
- userActions: { all: { total: 0, monthly: 0, weekly: 0, daily: 0, maxOnACase: 0 } },
- comments: { all: { total: 0, monthly: 0, weekly: 0, daily: 0, maxOnACase: 0 } },
- alerts: { all: { total: 0, monthly: 0, weekly: 0, daily: 0, maxOnACase: 0 } },
- connectors: {
- all: {
- all: { totalAttached: 0 },
- itsm: { totalAttached: 0 },
- sir: { totalAttached: 0 },
- jira: { totalAttached: 0 },
- resilient: { totalAttached: 0 },
- swimlane: { totalAttached: 0 },
- maxAttachedToACase: 0,
- },
- },
- pushes: {
- all: { total: 0, maxOnACase: 0 },
- },
- configuration: {
- all: {
- closure: {
- manually: 0,
- automatic: 0,
- },
- },
+const emptyAttachmentFramework = (): AttachmentFramework => ({
+ attachmentFramework: {
+ persistableAttachments: [],
+ externalAttachments: [],
+ files: emptyFileAttachment(),
},
});
+
+const emptyFileAttachment = (): FileAttachmentStats => ({
+ average: 0,
+ averageSize: 0,
+ maxOnACase: 0,
+ total: 0,
+});
diff --git a/x-pack/plugins/cases/server/telemetry/schema.ts b/x-pack/plugins/cases/server/telemetry/schema.ts
index 1f51ca134b577..59506c932a51f 100644
--- a/x-pack/plugins/cases/server/telemetry/schema.ts
+++ b/x-pack/plugins/cases/server/telemetry/schema.ts
@@ -14,6 +14,8 @@ import type {
TypeString,
SolutionTelemetrySchema,
AssigneesSchema,
+ AttachmentFrameworkSchema,
+ AttachmentItemsSchema,
} from './types';
const long: TypeLong = { type: 'long' };
@@ -26,6 +28,32 @@ const countSchema: CountSchema = {
daily: long,
};
+interface AttachmentRegistrySchema {
+ type: 'array';
+ items: AttachmentItemsSchema;
+}
+
+const attachmentRegistrySchema: AttachmentRegistrySchema = {
+ type: 'array',
+ items: {
+ average: long,
+ maxOnACase: long,
+ total: long,
+ type: string,
+ },
+};
+
+const attachmentFrameworkSchema: AttachmentFrameworkSchema = {
+ persistableAttachments: attachmentRegistrySchema,
+ externalAttachments: attachmentRegistrySchema,
+ files: {
+ average: long,
+ averageSize: long,
+ maxOnACase: long,
+ total: long,
+ },
+};
+
const assigneesSchema: AssigneesSchema = {
total: long,
totalWithZero: long,
@@ -35,6 +63,7 @@ const assigneesSchema: AssigneesSchema = {
const solutionTelemetry: SolutionTelemetrySchema = {
...countSchema,
assignees: assigneesSchema,
+ attachmentFramework: attachmentFrameworkSchema,
};
const statusSchema: StatusSchema = {
@@ -53,6 +82,7 @@ export const casesSchema: CasesTelemetrySchema = {
cases: {
all: {
...countSchema,
+ attachmentFramework: attachmentFrameworkSchema,
assignees: assigneesSchema,
status: statusSchema,
syncAlertsOn: long,
diff --git a/x-pack/plugins/cases/server/telemetry/types.ts b/x-pack/plugins/cases/server/telemetry/types.ts
index 095b967d1addf..e28c3abaf4d52 100644
--- a/x-pack/plugins/cases/server/telemetry/types.ts
+++ b/x-pack/plugins/cases/server/telemetry/types.ts
@@ -7,7 +7,7 @@
import type { ISavedObjectsRepository, Logger } from '@kbn/core/server';
import type { MakeSchemaFrom } from '@kbn/usage-collection-plugin/server';
-import type { OWNERS } from './constants';
+import type { Owner } from '../../common/constants/types';
export interface Buckets {
buckets: Array<{
@@ -57,8 +57,33 @@ export interface AssigneesFilters {
};
}
+export interface FileAttachmentAverageSize {
+ averageSize: number;
+}
+
+export type FileAttachmentAggregationResult = Record &
+ FileAttachmentAverageSize;
+
+export interface BucketsWithMaxOnCase {
+ buckets: Array<
+ {
+ doc_count: number;
+ key: string;
+ } & MaxBucketOnCaseAggregation
+ >;
+}
+
+export interface AttachmentFrameworkAggsResult {
+ externalReferenceTypes: BucketsWithMaxOnCase;
+ persistableReferenceTypes: BucketsWithMaxOnCase;
+}
+
+export type AttachmentAggregationResult = Record & {
+ participants: Cardinality;
+} & AttachmentFrameworkAggsResult;
+
export type CaseAggregationResult = Record<
- typeof OWNERS[number],
+ Owner,
{
counts: Buckets;
totalAssignees: ValueCount;
@@ -81,7 +106,29 @@ export interface Assignees {
totalWithAtLeastOne: number;
}
-export interface SolutionTelemetry extends Count {
+interface CommonAttachmentStats {
+ average: number;
+ maxOnACase: number;
+ total: number;
+}
+
+export interface AttachmentStats extends CommonAttachmentStats {
+ type: string;
+}
+
+export interface FileAttachmentStats extends CommonAttachmentStats {
+ averageSize: number;
+}
+
+export interface AttachmentFramework {
+ attachmentFramework: {
+ externalAttachments: AttachmentStats[];
+ persistableAttachments: AttachmentStats[];
+ files: FileAttachmentStats;
+ };
+}
+
+export interface SolutionTelemetry extends Count, AttachmentFramework {
assignees: Assignees;
}
@@ -92,25 +139,26 @@ export interface Status {
}
export interface LatestDates {
- createdAt: string | null;
- updatedAt: string | null;
- closedAt: string | null;
+ createdAt: string;
+ updatedAt: string;
+ closedAt: string;
}
export interface CasesTelemetry {
cases: {
- all: Count & {
- assignees: Assignees;
- status: Status;
- syncAlertsOn: number;
- syncAlertsOff: number;
- totalUsers: number;
- totalParticipants: number;
- totalTags: number;
- totalWithAlerts: number;
- totalWithConnectors: number;
- latestDates: LatestDates;
- };
+ all: Count &
+ AttachmentFramework & {
+ assignees: Assignees;
+ status: Status;
+ syncAlertsOn: number;
+ syncAlertsOff: number;
+ totalUsers: number;
+ totalParticipants: number;
+ totalTags: number;
+ totalWithAlerts: number;
+ totalWithConnectors: number;
+ latestDates: LatestDates;
+ };
sec: SolutionTelemetry;
obs: SolutionTelemetry;
main: SolutionTelemetry;
@@ -147,4 +195,6 @@ export type StatusSchema = MakeSchemaFrom;
export type LatestDatesSchema = MakeSchemaFrom;
export type CasesTelemetrySchema = MakeSchemaFrom;
export type AssigneesSchema = MakeSchemaFrom;
+export type AttachmentFrameworkSchema = MakeSchemaFrom;
+export type AttachmentItemsSchema = MakeSchemaFrom;
export type SolutionTelemetrySchema = MakeSchemaFrom;
diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
index 99e466ebbb357..fc3f0ac44d7f3 100644
--- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
+++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json
@@ -4783,6 +4783,64 @@
"daily": {
"type": "long"
},
+ "attachmentFramework": {
+ "properties": {
+ "persistableAttachments": {
+ "type": "array",
+ "items": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ }
+ },
+ "externalAttachments": {
+ "type": "array",
+ "items": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ }
+ },
+ "files": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "averageSize": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ }
+ }
+ }
+ }
+ },
"assignees": {
"properties": {
"total": {
@@ -4871,6 +4929,64 @@
"type": "long"
}
}
+ },
+ "attachmentFramework": {
+ "properties": {
+ "persistableAttachments": {
+ "type": "array",
+ "items": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ }
+ },
+ "externalAttachments": {
+ "type": "array",
+ "items": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ }
+ },
+ "files": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "averageSize": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ }
+ }
+ }
+ }
}
}
},
@@ -4900,6 +5016,64 @@
"type": "long"
}
}
+ },
+ "attachmentFramework": {
+ "properties": {
+ "persistableAttachments": {
+ "type": "array",
+ "items": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ }
+ },
+ "externalAttachments": {
+ "type": "array",
+ "items": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ }
+ },
+ "files": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "averageSize": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ }
+ }
+ }
+ }
}
}
},
@@ -4929,6 +5103,64 @@
"type": "long"
}
}
+ },
+ "attachmentFramework": {
+ "properties": {
+ "persistableAttachments": {
+ "type": "array",
+ "items": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ }
+ },
+ "externalAttachments": {
+ "type": "array",
+ "items": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ },
+ "type": {
+ "type": "keyword"
+ }
+ }
+ }
+ },
+ "files": {
+ "properties": {
+ "average": {
+ "type": "long"
+ },
+ "averageSize": {
+ "type": "long"
+ },
+ "maxOnACase": {
+ "type": "long"
+ },
+ "total": {
+ "type": "long"
+ }
+ }
+ }
+ }
}
}
}