unify endpoints and detections telemetry

x-pack/plugins/security_solution/server/plugin.ts

@@ -48,7 +48,6 @@ import { EndpointAppContextService } from './endpoint/endpoint_app_context_servi
 import { EndpointAppContext } from './endpoint/types';
 import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts';
 import { initUsageCollectors } from './usage';
-import { createSiemTelemetry } from './lib/telemetry';
 
 export interface SetupPlugins {
   alerts: AlertingSetup;
@@ -60,8 +59,6 @@ export interface SetupPlugins {
   security?: SecuritySetup;
   spaces?: SpacesSetup;
   taskManager?: TaskManagerSetupContract;
-  ml?: MlSetup;
-  lists?: ListPluginSetup;
   usageCollection?: UsageCollectionSetup;
 }
 
@@ -117,6 +114,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     initSavedObjects(core.savedObjects);
     initUiSettings(core.uiSettings);
     initUsageCollectors({
+      core,
       kibanaIndex: globalConfig.kibana.index,
       ml: plugins.ml,
       usageCollection: plugins.usageCollection,
@@ -251,14 +249,6 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
       });
     }
 
-    if (plugins.usageCollection) {
-      const { usageCollection } = plugins;
-      core.getStartServices().then(([{ savedObjects }]) => {
-        const savedObjectsClient = savedObjects.createInternalRepository();
-        createSiemTelemetry(usageCollection, savedObjectsClient);
-      });
-    }
-
     const libs = compose(core, plugins, this.context.env.mode.prod);
     initServer(libs);
 

x-pack/plugins/security_solution/server/usage/collector.ts

@@ -4,20 +4,32 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { LegacyAPICaller } from '../../../../../src/core/server';
+import { LegacyAPICaller, CoreSetup } from '../../../../../src/core/server';
 import { CollectorDependencies } from './types';
 import { DetectionsUsage, fetchDetectionsUsage } from './detections';
+import { EndpointUsage, getEndpointTelemetryFromFleet } from './endpoints';
 
 export type RegisterCollector = (deps: CollectorDependencies) => void;
 export interface UsageData {
   detections: DetectionsUsage;
+  endpoints: EndpointUsage;
 }
 
-export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCollection }) => {
+export async function getInternalSavedObjectsClient(core: CoreSetup) {
+  return core.getStartServices().then(async ([coreStart]) => {
+    return coreStart.savedObjects.createInternalRepository();
+  });
+}
+
+export const registerCollector: RegisterCollector = ({
+  core,
+  kibanaIndex,
+  ml,
+  usageCollection,
+}) => {
   if (!usageCollection) {
     return;
   }
-
   const collector = usageCollection.makeUsageCollector<UsageData>({
     type: 'security_solution',
     schema: {
@@ -43,11 +55,32 @@ export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCol
           },
         },
       },
+      endpoints: {
+        total_installed: { type: 'long' },
+        active_within_last_24_hours: { type: 'long' },
+        os: {
+          full_name: { type: 'keyword' },
+          platform: { type: 'keyword' },
+          version: { type: 'keyword' },
+          count: { type: 'long' },
+        },
+        policies: {
+          malware: {
+            success: { type: 'long' },
+            warning: { type: 'long' },
+            failure: { type: 'long' },
+          },
+        },
+      },
     },
     isReady: () => kibanaIndex.length > 0,
-    fetch: async (callCluster: LegacyAPICaller): Promise<UsageData> => ({
-      detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml),
-    }),
+    fetch: async (callCluster: LegacyAPICaller): Promise<UsageData> => {
+      const savedObjectsClient = await getInternalSavedObjectsClient(core);
+      return {
+        detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml),
+        endpoint: await getEndpointTelemetryFromFleet(savedObjectsClient),
+      };
+    },
   });
 
   usageCollection.registerCollector(collector);

x-pack/plugins/security_solution/server/usage/detections.mocks.ts → x-pack/plugins/security_solution/server/usage/detections/detections.mocks.ts

@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { INTERNAL_IMMUTABLE_KEY } from '../../common/constants';
+import { INTERNAL_IMMUTABLE_KEY } from '../../../common/constants';
 
 export const getMockJobSummaryResponse = () => [
   {

x-pack/plugins/security_solution/server/usage/detections.test.ts → x-pack/plugins/security_solution/server/usage/detections/detections.test.ts

@@ -4,20 +4,20 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { LegacyAPICaller } from '../../../../../src/core/server';
-import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
-import { jobServiceProvider } from '../../../ml/server/models/job_service';
-import { DataRecognizer } from '../../../ml/server/models/data_recognizer';
-import { mlServicesMock } from '../lib/machine_learning/mocks';
+import { LegacyAPICaller } from '../../../../../../src/core/server';
+import { elasticsearchServiceMock } from '../../../../../../src/core/server/mocks';
+import { jobServiceProvider } from '../../../../ml/server/models/job_service';
+import { DataRecognizer } from '../../../../ml/server/models/data_recognizer';
+import { mlServicesMock } from '../../lib/machine_learning/mocks';
 import {
   getMockJobSummaryResponse,
   getMockListModulesResponse,
   getMockRulesResponse,
 } from './detections.mocks';
-import { fetchDetectionsUsage } from './detections';
+import { fetchDetectionsUsage } from './index';
 
-jest.mock('../../../ml/server/models/job_service');
-jest.mock('../../../ml/server/models/data_recognizer');
+jest.mock('../../../../ml/server/models/job_service');
+jest.mock('../../../../ml/server/models/data_recognizer');
 
 describe('Detections Usage', () => {
   describe('fetchDetectionsUsage()', () => {

x-pack/plugins/security_solution/server/usage/detections_helpers.ts → x-pack/plugins/security_solution/server/usage/detections/detections_helpers.ts

@@ -6,15 +6,15 @@
 
 import { SearchParams } from 'elasticsearch';
 
-import { LegacyAPICaller, SavedObjectsClient } from '../../../../../src/core/server';
+import { LegacyAPICaller, SavedObjectsClient } from '../../../../../../src/core/server';
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
-import { jobServiceProvider } from '../../../ml/server/models/job_service';
+import { jobServiceProvider } from '../../../../ml/server/models/job_service';
 // eslint-disable-next-line @kbn/eslint/no-restricted-paths
-import { DataRecognizer } from '../../../ml/server/models/data_recognizer';
-import { MlPluginSetup } from '../../../ml/server';
-import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../common/constants';
-import { DetectionRulesUsage, MlJobsUsage } from './detections';
-import { isJobStarted } from '../../common/machine_learning/helpers';
+import { DataRecognizer } from '../../../../ml/server/models/data_recognizer';
+import { MlPluginSetup } from '../../../../ml/server';
+import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../../common/constants';
+import { DetectionRulesUsage, MlJobsUsage } from './index';
+import { isJobStarted } from '../../../common/machine_learning/helpers';
 
 interface DetectionsMetric {
   isElastic: boolean;

x-pack/plugins/security_solution/server/usage/detections.ts → x-pack/plugins/security_solution/server/usage/detections/index.ts

@@ -4,9 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { LegacyAPICaller } from '../../../../../src/core/server';
+import { LegacyAPICaller } from '../../../../../../src/core/server';
 import { getMlJobsUsage, getRulesUsage } from './detections_helpers';
-import { MlPluginSetup } from '../../../ml/server';
+import { MlPluginSetup } from '../../../../ml/server';
 
 interface FeatureUsage {
   enabled: number;

x-pack/plugins/security_solution/server/lib/telemetry/endpoint.mocks.ts → x-pack/plugins/security_solution/server/usage/endpoints/endpoint.mocks.ts

@@ -8,7 +8,7 @@ import { AgentEventSOAttributes } from './../../../../ingest_manager/common/type
 import {
   AGENT_SAVED_OBJECT_TYPE,
   AGENT_EVENT_SAVED_OBJECT_TYPE,
-} from './../../../../ingest_manager/common/constants/agent';
+} from '../../../../ingest_manager/common/constants/agent';
 import { Agent } from '../../../../ingest_manager/common';
 import { FLEET_ENDPOINT_PACKAGE_CONSTANT } from './fleet_saved_objects';
 

x-pack/plugins/security_solution/server/lib/telemetry/endpoint.test.ts → x-pack/plugins/security_solution/server/usage/endpoints/endpoint.test.ts

@@ -12,9 +12,9 @@ import {
   MockOSVersion,
 } from './endpoint.mocks';
 import { ISavedObjectsRepository, SavedObjectsFindResponse } from 'src/core/server';
-import { AgentEventSOAttributes } from './../../../../ingest_manager/common/types/models/agent';
+import { AgentEventSOAttributes } from '../../../../ingest_manager/common/types/models/agent';
 import { Agent } from '../../../../ingest_manager/common';
-import * as endpointTelemetry from './endpoint';
+import * as endpointTelemetry from './index';
 import * as fleetSavedObjects from './fleet_saved_objects';
 
 describe('test security solution endpoint telemetry', () => {

x-pack/plugins/security_solution/server/lib/telemetry/endpoint.ts → x-pack/plugins/security_solution/server/usage/endpoints/index.ts

@@ -90,7 +90,7 @@ export const trackEndpointOSTelemetry = (
  */
 export const getEndpointTelemetryFromFleet = async (
   savedObjectsClient: ISavedObjectsRepository
-) => {
+): Promise<EndpointUsage> => {
   // Retrieve every agent that references the endpoint as an installed package. It will not be listed if it was never installed
   const { saved_objects: endpointAgents } = await getFleetSavedObjectsMetadata(savedObjectsClient);
   const endpointTelemetry = getDefaultEndpointTelemetry();
@@ -100,13 +100,11 @@ export const getEndpointTelemetryFromFleet = async (
 
   // Use unique hosts to prevent any potential duplicates
   const uniqueHostIds: Set<string> = new Set();
-
   // Need unique agents to get events data for those that have run in last 24 hours
   const uniqueAgentIds: Set<string> = new Set();
 
   const aDayAgo = new Date();
   aDayAgo.setDate(aDayAgo.getDate() - 1);
-
   let osTracker: OSTracker = {};
 
   const endpointMetadataTelemetry = endpointAgents.reduce(
@@ -130,7 +128,7 @@ export const getEndpointTelemetryFromFleet = async (
     endpointTelemetry
   );
 
-  // All agents in the unique host.
+  // All unique agents with an endpoint installed. You can technically install a new agent on a host, so relying on most recently installed.
   endpointTelemetry.total_installed = uniqueHostIds.size;
 
   // Get the objects to populate our OS Telemetry

x-pack/plugins/security_solution/server/usage/types.ts

@@ -4,9 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { CoreSetup } from 'src/core/server';
 import { SetupPlugins } from '../plugin';
 
-export type CollectorDependencies = { kibanaIndex: string } & Pick<
+export type CollectorDependencies = { kibanaIndex: string; core: CoreSetup } & Pick<
   SetupPlugins,
   'ml' | 'usageCollection'
 >;

x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json

@@ -218,7 +218,7 @@
             }
           }
         },
-        "endpoint": {
+        "endpoints": {
           "properties": {
             "total_installed": {
               "type": "long"