[infra] Shorten IDs for ML jobs (elastic#47477)

miltonhultgren · Oct 6, 2023 · 9714423 · 9714423
1 parent baff0eb
commit 9714423
Show file tree

Hide file tree

Showing 62 changed files with 735 additions and 156 deletions.
diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts
@@ -8,6 +8,7 @@
 import * as rt from 'io-ts';
 
 import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common';
+import { idFormatByJobTypeRT } from '../../v1/id_formats';
 import { timeRangeRT, routeTimingMetadataRT } from '../../../shared';
 import {
   logEntryAnomalyRT,
@@ -54,6 +55,7 @@ export const getLogEntryAnomaliesRequestPayloadRT = rt.type({
     rt.type({
       // log view
       logView: persistedLogViewReferenceRT,
+      idFormats: idFormatByJobTypeRT,
       // the time range to fetch the log entry anomalies from
       timeRange: timeRangeRT,
     }),

diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts
@@ -14,6 +14,7 @@ import {
   timeRangeRT,
   routeTimingMetadataRT,
 } from '../../../shared';
+import { idFormatByJobTypeRT } from '../../v1/id_formats';
 
 export const LOG_ANALYSIS_GET_LOG_ENTRY_ANOMALIES_DATASETS_PATH =
   '/api/infra/log_analysis/results/log_entry_anomalies_datasets';
@@ -26,6 +27,7 @@ export const getLogEntryAnomaliesDatasetsRequestPayloadRT = rt.type({
   data: rt.type({
     // log view
     logView: persistedLogViewReferenceRT,
+    idFormats: idFormatByJobTypeRT,
     // the time range to fetch the anomalies datasets from
     timeRange: timeRangeRT,
   }),

diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts
@@ -8,6 +8,7 @@
 import * as rt from 'io-ts';
 
 import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common';
+import { idFormatRT } from '../../v1/id_formats';
 import {
   badRequestErrorRT,
   forbiddenErrorRT,
@@ -41,6 +42,7 @@ export const getLogEntryCategoriesRequestPayloadRT = rt.type({
       categoryCount: rt.number,
       // log view
       logView: persistedLogViewReferenceRT,
+      idFormat: idFormatRT,
       // the time range to fetch the categories from
       timeRange: timeRangeRT,
       // a list of histograms to create

diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts
@@ -8,6 +8,7 @@
 import * as rt from 'io-ts';
 
 import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common';
+import { idFormatRT } from '../../v1/id_formats';
 import {
   badRequestErrorRT,
   forbiddenErrorRT,
@@ -25,6 +26,7 @@ export const getLogEntryCategoryDatasetsRequestPayloadRT = rt.type({
   data: rt.type({
     // log view
     logView: persistedLogViewReferenceRT,
+    idFormat: idFormatRT,
     // the time range to fetch the category datasets from
     timeRange: timeRangeRT,
   }),

diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts
@@ -7,6 +7,7 @@
 
 import { logEntryContextRT, persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common';
 import * as rt from 'io-ts';
+import { idFormatRT } from '../../v1/id_formats';
 import {
   badRequestErrorRT,
   forbiddenErrorRT,
@@ -29,6 +30,7 @@ export const getLogEntryCategoryExamplesRequestPayloadRT = rt.type({
     exampleCount: rt.number,
     // log view
     logView: persistedLogViewReferenceRT,
+    idFormat: idFormatRT,
     // the time range to fetch the category examples from
     timeRange: timeRangeRT,
   }),

diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts
@@ -7,6 +7,7 @@
 
 import * as rt from 'io-ts';
 import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common';
+import { idFormatRT } from '../../v1/id_formats';
 import { logEntryExampleRT } from '../../../../log_analysis';
 import {
   badRequestErrorRT,
@@ -31,6 +32,7 @@ export const getLogEntryExamplesRequestPayloadRT = rt.type({
       exampleCount: rt.number,
       // logView
       logView: persistedLogViewReferenceRT,
+      idFormat: idFormatRT,
       // the time range to fetch the log rate examples from
       timeRange: timeRangeRT,
     }),

diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/v1/id_formats.ts b/x-pack/plugins/infra/common/http_api/log_analysis/v1/id_formats.ts
@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import * as rt from 'io-ts';
+import { logEntryRateJobTypeRT, logEntryCategoriesJobTypeRT } from '../../../log_analysis';
+
+export const idFormatRT = rt.union([rt.literal('legacy'), rt.literal('hashed')]);
+export type IdFormat = rt.TypeOf<typeof idFormatRT>;
+
+const jobTypeRT = rt.union([logEntryRateJobTypeRT, logEntryCategoriesJobTypeRT]);
+export type JobType = rt.TypeOf<typeof jobTypeRT>;
+
+export const idFormatByJobTypeRT = rt.record(jobTypeRT, idFormatRT);
+export type IdFormatByJobType = rt.TypeOf<typeof idFormatByJobTypeRT>;
+
+export const LOG_ANALYSIS_GET_ID_FORMATS = '/api/infra/log_analysis/id_formats';
+
+export const getLogAnalysisIdFormatsRequestPayloadRT = rt.type({
+  data: rt.type({
+    logViewId: rt.string,
+    spaceId: rt.string,
+  }),
+});
+
+export type GetLogAnalysisIdFormatsRequestPayload = rt.TypeOf<
+  typeof getLogAnalysisIdFormatsRequestPayloadRT
+>;
+
+export const getLogAnalysisIdFormatsSuccessResponsePayloadRT = rt.type({
+  data: rt.record(rt.union([logEntryRateJobTypeRT, logEntryCategoriesJobTypeRT]), idFormatRT),
+});
+
+export type GetLogAnalysisIdFormatsSuccessResponsePayload = rt.TypeOf<
+  typeof getLogAnalysisIdFormatsSuccessResponsePayloadRT
+>;
diff --git a/x-pack/plugins/infra/common/log_analysis/job_parameters.ts b/x-pack/plugins/infra/common/log_analysis/job_parameters.ts
@@ -6,21 +6,40 @@
  */
 
 import * as rt from 'io-ts';
+import { createHash } from 'crypto';
+import { IdFormat, JobType } from '../http_api/log_analysis/v1/id_formats';
 
 export const bucketSpan = 900000;
 
 export const categoriesMessageField = 'message';
 
 export const partitionField = 'event.dataset';
 
-export const getJobIdPrefix = (spaceId: string, sourceId: string) =>
-  `kibana-logs-ui-${spaceId}-${sourceId}-`;
-
-export const getJobId = (spaceId: string, logViewId: string, jobType: string) =>
-  `${getJobIdPrefix(spaceId, logViewId)}${jobType}`;
+export const getJobIdPrefix = (spaceId: string, sourceId: string, idFormat: IdFormat) => {
+  if (idFormat === 'legacy') {
+    return `kibana-logs-ui-${spaceId}-${sourceId}-`;
+  } else {
+    const hash = createHash('sha256')
+      .update(`${spaceId}-${sourceId}`)
+      .digest('hex')
+      .substring(0, 10);
+    return `logs-${hash}-`;
+  }
+};
 
-export const getDatafeedId = (spaceId: string, logViewId: string, jobType: string) =>
-  `datafeed-${getJobId(spaceId, logViewId, jobType)}`;
+export const getJobId = (
+  spaceId: string,
+  logViewId: string,
+  idFormat: IdFormat,
+  jobType: JobType
+) => `${getJobIdPrefix(spaceId, logViewId, idFormat)}${jobType}`;
+
+export const getDatafeedId = (
+  spaceId: string,
+  logViewId: string,
+  idFormat: IdFormat,
+  jobType: JobType
+) => `datafeed-${getJobId(spaceId, logViewId, idFormat, jobType)}`;
 
 export const datasetFilterRT = rt.union([
   rt.strict({

diff --git a/x-pack/plugins/infra/common/log_analysis/log_entry_categories_analysis.ts b/x-pack/plugins/infra/common/log_analysis/log_entry_categories_analysis.ts
@@ -8,16 +8,16 @@
 import * as rt from 'io-ts';
 import { sortRT } from './log_analysis_results';
 
-export const logEntryCategoriesJobTypeRT = rt.keyof({
-  'log-entry-categories-count': null,
-});
+export const logEntryCategoriesJobTypeRT = rt.literal('log-entry-categories-count');
 
 export type LogEntryCategoriesJobType = rt.TypeOf<typeof logEntryCategoriesJobTypeRT>;
 
 export const logEntryCategoriesJobTypes: LogEntryCategoriesJobType[] = [
   'log-entry-categories-count',
 ];
 
+export const logEntryCategoriesJobType: LogEntryCategoriesJobType = 'log-entry-categories-count';
+
 export const logEntryCategoryDatasetRT = rt.type({
   name: rt.string,
   maximumAnomalyScore: rt.number,

diff --git a/x-pack/plugins/infra/common/log_analysis/log_entry_rate_analysis.ts b/x-pack/plugins/infra/common/log_analysis/log_entry_rate_analysis.ts
@@ -7,10 +7,9 @@
 
 import * as rt from 'io-ts';
 
-export const logEntryRateJobTypeRT = rt.keyof({
-  'log-entry-rate': null,
-});
+export const logEntryRateJobTypeRT = rt.literal('log-entry-rate');
 
 export type LogEntryRateJobType = rt.TypeOf<typeof logEntryRateJobTypeRT>;
 
-export const logEntryRateJobTypes: LogEntryRateJobType[] = ['log-entry-rate'];
+export const logEntryRateJobType: LogEntryRateJobType = 'log-entry-rate';
+export const logEntryRateJobTypes: LogEntryRateJobType[] = [logEntryRateJobType];
diff --git a/...k/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/module_list.tsx b/...k/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/module_list.tsx
@@ -7,6 +7,7 @@
 
 import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import React, { useCallback } from 'react';
+import { logEntryCategoriesJobType, logEntryRateJobType } from '../../../../../common/log_analysis';
 import { useLogAnalysisCapabilitiesContext } from '../../../../containers/logs/log_analysis';
 import {
   logEntryCategoriesModule,
@@ -40,7 +41,7 @@ export const LogAnalysisModuleList: React.FC<{
       <EuiFlexGroup>
         <EuiFlexItem>
           <LogAnalysisModuleListCard
-            jobId={logEntryRateJobIds['log-entry-rate']}
+            jobId={logEntryRateJobIds[logEntryRateJobType]}
             hasSetupCapabilities={hasLogAnalysisSetupCapabilities}
             moduleDescription={logEntryRateModule.moduleDescription}
             moduleName={logEntryRateModule.moduleName}
@@ -50,7 +51,7 @@ export const LogAnalysisModuleList: React.FC<{
         </EuiFlexItem>
         <EuiFlexItem>
           <LogAnalysisModuleListCard
-            jobId={logEntryCategoriesJobIds['log-entry-categories-count']}
+            jobId={logEntryCategoriesJobIds[logEntryCategoriesJobType]}
             hasSetupCapabilities={hasLogAnalysisSetupCapabilities}
             moduleDescription={logEntryCategoriesModule.moduleDescription}
             moduleName={logEntryCategoriesModule.moduleName}

diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_cleanup.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_cleanup.ts
@@ -8,28 +8,30 @@
 import * as rt from 'io-ts';
 import type { HttpHandler } from '@kbn/core/public';
 
+import { IdFormat, JobType } from '../../../../../common/http_api/log_analysis/v1/id_formats';
 import { getDatafeedId, getJobId } from '../../../../../common/log_analysis';
 import { decodeOrThrow } from '../../../../../common/runtime_types';
 
-interface DeleteJobsRequestArgs<JobType extends string> {
+interface DeleteJobsRequestArgs<T extends JobType> {
   spaceId: string;
   logViewId: string;
-  jobTypes: JobType[];
+  idFormat: IdFormat;
+  jobTypes: T[];
 }
 
-export const callDeleteJobs = async <JobType extends string>(
-  requestArgs: DeleteJobsRequestArgs<JobType>,
+export const callDeleteJobs = async <T extends JobType>(
+  requestArgs: DeleteJobsRequestArgs<T>,
   fetch: HttpHandler
 ) => {
-  const { spaceId, logViewId, jobTypes } = requestArgs;
+  const { spaceId, logViewId, idFormat, jobTypes } = requestArgs;
 
   // NOTE: Deleting the jobs via this API will delete the datafeeds at the same time
   const deleteJobsResponse = await fetch('/internal/ml/jobs/delete_jobs', {
     method: 'POST',
     version: '1',
     body: JSON.stringify(
       deleteJobsRequestPayloadRT.encode({
-        jobIds: jobTypes.map((jobType) => getJobId(spaceId, logViewId, jobType)),
+        jobIds: jobTypes.map((jobType) => getJobId(spaceId, logViewId, idFormat, jobType)),
       })
     ),
   });
@@ -45,25 +47,28 @@ export const callGetJobDeletionTasks = async (fetch: HttpHandler) => {
   return decodeOrThrow(getJobDeletionTasksResponsePayloadRT)(jobDeletionTasksResponse);
 };
 
-interface StopDatafeedsRequestArgs<JobType extends string> {
+interface StopDatafeedsRequestArgs<T extends JobType> {
   spaceId: string;
   logViewId: string;
-  jobTypes: JobType[];
+  idFormat: IdFormat;
+  jobTypes: T[];
 }
 
-export const callStopDatafeeds = async <JobType extends string>(
-  requestArgs: StopDatafeedsRequestArgs<JobType>,
+export const callStopDatafeeds = async <T extends JobType>(
+  requestArgs: StopDatafeedsRequestArgs<T>,
   fetch: HttpHandler
 ) => {
-  const { spaceId, logViewId, jobTypes } = requestArgs;
+  const { spaceId, logViewId, idFormat, jobTypes } = requestArgs;
 
   // Stop datafeed due to https://github.com/elastic/kibana/issues/44652
   const stopDatafeedResponse = await fetch('/internal/ml/jobs/stop_datafeeds', {
     method: 'POST',
     version: '1',
     body: JSON.stringify(
       stopDatafeedsRequestPayloadRT.encode({
-        datafeedIds: jobTypes.map((jobType) => getDatafeedId(spaceId, logViewId, jobType)),
+        datafeedIds: jobTypes.map((jobType) =>
+          getDatafeedId(spaceId, logViewId, idFormat, jobType)
+        ),
       })
     ),
   });

diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts
@@ -8,26 +8,28 @@
 import * as rt from 'io-ts';
 import type { HttpHandler } from '@kbn/core/public';
 
+import { IdFormat, JobType } from '../../../../../common/http_api/log_analysis/v1/id_formats';
 import { getJobId, jobCustomSettingsRT } from '../../../../../common/log_analysis';
 import { decodeOrThrow } from '../../../../../common/runtime_types';
 
-interface RequestArgs<JobType extends string> {
+interface RequestArgs<T extends JobType> {
   spaceId: string;
   logViewId: string;
-  jobTypes: JobType[];
+  idFormat: IdFormat;
+  jobTypes: T[];
 }
 
-export const callJobsSummaryAPI = async <JobType extends string>(
-  requestArgs: RequestArgs<JobType>,
+export const callJobsSummaryAPI = async <T extends JobType>(
+  requestArgs: RequestArgs<T>,
   fetch: HttpHandler
 ) => {
-  const { spaceId, logViewId, jobTypes } = requestArgs;
+  const { spaceId, logViewId, idFormat, jobTypes } = requestArgs;
   const response = await fetch('/internal/ml/jobs/jobs_summary', {
     method: 'POST',
     version: '1',
     body: JSON.stringify(
       fetchJobStatusRequestPayloadRT.encode({
-        jobIds: jobTypes.map((jobType) => getJobId(spaceId, logViewId, jobType)),
+        jobIds: jobTypes.map((jobType) => getJobId(spaceId, logViewId, idFormat, jobType)),
       })
     ),
   });

diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts
@@ -46,7 +46,7 @@ export const callSetupMlModuleAPI = async (requestArgs: RequestArgs, fetch: Http
         start,
         end,
         indexPatternName: indexPattern,
-        prefix: getJobIdPrefix(spaceId, sourceId),
+        prefix: getJobIdPrefix(spaceId, sourceId, 'hashed'),
         startDatafeed: true,
         jobOverrides,
         datafeedOverrides,