Merge remote-tracking branch 'upstream/main' into 164059-untracked-di…

…sabled-alerts

.github/CODEOWNERS

@@ -502,6 +502,7 @@ x-pack/plugins/metrics_data_access @elastic/infra-monitoring-ui
 x-pack/packages/ml/agg_utils @elastic/ml-ui
 x-pack/packages/ml/anomaly_utils @elastic/ml-ui
 x-pack/packages/ml/category_validator @elastic/ml-ui
+x-pack/packages/ml/chi2test @elastic/ml-ui
 x-pack/packages/ml/data_frame_analytics_utils @elastic/ml-ui
 x-pack/packages/ml/data_grid @elastic/ml-ui
 x-pack/packages/ml/date_picker @elastic/ml-ui

config/serverless.oblt.yml

@@ -12,7 +12,6 @@ xpack.legacy_uptime.enabled: false
 xpack.serverless.observability.enabled: true
 
 ## Configure plugins
-xpack.infra.logs.app_target: discover
 
 ## Set the home route
 uiSettings.overrides.defaultRoute: /app/observability/landing

package.json

@@ -523,6 +523,7 @@
     "@kbn/ml-agg-utils": "link:x-pack/packages/ml/agg_utils",
     "@kbn/ml-anomaly-utils": "link:x-pack/packages/ml/anomaly_utils",
     "@kbn/ml-category-validator": "link:x-pack/packages/ml/category_validator",
+    "@kbn/ml-chi2test": "link:x-pack/packages/ml/chi2test",
     "@kbn/ml-data-frame-analytics-utils": "link:x-pack/packages/ml/data_frame_analytics_utils",
     "@kbn/ml-data-grid": "link:x-pack/packages/ml/data_grid",
     "@kbn/ml-date-picker": "link:x-pack/packages/ml/date_picker",

test/plugin_functional/test_suites/core_plugins/rendering.ts

@@ -250,11 +250,6 @@ export default function ({ getService }: PluginFunctionalProviderContext) {
         'xpack.index_management.dev.enableIndexDetailsPage (boolean)',
         'xpack.index_management.enableIndexStats (any)',
         'xpack.infra.sources.default.fields.message (array)',
-        /**
-         * xpack.infra.logs is conditional and will resolve to an object of properties
-         * - xpack.infra.logs.app_target (string)
-         */
-        'xpack.infra.logs (any)',
         'xpack.license_management.ui.enabled (boolean)',
         'xpack.maps.preserveDrawingBuffer (boolean)',
         'xpack.maps.showMapsInspectorAdapter (boolean)',

tsconfig.base.json

@@ -998,6 +998,8 @@
       "@kbn/ml-anomaly-utils/*": ["x-pack/packages/ml/anomaly_utils/*"],
       "@kbn/ml-category-validator": ["x-pack/packages/ml/category_validator"],
       "@kbn/ml-category-validator/*": ["x-pack/packages/ml/category_validator/*"],
+      "@kbn/ml-chi2test": ["x-pack/packages/ml/chi2test"],
+      "@kbn/ml-chi2test/*": ["x-pack/packages/ml/chi2test/*"],
       "@kbn/ml-data-frame-analytics-utils": ["x-pack/packages/ml/data_frame_analytics_utils"],
       "@kbn/ml-data-frame-analytics-utils/*": ["x-pack/packages/ml/data_frame_analytics_utils/*"],
       "@kbn/ml-data-grid": ["x-pack/packages/ml/data_grid"],

x-pack/packages/ml/chi2test/README.md

@@ -0,0 +1,4 @@
+# @kbn/ml-chi2test
+
+`computeChi2PValue` computes the p-value for how similar the datasets are.
+Returned value ranges from 0 to 1, with 1 meaning the datasets are identical.

x-pack/plugins/data_visualizer/public/application/data_drift/data_drift_utils.test.ts → x-pack/packages/ml/chi2test/compute_chi_2_pvalue.test.ts

@@ -5,8 +5,8 @@
  * 2.0.
  */
 
-import { computeChi2PValue } from './data_drift_utils';
-import { Histogram } from './types';
+import { computeChi2PValue } from './compute_chi_2_pvalue';
+import type { Histogram } from './types';
 
 describe('computeChi2PValue()', () => {
   test('should return close to 1 if datasets are both empty or nearly identical', () => {
@@ -83,6 +83,6 @@ describe('computeChi2PValue()', () => {
         percentage: 1,
       },
     ];
-    expect(computeChi2PValue(referenceTerms, comparisonTerms)).toStrictEqual(0);
+    expect(computeChi2PValue(referenceTerms, comparisonTerms)).toStrictEqual(0.000001);
   });
 });

x-pack/packages/ml/chi2test/compute_chi_2_pvalue.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { criticalTableLookup } from './critical_table_lookup';
+import type { Histogram } from './types';
+
+/**
+ * Compute the p-value for how similar the datasets are.
+ * Returned value ranges from 0 to 1, with 1 meaning the datasets are identical.
+ *
+ * @param {Histogram[]} normalizedBaselineTerms - An array of normalized baseline terms (Histogram objects).
+ * @param {Histogram[]} normalizedDriftedTerms - An array of normalized drifted terms (Histogram objects).
+ * @returns {number} The p-value indicating the similarity of the datasets.
+ */
+export const computeChi2PValue = (
+  normalizedBaselineTerms: Histogram[],
+  normalizedDriftedTerms: Histogram[]
+) => {
+  // Get all unique keys from both arrays
+  const allKeys: string[] = Array.from(
+    new Set([
+      ...normalizedBaselineTerms.map((term) => term.key.toString()),
+      ...normalizedDriftedTerms.map((term) => term.key.toString()),
+    ])
+  ).slice(0, 100);
+
+  // Calculate the chi-squared statistic and degrees of freedom
+  let chiSquared: number = 0;
+  const degreesOfFreedom: number = allKeys.length - 1;
+
+  if (degreesOfFreedom === 0) return 1;
+
+  allKeys.forEach((key) => {
+    const baselineTerm = normalizedBaselineTerms.find((term) => term.key === key);
+    const driftedTerm = normalizedDriftedTerms.find((term) => term.key === key);
+
+    const observed: number = driftedTerm?.percentage ?? 0;
+    const expected: number = baselineTerm?.percentage ?? 0;
+    chiSquared += Math.pow(observed - expected, 2) / (expected > 0 ? expected : 1e-6); // Prevent divide by zero
+  });
+
+  // Use the criticalTableLookup function to determine the p-value
+  return criticalTableLookup(chiSquared, degreesOfFreedom);
+};