Merge branch 'main' into remove_experimental_sec_solution

config/serverless.yml

@@ -1 +1,2 @@
 xpack.fleet.enableExperimental: ['fleetServerStandalone']
+xpack.fleet.internal.ILMPoliciesDisabled: true

package.json

@@ -1313,7 +1313,7 @@
     "backport": "^8.9.7",
     "callsites": "^3.1.0",
     "chance": "1.0.18",
-    "chromedriver": "^110.0.0",
+    "chromedriver": "^112.0.0",
     "clean-webpack-plugin": "^3.0.0",
     "compression-webpack-plugin": "^4.0.0",
     "copy-webpack-plugin": "^6.0.2",

src/dev/storybook/aliases.ts

@@ -50,4 +50,5 @@ export const storybookAliases = {
   triggers_actions_ui: 'x-pack/plugins/triggers_actions_ui/.storybook',
   ui_actions_enhanced: 'src/plugins/ui_actions_enhanced/.storybook',
   unified_search: 'src/plugins/unified_search/.storybook',
+  profiling: 'x-pack/plugins/profiling/.storybook',
 };

x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks.test.tsx

@@ -17,6 +17,7 @@ import { useCspBenchmarkIntegrations } from './use_csp_benchmark_integrations';
 import { useCspSetupStatusApi } from '../../common/api/use_setup_status_api';
 import { useSubscriptionStatus } from '../../common/hooks/use_subscription_status';
 import { useCspIntegrationLink } from '../../common/navigation/use_csp_integration_link';
+import { ERROR_STATE_TEST_SUBJECT } from './benchmarks_table';
 
 jest.mock('./use_csp_benchmark_integrations');
 jest.mock('../../common/api/use_setup_status_api');
@@ -80,7 +81,7 @@ describe('<Benchmarks />', () => {
     const error = new Error('message');
     renderBenchmarks(createReactQueryResponse({ status: 'error', error }));
 
-    expect(screen.getByText(error.message)).toBeInTheDocument();
+    expect(screen.getByTestId(ERROR_STATE_TEST_SUBJECT)).toBeInTheDocument();
   });
 
   it('renders the benchmarks table', () => {

x-pack/plugins/cloud_security_posture/public/pages/benchmarks/benchmarks_table.tsx

@@ -14,18 +14,24 @@ import {
   EuiLink,
   EuiToolTip,
   EuiAvatar,
+  EuiEmptyPrompt,
 } from '@elastic/eui';
 import React from 'react';
 import { generatePath } from 'react-router-dom';
 import { pagePathGetters } from '@kbn/fleet-plugin/public';
 import { i18n } from '@kbn/i18n';
 import type { PackagePolicy } from '@kbn/fleet-plugin/common';
+import { FormattedMessage } from '@kbn/i18n-react';
 import { TimestampTableCell } from '../../components/timestamp_table_cell';
 import type { Benchmark } from '../../../common/types';
 import { useKibana } from '../../common/hooks/use_kibana';
 import { benchmarksNavigation } from '../../common/navigation/constants';
 import * as TEST_SUBJ from './test_subjects';
 import { getEnabledCspIntegrationDetails } from '../../common/utils/get_enabled_csp_integration_details';
+import { isCommonError } from '../../components/cloud_posture_page';
+import { FullSizeCenteredPage } from '../../components/full_size_centered_page';
+
+export const ERROR_STATE_TEST_SUBJECT = 'benchmark_page_error';
 
 interface BenchmarksTableProps
   extends Pick<EuiBasicTableProps<Benchmark>, 'loading' | 'error' | 'noItemsMessage' | 'sorting'>,
@@ -67,6 +73,39 @@ const IntegrationButtonLink = ({
   );
 };
 
+const ErrorMessageComponent = (error: { error: unknown }) => (
+  <FullSizeCenteredPage>
+    <EuiEmptyPrompt
+      color="danger"
+      iconType="warning"
+      data-test-subj={ERROR_STATE_TEST_SUBJECT}
+      title={
+        <h2>
+          <FormattedMessage
+            id="xpack.csp.benchmarks.benchmarksTable.errorRenderer.errorTitle"
+            defaultMessage="We couldn't fetch your cloud security posture benchmark data"
+          />
+        </h2>
+      }
+      body={
+        isCommonError(error) ? (
+          <p>
+            <FormattedMessage
+              id="xpack.csp.benchmarks.benchmarksTable.errorRenderer.errorDescription"
+              defaultMessage="{error} {statusCode}: {body}"
+              values={{
+                error: error.body.error,
+                statusCode: error.body.statusCode,
+                body: error.body.message,
+              }}
+            />
+          </p>
+        ) : undefined
+      }
+    />
+  </FullSizeCenteredPage>
+);
+
 const BENCHMARKS_TABLE_COLUMNS: Array<EuiBasicTableColumn<Benchmark>> = [
   {
     field: 'package_policy.name',
@@ -193,6 +232,10 @@ export const BenchmarksTable = ({
     setQuery({ page: { ...page, index: page.index + 1 }, sort });
   };
 
+  if (error) {
+    return <ErrorMessageComponent error={error} />;
+  }
+
   return (
     <EuiBasicTable
       data-test-subj={rest['data-test-subj']}

x-pack/plugins/cloud_security_posture/public/pages/configurations/findings_flyout/findings_flyout.tsx

@@ -93,7 +93,7 @@ export const CisKubernetesIcons = ({
   benchmarkId: BenchmarkId;
   benchmarkName: BenchmarkName;
 }) => (
-  <EuiFlexGroup gutterSize="s">
+  <EuiFlexGroup gutterSize="s" alignItems="center">
     <EuiFlexItem grow={false}>
       <EuiIcon type={cisLogoIcon} size="xxl" />
     </EuiFlexItem>

x-pack/plugins/fleet/common/types/index.ts

@@ -44,6 +44,9 @@ export interface FleetConfigType {
     disableRegistryVersionCheck?: boolean;
     bundledPackageLocation?: string;
   };
+  internal?: {
+    disableILMPolicies: boolean;
+  };
 }
 
 // Calling Object.entries(PackagesGroupedByStatus) gave `status: string`

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -471,7 +471,7 @@ export type ExperimentalIndexingFeature =
 
 export interface ExperimentalDataStreamFeature {
   data_stream: string;
-  features: Record<ExperimentalIndexingFeature, boolean>;
+  features: Partial<Record<ExperimentalIndexingFeature, boolean>>;
 }
 
 export interface Installation extends SavedObjectAttributes {
@@ -493,7 +493,7 @@ export interface Installation extends SavedObjectAttributes {
   // TypeScript doesn't like using the `ExperimentalDataStreamFeature` type defined above here
   experimental_data_stream_features?: Array<{
     data_stream: string;
-    features: Record<ExperimentalIndexingFeature, boolean>;
+    features: Partial<Record<ExperimentalIndexingFeature, boolean>>;
   }>;
 }
 

x-pack/plugins/fleet/server/config.ts

@@ -158,6 +158,14 @@ export const config: PluginConfigDescriptor = {
         }
       },
     }),
+
+    internal: schema.maybe(
+      schema.object({
+        disableILMPolicies: schema.boolean({
+          defaultValue: false,
+        }),
+      })
+    ),
   }),
 };
 

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.ts

@@ -61,12 +61,18 @@ export function buildDefaultSettings({
       : defaultFields
   ).map((field) => field.name);
 
+  const isILMPolicyDisabled = appContextService.getConfig()?.internal?.disableILMPolicies ?? false;
+
   return {
     index: {
-      // ILM Policy must be added here, for now point to the default global ILM policy name
-      lifecycle: {
-        name: ilmPolicy ? ilmPolicy : type,
-      },
+      ...(isILMPolicyDisabled
+        ? {}
+        : {
+            // ILM Policy must be added here, for now point to the default global ILM policy name
+            lifecycle: {
+              name: ilmPolicy ? ilmPolicy : type,
+            },
+          }),
       // What should be our default for the compression?
       codec: 'best_compression',
       // All the default fields which should be queried have to be added here.

x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts

@@ -8,18 +8,24 @@
 import type { SavedObjectsClientContract, ElasticsearchClient } from '@kbn/core/server';
 import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks';
 import { loggerMock } from '@kbn/logging-mocks';
-
 import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants';
 
 import { appContextService } from '../../app_context';
 import { createAppContextStartContractMock } from '../../../mocks';
+import { saveArchiveEntries } from '../archive/storage';
+import { installILMPolicy } from '../elasticsearch/ilm/install';
+import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install';
 
 jest.mock('../elasticsearch/template/template');
 jest.mock('../kibana/assets/install');
 jest.mock('../kibana/index_pattern/install');
 jest.mock('./install');
 jest.mock('./get');
 
+jest.mock('../archive/storage');
+jest.mock('../elasticsearch/ilm/install');
+jest.mock('../elasticsearch/datastream_ilm/install');
+
 import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
 import { installKibanaAssetsAndReferences } from '../kibana/assets/install';
 
@@ -47,8 +53,22 @@ describe('_installPackage', () => {
 
   beforeEach(async () => {
     soClient = savedObjectsClientMock.create();
+
+    soClient.update.mockImplementation(async (type, id, attributes) => {
+      return { id, attributes } as any;
+    });
+
     esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
     appContextService.start(createAppContextStartContractMock());
+    jest.mocked(installILMPolicy).mockReset();
+    jest.mocked(installIlmForDataStream).mockReset();
+    jest.mocked(installIlmForDataStream).mockResolvedValue({
+      esReferences: [],
+      installedIlms: [],
+    });
+    jest.mocked(saveArchiveEntries).mockResolvedValue({
+      saved_objects: [],
+    });
   });
   afterEach(async () => {
     appContextService.stop();
@@ -96,4 +116,100 @@ describe('_installPackage', () => {
     await expect(installationPromise).rejects.toThrow('mocked');
     await expect(installationPromise).rejects.toThrow('should be caught');
   });
+
+  it('do not install ILM policies if disabled in config', async () => {
+    appContextService.start(
+      createAppContextStartContractMock({
+        internal: {
+          disableILMPolicies: true,
+        },
+      })
+    );
+    // force errors from this function
+    mockedInstallKibanaAssetsAndReferences.mockResolvedValue([]);
+    // pick any function between when those are called and when await Promise.all is defined later
+    // and force it to take long enough for the errors to occur
+    // @ts-expect-error about call signature
+    mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000));
+    mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({
+      installedTemplates: [],
+      esReferences: [],
+    });
+    await _installPackage({
+      savedObjectsClient: soClient,
+      // @ts-ignore
+      savedObjectsImporter: jest.fn(),
+      esClient,
+      logger: loggerMock.create(),
+      paths: [],
+      packageInfo: {
+        title: 'title',
+        name: 'xyz',
+        version: '4.5.6',
+        description: 'test',
+        type: 'integration',
+        categories: ['cloud', 'custom'],
+        format_version: 'string',
+        release: 'experimental',
+        conditions: { kibana: { version: 'x.y.z' } },
+        owner: { github: 'elastic/fleet' },
+      },
+      installType: 'install',
+      installSource: 'registry',
+      spaceId: DEFAULT_SPACE_ID,
+    });
+
+    expect(installILMPolicy).not.toBeCalled();
+    expect(installIlmForDataStream).not.toBeCalled();
+    // if we have a .catch this will fail nicely (test pass)
+    // otherwise the test will fail with either of the mocked errors
+    // await expect(installationPromise).rejects.toThrow('mocked');
+    // await expect(installationPromise).rejects.toThrow('should be caught');
+  });
+
+  it('install ILM policies if not disabled in config', async () => {
+    appContextService.start(
+      createAppContextStartContractMock({
+        internal: {
+          disableILMPolicies: false,
+        },
+      })
+    );
+    // force errors from this function
+    mockedInstallKibanaAssetsAndReferences.mockResolvedValue([]);
+    // pick any function between when those are called and when await Promise.all is defined later
+    // and force it to take long enough for the errors to occur
+    // @ts-expect-error about call signature
+    mockedUpdateCurrentWriteIndices.mockImplementation(async () => await sleep(1000));
+    mockedInstallIndexTemplatesAndPipelines.mockResolvedValue({
+      installedTemplates: [],
+      esReferences: [],
+    });
+    await _installPackage({
+      savedObjectsClient: soClient,
+      // @ts-ignore
+      savedObjectsImporter: jest.fn(),
+      esClient,
+      logger: loggerMock.create(),
+      paths: [],
+      packageInfo: {
+        title: 'title',
+        name: 'xyz',
+        version: '4.5.6',
+        description: 'test',
+        type: 'integration',
+        categories: ['cloud', 'custom'],
+        format_version: 'string',
+        release: 'experimental',
+        conditions: { kibana: { version: 'x.y.z' } },
+        owner: { github: 'elastic/fleet' },
+      },
+      installType: 'install',
+      installSource: 'registry',
+      spaceId: DEFAULT_SPACE_ID,
+    });
+
+    expect(installILMPolicy).toBeCalled();
+    expect(installIlmForDataStream).toBeCalled();
+  });
 });

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -153,20 +153,24 @@ export async function _installPackage({
     // currently only the base package has an ILM policy
     // at some point ILM policies can be installed/modified
     // per data stream and we should then save them
-    esReferences = await withPackageSpan('Install ILM policies', () =>
-      installILMPolicy(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences)
-    );
+    const isILMPoliciesDisabled =
+      appContextService.getConfig()?.internal?.disableILMPolicies ?? false;
+    if (!isILMPoliciesDisabled) {
+      esReferences = await withPackageSpan('Install ILM policies', () =>
+        installILMPolicy(packageInfo, paths, esClient, savedObjectsClient, logger, esReferences)
+      );
 
-    ({ esReferences } = await withPackageSpan('Install Data Stream ILM policies', () =>
-      installIlmForDataStream(
-        packageInfo,
-        paths,
-        esClient,
-        savedObjectsClient,
-        logger,
-        esReferences
-      )
-    ));
+      ({ esReferences } = await withPackageSpan('Install Data Stream ILM policies', () =>
+        installIlmForDataStream(
+          packageInfo,
+          paths,
+          esClient,
+          savedObjectsClient,
+          logger,
+          esReferences
+        )
+      ));
+    }
 
     // installs ml models
     esReferences = await withPackageSpan('Install ML models', () =>

x-pack/plugins/fleet/server/services/epm/packages/update.ts

@@ -56,7 +56,7 @@ export async function updateDatastreamExperimentalFeatures(
   pkgName: string,
   dataStreamFeatureMapping: Array<{
     data_stream: string;
-    features: Record<ExperimentalIndexingFeature, boolean>;
+    features: Partial<Record<ExperimentalIndexingFeature, boolean>>;
   }>
 ) {
   auditLoggingService.writeCustomSoAuditLog({