Add toggle to enable/disable rule install from SOs (#106189)

* Add toggle to enable/disable rule install from SOs
* Fix lint and type tests
* Fix types for tests
* Enable prebuiltRulesFromSavedObjects with extra rules in fleet
* Restore rules and disable prebuiltRulesFromSavedObjects
* Pass explicit arguments instead of full config
* Set values in 'endpoint' setup
* Propagate prebuiltRulesFrom{FileSystem,SavedObjects} from configs

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -389,6 +389,8 @@ kibana_vars=(
     xpack.securitySolution.maxTimelineImportExportSize
     xpack.securitySolution.maxTimelineImportPayloadBytes
     xpack.securitySolution.packagerTaskInterval
+    xpack.securitySolution.prebuiltRulesFromFileSystem
+    xpack.securitySolution.prebuiltRulesFromSavedObjects
     xpack.spaces.enabled
     xpack.spaces.maxSpaces
     xpack.task_manager.enabled

x-pack/plugins/security_solution/server/config.ts

@@ -64,6 +64,12 @@ export const configSchema = schema.object({
    * Artifacts Configuration
    */
   packagerTaskInterval: schema.string({ defaultValue: '60s' }),
+
+  /**
+   * Detection prebuilt rules
+   */
+  prebuiltRulesFromFileSystem: schema.boolean({ defaultValue: true }),
+  prebuiltRulesFromSavedObjects: schema.boolean({ defaultValue: true }),
 });
 
 export const createConfig = (context: PluginInitializerContext) =>

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts

@@ -94,6 +94,8 @@ export class EndpointAppContextService {
           this.manifestManager,
           dependencies.appClientFactory,
           dependencies.config.maxTimelineImportExportSize,
+          dependencies.config.prebuiltRulesFromFileSystem,
+          dependencies.config.prebuiltRulesFromSavedObjects,
           dependencies.security,
           dependencies.alerting,
           dependencies.licenseService,

x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts

@@ -42,6 +42,8 @@ describe('ingest_integration tests ', () => {
   let ctx: SecuritySolutionRequestHandlerContext;
   const exceptionListClient: ExceptionListClient = getExceptionListClientMock();
   const maxTimelineImportExportSize = createMockConfig().maxTimelineImportExportSize;
+  const prebuiltRulesFromFileSystem = createMockConfig().prebuiltRulesFromFileSystem;
+  const prebuiltRulesFromSavedObjects = createMockConfig().prebuiltRulesFromSavedObjects;
   let licenseEmitter: Subject<ILicense>;
   let licenseService: LicenseService;
   const Platinum = licenseMock.createLicense({ license: { type: 'platinum', mode: 'platinum' } });
@@ -80,6 +82,8 @@ describe('ingest_integration tests ', () => {
         manifestManager,
         endpointAppContextMock.appClientFactory,
         maxTimelineImportExportSize,
+        prebuiltRulesFromFileSystem,
+        prebuiltRulesFromSavedObjects,
         endpointAppContextMock.security,
         endpointAppContextMock.alerting,
         licenseService,

x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts

@@ -34,6 +34,8 @@ export const getPackagePolicyCreateCallback = (
   manifestManager: ManifestManager,
   appClientFactory: AppClientFactory,
   maxTimelineImportExportSize: number,
+  prebuiltRulesFromFileSystem: boolean,
+  prebuiltRulesFromSavedObjects: boolean,
   securityStart: SecurityPluginStart,
   alerts: AlertsStartContract,
   licenseService: LicenseService,
@@ -61,6 +63,8 @@ export const getPackagePolicyCreateCallback = (
           securityStart,
           alerts,
           maxTimelineImportExportSize,
+          prebuiltRulesFromFileSystem,
+          prebuiltRulesFromSavedObjects,
           exceptionsClient,
         }),
 

x-pack/plugins/security_solution/server/fleet_integration/handlers/install_prepackaged_rules.ts

@@ -22,6 +22,8 @@ export interface InstallPrepackagedRulesProps {
   securityStart: SecurityPluginStart;
   alerts: AlertsStartContract;
   maxTimelineImportExportSize: number;
+  prebuiltRulesFromFileSystem: boolean;
+  prebuiltRulesFromSavedObjects: boolean;
   exceptionsClient: ExceptionListClient;
 }
 
@@ -37,6 +39,8 @@ export const installPrepackagedRules = async ({
   securityStart,
   alerts,
   maxTimelineImportExportSize,
+  prebuiltRulesFromFileSystem,
+  prebuiltRulesFromSavedObjects,
   exceptionsClient,
 }: InstallPrepackagedRulesProps): Promise<void> => {
   // prep for detection rules creation
@@ -70,6 +74,8 @@ export const installPrepackagedRules = async ({
       alerts.getAlertsClientWithRequest(request),
       frameworkRequest,
       maxTimelineImportExportSize,
+      prebuiltRulesFromFileSystem,
+      prebuiltRulesFromSavedObjects,
       exceptionsClient
     );
   } catch (err) {

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/index.ts

@@ -26,6 +26,8 @@ export const createMockConfig = (): ConfigType => ({
   endpointResultListDefaultPageSize: 10,
   packagerTaskInterval: '60s',
   alertMergeStrategy: 'missingFields',
+  prebuiltRulesFromFileSystem: true,
+  prebuiltRulesFromSavedObjects: false,
 });
 
 export const mockGetCurrentUser = {

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts

@@ -292,13 +292,16 @@ describe('add_prepackaged_rules_route', () => {
         getExceptionListClient: jest.fn(),
         getListClient: jest.fn(),
       };
+      const config = createMockConfig();
 
       await createPrepackagedRules(
         context,
         siemMockClient,
         clients.alertsClient,
         {} as FrameworkRequest,
         1200,
+        config.prebuiltRulesFromFileSystem,
+        config.prebuiltRulesFromSavedObjects,
         mockExceptionsClient
       );
 
@@ -308,13 +311,16 @@ describe('add_prepackaged_rules_route', () => {
 
     test('uses passed in exceptions list client when lists client not available in context', async () => {
       const { lists, ...myContext } = context;
+      const config = createMockConfig();
 
       await createPrepackagedRules(
         myContext,
         siemMockClient,
         clients.alertsClient,
         {} as FrameworkRequest,
         1200,
+        config.prebuiltRulesFromFileSystem,
+        config.prebuiltRulesFromSavedObjects,
         mockExceptionsClient
       );
 

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts

@@ -77,7 +77,9 @@ export const addPrepackedRulesRoute = (
           siemClient,
           alertsClient,
           frameworkRequest,
-          config.maxTimelineImportExportSize
+          config.maxTimelineImportExportSize,
+          config.prebuiltRulesFromFileSystem,
+          config.prebuiltRulesFromSavedObjects
         );
         return response.ok({ body: validated ?? {} });
       } catch (err) {
@@ -104,7 +106,9 @@ export const createPrepackagedRules = async (
   siemClient: AppClient,
   alertsClient: AlertsClient,
   frameworkRequest: FrameworkRequest,
-  maxTimelineImportExportSize: number,
+  maxTimelineImportExportSize: ConfigType['maxTimelineImportExportSize'],
+  prebuiltRulesFromFileSystem: ConfigType['prebuiltRulesFromFileSystem'],
+  prebuiltRulesFromSavedObjects: ConfigType['prebuiltRulesFromSavedObjects'],
   exceptionsClient?: ExceptionListClient
 ): Promise<PrePackagedRulesAndTimelinesSchema | null> => {
   const esClient = context.core.elasticsearch.client;
@@ -121,7 +125,11 @@ export const createPrepackagedRules = async (
     await exceptionsListClient.createEndpointList();
   }
 
-  const latestPrepackagedRules = await getLatestPrepackagedRules(ruleAssetsClient);
+  const latestPrepackagedRules = await getLatestPrepackagedRules(
+    ruleAssetsClient,
+    prebuiltRulesFromFileSystem,
+    prebuiltRulesFromSavedObjects
+  );
   const prepackagedRules = await getExistingPrepackagedRules({ alertsClient });
   const rulesToInstall = getRulesToInstall(latestPrepackagedRules, prepackagedRules);
   const rulesToUpdate = getRulesToUpdate(latestPrepackagedRules, prepackagedRules);

x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts

@@ -53,7 +53,11 @@ export const getPrepackagedRulesStatusRoute = (
       }
 
       try {
-        const latestPrepackagedRules = await getLatestPrepackagedRules(ruleAssetsClient);
+        const latestPrepackagedRules = await getLatestPrepackagedRules(
+          ruleAssetsClient,
+          config.prebuiltRulesFromFileSystem,
+          config.prebuiltRulesFromSavedObjects
+        );
         const customRules = await findRules({
           alertsClient,
           perPage: 1,

x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts

@@ -21,6 +21,7 @@ import { rawRules } from './prepackaged_rules';
 import { RuleAssetSavedObjectsClient } from './rule_asset_saved_objects_client';
 import { IRuleAssetSOAttributes } from './types';
 import { SavedObjectAttributes } from '../../../../../../../src/core/types';
+import { ConfigType } from '../../../config';
 
 /**
  * Validate the rules from the file system and throw any errors indicating to the developer
@@ -103,21 +104,25 @@ export const getPrepackagedRules = (
 };
 
 export const getLatestPrepackagedRules = async (
-  client: RuleAssetSavedObjectsClient
+  client: RuleAssetSavedObjectsClient,
+  prebuiltRulesFromFileSystem: ConfigType['prebuiltRulesFromFileSystem'],
+  prebuiltRulesFromSavedObjects: ConfigType['prebuiltRulesFromSavedObjects']
 ): Promise<AddPrepackagedRulesSchemaDecoded[]> => {
   // build a map of the most recent version of each rule
-  const prepackaged = getPrepackagedRules();
+  const prepackaged = prebuiltRulesFromFileSystem ? getPrepackagedRules() : [];
   const ruleMap = new Map(prepackaged.map((r) => [r.rule_id, r]));
 
   // check the rules installed via fleet and create/update if the version is newer
-  const fleetRules = await getFleetInstalledRules(client);
-  const fleetUpdates = fleetRules.filter((r) => {
-    const rule = ruleMap.get(r.rule_id);
-    return rule == null || rule.version < r.version;
-  });
+  if (prebuiltRulesFromSavedObjects) {
+    const fleetRules = await getFleetInstalledRules(client);
+    const fleetUpdates = fleetRules.filter((r) => {
+      const rule = ruleMap.get(r.rule_id);
+      return rule == null || rule.version < r.version;
+    });
 
-  // add the new or updated rules to the map
-  fleetUpdates.forEach((r) => ruleMap.set(r.rule_id, r));
+    // add the new or updated rules to the map
+    fleetUpdates.forEach((r) => ruleMap.set(r.rule_id, r));
+  }
 
   return Array.from(ruleMap.values());
 };

x-pack/test/security_solution_cypress/config.ts

@@ -37,6 +37,9 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) {
         '--csp.strict=false',
         // define custom kibana server args here
         `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`,
+        // retrieve rules from the filesystem but not from fleet for Cypress tests
+        '--xpack.securitySolution.prebuiltRulesFromFileSystem=true',
+        '--xpack.securitySolution.prebuiltRulesFromSavedObjects=false',
       ],
     },
   };