[WIP][Fleet] Granular integrations, phase 1

x-pack/plugins/fleet/common/services/group_inputs.test.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { groupInputs } from './group_inputs';
+import { packageWithPolicyTemplates } from './fixtures/packages';
+
+describe('groupInputs()', () => {
+  it('should group inputs for a package with multiple policy templates and input groups correctly', () => {
+    expect(groupInputs(packageWithPolicyTemplates)).toMatchSnapshot();
+  });
+});

x-pack/plugins/fleet/common/services/group_inputs.ts

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import type {
+  PackageInfo,
+  RegistryInputGroup,
+  RegistryVarsEntry,
+  RegistryStream,
+  NewPackagePolicyInputStream,
+} from '../types';
+
+type InputGroupWithStreams = RegistryInputGroup & {
+  vars?: RegistryVarsEntry[];
+  streams: Array<StreamWithDataStreamMeta & { policyTemplate?: string }>;
+};
+
+type StreamWithDataStreamMeta = RegistryStream & Pick<NewPackagePolicyInputStream, 'data_stream'>;
+
+import { doesPackageHaveIntegrations } from './';
+
+const findDataStreamsByNames = (
+  names: string[] = [],
+  dataStreams: PackageInfo['data_streams'] = []
+): PackageInfo['data_streams'] => {
+  return names.length
+    ? dataStreams.filter((dataStream) => names.includes(dataStream.path))
+    : dataStreams;
+};
+
+const getStreamsForInputType = (
+  inputType: string,
+  dataStreams: PackageInfo['data_streams'] = []
+): StreamWithDataStreamMeta[] => {
+  const streams: StreamWithDataStreamMeta[] = [];
+
+  dataStreams.forEach((dataStream) => {
+    (dataStream.streams || []).forEach((stream) => {
+      if (stream.input === inputType) {
+        streams.push({
+          ...stream,
+          data_stream: {
+            type: dataStream.type,
+            dataset: dataStream.dataset,
+          },
+        });
+      }
+    });
+  });
+
+  return streams;
+};
+
+export const groupInputs = (packageInfo: PackageInfo): InputGroupWithStreams[] => {
+  const inputGroups: InputGroupWithStreams[] = [
+    ...(packageInfo.input_groups?.map((inputGroup) => ({ ...inputGroup, streams: [] })) || []),
+  ];
+  const hasIntegrations = doesPackageHaveIntegrations(packageInfo);
+  const { policy_templates: policyTemplates = [] } = packageInfo;
+
+  // If no policy templates, return no groups
+  if (policyTemplates.length === 0) {
+    return inputGroups;
+  }
+
+  // If no integrations, return first policy template's inputs as groups
+  if (!hasIntegrations) {
+    (policyTemplates[0].inputs || []).forEach(({ type, ...input }) => {
+      inputGroups.push({
+        ...input,
+        name: type,
+        streams: getStreamsForInputType(type, packageInfo.data_streams),
+      });
+    });
+    return inputGroups;
+  }
+
+  // Otherwise look in each policy template
+  policyTemplates.forEach((policyTemplate) => {
+    const policyTemplateDataStreams = findDataStreamsByNames(
+      policyTemplate.data_streams,
+      packageInfo.data_streams
+    );
+
+    (policyTemplate.inputs || []).forEach(({ type, input_group: inputGroupName }) => {
+      // Package should already have `input_groups` defined and each input should have an `input_group`
+      // If either one of these conditions aren't met, skip adding this input
+      const inputGroup = inputGroupName
+        ? inputGroups.find((group) => group.name === inputGroupName)
+        : undefined;
+
+      if (!inputGroup) {
+        return;
+      }
+
+      inputGroup.streams.push(
+        ...getStreamsForInputType(type, policyTemplateDataStreams).map((stream) => ({
+          ...stream,
+          policyTemplate: policyTemplate.name,
+        }))
+      );
+    });
+  });
+
+  return inputGroups;
+};

x-pack/plugins/fleet/common/services/index.ts

@@ -16,3 +16,15 @@ export { isValidNamespace } from './is_valid_namespace';
 export { isDiffPathProtocol } from './is_diff_path_protocol';
 export { LicenseService } from './license';
 export { isAgentUpgradeable } from './is_agent_upgradeable';
+export { doesPackageHaveIntegrations } from './packages_with_integrations';
+export { groupInputs } from './group_inputs';
+
+export {
+  PackagePolicyValidationResults,
+  PackagePolicyConfigValidationResults,
+  PackagePolicyInputValidationResults,
+  validatePackagePolicy,
+  validatePackagePolicyConfig,
+  validationHasErrors,
+  countValidationErrors,
+} from './validate_package_policy';

x-pack/plugins/fleet/common/services/package_to_package_policy.test.ts

@@ -7,6 +7,7 @@
 
 import type { PackageInfo } from '../types';
 
+import { packageWithPolicyTemplates } from './fixtures/packages';
 import { packageToPackagePolicy, packageToPackagePolicyInputs } from './package_to_package_policy';
 
 describe('Fleet - packageToPackagePolicy', () => {
@@ -324,9 +325,11 @@ describe('Fleet - packageToPackagePolicy', () => {
 
   describe('packageToPackagePolicy', () => {
     it('returns package policy with default name', () => {
-      expect(packageToPackagePolicy(mockPackage, '1', '2')).toEqual({
+      expect(
+        packageToPackagePolicy({ packageInfo: mockPackage, agentPolicyId: '1', outputId: '2' })
+      ).toEqual({
         policy_id: '1',
-        namespace: '',
+        namespace: 'default',
         enabled: true,
         inputs: [],
         name: 'mock-package-1',
@@ -338,8 +341,16 @@ describe('Fleet - packageToPackagePolicy', () => {
         },
       });
     });
+
     it('returns package policy with custom name', () => {
-      expect(packageToPackagePolicy(mockPackage, '1', '2', 'default', 'pkgPolicy-1')).toEqual({
+      expect(
+        packageToPackagePolicy({
+          name: 'pkgPolicy-1',
+          packageInfo: mockPackage,
+          agentPolicyId: '1',
+          outputId: '2',
+        })
+      ).toEqual({
         policy_id: '1',
         namespace: 'default',
         enabled: true,
@@ -353,16 +364,17 @@ describe('Fleet - packageToPackagePolicy', () => {
         },
       });
     });
+
     it('returns package policy with namespace and description', () => {
       expect(
-        packageToPackagePolicy(
-          mockPackage,
-          '1',
-          '2',
-          'mock-namespace',
-          'pkgPolicy-1',
-          'Test description'
-        )
+        packageToPackagePolicy({
+          name: 'pkgPolicy-1',
+          namespace: 'mock-namespace',
+          description: 'Test description',
+          packageInfo: mockPackage,
+          agentPolicyId: '1',
+          outputId: '2',
+        })
       ).toEqual({
         policy_id: '1',
         enabled: true,
@@ -378,27 +390,58 @@ describe('Fleet - packageToPackagePolicy', () => {
         },
       });
     });
-    it('returns package policy with inputs', () => {
-      const mockPackageWithPolicyTemplates = ({
+
+    it('returns package policy with inputs and package-level vars', () => {
+      const mockPackageWithPackageVars = ({
         ...mockPackage,
         policy_templates: [{ inputs: [{ type: 'foo' }] }],
+        vars: [{ default: 'foo-var-value', name: 'var-name', type: 'text' }],
       } as unknown) as PackageInfo;
 
       expect(
-        packageToPackagePolicy(mockPackageWithPolicyTemplates, '1', '2', 'default', 'pkgPolicy-1')
+        packageToPackagePolicy({
+          name: 'pkgPolicy-1',
+          packageInfo: mockPackageWithPackageVars,
+          agentPolicyId: '1',
+          outputId: '2',
+        })
       ).toEqual({
         policy_id: '1',
         namespace: 'default',
         enabled: true,
-        inputs: [{ type: 'foo', enabled: true, streams: [] }],
         name: 'pkgPolicy-1',
         output_id: '2',
         package: {
           name: 'mock-package',
           title: 'Mock package',
           version: '0.0.0',
         },
+        inputs: [{ type: 'foo', enabled: true, streams: [] }],
+        vars: { 'var-name': { value: 'foo-var-value', type: 'text' } },
       });
     });
+
+    it('returns package policy with multiple policy templates correctly', () => {
+      expect(
+        packageToPackagePolicy({
+          name: 'aws-1',
+          packageInfo: packageWithPolicyTemplates,
+          agentPolicyId: 'some-policy-id',
+          outputId: 'some-output-id',
+        })
+      ).toMatchSnapshot();
+    });
+
+    it('only enables streams matching a particular policy template if given', () => {
+      expect(
+        packageToPackagePolicy({
+          name: 'aws-1',
+          packageInfo: packageWithPolicyTemplates,
+          agentPolicyId: 'some-policy-id',
+          outputId: 'some-output-id',
+          enablePolicyTemplate: 'dynamodb',
+        })
+      ).toMatchSnapshot();
+    });
   });
 });