[Fleet] Support logstash as an output type

elastic · Feb 21, 2022 · c7db63b · c7db63b
1 parent 5555ac7
commit c7db63b
Show file tree

Hide file tree

Showing 12 changed files with 199 additions and 42 deletions.
diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts
@@ -11,6 +11,7 @@ export const OUTPUT_SAVED_OBJECT_TYPE = 'ingest-outputs';
 
 export const outputType = {
   Elasticsearch: 'elasticsearch',
+  Logstash: 'logstash',
 } as const;
 
 export const DEFAULT_OUTPUT_ID = 'fleet-default-output';

diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/output.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/output.yaml
@@ -24,6 +24,15 @@ properties:
     type: object
   config_yaml:
     type: string
+  ssl:
+    type: object
+    properties: 
+      certificate:
+        type: string
+      certificate_authorities:
+        type: string
+      key:
+        type: string
 required:
   - id
   - is_default

diff --git a/x-pack/plugins/fleet/common/types/models/output.ts b/x-pack/plugins/fleet/common/types/models/output.ts
@@ -13,13 +13,18 @@ export type OutputType = typeof outputType;
 export interface NewOutput {
   is_default: boolean;
   is_default_monitoring: boolean;
+  is_preconfigured?: boolean;
   name: string;
   type: ValueOf<OutputType>;
   hosts?: string[];
   ca_sha256?: string;
   ca_trusted_fingerprint?: string;
   config_yaml?: string;
-  is_preconfigured?: boolean;
+  ssl?: {
+    certificate_authorities?: string[];
+    certificate?: string;
+    key?: string;
+  };
 }
 
 export type OutputSOAttributes = NewOutput & {

diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -117,6 +117,7 @@ const getSavedObjectTypes = (
         config: { type: 'flattened' },
         config_yaml: { type: 'text' },
         is_preconfigured: { type: 'boolean', index: false },
+        ssl: { type: 'flattened', index: false },
       },
     },
     migrations: {

diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts
@@ -352,4 +352,54 @@ ssl.test: 123
       }
     `);
   });
+
+  it('should return placeholder ES_USERNAME and ES_PASSWORD for elasticsearch output type in standalone ', () => {
+    const policyOutput = transformOutputToFullPolicyOutput(
+      {
+        id: 'id123',
+        hosts: ['http://host.fr'],
+        is_default: false,
+        is_default_monitoring: false,
+        name: 'test output',
+        type: 'elasticsearch',
+      },
+      true
+    );
+
+    expect(policyOutput).toMatchInlineSnapshot(`
+      Object {
+        "ca_sha256": undefined,
+        "hosts": Array [
+          "http://host.fr",
+        ],
+        "password": "{ES_PASSWORD}",
+        "type": "elasticsearch",
+        "username": "{ES_USERNAME}",
+      }
+    `);
+  });
+
+  it('should not return placeholder ES_USERNAME and ES_PASSWORD for logstash output type in standalone ', () => {
+    const policyOutput = transformOutputToFullPolicyOutput(
+      {
+        id: 'id123',
+        hosts: ['host.fr:3332'],
+        is_default: false,
+        is_default_monitoring: false,
+        name: 'test output',
+        type: 'logstash',
+      },
+      true
+    );
+
+    expect(policyOutput).toMatchInlineSnapshot(`
+      Object {
+        "ca_sha256": undefined,
+        "hosts": Array [
+          "host.fr:3332",
+        ],
+        "type": "logstash",
+      }
+    `);
+  });
 });
diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts
@@ -171,17 +171,18 @@ export function transformOutputToFullPolicyOutput(
   standalone = false
 ): FullAgentPolicyOutput {
   // eslint-disable-next-line @typescript-eslint/naming-convention
-  const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint } = output;
+  const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint, ssl } = output;
   const configJs = config_yaml ? safeLoad(config_yaml) : {};
   const newOutput: FullAgentPolicyOutput = {
     ...configJs,
     type,
     hosts,
     ca_sha256,
+    ...(ssl ? { ssl } : {}),
     ...(ca_trusted_fingerprint ? { 'ssl.ca_trusted_fingerprint': ca_trusted_fingerprint } : {}),
   };
 
-  if (standalone) {
+  if (output.type === outputType.Elasticsearch && standalone) {
     newOutput.username = '{ES_USERNAME}';
     newOutput.password = '{ES_PASSWORD}';
   }

diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts
@@ -10,7 +10,7 @@ import uuid from 'uuid/v5';
 
 import type { NewOutput, Output, OutputSOAttributes } from '../types';
 import { DEFAULT_OUTPUT, DEFAULT_OUTPUT_ID, OUTPUT_SAVED_OBJECT_TYPE } from '../constants';
-import { decodeCloudId, normalizeHostsForAgents, SO_SEARCH_LIMIT } from '../../common';
+import { decodeCloudId, normalizeHostsForAgents, SO_SEARCH_LIMIT, outputType } from '../../common';
 import { OutputUnauthorizedError } from '../errors';
 
 import { appContextService } from './app_context';
@@ -149,7 +149,7 @@ class OutputService {
       }
     }
 
-    if (data.hosts) {
+    if (data.type === outputType.Elasticsearch && data.hosts) {
       data.hosts = data.hosts.map(normalizeHostsForAgents);
     }
 
@@ -260,7 +260,7 @@ class OutputService {
       );
     }
 
-    const updateData = { ...data };
+    const updateData = { type: originalOutput.type, ...data };
 
     // ensure only default output exists
     if (data.is_default) {
@@ -287,7 +287,7 @@ class OutputService {
       }
     }
 
-    if (updateData.hosts) {
+    if (updateData.type === outputType.Elasticsearch && updateData.hosts) {
       updateData.hosts = updateData.hosts.map(normalizeHostsForAgents);
     }
     const outputSO = await soClient.update<OutputSOAttributes>(

diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts
@@ -58,6 +58,7 @@ function isPreconfiguredOutputDifferentFromCurrent(
         existingOutput.hosts?.map(normalizeHostsForAgents),
         preconfiguredOutput.hosts.map(normalizeHostsForAgents)
       )) ||
+    (preconfiguredOutput.ssl && !isEqual(preconfiguredOutput.ssl, existingOutput.ssl)) ||
     existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 ||
     existingOutput.ca_trusted_fingerprint !== preconfiguredOutput.ca_trusted_fingerprint ||
     existingOutput.config_yaml !== preconfiguredOutput.config_yaml

diff --git a/x-pack/plugins/fleet/server/types/models/output.ts b/x-pack/plugins/fleet/server/types/models/output.ts
@@ -9,16 +9,76 @@ import { schema } from '@kbn/config-schema';
 
 import { outputType } from '../../../common/constants';
 
+export function validateLogstashHost(val: string) {
+  if (val.match(/^http([s]){0,1}:\/\//)) {
+    throw new Error('Invalid logstash host should not start with http(s)');
+  }
+
+  try {
+    const url = new URL(`http://${val}`);
+
+    if (url.host !== val) {
+      throw new Error('Invalid host');
+    }
+  } catch (err) {
+    throw new Error('Invalid logstash host');
+  }
+}
+
 const OutputBaseSchema = {
+  id: schema.maybe(schema.string()),
   name: schema.string(),
-  type: schema.oneOf([schema.literal(outputType.Elasticsearch)]),
-  hosts: schema.maybe(schema.arrayOf(schema.string())),
-  config: schema.maybe(schema.recordOf(schema.string(), schema.any())),
+  type: schema.oneOf([
+    schema.literal(outputType.Elasticsearch),
+    schema.literal(outputType.Logstash),
+  ]),
+  hosts: schema.conditional(
+    schema.siblingRef('type'),
+    schema.literal(outputType.Elasticsearch),
+    schema.arrayOf(schema.uri({ scheme: ['http', 'https'] })),
+    schema.arrayOf(schema.string({ validate: validateLogstashHost }))
+  ),
+  is_default: schema.boolean({ defaultValue: false }),
+  is_default_monitoring: schema.boolean({ defaultValue: false }),
+  ca_sha256: schema.maybe(schema.string()),
+  ca_trusted_fingerprint: schema.maybe(schema.string()),
   config_yaml: schema.maybe(schema.string()),
+  ssl: schema.maybe(
+    schema.object({
+      certificate_authorities: schema.maybe(schema.arrayOf(schema.string())),
+      certificate: schema.maybe(schema.string()),
+      key: schema.maybe(schema.string()),
+    })
+  ),
 };
 
-export const NewOutputSchema = schema.object({
-  ...OutputBaseSchema,
+export const NewOutputSchema = schema.object({ ...OutputBaseSchema });
+
+export const UpdateOutputSchema = schema.object({
+  name: schema.maybe(schema.string()),
+  type: schema.maybe(
+    schema.oneOf([schema.literal(outputType.Elasticsearch), schema.literal(outputType.Logstash)])
+  ),
+  hosts: schema.maybe(
+    schema.conditional(
+      schema.siblingRef('type'),
+      schema.literal(outputType.Elasticsearch),
+      schema.arrayOf(schema.uri({ scheme: ['http', 'https'] })),
+      schema.arrayOf(schema.string({ validate: validateLogstashHost }))
+    )
+  ),
+  is_default: schema.maybe(schema.boolean()),
+  is_default_monitoring: schema.maybe(schema.boolean()),
+  ca_sha256: schema.maybe(schema.string()),
+  ca_trusted_fingerprint: schema.maybe(schema.string()),
+  config_yaml: schema.maybe(schema.string()),
+  ssl: schema.maybe(
+    schema.object({
+      certificate_authorities: schema.maybe(schema.arrayOf(schema.string())),
+      certificate: schema.maybe(schema.string()),
+      key: schema.maybe(schema.string()),
+    })
+  ),
 });
 
 export const OutputSchema = schema.object({

diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts
@@ -10,10 +10,10 @@ import semverValid from 'semver/functions/valid';
 
 import { PRECONFIGURATION_LATEST_KEYWORD } from '../../constants';
 import type { PreconfiguredOutput } from '../../../common';
-import { outputType } from '../../../common';
 
 import { AgentPolicyBaseSchema } from './agent_policy';
 import { NamespaceSchema } from './package_policy';
+import { NewOutputSchema } from './output';
 
 const varsSchema = schema.maybe(
   schema.arrayOf(
@@ -74,16 +74,10 @@ function validatePreconfiguredOutputs(outputs: PreconfiguredOutput[]) {
 }
 
 export const PreconfiguredOutputsSchema = schema.arrayOf(
-  schema.object({
+  NewOutputSchema.extends({
     id: schema.string(),
-    is_default: schema.boolean({ defaultValue: false }),
-    is_default_monitoring: schema.boolean({ defaultValue: false }),
-    name: schema.string(),
-    type: schema.oneOf([schema.literal(outputType.Elasticsearch)]),
-    hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }))),
-    ca_sha256: schema.maybe(schema.string()),
-    ca_trusted_fingerprint: schema.maybe(schema.string()),
     config: schema.maybe(schema.object({}, { unknowns: 'allow' })),
+    config_yaml: schema.never(),
   }),
   {
     defaultValue: [],

diff --git a/x-pack/plugins/fleet/server/types/rest_spec/output.ts b/x-pack/plugins/fleet/server/types/rest_spec/output.ts
@@ -7,6 +7,8 @@
 
 import { schema } from '@kbn/config-schema';
 
+import { NewOutputSchema, UpdateOutputSchema } from '../models';
+
 export const GetOneOutputRequestSchema = {
   params: schema.object({
     outputId: schema.string(),
@@ -22,31 +24,12 @@ export const DeleteOutputRequestSchema = {
 export const GetOutputsRequestSchema = {};
 
 export const PostOutputRequestSchema = {
-  body: schema.object({
-    id: schema.maybe(schema.string()),
-    name: schema.string(),
-    type: schema.oneOf([schema.literal('elasticsearch')]),
-    is_default: schema.boolean({ defaultValue: false }),
-    is_default_monitoring: schema.boolean({ defaultValue: false }),
-    hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }))),
-    ca_sha256: schema.maybe(schema.string()),
-    ca_trusted_fingerprint: schema.maybe(schema.string()),
-    config_yaml: schema.maybe(schema.string()),
-  }),
+  body: NewOutputSchema,
 };
 
 export const PutOutputRequestSchema = {
   params: schema.object({
     outputId: schema.string(),
   }),
-  body: schema.object({
-    type: schema.maybe(schema.oneOf([schema.literal('elasticsearch')])),
-    name: schema.maybe(schema.string()),
-    is_default: schema.maybe(schema.boolean()),
-    is_default_monitoring: schema.maybe(schema.boolean()),
-    hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }))),
-    ca_sha256: schema.maybe(schema.string()),
-    ca_trusted_fingerprint: schema.maybe(schema.string()),
-    config_yaml: schema.maybe(schema.string()),
-  }),
+  body: UpdateOutputSchema,
 };