[Fleet] Support logstash as an output type in API and Kibana config

x-pack/plugins/fleet/common/constants/output.ts

@@ -11,6 +11,7 @@ export const OUTPUT_SAVED_OBJECT_TYPE = 'ingest-outputs';
 
 export const outputType = {
   Elasticsearch: 'elasticsearch',
+  Logstash: 'logstash',
 } as const;
 
 export const DEFAULT_OUTPUT_ID = 'fleet-default-output';

x-pack/plugins/fleet/common/openapi/bundled.json

@@ -4231,6 +4231,20 @@
           },
           "config_yaml": {
             "type": "string"
+          },
+          "ssl": {
+            "type": "object",
+            "properties": {
+              "certificate": {
+                "type": "string"
+              },
+              "certificate_authorities": {
+                "type": "string"
+              },
+              "key": {
+                "type": "string"
+              }
+            }
           }
         },
         "required": [

x-pack/plugins/fleet/common/openapi/bundled.yaml

@@ -2668,6 +2668,15 @@ components:
           type: object
         config_yaml:
           type: string
+        ssl:
+          type: object
+          properties:
+            certificate:
+              type: string
+            certificate_authorities:
+              type: string
+            key:
+              type: string
       required:
         - id
         - is_default

x-pack/plugins/fleet/common/openapi/components/schemas/output.yaml

@@ -24,6 +24,15 @@ properties:
     type: object
   config_yaml:
     type: string
+  ssl:
+    type: object
+    properties: 
+      certificate:
+        type: string
+      certificate_authorities:
+        type: string
+      key:
+        type: string
 required:
   - id
   - is_default

x-pack/plugins/fleet/common/types/models/output.ts

@@ -13,13 +13,18 @@ export type OutputType = typeof outputType;
 export interface NewOutput {
   is_default: boolean;
   is_default_monitoring: boolean;
+  is_preconfigured?: boolean;
   name: string;
   type: ValueOf<OutputType>;
   hosts?: string[];
   ca_sha256?: string;
   ca_trusted_fingerprint?: string;
   config_yaml?: string;
-  is_preconfigured?: boolean;
+  ssl?: {
+    certificate_authorities?: string[];
+    certificate?: string;
+    key?: string;
+  };
 }
 
 export type OutputSOAttributes = NewOutput & {

x-pack/plugins/fleet/common/types/rest_spec/output.ts

@@ -28,28 +28,38 @@ export interface PutOutputRequest {
     outputId: string;
   };
   body: {
-    type?: 'elasticsearch';
+    type?: 'elasticsearch' | 'logstash';
     name?: string;
     hosts?: string[];
     ca_sha256?: string;
     ca_trusted_fingerprint?: string;
     config_yaml?: string;
     is_default?: boolean;
     is_default_monitoring?: boolean;
+    ssl?: {
+      certificate_authorities?: string;
+      certificate?: string;
+      key?: string;
+    };
   };
 }
 
 export interface PostOutputRequest {
   body: {
     id?: string;
-    type: 'elasticsearch';
+    type: 'elasticsearch' | 'logstash';
     name: string;
     hosts?: string[];
     ca_sha256?: string;
     ca_trusted_fingerprint?: string;
     is_default?: boolean;
     is_default_monitoring?: boolean;
     config_yaml?: string;
+    ssl?: {
+      certificate_authorities?: string;
+      certificate?: string;
+      key?: string;
+    };
   };
 }
 

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -117,6 +117,7 @@ const getSavedObjectTypes = (
         config: { type: 'flattened' },
         config_yaml: { type: 'text' },
         is_preconfigured: { type: 'boolean', index: false },
+        ssl: { type: 'flattened', index: false },
       },
     },
     migrations: {

x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts

@@ -352,4 +352,54 @@ ssl.test: 123
       }
     `);
   });
+
+  it('should return placeholder ES_USERNAME and ES_PASSWORD for elasticsearch output type in standalone ', () => {
+    const policyOutput = transformOutputToFullPolicyOutput(
+      {
+        id: 'id123',
+        hosts: ['http://host.fr'],
+        is_default: false,
+        is_default_monitoring: false,
+        name: 'test output',
+        type: 'elasticsearch',
+      },
+      true
+    );
+
+    expect(policyOutput).toMatchInlineSnapshot(`
+      Object {
+        "ca_sha256": undefined,
+        "hosts": Array [
+          "http://host.fr",
+        ],
+        "password": "{ES_PASSWORD}",
+        "type": "elasticsearch",
+        "username": "{ES_USERNAME}",
+      }
+    `);
+  });
+
+  it('should not return placeholder ES_USERNAME and ES_PASSWORD for logstash output type in standalone ', () => {
+    const policyOutput = transformOutputToFullPolicyOutput(
+      {
+        id: 'id123',
+        hosts: ['host.fr:3332'],
+        is_default: false,
+        is_default_monitoring: false,
+        name: 'test output',
+        type: 'logstash',
+      },
+      true
+    );
+
+    expect(policyOutput).toMatchInlineSnapshot(`
+      Object {
+        "ca_sha256": undefined,
+        "hosts": Array [
+          "host.fr:3332",
+        ],
+        "type": "logstash",
+      }
+    `);
+  });
 });

x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts

@@ -171,17 +171,18 @@ export function transformOutputToFullPolicyOutput(
   standalone = false
 ): FullAgentPolicyOutput {
   // eslint-disable-next-line @typescript-eslint/naming-convention
-  const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint } = output;
+  const { config_yaml, type, hosts, ca_sha256, ca_trusted_fingerprint, ssl } = output;
   const configJs = config_yaml ? safeLoad(config_yaml) : {};
   const newOutput: FullAgentPolicyOutput = {
     ...configJs,
     type,
     hosts,
     ca_sha256,
+    ...(ssl ? { ssl } : {}),
     ...(ca_trusted_fingerprint ? { 'ssl.ca_trusted_fingerprint': ca_trusted_fingerprint } : {}),
   };
 
-  if (standalone) {
+  if (output.type === outputType.Elasticsearch && standalone) {
     newOutput.username = '{ES_USERNAME}';
     newOutput.password = '{ES_PASSWORD}';
   }

x-pack/plugins/fleet/server/services/output.ts

@@ -10,7 +10,7 @@ import uuid from 'uuid/v5';
 
 import type { NewOutput, Output, OutputSOAttributes } from '../types';
 import { DEFAULT_OUTPUT, DEFAULT_OUTPUT_ID, OUTPUT_SAVED_OBJECT_TYPE } from '../constants';
-import { decodeCloudId, normalizeHostsForAgents, SO_SEARCH_LIMIT } from '../../common';
+import { decodeCloudId, normalizeHostsForAgents, SO_SEARCH_LIMIT, outputType } from '../../common';
 import { OutputUnauthorizedError } from '../errors';
 
 import { appContextService } from './app_context';
@@ -149,7 +149,7 @@ class OutputService {
       }
     }
 
-    if (data.hosts) {
+    if (data.type === outputType.Elasticsearch && data.hosts) {
       data.hosts = data.hosts.map(normalizeHostsForAgents);
     }
 
@@ -260,7 +260,7 @@ class OutputService {
       );
     }
 
-    const updateData = { ...data };
+    const updateData = { type: originalOutput.type, ...data };
 
     // ensure only default output exists
     if (data.is_default) {
@@ -287,7 +287,7 @@ class OutputService {
       }
     }
 
-    if (updateData.hosts) {
+    if (updateData.type === outputType.Elasticsearch && updateData.hosts) {
       updateData.hosts = updateData.hosts.map(normalizeHostsForAgents);
     }
     const outputSO = await soClient.update<OutputSOAttributes>(

x-pack/plugins/fleet/server/services/preconfiguration.ts

@@ -57,6 +57,7 @@ function isPreconfiguredOutputDifferentFromCurrent(
         existingOutput.hosts?.map(normalizeHostsForAgents),
         preconfiguredOutput.hosts.map(normalizeHostsForAgents)
       )) ||
+    (preconfiguredOutput.ssl && !isEqual(preconfiguredOutput.ssl, existingOutput.ssl)) ||
     existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 ||
     existingOutput.ca_trusted_fingerprint !== preconfiguredOutput.ca_trusted_fingerprint ||
     existingOutput.config_yaml !== preconfiguredOutput.config_yaml

x-pack/plugins/fleet/server/types/models/output.ts

@@ -9,16 +9,74 @@ import { schema } from '@kbn/config-schema';
 
 import { outputType } from '../../../common/constants';
 
+export function validateLogstashHost(val: string) {
+  if (val.match(/^http([s]){0,1}:\/\//)) {
+    throw new Error('Invalid logstash host should not start with http(s)');
+  }
+
+  try {
+    const url = new URL(`http://${val}`);
+
+    if (url.host !== val) {
+      throw new Error('Invalid host');
+    }
+  } catch (err) {
+    throw new Error('Invalid logstash host');
+  }
+}
+
 const OutputBaseSchema = {
+  id: schema.maybe(schema.string()),
   name: schema.string(),
-  type: schema.oneOf([schema.literal(outputType.Elasticsearch)]),
-  hosts: schema.maybe(schema.arrayOf(schema.string())),
-  config: schema.maybe(schema.recordOf(schema.string(), schema.any())),
+  type: schema.oneOf([
+    schema.literal(outputType.Elasticsearch),
+    schema.literal(outputType.Logstash),
+  ]),
+  hosts: schema.conditional(
+    schema.siblingRef('type'),
+    schema.literal(outputType.Elasticsearch),
+    schema.arrayOf(schema.uri({ scheme: ['http', 'https'] })),
+    schema.arrayOf(schema.string({ validate: validateLogstashHost }))
+  ),
+  is_default: schema.boolean({ defaultValue: false }),
+  is_default_monitoring: schema.boolean({ defaultValue: false }),
+  ca_sha256: schema.maybe(schema.string()),
+  ca_trusted_fingerprint: schema.maybe(schema.string()),
   config_yaml: schema.maybe(schema.string()),
+  ssl: schema.maybe(
+    schema.object({
+      certificate_authorities: schema.maybe(schema.arrayOf(schema.string())),
+      certificate: schema.maybe(schema.string()),
+      key: schema.maybe(schema.string()),
+    })
+  ),
 };
 
-export const NewOutputSchema = schema.object({
-  ...OutputBaseSchema,
+export const NewOutputSchema = schema.object({ ...OutputBaseSchema });
+
+export const UpdateOutputSchema = schema.object({
+  name: schema.maybe(schema.string()),
+  type: schema.maybe(
+    schema.oneOf([schema.literal(outputType.Elasticsearch), schema.literal(outputType.Logstash)])
+  ),
+  hosts: schema.maybe(
+    schema.oneOf([
+      schema.arrayOf(schema.uri({ scheme: ['http', 'https'] })),
+      schema.arrayOf(schema.string({ validate: validateLogstashHost })),
+    ])
+  ),
+  is_default: schema.maybe(schema.boolean()),
+  is_default_monitoring: schema.maybe(schema.boolean()),
+  ca_sha256: schema.maybe(schema.string()),
+  ca_trusted_fingerprint: schema.maybe(schema.string()),
+  config_yaml: schema.maybe(schema.string()),
+  ssl: schema.maybe(
+    schema.object({
+      certificate_authorities: schema.maybe(schema.arrayOf(schema.string())),
+      certificate: schema.maybe(schema.string()),
+      key: schema.maybe(schema.string()),
+    })
+  ),
 });
 
 export const OutputSchema = schema.object({

x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts

@@ -17,12 +17,14 @@ describe('Test preconfiguration schema', () => {
             name: 'Output 1',
             type: 'elasticsearch',
             is_default: true,
+            hosts: ['http://test.fr:9200'],
           },
           {
             id: 'output-2',
             name: 'Output 2',
             type: 'elasticsearch',
             is_default: true,
+            hosts: ['http://test.fr:9200'],
           },
         ]);
       }).toThrowError('preconfigured outputs can only have one default output.');
@@ -35,12 +37,14 @@ describe('Test preconfiguration schema', () => {
             name: 'Output 1',
             type: 'elasticsearch',
             is_default_monitoring: true,
+            hosts: ['http://test.fr:9200'],
           },
           {
             id: 'output-2',
             name: 'Output 2',
             type: 'elasticsearch',
             is_default_monitoring: true,
+            hosts: ['http://test.fr:9200'],
           },
         ]);
       }).toThrowError('preconfigured outputs can only have one default monitoring output.');
@@ -52,11 +56,13 @@ describe('Test preconfiguration schema', () => {
             id: 'nonuniqueid',
             name: 'Output 1',
             type: 'elasticsearch',
+            hosts: ['http://test.fr:9200'],
           },
           {
             id: 'nonuniqueid',
             name: 'Output 2',
             type: 'elasticsearch',
+            hosts: ['http://test.fr:9200'],
           },
         ]);
       }).toThrowError('preconfigured outputs need to have unique ids.');
@@ -68,11 +74,13 @@ describe('Test preconfiguration schema', () => {
             id: 'output-1',
             name: 'nonuniquename',
             type: 'elasticsearch',
+            hosts: ['http://test.fr:9200'],
           },
           {
             id: 'output-2',
             name: 'nonuniquename',
             type: 'elasticsearch',
+            hosts: ['http://test.fr:9200'],
           },
         ]);
       }).toThrowError('preconfigured outputs need to have unique names.');