From 5d030c30b9dc655b53cb36ee42d704fbd58ed17c Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 15 Apr 2020 10:32:32 -0400 Subject: [PATCH 1/6] [Ingest] Allow to enable monitoring of elastic agent --- .../common/types/models/agent_config.ts | 7 + .../agent_config/components/config_form.tsx | 67 +++++++++- .../ingest_manager/server/saved_objects.ts | 1 + .../server/services/agent_config.test.ts | 123 ++++++++++++++++++ .../server/services/agent_config.ts | 39 ++++-- .../server/types/models/agent_config.ts | 3 + 6 files changed, 227 insertions(+), 13 deletions(-) create mode 100644 x-pack/plugins/ingest_manager/server/services/agent_config.test.ts diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts index 002c3784446a8..429aaf38f3953 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts @@ -23,6 +23,7 @@ export interface NewAgentConfig { namespace?: string; description?: string; is_default?: boolean; + monitoring_enabled?: Array<'logs' | 'metrics'>; } export interface AgentConfig extends NewAgentConfig, SavedObjectAttributes { @@ -58,4 +59,10 @@ export interface FullAgentConfig { }; datasources: FullAgentConfigDatasource[]; revision?: number; + 'settings.monitoring'?: { + use_output: string; + enabled: boolean; + metrics: boolean; + logs: boolean; + }; } diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx index 0d53ca34a1fef..f473438391c55 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx @@ -30,7 +30,7 @@ interface ValidationResults { const StyledEuiAccordion = styled(EuiAccordion)` .ingest-active-button { - color: ${props => props.theme.eui.euiColorPrimary}}; + color: ${props => props.theme.eui.euiColorPrimary}; } `; @@ -244,6 +244,71 @@ export const AgentConfigForm: React.FunctionComponent = ({ )} + + + +

+ +

+ + + + + } + checked={ + agentConfig.monitoring_enabled !== undefined && + agentConfig.monitoring_enabled.indexOf('logs') >= 0 + } + onChange={() => { + const hasLogs = + agentConfig.monitoring_enabled && + agentConfig.monitoring_enabled.indexOf('logs') >= 0; + + const previousValues = agentConfig.monitoring_enabled || []; + updateAgentConfig({ + monitoring_enabled: hasLogs + ? previousValues.filter(type => type !== 'logs') + : [...previousValues, 'logs'], + }); + }} + /> + + + } + checked={ + agentConfig.monitoring_enabled !== undefined && + agentConfig.monitoring_enabled.indexOf('metrics') >= 0 + } + onChange={() => { + const hasMetrics = + agentConfig.monitoring_enabled && + agentConfig.monitoring_enabled.indexOf('metrics') >= 0; + + const previousValues = agentConfig.monitoring_enabled || []; + updateAgentConfig({ + monitoring_enabled: hasMetrics + ? previousValues.filter(type => type !== 'metrics') + : [...previousValues, 'metrics'], + }); + }} + /> + + ); diff --git a/x-pack/plugins/ingest_manager/server/saved_objects.ts b/x-pack/plugins/ingest_manager/server/saved_objects.ts index 6800cb4056700..d4e23b635eb6c 100644 --- a/x-pack/plugins/ingest_manager/server/saved_objects.ts +++ b/x-pack/plugins/ingest_manager/server/saved_objects.ts @@ -77,6 +77,7 @@ export const savedObjectMappings = { updated_on: { type: 'keyword' }, updated_by: { type: 'keyword' }, revision: { type: 'integer' }, + monitoring_enabled: { type: 'keyword' }, }, }, [ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE]: { diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts new file mode 100644 index 0000000000000..ec9eaf6d9f083 --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { agentConfigService } from './agent_config'; +import { savedObjectsClientMock } from '../../../../../src/core/server/saved_objects/service/saved_objects_client.mock'; +import { Output } from '../types'; + +function getSavedObjectMock(configAttributes: any) { + const mock = savedObjectsClientMock.create(); + + mock.get.mockImplementation(async (type: string, id: string) => { + return { + type, + id, + references: [], + attributes: configAttributes, + }; + }); + + return mock; +} + +jest.mock('./output', () => { + return { + outputService: { + getDefaultOutputId: () => 'test-id', + get: (): Output => { + return { + id: 'test-id', + is_default: true, + name: 'default', + // @ts-ignore + type: 'elasticsearch', + hosts: ['http://127.0.0.1:9201'], + }; + }, + }, + }; +}); + +describe('agent config', () => { + describe('getFullConfig', () => { + it('should return a config without monitoring if not monitoring is not enabled', async () => { + const soClient = getSavedObjectMock({ + revision: 1, + }); + const config = await agentConfigService.getFullConfig(soClient, 'config'); + + expect(config).toMatchObject({ + id: 'config', + outputs: { + default: { + type: 'elasticsearch', + hosts: ['http://127.0.0.1:9201'], + ca_sha256: undefined, + api_key: undefined, + }, + }, + datasources: [], + revision: 1, + }); + }); + + it('should return a config with monitoring if monitoring is enabled for logs', async () => { + const soClient = getSavedObjectMock({ + revision: 1, + monitoring_enabled: ['logs'], + }); + const config = await agentConfigService.getFullConfig(soClient, 'config'); + + expect(config).toMatchObject({ + id: 'config', + outputs: { + default: { + type: 'elasticsearch', + hosts: ['http://127.0.0.1:9201'], + ca_sha256: undefined, + api_key: undefined, + }, + }, + datasources: [], + revision: 1, + 'settings.monitoring': { + use_output: 'default', + enabled: true, + logs: true, + metrics: false, + }, + }); + }); + + it('should return a config with monitoring if monitoring is enabled for metrics', async () => { + const soClient = getSavedObjectMock({ + revision: 1, + monitoring_enabled: ['metrics'], + }); + const config = await agentConfigService.getFullConfig(soClient, 'config'); + + expect(config).toMatchObject({ + id: 'config', + outputs: { + default: { + type: 'elasticsearch', + hosts: ['http://127.0.0.1:9201'], + ca_sha256: undefined, + api_key: undefined, + }, + }, + datasources: [], + revision: 1, + 'settings.monitoring': { + use_output: 'default', + enabled: true, + logs: false, + metrics: true, + }, + }); + }); + }); +}); diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.ts index 309ddca3784c2..e866e89bade38 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.ts @@ -301,28 +301,43 @@ class AgentConfigService { if (!config) { return null; } + const defaultOutput = await outputService.get( + soClient, + await outputService.getDefaultOutputId(soClient) + ); const agentConfig: FullAgentConfig = { id: config.id, outputs: { // TEMPORARY as we only support a default output - ...[ - await outputService.get(soClient, await outputService.getDefaultOutputId(soClient)), - ].reduce((outputs, { config: outputConfig, name, type, hosts, ca_sha256, api_key }) => { - outputs[name] = { - type, - hosts, - ca_sha256, - api_key, - ...outputConfig, - }; - return outputs; - }, {} as FullAgentConfig['outputs']), + ...[defaultOutput].reduce( + (outputs, { config: outputConfig, name, type, hosts, ca_sha256, api_key }) => { + outputs[name] = { + type, + hosts, + ca_sha256, + api_key, + ...outputConfig, + }; + return outputs; + }, + {} as FullAgentConfig['outputs'] + ), }, datasources: (config.datasources as Datasource[]) .filter(datasource => datasource.enabled) .map(ds => storedDatasourceToAgentDatasource(ds)), revision: config.revision, + ...(config.monitoring_enabled && config.monitoring_enabled.length > 0 + ? { + 'settings.monitoring': { + use_output: defaultOutput.name, + enabled: true, + logs: config.monitoring_enabled.indexOf('logs') >= 0, + metrics: config.monitoring_enabled.indexOf('metrics') >= 0, + }, + } + : {}), }; return agentConfig; diff --git a/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts index 040b2eb16289a..59cadf3bd7f74 100644 --- a/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/types/models/agent_config.ts @@ -11,6 +11,9 @@ const AgentConfigBaseSchema = { name: schema.string(), namespace: schema.maybe(schema.string()), description: schema.maybe(schema.string()), + monitoring_enabled: schema.maybe( + schema.arrayOf(schema.oneOf([schema.literal('logs'), schema.literal('metrics')])) + ), }; export const NewAgentConfigSchema = schema.object({ From a41950be8bb01fc468fe8ba0e8d73b7e0f8f138c Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 16 Apr 2020 13:12:28 -0400 Subject: [PATCH 2/6] Fix typo --- .../common/types/models/agent_config.ts | 12 ++++++---- .../server/services/agent_config.test.ts | 24 +++++++++++-------- .../server/services/agent_config.ts | 12 ++++++---- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts index 429aaf38f3953..5c9bfca883ca1 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts @@ -59,10 +59,12 @@ export interface FullAgentConfig { }; datasources: FullAgentConfigDatasource[]; revision?: number; - 'settings.monitoring'?: { - use_output: string; - enabled: boolean; - metrics: boolean; - logs: boolean; + settings?: { + monitoring: { + use_output: string; + enabled: boolean; + metrics: boolean; + logs: boolean; + }; }; } diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts index ec9eaf6d9f083..19203c83be099 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts @@ -83,11 +83,13 @@ describe('agent config', () => { }, datasources: [], revision: 1, - 'settings.monitoring': { - use_output: 'default', - enabled: true, - logs: true, - metrics: false, + settings: { + monitoring: { + use_output: 'default', + enabled: true, + logs: true, + metrics: false, + }, }, }); }); @@ -111,11 +113,13 @@ describe('agent config', () => { }, datasources: [], revision: 1, - 'settings.monitoring': { - use_output: 'default', - enabled: true, - logs: false, - metrics: true, + settings: { + monitoring: { + use_output: 'default', + enabled: true, + logs: false, + metrics: true, + }, }, }); }); diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.ts index e866e89bade38..bd6a7bf17506c 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.ts @@ -330,11 +330,13 @@ class AgentConfigService { revision: config.revision, ...(config.monitoring_enabled && config.monitoring_enabled.length > 0 ? { - 'settings.monitoring': { - use_output: defaultOutput.name, - enabled: true, - logs: config.monitoring_enabled.indexOf('logs') >= 0, - metrics: config.monitoring_enabled.indexOf('metrics') >= 0, + settings: { + monitoring: { + use_output: defaultOutput.name, + enabled: true, + logs: config.monitoring_enabled.indexOf('logs') >= 0, + metrics: config.monitoring_enabled.indexOf('metrics') >= 0, + }, }, } : {}), From 3c3464dc456b459a1b5a1eae0cd2f07591be55a1 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 21 Apr 2020 11:44:55 -0400 Subject: [PATCH 3/6] update UI --- .../agent_config/components/config_form.tsx | 86 +++++++++---------- 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx index f473438391c55..461a3a14a4f4b 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx @@ -18,6 +18,7 @@ import { EuiText, EuiComboBox, EuiIconTip, + EuiCheckboxGroup, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; @@ -244,66 +245,63 @@ export const AgentConfigForm: React.FunctionComponent = ({ )} +

+ + + + - - } - checked={ - agentConfig.monitoring_enabled !== undefined && - agentConfig.monitoring_enabled.indexOf('logs') >= 0 - } - onChange={() => { + { + acc[key] = true; + return acc; + }, + { logs: false, metrics: false } + )} + onChange={id => { + if (id !== 'logs' && id !== 'metrics') { + return; + } + const hasLogs = - agentConfig.monitoring_enabled && - agentConfig.monitoring_enabled.indexOf('logs') >= 0; + agentConfig.monitoring_enabled && agentConfig.monitoring_enabled.indexOf(id) >= 0; const previousValues = agentConfig.monitoring_enabled || []; updateAgentConfig({ monitoring_enabled: hasLogs - ? previousValues.filter(type => type !== 'logs') - : [...previousValues, 'logs'], - }); - }} - /> - - - } - checked={ - agentConfig.monitoring_enabled !== undefined && - agentConfig.monitoring_enabled.indexOf('metrics') >= 0 - } - onChange={() => { - const hasMetrics = - agentConfig.monitoring_enabled && - agentConfig.monitoring_enabled.indexOf('metrics') >= 0; - - const previousValues = agentConfig.monitoring_enabled || []; - updateAgentConfig({ - monitoring_enabled: hasMetrics - ? previousValues.filter(type => type !== 'metrics') - : [...previousValues, 'metrics'], + ? previousValues.filter(type => type !== id) + : [...previousValues, id], }); }} /> From ae0b8546aaa601a77e8d100c6c5dda8c5a8fed3c Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Thu, 23 Apr 2020 11:06:29 -0400 Subject: [PATCH 4/6] Enable by default --- x-pack/plugins/ingest_manager/common/constants/agent_config.ts | 1 + .../sections/agent_config/list_page/components/create_config.tsx | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/plugins/ingest_manager/common/constants/agent_config.ts b/x-pack/plugins/ingest_manager/common/constants/agent_config.ts index 337022e552278..68c68e2d370b6 100644 --- a/x-pack/plugins/ingest_manager/common/constants/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/constants/agent_config.ts @@ -14,6 +14,7 @@ export const DEFAULT_AGENT_CONFIG = { status: AgentConfigStatus.Active, datasources: [], is_default: true, + monitoring_enabled: ['logs', 'metrics'], }; export const DEFAULT_AGENT_CONFIGS_PACKAGES = [DefaultPackages.system]; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/components/create_config.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/components/create_config.tsx index 1fe116ef36090..9f582e7e2fbe6 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/components/create_config.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/list_page/components/create_config.tsx @@ -34,6 +34,7 @@ export const CreateAgentConfigFlyout: React.FunctionComponent = ({ onClos description: '', namespace: '', is_default: undefined, + monitoring_enabled: ['logs', 'metrics'], }); const [isLoading, setIsLoading] = useState(false); const [withSysMonitoring, setWithSysMonitoring] = useState(true); From b47703576097c69f169642664e2ff40564975618 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 29 Apr 2020 13:11:06 -0400 Subject: [PATCH 5/6] Fix type issue --- .../ingest_manager/common/constants/agent_config.ts | 2 +- .../ingest_manager/common/types/models/agent_config.ts | 2 +- .../ingest_manager/server/services/agent_config.test.ts | 9 ++++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/constants/agent_config.ts b/x-pack/plugins/ingest_manager/common/constants/agent_config.ts index 5b9a09719b0b1..9bc1293799d3c 100644 --- a/x-pack/plugins/ingest_manager/common/constants/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/constants/agent_config.ts @@ -14,7 +14,7 @@ export const DEFAULT_AGENT_CONFIG = { status: AgentConfigStatus.Active, datasources: [], is_default: true, - monitoring_enabled: ['logs', 'metrics'], + monitoring_enabled: ['logs', 'metrics'] as Array<'logs' | 'metrics'>, }; export const DEFAULT_AGENT_CONFIGS_PACKAGES = [DefaultPackages.system]; diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts index 8356527c95a8c..7705956590c16 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts @@ -63,7 +63,7 @@ export interface FullAgentConfig { revision?: number; settings?: { monitoring: { - use_output: string; + use_output?: string; enabled: boolean; metrics: boolean; logs: boolean; diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts index 19203c83be099..17758f6e3d7f1 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { savedObjectsClientMock } from 'src/core/server/mocks'; import { agentConfigService } from './agent_config'; -import { savedObjectsClientMock } from '../../../../../src/core/server/saved_objects/service/saved_objects_client.mock'; import { Output } from '../types'; function getSavedObjectMock(configAttributes: any) { @@ -61,6 +61,13 @@ describe('agent config', () => { }, datasources: [], revision: 1, + settings: { + monitoring: { + enabled: false, + logs: false, + metrics: false, + }, + }, }); }); From 41cd40ea0da725585f271d8bce2fbf45e06e6184 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 29 Apr 2020 14:49:50 -0400 Subject: [PATCH 6/6] Update x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx Co-Authored-By: Jen Huang --- .../sections/agent_config/components/config_form.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx index 461a3a14a4f4b..92c44d86e47c6 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/components/config_form.tsx @@ -260,7 +260,7 @@ export const AgentConfigForm: React.FunctionComponent = ({