From 7ef303eb362df466e53fe1ae5f38d719a782098f Mon Sep 17 00:00:00 2001 From: amyjtechwriter <61687663+amyjtechwriter@users.noreply.github.com> Date: Mon, 31 Jul 2023 12:26:23 +0100 Subject: [PATCH 1/6] =?UTF-8?q?[DOCS]Adding=20ESS=20icon=20to=20xpack.secu?= =?UTF-8?q?rity.session.=D1=81oncurrentSessions.maxSessi=E2=80=A6=20(#1624?= =?UTF-8?q?93)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `xpack.security.session.сoncurrentSessions.maxSessons` in Kibana docs. ## Summary Adding the Elastic Cloud icon to `xpack.security.session.сoncurrentSessions.maxSessons` security setting on [docs page](https://www.elastic.co/guide/en/kibana/current/security-settings-kb.html). Need to raise another PR from cloud repo to add security setting to [cloud docs page](https://www.elastic.co/guide/en/cloud/current/ec-manage-kibana-settings.html#ec-kibana-config). Relates to: #160958 --- docs/settings/security-settings.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/settings/security-settings.asciidoc b/docs/settings/security-settings.asciidoc index 98d7b13a1122b..06c8835d25c3c 100644 --- a/docs/settings/security-settings.asciidoc +++ b/docs/settings/security-settings.asciidoc @@ -210,7 +210,7 @@ Sets the interval at which {kib} tries to remove expired and invalid sessions fr + TIP: Use a string of `[ms\|s\|m\|h\|d\|w\|M\|Y]` (e.g. '20m', '24h', '7d', '1w'). -xpack.security.session.сoncurrentSessions.maxSessions:: +xpack.security.session.concurrentSessions.maxSessions {ess-icon}:: Set the maximum number of sessions each user is allowed to have active at any given time. By default, no limit is applied. If set, the value of this option should be an integer between `1` and `1000`. From 41af3d6c4dd120db9f7d021148f2ef5f22799117 Mon Sep 17 00:00:00 2001 From: Anton Dosov Date: Mon, 31 Jul 2023 13:38:05 +0200 Subject: [PATCH 2/6] Skip Serverless Observability Tests / serverless observability UI navigation active sidenav section is auto opened on load (#162782) ## Summary skip https://github.com/elastic/kibana/issues/162781 --- .../functional/test_suites/observability/navigation.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test_serverless/functional/test_suites/observability/navigation.ts b/x-pack/test_serverless/functional/test_suites/observability/navigation.ts index 2e8745680c528..6c90a2a410b61 100644 --- a/x-pack/test_serverless/functional/test_suites/observability/navigation.ts +++ b/x-pack/test_serverless/functional/test_suites/observability/navigation.ts @@ -69,7 +69,8 @@ export default function ({ getPageObject, getService }: FtrProviderContext) { await expectNoPageReload(); }); - it('active sidenav section is auto opened on load', async () => { + // FLAKY/BUG?: https://github.com/elastic/kibana/issues/162781 + it.skip('active sidenav section is auto opened on load', async () => { await svlCommonNavigation.sidenav.openSection('project_settings_project_nav'); await svlCommonNavigation.sidenav.clickLink({ deepLinkId: 'management' }); await browser.refresh(); From d74d33f757b2c665c0d0b133f6836f87c93a3663 Mon Sep 17 00:00:00 2001 From: Yngrid Coello Date: Mon, 31 Jul 2023 14:01:44 +0200 Subject: [PATCH 3/6] [APM] Improving overflow buckets documentation (#162774) --- .../plugins/apm/dev_docs/overflow_bucket_setup.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/apm/dev_docs/overflow_bucket_setup.md b/x-pack/plugins/apm/dev_docs/overflow_bucket_setup.md index 61e8bfb53d581..60bd87714609f 100644 --- a/x-pack/plugins/apm/dev_docs/overflow_bucket_setup.md +++ b/x-pack/plugins/apm/dev_docs/overflow_bucket_setup.md @@ -154,7 +154,11 @@ For more detailed instructions you can check [apm-server dev documentation](http - Copy paste the below script in a file called `tx_max_group.go` This file is responsible for generating 3 transactions per service. - ``` + +
+ tx_max_group.go + + ```go package main import ( @@ -185,9 +189,14 @@ For more detailed instructions you can check [apm-server dev documentation](http span.End() } ``` - +
+ - Now create a Bash Script file, name it anything - e.g., `service_max_group.sh` This file will generate services and then transactions for each service using the go script above. + +
+ tx_max_group.go + ```sh #!/usr/bin/env bash @@ -203,6 +212,7 @@ For more detailed instructions you can check [apm-server dev documentation](http echo "Ending script" ``` +
- Run `sh service_max_group` to generate the data From e66d949198f60eb6e6cc30eea93d99228b4cce4d Mon Sep 17 00:00:00 2001 From: Alexi Doak <109488926+doakalexi@users.noreply.github.com> Date: Mon, 31 Jul 2023 08:08:21 -0400 Subject: [PATCH 4/6] [ResponseOps][Alerting] Flaky test x-pack/plugins/event_log/server/es (#162558) https://github.com/elastic/kibana/issues/156061 ## Summary Fixes flaky test by adding ceil(). When it fails the time elapsed is close to the `MOCK_RETRY_DELAY` of 20 (like 19.7), so rounding up helps with the flakiness --- x-pack/plugins/event_log/server/es/init.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/event_log/server/es/init.test.ts b/x-pack/plugins/event_log/server/es/init.test.ts index 154a98fcb194c..30e220313b26b 100644 --- a/x-pack/plugins/event_log/server/es/init.test.ts +++ b/x-pack/plugins/event_log/server/es/init.test.ts @@ -455,8 +455,7 @@ describe('parseIndexAliases', () => { }); }); -// FLAKY: https://github.com/elastic/kibana/issues/156061 -describe.skip('retries', () => { +describe('retries', () => { let esContext = contextMock.create(); // set up context APIs to return defaults indicating already created beforeEach(() => { @@ -474,7 +473,7 @@ describe.skip('retries', () => { const timeStart = performance.now(); await initializeEs(esContext); - const timeElapsed = performance.now() - timeStart; + const timeElapsed = Math.ceil(performance.now() - timeStart); expect(timeElapsed).toBeGreaterThanOrEqual(MOCK_RETRY_DELAY); From d5761dc0ea5c6d347eff55028a53f23b64050069 Mon Sep 17 00:00:00 2001 From: Alexi Doak <109488926+doakalexi@users.noreply.github.com> Date: Mon, 31 Jul 2023 08:09:49 -0400 Subject: [PATCH 5/6] =?UTF-8?q?[ResponseOps][Alerting]=20Flaky=20test=20x-?= =?UTF-8?q?pack/test/alerting=5Fapi=5Fintegration/security=5Fand=5Fspaces/?= =?UTF-8?q?group2/tests/telemetry/index=C2=B7ts=20(#161096)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves https://github.com/elastic/kibana/issues/140973 Resolves https://github.com/elastic/kibana/issues/136153 ## Summary Fixes a telemetry bug and a flaky telemetry test ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/2717 x 250 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/2718 x 250 --- .../actions/server/usage/actions_telemetry.ts | 2 +- .../actions/server/usage/task_state.ts | 2 +- .../alerting_and_actions_telemetry.ts | 341 ++++++------------ .../group2/tests/telemetry/index.ts | 6 +- 4 files changed, 116 insertions(+), 235 deletions(-) diff --git a/x-pack/plugins/actions/server/usage/actions_telemetry.ts b/x-pack/plugins/actions/server/usage/actions_telemetry.ts index 7959d14775350..49ac9ff1978f8 100644 --- a/x-pack/plugins/actions/server/usage/actions_telemetry.ts +++ b/x-pack/plugins/actions/server/usage/actions_telemetry.ts @@ -476,7 +476,7 @@ export async function getExecutionsPerDayCount( countFailedByType: Record; avgExecutionTime: number; avgExecutionTimeByType: Record; - countRunOutcomeByConnectorType: Record; + countRunOutcomeByConnectorType: Record>; }> { const scriptedMetric = { scripted_metric: { diff --git a/x-pack/plugins/actions/server/usage/task_state.ts b/x-pack/plugins/actions/server/usage/task_state.ts index c41c76df6133c..3d4d96b525d5e 100644 --- a/x-pack/plugins/actions/server/usage/task_state.ts +++ b/x-pack/plugins/actions/server/usage/task_state.ts @@ -66,7 +66,7 @@ export const stateSchemaByVersion = { avg_execution_time_by_type_per_day: schema.recordOf(schema.string(), schema.number()), count_connector_types_by_action_run_outcome_per_day: schema.recordOf( schema.string(), - schema.number() + schema.recordOf(schema.string(), schema.number()) ), }), }, diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts index 7925bfe6f2eb9..02a41b6de7afa 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/alerting_and_actions_telemetry.ts @@ -8,7 +8,13 @@ import expect from '@kbn/expect'; import { ESTestIndexTool } from '@kbn/alerting-api-integration-helpers'; import { Spaces, Superuser } from '../../../scenarios'; -import { getUrlPrefix, getEventLog, getTestRuleData, TaskManagerDoc } from '../../../../common/lib'; +import { + getUrlPrefix, + getEventLog, + getTestRuleData, + TaskManagerDoc, + ObjectRemover, +} from '../../../../common/lib'; import { FtrProviderContext } from '../../../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export @@ -20,8 +26,8 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F const esTestIndexTool = new ESTestIndexTool(es, retry); const supertestWithoutAuth = getService('supertestWithoutAuth'); - // FLAKY: https://github.com/elastic/kibana/issues/140973 - describe.skip('telemetry', () => { + describe('telemetry', () => { + const objectRemover = new ObjectRemover(supertest); const alwaysFiringRuleId: { [key: string]: string } = {}; beforeEach(async () => { @@ -29,6 +35,12 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F await esTestIndexTool.setup(); }); + afterEach(() => objectRemover.removeAll()); + + after(async () => { + await esTestIndexTool.destroy(); + }); + async function createConnector(opts: { name: string; space: string; connectorTypeId: string }) { const { name, space, connectorTypeId } = opts; const { body: createdConnector } = await supertestWithoutAuth @@ -42,6 +54,7 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F secrets: {}, }) .expect(200); + objectRemover.add(space, createdConnector.id, 'connector', 'actions'); return createdConnector.id; } @@ -52,7 +65,8 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F .set('kbn-xsrf', 'foo') .auth(Superuser.username, Superuser.password) .send(getTestRuleData(ruleOverwrites)); - expect(ruleResponse.status).to.eql(200); + expect(ruleResponse.status).to.equal(200); + objectRemover.add(space, ruleResponse.body.id, 'rule', 'alerting'); return ruleResponse.body.id; } @@ -69,127 +83,81 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F space: space.id, connectorTypeId: 'test.throw', }); + // excluded connector await createConnector({ name: 'unused connector', space: space.id, connectorTypeId: 'test.excluded', }); - await createRule({ - space: space.id, - ruleOverwrites: { - rule_type_id: 'test.noop', - schedule: { interval: '30s' }, - throttle: '1s', - params: {}, - actions: [ - { - id: noopConnectorId, - group: 'default', - params: {}, - }, - ], - }, - }); - await createRule({ - space: space.id, - ruleOverwrites: { - rule_type_id: 'test.onlyContextVariables', - schedule: { interval: '10s' }, - throttle: '10m', - params: {}, - actions: [ - { - id: failingConnectorId, - group: 'default', - params: {}, - }, - ], - }, - }); - await createRule({ - space: space.id, - ruleOverwrites: { - rule_type_id: 'test.throw', - schedule: { interval: '1m' }, - throttle: '30s', - params: {}, - actions: [ - { - id: noopConnectorId, - group: 'default', - params: {}, - }, - ], - }, - }); alwaysFiringRuleId[space.id] = await createRule({ space: space.id, ruleOverwrites: { - rule_type_id: 'example.always-firing', - schedule: { interval: '3s' }, + rule_type_id: 'test.patternFiring', + schedule: { interval: '1h' }, throttle: null, - notify_when: 'onActiveAlert', - params: {}, + params: { + pattern: { instance: [true] }, + }, actions: [ { id: noopConnectorId, - group: 'small', + group: 'default', params: {}, }, { id: 'my-slack1', - group: 'medium', + group: 'default', params: {}, }, { id: failingConnectorId, - group: 'large', + group: 'default', params: {}, }, ], }, }); - + // disabled rule await createRule({ space: space.id, ruleOverwrites: { rule_type_id: 'test.noop', - schedule: { interval: '5m' }, + schedule: { interval: '1h' }, throttle: null, enabled: false, params: {}, actions: [], }, }); - + // throwing rule await createRule({ space: space.id, ruleOverwrites: { - rule_type_id: 'test.multipleSearches', - schedule: { interval: '40s' }, - throttle: '1m', - params: { numSearches: 2, delay: `2s` }, - actions: [], - }, - }); - - await createRule({ - space: space.id, - ruleOverwrites: { - rule_type_id: 'test.cumulative-firing', - schedule: { interval: '61s' }, - throttle: '2s', - notify_when: 'onActiveAlert', + rule_type_id: 'test.throw', + schedule: { interval: '1h' }, + throttle: null, + params: {}, actions: [ { - id: failingConnectorId, + id: noopConnectorId, group: 'default', params: {}, }, ], }, }); + // ES search rule + await createRule({ + space: space.id, + ruleOverwrites: { + rule_type_id: 'test.multipleSearches', + schedule: { interval: '1h' }, + throttle: '1s', + params: { numSearches: 2, delay: `2s` }, + actions: [], + }, + }); } } @@ -245,49 +213,44 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F ); expect( - telemetry.count_connector_types_by_action_run_outcome_per_day['test.throw'].failure - ).to.greaterThan(0); + telemetry.count_connector_types_by_action_run_outcome_per_day['test.throw'].failure > 0 + ).to.be(true); } function verifyAlertingTelemetry(telemetry: any) { logger.info(`alerting telemetry - ${JSON.stringify(telemetry)}`); // total number of enabled rules - expect(telemetry.count_active_total).to.equal(18); + expect(telemetry.count_active_total).to.equal(9); // total number of disabled rules expect(telemetry.count_disabled_total).to.equal(3); // total number of rules broken down by rule type - expect(telemetry.count_by_type.test__noop).to.equal(6); - expect(telemetry.count_by_type['example__always-firing']).to.equal(3); - expect(telemetry.count_by_type['test__cumulative-firing']).to.equal(3); + expect(telemetry.count_by_type.test__noop).to.equal(3); + expect(telemetry.count_by_type.test__patternFiring).to.equal(3); expect(telemetry.count_by_type.test__multipleSearches).to.equal(3); - expect(telemetry.count_by_type.test__onlyContextVariables).to.equal(3); expect(telemetry.count_by_type.test__throw).to.equal(3); // total number of enabled rules broken down by rule type - expect(telemetry.count_active_by_type['example__always-firing']).to.equal(3); - expect(telemetry.count_active_by_type['test__cumulative-firing']).to.equal(3); + expect(telemetry.count_active_by_type.test__patternFiring).to.equal(3); expect(telemetry.count_active_by_type.test__multipleSearches).to.equal(3); - expect(telemetry.count_active_by_type.test__noop).to.equal(3); - expect(telemetry.count_active_by_type.test__onlyContextVariables).to.equal(3); expect(telemetry.count_active_by_type.test__throw).to.equal(3); // throttle time stats expect(telemetry.throttle_time.min).to.equal('0s'); - expect(telemetry.throttle_time.avg).to.equal('115.5s'); - expect(telemetry.throttle_time.max).to.equal('600s'); + expect(telemetry.throttle_time.avg).to.equal('0.4s'); + expect(telemetry.throttle_time.max).to.equal('1s'); expect(telemetry.throttle_time_number_s.min).to.equal(0); - expect(telemetry.throttle_time_number_s.avg).to.equal(115.5); - expect(telemetry.throttle_time_number_s.max).to.equal(600); + expect(telemetry.throttle_time_number_s.avg).to.equal(0.4); + expect(telemetry.throttle_time_number_s.max).to.equal(1); // schedule interval stats - expect(telemetry.schedule_time.min).to.equal('3s'); - expect(telemetry.schedule_time.avg).to.equal('72s'); - expect(telemetry.schedule_time.max).to.equal('300s'); - expect(telemetry.schedule_time_number_s.min).to.equal(3); - expect(telemetry.schedule_time_number_s.avg).to.equal(72); - expect(telemetry.schedule_time_number_s.max).to.equal(300); + expect(telemetry.schedule_time.min).to.equal('3600s'); + expect(telemetry.schedule_time.avg).to.equal('3600s'); + expect(telemetry.schedule_time.max).to.equal('3600s'); + expect(telemetry.schedule_time_number_s.min).to.equal(3600); + expect(telemetry.schedule_time_number_s.avg).to.equal(3600); + expect(telemetry.schedule_time_number_s.max).to.equal(3600); // attached connectors stats expect(telemetry.connectors_per_alert.min).to.equal(0); @@ -299,12 +262,10 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F // number of rule executions - just checking for non-zero as we can't set an exact number // each rule should have had a chance to execute once - expect(telemetry.count_rules_executions_per_day >= 21).to.be(true); + expect(telemetry.count_rules_executions_per_day > 0).to.be(true); // number of rule executions broken down by rule type - expect(telemetry.count_by_type['example__always-firing'] >= 3).to.be(true); - expect(telemetry.count_by_type.test__onlyContextVariables >= 3).to.be(true); - expect(telemetry.count_by_type['test__cumulative-firing'] >= 3).to.be(true); + expect(telemetry.count_by_type.test__patternFiring >= 3).to.be(true); expect(telemetry.count_by_type.test__noop >= 3).to.be(true); expect(telemetry.count_by_type.test__multipleSearches >= 3).to.be(true); expect(telemetry.count_by_type.test__throw >= 3).to.be(true); @@ -313,16 +274,7 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F expect(telemetry.avg_execution_time_per_day > 0).to.be(true); // average execution time broken down by rule type - expect(telemetry.avg_execution_time_by_type_per_day['example__always-firing'] > 0).to.be( - true - ); - expect(telemetry.avg_execution_time_by_type_per_day.test__onlyContextVariables > 0).to.be( - true - ); - expect(telemetry.avg_execution_time_by_type_per_day['test__cumulative-firing'] > 0).to.be( - true - ); - expect(telemetry.avg_execution_time_by_type_per_day.test__noop > 0).to.be(true); + expect(telemetry.avg_execution_time_by_type_per_day.test__patternFiring > 0).to.be(true); expect(telemetry.avg_execution_time_by_type_per_day.test__multipleSearches > 0).to.be(true); expect(telemetry.avg_execution_time_by_type_per_day.test__throw > 0).to.be(true); @@ -330,17 +282,8 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F expect(telemetry.avg_es_search_duration_per_day > 0).to.be(true); // average es search time broken down by rule type, most of these rule types don't perform ES queries - expect( - telemetry.avg_es_search_duration_by_type_per_day['example__always-firing'] === 0 - ).to.be(true); - expect( - telemetry.avg_es_search_duration_by_type_per_day.test__onlyContextVariables === 0 - ).to.be(true); - expect( - telemetry.avg_es_search_duration_by_type_per_day['test__cumulative-firing'] === 0 - ).to.be(true); - expect(telemetry.avg_es_search_duration_by_type_per_day.test__noop === 0).to.be(true); - expect(telemetry.avg_es_search_duration_by_type_per_day.test__throw === 0).to.be(true); + expect(telemetry.avg_es_search_duration_by_type_per_day.test__patternFiring).to.equal(0); + expect(telemetry.avg_es_search_duration_by_type_per_day.test__throw).to.equal(0); // rule type that performs ES search expect(telemetry.avg_es_search_duration_by_type_per_day.test__multipleSearches > 0).to.be( @@ -351,17 +294,8 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F expect(telemetry.avg_total_search_duration_per_day > 0).to.be(true); // average total search time broken down by rule type, most of these rule types don't perform ES queries - expect( - telemetry.avg_total_search_duration_by_type_per_day['example__always-firing'] === 0 - ).to.be(true); - expect( - telemetry.avg_total_search_duration_by_type_per_day.test__onlyContextVariables === 0 - ).to.be(true); - expect( - telemetry.avg_total_search_duration_by_type_per_day['test__cumulative-firing'] === 0 - ).to.be(true); - expect(telemetry.avg_total_search_duration_by_type_per_day.test__noop === 0).to.be(true); - expect(telemetry.avg_total_search_duration_by_type_per_day.test__throw === 0).to.be(true); + expect(telemetry.avg_total_search_duration_by_type_per_day.test__patternFiring).to.equal(0); + expect(telemetry.avg_total_search_duration_by_type_per_day.test__throw).to.equal(0); // rule type that performs ES search expect(telemetry.avg_total_search_duration_by_type_per_day.test__multipleSearches > 0).to.be( @@ -391,32 +325,20 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F // percentile calculations for number of scheduled actions expect(telemetry.percentile_num_generated_actions_per_day.p50 >= 0).to.be(true); expect(telemetry.percentile_num_generated_actions_per_day.p90 >= 0).to.be(true); - expect(telemetry.percentile_num_generated_actions_per_day.p99).to.be.greaterThan(0); + expect(telemetry.percentile_num_generated_actions_per_day.p99 > 0).to.be(true); // percentile calculations by rule type. most of these rule types don't schedule actions so they should all be 0 - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p50['example__always-firing'] - ).to.equal(0); - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p90['example__always-firing'] - ).to.equal(0); - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p99['example__always-firing'] - ).to.equal(0); + // but this rule type does schedule actions so should be least 1 action scheduled expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p50.test__onlyContextVariables - ).to.equal(0); + telemetry.percentile_num_generated_actions_by_type_per_day.p50.test__patternFiring > 0 + ).to.be(true); expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p90.test__onlyContextVariables - ).to.equal(0); + telemetry.percentile_num_generated_actions_by_type_per_day.p90.test__patternFiring > 0 + ).to.be(true); expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p99.test__onlyContextVariables - ).to.equal(0); - - expect(telemetry.percentile_num_generated_actions_by_type_per_day.p50.test__noop).to.equal(0); - expect(telemetry.percentile_num_generated_actions_by_type_per_day.p90.test__noop).to.equal(0); - expect(telemetry.percentile_num_generated_actions_by_type_per_day.p99.test__noop).to.equal(0); + telemetry.percentile_num_generated_actions_by_type_per_day.p99.test__patternFiring > 0 + ).to.be(true); expect(telemetry.percentile_num_generated_actions_by_type_per_day.p50.test__throw).to.equal( 0 @@ -438,46 +360,23 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F telemetry.percentile_num_generated_actions_by_type_per_day.p99.test__multipleSearches ).to.equal(0); - // this rule type does schedule actions so should be least 1 action scheduled - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p50['test__cumulative-firing'] - ).to.be.greaterThan(0); - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p90['test__cumulative-firing'] - ).to.be.greaterThan(0); - expect( - telemetry.percentile_num_generated_actions_by_type_per_day.p99['test__cumulative-firing'] - ).to.be.greaterThan(0); - // percentile calculations for number of alerts expect(telemetry.percentile_num_alerts_per_day.p50 >= 0).to.be(true); expect(telemetry.percentile_num_alerts_per_day.p90 >= 0).to.be(true); - expect(telemetry.percentile_num_alerts_per_day.p99).to.be.greaterThan(0); + expect(telemetry.percentile_num_alerts_per_day.p99 > 0).to.be(true); // percentile calculations by rule type. most of these rule types don't generate alerts so they should all be 0 - expect( - telemetry.percentile_num_alerts_by_type_per_day.p50['example__always-firing'] - ).to.equal(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p90['example__always-firing'] - ).to.equal(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p99['example__always-firing'] - ).to.equal(0); - - expect( - telemetry.percentile_num_alerts_by_type_per_day.p50.test__onlyContextVariables - ).to.equal(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p90.test__onlyContextVariables - ).to.equal(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p99.test__onlyContextVariables - ).to.equal(0); + // but this rule type does generate alerts so should be least 1 alert - expect(telemetry.percentile_num_alerts_by_type_per_day.p50.test__noop).to.equal(0); - expect(telemetry.percentile_num_alerts_by_type_per_day.p90.test__noop).to.equal(0); - expect(telemetry.percentile_num_alerts_by_type_per_day.p99.test__noop).to.equal(0); + expect(telemetry.percentile_num_alerts_by_type_per_day.p50.test__patternFiring > 0).to.be( + true + ); + expect(telemetry.percentile_num_alerts_by_type_per_day.p90.test__patternFiring > 0).to.be( + true + ); + expect(telemetry.percentile_num_alerts_by_type_per_day.p99.test__patternFiring > 0).to.be( + true + ); expect(telemetry.percentile_num_alerts_by_type_per_day.p50.test__throw).to.equal(0); expect(telemetry.percentile_num_alerts_by_type_per_day.p90.test__throw).to.equal(0); @@ -493,45 +392,30 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F 0 ); - // this rule type does generate alerts so should be least 1 alert - expect( - telemetry.percentile_num_alerts_by_type_per_day.p50['test__cumulative-firing'] - ).to.be.greaterThan(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p90['test__cumulative-firing'] - ).to.be.greaterThan(0); - expect( - telemetry.percentile_num_alerts_by_type_per_day.p99['test__cumulative-firing'] - ).to.be.greaterThan(0); - // rules grouped by execution status - expect(telemetry.count_rules_by_execution_status).to.eql({ - success: 15, - error: 3, - warning: 0, - }); + expect(telemetry.count_rules_by_execution_status.success > 0).to.be(true); + expect(telemetry.count_rules_by_execution_status.error > 0).to.be(true); + expect(telemetry.count_rules_by_execution_status.warning).to.equal(0); + // number of rules that has tags - expect(telemetry.count_rules_with_tags).to.be(21); + expect(telemetry.count_rules_with_tags).to.equal(12); // rules grouped by notify when - expect(telemetry.count_rules_by_notify_when).to.eql({ - on_action_group_change: 0, - on_active_alert: 6, - on_throttle_interval: 15, - }); + expect(telemetry.count_rules_by_notify_when.on_action_group_change).to.equal(0); + expect(telemetry.count_rules_by_notify_when.on_active_alert).to.equal(0); + expect(telemetry.count_rules_by_notify_when.on_throttle_interval).to.equal(12); // rules snoozed - expect(telemetry.count_rules_snoozed).to.be(0); + expect(telemetry.count_rules_snoozed).to.equal(0); // rules muted - expect(telemetry.count_rules_muted).to.be(0); + expect(telemetry.count_rules_muted).to.equal(0); // rules with muted alerts - expect(telemetry.count_rules_with_muted_alerts).to.be(0); + expect(telemetry.count_rules_with_muted_alerts).to.equal(0); // Connector types grouped by consumers - expect(telemetry.count_connector_types_by_consumers).to.eql({ - // eslint-disable-next-line @typescript-eslint/naming-convention - alertsFixture: { test__noop: 9, test__throw: 9, __slack: 3 }, - }); + expect(telemetry.count_connector_types_by_consumers.alertsFixture.test__noop).to.equal(6); + expect(telemetry.count_connector_types_by_consumers.alertsFixture.test__throw).to.equal(3); + expect(telemetry.count_connector_types_by_consumers.alertsFixture.__slack).to.equal(3); - expect(telemetry.count_rules_by_execution_status_per_day.failure).to.greaterThan(0); - expect(telemetry.count_rules_by_execution_status_per_day.success).to.greaterThan(0); + expect(telemetry.count_rules_by_execution_status_per_day.failure > 0).to.be(true); + expect(telemetry.count_rules_by_execution_status_per_day.success > 0).to.be(true); } it('should retrieve telemetry data in the expected format', async () => { @@ -541,11 +425,11 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F await retry.try(async () => { return await getEventLog({ getService, - spaceId: Spaces[0].id, + spaceId: Spaces[2].id, type: 'alert', - id: alwaysFiringRuleId[Spaces[0].id], + id: alwaysFiringRuleId[Spaces[2].id], provider: 'alerting', - actions: new Map([['execute', { gte: 10 }]]), + actions: new Map([['execute', { gte: 1 }]]), }); }); @@ -566,10 +450,12 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F const taskState = telemetryTask!._source!.task?.state; expect(taskState).not.to.be(undefined); actionsTelemetry = JSON.parse(taskState!); - expect(actionsTelemetry.runs).to.equal(2); - expect(actionsTelemetry.count_total).to.equal(20); + expect(actionsTelemetry.runs > 0).to.be(true); + expect(actionsTelemetry.count_total).to.equal(21); }); + verifyActionsTelemetry(actionsTelemetry); + // request alerting telemetry task to run await supertest .post('/api/alerting_actions_telemetry/run_soon') @@ -587,11 +473,10 @@ export default function createAlertingAndActionsTelemetryTests({ getService }: F const taskState = telemetryTask!._source!.task?.state; expect(taskState).not.to.be(undefined); alertingTelemetry = JSON.parse(taskState!); - expect(alertingTelemetry.runs).to.equal(2); - expect(alertingTelemetry.count_total).to.equal(21); + expect(alertingTelemetry.runs > 0).to.be(true); + expect(alertingTelemetry.count_total).to.equal(12); }); - verifyActionsTelemetry(actionsTelemetry); verifyAlertingTelemetry(alertingTelemetry); }); }); diff --git a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/index.ts b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/index.ts index 829de0e1c6e71..9d393e73a4640 100644 --- a/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/index.ts +++ b/x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/telemetry/index.ts @@ -10,17 +10,13 @@ import { setupSpacesAndUsers, tearDown } from '../../../setup'; // eslint-disable-next-line import/no-default-export export default function telemetryTests({ loadTestFile, getService }: FtrProviderContext) { - const esArchiver = getService('esArchiver'); - // FLAKY: https://github.com/elastic/kibana/issues/136153 - describe.skip('Alerting and Actions Telemetry', () => { + describe('Alerting and Actions Telemetry', () => { before(async () => { - await esArchiver.load('x-pack/test/functional/es_archives/event_log_telemetry'); // reset the state in the telemetry task await setupSpacesAndUsers(getService); }); after(async () => { await tearDown(getService); - await esArchiver.unload('x-pack/test/functional/es_archives/event_log_telemetry'); }); loadTestFile(require.resolve('./alerting_and_actions_telemetry')); From 6b44fff5acd9cd10e53965446bc36a77769a6e50 Mon Sep 17 00:00:00 2001 From: Achyut Jhunjhunwala Date: Mon, 31 Jul 2023 14:27:12 +0200 Subject: [PATCH 6/6] [APM] Implementing Journey for APM (#162721) ## Summary Closes - https://github.com/elastic/kibana/issues/153844 As part of this PR, as its just the stepping stone, we will only cover a basic navigation flow and analyze the result obtained from Steps Dashboard and data collected by the APM Agents for this journey ## Scope - Generating a data set using Synthtrace instead of Archives - Capturing the flow from Service Inventory to Trace Waterfall loading on Transaction page - Capturing Event loop utilisation metrics enabled for APM Journey ## How to run it ``` node scripts/run_performance.js --journey-path x-pack/performance/journeys/apm_service_inventory.ts --skip-warmup ``` --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .buildkite/ftr_configs.yml | 1 + x-pack/performance/configs/apm_config.ts | 37 +++++++++ .../journeys/apm_service_inventory.ts | 53 +++++++++++++ .../performance/services/synthtrace/index.ts | 66 ++++++++++++++++ .../performance/synthtrace_data/apm_data.ts | 77 +++++++++++++++++++ x-pack/performance/tsconfig.json | 3 + 6 files changed, 237 insertions(+) create mode 100644 x-pack/performance/configs/apm_config.ts create mode 100644 x-pack/performance/journeys/apm_service_inventory.ts create mode 100644 x-pack/performance/services/synthtrace/index.ts create mode 100644 x-pack/performance/synthtrace_data/apm_data.ts diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml index 8a5d4c2ea3f9f..70be71f08184a 100644 --- a/.buildkite/ftr_configs.yml +++ b/.buildkite/ftr_configs.yml @@ -413,5 +413,6 @@ enabled: - x-pack/performance/journeys/ecommerce_dashboard_tsvb_gauge_only.ts - x-pack/performance/journeys/dashboard_listing_page.ts - x-pack/performance/journeys/cloud_security_dashboard.ts + - x-pack/performance/journeys/apm_service_inventory.ts - x-pack/test/custom_branding/config.ts - x-pack/test/profiling_api_integration/cloud/config.ts diff --git a/x-pack/performance/configs/apm_config.ts b/x-pack/performance/configs/apm_config.ts new file mode 100644 index 0000000000000..ebcfeb1aa3448 --- /dev/null +++ b/x-pack/performance/configs/apm_config.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FtrConfigProviderContext } from '@kbn/test'; +import { CA_CERT_PATH } from '@kbn/dev-utils'; + +// eslint-disable-next-line import/no-default-export +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const xpackFunctionalConfig = await readConfigFile( + // eslint-disable-next-line @kbn/imports/no_boundary_crossing + require.resolve('../../test/functional/config.base.js') + ); + + return { + ...xpackFunctionalConfig.getAll(), + + kbnTestServer: { + ...xpackFunctionalConfig.get('kbnTestServer'), + serverArgs: [ + ...xpackFunctionalConfig.get('kbnTestServer.serverArgs'), + '--home.disableWelcomeScreen=true', + '--csp.strict=false', + '--csp.warnLegacyBrowsers=false', + // define custom kibana server args here + `--elasticsearch.ssl.certificateAuthorities=${CA_CERT_PATH}`, + '--server.eluMonitor.enabled=true', + '--server.eluMonitor.logging.enabled=true', + '--server.eluMonitor.logging.threshold.ela=250', + '--server.eluMonitor.logging.threshold.elu=0.15', + ], + }, + }; +} diff --git a/x-pack/performance/journeys/apm_service_inventory.ts b/x-pack/performance/journeys/apm_service_inventory.ts new file mode 100644 index 0000000000000..03a5fce636dcd --- /dev/null +++ b/x-pack/performance/journeys/apm_service_inventory.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Journey } from '@kbn/journeys'; +import { SynthtraceClient } from '../services/synthtrace'; +import { generateData } from '../synthtrace_data/apm_data'; + +export const journey = new Journey({ + beforeSteps: async ({ kbnUrl, log, auth, es }) => { + // Install APM Package + const synthClient = new SynthtraceClient({ + kbnBaseUrl: kbnUrl.get(), + logger: log, + username: auth.getUsername(), + password: auth.getPassword(), + esClient: es, + }); + + await synthClient.installApmPackage(); + // Setup Synthtrace Client + await synthClient.initialiseEsClient(); + // Generate data using Synthtrace + const start = Date.now() - 1000; + const end = Date.now(); + await synthClient.index( + generateData({ + from: new Date(start).getTime(), + to: new Date(end).getTime(), + }) + ); + }, + ftrConfigPath: 'x-pack/performance/configs/apm_config.ts', +}) + .step('Navigate to Service Inventory Page', async ({ page, kbnUrl }) => { + await page.goto(kbnUrl.get(`app/apm/services`)); + await page.waitForSelector(`[data-test-subj="serviceLink_nodejs"]`); + }) + .step('Navigate to Service Overview Page', async ({ page, kbnUrl }) => { + await page.click(`[data-test-subj="serviceLink_nodejs"]`); + await page.waitForSelector(`[data-test-subj="apmMainTemplateHeaderServiceName"]`); + }) + .step('Navigate to Transactions tabs', async ({ page, kbnUrl }) => { + await page.click(`[data-test-subj="transactionsTab"]`); + await page.waitForSelector(`[data-test-subj="apmTransactionDetailLinkLink"]`); + }) + .step('Wait for Trace Waterfall on the page to load', async ({ page, kbnUrl }) => { + await page.click(`[data-test-subj="apmTransactionDetailLinkLink"]`); + await page.waitForSelector(`[data-test-subj="apmWaterfallButton"]`); + }); diff --git a/x-pack/performance/services/synthtrace/index.ts b/x-pack/performance/services/synthtrace/index.ts new file mode 100644 index 0000000000000..fc7ac5c306b62 --- /dev/null +++ b/x-pack/performance/services/synthtrace/index.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ApmSynthtraceEsClient, ApmSynthtraceKibanaClient } from '@kbn/apm-synthtrace'; +import Url from 'url'; +import { Readable } from 'stream'; +import { ApmFields, SynthtraceGenerator } from '@kbn/apm-synthtrace-client'; + +export interface SynthtraceClientParams { + kbnBaseUrl: string; + logger: any; + username: string; + password: string; + esClient: any; +} + +export class SynthtraceClient { + private synthtraceEsClient: ApmSynthtraceEsClient | undefined; + private packageVersion: string = ''; + private readonly kibanaUrlWithAuth: string; + + constructor(private readonly baseParams: SynthtraceClientParams) { + const kibanaUrl = new URL(this.baseParams.kbnBaseUrl); + this.kibanaUrlWithAuth = Url.format({ + protocol: kibanaUrl.protocol, + hostname: kibanaUrl.hostname, + port: kibanaUrl.port, + auth: `${this.baseParams.username}:${this.baseParams.password}`, + }); + } + + async installApmPackage() { + const kibanaClient = new ApmSynthtraceKibanaClient({ + logger: this.baseParams.logger, + target: this.kibanaUrlWithAuth, + }); + this.packageVersion = await kibanaClient.fetchLatestApmPackageVersion(); + + await kibanaClient.installApmPackage(this.packageVersion); + } + + async initialiseEsClient() { + this.synthtraceEsClient = new ApmSynthtraceEsClient({ + client: this.baseParams.esClient, + logger: this.baseParams.logger, + refreshAfterIndex: true, + version: this.packageVersion, + }); + + this.synthtraceEsClient.pipeline(this.synthtraceEsClient.getDefaultPipeline(false)); + } + + async index(events: SynthtraceGenerator) { + if (this.synthtraceEsClient) { + await this.synthtraceEsClient.index( + Readable.from(Array.from(events).flatMap((event) => event.serialize())) + ); + } else { + throw new Error('ES Client not initialised'); + } + } +} diff --git a/x-pack/performance/synthtrace_data/apm_data.ts b/x-pack/performance/synthtrace_data/apm_data.ts new file mode 100644 index 0000000000000..4a7ab835d37cd --- /dev/null +++ b/x-pack/performance/synthtrace_data/apm_data.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { apm, httpExitSpan, timerange } from '@kbn/apm-synthtrace-client'; + +export function generateData({ from, to }: { from: number; to: number }) { + const range = timerange(from, to); + const transactionName = '240rpm/75% 1000ms'; + + const synthRum = apm + .service({ name: 'synth-rum', environment: 'production', agentName: 'rum-js' }) + .instance('my-instance'); + const synthNode = apm + .service({ name: 'synth-node', environment: 'production', agentName: 'nodejs' }) + .instance('my-instance'); + const synthGo = apm + .service({ name: 'synth-go', environment: 'production', agentName: 'go' }) + .instance('my-instance'); + + return range.interval('1m').generator((timestamp) => { + return synthRum + .transaction({ transactionName }) + .duration(400) + .timestamp(timestamp) + .children( + // synth-rum -> synth-node + synthRum + .span( + httpExitSpan({ + spanName: 'GET /api/products/top', + destinationUrl: 'http://synth-node:3000', + }) + ) + .duration(300) + .timestamp(timestamp) + + .children( + // synth-node + synthNode + .transaction({ transactionName: 'Initial transaction in synth-node' }) + .duration(300) + .timestamp(timestamp) + .children( + synthNode + // synth-node -> synth-go + .span( + httpExitSpan({ + spanName: 'GET synth-go:3000', + destinationUrl: 'http://synth-go:3000', + }) + ) + .timestamp(timestamp) + .duration(400) + + .children( + // synth-go + synthGo + + .transaction({ transactionName: 'Initial transaction in synth-go' }) + .timestamp(timestamp) + .duration(200) + .children( + synthGo + .span({ spanName: 'custom_operation', spanType: 'custom' }) + .timestamp(timestamp) + .duration(100) + .success() + ) + ) + ) + ) + ); + }); +} diff --git a/x-pack/performance/tsconfig.json b/x-pack/performance/tsconfig.json index fb60be2e310ce..8dcd1caca873f 100644 --- a/x-pack/performance/tsconfig.json +++ b/x-pack/performance/tsconfig.json @@ -14,5 +14,8 @@ "@kbn/tooling-log", "@kbn/test", "@kbn/expect", + "@kbn/dev-utils", + "@kbn/apm-synthtrace", + "@kbn/apm-synthtrace-client", ] }