From 8353a7e160ef6030a6b213924d24d63ea89220b8 Mon Sep 17 00:00:00 2001 From: Navarone Feekery <13634519+navarone-feekery@users.noreply.github.com> Date: Fri, 29 Sep 2023 14:05:22 +0200 Subject: [PATCH 1/9] [Search] Add extraction service RCF to native connectors (#167527) ## Summary Add the `use_text_extraction_service` rich configurable field to some native connectors. Native connectors will not have access to the feature, but the field is required in case the user converts their native connector to a self-managed connector. --- .../types/native_connectors.ts | 107 +++++++++++++++--- 1 file changed, 93 insertions(+), 14 deletions(-) diff --git a/packages/kbn-search-connectors/types/native_connectors.ts b/packages/kbn-search-connectors/types/native_connectors.ts index 0e767b4bdd345..1c4b0bb7e1694 100644 --- a/packages/kbn-search-connectors/types/native_connectors.ts +++ b/packages/kbn-search-connectors/types/native_connectors.ts @@ -50,6 +50,22 @@ const MAX_CONCURRENT_DOWNLOADS_LABEL = i18n.translate( } ); +const USE_TEXT_EXTRACTION_SERVICE_LABEL = i18n.translate( + 'searchConnectors.nativeConnectors.textExtractionService.label', + { + defaultMessage: 'Use text extraction service', + } +); + +const USE_TEXT_EXTRACTION_SERVICE_TOOLTIP = i18n.translate( + 'searchConnectors.nativeConnectors.textExtractionService.tooltip', + { + defaultMessage: + 'Requires a separate deployment of the Elastic Data Extraction Service. ' + + 'Also requires that pipeline settings disable text extraction.', + } +); + const DATABASE_LABEL = i18n.translate('searchConnectors.nativeConnectors.databaseLabel', { defaultMessage: 'Database', }); @@ -157,6 +173,21 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record Date: Fri, 29 Sep 2023 08:05:35 -0400 Subject: [PATCH 2/9] chore(slo): Improve form field selectors (#167564) --- .../common/index_field_selector.tsx | 5 +- .../custom_metric/custom_metric_type_form.tsx | 9 +- .../custom_metric/metric_indicator.tsx | 98 +++++++------ .../histogram/histogram_indicator.tsx | 131 ++++++++++-------- .../histogram_indicator_type_form.tsx | 5 +- 5 files changed, 141 insertions(+), 107 deletions(-) diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/common/index_field_selector.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/common/index_field_selector.tsx index 6bb6996f95b2a..ef9121e871bbd 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/components/common/index_field_selector.tsx +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/common/index_field_selector.tsx @@ -6,7 +6,6 @@ */ import { EuiComboBox, EuiComboBoxOptionOption, EuiFlexItem, EuiFormRow } from '@elastic/eui'; -import { ALL_VALUE } from '@kbn/slo-schema'; import React, { useEffect, useState } from 'react'; import { Controller, useFormContext } from 'react-hook-form'; import { Field } from '../../../../hooks/slo/use_fetch_index_pattern_fields'; @@ -42,7 +41,7 @@ export function IndexFieldSelector({ { diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/custom_metric/custom_metric_type_form.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/custom_metric/custom_metric_type_form.tsx index c7fe7999f9be4..45870dcd68bd0 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/components/custom_metric/custom_metric_type_form.tsx +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/custom_metric/custom_metric_type_form.tsx @@ -27,6 +27,8 @@ import { MetricIndicator } from './metric_indicator'; export { NEW_CUSTOM_METRIC } from './metric_indicator'; +const SUPPORTED_METRIC_FIELD_TYPES = ['number', 'histogram']; + export function CustomMetricIndicatorTypeForm() { const { watch } = useFormContext(); const index = watch('indicator.params.index'); @@ -34,6 +36,9 @@ export function CustomMetricIndicatorTypeForm() { useFetchIndexPatternFields(index); const timestampFields = indexFields.filter((field) => field.type === 'date'); const partitionByFields = indexFields.filter((field) => field.aggregatable); + const metricFields = indexFields.filter((field) => + SUPPORTED_METRIC_FIELD_TYPES.includes(field.type) + ); return ( <> @@ -115,7 +120,7 @@ export function CustomMetricIndicatorTypeForm() { @@ -136,7 +141,7 @@ export function CustomMetricIndicatorTypeForm() { diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/custom_metric/metric_indicator.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/custom_metric/metric_indicator.tsx index a333a353f97d9..0c1fbce49d64e 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/components/custom_metric/metric_indicator.tsx +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/custom_metric/metric_indicator.tsx @@ -19,16 +19,16 @@ import { import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { first, range, xor } from 'lodash'; -import React from 'react'; +import React, { useEffect, useState } from 'react'; import { Controller, useFieldArray, useFormContext } from 'react-hook-form'; import { Field } from '../../../../hooks/slo/use_fetch_index_pattern_fields'; -import { createOptionsFromFields } from '../../helpers/create_options'; +import { createOptionsFromFields, Option } from '../../helpers/create_options'; import { CreateSLOForm } from '../../types'; import { QueryBuilder } from '../common/query_builder'; interface MetricIndicatorProps { type: 'good' | 'total'; - indexFields: Field[]; + metricFields: Field[]; isLoadingIndex: boolean; } @@ -47,51 +47,52 @@ function createEquationFromMetric(names: string[]) { return names.join(' + '); } -const SUPPORTED_FIELD_TYPES = ['number', 'histogram']; +const metricLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.customMetric.metricLabel', + { defaultMessage: 'Metric' } +); -export function MetricIndicator({ type, indexFields, isLoadingIndex }: MetricIndicatorProps) { - const metricLabel = i18n.translate( - 'xpack.observability.slo.sloEdit.sliType.customMetric.metricLabel', - { defaultMessage: 'Metric' } - ); +const filterLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.customMetric.filterLabel', + { defaultMessage: 'Filter' } +); - const filterLabel = i18n.translate( - 'xpack.observability.slo.sloEdit.sliType.customMetric.filterLabel', - { defaultMessage: 'Filter' } - ); +const metricTooltip = ( + +); - const metricTooltip = ( - - ); +const equationLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.customMetric.equationLabel', + { defaultMessage: 'Equation' } +); - const equationLabel = i18n.translate( - 'xpack.observability.slo.sloEdit.sliType.customMetric.equationLabel', - { defaultMessage: 'Equation' } - ); +const equationTooltip = ( + +); - const equationTooltip = ( - - ); +export function MetricIndicator({ type, metricFields, isLoadingIndex }: MetricIndicatorProps) { + const { control, watch, setValue, register, getFieldState } = useFormContext(); + const [options, setOptions] = useState(createOptionsFromFields(metricFields)); - const { control, watch, setValue, register } = useFormContext(); - const metricFields = indexFields.filter((field) => SUPPORTED_FIELD_TYPES.includes(field.type)); + useEffect(() => { + setOptions(createOptionsFromFields(metricFields)); + }, [metricFields]); const { fields, append, remove } = useFieldArray({ control, @@ -134,6 +135,7 @@ export function MetricIndicator({ type, indexFields, isLoadingIndex }: MetricInd {metricLabel} {metric.name} {metricTooltip} @@ -163,8 +165,9 @@ export function MetricIndicator({ type, indexFields, isLoadingIndex }: MetricInd 'xpack.observability.slo.sloEdit.sliType.customMetric.metricField.placeholder', { defaultMessage: 'Select a metric field' } )} + isClearable isInvalid={fieldState.invalid} - isDisabled={!indexPattern} + isDisabled={isLoadingIndex || !indexPattern} isLoading={!!indexPattern && isLoadingIndex} onChange={(selected: EuiComboBoxOptionOption[]) => { if (selected.length) { @@ -184,7 +187,14 @@ export function MetricIndicator({ type, indexFields, isLoadingIndex }: MetricInd ] : [] } - options={createOptionsFromFields(metricFields)} + onSearchChange={(searchValue: string) => { + setOptions( + createOptionsFromFields(metricFields, ({ value }) => + value.includes(searchValue) + ) + ); + }} + options={options} /> )} /> diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/histogram/histogram_indicator.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/histogram/histogram_indicator.tsx index 2b6f983470f82..1362674828d74 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/components/histogram/histogram_indicator.tsx +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/histogram/histogram_indicator.tsx @@ -5,9 +5,6 @@ * 2.0. */ -import React, { Fragment } from 'react'; -import { Controller, useFormContext } from 'react-hook-form'; -import { i18n } from '@kbn/i18n'; import { EuiComboBox, EuiComboBoxOptionOption, @@ -18,14 +15,17 @@ import { EuiIconTip, EuiSpacer, } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React, { Fragment, useEffect, useState } from 'react'; +import { Controller, useFormContext } from 'react-hook-form'; +import { Field } from '../../../../hooks/slo/use_fetch_index_pattern_fields'; +import { createOptionsFromFields, Option } from '../../helpers/create_options'; import { CreateSLOForm } from '../../types'; import { QueryBuilder } from '../common/query_builder'; -import { Field } from '../../../../hooks/slo/use_fetch_index_pattern_fields'; -import { createOptionsFromFields } from '../../helpers/create_options'; interface HistogramIndicatorProps { type: 'good' | 'total'; - indexFields: Field[]; + histogramFields: Field[]; isLoadingIndex: boolean; } @@ -46,61 +46,69 @@ const AGGREGATIONS = { const AGGREGATION_OPTIONS = Object.values(AGGREGATIONS); -export function HistogramIndicator({ type, indexFields, isLoadingIndex }: HistogramIndicatorProps) { - const { control, watch } = useFormContext(); +const aggregationTooltip = ( + +); - const histogramFields = indexFields.filter((field) => field.type === 'histogram'); - const indexPattern = watch('indicator.params.index'); - const aggregation = watch(`indicator.params.${type}.aggregation`); +const fromTooltip = ( + +); - const aggregationTooltip = ( - - ); +const toTooltip = ( + +); - const fromTooltip = ( - - ); +const aggregationLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.histogram.aggregationLabel', + { defaultMessage: 'Aggregation' } +); - const toTooltip = ( - - ); +const metricLabel = i18n.translate( + 'xpack.observability.slo.sloEdit.sliType.histogram.metricLabel', + { defaultMessage: 'Field' } +); - const aggregationLabel = i18n.translate( - 'xpack.observability.slo.sloEdit.sliType.histogram.aggregationLabel', - { defaultMessage: 'Aggregation' } - ); +const toLabel = i18n.translate('xpack.observability.slo.sloEdit.sliType.histogram.toLabel', { + defaultMessage: 'To', +}); - const metricLabel = i18n.translate( - 'xpack.observability.slo.sloEdit.sliType.histogram.metricLabel', - { defaultMessage: 'Field' } - ); +const fromLabel = i18n.translate('xpack.observability.slo.sloEdit.sliType.histogram.fromLabel', { + defaultMessage: 'From', +}); + +export function HistogramIndicator({ + type, + histogramFields, + isLoadingIndex, +}: HistogramIndicatorProps) { + const { control, watch, getFieldState } = useFormContext(); + const [options, setOptions] = useState(createOptionsFromFields(histogramFields)); - const toLabel = i18n.translate('xpack.observability.slo.sloEdit.sliType.histogram.toLabel', { - defaultMessage: 'To', - }); + useEffect(() => { + setOptions(createOptionsFromFields(histogramFields)); + }, [histogramFields]); - const fromLabel = i18n.translate('xpack.observability.slo.sloEdit.sliType.histogram.fromLabel', { - defaultMessage: 'From', - }); + const indexPattern = watch('indicator.params.index'); + const aggregation = watch(`indicator.params.${type}.aggregation`); return ( @@ -149,7 +157,11 @@ export function HistogramIndicator({ type, indexFields, isLoadingIndex }: Histog - {metricLabel}}> + {metricLabel}} + > { if (selected.length) { @@ -190,7 +202,14 @@ export function HistogramIndicator({ type, indexFields, isLoadingIndex }: Histog ] : [] } - options={createOptionsFromFields(histogramFields)} + onSearchChange={(searchValue: string) => { + setOptions( + createOptionsFromFields(histogramFields, ({ value }) => + value.includes(searchValue) + ) + ); + }} + options={options} /> )} /> diff --git a/x-pack/plugins/observability/public/pages/slo_edit/components/histogram/histogram_indicator_type_form.tsx b/x-pack/plugins/observability/public/pages/slo_edit/components/histogram/histogram_indicator_type_form.tsx index dfbf305235b41..eb18e0a363628 100644 --- a/x-pack/plugins/observability/public/pages/slo_edit/components/histogram/histogram_indicator_type_form.tsx +++ b/x-pack/plugins/observability/public/pages/slo_edit/components/histogram/histogram_indicator_type_form.tsx @@ -31,6 +31,7 @@ export function HistogramIndicatorTypeForm() { const { isLoading: isIndexFieldsLoading, data: indexFields = [] } = useFetchIndexPatternFields(index); + const histogramFields = indexFields.filter((field) => field.type === 'histogram'); const timestampFields = indexFields.filter((field) => field.type === 'date'); const partitionByFields = indexFields.filter((field) => field.aggregatable); @@ -109,7 +110,7 @@ export function HistogramIndicatorTypeForm() { @@ -128,7 +129,7 @@ export function HistogramIndicatorTypeForm() { From 706f6fa7a02253fdbfe9bba195d4703a4398b0b9 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Thu, 28 Sep 2023 20:13:10 -0500 Subject: [PATCH 3/9] Revert "[AO][SERVERLESS] Fix Custom Threshold rule tests for Serverless (#166942)" This reverts commit 4c1ca7e92fbc387279d0d41fd1238e790ce53518. --- .../composable/component/base.json | 25 -- .../composable/component/event.json | 24 -- .../composable/component/host.json | 189 --------------- .../composable/component/metricset.json | 18 -- .../composable/component/system.json | 69 ------ .../src/data_sources/composable/template.json | 52 ----- .../src/data_sources/fake_hosts/index.ts | 2 +- .../fake_hosts/index_template_def.ts | 38 --- .../src/data_sources/fake_hosts/template.ts | 218 ++++++++++++++++++ .../src/lib/install_template.ts | 28 +++ .../src/lib/manage_template.ts | 59 ----- .../packages/kbn-infra-forge/src/lib/queue.ts | 4 +- x-pack/packages/kbn-infra-forge/src/run.ts | 7 +- .../custom_threshold_rule/avg_pct_fired.ts | 7 +- .../custom_eq_avg_bytes_fired.ts | 7 +- .../documents_count_fired.ts | 7 +- .../custom_threshold_rule/group_by_fired.ts | 7 +- .../observability/metric_threshold_rule.ts | 6 - .../custom_threshold_rule/avg_pct_fired.ts | 23 +- .../custom_threshold_rule/avg_pct_no_data.ts | 10 +- .../custom_eq_avg_bytes_fired.ts | 14 +- .../documents_count_fired.ts | 16 +- .../custom_threshold_rule/group_by_fired.ts | 19 +- 23 files changed, 296 insertions(+), 553 deletions(-) delete mode 100644 x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/base.json delete mode 100644 x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/event.json delete mode 100644 x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/host.json delete mode 100644 x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/metricset.json delete mode 100644 x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/system.json delete mode 100644 x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json delete mode 100644 x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index_template_def.ts create mode 100644 x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/template.ts create mode 100644 x-pack/packages/kbn-infra-forge/src/lib/install_template.ts delete mode 100644 x-pack/packages/kbn-infra-forge/src/lib/manage_template.ts diff --git a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/base.json b/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/base.json deleted file mode 100644 index cebc9c2971745..0000000000000 --- a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/base.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "_meta": { - "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-base.html", - "ecs_version": "8.0.0" - }, - "template": { - "mappings": { - "properties": { - "@timestamp": { - "type": "date" - }, - "labels": { - "type": "object" - }, - "message": { - "type": "match_only_text" - }, - "tags": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } -} diff --git a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/event.json b/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/event.json deleted file mode 100644 index f235248f3ee83..0000000000000 --- a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/event.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "_meta": { - "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-event.html", - "ecs_version": "8.0.0" - }, - "template": { - "mappings": { - "properties": { - "event": { - "properties": { - "dataset": { - "ignore_above": 1024, - "type": "keyword" - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } -} diff --git a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/host.json b/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/host.json deleted file mode 100644 index cc370a5270eba..0000000000000 --- a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/host.json +++ /dev/null @@ -1,189 +0,0 @@ -{ - "_meta": { - "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-host.html", - "ecs_version": "8.0.0" - }, - "template": { - "mappings": { - "properties": { - "host": { - "properties": { - "architecture": { - "ignore_above": 1024, - "type": "keyword" - }, - "cpu": { - "properties": { - "usage": { - "scaling_factor": 1000, - "type": "scaled_float" - } - } - }, - "disk": { - "properties": { - "read": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "write": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "postal_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "timezone": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "network": { - "properties": { - "egress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - }, - "ingress": { - "properties": { - "bytes": { - "type": "long" - }, - "packets": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "family": { - "ignore_above": 1024, - "type": "keyword" - }, - "full": { - "fields": { - "text": { - "type": "match_only_text" - } - }, - "ignore_above": 1024, - "type": "keyword" - }, - "kernel": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "fields": { - "text": { - "type": "match_only_text" - } - }, - "ignore_above": 1024, - "type": "keyword" - }, - "platform": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "uptime": { - "type": "long" - } - } - } - } - } - } -} diff --git a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/metricset.json b/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/metricset.json deleted file mode 100644 index 06ef8b57bf4b3..0000000000000 --- a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/metricset.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "_meta": { - "ecs_version": "8.0.0" - }, - "template": { - "mappings": { - "properties": { - "metricset": { - "properties": { - "interval": { - "type": "long" - } - } - } - } - } - } -} diff --git a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/system.json b/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/system.json deleted file mode 100644 index 46335a6da442c..0000000000000 --- a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/component/system.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "_meta": { - "ecs_version": "8.0.0" - }, - "template": { - "mappings": { - "properties": { - "system": { - "properties": { - "cpu": { - "properties": { - "cores": { - "type": "integer" - }, - "system": { - "properties": { - "pct": { - "type": "float" - } - } - }, - "total": { - "properties": { - "norm": { - "properties": { - "pct": { - "type": "float" - } - } - } - } - }, - "user": { - "properties": { - "pct": { - "type": "float" - } - } - } - } - }, - "network": { - "properties": { - "in": { - "properties": { - "bytes": { - "type": "long" - } - } - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "out": { - "properties": { - "bytes": { - "type": "long" - } - } - } - } - } - } - } - } - } - } -} diff --git a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json b/x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json deleted file mode 100644 index 8064e7866e5f6..0000000000000 --- a/x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "_meta": { - "description": "Sample composable template that includes all ECS fields", - "ecs_version": "8.0.0" - }, - "composed_of": [ - "ecs_8.0.0_base", - "ecs_8.0.0_event", - "ecs_8.0.0_host", - "ecs_8.0.0_metricset", - "ecs_8.0.0_system" - ], - "index_patterns": [ - "kbn-data-forge-fake_hosts" - ], - "priority": 1, - "template": { - "mappings": { - "_meta": { - "version": "1.6.0" - }, - "date_detection": false, - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ] - }, - "settings": { - "index": { - "codec": "best_compression", - "mapping": { - "total_fields": { - "limit": 2000 - } - } - } - } - } -} diff --git a/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index.ts b/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index.ts index fe969726499b4..f6b685077ec6a 100644 --- a/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index.ts +++ b/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index.ts @@ -8,7 +8,7 @@ import lodash from 'lodash'; import type { Moment } from 'moment'; -export { indexTemplate } from './index_template_def'; +export { template } from './template'; const createGroupIndex = (index: number) => Math.floor(index / 1000) * 1000; diff --git a/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index_template_def.ts b/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index_template_def.ts deleted file mode 100644 index ab3876edabf5a..0000000000000 --- a/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/index_template_def.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as rt from 'io-ts'; -import base from '../composable/component/base.json'; -import event from '../composable/component/event.json'; -import host from '../composable/component/host.json'; -import metricset from '../composable/component/metricset.json'; -import system from '../composable/component/system.json'; - -import template from '../composable/template.json'; -const IndexTemplateDefRT = rt.type({ - namespace: rt.string, - template: rt.UnknownRecord, - components: rt.array(rt.type({ name: rt.string, template: rt.UnknownRecord })), -}); - -export type IndexTemplateDef = rt.TypeOf; - -const ECS_VERSION = template._meta.ecs_version; - -const components = [ - { name: `fake_hosts_${ECS_VERSION}_base`, template: base }, - { name: `fake_hosts_${ECS_VERSION}_event`, template: event }, - { name: `fake_hosts_${ECS_VERSION}_host`, template: host }, - { name: `fake_hosts_${ECS_VERSION}_metricset`, template: metricset }, - { name: `fake_hosts_${ECS_VERSION}_system`, template: system }, -]; - -export const indexTemplate: IndexTemplateDef = { - namespace: 'fake_hosts', - template: { ...template, composed_of: components.map(({ name }) => name) }, - components, -}; diff --git a/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/template.ts b/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/template.ts new file mode 100644 index 0000000000000..145002606a789 --- /dev/null +++ b/x-pack/packages/kbn-infra-forge/src/data_sources/fake_hosts/template.ts @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const template = { + order: 1, + index_patterns: ['kbn-data-forge*'], + settings: { + index: { + mapping: { + total_fields: { + limit: '10000', + }, + }, + number_of_shards: '1', + number_of_replicas: '0', + query: { + default_field: ['message', 'labels.*', 'event.*'], + }, + }, + }, + mappings: { + dynamic_templates: [ + { + labels: { + path_match: 'labels.*', + mapping: { + type: 'keyword', + }, + match_mapping_type: 'string', + }, + }, + { + strings_as_keyword: { + mapping: { + ignore_above: 1024, + type: 'keyword', + }, + match_mapping_type: 'string', + }, + }, + ], + date_detection: false, + properties: { + '@timestamp': { + type: 'date', + }, + tags: { + type: 'keyword', + }, + metricset: { + properties: { + period: { + type: 'long', + }, + }, + }, + host: { + properties: { + name: { + type: 'keyword', + ignore_above: 256, + }, + network: { + properties: { + name: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, + }, + }, + event: { + properties: { + dataset: { + type: 'keyword', + ignore_above: 256, + }, + module: { + type: 'keyword', + ignore_above: 256, + }, + }, + }, + system: { + properties: { + cpu: { + properties: { + cores: { + type: 'long', + }, + total: { + properties: { + norm: { + properties: { + pct: { + scaling_factor: 1000, + type: 'scaled_float', + }, + }, + }, + }, + }, + user: { + properties: { + pct: { + scaling_factor: 1000, + type: 'scaled_float', + }, + norm: { + properties: { + pct: { + scaling_factor: 1000, + type: 'scaled_float', + }, + }, + }, + }, + }, + system: { + properties: { + pct: { + scaling_factor: 1000, + type: 'scaled_float', + }, + }, + }, + }, + }, + network: { + properties: { + name: { + type: 'keyword', + ignore_above: 256, + }, + in: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + out: { + properties: { + bytes: { + type: 'long', + }, + }, + }, + }, + }, + }, + }, + container: { + properties: { + id: { + type: 'keyword', + ignore_above: 256, + }, + name: { + type: 'keyword', + ignore_above: 256, + }, + cpu: { + properties: { + cores: { + type: 'long', + }, + total: { + properties: { + norm: { + properties: { + pct: { + scaling_factor: 1000, + type: 'scaled_float', + }, + }, + }, + }, + }, + user: { + properties: { + pct: { + scaling_factor: 1000, + type: 'scaled_float', + }, + norm: { + properties: { + pct: { + scaling_factor: 1000, + type: 'scaled_float', + }, + }, + }, + }, + }, + system: { + properties: { + pct: { + scaling_factor: 1000, + type: 'scaled_float', + }, + }, + }, + }, + }, + }, + }, + }, + }, + aliases: { + 'metrics-fake_hosts': {}, + }, +}; diff --git a/x-pack/packages/kbn-infra-forge/src/lib/install_template.ts b/x-pack/packages/kbn-infra-forge/src/lib/install_template.ts new file mode 100644 index 0000000000000..02e9f1cb8442d --- /dev/null +++ b/x-pack/packages/kbn-infra-forge/src/lib/install_template.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Client } from '@elastic/elasticsearch'; +import type { ToolingLog } from '@kbn/tooling-log'; + +export function installTemplate( + client: Client, + template: object, + namespace: string, + logger: ToolingLog +) { + logger.debug(`installTemplate > template name: kbn-data-forge-${namespace}`); + return client.indices + .putTemplate({ name: `kbn-data-forge-${namespace}`, body: template }) + .catch((error: any) => logger.error(`installTemplate > ${JSON.stringify(error)}`)); +} + +export function deleteTemplate(client: Client, namespace: string, logger: ToolingLog) { + logger.debug(`deleteTemplate > template name: kbn-data-forge-${namespace}`); + return client.indices + .deleteTemplate({ name: `kbn-data-forge-${namespace}` }) + .catch((error: any) => logger.error(`deleteTemplate > ${JSON.stringify(error)}`)); +} diff --git a/x-pack/packages/kbn-infra-forge/src/lib/manage_template.ts b/x-pack/packages/kbn-infra-forge/src/lib/manage_template.ts deleted file mode 100644 index 722d1cb0b8ac4..0000000000000 --- a/x-pack/packages/kbn-infra-forge/src/lib/manage_template.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Client } from '@elastic/elasticsearch'; -import type { ToolingLog } from '@kbn/tooling-log'; -import { IndexTemplateDef } from '../data_sources/fake_hosts/index_template_def'; - -export async function installTemplate( - client: Client, - template: IndexTemplateDef, - namespace: string, - logger: ToolingLog -) { - logger.info(`Installing index templates (${namespace})`); - const componentNames = template.components.map(({ name }) => name); - logger.info(`Installing components for ${template.namespace} (${componentNames})`); - for (const component of template.components) { - await client.cluster - .putComponentTemplate({ - name: component.name, - ...component.template, - }) - .catch((error) => logger.error(`Failed installing component > ${JSON.stringify(error)}`)); - } - logger.info(`Installing index template (${template.namespace})`); - await client.indices - .putIndexTemplate({ - name: template.namespace, - ...template.template, - }) - .catch((error) => logger.error(`Failed installing template > ${JSON.stringify(error)}`)); -} - -export async function deleteTemplate( - client: Client, - template: IndexTemplateDef, - logger: ToolingLog -) { - logger.info(`deleteIndexTemplate > template name: ${template.namespace}`); - await client.indices - .deleteIndexTemplate({ - name: template.namespace, - }) - .catch((error: any) => - logger.error(`deleteIndexTemplate > ${template.namespace} ${JSON.stringify(error)}`) - ); - for (const component of template.components) { - logger.info(`deleteComponents > component name: ${component.name}`); - await client.cluster - .deleteComponentTemplate({ name: component.name }) - .catch((error: any) => - logger.error(`deleteComponents > ${component.name} ${JSON.stringify(error)}`) - ); - } -} diff --git a/x-pack/packages/kbn-infra-forge/src/lib/queue.ts b/x-pack/packages/kbn-infra-forge/src/lib/queue.ts index 75f3affb743d3..bf6553fe01b88 100644 --- a/x-pack/packages/kbn-infra-forge/src/lib/queue.ts +++ b/x-pack/packages/kbn-infra-forge/src/lib/queue.ts @@ -24,7 +24,7 @@ export const createQueue = ( logger.debug(`createQueue > index name: ${indexName}`); return async.cargoQueue( (docs: object[], callback) => { - const body: object[] = []; + const body: any[] = []; docs.forEach((doc) => { body.push({ create: { @@ -34,7 +34,7 @@ export const createQueue = ( body.push(omit(doc, 'namespace')); }); esClient - .bulk({ body, refresh: true }) + .bulk({ body }) .then((resp) => { if (resp.errors) { logger.error( diff --git a/x-pack/packages/kbn-infra-forge/src/run.ts b/x-pack/packages/kbn-infra-forge/src/run.ts index aad2adb24bc93..470c0afbdccc6 100644 --- a/x-pack/packages/kbn-infra-forge/src/run.ts +++ b/x-pack/packages/kbn-infra-forge/src/run.ts @@ -12,7 +12,7 @@ import type { Moment } from 'moment'; import type { ToolingLog } from '@kbn/tooling-log'; import type { Client } from '@elastic/elasticsearch'; import { createQueue, getIndexName } from './lib/queue'; -import { deleteTemplate, installTemplate } from './lib/manage_template'; +import { deleteTemplate, installTemplate } from './lib/install_template'; import * as fakeHosts from './data_sources/fake_hosts'; const generateEventsFns = { @@ -20,7 +20,7 @@ const generateEventsFns = { }; const templates = { - fake_hosts: fakeHosts.indexTemplate, + fake_hosts: fakeHosts.template, }; const EVENTS_PER_CYCLE = 1; @@ -68,6 +68,5 @@ export const generate = async ({ }; export const cleanup = async ({ esClient, logger }: { esClient: Client; logger: ToolingLog }) => { - const template = templates[DATASET]; - await deleteTemplate(esClient, template, logger); + await deleteTemplate(esClient, DATASET, logger); }; diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts index c2d91677af2a2..fd5a4054f0895 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts @@ -28,9 +28,6 @@ export default function ({ getService }: FtrProviderContext) { describe('Custom Threshold rule - AVG - PCT - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; - // DATE_VIEW should match the index template: - // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json - const DATE_VIEW = 'kbn-data-forge-fake_hosts'; const DATA_VIEW_ID = 'data-view-id'; let infraDataIndex: string; let actionId: string; @@ -40,9 +37,9 @@ export default function ({ getService }: FtrProviderContext) { infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger }); await createDataView({ supertest, - name: DATE_VIEW, + name: 'metrics-fake_hosts', id: DATA_VIEW_ID, - title: DATE_VIEW, + title: 'metrics-fake_hosts', }); }); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts index d17e0a1568603..7f7e66d050593 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts @@ -34,9 +34,6 @@ export default function ({ getService }: FtrProviderContext) { describe('Custom Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; - // DATE_VIEW should match the index template: - // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json - const DATE_VIEW = 'kbn-data-forge-fake_hosts'; const DATA_VIEW_ID = 'data-view-id'; let infraDataIndex: string; let actionId: string; @@ -46,9 +43,9 @@ export default function ({ getService }: FtrProviderContext) { infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger }); await createDataView({ supertest, - name: DATE_VIEW, + name: 'metrics-fake_hosts', id: DATA_VIEW_ID, - title: DATE_VIEW, + title: 'metrics-fake_hosts', }); }); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts index 4f5a1077c22d1..6a17340094e80 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts @@ -28,9 +28,6 @@ export default function ({ getService }: FtrProviderContext) { describe('Custom Threshold rule - DOCUMENTS_COUNT - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; - // DATE_VIEW should match the index template: - // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json - const DATE_VIEW = 'kbn-data-forge-fake_hosts'; const DATA_VIEW_ID = 'data-view-id'; let infraDataIndex: string; let actionId: string; @@ -40,9 +37,9 @@ export default function ({ getService }: FtrProviderContext) { infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger }); await createDataView({ supertest, - name: DATE_VIEW, + name: 'metrics-fake_hosts', id: DATA_VIEW_ID, - title: DATE_VIEW, + title: 'metrics-fake_hosts', }); }); diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts index 01386c9bd0250..da18b429c45c0 100644 --- a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts +++ b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts @@ -41,9 +41,6 @@ export default function ({ getService }: FtrProviderContext) { describe('Custom Threshold rule - GROUP_BY - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; - // DATE_VIEW should match the index template: - // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json - const DATE_VIEW = 'kbn-data-forge-fake_hosts'; const DATA_VIEW_ID = 'data-view-id'; let infraDataIndex: string; let actionId: string; @@ -53,9 +50,9 @@ export default function ({ getService }: FtrProviderContext) { infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger }); await createDataView({ supertest, - name: DATE_VIEW, + name: 'metrics-fake_hosts', id: DATA_VIEW_ID, - title: DATE_VIEW, + title: 'metrics-fake_hosts', }); }); diff --git a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts index 0de7e3d600612..2b727820ade70 100644 --- a/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts +++ b/x-pack/test/alerting_api_integration/observability/metric_threshold_rule.ts @@ -36,12 +36,6 @@ export default function ({ getService }: FtrProviderContext) { describe('alert and action creation', () => { before(async () => { - await supertest.patch(`/api/metrics/source/default`).set('kbn-xsrf', 'foo').send({ - anomalyThreshold: 50, - description: '', - metricAlias: 'kbn-data-forge*', - name: 'Default', - }); infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger }); actionId = await createIndexConnector({ supertest, diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_fired.ts index de26cb2e906d4..0389d0eace4e7 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_fired.ts @@ -23,11 +23,10 @@ export default function ({ getService }: FtrProviderContext) { const dataViewApi = getService('dataViewApi'); const logger = getService('log'); - describe('Custom Threshold rule - AVG - PCT - FIRED', () => { + // Blocked API: index_not_found_exception: no such index [.alerts-observability.threshold.alerts-default] + // Issue: https://github.com/elastic/kibana/issues/165138 + describe.skip('Custom Threshold rule - AVG - PCT - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; - // DATE_VIEW should match the index template: - // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json - const DATE_VIEW = 'kbn-data-forge-fake_hosts'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id'; let infraDataIndex: string; @@ -35,15 +34,11 @@ export default function ({ getService }: FtrProviderContext) { let ruleId: string; before(async () => { - infraDataIndex = await generate({ - esClient, - lookback: 'now-15m', - logger, - }); + infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger }); await dataViewApi.create({ - name: DATE_VIEW, + name: 'metrics-fake_hosts', id: DATA_VIEW_ID, - title: DATE_VIEW, + title: 'metrics-fake_hosts', }); }); @@ -62,7 +57,7 @@ export default function ({ getService }: FtrProviderContext) { }); await esClient.deleteByQuery({ index: '.kibana-event-log-*', - query: { term: { 'kibana.alert.rule.consumer': 'apm' } }, + query: { term: { 'kibana.alert.rule.consumer': 'logs' } }, }); await dataViewApi.delete({ id: DATA_VIEW_ID, @@ -80,7 +75,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await alertingApi.createRule({ tags: ['observability'], - consumer: 'apm', + consumer: 'logs', name: 'Threshold rule', ruleTypeId: OBSERVABILITY_THRESHOLD_RULE_TYPE_ID, params: { @@ -147,7 +142,7 @@ export default function ({ getService }: FtrProviderContext) { 'kibana.alert.rule.category', 'Custom threshold (BETA)' ); - expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'apm'); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'logs'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.name', 'Threshold rule'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.producer', 'observability'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.revision', 0); diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_no_data.ts b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_no_data.ts index b1cc5c41f6805..0bd7fdf7bbb6f 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_no_data.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_no_data.ts @@ -20,7 +20,9 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); - describe('Custom Threshold rule - AVG - PCT - NoData', () => { + // Blocked API: index_not_found_exception: no such index [.alerts-observability.threshold.alerts-default] + // Issue: https://github.com/elastic/kibana/issues/165138 + describe.skip('Custom Threshold rule - AVG - PCT - NoData', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id-no-data'; @@ -50,7 +52,7 @@ export default function ({ getService }: FtrProviderContext) { }); await esClient.deleteByQuery({ index: '.kibana-event-log-*', - query: { term: { 'kibana.alert.rule.consumer': 'apm' } }, + query: { term: { 'kibana.alert.rule.consumer': 'logs' } }, }); await dataViewApi.delete({ id: DATA_VIEW_ID, @@ -66,7 +68,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await alertingApi.createRule({ tags: ['observability'], - consumer: 'apm', + consumer: 'logs', name: 'Threshold rule', ruleTypeId: OBSERVABILITY_THRESHOLD_RULE_TYPE_ID, params: { @@ -133,7 +135,7 @@ export default function ({ getService }: FtrProviderContext) { 'kibana.alert.rule.category', 'Custom threshold (BETA)' ); - expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'apm'); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'logs'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.name', 'Threshold rule'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.producer', 'observability'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.revision', 0); diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts index b4b1858e9bac4..5ee54e1c9ad17 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts @@ -29,11 +29,9 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); - describe('Custom Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => { + // Issue: https://github.com/elastic/kibana/issues/165138 + describe.skip('Custom Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; - // DATE_VIEW should match the index template: - // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json - const DATE_VIEW = 'kbn-data-forge-fake_hosts'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id'; let infraDataIndex: string; @@ -43,9 +41,9 @@ export default function ({ getService }: FtrProviderContext) { before(async () => { infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger }); await dataViewApi.create({ - name: DATE_VIEW, + name: 'metrics-fake_hosts', id: DATA_VIEW_ID, - title: DATE_VIEW, + title: 'metrics-fake_hosts', }); }); @@ -82,7 +80,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await alertingApi.createRule({ tags: ['observability'], - consumer: 'apm', + consumer: 'logs', name: 'Threshold rule', ruleTypeId: OBSERVABILITY_THRESHOLD_RULE_TYPE_ID, params: { @@ -151,7 +149,7 @@ export default function ({ getService }: FtrProviderContext) { 'kibana.alert.rule.category', 'Custom threshold (BETA)' ); - expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'apm'); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'logs'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.name', 'Threshold rule'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.producer', 'observability'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.revision', 0); diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts index f5b0305993a91..56412a8380d2c 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts @@ -23,11 +23,9 @@ export default function ({ getService }: FtrProviderContext) { const alertingApi = getService('alertingApi'); const dataViewApi = getService('dataViewApi'); - describe('Custom Threshold rule - DOCUMENTS_COUNT - FIRED', () => { + // Issue: https://github.com/elastic/kibana/issues/165138 + describe.skip('Custom Threshold rule - DOCUMENTS_COUNT - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; - // DATE_VIEW should match the index template: - // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json - const DATE_VIEW = 'kbn-data-forge-fake_hosts'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id'; let infraDataIndex: string; @@ -37,9 +35,9 @@ export default function ({ getService }: FtrProviderContext) { before(async () => { infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger }); await dataViewApi.create({ - name: DATE_VIEW, + name: 'metrics-fake_hosts', id: DATA_VIEW_ID, - title: DATE_VIEW, + title: 'metrics-fake_hosts', }); }); @@ -58,7 +56,7 @@ export default function ({ getService }: FtrProviderContext) { }); await esClient.deleteByQuery({ index: '.kibana-event-log-*', - query: { term: { 'kibana.alert.rule.consumer': 'apm' } }, + query: { term: { 'kibana.alert.rule.consumer': 'logs' } }, }); await dataViewApi.delete({ id: DATA_VIEW_ID, @@ -76,7 +74,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await alertingApi.createRule({ tags: ['observability'], - consumer: 'apm', + consumer: 'logs', name: 'Threshold rule', ruleTypeId: OBSERVABILITY_THRESHOLD_RULE_TYPE_ID, params: { @@ -141,7 +139,7 @@ export default function ({ getService }: FtrProviderContext) { 'kibana.alert.rule.category', 'Custom threshold (BETA)' ); - expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'apm'); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'logs'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.name', 'Threshold rule'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.producer', 'observability'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.revision', 0); diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/group_by_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/group_by_fired.ts index 09ae231091369..f1b3c949421d4 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/group_by_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/group_by_fired.ts @@ -33,11 +33,9 @@ export default function ({ getService }: FtrProviderContext) { let alertId: string; let startedAt: string; - describe('Custom Threshold rule - GROUP_BY - FIRED', () => { + // Issue: https://github.com/elastic/kibana/issues/165138 + describe.skip('Custom Threshold rule - GROUP_BY - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; - // DATE_VIEW should match the index template: - // x-pack/packages/kbn-infra-forge/src/data_sources/composable/template.json - const DATE_VIEW = 'kbn-data-forge-fake_hosts'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; const DATA_VIEW_ID = 'data-view-id'; let infraDataIndex: string; @@ -47,9 +45,9 @@ export default function ({ getService }: FtrProviderContext) { before(async () => { infraDataIndex = await generate({ esClient, lookback: 'now-15m', logger }); await dataViewApi.create({ - name: DATE_VIEW, + name: 'metrics-fake_hosts', id: DATA_VIEW_ID, - title: DATE_VIEW, + title: 'metrics-fake_hosts', }); }); @@ -68,7 +66,7 @@ export default function ({ getService }: FtrProviderContext) { }); await esClient.deleteByQuery({ index: '.kibana-event-log-*', - query: { term: { 'kibana.alert.rule.consumer': 'apm' } }, + query: { term: { 'kibana.alert.rule.consumer': 'logs' } }, }); await dataViewApi.delete({ id: DATA_VIEW_ID, @@ -86,7 +84,7 @@ export default function ({ getService }: FtrProviderContext) { const createdRule = await alertingApi.createRule({ tags: ['observability'], - consumer: 'apm', + consumer: 'logs', name: 'Threshold rule', ruleTypeId: OBSERVABILITY_THRESHOLD_RULE_TYPE_ID, params: { @@ -160,7 +158,7 @@ export default function ({ getService }: FtrProviderContext) { 'kibana.alert.rule.category', 'Custom threshold (BETA)' ); - expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'apm'); + expect(resp.hits.hits[0]._source).property('kibana.alert.rule.consumer', 'logs'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.name', 'Threshold rule'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.producer', 'observability'); expect(resp.hits.hits[0]._source).property('kibana.alert.rule.revision', 0); @@ -226,8 +224,7 @@ export default function ({ getService }: FtrProviderContext) { expect(resp.hits.hits[0]._source?.ruleType).eql('observability.rules.custom_threshold'); expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql( - // Added the S to protocol.getUrlParts as not returning the correct value. - `${protocol}s://${hostname}:${port}/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)` + `${protocol}://${hostname}:${port}/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)` ); expect(resp.hits.hits[0]._source?.reason).eql( 'Custom equation is 0.8 in the last 1 min for host-0. Alert when >= 0.2.' From 6431d7c61ec1e1f81969e1750f03f233a076ff52 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 29 Sep 2023 08:29:23 -0400 Subject: [PATCH 4/9] skip failing test suite (#167619) --- .../observability/custom_threshold_rule/documents_count_fired.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts index 56412a8380d2c..7c67217ca96ee 100644 --- a/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts +++ b/x-pack/test_serverless/api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts @@ -24,6 +24,7 @@ export default function ({ getService }: FtrProviderContext) { const dataViewApi = getService('dataViewApi'); // Issue: https://github.com/elastic/kibana/issues/165138 + // Failing: See https://github.com/elastic/kibana/issues/167619 describe.skip('Custom Threshold rule - DOCUMENTS_COUNT - FIRED', () => { const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default'; const ALERT_ACTION_INDEX = 'alert-action-threshold'; From d7b7659322405e72c4d2dde86c4860fc057c82a6 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 29 Sep 2023 08:49:05 -0400 Subject: [PATCH 5/9] skip failing test suite (#167560) --- x-pack/test/saved_object_tagging/functional/tests/edit.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/saved_object_tagging/functional/tests/edit.ts b/x-pack/test/saved_object_tagging/functional/tests/edit.ts index 96a91c72c3f3e..f29212b251827 100644 --- a/x-pack/test/saved_object_tagging/functional/tests/edit.ts +++ b/x-pack/test/saved_object_tagging/functional/tests/edit.ts @@ -15,7 +15,8 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const tagManagementPage = PageObjects.tagManagement; - describe('edit tag', () => { + // Failing: See https://github.com/elastic/kibana/issues/167560 + describe.skip('edit tag', () => { let tagModal: typeof tagManagementPage['tagModal']; before(async () => { From 3545d3023e77238958fb8f3ee9902c6bb8be0ca9 Mon Sep 17 00:00:00 2001 From: Xavier Mouligneau Date: Fri, 29 Sep 2023 09:02:01 -0400 Subject: [PATCH 6/9] [RAM] Allow conditional action for rule type with AAD (#167556) ## Summary With this change, we will let rule type to use conditional action if the rule type create alert in the alert index. We will need to some clean up on these two fields `HasAlertsMappings` and `hasFieldsForAAD` in future release. (https://github.com/elastic/kibana/issues/167558) --- .../public/connector_types/slack/action_form.test.tsx | 1 + .../sections/action_connector_form/action_type_form.tsx | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/stack_connectors/public/connector_types/slack/action_form.test.tsx b/x-pack/plugins/stack_connectors/public/connector_types/slack/action_form.test.tsx index e4e1cb533aa93..f06461dd5fbc9 100644 --- a/x-pack/plugins/stack_connectors/public/connector_types/slack/action_form.test.tsx +++ b/x-pack/plugins/stack_connectors/public/connector_types/slack/action_form.test.tsx @@ -164,6 +164,7 @@ describe('ActionForm - Slack API Connector', () => { const testProps = { ...baseProps, + hasAlertsMappings: false, actions: testActions, }; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.tsx index 2cd0c43f5aa4f..3f0982b1b24f3 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/action_connector_form/action_type_form.tsx @@ -358,7 +358,8 @@ export const ActionTypeForm = ({ setActionGroupIdByIndex && !actionItem.frequency?.summary; - const showActionAlertsFilter = hasFieldsForAAD || producerId === AlertConsumers.SIEM; + const showActionAlertsFilter = + hasFieldsForAAD || producerId === AlertConsumers.SIEM || hasAlertsMappings; const accordionContent = checkEnabledResult.isEnabled ? ( <> From d9b026d7e1a87e4b4575cef795be16761318d1f3 Mon Sep 17 00:00:00 2001 From: Chris Cressman Date: Fri, 29 Sep 2023 09:08:05 -0400 Subject: [PATCH 7/9] [Enterprise Search] Update URLs for docs migrating from Enterprise Search (#167565) Several docs are going to migrate from Enterprise Search to Elasticsearch. Replace Enterprise Search URLs with Elasticsearch URLs. --- packages/kbn-doc-links/src/get_doc_links.ts | 26 ++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/kbn-doc-links/src/get_doc_links.ts b/packages/kbn-doc-links/src/get_doc_links.ts index d9cbd6aed1315..ea463a75d474b 100644 --- a/packages/kbn-doc-links/src/get_doc_links.ts +++ b/packages/kbn-doc-links/src/get_doc_links.ts @@ -133,9 +133,9 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => { aiSearchDoc: `${ESRE_DOCS}`, aiSearchHelp: `${ESRE_DOCS}help.html`, apiKeys: `${KIBANA_DOCS}api-keys.html`, - behavioralAnalytics: `${ENTERPRISE_SEARCH_DOCS}analytics-overview.html`, - behavioralAnalyticsCORS: `${ENTERPRISE_SEARCH_DOCS}analytics-cors-proxy.html`, - behavioralAnalyticsEvents: `${ENTERPRISE_SEARCH_DOCS}analytics-events.html`, + behavioralAnalytics: `${ELASTICSEARCH_DOCS}behavioral-analytics-overview.html`, + behavioralAnalyticsCORS: `${ELASTICSEARCH_DOCS}behavioral-analytics-cors.html`, + behavioralAnalyticsEvents: `${ELASTICSEARCH_DOCS}behavioral-analytics-event.html`, buildConnector: `${ENTERPRISE_SEARCH_DOCS}build-connector.html`, bulkApi: `${ELASTICSEARCH_DOCS}docs-bulk.html`, configuration: `${ENTERPRISE_SEARCH_DOCS}configuration.html`, @@ -170,24 +170,24 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => { crawlerOverview: `${ENTERPRISE_SEARCH_DOCS}crawler.html`, deployTrainedModels: `${MACHINE_LEARNING_DOCS}ml-nlp-deploy-models.html`, documentLevelSecurity: `${ELASTICSEARCH_DOCS}document-level-security.html`, - elser: `${ENTERPRISE_SEARCH_DOCS}elser-text-expansion.html`, + elser: `${ELASTICSEARCH_DOCS}semantic-search-elser.html`, engines: `${ENTERPRISE_SEARCH_DOCS}engines.html`, indexApi: `${ELASTICSEARCH_DOCS}docs-index_.html`, - ingestionApis: `${ENTERPRISE_SEARCH_DOCS}ingestion-apis.html`, - ingestPipelines: `${ENTERPRISE_SEARCH_DOCS}ingest-pipelines.html`, + ingestionApis: `${ELASTICSEARCH_DOCS}search-your-data.html`, + ingestPipelines: `${ELASTICSEARCH_DOCS}ingest-pipeline-search.html`, knnSearch: `${ELASTICSEARCH_DOCS}knn-search.html`, knnSearchCombine: `${ELASTICSEARCH_DOCS}knn-search.html#_combine_approximate_knn_with_other_features`, languageAnalyzers: `${ELASTICSEARCH_DOCS}analysis-lang-analyzer.html`, languageClients: `${ENTERPRISE_SEARCH_DOCS}programming-language-clients.html`, licenseManagement: `${ENTERPRISE_SEARCH_DOCS}license-management.html`, - machineLearningStart: `${ENTERPRISE_SEARCH_DOCS}machine-learning-start.html`, + machineLearningStart: `${ELASTICSEARCH_DOCS}nlp-example.html`, mailService: `${ENTERPRISE_SEARCH_DOCS}mailer-configuration.html`, - mlDocumentEnrichment: `${ENTERPRISE_SEARCH_DOCS}document-enrichment.html`, - mlDocumentEnrichmentUpdateMappings: `${ENTERPRISE_SEARCH_DOCS}document-enrichment.html#document-enrichment-update-mappings`, - searchApplicationsTemplates: `${ENTERPRISE_SEARCH_DOCS}search-applications-templates.html`, - searchApplicationsSearchApi: `${ENTERPRISE_SEARCH_DOCS}search-applications-safe-search.html`, - searchApplications: `${ENTERPRISE_SEARCH_DOCS}search-applications.html`, - searchApplicationsSearch: `${ENTERPRISE_SEARCH_DOCS}search-applications-search.html`, + mlDocumentEnrichment: `${ELASTICSEARCH_DOCS}ingest-pipeline-search-inference.html`, + mlDocumentEnrichmentUpdateMappings: `${ELASTICSEARCH_DOCS}ingest-pipeline-search-inference.html#ingest-pipeline-search-inference-update-mapping`, + searchApplicationsTemplates: `${ELASTICSEARCH_DOCS}search-application-api.html`, + searchApplicationsSearchApi: `${ELASTICSEARCH_DOCS}search-application-security.html`, + searchApplications: `${ELASTICSEARCH_DOCS}search-application-overview.html`, + searchApplicationsSearch: `${ELASTICSEARCH_DOCS}search-application-client.html`, searchLabs: `${SEARCH_LABS_URL}`, searchLabsRepo: `${SEARCH_LABS_REPO}`, searchTemplates: `${ELASTICSEARCH_DOCS}search-template.html`, From 4c4b2d449715338c89e5df312de5beebb9cd8656 Mon Sep 17 00:00:00 2001 From: Gabriel Landau <42078554+gabriellandau@users.noreply.github.com> Date: Fri, 29 Sep 2023 09:31:12 -0400 Subject: [PATCH 8/9] Endpoint Advanced Policy Option: `advanced.events.check_debug_registers` (#167308) ## Summary New advanced Endpoint/Defend option. See its description for details. ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../pages/policy/models/advanced_policy_schema.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/models/advanced_policy_schema.ts b/x-pack/plugins/security_solution/public/management/pages/policy/models/advanced_policy_schema.ts index 056577f7944ef..02d856fa4ebdd 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/models/advanced_policy_schema.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/models/advanced_policy_schema.ts @@ -1351,4 +1351,15 @@ export const AdvancedPolicySchema: AdvancedPolicySchemaType[] = [ } ), }, + { + key: 'windows.advanced.events.check_debug_registers', + first_supported_version: '8.11', + documentation: i18n.translate( + 'xpack.securitySolution.endpoint.policy.advanced.windows.advanced.events.check_debug_registers', + { + defaultMessage: + 'Check debug registers inline to detect the use of hardware breakpoints. Malware may use hardware breakpoints to forge benign-looking call stacks. Default: true', + } + ), + }, ]; From 7393bfefbae0666234e896a29780ddf9fed8ccbf Mon Sep 17 00:00:00 2001 From: Stratoula Kalafateli Date: Fri, 29 Sep 2023 16:35:52 +0300 Subject: [PATCH 9/9] [Lens] Fixes mosaic with 2 axis coloring (#167035) ## Summary Fixes https://github.com/elastic/kibana/issues/164964 The Other label was not formatted correctly. image ### Checklist - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../public/utils/layers/get_color.test.ts | 62 ++++++++++--------- .../public/utils/layers/get_color.ts | 16 ++++- .../public/utils/layers/get_layers.ts | 6 +- 3 files changed, 50 insertions(+), 34 deletions(-) diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.test.ts b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.test.ts index c6fffce60ba0b..f8728ebf7a51c 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.test.ts +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.test.ts @@ -8,7 +8,6 @@ import type { PaletteOutput, PaletteDefinition } from '@kbn/coloring'; import { chartPluginMock } from '@kbn/charts-plugin/public/mocks'; -import { Datatable } from '@kbn/expressions-plugin/common'; import { byDataColorPaletteMap, SimplifiedArrayNode } from './get_color'; import type { SeriesLayer } from '@kbn/coloring'; import { dataPluginMock } from '@kbn/data-plugin/public/mocks'; @@ -21,29 +20,28 @@ import { ChartTypes } from '../../../common/types'; import { getDistinctSeries } from '..'; describe('#byDataColorPaletteMap', () => { - let datatable: Datatable; let paletteDefinition: PaletteDefinition; let palette: PaletteOutput; - const columnId = 'foo'; + const visData = createMockVisData(); + const defaultFormatter = jest.fn((...args) => fieldFormatsMock.deserialize(...args)); + const formatters = generateFormatters(visData, defaultFormatter); beforeEach(() => { - datatable = { - rows: [ - { - [columnId]: '1', - }, - { - [columnId]: '2', - }, - ], - } as unknown as Datatable; paletteDefinition = chartPluginMock.createPaletteRegistry().get('default'); palette = { type: 'palette' } as PaletteOutput; }); it('should create byDataColorPaletteMap', () => { - expect(byDataColorPaletteMap(datatable.rows, columnId, paletteDefinition, palette)) - .toMatchInlineSnapshot(` + expect( + byDataColorPaletteMap( + visData.rows, + visData.columns[0], + paletteDefinition, + palette, + formatters, + fieldFormatsMock + ) + ).toMatchInlineSnapshot(` Object { "getColor": [Function], } @@ -52,21 +50,25 @@ describe('#byDataColorPaletteMap', () => { it('should get color', () => { const colorPaletteMap = byDataColorPaletteMap( - datatable.rows, - columnId, + visData.rows, + visData.columns[0], paletteDefinition, - palette + palette, + formatters, + fieldFormatsMock ); - expect(colorPaletteMap.getColor('1')).toBe('black'); + expect(colorPaletteMap.getColor('Logstash Airways')).toBe('black'); }); it('should return undefined in case if values not in datatable', () => { const colorPaletteMap = byDataColorPaletteMap( - datatable.rows, - columnId, + visData.rows, + visData.columns[0], paletteDefinition, - palette + palette, + formatters, + fieldFormatsMock ); expect(colorPaletteMap.getColor('wrong')).toBeUndefined(); @@ -74,24 +76,26 @@ describe('#byDataColorPaletteMap', () => { it('should increase rankAtDepth for each new value', () => { const colorPaletteMap = byDataColorPaletteMap( - datatable.rows, - columnId, + visData.rows, + visData.columns[0], paletteDefinition, - palette + palette, + formatters, + fieldFormatsMock ); - colorPaletteMap.getColor('1'); - colorPaletteMap.getColor('2'); + colorPaletteMap.getColor('Logstash Airways'); + colorPaletteMap.getColor('JetBeats'); expect(paletteDefinition.getCategoricalColor).toHaveBeenNthCalledWith( 1, - [{ name: '1', rankAtDepth: 0, totalSeriesAtDepth: 2 }], + [{ name: 'Logstash Airways', rankAtDepth: 0, totalSeriesAtDepth: 4 }], { behindText: false }, undefined ); expect(paletteDefinition.getCategoricalColor).toHaveBeenNthCalledWith( 2, - [{ name: '2', rankAtDepth: 1, totalSeriesAtDepth: 2 }], + [{ name: 'JetBeats', rankAtDepth: 1, totalSeriesAtDepth: 4 }], { behindText: false }, undefined ); diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.ts b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.ts index 2f93b9bc8d373..bb9538a02fb27 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.ts +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_color.ts @@ -21,12 +21,22 @@ const isTreemapOrMosaicChart = (shape: ChartTypes) => export const byDataColorPaletteMap = ( rows: Datatable['rows'], - columnId: string, + column: Partial, paletteDefinition: PaletteDefinition, - { params }: PaletteOutput + { params }: PaletteOutput, + formatters: Record, + formatter: FieldFormatsStart ) => { const colorMap = new Map( - rows.map((item) => [String(item[columnId]), undefined]) + rows.map((item) => { + const formattedName = getNodeLabel( + item[column.id ?? ''], + column, + formatters, + formatter.deserialize + ); + return [formattedName, undefined]; + }) ); let rankAtDepth = 0; diff --git a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_layers.ts b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_layers.ts index 6f40097809e18..73d6e29a5b1ed 100644 --- a/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_layers.ts +++ b/src/plugins/chart_expressions/expression_partition_vis/public/utils/layers/get_layers.ts @@ -61,9 +61,11 @@ export const getLayers = ( if (!syncColors && columns[1]?.id && palettes && visParams.palette) { byDataPalette = byDataColorPaletteMap( rows, - columns[1].id, + columns[1], palettes?.get(visParams.palette.name), - visParams.palette + visParams.palette, + formatters, + formatter ); }