Merge branch '7.x' into backport/7.x/pr-70828

docs/apm/api.asciidoc

@@ -355,6 +355,7 @@ allowing you to easily see how these events are impacting the performance of you
 
 By default, annotations are stored in a newly created `observability-annotations` index.
 The name of this index can be changed in your `config.yml` by editing `xpack.observability.annotations.index`.
+If you change the default index name, you'll also need to <<apm-app-annotation-user-create,update your user privileges>> accordingly.
 
 The following APIs are available:
 

docs/apm/apm-app-users.asciidoc

@@ -4,7 +4,7 @@
 
 :beat_default_index_prefix: apm
 :beat_kib_app: APM app
-:annotation_index: `observability-annotations`
+:annotation_index: observability-annotations
 
 ++++
 <titleabbrev>Users and privileges</titleabbrev>
@@ -102,6 +102,54 @@ Here are two examples:
 *********************************** ***********************************
 ////
 
+[role="xpack"]
+[[apm-app-annotation-user-create]]
+=== APM app annotation user
+
+++++
+<titleabbrev>Create an annotation user</titleabbrev>
+++++
+
+NOTE: By default, the `apm_user` built-in role provides access to Observability annotations.
+You only need to create an annotation user if the default annotation index
+defined in <<apm-settings-kb,`xpack.observability.annotations.index`>> has been customized.
+
+[[apm-app-annotation-user]]
+==== Annotation user
+
+View deployment annotations in the APM app.
+
+. Create a new role, named something like `annotation_user`,
+and assign the following privileges:
++
+[options="header"]
+|====
+|Type | Privilege | Purpose
+
+|Index
+|`read` on +\{ANNOTATION_INDEX\}+^1^
+|Read-only access to the observability annotation index
+
+|Index
+|`view_index_metadata` on +\{ANNOTATION_INDEX\}+^1^
+|Read-only access to observability annotation index metadata
+|====
++
+^1^ +\{ANNOTATION_INDEX\}+ should be the index name you've defined in
+<<apm-settings-kb,`xpack.observability.annotations.index`>>.
+
+. Assign the `annotation_user` created previously, and the built-in roles necessary to create
+a <<apm-app-reader-full,full>> or <<apm-app-reader-partial,partial>> APM reader to any users that need to view annotations in the APM app
+
+[[apm-app-annotation-api]]
+==== Annotation API
+
+See <<apm-app-api-user>>.
+
+////
+*********************************** ***********************************
+////
+
 [role="xpack"]
 [[apm-app-central-config-user]]
 === APM app central config user

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { MetricExpressionParams } from '../types';
+import { getElasticsearchMetricQuery } from './metric_query';
+
+describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
+ const expressionParams = {
+ metric: 'system.is.a.good.puppy.dog',
+ aggType: 'avg',
+ timeUnit: 'm',
+ timeSize: 1,
+ } as MetricExpressionParams;
+
+ const timefield = '@timestamp';
+ const groupBy = 'host.doggoname';
+
+ describe('when passed no filterQuery', () => {
+ const searchBody = getElasticsearchMetricQuery(expressionParams, timefield, groupBy);
+ test('includes a range filter', () => {
+ expect(
+ searchBody.query.bool.filter.find((filter) => filter.hasOwnProperty('range'))
+ ).toBeTruthy();
+ });
+
+ test('includes a metric field filter', () => {
+ expect(searchBody.query.bool.filter).toMatchObject(
+ expect.arrayContaining([{ exists: { field: 'system.is.a.good.puppy.dog' } }])
+ );
+ });
+ });
+
+ describe('when passed a filterQuery', () => {
+ const filterQuery =
+ // This is adapted from a real-world query that previously broke alerts
+ // We want to make sure it doesn't override any existing filters
+ '{"bool":{"filter":[{"bool":{"filter":[{"bool":{"must_not":[{"bool":{"should":[{"query_string":{"query":"bark*","fields":["host.name^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1}}],"adjust_pure_negative":true,"minimum_should_match":"1","boost":1}}],"adjust_pure_negative":true,"boost":1}},{"bool":{"must_not":[{"bool":{"should":[{"query_string":{"query":"woof*","fields":["host.name^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1}}],"adjust_pure_negative":true,"minimum_should_match":"1","boost":1}}],"adjust_pure_negative":true,"boost":1}}],"adjust_pure_negative":true,"boost":1}}],"adjust_pure_negative":true,"boost":1}}';
+
+ const searchBody = getElasticsearchMetricQuery(
+ expressionParams,
+ timefield,
+ groupBy,
+ filterQuery
+ );
+ test('includes a range filter', () => {
+ expect(
+ searchBody.query.bool.filter.find((filter) => filter.hasOwnProperty('range'))
+ ).toBeTruthy();
+ });
+
+ test('includes a metric field filter', () => {
+ expect(searchBody.query.bool.filter).toMatchObject(
+ expect.arrayContaining([{ exists: { field: 'system.is.a.good.puppy.dog' } }])
+ );
+ });
+ });
+});

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts

@@ -11,11 +11,11 @@ import { createPercentileAggregation } from './create_percentile_aggregation';
 
 const MINIMUM_BUCKETS = 5;
 
-const getParsedFilterQuery: (
- filterQuery: string | undefined
-) => Record<string, any> | Array<Record<string, any>> = (filterQuery) => {
- if (!filterQuery) return {};
- return JSON.parse(filterQuery).bool;
+const getParsedFilterQuery: (filterQuery: string | undefined) => Record<string, any> | null = (
+ filterQuery
+) => {
+ if (!filterQuery) return null;
+ return JSON.parse(filterQuery);
 };
 
 export const getElasticsearchMetricQuery = (
@@ -129,9 +129,8 @@ export const getElasticsearchMetricQuery = (
  filter: [
  ...rangeFilters,
  ...metricFieldFilters,
- ...(Array.isArray(parsedFilterQuery) ? parsedFilterQuery : []),
+ ...(parsedFilterQuery ? [parsedFilterQuery] : []),
  ],
- ...(!Array.isArray(parsedFilterQuery) ? parsedFilterQuery : {}),
  },
  },
  size: 0,

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -175,6 +175,12 @@ export interface Dataset {
  package: string;
  path: string;
  ingest_pipeline: string;
+ elasticsearch?: RegistryElasticsearch;
+}
+
+export interface RegistryElasticsearch {
+ 'index_template.settings'?: object;
+ 'index_template.mappings'?: object;
 }
 
 // EPR types this as `[]map[string]interface{}`
@@ -272,6 +278,7 @@ export interface IndexTemplate {
  data_stream: {
  timestamp_field: string;
  };
+ composed_of: string[];
  _meta: object;
 }
 

x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/page_paths.ts

@@ -53,7 +53,7 @@ export const PAGE_ROUTING_PATHS = {
  fleet_agent_details_events: '/fleet/agents/:agentId',
  fleet_agent_details_details: '/fleet/agents/:agentId/details',
  fleet_enrollment_tokens: '/fleet/enrollment-tokens',
- data_streams: '/data-streams',
+ data_streams: '/datasets',
 };
 
 export const pagePathGetters: {
@@ -80,5 +80,5 @@ export const pagePathGetters: {
  fleet_agent_details: ({ agentId, tabId }) =>
  `/fleet/agents/${agentId}${tabId ? `/${tabId}` : ''}`,
  fleet_enrollment_tokens: () => '/fleet/enrollment-tokens',
- data_streams: () => '/data-streams',
+ data_streams: () => '/datasets',
 };

x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_breadcrumbs.tsx

@@ -207,7 +207,7 @@ const breadcrumbGetters: {
  BASE_BREADCRUMB,
  {
  text: i18n.translate('xpack.ingestManager.breadcrumbs.datastreamsPageTitle', {
- defaultMessage: 'Data streams',
+ defaultMessage: 'Datasets',
  }),
  },
  ],

x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx

@@ -103,7 +103,7 @@ export const DefaultLayout: React.FunctionComponent<Props> = ({
  <EuiTab isSelected={section === 'data_stream'} href={getHref('data_streams')}>
  <FormattedMessage
  id="xpack.ingestManager.appNavigation.dataStreamsLinkText"
- defaultMessage="Data streams"
+ defaultMessage="Datasets"
  />
  </EuiTab>
  </EuiTabs>

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx

@@ -32,7 +32,7 @@ const DataStreamListPageLayout: React.FunctionComponent = ({ children }) => (
  <h1>
  <FormattedMessage
  id="xpack.ingestManager.dataStreamList.pageTitle"
- defaultMessage="Data streams"
+ defaultMessage="Datasets"
  />
  </h1>
  </EuiText>
@@ -177,7 +177,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
  <h2>
  <FormattedMessage
  id="xpack.ingestManager.dataStreamList.noDataStreamsPrompt"
- defaultMessage="No data streams"
+ defaultMessage="No datasets"
  />
  </h2>
  }
@@ -220,14 +220,14 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
  isLoading ? (
  <FormattedMessage
  id="xpack.ingestManager.dataStreamList.loadingDataStreamsMessage"
- defaultMessage="Loading data streams…"
+ defaultMessage="Loading datasets…"
  />
  ) : dataStreamsData && !dataStreamsData.data_streams.length ? (
  emptyPrompt
  ) : (
  <FormattedMessage
  id="xpack.ingestManager.dataStreamList.noFilteredDataStreamsMessage"
- defaultMessage="No matching data streams found"
+ defaultMessage="No matching datasets found"
  />
  )
  }
@@ -257,7 +257,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {
  placeholder: i18n.translate(
  'xpack.ingestManager.dataStreamList.searchPlaceholderTitle',
  {
- defaultMessage: 'Filter data streams',
+ defaultMessage: 'Filter datasets',
  }
  ),
  incremental: true,

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/overview/components/datastream_section.tsx

@@ -51,14 +51,14 @@ export const OverviewDatastreamSection: React.FC = () => {
  <h2>
  <FormattedMessage
  id="xpack.ingestManager.overviewPageDataStreamsPanelTitle"
- defaultMessage="Data streams"
+ defaultMessage="Datasets"
  />
  </h2>
  </EuiTitle>
  <EuiButtonEmpty size="xs" flush="right" href={getHref('data_streams')}>
  <FormattedMessage
  id="xpack.ingestManager.overviewPageDataStreamsPanelAction"
- defaultMessage="View data streams"
+ defaultMessage="View datasets"
  />
  </EuiButtonEmpty>
  </header>
@@ -70,7 +70,7 @@ export const OverviewDatastreamSection: React.FC = () => {
  <EuiDescriptionListTitle>
  <FormattedMessage
  id="xpack.ingestManager.overviewDatastreamTotalTitle"
- defaultMessage="Data streams"
+ defaultMessage="Datasets"
  />
  </EuiDescriptionListTitle>
  <EuiDescriptionListDescription>