diff --git a/docs/apm/api.asciidoc b/docs/apm/api.asciidoc index 54159b642dd1a8..2fbeea0534fc05 100644 --- a/docs/apm/api.asciidoc +++ b/docs/apm/api.asciidoc @@ -355,6 +355,7 @@ allowing you to easily see how these events are impacting the performance of you By default, annotations are stored in a newly created `observability-annotations` index. The name of this index can be changed in your `config.yml` by editing `xpack.observability.annotations.index`. +If you change the default index name, you'll also need to <> accordingly. The following APIs are available: diff --git a/docs/apm/apm-app-users.asciidoc b/docs/apm/apm-app-users.asciidoc index 442a07d2797252..d766c866f87e43 100644 --- a/docs/apm/apm-app-users.asciidoc +++ b/docs/apm/apm-app-users.asciidoc @@ -4,7 +4,7 @@ :beat_default_index_prefix: apm :beat_kib_app: APM app -:annotation_index: `observability-annotations` +:annotation_index: observability-annotations ++++ Users and privileges @@ -102,6 +102,54 @@ Here are two examples: *********************************** *********************************** //// +[role="xpack"] +[[apm-app-annotation-user-create]] +=== APM app annotation user + +++++ +Create an annotation user +++++ + +NOTE: By default, the `apm_user` built-in role provides access to Observability annotations. +You only need to create an annotation user if the default annotation index +defined in <> has been customized. + +[[apm-app-annotation-user]] +==== Annotation user + +View deployment annotations in the APM app. + +. Create a new role, named something like `annotation_user`, +and assign the following privileges: ++ +[options="header"] +|==== +|Type | Privilege | Purpose + +|Index +|`read` on +\{ANNOTATION_INDEX\}+^1^ +|Read-only access to the observability annotation index + +|Index +|`view_index_metadata` on +\{ANNOTATION_INDEX\}+^1^ +|Read-only access to observability annotation index metadata +|==== ++ +^1^ +\{ANNOTATION_INDEX\}+ should be the index name you've defined in +<>. + +. Assign the `annotation_user` created previously, and the built-in roles necessary to create +a <> or <> APM reader to any users that need to view annotations in the APM app + +[[apm-app-annotation-api]] +==== Annotation API + +See <>. + +//// +*********************************** *********************************** +//// + [role="xpack"] [[apm-app-central-config-user]] === APM app central config user diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts new file mode 100644 index 00000000000000..3ad1031f574e25 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { MetricExpressionParams } from '../types'; +import { getElasticsearchMetricQuery } from './metric_query'; + +describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => { + const expressionParams = { + metric: 'system.is.a.good.puppy.dog', + aggType: 'avg', + timeUnit: 'm', + timeSize: 1, + } as MetricExpressionParams; + + const timefield = '@timestamp'; + const groupBy = 'host.doggoname'; + + describe('when passed no filterQuery', () => { + const searchBody = getElasticsearchMetricQuery(expressionParams, timefield, groupBy); + test('includes a range filter', () => { + expect( + searchBody.query.bool.filter.find((filter) => filter.hasOwnProperty('range')) + ).toBeTruthy(); + }); + + test('includes a metric field filter', () => { + expect(searchBody.query.bool.filter).toMatchObject( + expect.arrayContaining([{ exists: { field: 'system.is.a.good.puppy.dog' } }]) + ); + }); + }); + + describe('when passed a filterQuery', () => { + const filterQuery = + // This is adapted from a real-world query that previously broke alerts + // We want to make sure it doesn't override any existing filters + '{"bool":{"filter":[{"bool":{"filter":[{"bool":{"must_not":[{"bool":{"should":[{"query_string":{"query":"bark*","fields":["host.name^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1}}],"adjust_pure_negative":true,"minimum_should_match":"1","boost":1}}],"adjust_pure_negative":true,"boost":1}},{"bool":{"must_not":[{"bool":{"should":[{"query_string":{"query":"woof*","fields":["host.name^1.0"],"type":"best_fields","default_operator":"or","max_determinized_states":10000,"enable_position_increments":true,"fuzziness":"AUTO","fuzzy_prefix_length":0,"fuzzy_max_expansions":50,"phrase_slop":0,"escape":false,"auto_generate_synonyms_phrase_query":true,"fuzzy_transpositions":true,"boost":1}}],"adjust_pure_negative":true,"minimum_should_match":"1","boost":1}}],"adjust_pure_negative":true,"boost":1}}],"adjust_pure_negative":true,"boost":1}}],"adjust_pure_negative":true,"boost":1}}'; + + const searchBody = getElasticsearchMetricQuery( + expressionParams, + timefield, + groupBy, + filterQuery + ); + test('includes a range filter', () => { + expect( + searchBody.query.bool.filter.find((filter) => filter.hasOwnProperty('range')) + ).toBeTruthy(); + }); + + test('includes a metric field filter', () => { + expect(searchBody.query.bool.filter).toMatchObject( + expect.arrayContaining([{ exists: { field: 'system.is.a.good.puppy.dog' } }]) + ); + }); + }); +}); diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts index 5680035d9d609b..15506a30529c4e 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts @@ -11,11 +11,11 @@ import { createPercentileAggregation } from './create_percentile_aggregation'; const MINIMUM_BUCKETS = 5; -const getParsedFilterQuery: ( - filterQuery: string | undefined -) => Record | Array> = (filterQuery) => { - if (!filterQuery) return {}; - return JSON.parse(filterQuery).bool; +const getParsedFilterQuery: (filterQuery: string | undefined) => Record | null = ( + filterQuery +) => { + if (!filterQuery) return null; + return JSON.parse(filterQuery); }; export const getElasticsearchMetricQuery = ( @@ -129,9 +129,8 @@ export const getElasticsearchMetricQuery = ( filter: [ ...rangeFilters, ...metricFieldFilters, - ...(Array.isArray(parsedFilterQuery) ? parsedFilterQuery : []), + ...(parsedFilterQuery ? [parsedFilterQuery] : []), ], - ...(!Array.isArray(parsedFilterQuery) ? parsedFilterQuery : {}), }, }, size: 0, diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 3ee3039e9e1c49..0d2825f0aa80dd 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -175,6 +175,12 @@ export interface Dataset { package: string; path: string; ingest_pipeline: string; + elasticsearch?: RegistryElasticsearch; +} + +export interface RegistryElasticsearch { + 'index_template.settings'?: object; + 'index_template.mappings'?: object; } // EPR types this as `[]map[string]interface{}` @@ -272,6 +278,7 @@ export interface IndexTemplate { data_stream: { timestamp_field: string; }; + composed_of: string[]; _meta: object; } diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/page_paths.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/page_paths.ts index 9881d5e40d8ab2..9f1088a94aa946 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/page_paths.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/constants/page_paths.ts @@ -53,7 +53,7 @@ export const PAGE_ROUTING_PATHS = { fleet_agent_details_events: '/fleet/agents/:agentId', fleet_agent_details_details: '/fleet/agents/:agentId/details', fleet_enrollment_tokens: '/fleet/enrollment-tokens', - data_streams: '/data-streams', + data_streams: '/datasets', }; export const pagePathGetters: { @@ -80,5 +80,5 @@ export const pagePathGetters: { fleet_agent_details: ({ agentId, tabId }) => `/fleet/agents/${agentId}${tabId ? `/${tabId}` : ''}`, fleet_enrollment_tokens: () => '/fleet/enrollment-tokens', - data_streams: () => '/data-streams', + data_streams: () => '/datasets', }; diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_breadcrumbs.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_breadcrumbs.tsx index 2b92987963ef6a..293638cff50bf1 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_breadcrumbs.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/hooks/use_breadcrumbs.tsx @@ -207,7 +207,7 @@ const breadcrumbGetters: { BASE_BREADCRUMB, { text: i18n.translate('xpack.ingestManager.breadcrumbs.datastreamsPageTitle', { - defaultMessage: 'Data streams', + defaultMessage: 'Datasets', }), }, ], diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx index 5e0cba7383e9ca..1f356301b714ac 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/layouts/default.tsx @@ -103,7 +103,7 @@ export const DefaultLayout: React.FunctionComponent = ({ diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx index e1583d2e426bc2..a6e458a4615cdb 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx @@ -32,7 +32,7 @@ const DataStreamListPageLayout: React.FunctionComponent = ({ children }) => (

@@ -177,7 +177,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => {

} @@ -220,14 +220,14 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { isLoading ? ( ) : dataStreamsData && !dataStreamsData.data_streams.length ? ( emptyPrompt ) : ( ) } @@ -257,7 +257,7 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { placeholder: i18n.translate( 'xpack.ingestManager.dataStreamList.searchPlaceholderTitle', { - defaultMessage: 'Filter data streams', + defaultMessage: 'Filter datasets', } ), incremental: true, diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/overview/components/datastream_section.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/overview/components/datastream_section.tsx index 87906afb4122ab..eab6cf087e1274 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/overview/components/datastream_section.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/overview/components/datastream_section.tsx @@ -51,14 +51,14 @@ export const OverviewDatastreamSection: React.FC = () => {

@@ -70,7 +70,7 @@ export const OverviewDatastreamSection: React.FC = () => { diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index f5fec020bf5b4d..848e65b7931ebc 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -94,6 +94,7 @@ exports[`tests loading base.yml: base.yml 1`] = ` "data_stream": { "timestamp_field": "@timestamp" }, + "composed_of": [], "_meta": { "package": { "name": "nginx" @@ -197,6 +198,7 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` "data_stream": { "timestamp_field": "@timestamp" }, + "composed_of": [], "_meta": { "package": { "name": "coredns" @@ -1684,6 +1686,7 @@ exports[`tests loading system.yml: system.yml 1`] = ` "data_stream": { "timestamp_field": "@timestamp" }, + "composed_of": [], "_meta": { "package": { "name": "system" diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts index a318aecf347d62..e14645bbbf5fb0 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts @@ -5,7 +5,13 @@ */ import Boom from 'boom'; -import { Dataset, RegistryPackage, ElasticsearchAssetType, TemplateRef } from '../../../../types'; +import { + Dataset, + RegistryPackage, + ElasticsearchAssetType, + TemplateRef, + RegistryElasticsearch, +} from '../../../../types'; import { CallESAsCurrentUser } from '../../../../types'; import { Field, loadFieldsFromYaml, processFields } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; @@ -157,6 +163,98 @@ export async function installTemplateForDataset({ }); } +function putComponentTemplate( + body: object | undefined, + name: string, + callCluster: CallESAsCurrentUser +): { clusterPromise: Promise; name: string } | undefined { + if (body) { + const callClusterParams: { + method: string; + path: string; + ignore: number[]; + body: any; + } = { + method: 'PUT', + path: `/_component_template/${name}`, + ignore: [404], + body, + }; + + return { clusterPromise: callCluster('transport.request', callClusterParams), name }; + } +} + +function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | undefined) { + let mappingsTemplate; + let settingsTemplate; + + if (registryElasticsearch && registryElasticsearch['index_template.mappings']) { + mappingsTemplate = { + template: { + mappings: { + ...registryElasticsearch['index_template.mappings'], + // temporary change until https://github.com/elastic/elasticsearch/issues/58956 is resolved + // hopefully we'll be able to remove the entire properties section once that issue is resolved + properties: { + // if the timestamp_field changes here: https://github.com/elastic/kibana/blob/master/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts#L309 + // we'll need to update this as well + '@timestamp': { + type: 'date', + }, + }, + }, + }, + }; + } + + if (registryElasticsearch && registryElasticsearch['index_template.settings']) { + settingsTemplate = { + template: { + settings: registryElasticsearch['index_template.settings'], + }, + }; + } + return { settingsTemplate, mappingsTemplate }; +} + +async function installDatasetComponentTemplates( + templateName: string, + registryElasticsearch: RegistryElasticsearch | undefined, + callCluster: CallESAsCurrentUser +) { + const templates: string[] = []; + const componentPromises: Array> = []; + + const compTemplates = buildComponentTemplates(registryElasticsearch); + + const mappings = putComponentTemplate( + compTemplates.mappingsTemplate, + `${templateName}-mappings`, + callCluster + ); + + const settings = putComponentTemplate( + compTemplates.settingsTemplate, + `${templateName}-settings`, + callCluster + ); + + if (mappings) { + templates.push(mappings.name); + componentPromises.push(mappings.clusterPromise); + } + + if (settings) { + templates.push(settings.name); + componentPromises.push(settings.clusterPromise); + } + + // TODO: Check return values for errors + await Promise.all(componentPromises); + return templates; +} + export async function installTemplate({ callCluster, fields, @@ -180,13 +278,22 @@ export async function installTemplate({ packageVersion, }); } + + const composedOfTemplates = await installDatasetComponentTemplates( + templateName, + dataset.elasticsearch, + callCluster + ); + const template = getTemplate({ type: dataset.type, templateName, mappings, pipelineName, packageName, + composedOfTemplates, }); + // TODO: Check return values for errors const callClusterParams: { method: string; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts index 73a6767f6b947c..99e568bf771f83 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts @@ -29,10 +29,37 @@ test('get template', () => { templateName, packageName: 'nginx', mappings: { properties: {} }, + composedOfTemplates: [], }); expect(template.index_patterns).toStrictEqual([`${templateName}-*`]); }); +test('adds composed_of correctly', () => { + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + type: 'logs', + templateName: 'name', + packageName: 'nginx', + mappings: { properties: {} }, + composedOfTemplates, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); +}); + +test('adds empty composed_of correctly', () => { + const composedOfTemplates: string[] = []; + + const template = getTemplate({ + type: 'logs', + templateName: 'name', + packageName: 'nginx', + mappings: { properties: {} }, + composedOfTemplates, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); +}); + test('tests loading base.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); @@ -45,6 +72,7 @@ test('tests loading base.yml', () => { templateName: 'foo', packageName: 'nginx', mappings, + composedOfTemplates: [], }); expect(template).toMatchSnapshot(path.basename(ymlPath)); @@ -62,6 +90,7 @@ test('tests loading coredns.logs.yml', () => { templateName: 'foo', packageName: 'coredns', mappings, + composedOfTemplates: [], }); expect(template).toMatchSnapshot(path.basename(ymlPath)); @@ -79,6 +108,7 @@ test('tests loading system.yml', () => { templateName: 'whatsthis', packageName: 'system', mappings, + composedOfTemplates: [], }); expect(template).toMatchSnapshot(path.basename(ymlPath)); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index 2de378f7175348..e7867532ed1762 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -43,14 +43,16 @@ export function getTemplate({ mappings, pipelineName, packageName, + composedOfTemplates, }: { type: string; templateName: string; mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; + composedOfTemplates: string[]; }): IndexTemplate { - const template = getBaseTemplate(type, templateName, mappings, packageName); + const template = getBaseTemplate(type, templateName, mappings, packageName, composedOfTemplates); if (pipelineName) { template.template.settings.index.default_pipeline = pipelineName; } @@ -244,7 +246,8 @@ function getBaseTemplate( type: string, templateName: string, mappings: IndexTemplateMappings, - packageName: string + packageName: string, + composedOfTemplates: string[] ): IndexTemplate { return { // This takes precedence over all index templates installed by ES by default (logs-*-* and metrics-*-*) @@ -308,6 +311,7 @@ function getBaseTemplate( data_stream: { timestamp_field: '@timestamp', }, + composed_of: composedOfTemplates, _meta: { package: { name: packageName, diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts new file mode 100644 index 00000000000000..ae6493d4716e81 --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + SavedObject, + SavedObjectsBulkCreateObject, + SavedObjectsClientContract, +} from 'src/core/server'; +import * as Registry from '../../registry'; +import { AssetType, KibanaAssetType, AssetReference } from '../../../../types'; + +type SavedObjectToBe = Required & { type: AssetType }; +export type ArchiveAsset = Pick< + SavedObject, + 'id' | 'attributes' | 'migrationVersion' | 'references' +> & { + type: AssetType; +}; + +export async function getKibanaAsset(key: string) { + const buffer = Registry.getAsset(key); + + // cache values are buffers. convert to string / JSON + return JSON.parse(buffer.toString('utf8')); +} + +export function createSavedObjectKibanaAsset( + jsonAsset: ArchiveAsset, + pkgName: string +): SavedObjectToBe { + // convert that to an object + const asset = changeAssetIds(jsonAsset, pkgName); + + return { + type: asset.type, + id: asset.id, + attributes: asset.attributes, + references: asset.references || [], + migrationVersion: asset.migrationVersion || {}, + }; +} + +// modifies id property and the id property of references objects (not index-pattern) +// to be prepended with the package name to distinguish assets from Beats modules' assets +export const changeAssetIds = (asset: ArchiveAsset, pkgName: string): ArchiveAsset => { + const references = asset.references.map((ref) => { + if (ref.type === KibanaAssetType.indexPattern) return ref; + const id = getAssetId(ref.id, pkgName); + return { ...ref, id }; + }); + return { + ...asset, + id: getAssetId(asset.id, pkgName), + references, + }; +}; + +export const getAssetId = (id: string, pkgName: string) => { + return `${pkgName}-${id}`; +}; + +// TODO: make it an exhaustive list +// e.g. switch statement with cases for each enum key returning `never` for default case +export async function installKibanaAssets(options: { + savedObjectsClient: SavedObjectsClientContract; + pkgName: string; + paths: string[]; +}) { + const { savedObjectsClient, paths, pkgName } = options; + + // Only install Kibana assets during package installation. + const kibanaAssetTypes = Object.values(KibanaAssetType); + const installationPromises = kibanaAssetTypes.map((assetType) => + installKibanaSavedObjects({ savedObjectsClient, assetType, paths, pkgName }) + ); + + // installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][] + // call .flat to flatten into one dimensional array + return Promise.all(installationPromises).then((results) => results.flat()); +} + +async function installKibanaSavedObjects({ + savedObjectsClient, + assetType, + paths, + pkgName, +}: { + savedObjectsClient: SavedObjectsClientContract; + assetType: KibanaAssetType; + paths: string[]; + pkgName: string; +}) { + const isSameType = (path: string) => assetType === Registry.pathParts(path).type; + const pathsOfType = paths.filter((path) => isSameType(path)); + const kibanaAssets = await Promise.all(pathsOfType.map((path) => getKibanaAsset(path))); + const toBeSavedObjects = await Promise.all( + kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset, pkgName)) + ); + + if (toBeSavedObjects.length === 0) { + return []; + } else { + const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, { + overwrite: true, + }); + const createdObjects = createResults.saved_objects; + const installed = createdObjects.map(toAssetReference); + return installed; + } +} + +function toAssetReference({ id, type }: SavedObject) { + const reference: AssetReference = { id, type: type as KibanaAssetType }; + + return reference; +} diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap new file mode 100644 index 00000000000000..638ed4b6118c99 --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap @@ -0,0 +1,133 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`a kibana asset id and its reference ids are appended with package name changeAssetIds output matches snapshot: dashboard.json 1`] = ` +{ + "attributes": { + "description": "Overview dashboard for the Nginx integration in Metrics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false, + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "1", + "w": 24, + "x": 24, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "2", + "w": 24, + "x": 24, + "y": 12 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "3", + "w": 24, + "x": 0, + "y": 12 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "4", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "5", + "w": 48, + "x": 0, + "y": 24 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "version": "7.3.0" + } + ], + "timeRestore": false, + "title": "[Metrics Nginx] Overview ECS", + "version": 1 + }, + "id": "nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs", + "migrationVersion": { + "dashboard": "7.3.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_0", + "type": "search" + }, + { + "id": "nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_1", + "type": "map" + }, + { + "id": "nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_2", + "type": "dashboard" + }, + { + "id": "nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_4", + "type": "visualization" + } + ], + "type": "dashboard" +} +`; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json new file mode 100644 index 00000000000000..e28a61ae5e18c3 --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json @@ -0,0 +1,129 @@ +{ + "attributes": { + "description": "Overview dashboard for the Nginx integration in Metrics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false, + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "1", + "w": 24, + "x": 24, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "2", + "w": 24, + "x": 24, + "y": 12 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "3", + "w": 24, + "x": 0, + "y": 12 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "4", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "5", + "w": 48, + "x": 0, + "y": 24 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "version": "7.3.0" + } + ], + "timeRestore": false, + "title": "[Metrics Nginx] Overview ECS", + "version": 1 + }, + "id": "023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs", + "migrationVersion": { + "dashboard": "7.3.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_0", + "type": "search" + }, + { + "id": "a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_1", + "type": "map" + }, + { + "id": "d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_2", + "type": "dashboard" + }, + { + "id": "47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_4", + "type": "visualization" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts new file mode 100644 index 00000000000000..f9bc4cdbf203fd --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { readFileSync } from 'fs'; +import path from 'path'; +import { getAssetId, changeAssetIds } from '../install'; + +expect.addSnapshotSerializer({ + print(val) { + return JSON.stringify(val, null, 2); + }, + + test(val) { + return val; + }, +}); + +describe('a kibana asset id and its reference ids are appended with package name', () => { + const assetPath = path.join(__dirname, './dashboard.json'); + const kibanaAsset = JSON.parse(readFileSync(assetPath, 'utf-8')); + const pkgName = 'nginx'; + const modifiedAssetObject = changeAssetIds(kibanaAsset, pkgName); + + test('changeAssetIds output matches snapshot', () => { + expect(modifiedAssetObject).toMatchSnapshot(path.basename(assetPath)); + }); + + test('getAssetId', () => { + const id = '47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs'; + expect(getAssetId(id, pkgName)).toBe(`${pkgName}-${id}`); + }); +}); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts deleted file mode 100644 index b623295c5e0604..00000000000000 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SavedObject, SavedObjectsBulkCreateObject } from 'src/core/server'; -import { AssetType } from '../../../types'; -import * as Registry from '../registry'; - -type ArchiveAsset = Pick; -type SavedObjectToBe = Required & { type: AssetType }; - -export async function getObject(key: string) { - const buffer = Registry.getAsset(key); - - // cache values are buffers. convert to string / JSON - const json = buffer.toString('utf8'); - // convert that to an object - const asset: ArchiveAsset = JSON.parse(json); - - const { type, file } = Registry.pathParts(key); - const savedObject: SavedObjectToBe = { - type, - id: file.replace('.json', ''), - attributes: asset.attributes, - references: asset.references || [], - migrationVersion: asset.migrationVersion || {}, - }; - - return savedObject; -} diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts index b79f9178ad6af4..53ffd5c6e70328 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts @@ -23,7 +23,7 @@ export { SearchParams, } from './get'; -export { installKibanaAssets, installPackage, ensureInstalledPackage } from './install'; +export { installPackage, ensureInstalledPackage } from './install'; export { removeInstallation } from './remove'; type RequiredPackage = 'system' | 'endpoint'; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 910283549abdfc..8f73bc9a027653 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -4,13 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import { SavedObjectsClientContract } from 'src/core/server'; import Boom from 'boom'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { AssetReference, Installation, - KibanaAssetType, CallESAsCurrentUser, DefaultPackages, ElasticsearchAssetType, @@ -18,7 +17,7 @@ import { } from '../../../types'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import * as Registry from '../registry'; -import { getObject } from './get_objects'; +import { installKibanaAssets } from '../kibana/assets/install'; import { getInstallation, getInstallationObject, isRequiredPackage } from './index'; import { installTemplates } from '../elasticsearch/template/install'; import { generateESIndexPatterns } from '../elasticsearch/template/template'; @@ -121,7 +120,6 @@ export async function installPackage(options: { installKibanaAssets({ savedObjectsClient, pkgName, - pkgVersion, paths, }), installPipelines(registryPackageInfo, paths, callCluster), @@ -185,27 +183,6 @@ export async function installPackage(options: { }); } -// TODO: make it an exhaustive list -// e.g. switch statement with cases for each enum key returning `never` for default case -export async function installKibanaAssets(options: { - savedObjectsClient: SavedObjectsClientContract; - pkgName: string; - pkgVersion: string; - paths: string[]; -}) { - const { savedObjectsClient, paths } = options; - - // Only install Kibana assets during package installation. - const kibanaAssetTypes = Object.values(KibanaAssetType); - const installationPromises = kibanaAssetTypes.map(async (assetType) => - installKibanaSavedObjects({ savedObjectsClient, assetType, paths }) - ); - - // installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][] - // call .flat to flatten into one dimensional array - return Promise.all(installationPromises).then((results) => results.flat()); -} - export async function saveInstallationReferences(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; @@ -240,34 +217,3 @@ export async function saveInstallationReferences(options: { return toSaveAssetRefs; } - -async function installKibanaSavedObjects({ - savedObjectsClient, - assetType, - paths, -}: { - savedObjectsClient: SavedObjectsClientContract; - assetType: KibanaAssetType; - paths: string[]; -}) { - const isSameType = (path: string) => assetType === Registry.pathParts(path).type; - const pathsOfType = paths.filter((path) => isSameType(path)); - const toBeSavedObjects = await Promise.all(pathsOfType.map(getObject)); - - if (toBeSavedObjects.length === 0) { - return []; - } else { - const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, { - overwrite: true, - }); - const createdObjects = createResults.saved_objects; - const installed = createdObjects.map(toAssetReference); - return installed; - } -} - -function toAssetReference({ id, type }: SavedObject) { - const reference: AssetReference = { id, type: type as KibanaAssetType }; - - return reference; -} diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index 8239302a97832d..a559ca18cfedef 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -41,6 +41,7 @@ export { PackageInfo, RegistryVarsEntry, Dataset, + RegistryElasticsearch, AssetReference, ElasticsearchAssetType, IngestAssetType, diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/fields/fields.yml b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/fields/fields.yml new file mode 100644 index 00000000000000..12a9a03c1337b4 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/fields/fields.yml @@ -0,0 +1,16 @@ +- name: dataset.type + type: constant_keyword + description: > + Dataset type. +- name: dataset.name + type: constant_keyword + description: > + Dataset name. +- name: dataset.namespace + type: constant_keyword + description: > + Dataset namespace. +- name: '@timestamp' + type: date + description: > + Event timestamp. diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/manifest.yml b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/manifest.yml new file mode 100644 index 00000000000000..9ac3c68a0be9ec --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/manifest.yml @@ -0,0 +1,9 @@ +title: Test Dataset + +type: logs + +elasticsearch: + index_template.mappings: + dynamic: false + index_template.settings: + index.lifecycle.name: reference diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/docs/README.md b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/docs/README.md new file mode 100644 index 00000000000000..17fb41ceae242d --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/docs/README.md @@ -0,0 +1,3 @@ +# Test package + +For testing the that the settings and mappings section get used diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/img/logo_overrides_64_color.svg b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/img/logo_overrides_64_color.svg new file mode 100644 index 00000000000000..b03007a76ffcc5 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/img/logo_overrides_64_color.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/manifest.yml b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/manifest.yml new file mode 100644 index 00000000000000..ba9fd0fada006d --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/manifest.yml @@ -0,0 +1,20 @@ +format_version: 1.0.0 +name: overrides +title: Mappings Settings Test +description: This is a test package for testing that the mappings and settings sections in the dataset manifest are applied. +version: 0.1.0 +categories: ['security'] +release: beta +type: integration +license: basic + +requirement: + elasticsearch: + versions: '>7.7.0' + kibana: + versions: '>7.7.0' + +icons: + - src: '/img/logo_overrides_64_color.svg' + size: '16x16' + type: 'image/svg+xml' diff --git a/x-pack/test/ingest_manager_api_integration/apis/index.js b/x-pack/test/ingest_manager_api_integration/apis/index.js index ef8880f86078b3..3f8df8379e743a 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/index.js @@ -11,5 +11,6 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./file')); //loadTestFile(require.resolve('./template')); loadTestFile(require.resolve('./ilm')); + loadTestFile(require.resolve('./install')); }); } diff --git a/x-pack/test/ingest_manager_api_integration/apis/install.ts b/x-pack/test/ingest_manager_api_integration/apis/install.ts new file mode 100644 index 00000000000000..92078c25419dfd --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/install.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../helpers'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const es = getService('es'); + const dockerServers = getService('dockerServers'); + const log = getService('log'); + + const deletePackage = async (pkgkey: string) => { + await supertest.delete(`/api/ingest_manager/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx'); + }; + + const mappingsPackage = 'overrides-0.1.0'; + const server = dockerServers.get('registry'); + + describe('installs packages that include settings and mappings overrides', async () => { + after(async () => { + if (server.enabled) { + // remove the package just in case it being installed will affect other tests + await deletePackage(mappingsPackage); + } + }); + + it('should install the overrides package correctly', async function () { + if (server.enabled) { + let { body } = await supertest + .post(`/api/ingest_manager/epm/packages/${mappingsPackage}`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + const templateName = body.response[0].id; + + ({ body } = await es.transport.request({ + method: 'GET', + path: `/_index_template/${templateName}`, + })); + + // make sure it has the right composed_of array, the contents should be the component templates + // that were installed + expect(body.index_templates[0].index_template.composed_of).to.contain( + `${templateName}-mappings` + ); + expect(body.index_templates[0].index_template.composed_of).to.contain( + `${templateName}-settings` + ); + + ({ body } = await es.transport.request({ + method: 'GET', + path: `/_component_template/${templateName}-mappings`, + })); + + // Make sure that the `dynamic` field exists and is set to false (as it is in the package) + expect(body.component_templates[0].component_template.template.mappings.dynamic).to.be( + false + ); + // Make sure that the `@timestamp` field exists and is set to date + // this can be removed once https://github.com/elastic/elasticsearch/issues/58956 is resolved + expect( + body.component_templates[0].component_template.template.mappings.properties['@timestamp'] + .type + ).to.be('date'); + + ({ body } = await es.transport.request({ + method: 'GET', + path: `/_component_template/${templateName}-settings`, + })); + + // Make sure that the lifecycle name gets set correct in the settings + expect( + body.component_templates[0].component_template.template.settings.index.lifecycle.name + ).to.be('reference'); + } else { + warnAndSkipTest(this, log); + } + }); + }); +} diff --git a/x-pack/test/ingest_manager_api_integration/apis/list.ts b/x-pack/test/ingest_manager_api_integration/apis/list.ts index 200358cb6f8f03..abed9a7b859599 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/list.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/list.ts @@ -29,7 +29,7 @@ export default function ({ getService }: FtrProviderContext) { return response.body; }; const listResponse = await fetchPackageList(); - expect(listResponse.response.length).to.be(11); + expect(listResponse.response.length).to.be(12); } else { warnAndSkipTest(this, log); } diff --git a/x-pack/test/ingest_manager_api_integration/apis/template.ts b/x-pack/test/ingest_manager_api_integration/apis/template.ts index 8911dd28dc2437..f7e5a894b83ff1 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/template.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/template.ts @@ -27,6 +27,7 @@ export default function ({ getService }: FtrProviderContext) { templateName, mappings, packageName: 'system', + composedOfTemplates: [], }); // This test is not an API integration test with Kibana