diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx index 16a16205261c9..dc931f835b043 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx @@ -11,6 +11,8 @@ import { FormattedMessage } from '@kbn/i18n-react'; import { EuiFlexGroup, EuiFlexItem, EuiTitle, EuiSpacer } from '@elastic/eui'; import { groupBy } from 'lodash'; +import type { ResolvedSimpleSavedObject } from 'src/core/public'; + import { Loading, Error, ExtensionWrapper } from '../../../../../components'; import type { PackageInfo } from '../../../../../types'; @@ -27,6 +29,7 @@ import type { AssetSavedObject } from './types'; import { allowedAssetTypes } from './constants'; import { AssetsAccordion } from './assets_accordion'; +const allowedAssetTypesLookup = new Set(allowedAssetTypes); interface AssetsPanelProps { packageInfo: PackageInfo; } @@ -74,19 +77,32 @@ export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => { const objectsByType = await Promise.all( Object.entries(groupBy(objectsToGet, 'type')).map(([type, objects]) => savedObjectsClient - .bulkGet(objects) + .bulkResolve(objects) // Ignore privilege errors .catch((e: any) => { if (e?.body?.statusCode === 403) { - return { savedObjects: [] }; + return { resolved_objects: [] }; } else { throw e; } }) - .then(({ savedObjects }) => savedObjects as AssetSavedObject[]) + .then( + ({ + resolved_objects: resolvedObjects, + }: { + resolved_objects: ResolvedSimpleSavedObject[]; + }) => { + return resolvedObjects + .map(({ saved_object: savedObject }) => savedObject) + .filter( + (savedObject) => + savedObject?.error?.statusCode !== 404 && + allowedAssetTypesLookup.has(savedObject.type) + ) as AssetSavedObject[]; + } + ) ) ); - setAssetsSavedObjects(objectsByType.flat()); } catch (e) { setFetchError(e); @@ -107,7 +123,6 @@ export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => { } let content: JSX.Element | Array; - if (isLoading) { content = ; } else if (fetchError) { @@ -122,7 +137,7 @@ export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => { error={fetchError} /> ); - } else if (assetSavedObjects === undefined) { + } else if (assetSavedObjects === undefined || assetSavedObjects.length === 0) { if (customAssetsExtension) { // If a UI extension for custom asset entries is defined, render the custom component here depisite // there being no saved objects found diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts index 50c0239cd8c56..5ab15a1f52e75 100644 --- a/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts @@ -9,15 +9,27 @@ import type { SavedObject, SavedObjectsBulkCreateObject, SavedObjectsClientContract, + SavedObjectsImporter, + Logger, } from 'src/core/server'; +import type { SavedObjectsImportSuccess, SavedObjectsImportFailure } from 'src/core/server/types'; + +import { createListStream } from '@kbn/utils'; +import { partition } from 'lodash'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common'; import { getAsset, getPathParts } from '../../archive'; import { KibanaAssetType, KibanaSavedObjectType } from '../../../../types'; import type { AssetType, AssetReference, AssetParts } from '../../../../types'; import { savedObjectTypes } from '../../packages'; -import { indexPatternTypes } from '../index_pattern/install'; +import { indexPatternTypes, getIndexPatternSavedObjects } from '../index_pattern/install'; +type SavedObjectsImporterContract = Pick; +const formatImportErrorsForLog = (errors: SavedObjectsImportFailure[]) => + JSON.stringify( + errors.map(({ type, id, error }) => ({ type, id, error })) // discard other fields + ); +const validKibanaAssetTypes = new Set(Object.values(KibanaAssetType)); type SavedObjectToBe = Required> & { type: KibanaSavedObjectType; }; @@ -42,23 +54,8 @@ const KibanaSavedObjectTypeMapping: Record Promise>> -> = { - [KibanaAssetType.dashboard]: installKibanaSavedObjects, - [KibanaAssetType.indexPattern]: installKibanaIndexPatterns, - [KibanaAssetType.map]: installKibanaSavedObjects, - [KibanaAssetType.search]: installKibanaSavedObjects, - [KibanaAssetType.visualization]: installKibanaSavedObjects, - [KibanaAssetType.lens]: installKibanaSavedObjects, - [KibanaAssetType.mlModule]: installKibanaSavedObjects, - [KibanaAssetType.securityRule]: installKibanaSavedObjects, - [KibanaAssetType.tag]: installKibanaSavedObjects, +const AssetFilters: Record ArchiveAsset[]> = { + [KibanaAssetType.indexPattern]: removeReservedIndexPatterns, }; export async function getKibanaAsset(key: string): Promise { @@ -79,29 +76,46 @@ export function createSavedObjectKibanaAsset(asset: ArchiveAsset): SavedObjectTo }; } -// TODO: make it an exhaustive list -// e.g. switch statement with cases for each enum key returning `never` for default case export async function installKibanaAssets(options: { - savedObjectsClient: SavedObjectsClientContract; + savedObjectsImporter: SavedObjectsImporterContract; + logger: Logger; pkgName: string; kibanaAssets: Record; -}): Promise { - const { savedObjectsClient, kibanaAssets } = options; +}): Promise { + const { kibanaAssets, savedObjectsImporter, logger } = options; + const assetsToInstall = Object.entries(kibanaAssets).flatMap(([assetType, assets]) => { + if (!validKibanaAssetTypes.has(assetType as KibanaAssetType)) { + return []; + } - // install the assets - const kibanaAssetTypes = Object.values(KibanaAssetType); - const installedAssets = await Promise.all( - kibanaAssetTypes.map((assetType) => { - if (kibanaAssets[assetType]) { - return AssetInstallers[assetType]({ - savedObjectsClient, - kibanaAssets: kibanaAssets[assetType], - }); - } + if (!assets.length) { return []; - }) - ); - return installedAssets.flat(); + } + + const assetFilter = AssetFilters[assetType]; + if (assetFilter) { + return assetFilter(assets); + } + + return assets; + }); + + if (!assetsToInstall.length) { + return []; + } + + // As we use `import` to create our saved objects, we have to install + // their references (the index patterns) at the same time + // to prevent a reference error + const indexPatternSavedObjects = getIndexPatternSavedObjects() as ArchiveAsset[]; + + const installedAssets = await installKibanaSavedObjects({ + logger, + savedObjectsImporter, + kibanaAssets: [...indexPatternSavedObjects, ...assetsToInstall], + }); + + return installedAssets; } export const deleteKibanaInstalledRefs = async ( savedObjectsClient: SavedObjectsClientContract, @@ -153,39 +167,95 @@ export async function getKibanaAssets( } async function installKibanaSavedObjects({ - savedObjectsClient, + savedObjectsImporter, kibanaAssets, + logger, }: { - savedObjectsClient: SavedObjectsClientContract; kibanaAssets: ArchiveAsset[]; + savedObjectsImporter: SavedObjectsImporterContract; + logger: Logger; }) { const toBeSavedObjects = await Promise.all( kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset)) ); + let allSuccessResults = []; + if (toBeSavedObjects.length === 0) { return []; } else { - const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, { - overwrite: true, - }); - return createResults.saved_objects; + const { successResults: importSuccessResults = [], errors: importErrors = [] } = + await savedObjectsImporter.import({ + overwrite: true, + readStream: createListStream(toBeSavedObjects), + createNewCopies: false, + }); + + allSuccessResults = importSuccessResults; + const [referenceErrors, otherErrors] = partition( + importErrors, + (e) => e?.error?.type === 'missing_references' + ); + + if (otherErrors?.length) { + throw new Error( + `Encountered ${ + otherErrors.length + } errors creating saved objects: ${formatImportErrorsForLog(otherErrors)}` + ); + } + /* + A reference error here means that a saved object reference in the references + array cannot be found. This is an error in the package its-self but not a fatal + one. For example a dashboard may still refer to the legacy `metricbeat-*` index + pattern. We ignore reference errors here so that legacy version of a package + can still be installed, but if a warning is logged it should be reported to + the integrations team. */ + if (referenceErrors.length) { + logger.debug( + `Resolving ${ + referenceErrors.length + } reference errors creating saved objects: ${formatImportErrorsForLog(referenceErrors)}` + ); + + const idsToResolve = new Set(referenceErrors.map(({ id }) => id)); + + const resolveSavedObjects = toBeSavedObjects.filter(({ id }) => idsToResolve.has(id)); + const retries = referenceErrors.map(({ id, type }) => ({ + id, + type, + ignoreMissingReferences: true, + replaceReferences: [], + overwrite: true, + })); + + const { successResults: resolveSuccessResults = [], errors: resolveErrors = [] } = + await savedObjectsImporter.resolveImportErrors({ + readStream: createListStream(resolveSavedObjects), + createNewCopies: false, + retries, + }); + + if (resolveErrors?.length) { + throw new Error( + `Encountered ${ + resolveErrors.length + } errors resolving reference errors: ${formatImportErrorsForLog(resolveErrors)}` + ); + } + + allSuccessResults = [...allSuccessResults, ...resolveSuccessResults]; + } + + return allSuccessResults; } } -async function installKibanaIndexPatterns({ - savedObjectsClient, - kibanaAssets, -}: { - savedObjectsClient: SavedObjectsClientContract; - kibanaAssets: ArchiveAsset[]; -}) { - // Filter out any reserved index patterns +// Filter out any reserved index patterns +function removeReservedIndexPatterns(kibanaAssets: ArchiveAsset[]) { const reservedPatterns = indexPatternTypes.map((pattern) => `${pattern}-*`); - const nonReservedPatterns = kibanaAssets.filter((asset) => !reservedPatterns.includes(asset.id)); - - return installKibanaSavedObjects({ savedObjectsClient, kibanaAssets: nonReservedPatterns }); + return kibanaAssets.filter((asset) => !reservedPatterns.includes(asset.id)); } export function toAssetReference({ id, type }: SavedObject) { diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap deleted file mode 100644 index da870290329a8..0000000000000 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap +++ /dev/null @@ -1,935 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`creating index patterns from yaml fields createFieldFormatMap creates correct map based on inputs all variations and all the params get passed through: createFieldFormatMap 1`] = ` -{ - "fieldPattern": { - "params": { - "pattern": "patternVal" - } - }, - "fieldFormat": { - "id": "formatVal" - }, - "fieldFormatWithParam": { - "id": "formatVal", - "params": { - "outputPrecision": 2 - } - }, - "fieldFormatAndPattern": { - "id": "formatVal", - "params": { - "pattern": "patternVal" - } - }, - "fieldFormatAndAllParams": { - "id": "formatVal", - "params": { - "pattern": "pattenVal", - "inputFormat": "inputFormatVal", - "outputFormat": "outputFormalVal", - "outputPrecision": 3, - "labelTemplate": "labelTemplateVal", - "urlTemplate": "urlTemplateVal" - } - } -} -`; - -exports[`creating index patterns from yaml fields createIndexPattern function creates Kibana index pattern: createIndexPattern 1`] = ` -{ - "title": "logs-*", - "timeFieldName": "@timestamp", - "fields": "[{\\"name\\":\\"coredns.id\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.allParams\\",\\"type\\":\\"number\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.query.length\\",\\"type\\":\\"number\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.query.size\\",\\"type\\":\\"number\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.query.class\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.query.name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.query.type\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.response.code\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.response.flags\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.response.size\\",\\"type\\":\\"number\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"coredns.dnssec_ok\\",\\"type\\":\\"boolean\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"@timestamp\\",\\"type\\":\\"date\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"labels\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"message\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":true},{\\"name\\":\\"tags\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"agent.ephemeral_id\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"agent.id\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"agent.name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"agent.type\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"agent.version\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"as.number\\",\\"type\\":\\"number\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"as.organization.name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.remote_ip_list\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.body_sent.bytes\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.method\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.url\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.http_version\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.response_code\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.referrer\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.agent\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.device\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.os\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.os_name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.user_agent.original\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.continent_name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.country_iso_code\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.location\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.region_name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.city_name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"nginx.access.geoip.region_iso_code\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"source.geo.continent_name\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":true},{\\"name\\":\\"country\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"country.keyword\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"country.text\\",\\"type\\":\\"string\\",\\"count\\":0,\\"scripted\\":false,\\"indexed\\":true,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":true}]", - "fieldFormatMap": "{\\"coredns.allParams\\":{\\"id\\":\\"bytes\\",\\"params\\":{\\"pattern\\":\\"patternValQueryWeight\\",\\"inputFormat\\":\\"inputFormatVal,\\",\\"outputFormat\\":\\"outputFormalVal,\\",\\"outputPrecision\\":\\"3,\\",\\"labelTemplate\\":\\"labelTemplateVal,\\",\\"urlTemplate\\":\\"urlTemplateVal,\\"}},\\"coredns.query.length\\":{\\"params\\":{\\"pattern\\":\\"patternValQueryLength\\"}},\\"coredns.query.size\\":{\\"id\\":\\"bytes\\",\\"params\\":{\\"pattern\\":\\"patternValQuerySize\\"}},\\"coredns.response.size\\":{\\"id\\":\\"bytes\\"}}", - "allowNoIndex": true -} -`; - -exports[`creating index patterns from yaml fields createIndexPatternFields function creates Kibana index pattern fields and fieldFormatMap: createIndexPatternFields 1`] = ` -{ - "indexPatternFields": [ - { - "name": "coredns.id", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.allParams", - "type": "number", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.query.length", - "type": "number", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.query.size", - "type": "number", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.query.class", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.query.name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.query.type", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.response.code", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.response.flags", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.response.size", - "type": "number", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "coredns.dnssec_ok", - "type": "boolean", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "@timestamp", - "type": "date", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "labels", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "message", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": false, - "readFromDocValues": true - }, - { - "name": "tags", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "agent.ephemeral_id", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "agent.id", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "agent.name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "agent.type", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "agent.version", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "as.number", - "type": "number", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "as.organization.name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.remote_ip_list", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.body_sent.bytes", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.user_name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.method", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.url", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.http_version", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.response_code", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.referrer", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.agent", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.user_agent.device", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.user_agent.name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.user_agent.os", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.user_agent.os_name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.user_agent.original", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.geoip.continent_name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": false, - "readFromDocValues": true - }, - { - "name": "nginx.access.geoip.country_iso_code", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.geoip.location", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.geoip.region_name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.geoip.city_name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "nginx.access.geoip.region_iso_code", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "source.geo.continent_name", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": false, - "readFromDocValues": true - }, - { - "name": "country", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "country.keyword", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": true, - "readFromDocValues": true - }, - { - "name": "country.text", - "type": "string", - "count": 0, - "scripted": false, - "indexed": true, - "searchable": true, - "aggregatable": false, - "readFromDocValues": true - } - ], - "fieldFormatMap": { - "coredns.allParams": { - "id": "bytes", - "params": { - "pattern": "patternValQueryWeight", - "inputFormat": "inputFormatVal,", - "outputFormat": "outputFormalVal,", - "outputPrecision": "3,", - "labelTemplate": "labelTemplateVal,", - "urlTemplate": "urlTemplateVal," - } - }, - "coredns.query.length": { - "params": { - "pattern": "patternValQueryLength" - } - }, - "coredns.query.size": { - "id": "bytes", - "params": { - "pattern": "patternValQuerySize" - } - }, - "coredns.response.size": { - "id": "bytes" - } - } -} -`; - -exports[`creating index patterns from yaml fields flattenFields function flattens recursively and handles copying alias fields flattenFields matches snapshot: flattenFields 1`] = ` -[ - { - "name": "coredns.id", - "type": "keyword", - "description": "id of the DNS transaction\\n" - }, - { - "name": "coredns.allParams", - "type": "integer", - "format": "bytes", - "pattern": "patternValQueryWeight", - "input_format": "inputFormatVal,", - "output_format": "outputFormalVal,", - "output_precision": "3,", - "label_template": "labelTemplateVal,", - "url_template": "urlTemplateVal,", - "openLinkInCurrentTab": "true,", - "description": "weight of the DNS query\\n" - }, - { - "name": "coredns.query.length", - "type": "integer", - "pattern": "patternValQueryLength", - "description": "length of the DNS query\\n" - }, - { - "name": "coredns.query.size", - "type": "integer", - "format": "bytes", - "pattern": "patternValQuerySize", - "description": "size of the DNS query\\n" - }, - { - "name": "coredns.query.class", - "type": "keyword", - "description": "DNS query class\\n" - }, - { - "name": "coredns.query.name", - "type": "keyword", - "description": "DNS query name\\n" - }, - { - "name": "coredns.query.type", - "type": "keyword", - "description": "DNS query type\\n" - }, - { - "name": "coredns.response.code", - "type": "keyword", - "description": "DNS response code\\n" - }, - { - "name": "coredns.response.flags", - "type": "keyword", - "description": "DNS response flags\\n" - }, - { - "name": "coredns.response.size", - "type": "integer", - "format": "bytes", - "description": "size of the DNS response\\n" - }, - { - "name": "coredns.dnssec_ok", - "type": "boolean", - "description": "dnssec flag\\n" - }, - { - "name": "@timestamp", - "level": "core", - "required": true, - "type": "date", - "description": "Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events.", - "example": "2016-05-23T08:05:34.853Z" - }, - { - "name": "labels", - "level": "core", - "type": "object", - "object_type": "keyword", - "description": "Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: \`docker\` and \`k8s\` labels.", - "example": { - "application": "foo-bar", - "env": "production" - } - }, - { - "name": "message", - "level": "core", - "type": "text", - "description": "For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message.", - "example": "Hello World" - }, - { - "name": "tags", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "List of keywords used to tag each event.", - "example": "[\\"production\\", \\"env2\\"]" - }, - { - "name": "agent.ephemeral_id", - "level": "extended", - "type": "keyword", - "ignore_above": 1024, - "description": "Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but \`agent.id\` does not.", - "example": "8a4f500f" - }, - { - "name": "agent.id", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.", - "example": "8a4f500d" - }, - { - "name": "agent.name", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "Custom name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.", - "example": "foo" - }, - { - "name": "agent.type", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.", - "example": "filebeat" - }, - { - "name": "agent.version", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "Version of the agent.", - "example": "6.0.0-rc2" - }, - { - "name": "as.number", - "level": "extended", - "type": "long", - "description": "Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet.", - "example": 15169 - }, - { - "name": "as.organization.name", - "level": "extended", - "type": "keyword", - "ignore_above": 1024, - "description": "Organization name.", - "example": "Google LLC" - }, - { - "name": "@timestamp", - "level": "core", - "required": true, - "type": "date", - "description": "Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events.", - "example": "2016-05-23T08:05:34.853Z" - }, - { - "name": "labels", - "level": "core", - "type": "object", - "object_type": "keyword", - "description": "Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: \`docker\` and \`k8s\` labels.", - "example": { - "application": "foo-bar", - "env": "production" - } - }, - { - "name": "message", - "level": "core", - "type": "text", - "description": "For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message.", - "example": "Hello World" - }, - { - "name": "tags", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "List of keywords used to tag each event.", - "example": "[\\"production\\", \\"env2\\"]" - }, - { - "name": "agent.ephemeral_id", - "level": "extended", - "type": "keyword", - "ignore_above": 1024, - "description": "Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but \`agent.id\` does not.", - "example": "8a4f500f" - }, - { - "name": "agent.id", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id.", - "example": "8a4f500d" - }, - { - "name": "agent.name", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "Custom name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty.", - "example": "foo" - }, - { - "name": "agent.type", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine.", - "example": "filebeat" - }, - { - "name": "agent.version", - "level": "core", - "type": "keyword", - "ignore_above": 1024, - "description": "Version of the agent.", - "example": "6.0.0-rc2" - }, - { - "name": "as.number", - "level": "extended", - "type": "long", - "description": "Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet.", - "example": 15169 - }, - { - "name": "as.organization.name", - "level": "extended", - "type": "keyword", - "ignore_above": 1024, - "description": "Organization name.", - "example": "Google LLC" - }, - { - "name": "nginx.access.remote_ip_list", - "type": "array", - "description": "An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like \`X-Forwarded-For\`. Real source IP is restored to \`source.ip\`.\\n" - }, - { - "name": "nginx.access.body_sent.bytes", - "type": "alias", - "path": "http.response.body.bytes", - "migration": true - }, - { - "name": "nginx.access.user_name", - "type": "alias", - "path": "user.name", - "migration": true - }, - { - "name": "nginx.access.method", - "type": "alias", - "path": "http.request.method", - "migration": true - }, - { - "name": "nginx.access.url", - "type": "alias", - "path": "url.original", - "migration": true - }, - { - "name": "nginx.access.http_version", - "type": "alias", - "path": "http.version", - "migration": true - }, - { - "name": "nginx.access.response_code", - "type": "alias", - "path": "http.response.status_code", - "migration": true - }, - { - "name": "nginx.access.referrer", - "type": "alias", - "path": "http.request.referrer", - "migration": true - }, - { - "name": "nginx.access.agent", - "type": "alias", - "path": "user_agent.original", - "migration": true - }, - { - "name": "nginx.access.user_agent.device", - "type": "alias", - "path": "user_agent.device.name", - "migration": true - }, - { - "name": "nginx.access.user_agent.name", - "type": "alias", - "path": "user_agent.name", - "migration": true - }, - { - "name": "nginx.access.user_agent.os", - "type": "alias", - "path": "user_agent.os.full_name", - "migration": true - }, - { - "name": "nginx.access.user_agent.os_name", - "type": "alias", - "path": "user_agent.os.name", - "migration": true - }, - { - "name": "nginx.access.user_agent.original", - "type": "alias", - "path": "user_agent.original", - "migration": true - }, - { - "name": "nginx.access.geoip.continent_name", - "type": "text", - "path": "source.geo.continent_name" - }, - { - "name": "nginx.access.geoip.country_iso_code", - "type": "alias", - "path": "source.geo.country_iso_code", - "migration": true - }, - { - "name": "nginx.access.geoip.location", - "type": "alias", - "path": "source.geo.location", - "migration": true - }, - { - "name": "nginx.access.geoip.region_name", - "type": "alias", - "path": "source.geo.region_name", - "migration": true - }, - { - "name": "nginx.access.geoip.city_name", - "type": "alias", - "path": "source.geo.city_name", - "migration": true - }, - { - "name": "nginx.access.geoip.region_iso_code", - "type": "alias", - "path": "source.geo.region_iso_code", - "migration": true - }, - { - "name": "source.geo.continent_name", - "type": "text" - }, - { - "name": "country", - "type": "", - "multi_fields": [ - { - "name": "keyword", - "type": "keyword" - }, - { - "name": "text", - "type": "text" - } - ] - }, - { - "name": "country.keyword", - "type": "keyword" - }, - { - "name": "country.text", - "type": "text" - } -] -`; diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.test.ts b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.test.ts deleted file mode 100644 index dfdaa66a7b43e..0000000000000 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.test.ts +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import path from 'path'; -import { readFileSync } from 'fs'; - -import glob from 'glob'; -import { safeLoad } from 'js-yaml'; - -import type { FieldSpec } from 'src/plugins/data/common'; - -import type { Fields, Field } from '../../fields/field'; - -import { - flattenFields, - dedupeFields, - transformField, - findFieldByPath, - createFieldFormatMap, - createIndexPatternFields, - createIndexPattern, -} from './install'; -import { dupeFields } from './tests/test_data'; - -// Add our own serialiser to just do JSON.stringify -expect.addSnapshotSerializer({ - print(val) { - return JSON.stringify(val, null, 2); - }, - - test(val) { - return val; - }, -}); -const files = glob.sync(path.join(__dirname, '/tests/*.yml')); -let fields: Fields = []; -for (const file of files) { - const fieldsYML = readFileSync(file, 'utf-8'); - fields = fields.concat(safeLoad(fieldsYML)); -} - -describe('creating index patterns from yaml fields', () => { - interface Test { - fields: Field[]; - expect: string | number | boolean | undefined; - } - - const name = 'testField'; - - test('createIndexPatternFields function creates Kibana index pattern fields and fieldFormatMap', () => { - const indexPatternFields = createIndexPatternFields(fields); - expect(indexPatternFields).toMatchSnapshot('createIndexPatternFields'); - }); - - test('createIndexPattern function creates Kibana index pattern', () => { - const indexPattern = createIndexPattern('logs', fields); - expect(indexPattern).toMatchSnapshot('createIndexPattern'); - }); - - describe('flattenFields function flattens recursively and handles copying alias fields', () => { - test('a field of type group with no nested fields is skipped', () => { - const flattened = flattenFields([{ name: 'nginx', type: 'group' }]); - expect(flattened.length).toBe(0); - }); - test('flattenFields matches snapshot', () => { - const flattened = flattenFields(fields); - expect(flattened).toMatchSnapshot('flattenFields'); - }); - }); - - describe('dedupFields', () => { - const deduped = dedupeFields(dupeFields); - const checkIfDup = (field: Field) => { - return deduped.filter((item) => item.name === field.name); - }; - test('there there is one field object with name of "1"', () => { - expect(checkIfDup({ name: '1' }).length).toBe(1); - }); - test('there there is one field object with name of "1.1"', () => { - expect(checkIfDup({ name: '1.1' }).length).toBe(1); - }); - test('there there is one field object with name of "2"', () => { - expect(checkIfDup({ name: '2' }).length).toBe(1); - }); - test('there there is one field object with name of "4"', () => { - expect(checkIfDup({ name: '4' }).length).toBe(1); - }); - // existing field takes precendence - test('the new merged field has correct attributes', () => { - const mergedField = deduped.find((field) => field.name === '1'); - expect(mergedField?.searchable).toBe(true); - expect(mergedField?.aggregatable).toBe(true); - expect(mergedField?.count).toBe(0); - }); - }); - - describe('getFieldByPath searches recursively for field in fields given dot separated path', () => { - const searchFields: Fields = [ - { - name: '1', - fields: [ - { - name: '1-1', - }, - { - name: '1-2', - }, - ], - }, - { - name: '2', - fields: [ - { - name: '2-1', - }, - { - name: '2-2', - fields: [ - { - name: '2-2-1', - }, - { - name: '2-2-2', - }, - ], - }, - ], - }, - ]; - test('returns undefined when the field does not exist', () => { - expect(findFieldByPath(searchFields, '0')).toBe(undefined); - }); - test('returns undefined if the field is not a leaf node', () => { - expect(findFieldByPath(searchFields, '1')?.name).toBe(undefined); - }); - test('returns undefined searching for a nested field that does not exist', () => { - expect(findFieldByPath(searchFields, '1.1-3')?.name).toBe(undefined); - }); - test('returns nested field that is a leaf node', () => { - expect(findFieldByPath(searchFields, '2.2-2.2-2-1')?.name).toBe('2-2-1'); - }); - }); - - test('transformField maps field types to kibana index pattern data types', () => { - const tests: Test[] = [ - { fields: [{ name: 'testField' }], expect: 'string' }, - { fields: [{ name: 'testField', type: 'half_float' }], expect: 'number' }, - { fields: [{ name: 'testField', type: 'scaled_float' }], expect: 'number' }, - { fields: [{ name: 'testField', type: 'float' }], expect: 'number' }, - { fields: [{ name: 'testField', type: 'integer' }], expect: 'number' }, - { fields: [{ name: 'testField', type: 'long' }], expect: 'number' }, - { fields: [{ name: 'testField', type: 'short' }], expect: 'number' }, - { fields: [{ name: 'testField', type: 'byte' }], expect: 'number' }, - { fields: [{ name: 'testField', type: 'keyword' }], expect: 'string' }, - { fields: [{ name: 'testField', type: 'invalidType' }], expect: 'string' }, - { fields: [{ name: 'testField', type: 'text' }], expect: 'string' }, - { fields: [{ name: 'testField', type: 'date' }], expect: 'date' }, - { fields: [{ name: 'testField', type: 'geo_point' }], expect: 'geo_point' }, - { fields: [{ name: 'testField', type: 'constant_keyword' }], expect: 'string' }, - ]; - - tests.forEach((test) => { - const res = test.fields.map(transformField); - expect(res[0].type).toBe(test.expect); - }); - }); - - test('transformField changes values based on other values', () => { - interface TestWithAttr extends Test { - attr: keyof FieldSpec; - } - - const tests: TestWithAttr[] = [ - // count - { fields: [{ name }], expect: 0, attr: 'count' }, - { fields: [{ name, count: 4 }], expect: 4, attr: 'count' }, - - // searchable - { fields: [{ name }], expect: true, attr: 'searchable' }, - { fields: [{ name, searchable: true }], expect: true, attr: 'searchable' }, - { fields: [{ name, searchable: false }], expect: false, attr: 'searchable' }, - { fields: [{ name, type: 'binary' }], expect: false, attr: 'searchable' }, - { fields: [{ name, searchable: true, type: 'binary' }], expect: false, attr: 'searchable' }, - { - fields: [{ name, searchable: true, type: 'object', enabled: false }], - expect: false, - attr: 'searchable', - }, - - // aggregatable - { fields: [{ name }], expect: true, attr: 'aggregatable' }, - { fields: [{ name, aggregatable: true }], expect: true, attr: 'aggregatable' }, - { fields: [{ name, aggregatable: false }], expect: false, attr: 'aggregatable' }, - { fields: [{ name, type: 'binary' }], expect: false, attr: 'aggregatable' }, - { - fields: [{ name, aggregatable: true, type: 'binary' }], - expect: false, - attr: 'aggregatable', - }, - { fields: [{ name, type: 'keyword' }], expect: true, attr: 'aggregatable' }, - { fields: [{ name, type: 'constant_keyword' }], expect: true, attr: 'aggregatable' }, - { fields: [{ name, type: 'text', aggregatable: true }], expect: false, attr: 'aggregatable' }, - { fields: [{ name, type: 'text' }], expect: false, attr: 'aggregatable' }, - { - fields: [{ name, aggregatable: true, type: 'object', enabled: false }], - expect: false, - attr: 'aggregatable', - }, - - // indexed - { fields: [{ name, type: 'binary' }], expect: false, attr: 'indexed' }, - { - fields: [{ name, index: true, type: 'binary' }], - expect: false, - attr: 'indexed', - }, - { - fields: [{ name, index: true, type: 'object', enabled: false }], - expect: false, - attr: 'indexed', - }, - - // script, scripted - { fields: [{ name }], expect: false, attr: 'scripted' }, - { fields: [{ name }], expect: undefined, attr: 'script' }, - { fields: [{ name, script: 'doc[]' }], expect: true, attr: 'scripted' }, - { fields: [{ name, script: 'doc[]' }], expect: 'doc[]', attr: 'script' }, - - // lang - { fields: [{ name }], expect: undefined, attr: 'lang' }, - { fields: [{ name, script: 'doc[]' }], expect: 'painless', attr: 'lang' }, - ]; - tests.forEach((test) => { - const res = test.fields.map(transformField); - expect(res[0][test.attr]).toBe(test.expect); - }); - }); - - describe('createFieldFormatMap creates correct map based on inputs', () => { - test('field with no format or pattern have empty fieldFormatMap', () => { - const fieldsToFormat = [{ name: 'fieldName', input_format: 'inputFormatVal' }]; - const fieldFormatMap = createFieldFormatMap(fieldsToFormat); - expect(fieldFormatMap).toEqual({}); - }); - test('field with pattern and no format creates fieldFormatMap with no id', () => { - const fieldsToFormat = [ - { name: 'fieldName', pattern: 'patternVal', input_format: 'inputFormatVal' }, - ]; - const fieldFormatMap = createFieldFormatMap(fieldsToFormat); - const expectedFieldFormatMap = { - fieldName: { - params: { - pattern: 'patternVal', - inputFormat: 'inputFormatVal', - }, - }, - }; - expect(fieldFormatMap).toEqual(expectedFieldFormatMap); - }); - - test('field with format and params creates fieldFormatMap with id', () => { - const fieldsToFormat = [ - { - name: 'fieldName', - format: 'formatVal', - pattern: 'patternVal', - input_format: 'inputFormatVal', - }, - ]; - const fieldFormatMap = createFieldFormatMap(fieldsToFormat); - const expectedFieldFormatMap = { - fieldName: { - id: 'formatVal', - params: { - pattern: 'patternVal', - inputFormat: 'inputFormatVal', - }, - }, - }; - expect(fieldFormatMap).toEqual(expectedFieldFormatMap); - }); - - test('all variations and all the params get passed through', () => { - const fieldsToFormat = [ - { name: 'fieldPattern', pattern: 'patternVal' }, - { name: 'fieldFormat', format: 'formatVal' }, - { name: 'fieldFormatWithParam', format: 'formatVal', output_precision: 2 }, - { name: 'fieldFormatAndPattern', format: 'formatVal', pattern: 'patternVal' }, - { - name: 'fieldFormatAndAllParams', - format: 'formatVal', - pattern: 'pattenVal', - input_format: 'inputFormatVal', - output_format: 'outputFormalVal', - output_precision: 3, - label_template: 'labelTemplateVal', - url_template: 'urlTemplateVal', - openLinkInCurrentTab: true, - }, - ]; - const fieldFormatMap = createFieldFormatMap(fieldsToFormat); - expect(fieldFormatMap).toMatchSnapshot('createFieldFormatMap'); - }); - }); -}); diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts index 61d6f6ed8818a..c42029f2c453d 100644 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.ts @@ -5,380 +5,58 @@ * 2.0. */ -import type { SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server'; -import type { FieldSpec } from 'src/plugins/data/common'; +import type { SavedObjectsClientContract } from 'src/core/server'; -import { loadFieldsFromYaml } from '../../fields/field'; -import type { Fields, Field } from '../../fields/field'; import { dataTypes, installationStatuses } from '../../../../../common/constants'; -import type { - ArchivePackage, - Installation, - InstallSource, - ValueOf, -} from '../../../../../common/types'; import { appContextService } from '../../../../services'; -import type { RegistryPackage, DataType } from '../../../../types'; -import { getInstallation, getPackageFromSource, getPackageSavedObjects } from '../../packages/get'; - -interface FieldFormatMap { - [key: string]: FieldFormatMapItem; -} -interface FieldFormatMapItem { - id?: string; - params?: FieldFormatParams; -} -interface FieldFormatParams { - pattern?: string; - inputFormat?: string; - outputFormat?: string; - outputPrecision?: number; - labelTemplate?: string; - urlTemplate?: string; - openLinkInCurrentTab?: boolean; -} -/* this should match https://github.com/elastic/beats/blob/d9a4c9c240a9820fab15002592e5bb6db318543b/libbeat/kibana/fields_transformer.go */ -interface TypeMap { - [key: string]: string; -} -const typeMap: TypeMap = { - binary: 'binary', - half_float: 'number', - scaled_float: 'number', - float: 'number', - integer: 'number', - long: 'number', - short: 'number', - byte: 'number', - text: 'string', - keyword: 'string', - '': 'string', - geo_point: 'geo_point', - date: 'date', - ip: 'ip', - boolean: 'boolean', - constant_keyword: 'string', -}; - +import { getPackageSavedObjects } from '../../packages/get'; const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern'; export const indexPatternTypes = Object.values(dataTypes); -export async function installIndexPatterns({ - savedObjectsClient, - pkgName, - pkgVersion, - installSource, -}: { - savedObjectsClient: SavedObjectsClientContract; - esClient: ElasticsearchClient; - pkgName?: string; - pkgVersion?: string; - installSource?: InstallSource; -}) { - const logger = appContextService.getLogger(); - logger.debug( - `kicking off installation of index patterns for ${ - pkgName && pkgVersion ? `${pkgName}-${pkgVersion}` : 'no specific package' - }` - ); +export function getIndexPatternSavedObjects() { + return indexPatternTypes.map((indexPatternType) => ({ + id: `${indexPatternType}-*`, + type: INDEX_PATTERN_SAVED_OBJECT_TYPE, + attributes: { + title: `${indexPatternType}-*`, + timeFieldName: '@timestamp', + allowNoIndex: true, + }, + })); +} +export async function removeUnusedIndexPatterns(savedObjectsClient: SavedObjectsClientContract) { + const logger = appContextService.getLogger(); // get all user installed packages const installedPackagesRes = await getPackageSavedObjects(savedObjectsClient); const installedPackagesSavedObjects = installedPackagesRes.saved_objects.filter( (so) => so.attributes.install_status === installationStatuses.Installed ); - const packagesToFetch = installedPackagesSavedObjects.reduce< - Array<{ name: string; version: string; installedPkg: Installation | undefined }> - >((acc, pkg) => { - acc.push({ - name: pkg.attributes.name, - version: pkg.attributes.version, - installedPkg: pkg.attributes, - }); - return acc; - }, []); - - if (pkgName && pkgVersion && installSource) { - const packageToInstall = packagesToFetch.find((pkg) => pkg.name === pkgName); - if (packageToInstall) { - // set the version to the one we want to install - // if we're reinstalling the number will be the same - // if this is an upgrade then we'll be modifying the version number to the upgrade version - packageToInstall.version = pkgVersion; - } else { - // if we're installing for the first time, add to the list - packagesToFetch.push({ - name: pkgName, - version: pkgVersion, - installedPkg: await getInstallation({ savedObjectsClient, pkgName }), - }); - } + if (installedPackagesSavedObjects.length > 0) { + return []; } - // get each package's registry info - const packagesToFetchPromise = packagesToFetch.map((pkg) => - getPackageFromSource({ - pkgName: pkg.name, - pkgVersion: pkg.version, - installedPkg: pkg.installedPkg, - savedObjectsClient, - }) + const patternsToDelete = indexPatternTypes.map((indexPatternType) => `${indexPatternType}-*`); + + const { resolved_objects: resolvedObjects } = await savedObjectsClient.bulkResolve( + patternsToDelete.map((pattern) => ({ id: pattern, type: INDEX_PATTERN_SAVED_OBJECT_TYPE })) ); - const packages = await Promise.all(packagesToFetchPromise); - // for each index pattern type, create an index pattern + // eslint-disable-next-line @typescript-eslint/naming-convention + const idsToDelete = resolvedObjects.map(({ saved_object }) => saved_object.id); + return Promise.all( - indexPatternTypes.map(async (indexPatternType) => { - // if this is an update because a package is being uninstalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern - if (!pkgName && installedPackagesSavedObjects.length === 0) { - try { - logger.debug(`deleting index pattern ${indexPatternType}-*`); - await savedObjectsClient.delete(INDEX_PATTERN_SAVED_OBJECT_TYPE, `${indexPatternType}-*`); - } catch (err) { - // index pattern was probably deleted by the user already - } - return; + idsToDelete.map(async (id) => { + try { + logger.debug(`deleting index pattern ${id}`); + await savedObjectsClient.delete(INDEX_PATTERN_SAVED_OBJECT_TYPE, id); + } catch (err) { + // index pattern was probably deleted by the user already + logger.debug(`Non fatal error encountered deleting index pattern ${id} : ${err}`); } - const packagesWithInfo = packages.map((pkg) => pkg.packageInfo); - // get all data stream fields from all installed packages - const fields = await getAllDataStreamFieldsByType(packagesWithInfo, indexPatternType); - const kibanaIndexPattern = createIndexPattern(indexPatternType, fields); - - // create or overwrite the index pattern - await savedObjectsClient.create(INDEX_PATTERN_SAVED_OBJECT_TYPE, kibanaIndexPattern, { - id: `${indexPatternType}-*`, - overwrite: true, - }); - logger.debug(`created index pattern ${kibanaIndexPattern.title}`); + return; }) ); } - -// loops through all given packages and returns an array -// of all fields from all data streams matching data stream type -export const getAllDataStreamFieldsByType = async ( - packages: Array, - dataStreamType: ValueOf -): Promise => { - const dataStreamsPromises = packages.reduce>>((acc, pkg) => { - if (pkg.data_streams) { - // filter out data streams by data stream type - const matchingDataStreams = pkg.data_streams.filter( - (dataStream) => dataStream.type === dataStreamType - ); - matchingDataStreams.forEach((dataStream) => { - acc.push(loadFieldsFromYaml(pkg, dataStream.path)); - }); - } - return acc; - }, []); - - // get all the data stream fields for each installed package into one array - const allDataStreamFields: Fields[] = await Promise.all(dataStreamsPromises); - return allDataStreamFields.flat(); -}; - -// creates or updates index pattern -export const createIndexPattern = (indexPatternType: string, fields: Fields) => { - const { indexPatternFields, fieldFormatMap } = createIndexPatternFields(fields); - - return { - title: `${indexPatternType}-*`, - timeFieldName: '@timestamp', - fields: JSON.stringify(indexPatternFields), - fieldFormatMap: JSON.stringify(fieldFormatMap), - allowNoIndex: true, - }; -}; - -// takes fields from yaml files and transforms into Kibana Index Pattern fields -// and also returns the fieldFormatMap -export const createIndexPatternFields = ( - fields: Fields -): { indexPatternFields: FieldSpec[]; fieldFormatMap: FieldFormatMap } => { - const flattenedFields = flattenFields(fields); - const fieldFormatMap = createFieldFormatMap(flattenedFields); - const transformedFields = flattenedFields.map(transformField); - const dedupedFields = dedupeFields(transformedFields); - return { indexPatternFields: dedupedFields, fieldFormatMap }; -}; - -// merges fields that are duplicates with the existing taking precedence -export const dedupeFields = (fields: FieldSpec[]) => { - const uniqueObj = fields.reduce<{ [name: string]: FieldSpec }>((acc, field) => { - // if field doesn't exist yet - if (!acc[field.name]) { - acc[field.name] = field; - // if field exists already - } else { - const existingField = acc[field.name]; - // if the existing field and this field have the same type, merge - if (existingField.type === field.type) { - const mergedField = { ...field, ...existingField }; - acc[field.name] = mergedField; - } else { - // log when there is a dup with different types - } - } - return acc; - }, {}); - - return Object.values(uniqueObj); -}; - -/** - * search through fields with field's path property - * returns undefined if field not found or field is not a leaf node - * @param allFields fields to search - * @param path dot separated path from field.path - */ -export const findFieldByPath = (allFields: Fields, path: string): Field | undefined => { - const pathParts = path.split('.'); - return getField(allFields, pathParts); -}; - -const getField = (fields: Fields, pathNames: string[]): Field | undefined => { - if (!pathNames.length) return undefined; - // get the first rest of path names - const [name, ...restPathNames] = pathNames; - for (const field of fields) { - if (field.name === name) { - // check field's fields, passing in the remaining path names - if (field.fields && field.fields.length > 0) { - return getField(field.fields, restPathNames); - } - // no nested fields to search, but still more names - not found - if (restPathNames.length) { - return undefined; - } - return field; - } - } - return undefined; -}; - -export const transformField = (field: Field, i: number, fields: Fields): FieldSpec => { - const newField: FieldSpec = { - name: field.name, - type: field.type && typeMap[field.type] ? typeMap[field.type] : 'string', - count: field.count ?? 0, - scripted: false, - indexed: field.index ?? true, - searchable: field.searchable ?? true, - aggregatable: field.aggregatable ?? true, - readFromDocValues: field.doc_values ?? true, - }; - - if (newField.type === 'binary') { - newField.aggregatable = false; - newField.readFromDocValues = field.doc_values ?? false; - newField.indexed = false; - newField.searchable = false; - } - - if (field.type === 'object' && field.hasOwnProperty('enabled')) { - const enabled = field.enabled ?? true; - if (!enabled) { - newField.aggregatable = false; - newField.readFromDocValues = false; - newField.indexed = false; - newField.searchable = false; - } - } - - if (field.type === 'text') { - newField.aggregatable = false; - } - - if (field.hasOwnProperty('script')) { - newField.scripted = true; - newField.script = field.script; - newField.lang = 'painless'; - newField.readFromDocValues = false; - } - - return newField; -}; - -/** - * flattenFields - * - * flattens fields and renames them with a path of the parent names - */ - -export const flattenFields = (allFields: Fields): Fields => { - const flatten = (fields: Fields): Fields => - fields.reduce((acc, field) => { - // if this is a group fields with no fields, skip the field - if (field.type === 'group' && !field.fields?.length) { - return acc; - } - // recurse through nested fields - if (field.type === 'group' && field.fields?.length) { - // skip if field.enabled is not explicitly set to false - if (!field.hasOwnProperty('enabled') || field.enabled === true) { - acc = renameAndFlatten(field, field.fields, [...acc]); - } - } else { - // handle alias type fields - if (field.type === 'alias' && field.path) { - const foundField = findFieldByPath(allFields, field.path); - // if aliased leaf field is found copy its props over except path and name - if (foundField) { - const { path, name } = field; - field = { ...foundField, path, name }; - } - } - // add field before going through multi_fields because we still want to add the parent field - acc.push(field); - - // for each field in multi_field add new field - if (field.multi_fields?.length) { - acc = renameAndFlatten(field, field.multi_fields, [...acc]); - } - } - return acc; - }, []); - - // helper function to call flatten() and rename the fields - const renameAndFlatten = (field: Field, fields: Fields, acc: Fields): Fields => { - const flattenedFields = flatten(fields); - flattenedFields.forEach((nestedField) => { - acc.push({ - ...nestedField, - name: `${field.name}.${nestedField.name}`, - }); - }); - return acc; - }; - - return flatten(allFields); -}; - -export const createFieldFormatMap = (fields: Fields): FieldFormatMap => - fields.reduce((acc, field) => { - if (field.format || field.pattern) { - const fieldFormatMapItem: FieldFormatMapItem = {}; - if (field.format) { - fieldFormatMapItem.id = field.format; - } - const params = getFieldFormatParams(field); - if (Object.keys(params).length) fieldFormatMapItem.params = params; - acc[field.name] = fieldFormatMapItem; - } - return acc; - }, {}); - -const getFieldFormatParams = (field: Field): FieldFormatParams => { - const params: FieldFormatParams = {}; - if (field.pattern) params.pattern = field.pattern; - if (field.input_format) params.inputFormat = field.input_format; - if (field.output_format) params.outputFormat = field.output_format; - if (field.output_precision) params.outputPrecision = field.output_precision; - if (field.label_template) params.labelTemplate = field.label_template; - if (field.url_template) params.urlTemplate = field.url_template; - if (field.open_link_in_current_tab) params.openLinkInCurrentTab = field.open_link_in_current_tab; - return params; -}; diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/coredns.logs.yml b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/coredns.logs.yml deleted file mode 100644 index d66a4cf62bc41..0000000000000 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/coredns.logs.yml +++ /dev/null @@ -1,71 +0,0 @@ -- name: coredns - type: group - description: > - coredns fields after normalization - fields: - - name: id - type: keyword - description: > - id of the DNS transaction - - - name: allParams - type: integer - format: bytes - pattern: patternValQueryWeight - input_format: inputFormatVal, - output_format: outputFormalVal, - output_precision: 3, - label_template: labelTemplateVal, - url_template: urlTemplateVal, - openLinkInCurrentTab: true, - description: > - weight of the DNS query - - - name: query.length - type: integer - pattern: patternValQueryLength - description: > - length of the DNS query - - - name: query.size - type: integer - format: bytes - pattern: patternValQuerySize - description: > - size of the DNS query - - - name: query.class - type: keyword - description: > - DNS query class - - - name: query.name - type: keyword - description: > - DNS query name - - - name: query.type - type: keyword - description: > - DNS query type - - - name: response.code - type: keyword - description: > - DNS response code - - - name: response.flags - type: keyword - description: > - DNS response flags - - - name: response.size - type: integer - format: bytes - description: > - size of the DNS response - - - name: dnssec_ok - type: boolean - description: > - dnssec flag diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/nginx.access.ecs.yml b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/nginx.access.ecs.yml deleted file mode 100644 index 51090a0fe7cf0..0000000000000 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/nginx.access.ecs.yml +++ /dev/null @@ -1,112 +0,0 @@ -- name: '@timestamp' - level: core - required: true - type: date - description: 'Date/time when the event originated. - This is the date/time extracted from the event, typically representing when - the event was generated by the source. - If the event source has no original timestamp, this value is typically populated - by the first time the event was received by the pipeline. - Required field for all events.' - example: '2016-05-23T08:05:34.853Z' -- name: labels - level: core - type: object - object_type: keyword - description: 'Custom key/value pairs. - Can be used to add meta information to events. Should not contain nested objects. - All values are stored as keyword. - Example: `docker` and `k8s` labels.' - example: - application: foo-bar - env: production -- name: message - level: core - type: text - description: 'For log events the message field contains the log message, optimized - for viewing in a log viewer. - For structured logs without an original message field, other fields can be concatenated - to form a human-readable summary of the event. - If multiple messages exist, they can be combined into one message.' - example: Hello World -- name: tags - level: core - type: keyword - ignore_above: 1024 - description: List of keywords used to tag each event. - example: '["production", "env2"]' -- name: agent - title: Agent - group: 2 - description: 'The agent fields contain the data about the software entity, if - any, that collects, detects, or observes events on a host, or takes measurements - on a host. - Examples include Beats. Agents may also run on observers. ECS agent.* fields - shall be populated with details of the agent running on the host or observer - where the event happened or the measurement was taken.' - footnote: 'Examples: In the case of Beats for logs, the agent.name is filebeat. - For APM, it is the agent running in the app/service. The agent information does - not change if data is sent through queuing systems like Kafka, Redis, or processing - systems such as Logstash or APM Server.' - type: group - fields: - - name: ephemeral_id - level: extended - type: keyword - ignore_above: 1024 - description: 'Ephemeral identifier of this agent (if one exists). - This id normally changes across restarts, but `agent.id` does not.' - example: 8a4f500f - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique identifier of this agent (if one exists). - Example: For Beats this would be beat.id.' - example: 8a4f500d - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Custom name of the agent. - This is a name that can be given to an agent. This can be helpful if for example - two Filebeat instances are running on the same host but a human readable separation - is needed on which Filebeat instance data is coming from. - If no name is given, the name is often left empty.' - example: foo - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of the agent. - The agent type stays always the same and should be given by the agent used. - In case of Filebeat the agent would always be Filebeat also if two Filebeat - instances are run on the same machine.' - example: filebeat - - name: version - level: core - type: keyword - ignore_above: 1024 - description: Version of the agent. - example: 6.0.0-rc2 -- name: as - title: Autonomous System - group: 2 - description: An autonomous system (AS) is a collection of connected Internet Protocol - (IP) routing prefixes under the control of one or more network operators on - behalf of a single administrative entity or domain that presents a common, clearly - defined routing policy to the internet. - type: group - fields: - - name: number - level: extended - type: long - description: Unique number allocated to the autonomous system. The autonomous - system number (ASN) uniquely identifies each network on the Internet. - example: 15169 - - name: organization.name - level: extended - type: keyword - ignore_above: 1024 - description: Organization name. - example: Google LLC \ No newline at end of file diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/nginx.error.ecs.yml b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/nginx.error.ecs.yml deleted file mode 100644 index 51090a0fe7cf0..0000000000000 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/nginx.error.ecs.yml +++ /dev/null @@ -1,112 +0,0 @@ -- name: '@timestamp' - level: core - required: true - type: date - description: 'Date/time when the event originated. - This is the date/time extracted from the event, typically representing when - the event was generated by the source. - If the event source has no original timestamp, this value is typically populated - by the first time the event was received by the pipeline. - Required field for all events.' - example: '2016-05-23T08:05:34.853Z' -- name: labels - level: core - type: object - object_type: keyword - description: 'Custom key/value pairs. - Can be used to add meta information to events. Should not contain nested objects. - All values are stored as keyword. - Example: `docker` and `k8s` labels.' - example: - application: foo-bar - env: production -- name: message - level: core - type: text - description: 'For log events the message field contains the log message, optimized - for viewing in a log viewer. - For structured logs without an original message field, other fields can be concatenated - to form a human-readable summary of the event. - If multiple messages exist, they can be combined into one message.' - example: Hello World -- name: tags - level: core - type: keyword - ignore_above: 1024 - description: List of keywords used to tag each event. - example: '["production", "env2"]' -- name: agent - title: Agent - group: 2 - description: 'The agent fields contain the data about the software entity, if - any, that collects, detects, or observes events on a host, or takes measurements - on a host. - Examples include Beats. Agents may also run on observers. ECS agent.* fields - shall be populated with details of the agent running on the host or observer - where the event happened or the measurement was taken.' - footnote: 'Examples: In the case of Beats for logs, the agent.name is filebeat. - For APM, it is the agent running in the app/service. The agent information does - not change if data is sent through queuing systems like Kafka, Redis, or processing - systems such as Logstash or APM Server.' - type: group - fields: - - name: ephemeral_id - level: extended - type: keyword - ignore_above: 1024 - description: 'Ephemeral identifier of this agent (if one exists). - This id normally changes across restarts, but `agent.id` does not.' - example: 8a4f500f - - name: id - level: core - type: keyword - ignore_above: 1024 - description: 'Unique identifier of this agent (if one exists). - Example: For Beats this would be beat.id.' - example: 8a4f500d - - name: name - level: core - type: keyword - ignore_above: 1024 - description: 'Custom name of the agent. - This is a name that can be given to an agent. This can be helpful if for example - two Filebeat instances are running on the same host but a human readable separation - is needed on which Filebeat instance data is coming from. - If no name is given, the name is often left empty.' - example: foo - - name: type - level: core - type: keyword - ignore_above: 1024 - description: 'Type of the agent. - The agent type stays always the same and should be given by the agent used. - In case of Filebeat the agent would always be Filebeat also if two Filebeat - instances are run on the same machine.' - example: filebeat - - name: version - level: core - type: keyword - ignore_above: 1024 - description: Version of the agent. - example: 6.0.0-rc2 -- name: as - title: Autonomous System - group: 2 - description: An autonomous system (AS) is a collection of connected Internet Protocol - (IP) routing prefixes under the control of one or more network operators on - behalf of a single administrative entity or domain that presents a common, clearly - defined routing policy to the internet. - type: group - fields: - - name: number - level: extended - type: long - description: Unique number allocated to the autonomous system. The autonomous - system number (ASN) uniquely identifies each network on the Internet. - example: 15169 - - name: organization.name - level: extended - type: keyword - ignore_above: 1024 - description: Organization name. - example: Google LLC \ No newline at end of file diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/nginx.fields.yml b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/nginx.fields.yml deleted file mode 100644 index 7c2e721d564e7..0000000000000 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/nginx.fields.yml +++ /dev/null @@ -1,120 +0,0 @@ -- name: nginx.access - type: group - description: > - Contains fields for the Nginx access logs. - fields: - - name: group_disabled - type: group - enabled: false - fields: - - name: message - type: text - - name: remote_ip_list - type: array - description: > - An array of remote IP addresses. It is a list because it is common to include, besides the client - IP address, IP addresses from headers like `X-Forwarded-For`. - Real source IP is restored to `source.ip`. - - - name: body_sent.bytes - type: alias - path: http.response.body.bytes - migration: true - - name: user_name - type: alias - path: user.name - migration: true - - name: method - type: alias - path: http.request.method - migration: true - - name: url - type: alias - path: url.original - migration: true - - name: http_version - type: alias - path: http.version - migration: true - - name: response_code - type: alias - path: http.response.status_code - migration: true - - name: referrer - type: alias - path: http.request.referrer - migration: true - - name: agent - type: alias - path: user_agent.original - migration: true - - - name: user_agent - type: group - fields: - - name: device - type: alias - path: user_agent.device.name - migration: true - - name: name - type: alias - path: user_agent.name - migration: true - - name: os - type: alias - path: user_agent.os.full_name - migration: true - - name: os_name - type: alias - path: user_agent.os.name - migration: true - - name: original - type: alias - path: user_agent.original - migration: true - - - name: geoip - type: group - fields: - - name: continent_name - type: alias - path: source.geo.continent_name - migration: true - - name: country_iso_code - type: alias - path: source.geo.country_iso_code - migration: true - - name: location - type: alias - path: source.geo.location - migration: true - - name: region_name - type: alias - path: source.geo.region_name - migration: true - - name: city_name - type: alias - path: source.geo.city_name - migration: true - - name: region_iso_code - type: alias - path: source.geo.region_iso_code - migration: true - -- name: source - type: group - fields: - - name: geo - type: group - fields: - - name: continent_name - type: text -- name: country - type: "" - multi_fields: - - name: keyword - type: keyword - - name: text - type: text -- name: nginx - type: group \ No newline at end of file diff --git a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/test_data.ts b/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/test_data.ts deleted file mode 100644 index d9bcf36651081..0000000000000 --- a/x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/tests/test_data.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { FieldSpec } from 'src/plugins/data/common'; - -export const dupeFields: FieldSpec[] = [ - { - name: '1', - type: 'integer', - searchable: true, - aggregatable: true, - count: 0, - indexed: true, - readFromDocValues: true, - scripted: false, - }, - { - name: '2', - type: 'integer', - searchable: true, - aggregatable: true, - count: 0, - indexed: true, - readFromDocValues: true, - scripted: false, - }, - { - name: '3', - type: 'integer', - searchable: true, - aggregatable: true, - count: 0, - indexed: true, - readFromDocValues: true, - scripted: false, - }, - { - name: '1', - type: 'integer', - searchable: false, - aggregatable: false, - count: 2, - indexed: true, - readFromDocValues: true, - scripted: false, - }, - { - name: '1.1', - type: 'integer', - searchable: false, - aggregatable: false, - count: 0, - indexed: true, - readFromDocValues: true, - scripted: false, - }, - { - name: '4', - type: 'integer', - searchable: false, - aggregatable: false, - count: 0, - indexed: true, - readFromDocValues: true, - scripted: false, - }, - { - name: '2', - type: 'integer', - searchable: false, - aggregatable: false, - count: 0, - indexed: true, - readFromDocValues: true, - scripted: false, - }, - { - name: '1', - type: 'integer', - searchable: false, - aggregatable: false, - count: 1, - indexed: true, - readFromDocValues: true, - scripted: false, - }, -]; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts index 5ee0f57b6e03a..dbec18851cfc9 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.test.ts @@ -20,7 +20,6 @@ jest.mock('./get'); import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; import { installKibanaAssets } from '../kibana/assets/install'; -import { installIndexPatterns } from '../kibana/index_pattern/install'; import { _installPackage } from './_install_package'; @@ -30,9 +29,6 @@ const mockedUpdateCurrentWriteIndices = updateCurrentWriteIndices as jest.Mocked const mockedGetKibanaAssets = installKibanaAssets as jest.MockedFunction< typeof installKibanaAssets >; -const mockedInstallIndexPatterns = installIndexPatterns as jest.MockedFunction< - typeof installIndexPatterns ->; function sleep(millis: number) { return new Promise((resolve) => setTimeout(resolve, millis)); @@ -50,14 +46,11 @@ describe('_installPackage', () => { afterEach(async () => { appContextService.stop(); }); - it('handles errors from installIndexPatterns or installKibanaAssets', async () => { - // force errors from either/both these functions + it('handles errors from installKibanaAssets', async () => { + // force errors from this function mockedGetKibanaAssets.mockImplementation(async () => { throw new Error('mocked async error A: should be caught'); }); - mockedInstallIndexPatterns.mockImplementation(async () => { - throw new Error('mocked async error B: should be caught'); - }); // pick any function between when those are called and when await Promise.all is defined later // and force it to take long enough for the errors to occur @@ -66,6 +59,8 @@ describe('_installPackage', () => { const installationPromise = _installPackage({ savedObjectsClient: soClient, + // @ts-ignore + savedObjectsImporter: jest.fn(), esClient, logger: loggerMock.create(), paths: [], diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index e2027a99463fc..ac0c7e1729913 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -10,6 +10,7 @@ import type { Logger, SavedObject, SavedObjectsClientContract, + SavedObjectsImporter, } from 'src/core/server'; import { @@ -36,7 +37,6 @@ import { installMlModel } from '../elasticsearch/ml_model/'; import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install'; import { saveArchiveEntries } from '../archive/storage'; import { ConcurrentInstallOperationError } from '../../../errors'; - import { packagePolicyService } from '../..'; import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install'; @@ -48,6 +48,7 @@ import { deleteKibanaSavedObjectsAssets } from './remove'; export async function _installPackage({ savedObjectsClient, + savedObjectsImporter, esClient, logger, installedPkg, @@ -57,6 +58,7 @@ export async function _installPackage({ installSource, }: { savedObjectsClient: SavedObjectsClientContract; + savedObjectsImporter: Pick; esClient: ElasticsearchClient; logger: Logger; installedPkg?: SavedObject; @@ -100,21 +102,6 @@ export async function _installPackage({ }); } - // kick off `installKibanaAssets` as early as possible because they're the longest running operations - // we don't `await` here because we don't want to delay starting the many other `install*` functions - // however, without an `await` or a `.catch` we haven't defined how to handle a promise rejection - // we define it many lines and potentially seconds of wall clock time later in - // `await installKibanaAssetsPromise` - // if we encounter an error before we there, we'll have an "unhandled rejection" which causes its own problems - // the program will log something like this _and exit/crash_ - // Unhandled Promise rejection detected: - // RegistryResponseError or some other error - // Terminating process... - // server crashed with status code 1 - // - // add a `.catch` to prevent the "unhandled rejection" case - // in that `.catch`, set something that indicates a failure - // check for that failure later and act accordingly (throw, ignore, return) const kibanaAssets = await getKibanaAssets(paths); if (installedPkg) await deleteKibanaSavedObjectsAssets( @@ -127,12 +114,13 @@ export async function _installPackage({ pkgName, kibanaAssets ); - let installKibanaAssetsError; - const installKibanaAssetsPromise = installKibanaAssets({ - savedObjectsClient, + + await installKibanaAssets({ + logger, + savedObjectsImporter, pkgName, kibanaAssets, - }).catch((reason) => (installKibanaAssetsError = reason)); + }); // the rest of the installation must happen in sequential order // currently only the base package has an ILM policy @@ -211,10 +199,6 @@ export async function _installPackage({ } const installedTemplateRefs = getAllTemplateRefs(installedTemplates); - // make sure the assets are installed (or didn't error) - if (installKibanaAssetsError) throw installKibanaAssetsError; - await installKibanaAssetsPromise; - const packageAssetResults = await saveArchiveEntries({ savedObjectsClient, paths, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts index c77e2a0a22a0a..8a7fb9ae005d0 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/bulk_install_packages.ts @@ -9,7 +9,6 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/s import { appContextService } from '../../app_context'; import * as Registry from '../registry'; -import { installIndexPatterns } from '../kibana/index_pattern/install'; import type { InstallResult } from '../../../types'; @@ -71,7 +70,6 @@ export async function bulkInstallPackages({ esClient, pkgkey: Registry.pkgToPkgKey(pkgKeyProps), installSource, - skipPostInstall: true, force, }); if (installResult.error) { @@ -92,19 +90,6 @@ export async function bulkInstallPackages({ }) ); - // only install index patterns if we completed install for any package-version for the - // first time, aka fresh installs or upgrades - if ( - bulkInstallResults.find( - (result) => - result.status === 'fulfilled' && - !result.value.result?.error && - result.value.result?.status === 'installed' - ) - ) { - await installIndexPatterns({ savedObjectsClient, esClient, installSource }); - } - return bulkInstallResults.map((result, index) => { const packageName = getNameFromPackagesToInstall(packagesToInstall, index); if (result.status === 'fulfilled') { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/index.ts b/x-pack/plugins/fleet/server/services/epm/packages/index.ts index a6970a8d19db4..feee4277ab0e1 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/index.ts @@ -7,7 +7,11 @@ import type { SavedObject } from 'src/core/server'; -import { unremovablePackages, installationStatuses } from '../../../../common'; +import { + unremovablePackages, + installationStatuses, + KibanaSavedObjectType, +} from '../../../../common'; import { KibanaAssetType } from '../../../types'; import type { AssetType, Installable, Installation } from '../../../types'; @@ -40,7 +44,7 @@ export class PackageNotInstalledError extends Error { // only Kibana Assets use Saved Objects at this point export const savedObjectTypes: AssetType[] = Object.values(KibanaAssetType); - +export const kibanaSavedObjectTypes: KibanaSavedObjectType[] = Object.values(KibanaSavedObjectType); export function createInstallableFrom( from: T, savedObject?: SavedObject diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index a9bb235c22cb8..261a0d9a6d688 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -26,6 +26,9 @@ jest.mock('../../app_context', () => { return { error: jest.fn(), debug: jest.fn(), warn: jest.fn() }; }), getTelemetryEventsSender: jest.fn(), + getSavedObjects: jest.fn(() => ({ + createImporter: jest.fn(), + })), }, }; }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 330fd84e789b8..a580248b43731 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -39,7 +39,6 @@ import * as Registry from '../registry'; import { setPackageInfo, parseAndVerifyArchiveEntries, unpackBufferToCache } from '../archive'; import { toAssetReference } from '../kibana/assets/install'; import type { ArchiveAsset } from '../kibana/assets/install'; -import { installIndexPatterns } from '../kibana/index_pattern/install'; import type { PackageUpdateEvent } from '../../upgrade_sender'; import { sendTelemetryEvents, UpdateEventType } from '../../upgrade_sender'; @@ -303,10 +302,15 @@ async function installPackageFromRegistry({ return { error: err, installType }; } + const savedObjectsImporter = appContextService + .getSavedObjects() + .createImporter(savedObjectsClient); + // try installing the package, if there was an error, call error handler and rethrow // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed' return _installPackage({ savedObjectsClient, + savedObjectsImporter, esClient, logger, installedPkg, @@ -407,9 +411,15 @@ async function installPackageByUpload({ version: packageInfo.version, packageInfo, }); + + const savedObjectsImporter = appContextService + .getSavedObjects() + .createImporter(savedObjectsClient); + // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed' return _installPackage({ savedObjectsClient, + savedObjectsImporter, esClient, logger, installedPkg, @@ -441,41 +451,25 @@ async function installPackageByUpload({ } } -export type InstallPackageParams = { - skipPostInstall?: boolean; -} & ( +export type InstallPackageParams = | ({ installSource: Extract } & InstallRegistryPackageParams) - | ({ installSource: Extract } & InstallUploadedArchiveParams) -); + | ({ installSource: Extract } & InstallUploadedArchiveParams); export async function installPackage(args: InstallPackageParams) { if (!('installSource' in args)) { throw new Error('installSource is required'); } const logger = appContextService.getLogger(); - const { savedObjectsClient, esClient, skipPostInstall = false, installSource } = args; + const { savedObjectsClient, esClient } = args; if (args.installSource === 'registry') { const { pkgkey, force } = args; - const { pkgName, pkgVersion } = Registry.splitPkgKey(pkgkey); logger.debug(`kicking off install of ${pkgkey} from registry`); const response = installPackageFromRegistry({ savedObjectsClient, pkgkey, esClient, force, - }).then(async (installResult) => { - if (skipPostInstall || installResult.error) { - return installResult; - } - logger.debug(`install of ${pkgkey} finished, running post-install`); - return installIndexPatterns({ - savedObjectsClient, - esClient, - pkgName, - pkgVersion, - installSource, - }).then(() => installResult); }); return response; } else if (args.installSource === 'upload') { @@ -486,16 +480,6 @@ export async function installPackage(args: InstallPackageParams) { esClient, archiveBuffer, contentType, - }).then(async (installResult) => { - if (skipPostInstall || installResult.error) { - return installResult; - } - logger.debug(`install of uploaded package finished, running post-install`); - return installIndexPatterns({ - savedObjectsClient, - esClient, - installSource, - }).then(() => installResult); }); return response; } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index cd85eecbf1e78..957dac8c1aacb 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -18,7 +18,7 @@ import type { Installation, } from '../../../types'; import { deletePipeline } from '../elasticsearch/ingest_pipeline/'; -import { installIndexPatterns } from '../kibana/index_pattern/install'; +import { removeUnusedIndexPatterns } from '../kibana/index_pattern/install'; import { deleteTransforms } from '../elasticsearch/transform/remove'; import { deleteMlModel } from '../elasticsearch/ml_model'; import { packagePolicyService, appContextService } from '../..'; @@ -27,7 +27,7 @@ import { deletePackageCache } from '../archive'; import { deleteIlms } from '../elasticsearch/datastream_ilm/remove'; import { removeArchiveEntries } from '../archive/storage'; -import { getInstallation, savedObjectTypes } from './index'; +import { getInstallation, kibanaSavedObjectTypes } from './index'; export async function removeInstallation(options: { savedObjectsClient: SavedObjectsClientContract; @@ -62,10 +62,10 @@ export async function removeInstallation(options: { // could also update with [] or some other state await savedObjectsClient.delete(PACKAGES_SAVED_OBJECT_TYPE, pkgName); - // recreate or delete index patterns when a package is uninstalled + // delete the index patterns if no packages are installed // this must be done after deleting the saved object for the current package otherwise it will retrieve the package - // from the registry again and reinstall the index patterns - await installIndexPatterns({ savedObjectsClient, esClient }); + // from the registry again and keep the index patterns + await removeUnusedIndexPatterns(savedObjectsClient); // remove the package archive and its contents from the cache so that a reinstall fetches // a fresh copy from the registry @@ -80,14 +80,26 @@ export async function removeInstallation(options: { return installedAssets; } -// TODO: this is very much like deleteKibanaSavedObjectsAssets below -function deleteKibanaAssets( +async function deleteKibanaAssets( installedObjects: KibanaAssetReference[], savedObjectsClient: SavedObjectsClientContract ) { - return installedObjects.map(async ({ id, type }) => { + const { resolved_objects: resolvedObjects } = await savedObjectsClient.bulkResolve( + installedObjects + ); + + const foundObjects = resolvedObjects.filter( + ({ saved_object: savedObject }) => savedObject?.error?.statusCode !== 404 + ); + + // in the case of a partial install, it is expected that some assets will be not found + // we filter these out before calling delete + const assetsToDelete = foundObjects.map(({ saved_object: { id, type } }) => ({ id, type })); + const promises = assetsToDelete.map(async ({ id, type }) => { return savedObjectsClient.delete(type, id); }); + + return Promise.all(promises); } function deleteESAssets( @@ -145,7 +157,7 @@ async function deleteAssets( // then the other asset types await Promise.all([ ...deleteESAssets(otherAssets, esClient), - ...deleteKibanaAssets(installedKibana, savedObjectsClient), + deleteKibanaAssets(installedKibana, savedObjectsClient), ]); } catch (err) { // in the rollback case, partial installs are likely, so missing assets are not an error @@ -177,23 +189,19 @@ async function deleteComponentTemplate(esClient: ElasticsearchClient, name: stri } } -// TODO: this is very much like deleteKibanaAssets above export async function deleteKibanaSavedObjectsAssets( savedObjectsClient: SavedObjectsClientContract, - installedRefs: AssetReference[] + installedRefs: KibanaAssetReference[] ) { if (!installedRefs.length) return; const logger = appContextService.getLogger(); - const deletePromises = installedRefs.map(({ id, type }) => { - const assetType = type as AssetType; + const assetsToDelete = installedRefs + .filter(({ type }) => kibanaSavedObjectTypes.includes(type)) + .map(({ id, type }) => ({ id, type })); - if (savedObjectTypes.includes(assetType)) { - return savedObjectsClient.delete(assetType, id); - } - }); try { - await Promise.all(deletePromises); + await deleteKibanaAssets(assetsToDelete, savedObjectsClient); } catch (err) { // in the rollback case, partial installs are likely, so missing assets are not an error if (!savedObjectsClient.errors.isNotFoundError(err)) { diff --git a/x-pack/plugins/ml/server/saved_objects/saved_objects.ts b/x-pack/plugins/ml/server/saved_objects/saved_objects.ts index 004b5e8e554cc..f3061d3e38d7a 100644 --- a/x-pack/plugins/ml/server/saved_objects/saved_objects.ts +++ b/x-pack/plugins/ml/server/saved_objects/saved_objects.ts @@ -25,6 +25,10 @@ export function setupSavedObjects(savedObjects: SavedObjectsServiceSetup) { savedObjects.registerType({ name: ML_MODULE_SAVED_OBJECT_TYPE, hidden: false, + management: { + importableAndExportable: true, + visibleInManagement: false, + }, namespaceType: 'agnostic', migrations, mappings: mlModule, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_object_mappings.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_object_mappings.ts index e2941b503664b..d42248823e733 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset/rule_asset_saved_object_mappings.ts @@ -27,6 +27,10 @@ export const ruleAssetSavedObjectMappings: SavedObjectsType['mappings'] = { export const ruleAssetType: SavedObjectsType = { name: ruleAssetSavedObjectType, hidden: false, + management: { + importableAndExportable: true, + visibleInManagement: false, + }, namespaceType: 'agnostic', mappings: ruleAssetSavedObjectMappings, }; diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 0915af7e25f0c..e553ee35a6eb6 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -255,48 +255,6 @@ export default function (providerContext: FtrProviderContext) { } expect(resIndexPattern.response.data.statusCode).equal(404); }); - it('should have removed the fields from the index patterns', async () => { - // The reason there is an expect inside the try and inside the catch in this test case is to guard against two - // different scenarios. - // - // If a test case in another file calls /setup then the system and endpoint packages will be installed and - // will be present for the remainder of the tests (because they cannot be removed). If that is the case the - // expect in the try will work because the logs-* and metrics-* index patterns will still be present even - // after this test uninstalls its package. - // - // If /setup was never called prior to this test, when the test package is uninstalled the index pattern code - // checks to see if there are no packages installed and completely removes the logs-* and metrics-* index - // patterns. If that happens this code will throw an error and indicate that the index pattern being searched - // for was completely removed. In this case the catch's expect will test to make sure the error thrown was - // a 404 because all of the packages have been removed. - try { - const resIndexPatternLogs = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'logs-*', - }); - const fields = JSON.parse(resIndexPatternLogs.attributes.fields); - const exists = fields.find((field: { name: string }) => field.name === 'logs_test_name'); - expect(exists).to.be(undefined); - } catch (err) { - // if all packages are uninstalled there won't be a logs-* index pattern - expect(err.response.data.statusCode).equal(404); - } - - try { - const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'metrics-*', - }); - const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields); - const existsMetrics = fieldsMetrics.find( - (field: { name: string }) => field.name === 'metrics_test_name' - ); - expect(existsMetrics).to.be(undefined); - } catch (err) { - // if all packages are uninstalled there won't be a metrics-* index pattern - expect(err.response.data.statusCode).equal(404); - } - }); it('should have removed the saved object', async function () { let res; try { @@ -512,23 +470,19 @@ const expectAssetsInstalled = ({ } expect(resInvalidTypeIndexPattern.response.data.statusCode).equal(404); }); - it('should create an index pattern with the package fields', async () => { + it('should not add fields to the index patterns', async () => { const resIndexPatternLogs = await kibanaServer.savedObjects.get({ type: 'index-pattern', id: 'logs-*', }); - const fields = JSON.parse(resIndexPatternLogs.attributes.fields); - const exists = fields.find((field: { name: string }) => field.name === 'logs_test_name'); - expect(exists).not.to.be(undefined); + const logsAttributes = resIndexPatternLogs.attributes; + expect(logsAttributes.fields).to.be(undefined); const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ type: 'index-pattern', id: 'metrics-*', }); - const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields); - const metricsExists = fieldsMetrics.find( - (field: { name: string }) => field.name === 'metrics_test_name' - ); - expect(metricsExists).not.to.be(undefined); + const metricsAttributes = resIndexPatternMetrics.attributes; + expect(metricsAttributes.fields).to.be(undefined); }); it('should have created the correct saved object', async function () { const res = await kibanaServer.savedObjects.get({ diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_multiple.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_multiple.ts index 5e9aa415d6c90..b3c70539579c5 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_multiple.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_multiple.ts @@ -61,14 +61,7 @@ export default function (providerContext: FtrProviderContext) { const uninstallingPackagesPromise = pkgs.map((pkg) => uninstallPackage(pkg)); return Promise.all(uninstallingPackagesPromise); }; - const expectPkgFieldToExist = (fields: any[], fieldName: string, exists: boolean = true) => { - const fieldExists = fields.find((field: { name: string }) => field.name === fieldName); - if (exists) { - expect(fieldExists).not.to.be(undefined); - } else { - expect(fieldExists).to.be(undefined); - } - }; + describe('installs and uninstalls multiple packages side effects', async () => { skipIfNoDockerRegistry(providerContext); setupFleetAndAgents(providerContext); @@ -81,69 +74,17 @@ export default function (providerContext: FtrProviderContext) { if (!server.enabled) return; await uninstallPackages([pkgKey, experimentalPkgKey]); }); - it('should create index patterns from all installed packages: uploaded, experimental, beta', async () => { - const resIndexPatternLogs = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'logs-*', - }); - - const fieldsLogs = JSON.parse(resIndexPatternLogs.attributes.fields); - - expectPkgFieldToExist(fieldsLogs, 'logs_test_name'); - expectPkgFieldToExist(fieldsLogs, 'logs_experimental_name'); - expectPkgFieldToExist(fieldsLogs, 'logs_experimental2_name'); - expectPkgFieldToExist(fieldsLogs, 'apache.error.uploadtest'); - const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'metrics-*', - }); - const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields); - expectPkgFieldToExist(fieldsMetrics, 'metrics_test_name'); - expectPkgFieldToExist(fieldsMetrics, 'metrics_experimental_name'); - expectPkgFieldToExist(fieldsMetrics, 'metrics_experimental2_name'); - expectPkgFieldToExist(fieldsMetrics, 'apache.status.uploadtest'); - }); - it('should correctly recreate index patterns when a package is uninstalled', async () => { - await uninstallPackage(experimental2PkgKey); - const resIndexPatternLogs = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'logs-*', - }); - const fieldsLogs = JSON.parse(resIndexPatternLogs.attributes.fields); - expectPkgFieldToExist(fieldsLogs, 'logs_test_name'); - expectPkgFieldToExist(fieldsLogs, 'logs_experimental_name'); - expectPkgFieldToExist(fieldsLogs, 'logs_experimental2_name', false); - expectPkgFieldToExist(fieldsLogs, 'apache.error.uploadtest'); - const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'metrics-*', - }); - const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields); - - expectPkgFieldToExist(fieldsMetrics, 'metrics_test_name'); - expectPkgFieldToExist(fieldsMetrics, 'metrics_experimental_name'); - expectPkgFieldToExist(fieldsMetrics, 'metrics_experimental2_name', false); - expectPkgFieldToExist(fieldsMetrics, 'apache.status.uploadtest'); - }); - it('should correctly recreate index patterns when an uploaded package is uninstalled', async () => { - await uninstallPackage(uploadPkgKey); + it('should create index patterns (without fields)', async () => { const resIndexPatternLogs = await kibanaServer.savedObjects.get({ type: 'index-pattern', id: 'logs-*', }); - const fieldsLogs = JSON.parse(resIndexPatternLogs.attributes.fields); - expectPkgFieldToExist(fieldsLogs, 'logs_test_name'); - expectPkgFieldToExist(fieldsLogs, 'logs_experimental_name'); - expectPkgFieldToExist(fieldsLogs, 'apache.error.uploadtest', false); + expect(resIndexPatternLogs.attributes.fields).to.be(undefined); const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ type: 'index-pattern', id: 'metrics-*', }); - const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields); - - expectPkgFieldToExist(fieldsMetrics, 'metrics_test_name'); - expectPkgFieldToExist(fieldsMetrics, 'metrics_experimental_name'); - expectPkgFieldToExist(fieldsMetrics, 'apache.status.uploadtest', false); + expect(resIndexPatternMetrics.attributes.fields).to.be(undefined); }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 032abac4be4de..b46c932373cdf 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -287,24 +287,6 @@ export default function (providerContext: FtrProviderContext) { ], }); }); - it('should have updated the index patterns', async function () { - const resIndexPatternLogs = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'logs-*', - }); - const fields = JSON.parse(resIndexPatternLogs.attributes.fields); - const updated = fields.filter((field: { name: string }) => field.name === 'new_field_name'); - expect(!!updated.length).equal(true); - const resIndexPatternMetrics = await kibanaServer.savedObjects.get({ - type: 'index-pattern', - id: 'metrics-*', - }); - const fieldsMetrics = JSON.parse(resIndexPatternMetrics.attributes.fields); - const updatedMetrics = fieldsMetrics.filter( - (field: { name: string }) => field.name === 'metrics_test_name2' - ); - expect(!!updatedMetrics.length).equal(true); - }); it('should have updated the kibana assets', async function () { const resDashboard = await kibanaServer.savedObjects.get({ type: 'dashboard', diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/all_logs.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/all_logs.json new file mode 100644 index 0000000000000..52fb2fd62957d --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/all_logs.json @@ -0,0 +1,36 @@ +{ + "attributes": { + "columns": [ + "log.level", + "kafka.log.component", + "message" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\",\"key\":\"dataset.name\",\"negate\":false,\"params\":{\"query\":\"kafka.log\",\"type\":\"phrase\"},\"type\":\"phrase\",\"value\":\"log\"},\"query\":{\"match\":{\"dataset.name\":{\"query\":\"kafka.log\",\"type\":\"phrase\"}}}}],\"highlightAll\":true,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\",\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"version\":true}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "All logs [Logs Kafka] ECS", + "version": 1 + }, + "id": "Kafka stacktraces-ecs", + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search" +} \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/ecs_logs.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/ecs_logs.json new file mode 100644 index 0000000000000..1b34746cec89e --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/ecs_logs.json @@ -0,0 +1,36 @@ +{ + "attributes": { + "columns": [ + "log.level", + "kafka.log.component", + "message" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\",\"key\":\"dataset.name\",\"negate\":false,\"params\":{\"query\":\"kafka.log\",\"type\":\"phrase\"},\"type\":\"phrase\",\"value\":\"log\"},\"query\":{\"match\":{\"dataset.name\":{\"query\":\"kafka.log\",\"type\":\"phrase\"}}}}],\"highlightAll\":true,\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\",\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"version\":true}" + }, + "sort": [ + [ + "@timestamp", + "desc" + ] + ], + "title": "All logs [Logs Kafka] ECS", + "version": 1 + }, + "id": "All Kafka logs-ecs", + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "search" +} \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/sample_search.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/sample_search.json index 1b34746cec89e..2698337d0b6fd 100644 --- a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/sample_search.json +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/search/sample_search.json @@ -19,7 +19,7 @@ "title": "All logs [Logs Kafka] ECS", "version": 1 }, - "id": "All Kafka logs-ecs", + "id": "sample_search", "references": [ { "id": "logs-*", diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_0.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_0.json new file mode 100644 index 0000000000000..095bf2ee94b59 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_0.json @@ -0,0 +1,22 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchRefName": "search_0", + "title": "Log levels over time [Logs Kafka] ECS", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}" + }, + "id": "panel_0", + "references": [ + { + "id": "All Kafka logs-ecs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_1.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_1.json new file mode 100644 index 0000000000000..9f98c35881c04 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_1.json @@ -0,0 +1,22 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchRefName": "search_0", + "title": "Log levels over time [Logs Kafka] ECS", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}" + }, + "id": "panel_1", + "references": [ + { + "id": "All Kafka logs-ecs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_2.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_2.json new file mode 100644 index 0000000000000..25a175ab040dc --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_2.json @@ -0,0 +1,22 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchRefName": "search_0", + "title": "Log levels over time [Logs Kafka] ECS", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}" + }, + "id": "panel_2", + "references": [ + { + "id": "All Kafka logs-ecs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_3.json b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_3.json new file mode 100644 index 0000000000000..4987ac23b8029 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/filetest/0.1.0/kibana/visualization/panel_3.json @@ -0,0 +1,22 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\"filter\":[]}" + }, + "savedSearchRefName": "search_0", + "title": "Log levels over time [Logs Kafka] ECS", + "uiStateJSON": "{}", + "version": 1, + "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Log Level\",\"field\":\"log.level\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per day\"},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"mode\":\"stacked\",\"show\":\"true\",\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"Log levels over time [Logs Kafka] ECS\",\"type\":\"histogram\"}" + }, + "id": "panel_3", + "references": [ + { + "id": "All Kafka logs-ecs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization" +} \ No newline at end of file