Skip to content

Commit

Permalink
[Fleet] Handle Saved Object ID changes (#108959) (#119527)
Browse files Browse the repository at this point in the history 
* Do not add fields to index patterns

* remove redundant tests

* install index patterns before package install

* update remove comment

* use import to create package assets

Here I have also moved to importing all assets at once. This is essential when importing to ensure that all saved objects references are imported at once. There is also an efficiencey improvement.

* Import index patterns

* use resolve when deleting index patterns

* fix: asset type validation

* add option to override supported import types

* make ml-module importable

* Revert "add option to override supported import types"

This reverts commit 1f48e6ee193fea5e5cb0f37c70cbfa7ae47eeab5.

* remove hidden: false from ml-module

* use resolve when deleting assets

* make security-rule SO type importable

* use bulkResolve to get package assets

* fix tests

* fix 'multiple' tests

* remove unused function

* create index patterns at the same time as other assets

* remove unused test

* Fix integration tests
We were checking for an error before the import was complete.

* tidy for PR

* add missing test assets

* do not attempt to delete missing assets

* resolve any reference errors that occur on import

* await installKibanaAssets immediately

* show assets not found when assets installed in a different space

* fix delete asset check on force upgrade

* add comment about reference errors

* remove a couple of appContextService dependencies
  • Loading branch information
@hop-dev
hop-dev authored Dec 6, 2021
1 parent 27d0b90 commit 7e3c3d1
Show file tree
Hide file tree
Showing 29 changed files with 415 additions and 2,394 deletions.
27 changes: 21 additions & 6 deletions ...gins/fleet/public/applications/integrations/sections/epm/screens/detail/assets/assets.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import { FormattedMessage } from '@kbn/i18n-react';
import { EuiFlexGroup, EuiFlexItem, EuiTitle, EuiSpacer } from '@elastic/eui';
import { groupBy } from 'lodash';

import type { ResolvedSimpleSavedObject } from 'src/core/public';

import { Loading, Error, ExtensionWrapper } from '../../../../../components';

import type { PackageInfo } from '../../../../../types';
Expand All @@ -27,6 +29,7 @@ import type { AssetSavedObject } from './types';
import { allowedAssetTypes } from './constants';
import { AssetsAccordion } from './assets_accordion';

const allowedAssetTypesLookup = new Set<string>(allowedAssetTypes);
interface AssetsPanelProps {
packageInfo: PackageInfo;
}
Expand Down Expand Up @@ -74,19 +77,32 @@ export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => {
const objectsByType = await Promise.all(
Object.entries(groupBy(objectsToGet, 'type')).map(([type, objects]) =>
savedObjectsClient
.bulkGet(objects)
.bulkResolve(objects)
// Ignore privilege errors
.catch((e: any) => {
if (e?.body?.statusCode === 403) {
return { savedObjects: [] };
return { resolved_objects: [] };
} else {
throw e;
}
})
.then(({ savedObjects }) => savedObjects as AssetSavedObject[])
.then(
({
resolved_objects: resolvedObjects,
}: {
resolved_objects: ResolvedSimpleSavedObject[];
}) => {
return resolvedObjects
.map(({ saved_object: savedObject }) => savedObject)
.filter(
(savedObject) =>
savedObject?.error?.statusCode !== 404 &&
allowedAssetTypesLookup.has(savedObject.type)
) as AssetSavedObject[];
}
)
)
);

setAssetsSavedObjects(objectsByType.flat());
} catch (e) {
setFetchError(e);
Expand All @@ -107,7 +123,6 @@ export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => {
}

let content: JSX.Element | Array<JSX.Element | null>;

if (isLoading) {
content = <Loading />;
} else if (fetchError) {
Expand All @@ -122,7 +137,7 @@ export const AssetsPage = ({ packageInfo }: AssetsPanelProps) => {
error={fetchError}
/>
);
} else if (assetSavedObjects === undefined) {
} else if (assetSavedObjects === undefined || assetSavedObjects.length === 0) {
if (customAssetsExtension) {
// If a UI extension for custom asset entries is defined, render the custom component here depisite
// there being no saved objects found
Expand Down
176 changes: 123 additions & 53 deletions x-pack/plugins/fleet/server/services/epm/kibana/assets/install.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,27 @@ import type {
SavedObject,
SavedObjectsBulkCreateObject,
SavedObjectsClientContract,
SavedObjectsImporter,
Logger,
} from 'src/core/server';
import type { SavedObjectsImportSuccess, SavedObjectsImportFailure } from 'src/core/server/types';

import { createListStream } from '@kbn/utils';
import { partition } from 'lodash';

import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common';
import { getAsset, getPathParts } from '../../archive';
import { KibanaAssetType, KibanaSavedObjectType } from '../../../../types';
import type { AssetType, AssetReference, AssetParts } from '../../../../types';
import { savedObjectTypes } from '../../packages';
import { indexPatternTypes } from '../index_pattern/install';
import { indexPatternTypes, getIndexPatternSavedObjects } from '../index_pattern/install';

type SavedObjectsImporterContract = Pick<SavedObjectsImporter, 'import' | 'resolveImportErrors'>;
const formatImportErrorsForLog = (errors: SavedObjectsImportFailure[]) =>
JSON.stringify(
errors.map(({ type, id, error }) => ({ type, id, error })) // discard other fields
);
const validKibanaAssetTypes = new Set(Object.values(KibanaAssetType));
type SavedObjectToBe = Required<Pick<SavedObjectsBulkCreateObject, keyof ArchiveAsset>> & {
type: KibanaSavedObjectType;
};
Expand All @@ -42,23 +54,8 @@ const KibanaSavedObjectTypeMapping: Record<KibanaAssetType, KibanaSavedObjectTyp
[KibanaAssetType.tag]: KibanaSavedObjectType.tag,
};

// Define how each asset type will be installed
const AssetInstallers: Record<
KibanaAssetType,
(args: {
savedObjectsClient: SavedObjectsClientContract;
kibanaAssets: ArchiveAsset[];
}) => Promise<Array<SavedObject<unknown>>>
> = {
[KibanaAssetType.dashboard]: installKibanaSavedObjects,
[KibanaAssetType.indexPattern]: installKibanaIndexPatterns,
[KibanaAssetType.map]: installKibanaSavedObjects,
[KibanaAssetType.search]: installKibanaSavedObjects,
[KibanaAssetType.visualization]: installKibanaSavedObjects,
[KibanaAssetType.lens]: installKibanaSavedObjects,
[KibanaAssetType.mlModule]: installKibanaSavedObjects,
[KibanaAssetType.securityRule]: installKibanaSavedObjects,
[KibanaAssetType.tag]: installKibanaSavedObjects,
const AssetFilters: Record<string, (kibanaAssets: ArchiveAsset[]) => ArchiveAsset[]> = {
[KibanaAssetType.indexPattern]: removeReservedIndexPatterns,
};

export async function getKibanaAsset(key: string): Promise<ArchiveAsset> {
Expand All @@ -79,29 +76,46 @@ export function createSavedObjectKibanaAsset(asset: ArchiveAsset): SavedObjectTo
};
}

// TODO: make it an exhaustive list
// e.g. switch statement with cases for each enum key returning `never` for default case
export async function installKibanaAssets(options: {
savedObjectsClient: SavedObjectsClientContract;
savedObjectsImporter: SavedObjectsImporterContract;
logger: Logger;
pkgName: string;
kibanaAssets: Record<KibanaAssetType, ArchiveAsset[]>;
}): Promise<SavedObject[]> {
const { savedObjectsClient, kibanaAssets } = options;
}): Promise<SavedObjectsImportSuccess[]> {
const { kibanaAssets, savedObjectsImporter, logger } = options;
const assetsToInstall = Object.entries(kibanaAssets).flatMap(([assetType, assets]) => {
if (!validKibanaAssetTypes.has(assetType as KibanaAssetType)) {
return [];
}

// install the assets
const kibanaAssetTypes = Object.values(KibanaAssetType);
const installedAssets = await Promise.all(
kibanaAssetTypes.map((assetType) => {
if (kibanaAssets[assetType]) {
return AssetInstallers[assetType]({
savedObjectsClient,
kibanaAssets: kibanaAssets[assetType],
});
}
if (!assets.length) {
return [];
})
);
return installedAssets.flat();
}

const assetFilter = AssetFilters[assetType];
if (assetFilter) {
return assetFilter(assets);
}

return assets;
});

if (!assetsToInstall.length) {
return [];
}

// As we use `import` to create our saved objects, we have to install
// their references (the index patterns) at the same time
// to prevent a reference error
const indexPatternSavedObjects = getIndexPatternSavedObjects() as ArchiveAsset[];

const installedAssets = await installKibanaSavedObjects({
logger,
savedObjectsImporter,
kibanaAssets: [...indexPatternSavedObjects, ...assetsToInstall],
});

return installedAssets;
}
export const deleteKibanaInstalledRefs = async (
savedObjectsClient: SavedObjectsClientContract,
Expand Down Expand Up @@ -153,39 +167,95 @@ export async function getKibanaAssets(
}

async function installKibanaSavedObjects({
savedObjectsClient,
savedObjectsImporter,
kibanaAssets,
logger,
}: {
savedObjectsClient: SavedObjectsClientContract;
kibanaAssets: ArchiveAsset[];
savedObjectsImporter: SavedObjectsImporterContract;
logger: Logger;
}) {
const toBeSavedObjects = await Promise.all(
kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset))
);

let allSuccessResults = [];

if (toBeSavedObjects.length === 0) {
return [];
} else {
const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, {
overwrite: true,
});
return createResults.saved_objects;
const { successResults: importSuccessResults = [], errors: importErrors = [] } =
await savedObjectsImporter.import({
overwrite: true,
readStream: createListStream(toBeSavedObjects),
createNewCopies: false,
});

allSuccessResults = importSuccessResults;
const [referenceErrors, otherErrors] = partition(
importErrors,
(e) => e?.error?.type === 'missing_references'
);

if (otherErrors?.length) {
throw new Error(
`Encountered ${
otherErrors.length
} errors creating saved objects: ${formatImportErrorsForLog(otherErrors)}`
);
}
/*
A reference error here means that a saved object reference in the references
array cannot be found. This is an error in the package its-self but not a fatal
one. For example a dashboard may still refer to the legacy `metricbeat-*` index
pattern. We ignore reference errors here so that legacy version of a package
can still be installed, but if a warning is logged it should be reported to
the integrations team. */
if (referenceErrors.length) {
logger.debug(
`Resolving ${
referenceErrors.length
} reference errors creating saved objects: ${formatImportErrorsForLog(referenceErrors)}`
);

const idsToResolve = new Set(referenceErrors.map(({ id }) => id));

const resolveSavedObjects = toBeSavedObjects.filter(({ id }) => idsToResolve.has(id));
const retries = referenceErrors.map(({ id, type }) => ({
id,
type,
ignoreMissingReferences: true,
replaceReferences: [],
overwrite: true,
}));

const { successResults: resolveSuccessResults = [], errors: resolveErrors = [] } =
await savedObjectsImporter.resolveImportErrors({
readStream: createListStream(resolveSavedObjects),
createNewCopies: false,
retries,
});

if (resolveErrors?.length) {
throw new Error(
`Encountered ${
resolveErrors.length
} errors resolving reference errors: ${formatImportErrorsForLog(resolveErrors)}`
);
}

allSuccessResults = [...allSuccessResults, ...resolveSuccessResults];
}

return allSuccessResults;
}
}

async function installKibanaIndexPatterns({
savedObjectsClient,
kibanaAssets,
}: {
savedObjectsClient: SavedObjectsClientContract;
kibanaAssets: ArchiveAsset[];
}) {
// Filter out any reserved index patterns
// Filter out any reserved index patterns
function removeReservedIndexPatterns(kibanaAssets: ArchiveAsset[]) {
const reservedPatterns = indexPatternTypes.map((pattern) => `${pattern}-*`);

const nonReservedPatterns = kibanaAssets.filter((asset) => !reservedPatterns.includes(asset.id));

return installKibanaSavedObjects({ savedObjectsClient, kibanaAssets: nonReservedPatterns });
return kibanaAssets.filter((asset) => !reservedPatterns.includes(asset.id));
}

export function toAssetReference({ id, type }: SavedObject) {
Expand Down
Loading

0 comments on commit 7e3c3d1

Please sign in to comment.