From 0d65f32ecfa88844be202b17bc48c0bba63108e8 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 21 Sep 2021 13:28:25 -0400 Subject: [PATCH 01/21] Add initial implementation for keep policies up to date functionality --- .../plugins/fleet/common/services/routes.ts | 4 + .../plugins/fleet/common/types/models/epm.ts | 2 + .../fleet/common/types/rest_spec/epm.ts | 13 +++ .../epm/screens/detail/settings/settings.tsx | 96 ++++++++++++++++++- .../fleet/public/hooks/use_request/epm.ts | 10 ++ x-pack/plugins/fleet/public/types/index.ts | 2 + .../fleet/server/routes/epm/handlers.ts | 25 +++++ .../plugins/fleet/server/routes/epm/index.ts | 11 +++ .../fleet/server/saved_objects/index.ts | 1 + .../fleet/server/services/epm/packages/get.ts | 1 + .../server/services/epm/packages/update.ts | 42 ++++++++ .../fleet/server/types/rest_spec/epm.ts | 9 ++ 12 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/update.ts diff --git a/x-pack/plugins/fleet/common/services/routes.ts b/x-pack/plugins/fleet/common/services/routes.ts index 5294c31d6a289..79ea19360c849 100644 --- a/x-pack/plugins/fleet/common/services/routes.ts +++ b/x-pack/plugins/fleet/common/services/routes.ts @@ -59,6 +59,10 @@ export const epmRouteService = { getRemovePath: (pkgkey: string) => { return EPM_API_ROUTES.DELETE_PATTERN.replace('{pkgkey}', pkgkey).replace(/\/$/, ''); // trim trailing slash }, + + getUpdatePath: (pkgkey: string) => { + return EPM_API_ROUTES.INFO_PATTERN.replace('{pkgkey}', pkgkey); + }, }; export const packagePolicyRouteService = { diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 66852bc965b07..b7268034f6fb8 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -349,6 +349,7 @@ export interface EpmPackageAdditions { assets: AssetsGroupedByServiceByType; removable?: boolean; notice?: string; + keepPoliciesUpToDate?: boolean; } type Merge = Omit> & @@ -377,6 +378,7 @@ export interface Installation extends SavedObjectAttributes { install_version: string; install_started_at: string; install_source: InstallSource; + keep_policies_up_to_date: boolean; } export interface PackageUsageStats { diff --git a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts index 51772eadca69e..cfe0b4abdcd3c 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/epm.ts @@ -57,6 +57,19 @@ export interface GetInfoResponse { response: PackageInfo; } +export interface UpdatePackageRequest { + params: { + pkgkey: string; + }; + body: { + keepPoliciesUpToDate?: boolean; + }; +} + +export interface UpdatePackageResponse { + response: PackageInfo; +} + export interface GetStatsRequest { params: { pkgname: string; diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx index 07c95e0d77ec7..fae987394a7a7 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { memo, useEffect, useMemo, useState } from 'react'; +import React, { memo, useCallback, useEffect, useMemo, useState } from 'react'; import styled from 'styled-components'; import { FormattedMessage } from '@kbn/i18n/react'; import semverLt from 'semver/functions/lt'; @@ -18,6 +18,8 @@ import { EuiText, EuiSpacer, EuiLink, + EuiSwitch, + EuiIcon, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; @@ -29,9 +31,13 @@ import { useGetPackageInstallStatus, useLink, sendUpgradePackagePolicyDryRun, + sendUpdatePackage, + useStartServices, } from '../../../../../hooks'; import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../../../constants'; +import { toMountPoint } from '../../../../../../../../../../../src/plugins/kibana_react/public'; + import { InstallButton } from './install_button'; import { UpdateButton } from './update_button'; import { UninstallButton } from './uninstall_button'; @@ -85,7 +91,7 @@ interface Props { } export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { - const { name, title, removable, latestVersion, version } = packageInfo; + const { name, title, removable, latestVersion, version, keepPoliciesUpToDate } = packageInfo; const [dryRunData, setDryRunData] = useState(); const [isUpgradingPackagePolicies, setIsUpgradingPackagePolicies] = useState(false); const getPackageInstallStatus = useGetPackageInstallStatus(); @@ -95,6 +101,67 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${name}`, }); + const { notifications } = useStartServices(); + + const [keepPoliciesUpToDateSwitchValue, setKeepPoliciesUpToDateSwitchValue] = useState( + keepPoliciesUpToDate ?? false + ); + + const handleKeepPoliciesUpToDateSwitchChange = useCallback(() => { + const saveKeepPoliciesUpToDate = async () => { + try { + setKeepPoliciesUpToDateSwitchValue((prev) => !prev); + + await sendUpdatePackage(`${packageInfo.name}-${packageInfo.version}`, { + keepPoliciesUpToDate: !keepPoliciesUpToDateSwitchValue, + }); + + notifications.toasts.addSuccess({ + title: toMountPoint( + !keepPoliciesUpToDateSwitchValue ? ( + + ) : ( + + ) + ), + text: toMountPoint( + + ), + }); + } catch (error) { + notifications.toasts.addError(error, { + title: toMountPoint( + + ), + }); + } + }; + + saveKeepPoliciesUpToDate(); + }, [ + keepPoliciesUpToDateSwitchValue, + notifications.toasts, + packageInfo.name, + packageInfo.version, + title, + ]); + const { status: installationStatus, version: installedVersion } = getPackageInstallStatus(name); const packageHasUsages = !!packagePoliciesData?.total; @@ -199,6 +266,31 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { + + + + + + + + + + + + + + + {(updateAvailable || isUpgradingPackagePolicies) && ( <> diff --git a/x-pack/plugins/fleet/public/hooks/use_request/epm.ts b/x-pack/plugins/fleet/public/hooks/use_request/epm.ts index 8599b4f2c703c..22667be2a0a4b 100644 --- a/x-pack/plugins/fleet/public/hooks/use_request/epm.ts +++ b/x-pack/plugins/fleet/public/hooks/use_request/epm.ts @@ -15,6 +15,8 @@ import type { GetInfoResponse, InstallPackageResponse, DeletePackageResponse, + UpdatePackageRequest, + UpdatePackageResponse, } from '../../types'; import type { GetStatsResponse } from '../../../common'; @@ -99,3 +101,11 @@ export const sendRemovePackage = (pkgkey: string) => { method: 'delete', }); }; + +export const sendUpdatePackage = (pkgkey: string, body: UpdatePackageRequest['body']) => { + return sendRequest({ + path: epmRouteService.getUpdatePath(pkgkey), + method: 'put', + body, + }); +}; diff --git a/x-pack/plugins/fleet/public/types/index.ts b/x-pack/plugins/fleet/public/types/index.ts index 2328ca826da71..3ff0a760b5882 100644 --- a/x-pack/plugins/fleet/public/types/index.ts +++ b/x-pack/plugins/fleet/public/types/index.ts @@ -128,6 +128,8 @@ export { Installable, RegistryRelease, PackageSpecCategory, + UpdatePackageRequest, + UpdatePackageResponse, } from '../../common'; export * from './intra_app_route_state'; diff --git a/x-pack/plugins/fleet/server/routes/epm/handlers.ts b/x-pack/plugins/fleet/server/routes/epm/handlers.ts index 16d583f8a8d1f..2324d1a423bfc 100644 --- a/x-pack/plugins/fleet/server/routes/epm/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/epm/handlers.ts @@ -22,6 +22,7 @@ import type { BulkInstallPackagesResponse, IBulkInstallPackageHTTPError, GetStatsResponse, + UpdatePackageResponse, } from '../../../common'; import type { GetCategoriesRequestSchema, @@ -33,6 +34,7 @@ import type { DeletePackageRequestSchema, BulkUpgradePackagesFromRegistryRequestSchema, GetStatsRequestSchema, + UpdatePackageRequestSchema, } from '../../types'; import { bulkInstallPackages, @@ -53,6 +55,7 @@ import { licenseService } from '../../services'; import { getArchiveEntry } from '../../services/epm/archive/cache'; import { getAsset } from '../../services/epm/archive/storage'; import { getPackageUsageStats } from '../../services/epm/packages/get'; +import { updatePackage } from '../../services/epm/packages/update'; export const getCategoriesHandler: RequestHandler< undefined, @@ -201,6 +204,28 @@ export const getInfoHandler: RequestHandler, + unknown, + TypeOf +> = async (context, request, response) => { + try { + const { pkgkey } = request.params; + const savedObjectsClient = context.core.savedObjects.client; + + const { pkgName } = splitPkgKey(pkgkey); + + const res = await updatePackage({ savedObjectsClient, pkgName, ...request.body }); + const body: UpdatePackageResponse = { + response: res, + }; + + return response.ok({ body }); + } catch (error) { + return defaultIngestErrorHandler({ error, response }); + } +}; + export const getStatsHandler: RequestHandler> = async ( context, request, diff --git a/x-pack/plugins/fleet/server/routes/epm/index.ts b/x-pack/plugins/fleet/server/routes/epm/index.ts index 40316bd102e5f..684547dc1862c 100644 --- a/x-pack/plugins/fleet/server/routes/epm/index.ts +++ b/x-pack/plugins/fleet/server/routes/epm/index.ts @@ -18,6 +18,7 @@ import { DeletePackageRequestSchema, BulkUpgradePackagesFromRegistryRequestSchema, GetStatsRequestSchema, + UpdatePackageRequestSchema, } from '../../types'; import { @@ -31,6 +32,7 @@ import { deletePackageHandler, bulkInstallPackagesFromRegistryHandler, getStatsHandler, + updatePackageHandler, } from './handlers'; const MAX_FILE_SIZE_BYTES = 104857600; // 100MB @@ -90,6 +92,15 @@ export const registerRoutes = (router: IRouter) => { getInfoHandler ); + router.put( + { + path: EPM_API_ROUTES.INFO_PATTERN, + validate: UpdatePackageRequestSchema, + options: { tags: [`access:${PLUGIN_ID}-all`] }, + }, + updatePackageHandler + ); + router.post( { path: EPM_API_ROUTES.INSTALL_FROM_REGISTRY_PATTERN, diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 5c117909432b0..a3d3cc959b3d9 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -294,6 +294,7 @@ const getSavedObjectTypes = ( version: { type: 'keyword' }, internal: { type: 'boolean' }, removable: { type: 'boolean' }, + keep_policies_up_to_date: { type: 'boolean' }, es_index_patterns: { enabled: false, type: 'object', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 0e23981b95fcd..d4f988e5fba8c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -137,6 +137,7 @@ export async function getPackageInfo(options: { assets: Registry.groupPathsByService(paths || []), removable: !isUnremovablePackage(pkgName), notice: Registry.getNoticePath(paths || []), + keepPoliciesUpToDate: savedObject?.attributes.keep_policies_up_to_date ?? false, }; const updated = { ...packageInfo, ...additions }; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/update.ts b/x-pack/plugins/fleet/server/services/epm/packages/update.ts new file mode 100644 index 0000000000000..84c756983fa07 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/update.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from 'kibana/server'; +import type { TypeOf } from '@kbn/config-schema'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; +import type { Installation, UpdatePackageRequestSchema } from '../../../types'; +import { IngestManagerError } from '../../../errors'; + +import { getInstallationObject, getPackageInfo } from './get'; + +export async function updatePackage( + options: { + savedObjectsClient: SavedObjectsClientContract; + pkgName: string; + keepPoliciesUpToDate?: boolean; + } & TypeOf +) { + const { savedObjectsClient, pkgName, keepPoliciesUpToDate } = options; + const installedPackage = await getInstallationObject({ savedObjectsClient, pkgName }); + + if (!installedPackage) { + throw new IngestManagerError(`package ${pkgName} is not installed`); + } + + await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, installedPackage.id, { + keep_policies_up_to_date: keepPoliciesUpToDate ?? false, + }); + + const packageInfo = await getPackageInfo({ + savedObjectsClient, + pkgName, + pkgVersion: installedPackage.attributes.version, + }); + + return packageInfo; +} diff --git a/x-pack/plugins/fleet/server/types/rest_spec/epm.ts b/x-pack/plugins/fleet/server/types/rest_spec/epm.ts index 25f1e766a7476..918def62a9d0e 100644 --- a/x-pack/plugins/fleet/server/types/rest_spec/epm.ts +++ b/x-pack/plugins/fleet/server/types/rest_spec/epm.ts @@ -35,6 +35,15 @@ export const GetInfoRequestSchema = { }), }; +export const UpdatePackageRequestSchema = { + params: schema.object({ + pkgkey: schema.string(), + }), + body: schema.object({ + keepPoliciesUpToDate: schema.boolean(), + }), +}; + export const GetStatsRequestSchema = { params: schema.object({ pkgName: schema.string(), From f856653ee68a34eb634c9f06b08aacaae0593ca2 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 22 Sep 2021 13:32:22 -0400 Subject: [PATCH 02/21] Upgrade package policies during preconfiguration check --- x-pack/plugins/fleet/server/constants/index.ts | 1 + .../fleet/server/services/preconfiguration.ts | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/constants/index.ts b/x-pack/plugins/fleet/server/constants/index.ts index 0dcd5e7f47800..bfa48c38ee2ea 100644 --- a/x-pack/plugins/fleet/server/constants/index.ts +++ b/x-pack/plugins/fleet/server/constants/index.ts @@ -57,6 +57,7 @@ export { // Preconfiguration PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, PRECONFIGURATION_LATEST_KEYWORD, + AUTO_UPDATE_PACKAGES, } from '../../common'; export { diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 37ed98a6f4aa0..4f5b4db037726 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -21,6 +21,7 @@ import type { import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common'; import { + AUTO_UPDATE_PACKAGES, PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, PRECONFIGURATION_LATEST_KEYWORD, } from '../constants'; @@ -33,7 +34,7 @@ import { ensurePackagesCompletedInstall } from './epm/packages/install'; import { bulkInstallPackages } from './epm/packages/bulk_install_packages'; import { agentPolicyService, addPackageToAgentPolicy } from './agent_policy'; import type { InputsOverride } from './package_policy'; -import { overridePackageInputs } from './package_policy'; +import { overridePackageInputs, packagePolicyService } from './package_policy'; import { appContextService } from './app_context'; interface PreconfigurationResult { @@ -229,6 +230,17 @@ export async function ensurePreconfiguredPackagesAndPolicies( } } + // Upgrade package policies for any packages designated as AUTO_UPDATE + const policyIdsToUpgrade = await packagePolicyService.listIds(soClient, { + page: 1, + perPage: 10000, + kuery: `ingest-package-policies.package.name:${AUTO_UPDATE_PACKAGES.map( + ({ name }) => name + ).join(' or ')}`, + }); + + await packagePolicyService.upgrade(soClient, esClient, policyIdsToUpgrade.items); + return { policies: fulfilledPolicies.map((p) => p.policy From 75f32fcc7b39dbd6be458565a49968b9e068a105 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 22 Sep 2021 15:06:02 -0400 Subject: [PATCH 03/21] Only show keep policies up to date switch for default/auto-update packages --- .../epm/screens/detail/settings/settings.tsx | 73 +++++++++++-------- .../plugins/fleet/public/constants/index.ts | 3 + .../services/epm/packages/_install_package.ts | 34 +++++++-- 3 files changed, 72 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx index fae987394a7a7..0e0d954bbfc26 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx @@ -9,6 +9,7 @@ import React, { memo, useCallback, useEffect, useMemo, useState } from 'react'; import styled from 'styled-components'; import { FormattedMessage } from '@kbn/i18n/react'; import semverLt from 'semver/functions/lt'; +import { uniq } from 'lodash'; import { EuiCallOut, @@ -34,7 +35,11 @@ import { sendUpdatePackage, useStartServices, } from '../../../../../hooks'; -import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../../../constants'; +import { + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + AUTO_UPDATE_PACKAGES, + DEFAULT_PACKAGES, +} from '../../../../../constants'; import { toMountPoint } from '../../../../../../../../../../../src/plugins/kibana_react/public'; @@ -103,6 +108,14 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { const { notifications } = useStartServices(); + const shouldShowKeepPoliciesUpToDateSwitch = useMemo(() => { + const packages = [...DEFAULT_PACKAGES, ...AUTO_UPDATE_PACKAGES]; + + const packageNames = uniq(packages.map((pkg) => pkg.name)); + + return packageNames.includes(name); + }, [name]); + const [keepPoliciesUpToDateSwitchValue, setKeepPoliciesUpToDateSwitchValue] = useState( keepPoliciesUpToDate ?? false ); @@ -132,13 +145,6 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { /> ) ), - text: toMountPoint( - - ), }); } catch (error) { notifications.toasts.addError(error, { @@ -266,30 +272,33 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { - - - - - - - - - - - - - - + {shouldShowKeepPoliciesUpToDateSwitch && ( + <> + + + + + + + + + + + + + + + )} {(updateAvailable || isUpgradingPackagePolicies) && ( <> diff --git a/x-pack/plugins/fleet/public/constants/index.ts b/x-pack/plugins/fleet/public/constants/index.ts index a0e88bc58726a..32dd732c53dec 100644 --- a/x-pack/plugins/fleet/public/constants/index.ts +++ b/x-pack/plugins/fleet/public/constants/index.ts @@ -19,6 +19,9 @@ export { // Fleet Server index AGENTS_INDEX, ENROLLMENT_API_KEYS_INDEX, + // Preconfiguration + AUTO_UPDATE_PACKAGES, + DEFAULT_PACKAGES, } from '../../common/constants'; export * from './page_paths'; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 1bbbb1bb9b6a2..64e0edd8848a4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -7,7 +7,11 @@ import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'src/core/server'; -import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE } from '../../../../common'; +import { + MAX_TIME_COMPLETE_INSTALL, + ASSETS_SAVED_OBJECT_TYPE, + AUTO_UPDATE_PACKAGES, +} from '../../../../common'; import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import type { AssetReference, Installation, InstallType } from '../../../types'; @@ -22,6 +26,8 @@ import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install import { saveArchiveEntries } from '../archive/storage'; import { ConcurrentInstallOperationError } from '../../../errors'; +import { packagePolicyService } from '../..'; + import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install'; import { deleteKibanaSavedObjectsAssets } from './remove'; @@ -192,11 +198,27 @@ export async function _installPackage({ // update to newly installed version when all assets are successfully installed if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion); - await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { - install_version: pkgVersion, - install_status: 'installed', - package_assets: packageAssetRefs, - }); + const updatedPackage = await savedObjectsClient.update( + PACKAGES_SAVED_OBJECT_TYPE, + pkgName, + { + install_version: pkgVersion, + install_status: 'installed', + package_assets: packageAssetRefs, + } + ); + + // If the package is flagged with the `keep_policies_up_to_date` flag, upgrade its + // associated package policies after installation + if (updatedPackage.attributes.keep_policies_up_to_date) { + const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, { + page: 1, + perPage: 10000, + kuery: `ingest-package-policies.package.name:${pkgName}`, + }); + + await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); + } return [ ...installedKibanaAssetsRefs, From 6e3b21f4fbfbb4e0cd0cfe7a35fe69b277d2a45e Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Wed, 22 Sep 2021 15:36:54 -0400 Subject: [PATCH 04/21] Fix type error --- .../sections/epm/screens/detail/settings/settings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx index 0e0d954bbfc26..b0c0d52a905db 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx @@ -148,7 +148,7 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { }); } catch (error) { notifications.toasts.addError(error, { - title: toMountPoint( + title: ( Date: Thu, 23 Sep 2021 12:14:23 -0400 Subject: [PATCH 05/21] Fixup setup policy upgrade logic --- .../server/services/epm/packages/install.ts | 1 + .../server/services/preconfiguration.test.ts | 29 +++++++++++ .../fleet/server/services/preconfiguration.ts | 50 +++++++++++++++---- 3 files changed, 71 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 2568f40594f10..7a7ad2128dacc 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -451,6 +451,7 @@ export async function createInstallation(options: { install_status: 'installing', install_started_at: new Date().toISOString(), install_source: installSource, + keep_policies_up_to_date: false, }, { id: pkgName, overwrite: true } ); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 86fdd2f0aa800..159074b2fc62c 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -15,6 +15,7 @@ import type { AgentPolicy, NewPackagePolicy, Output } from '../types'; import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants'; import * as agentPolicy from './agent_policy'; +import * as packagePolicy from './package_policy'; import { ensurePreconfiguredPackagesAndPolicies, @@ -131,6 +132,7 @@ jest.mock('./package_policy', () => ({ ...jest.requireActual('./package_policy'), packagePolicyService: { getByIDs: jest.fn().mockReturnValue([]), + listIds: jest.fn().mockReturnValue({ items: [] }), create(soClient: any, esClient: any, newPackagePolicy: NewPackagePolicy) { return { id: 'mocked', @@ -156,6 +158,7 @@ jest.mock('./app_context', () => ({ })); const spyAgentPolicyServiceUpdate = jest.spyOn(agentPolicy.agentPolicyService, 'update'); +const spyPackagePolicyServiceUpgrade = jest.spyOn(packagePolicy.packagePolicyService, 'upgrade'); describe('policy preconfiguration', () => { beforeEach(() => { @@ -479,4 +482,30 @@ describe('comparePreconfiguredPolicyToCurrent', () => { ); expect(hasChanged).toBe(false); }); + + it('should upgrade package policies if package is set as AUTO_UPDATE', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [] as PreconfiguredAgentPolicy[], + [], + mockDefaultOutput + ); + }); + + it('should upgrade package policies if package has `keep_policies_up_to_date: true`', async () => { + const soClient = getPutPreconfiguredPackagesMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + await ensurePreconfiguredPackagesAndPolicies( + soClient, + esClient, + [] as PreconfiguredAgentPolicy[], + [], + mockDefaultOutput + ); + }); }); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 4f5b4db037726..89721e78369fb 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -17,6 +17,7 @@ import type { PreconfiguredAgentPolicy, PreconfiguredPackage, PreconfigurationError, + PackagePolicy, } from '../../common'; import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common'; @@ -230,16 +231,47 @@ export async function ensurePreconfiguredPackagesAndPolicies( } } - // Upgrade package policies for any packages designated as AUTO_UPDATE - const policyIdsToUpgrade = await packagePolicyService.listIds(soClient, { - page: 1, - perPage: 10000, - kuery: `ingest-package-policies.package.name:${AUTO_UPDATE_PACKAGES.map( - ({ name }) => name - ).join(' or ')}`, - }); + // Upgrade package policies for any packages designated as AUTO_UPDATE or that have `keep_policies_up_to_date: true` + const policyIdsToUpgrade: string[] = []; + const fulfilledPolicyPackagePolicyIds = fulfilledPolicies.flatMap( + ({ policy }) => policy?.package_policies as string[] + ); + + for (const packagePolicyId of fulfilledPolicyPackagePolicyIds) { + const packagePolicy = await packagePolicyService.get(soClient, packagePolicyId); - await packagePolicyService.upgrade(soClient, esClient, policyIdsToUpgrade.items); + if (!packagePolicy || !packagePolicy.package) { + continue; + } + + const packageInfo = await getPackageInfo({ + savedObjectsClient: soClient, + pkgName: packagePolicy.package.name, + pkgVersion: packagePolicy.package.version, + }); + + const shouldUpgradePolicies = + AUTO_UPDATE_PACKAGES.some((pkg) => pkg.name === packageInfo.name) || + packageInfo.keepPoliciesUpToDate; + + if (shouldUpgradePolicies) { + policyIdsToUpgrade.push(packagePolicy.id); + } + } + + if (policyIdsToUpgrade.length) { + appContextService + .getLogger() + .debug( + `Upgrading ${policyIdsToUpgrade.length} package policies: ${policyIdsToUpgrade.join(', ')}` + ); + try { + await packagePolicyService.upgrade(soClient, esClient, policyIdsToUpgrade); + // Swallow errors from upgrades + } catch (error) { + appContextService.getLogger().error(error); + } + } return { policies: fulfilledPolicies.map((p) => From 9ab7620d439ad0ca9bb808975e017870e74c73ea Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Thu, 23 Sep 2021 12:38:52 -0400 Subject: [PATCH 06/21] Add migration for keep policies up to date flag --- .../fleet/server/saved_objects/index.ts | 2 ++ .../saved_objects/migrations/to_v7_16_0.ts | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_16_0.ts diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index a3d3cc959b3d9..155bf800a1aef 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -44,6 +44,7 @@ import { } from './migrations/to_v7_13_0'; import { migratePackagePolicyToV7140, migrateInstallationToV7140 } from './migrations/to_v7_14_0'; import { migratePackagePolicyToV7150 } from './migrations/to_v7_15_0'; +import { migrateInstallationToV7160 } from './migrations/to_v7_16_0'; /* * Saved object types and mappings @@ -329,6 +330,7 @@ const getSavedObjectTypes = ( migrations: { '7.14.0': migrateInstallationToV7140, '7.14.1': migrateInstallationToV7140, + '7.16.0': migrateInstallationToV7160, }, }, [ASSETS_SAVED_OBJECT_TYPE]: { diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_16_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_16_0.ts new file mode 100644 index 0000000000000..7d12c550ec406 --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_16_0.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectMigrationFn } from 'kibana/server'; + +import type { Installation } from '../../../common'; +import { AUTO_UPDATE_PACKAGES, DEFAULT_PACKAGES } from '../../../common'; + +export const migrateInstallationToV7160: SavedObjectMigrationFn = ( + installationDoc, + migrationContext +) => { + const updatedInstallationDoc = installationDoc; + + if ( + [...AUTO_UPDATE_PACKAGES, ...DEFAULT_PACKAGES].some( + (pkg) => pkg.name === updatedInstallationDoc.attributes.name + ) + ) { + updatedInstallationDoc.attributes.keep_policies_up_to_date = true; + } + + return updatedInstallationDoc; +}; From b7d383c6b178c24e167948e9e7927ce068e4a54b Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 27 Sep 2021 10:34:06 -0400 Subject: [PATCH 07/21] Move setup package policy logic to new module + add tests --- .../services/managed_package_policies.test.ts | 106 ++++++++++++++++++ .../services/managed_package_policies.ts | 62 ++++++++++ .../server/services/preconfiguration.test.ts | 34 +----- .../fleet/server/services/preconfiguration.ts | 43 +------ 4 files changed, 177 insertions(+), 68 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/managed_package_policies.test.ts create mode 100644 x-pack/plugins/fleet/server/services/managed_package_policies.ts diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts b/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts new file mode 100644 index 0000000000000..a53b1fe648905 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks'; + +import { upgradeManagedPackagePolicies } from './managed_package_policies'; +import { packagePolicyService } from './package_policy'; +import { getPackageInfo } from './epm/packages'; + +jest.mock('./package_policy'); +jest.mock('./epm/packages'); +jest.mock('./app_context', () => { + return { + ...jest.requireActual('./app_context'), + appContextService: { + getLogger: jest.fn(() => { + return { debug: jest.fn() }; + }), + }, + }; +}); + +describe('managed package policies', () => { + afterEach(() => { + (packagePolicyService.get as jest.Mock).mockReset(); + (getPackageInfo as jest.Mock).mockReset(); + }); + + it('should not upgrade policies for non-managed package', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const soClient = savedObjectsClientMock.create(); + + (packagePolicyService.get as jest.Mock).mockImplementationOnce( + (savedObjectsClient: any, id: string) => { + return { + id, + inputs: {}, + version: '', + revision: 1, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + package: { + name: 'non-managed-package', + title: 'Non-Managed Package', + version: '0.0.1', + }, + }; + } + ); + + (getPackageInfo as jest.Mock).mockImplementationOnce( + ({ savedObjectsClient, pkgName, pkgVersion }) => ({ + name: pkgName, + version: pkgVersion, + keepPoliciesUpToDate: false, + }) + ); + + await upgradeManagedPackagePolicies(soClient, esClient, ['non-managed-package-id']); + + expect(packagePolicyService.upgrade).not.toBeCalled(); + }); + + it('should upgrade policies for managed package', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const soClient = savedObjectsClientMock.create(); + + (packagePolicyService.get as jest.Mock).mockImplementationOnce( + (savedObjectsClient: any, id: string) => { + return { + id, + inputs: {}, + version: '', + revision: 1, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + package: { + name: 'managed-package', + title: 'Managed Package', + version: '0.0.1', + }, + }; + } + ); + + (getPackageInfo as jest.Mock).mockImplementationOnce( + ({ savedObjectsClient, pkgName, pkgVersion }) => ({ + name: pkgName, + version: pkgVersion, + keepPoliciesUpToDate: true, + }) + ); + + await upgradeManagedPackagePolicies(soClient, esClient, ['managed-package-id']); + + expect(packagePolicyService.upgrade).toBeCalledWith(soClient, esClient, ['managed-package-id']); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.ts b/x-pack/plugins/fleet/server/services/managed_package_policies.ts new file mode 100644 index 0000000000000..780e1ea983022 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/managed_package_policies.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; + +import { AUTO_UPDATE_PACKAGES } from '../../common'; + +import { appContextService } from './app_context'; +import { getPackageInfo } from './epm/packages'; +import { packagePolicyService } from './package_policy'; + +/** + * Upgrade any package policies for packages installed through setup that are denoted as `AUTO_UPGRADE` packages + * or have the `keep_policies_up_to_date` flag set to `true` + */ +export const upgradeManagedPackagePolicies = async ( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + packagePolicyIds: string[] +) => { + const policyIdsToUpgrade: string[] = []; + + for (const packagePolicyId of packagePolicyIds) { + const packagePolicy = await packagePolicyService.get(soClient, packagePolicyId); + + if (!packagePolicy || !packagePolicy.package) { + continue; + } + + const packageInfo = await getPackageInfo({ + savedObjectsClient: soClient, + pkgName: packagePolicy.package.name, + pkgVersion: packagePolicy.package.version, + }); + + const shouldUpgradePolicies = + AUTO_UPDATE_PACKAGES.some((pkg) => pkg.name === packageInfo.name) || + packageInfo.keepPoliciesUpToDate; + + if (shouldUpgradePolicies) { + policyIdsToUpgrade.push(packagePolicy.id); + } + } + + if (policyIdsToUpgrade.length) { + appContextService + .getLogger() + .debug( + `Upgrading ${policyIdsToUpgrade.length} package policies: ${policyIdsToUpgrade.join(', ')}` + ); + try { + await packagePolicyService.upgrade(soClient, esClient, policyIdsToUpgrade); + // Swallow errors from upgrades + } catch (error) { + appContextService.getLogger().error(error); + } + } +}; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 159074b2fc62c..d6e47b005ed1e 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -15,7 +15,6 @@ import type { AgentPolicy, NewPackagePolicy, Output } from '../types'; import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants'; import * as agentPolicy from './agent_policy'; -import * as packagePolicy from './package_policy'; import { ensurePreconfiguredPackagesAndPolicies, @@ -140,6 +139,12 @@ jest.mock('./package_policy', () => ({ ...newPackagePolicy, }; }, + get(soClient: any, id: string) { + return { + id: 'mocked', + version: 'mocked', + }; + }, }, })); @@ -158,7 +163,6 @@ jest.mock('./app_context', () => ({ })); const spyAgentPolicyServiceUpdate = jest.spyOn(agentPolicy.agentPolicyService, 'update'); -const spyPackagePolicyServiceUpgrade = jest.spyOn(packagePolicy.packagePolicyService, 'upgrade'); describe('policy preconfiguration', () => { beforeEach(() => { @@ -482,30 +486,4 @@ describe('comparePreconfiguredPolicyToCurrent', () => { ); expect(hasChanged).toBe(false); }); - - it('should upgrade package policies if package is set as AUTO_UPDATE', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - - await ensurePreconfiguredPackagesAndPolicies( - soClient, - esClient, - [] as PreconfiguredAgentPolicy[], - [], - mockDefaultOutput - ); - }); - - it('should upgrade package policies if package has `keep_policies_up_to_date: true`', async () => { - const soClient = getPutPreconfiguredPackagesMock(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - - await ensurePreconfiguredPackagesAndPolicies( - soClient, - esClient, - [] as PreconfiguredAgentPolicy[], - [], - mockDefaultOutput - ); - }); }); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 89721e78369fb..9f257aea9e3ec 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -17,12 +17,10 @@ import type { PreconfiguredAgentPolicy, PreconfiguredPackage, PreconfigurationError, - PackagePolicy, } from '../../common'; import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common'; import { - AUTO_UPDATE_PACKAGES, PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE, PRECONFIGURATION_LATEST_KEYWORD, } from '../constants'; @@ -35,8 +33,9 @@ import { ensurePackagesCompletedInstall } from './epm/packages/install'; import { bulkInstallPackages } from './epm/packages/bulk_install_packages'; import { agentPolicyService, addPackageToAgentPolicy } from './agent_policy'; import type { InputsOverride } from './package_policy'; -import { overridePackageInputs, packagePolicyService } from './package_policy'; +import { overridePackageInputs } from './package_policy'; import { appContextService } from './app_context'; +import { upgradeManagedPackagePolicies } from './managed_package_policies'; interface PreconfigurationResult { policies: Array<{ id: string; updated_at: string }>; @@ -231,47 +230,11 @@ export async function ensurePreconfiguredPackagesAndPolicies( } } - // Upgrade package policies for any packages designated as AUTO_UPDATE or that have `keep_policies_up_to_date: true` - const policyIdsToUpgrade: string[] = []; const fulfilledPolicyPackagePolicyIds = fulfilledPolicies.flatMap( ({ policy }) => policy?.package_policies as string[] ); - for (const packagePolicyId of fulfilledPolicyPackagePolicyIds) { - const packagePolicy = await packagePolicyService.get(soClient, packagePolicyId); - - if (!packagePolicy || !packagePolicy.package) { - continue; - } - - const packageInfo = await getPackageInfo({ - savedObjectsClient: soClient, - pkgName: packagePolicy.package.name, - pkgVersion: packagePolicy.package.version, - }); - - const shouldUpgradePolicies = - AUTO_UPDATE_PACKAGES.some((pkg) => pkg.name === packageInfo.name) || - packageInfo.keepPoliciesUpToDate; - - if (shouldUpgradePolicies) { - policyIdsToUpgrade.push(packagePolicy.id); - } - } - - if (policyIdsToUpgrade.length) { - appContextService - .getLogger() - .debug( - `Upgrading ${policyIdsToUpgrade.length} package policies: ${policyIdsToUpgrade.join(', ')}` - ); - try { - await packagePolicyService.upgrade(soClient, esClient, policyIdsToUpgrade); - // Swallow errors from upgrades - } catch (error) { - appContextService.getLogger().error(error); - } - } + await upgradeManagedPackagePolicies(soClient, esClient, fulfilledPolicyPackagePolicyIds); return { policies: fulfilledPolicies.map((p) => From 56dcc0c8c35aea1e15502e81c78235d6deebd5bc Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 27 Sep 2021 12:28:27 -0400 Subject: [PATCH 08/21] Update snapshots to include keepPoliciesUpToDate field --- .../apis/epm/__snapshots__/install_by_upload.snap | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap index 8f9428d8a12db..8e06e62385315 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap +++ b/x-pack/test/fleet_api_integration/apis/epm/__snapshots__/install_by_upload.snap @@ -276,6 +276,7 @@ Object { "type": "image/svg+xml", }, ], + "keepPoliciesUpToDate": false, "license": "basic", "name": "apache", "owner": Object { @@ -449,6 +450,7 @@ Object { }, ], "internal": false, + "keep_policies_up_to_date": false, "name": "apache", "package_assets": Array [ Object { From 224e04eafb83fa05eb26acb475c309fb70f5b85f Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 27 Sep 2021 13:07:15 -0400 Subject: [PATCH 09/21] Fix type errors --- .../sections/epm/screens/detail/settings/settings.tsx | 11 ++++------- .../services/epm/packages/get_install_type.test.ts | 2 ++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx index b0c0d52a905db..b2fb819d62c45 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx @@ -148,13 +148,10 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { }); } catch (error) { notifications.toasts.addError(error, { - title: ( - - ), + title: i18n.translate('xpack.fleet.integrations.keepPoliciesUpToDateError', { + defaultMessage: 'Error saving integration settings for {title}', + values: { title }, + }), }); } }; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts index 155cd67e60287..6bc962165f1d2 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get_install_type.test.ts @@ -28,6 +28,7 @@ const mockInstallation: SavedObject = { install_version: '1.0.0', install_started_at: new Date().toISOString(), install_source: 'registry', + keep_policies_up_to_date: false, }, }; const mockInstallationUpdateFail: SavedObject = { @@ -46,6 +47,7 @@ const mockInstallationUpdateFail: SavedObject = { install_version: '1.0.1', install_started_at: new Date().toISOString(), install_source: 'registry', + keep_policies_up_to_date: false, }, }; From 898bdc1c78a82d086d195ee39ea3ac089cc50355 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 27 Sep 2021 19:23:22 -0400 Subject: [PATCH 10/21] Fix some CI failures --- .../sections/epm/components/package_card.stories.tsx | 1 + .../sections/epm/components/package_list_grid.stories.tsx | 1 + .../fleet/server/services/epm/packages/_install_package.ts | 6 +----- .../fleet_api_integration/apis/epm/install_remove_assets.ts | 1 + x-pack/test/fleet_api_integration/apis/epm/update_assets.ts | 1 + 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_card.stories.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_card.stories.tsx index 69c70bba5be1d..df09bd20b2596 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_card.stories.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_card.stories.tsx @@ -66,6 +66,7 @@ export const Installed = ({ width, ...props }: Args) => { install_status: 'installed', install_source: 'registry', install_started_at: '2020-01-01T00:00:00.000Z', + keep_policies_up_to_date: false, }, references: [], }; diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.stories.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.stories.tsx index 737a83d5f5da8..accae33241801 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.stories.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/components/package_list_grid.stories.tsx @@ -44,6 +44,7 @@ const savedObject: SavedObject = { install_status: 'installed', install_source: 'registry', install_started_at: '2020-01-01T00:00:00.000Z', + keep_policies_up_to_date: false, }, references: [], }; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 64e0edd8848a4..f10f2d26a4cc4 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -7,11 +7,7 @@ import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'src/core/server'; -import { - MAX_TIME_COMPLETE_INSTALL, - ASSETS_SAVED_OBJECT_TYPE, - AUTO_UPDATE_PACKAGES, -} from '../../../../common'; +import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE } from '../../../../common'; import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import type { AssetReference, Installation, InstallType } from '../../../types'; diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 348b4bef59b30..e57899531e939 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -618,6 +618,7 @@ const expectAssetsInstalled = ({ install_status: 'installed', install_started_at: res.attributes.install_started_at, install_source: 'registry', + keep_policies_up_to_date: false, }); }); }; diff --git a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts index 390be9bf6ea19..3516eccf9bb15 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/update_assets.ts @@ -432,6 +432,7 @@ export default function (providerContext: FtrProviderContext) { install_status: 'installed', install_started_at: res.attributes.install_started_at, install_source: 'registry', + keep_policies_up_to_date: false, }); }); }); From 532b8961ecf5b90691bba8aaa3dca934a5be3ab4 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 27 Sep 2021 20:36:53 -0400 Subject: [PATCH 11/21] Fix more type errors --- .../plugins/security_solution/common/endpoint/generate_data.ts | 1 + .../server/endpoint/routes/metadata/metadata.test.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index a1b8ca98afc20..1492e0e8c82c9 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1663,6 +1663,7 @@ export class EndpointDocGenerator extends BaseDataGenerator { install_status: 'installed', install_started_at: '2020-06-24T14:41:23.098Z', install_source: 'registry', + keep_policies_up_to_date: false, }, references: [], updated_at: '2020-06-24T14:41:23.098Z', diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts index bb1917ba12323..143bdc1f7eec6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts @@ -120,6 +120,7 @@ describe('test endpoint route', () => { type: ElasticsearchAssetType.transform, }, ], + keep_policies_up_to_date: false, }) ); endpointAppContextService.start({ ...startContract, packageService: mockPackageService }); From 3aa5e4e01d64dc55f2a0efa61a961147a1718205 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 Sep 2021 08:05:33 -0400 Subject: [PATCH 12/21] Fix type error in isolation test --- .../server/endpoint/routes/actions/isolation.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts index 0e25bb3561788..8089cc720fb7a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts @@ -151,6 +151,7 @@ describe('Host Isolation', () => { type: ElasticsearchAssetType.transform, }, ], + keep_policies_up_to_date: false, }) ); licenseEmitter = new Subject(); From 31d27e9b744ea76882f53da64ec69e09222ef9e2 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 Sep 2021 10:54:47 -0400 Subject: [PATCH 13/21] Fix package fixtures types --- x-pack/plugins/fleet/storybook/context/fixtures/packages.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/fleet/storybook/context/fixtures/packages.ts b/x-pack/plugins/fleet/storybook/context/fixtures/packages.ts index 251024a4e7cdb..2a6012272d4b8 100644 --- a/x-pack/plugins/fleet/storybook/context/fixtures/packages.ts +++ b/x-pack/plugins/fleet/storybook/context/fixtures/packages.ts @@ -985,6 +985,7 @@ export const response: GetPackagesResponse['response'] = [ install_status: 'installed', install_started_at: '2021-08-25T19:44:41.090Z', install_source: 'registry', + keep_policies_up_to_date: false, }, references: [], coreMigrationVersion: '7.14.0', @@ -1113,6 +1114,7 @@ export const response: GetPackagesResponse['response'] = [ install_status: 'installed', install_started_at: '2021-08-25T19:44:37.078Z', install_source: 'registry', + keep_policies_up_to_date: false, }, references: [], coreMigrationVersion: '7.14.0', @@ -4268,6 +4270,7 @@ export const response: GetPackagesResponse['response'] = [ install_status: 'installed', install_started_at: '2021-08-25T19:44:43.380Z', install_source: 'registry', + keep_policies_up_to_date: false, }, references: [], coreMigrationVersion: '7.14.0', From d1f4d3a15d5f99b7d31c4c951769b9db8d82dc7e Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 Sep 2021 11:24:30 -0400 Subject: [PATCH 14/21] Fix another type error --- .../server/endpoint/routes/metadata/metadata.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts index db647c7c8462a..d9016e7a9c7cb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts @@ -391,6 +391,7 @@ describe('test endpoint route', () => { type: ElasticsearchAssetType.transform, }, ], + keep_policies_up_to_date: false, }) ); endpointAppContextService.start({ ...startContract, packageService: mockPackageService }); From bea42e9431931aeaf5479af0045b94d7f3744203 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 Sep 2021 12:57:40 -0400 Subject: [PATCH 15/21] Move policy upgrade error swallowing up a level in setup --- .../server/services/managed_package_policies.ts | 8 ++------ .../fleet/server/services/preconfiguration.ts | 13 +++++++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.ts b/x-pack/plugins/fleet/server/services/managed_package_policies.ts index 780e1ea983022..73f85525f4c60 100644 --- a/x-pack/plugins/fleet/server/services/managed_package_policies.ts +++ b/x-pack/plugins/fleet/server/services/managed_package_policies.ts @@ -52,11 +52,7 @@ export const upgradeManagedPackagePolicies = async ( .debug( `Upgrading ${policyIdsToUpgrade.length} package policies: ${policyIdsToUpgrade.join(', ')}` ); - try { - await packagePolicyService.upgrade(soClient, esClient, policyIdsToUpgrade); - // Swallow errors from upgrades - } catch (error) { - appContextService.getLogger().error(error); - } + + await packagePolicyService.upgrade(soClient, esClient, policyIdsToUpgrade); } }; diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 782cb504a92bb..a444f8bdaa4da 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -314,11 +314,16 @@ export async function ensurePreconfiguredPackagesAndPolicies( } } - const fulfilledPolicyPackagePolicyIds = fulfilledPolicies.flatMap( - ({ policy }) => policy?.package_policies as string[] - ); + try { + const fulfilledPolicyPackagePolicyIds = fulfilledPolicies.flatMap( + ({ policy }) => policy?.package_policies as string[] + ); - await upgradeManagedPackagePolicies(soClient, esClient, fulfilledPolicyPackagePolicyIds); + await upgradeManagedPackagePolicies(soClient, esClient, fulfilledPolicyPackagePolicyIds); + // Swallow errors that occur when upgrading + } catch (error) { + appContextService.getLogger().error(error); + } return { policies: fulfilledPolicies.map((p) => From f332f7e813073b03a002f1f2da85d5a55bd82967 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 28 Sep 2021 16:01:37 -0400 Subject: [PATCH 16/21] Address PR feedback - Move keep policies up to date switch to separate component - Use PACKAGE_POLICY_SAVED_OBJECT_TYPE instead of magic string --- .../epm/screens/detail/components/index.tsx | 1 + .../keep_policies_up_to_date_switch.tsx | 46 +++++++++++++++++++ .../epm/screens/detail/settings/settings.tsx | 39 ++++++---------- .../services/epm/packages/_install_package.ts | 8 +++- 4 files changed, 68 insertions(+), 26 deletions(-) create mode 100644 x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/keep_policies_up_to_date_switch.tsx diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/index.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/index.tsx index 8424fecad08cd..8716d78dfb7bd 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/index.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/index.tsx @@ -7,3 +7,4 @@ export { UpdateIcon } from './update_icon'; export { IntegrationAgentPolicyCount } from './integration_agent_policy_count'; export { IconPanel, LoadingIconPanel } from './icon_panel'; +export { KeepPoliciesUpToDateSwitch } from './keep_policies_up_to_date_switch'; diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/keep_policies_up_to_date_switch.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/keep_policies_up_to_date_switch.tsx new file mode 100644 index 0000000000000..751282cc42288 --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/components/keep_policies_up_to_date_switch.tsx @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiSwitch, EuiSpacer, EuiText, EuiFlexGroup, EuiFlexItem, EuiIcon } from '@elastic/eui'; + +interface Props { + checked: boolean; + onChange: () => void; +} + +export const KeepPoliciesUpToDateSwitch: React.FunctionComponent = ({ + checked, + onChange, +}) => ( + <> + + + + + + + + + + + + + +); diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx index b2fb819d62c45..a4aa877164e74 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx @@ -19,8 +19,6 @@ import { EuiText, EuiSpacer, EuiLink, - EuiSwitch, - EuiIcon, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; @@ -43,6 +41,8 @@ import { import { toMountPoint } from '../../../../../../../../../../../src/plugins/kibana_react/public'; +import { KeepPoliciesUpToDateSwitch } from '../components'; + import { InstallButton } from './install_button'; import { UpdateButton } from './update_button'; import { UninstallButton } from './uninstall_button'; @@ -131,16 +131,22 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { notifications.toasts.addSuccess({ title: toMountPoint( + + ), + text: toMountPoint( !keepPoliciesUpToDateSwitchValue ? ( ) : ( ) @@ -148,7 +154,10 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { }); } catch (error) { notifications.toasts.addError(error, { - title: i18n.translate('xpack.fleet.integrations.keepPoliciesUpToDateError', { + title: i18n.translate('xpack.fleet.integrations.integrationSavedError', { + defaultMessage: 'Error saving integration settings', + }), + toastMessage: i18n.translate('xpack.fleet.integrations.keepPoliciesUpToDateError', { defaultMessage: 'Error saving integration settings for {title}', values: { title }, }), @@ -271,28 +280,10 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { {shouldShowKeepPoliciesUpToDateSwitch && ( <> - - - - - - - - - - - - )} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index f10f2d26a4cc4..c189825afcc32 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -7,7 +7,11 @@ import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } from 'src/core/server'; -import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE } from '../../../../common'; +import { + MAX_TIME_COMPLETE_INSTALL, + ASSETS_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, +} from '../../../../common'; import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import type { AssetReference, Installation, InstallType } from '../../../types'; @@ -210,7 +214,7 @@ export async function _installPackage({ const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, { page: 1, perPage: 10000, - kuery: `ingest-package-policies.package.name:${pkgName}`, + kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`, }); await packagePolicyService.upgrade(savedObjectsClient, esClient, policyIdsToUpgrade.items); From 08811a1054698fd9ead5f599ed537a4bdc14ee47 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 4 Oct 2021 11:44:56 -0400 Subject: [PATCH 17/21] Fix overwriting user values when upgrading Fixes #113731 --- .../fleet/server/services/package_policy.ts | 11 +- .../apis/package_policy/upgrade.ts | 116 ++++++++++++++++++ 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 9b02d6eaff495..93f04a55d233b 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -980,7 +980,7 @@ export function overridePackageInputs( ({ name }) => name === input.policy_template ); - // Ignore any policy template removes in the new package version + // Ignore any policy templates removed in the new package version if (!policyTemplate) { return false; } @@ -1000,7 +1000,7 @@ export function overridePackageInputs( // If there's no corresponding input on the original package policy, just // take the override value from the new package as-is. This case typically - // occurs when inputs or package policies are added/removed between versions. + // occurs when inputs or package policy templates are added/removed between versions. if (originalInput === undefined) { inputs.push(override as NewPackagePolicyInput); continue; @@ -1092,7 +1092,14 @@ function deepMergeVars(original: any, override: any): any { for (const { name, ...overrideVal } of overrideVars) { const originalVar = original.vars[name]; + result.vars[name] = { ...originalVar, ...overrideVal }; + + // Ensure that any value from the original object is persisted on the newly merged resulting object, + // even if we merge other data about the given variable + if (originalVar?.value) { + result.vars[name].value = originalVar.value; + } } return result; diff --git a/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts b/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts index 3a7d6f5d6b19e..0be2d7d0a7468 100644 --- a/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts +++ b/x-pack/test/fleet_api_integration/apis/package_policy/upgrade.ts @@ -162,6 +162,122 @@ export default function (providerContext: FtrProviderContext) { }); }); + describe('when upgrading to a version with no breaking changes', function () { + withTestPackageVersion('0.2.5-non-breaking-change'); + + beforeEach(async function () { + const { body: agentPolicyResponse } = await supertest + .post(`/api/fleet/agent_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Test policy', + namespace: 'default', + }) + .expect(200); + + agentPolicyId = agentPolicyResponse.item.id; + + const { body: packagePolicyResponse } = await supertest + .post(`/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'package_policy_upgrade_1', + description: '', + namespace: 'default', + policy_id: agentPolicyId, + enabled: true, + output_id: '', + inputs: [ + { + policy_template: 'package_policy_upgrade', + type: 'test_input', + enabled: true, + streams: [ + { + id: 'test-package_policy_upgrade-xxxx', + enabled: true, + data_stream: { + type: 'test_stream', + dataset: 'package_policy_upgrade.test_stream', + }, + vars: { + test_var: { + value: 'My custom test value', + }, + }, + }, + ], + }, + ], + package: { + name: 'package_policy_upgrade', + title: 'This is a test package for upgrading package policies', + version: '0.2.0-add-non-required-test-var', + }, + }) + .expect(200); + + packagePolicyId = packagePolicyResponse.item.id; + }); + + afterEach(async function () { + await supertest + .post(`/api/fleet/package_policies/delete`) + .set('kbn-xsrf', 'xxxx') + .send({ packagePolicyIds: [packagePolicyId] }) + .expect(200); + + await supertest + .post('/api/fleet/agent_policies/delete') + .set('kbn-xsrf', 'xxxx') + .send({ agentPolicyId }) + .expect(200); + }); + + describe('dry run', function () { + it('returns a valid diff', async function () { + const { body }: { body: UpgradePackagePolicyDryRunResponse } = await supertest + .post(`/api/fleet/package_policies/upgrade`) + .set('kbn-xsrf', 'xxxx') + .send({ + packagePolicyIds: [packagePolicyId], + dryRun: true, + }) + .expect(200); + + expect(body.length).to.be(1); + expect(body[0].diff?.length).to.be(2); + expect(body[0].hasErrors).to.be(false); + + const [currentPackagePolicy, proposedPackagePolicy] = body[0].diff ?? []; + + expect(currentPackagePolicy?.package?.version).to.be('0.2.0-add-non-required-test-var'); + expect(proposedPackagePolicy?.package?.version).to.be('0.2.5-non-breaking-change'); + + const testInput = proposedPackagePolicy?.inputs.find(({ type }) => type === 'test_input'); + const testStream = testInput?.streams[0]; + + expect(testStream?.vars?.test_var.value).to.be('My custom test value'); + }); + }); + + describe('upgrade', function () { + it('successfully upgrades package policy', async function () { + const { body }: { body: UpgradePackagePolicyResponse } = await supertest + .post(`/api/fleet/package_policies/upgrade`) + .set('kbn-xsrf', 'xxxx') + .send({ + packagePolicyIds: [packagePolicyId], + dryRun: false, + }) + .expect(200); + + expect(body.length).to.be(1); + expect(body[0].success).to.be(true); + }); + }); + }); + describe('when upgrading to a version where a non-required variable has been added', function () { withTestPackageVersion('0.2.0-add-non-required-test-var'); From 7ef307ac84552f9488ff040df3f17a83d032ac58 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 4 Oct 2021 11:45:43 -0400 Subject: [PATCH 18/21] Add test package --- .../test_stream/agent/stream/stream.yml.hbs | 1 + .../data_stream/test_stream/fields/fields.yml | 16 +++++++++++++ .../data_stream/test_stream/manifest.yml | 15 ++++++++++++ .../0.2.5-non-breaking-change/docs/README.md | 3 +++ .../0.2.5-non-breaking-change/manifest.yml | 23 +++++++++++++++++++ 5 files changed, 58 insertions(+) create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/agent/stream/stream.yml.hbs create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/fields/fields.yml create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/manifest.yml create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/docs/README.md create mode 100644 x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/manifest.yml diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/agent/stream/stream.yml.hbs b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000000..2870385f21f95 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/agent/stream/stream.yml.hbs @@ -0,0 +1 @@ +config.version: "2" diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/fields/fields.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/fields/fields.yml new file mode 100644 index 0000000000000..6e003ed0ad147 --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/fields/fields.yml @@ -0,0 +1,16 @@ +- name: data_stream.type + type: constant_keyword + description: > + Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: > + Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: > + Data stream namespace. +- name: '@timestamp' + type: date + description: > + Event timestamp. diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/manifest.yml new file mode 100644 index 0000000000000..95b72f0122aec --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/data_stream/test_stream/manifest.yml @@ -0,0 +1,15 @@ +title: Test stream +type: logs +streams: + - input: test_input + vars: + - name: test_var + type: text + title: Test Var + show_user: true + default: Test Value + - name: test_var_2 + type: text + title: Test Var 2 + show_user: true + default: Test Value 2 diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/docs/README.md b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/docs/README.md new file mode 100644 index 0000000000000..0b9b18421c9dc --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/docs/README.md @@ -0,0 +1,3 @@ +# Test package + +This is a test package for testing automated upgrades for package policies diff --git a/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/manifest.yml b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/manifest.yml new file mode 100644 index 0000000000000..2105ee451ffae --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fixtures/test_packages/package_policy_upgrade/0.2.5-non-breaking-change/manifest.yml @@ -0,0 +1,23 @@ +format_version: 1.0.0 +name: package_policy_upgrade +title: Tests package policy upgrades +description: This is a test package for upgrading package policies +version: 0.2.5-non-breaking-change +categories: [] +release: beta +type: integration +license: basic +requirement: + elasticsearch: + versions: '>7.7.0' + kibana: + versions: '>7.7.0' +policy_templates: + - name: package_policy_upgrade + title: Package Policy Upgrade + description: Test Package for Upgrading Package Policies + inputs: + - type: test_input + title: Test Input + description: Test Input + enabled: true From 5cc5712fcc5cab714c9051b0e5de863a7514e8d3 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 4 Oct 2021 15:15:28 -0400 Subject: [PATCH 19/21] Fix tests for overridePackageVars --- .../server/services/package_policy.test.ts | 103 +++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index 31f1440135436..c04134e97b415 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -1085,7 +1085,98 @@ describe('Package policy service', () => { }); describe('overridePackageInputs', () => { - it('should override variable in base package policy', () => { + describe('when variable is already defined', () => { + it('preserves original variable value without overwriting', () => { + const basePackagePolicy: NewPackagePolicy = { + name: 'base-package-policy', + description: 'Base Package Policy', + namespace: 'default', + enabled: true, + policy_id: 'xxxx', + output_id: 'xxxx', + package: { + name: 'test-package', + title: 'Test Package', + version: '0.0.1', + }, + inputs: [ + { + type: 'logs', + policy_template: 'template_1', + enabled: true, + vars: { + path: { + type: 'text', + value: ['/var/log/logfile.log'], + }, + }, + streams: [], + }, + ], + }; + + const packageInfo: PackageInfo = { + name: 'test-package', + description: 'Test Package', + title: 'Test Package', + version: '0.0.1', + latestVersion: '0.0.1', + release: 'experimental', + format_version: '1.0.0', + owner: { github: 'elastic/fleet' }, + policy_templates: [ + { + name: 'template_1', + title: 'Template 1', + description: 'Template 1', + inputs: [ + { + type: 'logs', + title: 'Log', + description: 'Log Input', + vars: [ + { + name: 'path', + type: 'text', + }, + ], + }, + ], + }, + ], + // @ts-ignore + assets: {}, + }; + + const inputsOverride: NewPackagePolicyInput[] = [ + { + type: 'logs', + enabled: true, + streams: [], + vars: { + path: { + type: 'text', + value: '/var/log/new-logfile.log', + }, + }, + }, + ]; + + const result = overridePackageInputs( + basePackagePolicy, + packageInfo, + // TODO: Update this type assertion when the `InputsOverride` type is updated such + // that it no longer causes unresolvable type errors when used directly + inputsOverride as InputsOverride[], + false + ); + expect(result.inputs[0]?.vars?.path.value).toEqual(['/var/log/logfile.log']); + }); + }); + }); + + describe('when variable is undefined in original object', () => { + it('adds the variable definition to the resulting object', () => { const basePackagePolicy: NewPackagePolicy = { name: 'base-package-policy', description: 'Base Package Policy', @@ -1138,6 +1229,10 @@ describe('Package policy service', () => { name: 'path', type: 'text', }, + { + name: 'path_2', + type: 'text', + }, ], }, ], @@ -1157,6 +1252,10 @@ describe('Package policy service', () => { type: 'text', value: '/var/log/new-logfile.log', }, + path_2: { + type: 'text', + value: '/var/log/custom.log', + }, }, }, ]; @@ -1169,7 +1268,7 @@ describe('Package policy service', () => { inputsOverride as InputsOverride[], false ); - expect(result.inputs[0]?.vars?.path.value).toBe('/var/log/new-logfile.log'); + expect(result.inputs[0]?.vars?.path_2.value).toEqual('/var/log/custom.log'); }); }); }); From a513f054849228ebf4ad4eebb5656bf6e4186dc6 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 5 Oct 2021 09:34:57 -0400 Subject: [PATCH 20/21] Address PR feedback - Don't index keep_policies_up_to_date field - Use SO_SEARCH_LIMIT constant instead of magic number --- x-pack/plugins/fleet/server/saved_objects/index.ts | 2 +- .../fleet/server/services/epm/packages/_install_package.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 0ddfd59dc7705..ac5ca401da000 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -299,7 +299,7 @@ const getSavedObjectTypes = ( version: { type: 'keyword' }, internal: { type: 'boolean' }, removable: { type: 'boolean' }, - keep_policies_up_to_date: { type: 'boolean' }, + keep_policies_up_to_date: { type: 'boolean', index: false }, es_index_patterns: { enabled: false, type: 'object', diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index c189825afcc32..9f66b5dd379ec 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -11,6 +11,7 @@ import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE, + SO_SEARCH_LIMIT, } from '../../../../common'; import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; @@ -213,7 +214,7 @@ export async function _installPackage({ if (updatedPackage.attributes.keep_policies_up_to_date) { const policyIdsToUpgrade = await packagePolicyService.listIds(savedObjectsClient, { page: 1, - perPage: 10000, + perPage: SO_SEARCH_LIMIT, kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName}`, }); From 9f1b9cbef4e2e2051a33cab246de81d475088050 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Tue, 5 Oct 2021 09:44:02 -0400 Subject: [PATCH 21/21] Make toast translation usage more consistent --- .../epm/screens/detail/settings/settings.tsx | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx index a4aa877164e74..185ae10bcafd2 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/settings/settings.tsx @@ -39,8 +39,6 @@ import { DEFAULT_PACKAGES, } from '../../../../../constants'; -import { toMountPoint } from '../../../../../../../../../../../src/plugins/kibana_react/public'; - import { KeepPoliciesUpToDateSwitch } from '../components'; import { InstallButton } from './install_button'; @@ -130,27 +128,20 @@ export const SettingsPage: React.FC = memo(({ packageInfo }: Props) => { }); notifications.toasts.addSuccess({ - title: toMountPoint( - - ), - text: toMountPoint( - !keepPoliciesUpToDateSwitchValue ? ( - - ) : ( - - ) - ), + title: i18n.translate('xpack.fleet.integrations.integrationSaved', { + defaultMessage: 'Integration settings saved', + }), + text: !keepPoliciesUpToDateSwitchValue + ? i18n.translate('xpack.fleet.integrations.keepPoliciesUpToDateEnabledSuccess', { + defaultMessage: + 'Fleet will automatically keep integration policies up to date for {title}', + values: { title }, + }) + : i18n.translate('xpack.fleet.integrations.keepPoliciesUpToDateDisabledSuccess', { + defaultMessage: + 'Fleet will not automatically keep integration policies up to date for {title}', + values: { title }, + }), }); } catch (error) { notifications.toasts.addError(error, {