diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index ac6eca6b3b97d..af0770b61f180 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -125,6 +125,8 @@ export class SigningServiceNotFoundError extends FleetNotFoundError {} export class InputNotFoundError extends FleetNotFoundError {} export class OutputNotFoundError extends FleetNotFoundError {} export class PackageNotFoundError extends FleetNotFoundError {} +export class ArchiveNotFoundError extends FleetNotFoundError {} + export class PackagePolicyNotFoundError extends FleetNotFoundError<{ /** The package policy ID that was not found */ packagePolicyId: string; diff --git a/x-pack/plugins/fleet/server/routes/epm/file_handler.ts b/x-pack/plugins/fleet/server/routes/epm/file_handler.ts index 701915d384f47..8ae344cfc6264 100644 --- a/x-pack/plugins/fleet/server/routes/epm/file_handler.ts +++ b/x-pack/plugins/fleet/server/routes/epm/file_handler.ts @@ -105,10 +105,17 @@ export const getFileHandler: FleetRequestHandler< }); } else { const registryResponse = await getFile(pkgName, pkgVersion, filePath); + if (!registryResponse) + return response.custom({ + body: {}, + statusCode: 400, + }); + const headersToProxy: KnownHeaders[] = ['content-type']; const proxiedHeaders = headersToProxy.reduce((headers, knownHeader) => { - const value = registryResponse.headers.get(knownHeader); - if (value !== null) { + const value = registryResponse?.headers.get(knownHeader); + + if (!!value) { headers[knownHeader] = value; } return headers; diff --git a/x-pack/plugins/fleet/server/services/epm/airgapped.ts b/x-pack/plugins/fleet/server/services/epm/airgapped.ts new file mode 100644 index 0000000000000..4507de3ea15eb --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/airgapped.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { appContextService } from '..'; + +export const airGappedUtils = () => { + const config = appContextService.getConfig(); + const hasRegistryUrls = config?.registryUrl || config?.registryProxyUrl; + const isAirGapped = config?.isAirGapped; + + const shouldSkipRegistryRequests = isAirGapped && !hasRegistryUrls; + + return { hasRegistryUrls, isAirGapped, shouldSkipRegistryRequests }; +}; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts index 5711c8fcccaf4..c3b0eee26cb5c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts @@ -925,6 +925,32 @@ owner: elastic`, }); }); + it('should avoid loading archive when isAirGapped == true', async () => { + const mockContract = createAppContextStartContractMock({ isAirGapped: true }); + appContextService.start(mockContract); + + const soClient = savedObjectsClientMock.create(); + soClient.get.mockRejectedValue(SavedObjectsErrorHelpers.createGenericNotFoundError()); + MockRegistry.fetchInfo.mockResolvedValue({ + name: 'my-package', + version: '1.0.0', + assets: [], + } as unknown as RegistryPackage); + + await expect( + getPackageInfo({ + savedObjectsClient: soClient, + pkgName: 'my-package', + pkgVersion: '1.0.0', + }) + ).resolves.toMatchObject({ + latestVersion: '1.0.0', + status: 'not_installed', + }); + + expect(MockRegistry.getPackage).not.toHaveBeenCalled(); + }); + describe('installation status', () => { it('should be not_installed when no package SO exists', async () => { const soClient = savedObjectsClientMock.create(); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 5b0e3df279cdc..824d3f4b30f72 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -68,6 +68,8 @@ import { auditLoggingService } from '../../audit_logging'; import { getFilteredSearchPackages } from '../filtered_packages'; +import { airGappedUtils } from '../airgapped'; + import { createInstallableFrom } from '.'; import { getPackageAssetsMapCache, @@ -478,7 +480,11 @@ export async function getPackageInfo({ let packageInfo; // We need to get input only packages from source to get all fields // see https://github.com/elastic/package-registry/issues/864 - if (registryInfo && skipArchive && registryInfo.type !== 'input') { + if ( + registryInfo && + (skipArchive || airGappedUtils().shouldSkipRegistryRequests) && + registryInfo.type !== 'input' + ) { packageInfo = registryInfo; // Fix the paths paths = @@ -629,6 +635,7 @@ export async function getPackageFromSource(options: { } catch (err) { if (err instanceof RegistryResponseError && err.status === 404) { res = await Registry.getBundledArchive(pkgName, pkgVersion); + logger.debug(`retrieved bundled package ${pkgName}-${pkgVersion}`); } else { throw err; } @@ -763,7 +770,7 @@ export async function getPackageAssetsMap({ packageInfo: PackageInfo; logger: Logger; ignoreUnverified?: boolean; -}) { +}): Promise { const cache = getPackageAssetsMapCache(packageInfo.name, packageInfo.version); if (cache) { return cache; @@ -774,17 +781,22 @@ export async function getPackageAssetsMap({ logger, }); - let assetsMap: AssetsMap | undefined; - if (installedPackageWithAssets?.installation.version !== packageInfo.version) { - // Try to get from registry - const pkg = await Registry.getPackage(packageInfo.name, packageInfo.version, { - ignoreUnverified, - }); - assetsMap = pkg.assetsMap; - } else { - assetsMap = installedPackageWithAssets.assetsMap; - } - setPackageAssetsMapCache(packageInfo.name, packageInfo.version, assetsMap); + try { + let assetsMap: AssetsMap | undefined; + if (installedPackageWithAssets?.installation.version !== packageInfo.version) { + // Try to get from registry + const pkg = await Registry.getPackage(packageInfo.name, packageInfo.version, { + ignoreUnverified, + }); + assetsMap = pkg.assetsMap; + } else { + assetsMap = installedPackageWithAssets.assetsMap; + } + setPackageAssetsMapCache(packageInfo.name, packageInfo.version, assetsMap); - return assetsMap; + return assetsMap; + } catch (error) { + logger.warn(`getPackageAssetsMap error: ${error}`); + throw error; + } } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index ebe5acc35178d..07d7392971d74 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -1354,44 +1354,48 @@ export async function installAssetsForInputPackagePolicy(opts: { `Error while creating index templates: unable to find installed package ${pkgInfo.name}` ); } - if (installedPkgWithAssets.installation.version !== pkgInfo.version) { - const pkg = await Registry.getPackage(pkgInfo.name, pkgInfo.version, { - ignoreUnverified: force, - }); + try { + if (installedPkgWithAssets.installation.version !== pkgInfo.version) { + const pkg = await Registry.getPackage(pkgInfo.name, pkgInfo.version, { + ignoreUnverified: force, + }); - const archiveIterator = createArchiveIteratorFromMap(pkg.assetsMap); - packageInstallContext = { - assetsMap: pkg.assetsMap, - packageInfo: pkg.packageInfo, - paths: pkg.paths, - archiveIterator, - }; - } else { - const archiveIterator = createArchiveIteratorFromMap(installedPkgWithAssets.assetsMap); - packageInstallContext = { - assetsMap: installedPkgWithAssets.assetsMap, - packageInfo: installedPkgWithAssets.packageInfo, - paths: installedPkgWithAssets.paths, - archiveIterator, - }; - } + const archiveIterator = createArchiveIteratorFromMap(pkg.assetsMap); + packageInstallContext = { + assetsMap: pkg.assetsMap, + packageInfo: pkg.packageInfo, + paths: pkg.paths, + archiveIterator, + }; + } else { + const archiveIterator = createArchiveIteratorFromMap(installedPkgWithAssets.assetsMap); + packageInstallContext = { + assetsMap: installedPkgWithAssets.assetsMap, + packageInfo: installedPkgWithAssets.packageInfo, + paths: installedPkgWithAssets.paths, + archiveIterator, + }; + } - await installIndexTemplatesAndPipelines({ - installedPkg: installedPkgWithAssets.installation, - packageInstallContext, - esReferences: installedPkgWithAssets.installation.installed_es || [], - savedObjectsClient: soClient, - esClient, - logger, - onlyForDataStreams: [dataStream], - }); - // Upate ES index patterns - await optimisticallyAddEsAssetReferences( - soClient, - installedPkgWithAssets.installation.name, - [], - generateESIndexPatterns([dataStream]) - ); + await installIndexTemplatesAndPipelines({ + installedPkg: installedPkgWithAssets.installation, + packageInstallContext, + esReferences: installedPkgWithAssets.installation.installed_es || [], + savedObjectsClient: soClient, + esClient, + logger, + onlyForDataStreams: [dataStream], + }); + // Upate ES index patterns + await optimisticallyAddEsAssetReferences( + soClient, + installedPkgWithAssets.installation.name, + [], + generateESIndexPatterns([dataStream]) + ); + } catch (error) { + logger.warn(`installAssetsForInputPackagePolicy error: ${error}`); + } } interface NoPkgArgs { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts index 92068dfcd424d..d95d387322120 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/package_verification.ts @@ -75,7 +75,7 @@ export async function verifyPackageArchiveSignature({ }: { pkgName: string; pkgVersion: string; - pkgArchiveBuffer: Buffer; + pkgArchiveBuffer: Buffer | undefined; logger: Logger; }): Promise { const verificationKey = await getGpgKeyOrUndefined(); @@ -97,6 +97,11 @@ export async function verifyPackageArchiveSignature({ return result; } + if (!pkgArchiveBuffer) { + logger.warn(`Archive not found for package ${pkgName}-${pkgVersion}. Skipping verification.`); + return result; + } + const { isVerified, keyId } = await _verifyPackageSignature({ pkgArchiveBuffer, pkgArchiveSignature, diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts index 6de0577e33238..34a474c0900e8 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.test.ts @@ -84,6 +84,7 @@ describe('splitPkgKey', () => { describe('fetch package', () => { afterEach(() => { mockFetchUrl.mockReset(); + mockGetConfig.mockReset(); mockGetBundledPackageByName.mockReset(); }); @@ -100,6 +101,22 @@ describe('fetch package', () => { expect(result).toEqual(registryPackage); }); + it('Should return bundled package when isAirGapped = true', async () => { + mockGetConfig.mockReturnValue({ + isAirGapped: true, + enabled: true, + agents: { enabled: true, elasticsearch: {} }, + }); + const bundledPackage = { name: 'testpkg', version: '1.0.0' }; + const registryPackage = { name: 'testpkg', version: '1.0.1' }; + + mockFetchUrl.mockResolvedValue(JSON.stringify([registryPackage])); + + mockGetBundledPackageByName.mockResolvedValue(bundledPackage); + const result = await fetchMethodToTest('testpkg'); + expect(result).toEqual(bundledPackage); + }); + it('Should return bundled package if bundled package is newer version', async () => { const bundledPackage = { name: 'testpkg', version: '1.0.1' }; const registryPackage = { name: 'testpkg', version: '1.0.0' }; @@ -220,6 +237,15 @@ describe('fetchInfo', () => { expect(e).toBeInstanceOf(PackageNotFoundError); } }); + + it('falls back to bundled package when isAirGapped config == true', async () => { + mockGetConfig.mockReturnValue({ + isAirGapped: true, + }); + + const fetchedInfo = await fetchInfo('test-package', '1.0.0'); + expect(fetchedInfo).toBeTruthy(); + }); }); describe('fetchCategories', () => { @@ -317,6 +343,13 @@ describe('fetchList', () => { expect(callUrl.searchParams.get('capabilities')).toBeNull(); }); + it('does not call registry if isAirGapped == true', async () => { + mockGetConfig.mockReturnValue({ isAirGapped: true }); + mockFetchUrl.mockResolvedValue(JSON.stringify([])); + await fetchList(); + expect(mockFetchUrl).toBeCalledTimes(0); + }); + it('does call registry with kibana.version if not explictly disabled', async () => { mockGetConfig.mockReturnValue({ internal: { diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 75b9869d0a7c6..3fd0e09578ca7 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -46,6 +46,7 @@ import { RegistryResponseError, PackageFailedVerificationError, PackageUnsupportedMediaTypeError, + ArchiveNotFoundError, } from '../../../errors'; import { getBundledPackageByName } from '../packages/bundled_packages'; @@ -56,6 +57,8 @@ import { verifyPackageArchiveSignature } from '../packages/package_verification' import type { ArchiveIterator } from '../../../../common/types'; +import { airGappedUtils } from '../airgapped'; + import { fetchUrl, getResponse, getResponseStream } from './requests'; import { getRegistryUrl } from './registry_url'; @@ -67,6 +70,15 @@ export const pkgToPkgKey = ({ name, version }: { name: string; version: string } export async function fetchList( params?: GetPackagesRequest['query'] ): Promise { + if (airGappedUtils().shouldSkipRegistryRequests) { + appContextService + .getLogger() + .debug( + 'fetchList: isAirGapped enabled and no registryUrl or RegistryProxyUrl configured, skipping registry requests' + ); + return []; + } + const registryUrl = getRegistryUrl(); const url = new URL(`${registryUrl}/search`); if (params) { @@ -106,16 +118,24 @@ async function _fetchFindLatestPackage( } try { - const res = await fetchUrl(url.toString(), 1); - const searchResults: RegistryPackage[] = JSON.parse(res); - - const latestPackageFromRegistry = searchResults[0] ?? null; - - if (bundledPackage && semverGte(bundledPackage.version, latestPackageFromRegistry.version)) { + if (!airGappedUtils().shouldSkipRegistryRequests) { + const res = await fetchUrl(url.toString(), 1); + const searchResults: RegistryPackage[] = JSON.parse(res); + + const latestPackageFromRegistry = searchResults[0] ?? null; + + if ( + bundledPackage && + semverGte(bundledPackage.version, latestPackageFromRegistry.version) + ) { + return bundledPackage; + } + return latestPackageFromRegistry; + } else if (airGappedUtils().shouldSkipRegistryRequests && bundledPackage) { return bundledPackage; + } else { + return null; } - - return latestPackageFromRegistry; } catch (error) { logger.error( `Failed to fetch latest version of ${packageName} from registry: ${error.message}` @@ -169,6 +189,13 @@ export async function fetchInfo( pkgVersion: string ): Promise { const registryUrl = getRegistryUrl(); + // if isAirGapped config enabled and bundled package, use the bundled version + if (airGappedUtils().shouldSkipRegistryRequests) { + const archivePackage = await getBundledArchive(pkgName, pkgVersion); + if (archivePackage) { + return archivePackage.packageInfo; + } + } try { // Trailing slash avoids 301 redirect / extra hop const res = await fetchUrl(`${registryUrl}/package/${pkgName}/${pkgVersion}/`).then(JSON.parse); @@ -208,12 +235,20 @@ export async function getFile( pkgName: string, pkgVersion: string, relPath: string -): Promise { +): Promise { const filePath = `/package/${pkgName}/${pkgVersion}/${relPath}`; return fetchFile(filePath); } -export async function fetchFile(filePath: string): Promise { +export async function fetchFile(filePath: string): Promise { + if (airGappedUtils().shouldSkipRegistryRequests) { + appContextService + .getLogger() + .debug( + 'fetchFile: isAirGapped enabled and no registryUrl or RegistryProxyUrl configured, skipping registry requests' + ); + return null; + } const registryUrl = getRegistryUrl(); return getResponse(`${registryUrl}${filePath}`); } @@ -264,6 +299,15 @@ function setConstraints(url: URL) { export async function fetchCategories( params?: GetCategoriesRequest['query'] ): Promise { + if (airGappedUtils().shouldSkipRegistryRequests) { + appContextService + .getLogger() + .debug( + 'fetchCategories: isAirGapped enabled and no registryUrl or RegistryProxyUrl configured, skipping registry requests' + ); + return []; + } + const registryUrl = getRegistryUrl(); const url = new URL(`${registryUrl}/categories`); if (params) { @@ -319,42 +363,52 @@ export async function getPackage( archiveIterator: ArchiveIterator; verificationResult?: PackageVerificationResult; }> { + const logger = appContextService.getLogger(); const verifyPackage = appContextService.getExperimentalFeatures().packageVerification; let packageInfo: ArchivePackage | undefined = getPackageInfo({ name, version }); let verificationResult: PackageVerificationResult | undefined = verifyPackage ? getVerificationResult({ name, version }) : undefined; + try { + const { + archiveBuffer, + archivePath, + verificationResult: latestVerificationResult, + } = await withPackageSpan('Fetch package archive from archive buffer', () => + fetchArchiveBuffer({ + pkgName: name, + pkgVersion: version, + shouldVerify: verifyPackage, + ignoreUnverified: options?.ignoreUnverified, + }) + ); - const { - archiveBuffer, - archivePath, - verificationResult: latestVerificationResult, - } = await withPackageSpan('Fetch package archive from archive buffer', () => - fetchArchiveBuffer({ - pkgName: name, - pkgVersion: version, - shouldVerify: verifyPackage, - ignoreUnverified: options?.ignoreUnverified, - }) - ); - - if (latestVerificationResult) { - verificationResult = latestVerificationResult; - setVerificationResult({ name, version }, latestVerificationResult); - } + if (latestVerificationResult) { + verificationResult = latestVerificationResult; + setVerificationResult({ name, version }, latestVerificationResult); + } - const contentType = ensureContentType(archivePath); - const { paths, assetsMap, archiveIterator } = await unpackBufferToAssetsMap({ - archiveBuffer, - contentType, - useStreaming: options?.useStreaming, - }); + const contentType = ensureContentType(archivePath); + const { paths, assetsMap, archiveIterator } = await unpackBufferToAssetsMap({ + archiveBuffer, + contentType, + useStreaming: options?.useStreaming, + }); - if (!packageInfo) { - packageInfo = await getPackageInfoFromArchiveOrCache(name, version, archiveBuffer, archivePath); - } + if (!packageInfo) { + packageInfo = await getPackageInfoFromArchiveOrCache( + name, + version, + archiveBuffer, + archivePath + ); + } - return { paths, packageInfo, assetsMap, archiveIterator, verificationResult }; + return { paths, packageInfo, assetsMap, archiveIterator, verificationResult }; + } catch (error) { + logger.warn(`getPackage failed with error: ${error}`); + throw error; + } } export async function getPackageFieldsMetadata( @@ -367,36 +421,40 @@ export async function getPackageFieldsMetadata( // Attempt retrieving latest package name and version const latestPackage = await fetchFindLatestPackageOrThrow(packageName); const { name, version } = latestPackage; - - // Attempt retrieving latest package - const resolvedPackage = await getPackage(name, version); - - // We need to collect all the available data streams for the package. - // In case a dataset is specified from the parameter, it will load the fields only for that specific dataset. - // As a fallback case, we'll try to read the fields for all the data streams in the package. - const dataStreamsMap = resolveDataStreamsMap(resolvedPackage.packageInfo.data_streams); - - const { assetsMap } = resolvedPackage; - - const dataStream = datasetName ? dataStreamsMap.get(datasetName) : null; - - if (dataStream) { - // Resolve a single data stream fields when the `datasetName` parameter is specified - return resolveDataStreamFields({ dataStream, assetsMap, excludedFieldsAssets }); - } else { - // Resolve and merge all the integration data streams fields otherwise - return [...dataStreamsMap.values()].reduce( - (packageDataStreamsFields, currentDataStream) => - Object.assign( - packageDataStreamsFields, - resolveDataStreamFields({ - dataStream: currentDataStream, - assetsMap, - excludedFieldsAssets, - }) - ), - {} - ); + try { + // Attempt retrieving latest package + const resolvedPackage = await getPackage(name, version); + + // We need to collect all the available data streams for the package. + // In case a dataset is specified from the parameter, it will load the fields only for that specific dataset. + // As a fallback case, we'll try to read the fields for all the data streams in the package. + const dataStreamsMap = resolveDataStreamsMap(resolvedPackage.packageInfo.data_streams); + + const assetsMap = resolvedPackage.assetsMap; + + const dataStream = datasetName ? dataStreamsMap.get(datasetName) : null; + + if (dataStream) { + // Resolve a single data stream fields when the `datasetName` parameter is specified + return resolveDataStreamFields({ dataStream, assetsMap, excludedFieldsAssets }); + } else { + // Resolve and merge all the integration data streams fields otherwise + return [...dataStreamsMap.values()].reduce( + (packageDataStreamsFields, currentDataStream) => + Object.assign( + packageDataStreamsFields, + resolveDataStreamFields({ + dataStream: currentDataStream, + assetsMap, + excludedFieldsAssets, + }) + ), + {} + ); + } + } catch (error) { + appContextService.getLogger().warn(`getPackageFieldsMetadata error: ${error}`); + throw error; } } @@ -436,22 +494,31 @@ export async function fetchArchiveBuffer({ } const registryUrl = getRegistryUrl(); const archiveUrl = `${registryUrl}${archivePath}`; - const archiveBuffer = await getResponseStream(archiveUrl).then(streamToBuffer); - - if (shouldVerify) { - const verificationResult = await verifyPackageArchiveSignature({ - pkgName, - pkgVersion, - pkgArchiveBuffer: archiveBuffer, - logger, - }); + try { + const archiveBuffer = await getResponseStream(archiveUrl).then(streamToBuffer); + if (!archiveBuffer) { + logger.warn(`Archive Buffer not found`); + throw new ArchiveNotFoundError('Archive Buffer not found'); + } + + if (shouldVerify) { + const verificationResult = await verifyPackageArchiveSignature({ + pkgName, + pkgVersion, + pkgArchiveBuffer: archiveBuffer, + logger, + }); - if (verificationResult.verificationStatus === 'unverified' && !ignoreUnverified) { - throw new PackageFailedVerificationError(pkgName, pkgVersion); + if (verificationResult.verificationStatus === 'unverified' && !ignoreUnverified) { + throw new PackageFailedVerificationError(pkgName, pkgVersion); + } + return { archiveBuffer, archivePath, verificationResult }; } - return { archiveBuffer, archivePath, verificationResult }; + return { archiveBuffer, archivePath }; + } catch (error) { + logger.warn(`fetchArchiveBuffer failed with error: ${error}`); + throw error; } - return { archiveBuffer, archivePath }; } export async function getPackageArchiveSignatureOrUndefined({ @@ -473,9 +540,10 @@ export async function getPackageArchiveSignatureOrUndefined({ } try { - const { body } = await fetchFile(signaturePath); + const res = await fetchFile(signaturePath); - return streamToString(body); + if (res?.body) return streamToString(res.body); + return undefined; } catch (e) { logger.error(`Error retrieving package signature at '${signaturePath}' : ${e}`); return undefined; diff --git a/x-pack/plugins/fleet/server/services/epm/registry/requests.test.ts b/x-pack/plugins/fleet/server/services/epm/registry/requests.test.ts index e4e34d25671f6..367f68f50acd2 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/requests.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/requests.test.ts @@ -5,11 +5,17 @@ * 2.0. */ +import { securityMock } from '@kbn/security-plugin/server/mocks'; +import { loggerMock } from '@kbn/logging-mocks'; + +import type { Logger } from '@kbn/core/server'; + import { RegistryError, RegistryConnectionError, RegistryResponseError } from '../../../errors'; import { appContextService } from '../../app_context'; import { fetchUrl, getResponse } from './requests'; jest.mock('node-fetch'); +jest.mock('../../app_context'); let mockRegistryProxyUrl: string | undefined; jest.mock('./proxy', () => ({ @@ -17,11 +23,9 @@ jest.mock('./proxy', () => ({ getRegistryProxyUrl: () => mockRegistryProxyUrl, })); -jest.mock('../../app_context', () => ({ - appContextService: { - getKibanaVersion: jest.fn(), - getLogger: jest.fn().mockReturnValue({ debug: jest.fn() }), - }, +const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), })); const { Response, FetchError } = jest.requireActual('node-fetch'); @@ -29,18 +33,23 @@ const { Response, FetchError } = jest.requireActual('node-fetch'); const fetchMock = require('node-fetch') as jest.Mock; jest.setTimeout(120 * 1000); -describe('Registry request', () => { + +describe('Registry requests', () => { beforeEach(async () => {}); afterEach(async () => { jest.clearAllMocks(); }); + let mockedLogger: jest.Mocked; describe('fetch options', () => { beforeEach(() => { fetchMock.mockImplementationOnce(() => Promise.resolve(new Response(''))); - (appContextService.getKibanaVersion as jest.Mock).mockReturnValue('8.0.0'); + mockedAppContextService.getKibanaVersion.mockReturnValue('8.0.0'); + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); }); + it('should set User-Agent header including kibana version', async () => { await getResponse(''); @@ -219,5 +228,32 @@ describe('Registry request', () => { expect(fetchMock).toHaveBeenCalledTimes(1); }); }); + + describe('with config.isAirGapped == true', () => { + beforeEach(() => { + mockedLogger = loggerMock.create(); + mockedAppContextService.getLogger.mockReturnValue(mockedLogger); + mockedAppContextService.getConfig.mockReturnValue({ + isAirGapped: true, + enabled: true, + agents: { enabled: true, elasticsearch: {} }, + }); + }); + + it('should not call fetch', async () => { + await getResponse(''); + + expect(fetchMock).toHaveBeenCalledTimes(0); + }); + + it('fetchUrl throws error', async () => { + fetchMock.mockImplementationOnce(() => { + throw new Error('mocked'); + }); + const promise = fetchUrl(''); + await expect(promise).rejects.toThrow(RegistryResponseError); + expect(fetchMock).toHaveBeenCalledTimes(0); + }); + }); }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/registry/requests.ts b/x-pack/plugins/fleet/server/services/epm/registry/requests.ts index 47084b601a27e..a67e01408a1e9 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/requests.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/requests.ts @@ -13,6 +13,8 @@ import { streamToString } from '../streams'; import { appContextService } from '../../app_context'; import { RegistryError, RegistryConnectionError, RegistryResponseError } from '../../../errors'; +import { airGappedUtils } from '../airgapped'; + import { getProxyAgent, getRegistryProxyUrl } from './proxy'; type FailedAttemptErrors = pRetry.FailedAttemptError | FetchError | Error; @@ -20,6 +22,7 @@ type FailedAttemptErrors = pRetry.FailedAttemptError | FetchError | Error; // not sure what to call this function, but we're not exporting it async function registryFetch(url: string) { const response = await fetch(url, getFetchOptions(url)); + if (response.ok) { return response; } else { @@ -34,7 +37,16 @@ async function registryFetch(url: string) { } } -export async function getResponse(url: string, retries: number = 5): Promise { +export async function getResponse(url: string, retries: number = 5): Promise { + const logger = appContextService.getLogger(); + + if (airGappedUtils().shouldSkipRegistryRequests) { + logger.debug( + 'getResponse: isAirGapped enabled and no registryUrl or RegistryProxyUrl configured, skipping registry requests' + ); + return null; + } + try { // we only want to retry certain failures like network issues // the rest should only try the one time then fail as they do now @@ -72,11 +84,20 @@ export async function getResponseStream( retries?: number ): Promise { const res = await getResponse(url, retries); - return res.body; + if (res) { + return res?.body; + } + throw new RegistryResponseError('isAirGapped config enabled, registry not reacheable'); } export async function fetchUrl(url: string, retries?: number): Promise { - return getResponseStream(url, retries).then(streamToString); + const logger = appContextService.getLogger(); + try { + return getResponseStream(url, retries).then(streamToString); + } catch (error) { + logger.warn(`getResponseStream failed with error: ${error}`); + throw error; + } } // node-fetch throws a FetchError for those types of errors and diff --git a/x-pack/plugins/fleet/server/services/register_fields_metadata_extractors.ts b/x-pack/plugins/fleet/server/services/register_fields_metadata_extractors.ts index 9a53c235622bf..a468c20137b38 100644 --- a/x-pack/plugins/fleet/server/services/register_fields_metadata_extractors.ts +++ b/x-pack/plugins/fleet/server/services/register_fields_metadata_extractors.ts @@ -10,6 +10,8 @@ import type { FieldsMetadataServerSetup } from '@kbn/fields-metadata-plugin/serv import type { FleetStartContract, FleetStartDeps } from '../plugin'; +import { appContextService } from '.'; + interface RegistrationDeps { core: CoreSetup; fieldsMetadata: FieldsMetadataServerSetup; @@ -17,12 +19,17 @@ interface RegistrationDeps { export const registerFieldsMetadataExtractors = ({ core, fieldsMetadata }: RegistrationDeps) => { fieldsMetadata.registerIntegrationFieldsExtractor(async ({ integration, dataset }) => { - const [_core, _startDeps, { packageService }] = await core.getStartServices(); + try { + const [_core, _startDeps, { packageService }] = await core.getStartServices(); - return packageService.asInternalUser.getPackageFieldsMetadata({ - packageName: integration, - datasetName: dataset, - }); + return packageService.asInternalUser.getPackageFieldsMetadata({ + packageName: integration, + datasetName: dataset, + }); + } catch (error) { + appContextService.getLogger().warn(`registerIntegrationFieldsExtractor error: ${error}`); + throw error; + } }); fieldsMetadata.registerIntegrationListExtractor(async () => { diff --git a/x-pack/plugins/fleet/server/telemetry/sender.ts b/x-pack/plugins/fleet/server/telemetry/sender.ts index 8fb71683b2c9c..bdecd6cc8d0bf 100644 --- a/x-pack/plugins/fleet/server/telemetry/sender.ts +++ b/x-pack/plugins/fleet/server/telemetry/sender.ts @@ -14,10 +14,10 @@ import axios from 'axios'; import type { InfoResponse, LicenseGetResponse } from '@elastic/elasticsearch/lib/api/types'; -import { appContextService } from '../services'; - import { exhaustMap, Subject, takeUntil, timer } from 'rxjs'; +import { appContextService } from '../services'; + import { TelemetryQueue } from './queue'; import type { FleetTelemetryChannel, FleetTelemetryChannelEvents } from './types';