From 17bf7d7ed9626c359f6f1e74a7de72f55f024da0 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 7 Aug 2023 23:02:57 +0300 Subject: [PATCH] refactor: Reuse datasource filtering (#23716) --- lib/modules/datasource/common.spec.ts | 23 ++++++++--------- lib/modules/datasource/common.ts | 17 +++++++------ lib/modules/datasource/index.ts | 21 +++++++++++----- .../repository/process/lookup/index.ts | 25 ++++++++++--------- .../repository/process/lookup/types.ts | 1 + 5 files changed, 48 insertions(+), 39 deletions(-) diff --git a/lib/modules/datasource/common.spec.ts b/lib/modules/datasource/common.spec.ts index 53147a03f7a873..7a57cd455a613d 100644 --- a/lib/modules/datasource/common.spec.ts +++ b/lib/modules/datasource/common.spec.ts @@ -89,31 +89,28 @@ describe('modules/datasource/common', () => { describe('applyExtractVersion', () => { it('should return the same release result if extractVersion is not defined', () => { - const config = {}; const releaseResult: ReleaseResult = { releases: [{ version: '1.0.0' }, { version: '2.0.0' }], }; - const res = applyExtractVersion(config, releaseResult); + const res = applyExtractVersion(releaseResult, undefined); expect(res).toBe(releaseResult); }); it('should extract version from release using provided regex', () => { - const config = { extractVersion: '^v(?.+)$' }; const releaseResult: ReleaseResult = { releases: [{ version: 'v1.0.0' }, { version: 'v2.0.0' }], }; - const res = applyExtractVersion(config, releaseResult); + const res = applyExtractVersion(releaseResult, '^v(?.+)$'); expect(res).toEqual({ releases: [{ version: '1.0.0' }, { version: '2.0.0' }], }); }); it('should return null for releases with invalid version', () => { - const config = { extractVersion: '^v(?.+)$' }; const releaseResult: ReleaseResult = { releases: [{ version: 'v1.0.0' }, { version: 'invalid' }], }; - const result = applyExtractVersion(config, releaseResult); + const result = applyExtractVersion(releaseResult, '^v(?.+)$'); expect(result).toEqual({ releases: [{ version: '1.0.0' }], }); @@ -131,7 +128,7 @@ describe('modules/datasource/common', () => { it('should filter out invalid versions', () => { const config = { datasource: 'npm' }; - const res = filterValidVersions(config, releaseResult); + const res = filterValidVersions(releaseResult, config); expect(res).toEqual({ releases: [{ version: '1.0.0' }, { version: '2.0.0' }], }); @@ -139,7 +136,7 @@ describe('modules/datasource/common', () => { it('should use default versioning if none is specified', () => { const config = { datasource: 'foobar' }; - const res = filterValidVersions(config, releaseResult); + const res = filterValidVersions(releaseResult, config); expect(res).toEqual({ releases: [{ version: '1.0.0' }, { version: '2.0.0' }], }); @@ -147,7 +144,7 @@ describe('modules/datasource/common', () => { it('should use specified versioning if provided', () => { const config = { datasource: 'npm', versioning: 'semver' }; - const res = filterValidVersions(config, releaseResult); + const res = filterValidVersions(releaseResult, config); expect(res).toEqual({ releases: [{ version: '1.0.0' }, { version: '2.0.0' }], }); @@ -172,7 +169,7 @@ describe('modules/datasource/common', () => { { version: '3.0.0' }, ], }; - const result = sortAndRemoveDuplicates(config, releaseResult); + const result = sortAndRemoveDuplicates(releaseResult, config); expect(result).toEqual(expected); }); @@ -181,7 +178,7 @@ describe('modules/datasource/common', () => { const releaseResult: ReleaseResult = { releases: [{ version: '1.0.0' }, { version: '2.0.0' }], }; - const result = sortAndRemoveDuplicates(config, releaseResult); + const result = sortAndRemoveDuplicates(releaseResult, config); expect(result).toEqual({ releases: [{ version: '1.0.0' }, { version: '2.0.0' }], }); @@ -205,7 +202,7 @@ describe('modules/datasource/common', () => { { version: '2.0.0', constraints: { foo: ['^2.0.0'] } }, ], }; - expect(applyConstraintsFiltering(config, releaseResult)).toEqual({ + expect(applyConstraintsFiltering(releaseResult, config)).toEqual({ releases: [{ version: '1.0.0' }, { version: '2.0.0' }], }); }); @@ -224,7 +221,7 @@ describe('modules/datasource/common', () => { { version: '3.0.0', constraints: { baz: ['^0.9.0'] } }, ], }; - expect(applyConstraintsFiltering(config, releaseResult)).toEqual({ + expect(applyConstraintsFiltering(releaseResult, config)).toEqual({ releases: [{ version: '1.0.0' }, { version: '2.0.0' }], }); }); diff --git a/lib/modules/datasource/common.ts b/lib/modules/datasource/common.ts index cbb34e7b3d7a1e..c839724b7f8d5b 100644 --- a/lib/modules/datasource/common.ts +++ b/lib/modules/datasource/common.ts @@ -53,14 +53,15 @@ export function isGetPkgReleasesConfig( ); } -export function applyExtractVersion< - Config extends Pick ->(config: Config, releaseResult: ReleaseResult): ReleaseResult { - if (!config.extractVersion) { +export function applyExtractVersion( + releaseResult: ReleaseResult, + extractVersion: string | undefined +): ReleaseResult { + if (!extractVersion) { return releaseResult; } - const extractVersionRegEx = regEx(config.extractVersion); + const extractVersionRegEx = regEx(extractVersion); releaseResult.releases = filterMap(releaseResult.releases, (release) => { const version = extractVersionRegEx.exec(release.version)?.groups?.version; if (!version) { @@ -76,7 +77,7 @@ export function applyExtractVersion< export function filterValidVersions< Config extends Pick ->(config: Config, releaseResult: ReleaseResult): ReleaseResult { +>(releaseResult: ReleaseResult, config: Config): ReleaseResult { const versioningName = config.versioning ?? getDefaultVersioning(config.datasource); const versioning = allVersioning.get(versioningName); @@ -90,7 +91,7 @@ export function filterValidVersions< export function sortAndRemoveDuplicates< Config extends Pick ->(config: Config, releaseResult: ReleaseResult): ReleaseResult { +>(releaseResult: ReleaseResult, config: Config): ReleaseResult { const versioningName = config.versioning ?? getDefaultVersioning(config.datasource); const versioning = allVersioning.get(versioningName); @@ -121,7 +122,7 @@ export function applyConstraintsFiltering< | 'constraints' | 'packageName' > ->(config: Config, releaseResult: ReleaseResult): ReleaseResult { +>(releaseResult: ReleaseResult, config: Config): ReleaseResult { if (config?.constraintsFiltering !== 'strict') { for (const release of releaseResult.releases) { delete release.constraints; diff --git a/lib/modules/datasource/index.ts b/lib/modules/datasource/index.ts index 5edb609a5a9686..3bc7181d22bd81 100644 --- a/lib/modules/datasource/index.ts +++ b/lib/modules/datasource/index.ts @@ -310,7 +310,7 @@ async function fetchReleases( return dep; } -function getRawReleases( +function fetchCachedReleases( config: GetReleasesInternalConfig ): Promise { const { datasource, packageName, registryUrls } = config; @@ -345,7 +345,7 @@ export function getRawPkgReleases( return AsyncResult.err('no-package-name'); } - return Result.wrapNullable(getRawReleases(config), 'no-result' as const) + return Result.wrapNullable(fetchCachedReleases(config), 'no-result' as const) .catch((e) => { if (e instanceof ExternalHostError) { e.hostType = config.datasource; @@ -356,14 +356,23 @@ export function getRawPkgReleases( .transform(clone); } +export function applyDatasourceFilters( + releaseResult: ReleaseResult, + config: GetPkgReleasesConfig +): ReleaseResult { + let res = releaseResult; + res = applyExtractVersion(res, config.extractVersion); + res = filterValidVersions(res, config); + res = sortAndRemoveDuplicates(res, config); + res = applyConstraintsFiltering(res, config); + return res; +} + export async function getPkgReleases( config: GetPkgReleasesConfig ): Promise { const { val = null, err } = await getRawPkgReleases(config) - .transform((res) => applyExtractVersion(config, res)) - .transform((res) => filterValidVersions(config, res)) - .transform((res) => sortAndRemoveDuplicates(config, res)) - .transform((res) => applyConstraintsFiltering(config, res)) + .transform((res) => applyDatasourceFilters(res, config)) .unwrap(); if (err instanceof Error) { diff --git a/lib/workers/repository/process/lookup/index.ts b/lib/workers/repository/process/lookup/index.ts index ade4a9ba292d45..77f6dd5b59a7e4 100644 --- a/lib/workers/repository/process/lookup/index.ts +++ b/lib/workers/repository/process/lookup/index.ts @@ -6,8 +6,9 @@ import { logger } from '../../../../logger'; import { Release, ReleaseResult, + applyDatasourceFilters, getDigest, - getPkgReleases, + getRawPkgReleases, isGetPkgReleasesConfig, supportsDigests, } from '../../../../modules/datasource'; @@ -19,10 +20,8 @@ import { getRangeStrategy } from '../../../../modules/manager'; import * as allVersioning from '../../../../modules/versioning'; import { ExternalHostError } from '../../../../types/errors/external-host-error'; import { assignKeys } from '../../../../util/assign-keys'; -import { clone } from '../../../../util/clone'; import { applyPackageRules } from '../../../../util/package-rules'; import { regEx } from '../../../../util/regex'; -import { Result } from '../../../../util/result'; import { getBucket } from './bucket'; import { getCurrentVersion } from './current'; import { filterVersions } from './filter'; @@ -84,14 +83,17 @@ export async function lookupUpdates( return res; } - const { val: lookupValue, err: lookupError } = await Result.wrapNullable( - getPkgReleases(config), - 'no-releases' as const + const { val: releaseResult, err: lookupError } = await getRawPkgReleases( + config ) - .transform((x) => Result.ok(clone(x))) + .transform((res) => applyDatasourceFilters(res, config)) .unwrap(); - if (lookupError === 'no-releases') { + if (lookupError instanceof Error) { + throw lookupError; + } + + if (lookupError) { // If dependency lookup fails then warn and return const warning: ValidationMessage = { topic: packageName, @@ -101,18 +103,17 @@ export async function lookupUpdates( // TODO: return warnings in own field res.warnings.push(warning); return res; - } else if (lookupError) { - throw lookupError; } - dependency = lookupValue; + dependency = releaseResult; + if (dependency.deprecationMessage) { logger.debug( `Found deprecationMessage for ${datasource} package ${packageName}` ); } - assignKeys(res, lookupValue, [ + assignKeys(res, dependency, [ 'deprecationMessage', 'sourceUrl', 'registryUrl', diff --git a/lib/workers/repository/process/lookup/types.ts b/lib/workers/repository/process/lookup/types.ts index 5d20ea72bc1e80..3b5551f7e2a198 100644 --- a/lib/workers/repository/process/lookup/types.ts +++ b/lib/workers/repository/process/lookup/types.ts @@ -47,6 +47,7 @@ export interface LookupUpdateConfig replacementName?: string; replacementNameTemplate?: string; replacementVersion?: string; + extractVersion?: string; } export interface UpdateResult {