Skip to content

Commit

Permalink
fix: incorrect latest tag in strict specific version (#610)
Browse files Browse the repository at this point in the history 
1. 添加新建任务时的重复版本校验，去除执行任务时的版本去重
2.
在 strictSyncSpecificVersion 开启时，修改为从可用版本中选择 latestTag。之前从任务的指定版本中生成 latestTag 逻辑不正确
  • Loading branch information
@hezhengxu2018
hezhengxu2018 authored Nov 28, 2023
1 parent 072e146 commit acfd667
Show file tree
Hide file tree
Showing 5 changed files with 67 additions and 19 deletions.
40 changes: 27 additions & 13 deletions app/core/service/PackageSyncerService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ import { setTimeout } from 'timers/promises';
import { rm } from 'fs/promises';
import { isEqual, isEmpty } from 'lodash';
import semver from 'semver';
import semverRcompare from 'semver/functions/rcompare';
import semverPrerelease from 'semver/functions/prerelease';
import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry';
import { detectInstallScript, getScopeAndName } from '../../common/PackageUtil';
import { downloadToTempfile } from '../../common/FileUtil';
Expand Down Expand Up @@ -562,16 +560,34 @@ export class PackageSyncerService extends AbstractService {
logs.push(`[${isoNow()}] 📦 Add latest tag version "${fullname}: ${distTags.latest}"`);
specificVersions.push(distTags.latest);
}
const versions: PackageJSONType[] = specificVersions ? Object.values<any>(versionMap).filter(verItem => specificVersions.includes(verItem.version)) : Object.values<any>(versionMap);
logs.push(`[${isoNow()}] 🚧 Syncing versions ${existsVersionCount} => ${versions.length}`);
const versions = specificVersions ?
Object.values<PackageJSONType>(versionMap).filter(verItem => specificVersions.includes(verItem.version)) :
Object.values<PackageJSONType>(versionMap);
// 全量同步时跳过排序
const sortedAvailableVersions = specificVersions ?
versions.map(item => item.version).sort(semver.rcompare) : [];
// 在strictSyncSpecivicVersion模式下（不同步latest）且所有传入的version均不可用
if (specificVersions && sortedAvailableVersions.length === 0) {
logs.push(`[${isoNow()}] ❌ `);
task.error = 'There is no available specific versions, stop task.';
logs.push(`[${isoNow()}] ${task.error}, log: ${logUrl}`);
logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
this.logger.info('[PackageSyncerService.executeTask:fail-empty-list] taskId: %s, targetName: %s, %s',
task.taskId, task.targetName, task.error);
return;
}
if (specificVersions) {
const availableVersionList = versions.map(item => item.version);
let notAvailableVersionList = specificVersions.filter(i => !availableVersionList.includes(i));
// specific versions may not in manifest.
const notAvailableVersionList = specificVersions.filter(i => !sortedAvailableVersions.includes(i));
logs.push(`[${isoNow()}] 🚧 Syncing specific versions: ${sortedAvailableVersions.join(' | ')}`);
if (notAvailableVersionList.length > 0) {
notAvailableVersionList = Array.from(new Set(notAvailableVersionList));
logs.push(`[${isoNow()}] 🚧 Some specific versions are not available: 👉 ${notAvailableVersionList.join(' | ')} 👈`);
logs.push(`🚧 Some specific versions are not available: 👉 ${notAvailableVersionList.join(' | ')} 👈`);
}
} else {
logs.push(`[${isoNow()}] 🚧 Syncing versions ${existsVersionCount} => ${versions.length}`);
}

const updateVersions: string[] = [];
const differentMetas: [PackageJSONType, Partial<PackageJSONType>][] = [];
let syncIndex = 0;
Expand Down Expand Up @@ -818,14 +834,12 @@ export class PackageSyncerService extends AbstractService {
// 在同步 sepcific version 时如果没有同步 latestTag 的版本会出现 latestTag 丢失或指向版本不正确的情况
if (specificVersions && this.config.cnpmcore.strictSyncSpecivicVersion) {
// 不允许自动同步 latest 版本，从已同步版本中选出 latest
let latestStableVersion: string;
const sortedVersionList = specificVersions.sort(semverRcompare);
latestStableVersion = sortedVersionList.filter(i => !semverPrerelease(i))[0];
let latestStableVersion = semver.maxSatisfying(sortedAvailableVersions, '*');
// 所有版本都不是稳定版本则指向非稳定版本保证 latest 存在
if (!latestStableVersion) {
latestStableVersion = sortedVersionList[0];
latestStableVersion = sortedAvailableVersions[0];
}
if (!existsDistTags.latest || semverRcompare(existsDistTags.latest, latestStableVersion) === 1) {
if (!existsDistTags.latest || semver.rcompare(existsDistTags.latest, latestStableVersion) === 1) {
logs.push(`[${isoNow()}] 🚧 patch latest tag from specific versions 🚧`);
changedTags.push({ action: 'change', tag: 'latest', version: latestStableVersion });
await this.packageManagerService.savePackageTag(pkg, 'latest', latestStableVersion);
Expand Down
11 changes: 7 additions & 4 deletions app/port/typebox.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import semver from 'semver';
import npa from 'npm-package-arg';
import { HookType } from '../common/enum/Hook';
import binaryConfig, { BinaryName } from '../../config/binaries';
import { uniq } from 'lodash';

export const Name = Type.String({
transform: [ 'trim' ],
Expand Down Expand Up @@ -55,7 +56,7 @@ export const Version = Type.String({
});

export const VersionStringArray = Type.String({
format: 'semver-version-array',
format: 'unique-semver-version-array',
transform: [ 'trim' ],
});

Expand Down Expand Up @@ -149,7 +150,7 @@ export function patchAjv(ajv: any) {
return binaryConfig[binaryName];
},
});
ajv.addFormat('semver-version-array', {
ajv.addFormat('unique-semver-version-array', {
type: 'string',
validate: (versionStringList: string) => {
let versionList;
Expand All @@ -158,8 +159,10 @@ export function patchAjv(ajv: any) {
} catch (error) {
return false;
}
if (versionList instanceof Array) {
return versionList.every(version => !!semver.valid(version));
if (Array.isArray(versionList)) {
const isSemver = versionList.every(version => !!semver.valid(version));
const isUnique = uniq(versionList).length === versionList.length;
return isSemver && isUnique;
}
return false;
},
Expand Down
3 changes: 2 additions & 1 deletion app/repository/TaskRepository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import type { HistoryTask as HistoryTaskModel } from './model/HistoryTask';
import { Task as TaskEntity, TaskUpdateCondition } from '../core/entity/Task';
import { AbstractRepository } from './AbstractRepository';
import { TaskType, TaskState } from '../../app/common/enum/Task';
import { uniq } from 'lodash';

@SingletonProto({
accessLevel: AccessLevel.PUBLIC,
Expand Down Expand Up @@ -72,7 +73,7 @@ export class TaskRepository extends AbstractRepository {
if (!model || !model.data.specificVersions) return;
if (specificVersions) {
const data = model.data;
const combinedVersions = Array.from(new Set(data.specificVersions.concat(specificVersions)));
const combinedVersions = uniq<string>(data.specificVersions.concat(specificVersions));
data.specificVersions = combinedVersions;
await model.update({ data });
} else {
Expand Down
25 changes: 25 additions & 0 deletions test/core/service/PackageSyncerService/executeTask.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1346,6 +1346,31 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
assert(log.includes('🚧 Some specific versions are not available: 👉 9.99.9'));
});

it('should not sync nonexistent specific versions in strict mode.', async () => {
mock(app.config.cnpmcore, 'strictSyncSpecivicVersion', true);
app.mockHttpclient('https://registry.npmjs.org/%40cnpmcore%2Ftest-sync-package-has-two-versions', 'GET', {
data: '{"_id":"@cnpmcore/test-sync-package-has-two-versions","_rev":"4-541287ae0a14039fea89ac08fa5ec53d","name":"@cnpmcore/test-sync-package-has-two-versions","dist-tags":{"latest":"2.0.0","next":"2.0.0"},"versions":{"1.0.0":{"name":"@cnpmcore/test-sync-package-has-two-versions","version":"1.0.0","description":"cnpmcore local test package","main":"index.js","scripts":{"test":"echo \\"hello\\""},"author":"","license":"MIT","gitHead":"60cfb1cf401f87a60a1b0dfd7ee739f98ffd7847","_id":"@cnpmcore/test-sync-package-has-two-versions@1.0.0","_nodeVersion":"16.13.1","_npmVersion":"8.1.2","dist":{"integrity":"sha512-WR0T96H8t7ss1FK8GWPPblx+usbjU4bNGRjMHS9t/oVA5DgJDxitydPSFPeIUtXciyekI7R47do9Lc3GgC4P5A==","shasum":"2ddc6ee93b92be6d64139fb1a631d2610f43e946","tarball":"https://registry.npmjs.org/@cnpmcore/test-sync-package-has-two-versions/-/test-sync-package-has-two-versions-1.0.0.tgz","fileCount":2,"unpackedSize":238,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDj5Ui2GU8nVmHFk0hCt/i3gPW9eQdOCZgKzpAlkvERwQIhAPZ0NCefLoEfOpnbdKAUr7Ng9Sy6FMnTsDxDaM2dQHNw"}]},"_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"directories":{},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/test-sync-package-has-two-versions_1.0.0_1639442699824_0.6948988437963031"},"_hasShrinkwrap":false},"2.0.0":{"name":"@cnpmcore/test-sync-package-has-two-versions","version":"2.0.0","description":"cnpmcore local test package","main":"index.js","scripts":{"test":"echo \\"hello\\""},"author":"","license":"MIT","gitHead":"60cfb1cf401f87a60a1b0dfd7ee739f98ffd7847","_id":"@cnpmcore/test-sync-package-has-two-versions@2.0.0","_nodeVersion":"16.13.1","_npmVersion":"8.1.2","dist":{"integrity":"sha512-qgHLQzXq+VN7q0JWibeBYrqb3Iajl4lpVuxlQstclRz4ejujfDFswBGSXmCv9FyIIdmSAe5bZo0oHQLsod3pAA==","shasum":"891eb8e08ceadbd86e75b6d66f31f7e5a28a8d68","tarball":"https://registry.npmjs.org/@cnpmcore/test-sync-package-has-two-versions/-/test-sync-package-has-two-versions-2.0.0.tgz","fileCount":2,"unpackedSize":238,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIAWVz7mIHF23Gq4a+Swsj2ZSdn87991HcE1+fQm8shNCAiByOIuhaZAbo9hct24qYf7FWqx6Lyluo+Rpnrn91//Ibg=="}]},"_npmUser":{"name":"fengmk2","email":"fengmk2@gmail.com"},"directories":{},"maintainers":[{"name":"fengmk2","email":"fengmk2@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/test-sync-package-has-two-versions_2.0.0_1639442732240_0.33204392278137207"},"_hasShrinkwrap":false}},"time":{"created":"2021-12-14T00:44:59.775Z","1.0.0":"2021-12-14T00:44:59.940Z","modified":"2022-05-23T02:33:52.613Z","2.0.0":"2021-12-14T00:45:32.457Z"},"maintainers":[{"email":"killa07071201@gmail.com","name":"killagu"},{"email":"fengmk2@gmail.com","name":"fengmk2"}],"description":"cnpmcore local test package","license":"MIT","readme":"ERROR: No README data found!","readmeFilename":""}',
persist: false,
});
app.mockHttpclient('https://registry.npmjs.org/@cnpmcore/test-sync-package-has-two-versions/-/test-sync-package-has-two-versions-1.0.0.tgz', 'GET', {
data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'),
persist: false,
});
const name = '@cnpmcore/test-sync-package-has-two-versions';
await packageSyncerService.createTask(name, { specificVersions: [ '1.0.0', '9.99.9' ] });
const task = await packageSyncerService.findExecuteTask();
assert(task);
assert.equal(task.targetName, name);
await packageSyncerService.executeTask(task);
const stream = await packageSyncerService.findTaskLog(task);
assert(stream);
const log = await TestUtil.readStreamToLog(stream);
assert(log.includes('🚧 Some specific versions are not available: 👉 9.99.9'));
const manifest = await packageManagerService.listPackageAbbreviatedManifests('@cnpmcore', 'test-sync-package-has-two-versions');
assert(manifest.data?.['dist-tags']);
assert.equal(manifest.data?.['dist-tags'].latest, '1.0.0');
});

// 有任务积压，不一定能够同步完
it.skip('should sync sourceRegistryIsCNpm = true && syncUpstreamFirst = true', async () => {
app.mockHttpclient('https://r.cnpmjs.org/cnpmcore-test-sync-deprecated', 'GET', {
Expand Down
7 changes: 6 additions & 1 deletion test/port/controller/PackageSyncController/createSyncTask.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,16 @@ describe('test/port/controller/PackageSyncController/createSyncTask.test.ts', ()
assert(res.body.error === '[FORBIDDEN] Can\'t sync private package "@cnpm/koa"');
});

it('should 422 if specificVersions cannot parse is not valideted', async () => {
it('should 422 if specificVersions cannot parse or not valideted', async () => {
await app.httpRequest()
.put('/-/package/koa/syncs')
.send({ specificVersions: '1.0.0' })
.expect(422);

await app.httpRequest()
.put('/-/package/koa/syncs')
.send({ specificVersions: '["1.0.0", "1.0.0"]' })
.expect(422);
});

it('should 201 if user login when alwaysAuth = true', async () => {
Expand Down

0 comments on commit acfd667

Please sign in to comment.