Skip to content

Commit

Permalink
feat: sync package readme (#512)
Browse files Browse the repository at this point in the history
  • Loading branch information
@fengmk2
fengmk2 authored Jun 13, 2023
1 parent 56d8e1a commit f64e273
Show file tree
Hide file tree
Showing 5 changed files with 147 additions and 38 deletions.
26 changes: 25 additions & 1 deletion app/core/event/SyncPackageVersionFile.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Event, Inject } from '@eggjs/tegg';
import {
EggAppConfig,
} from 'egg';
import { PACKAGE_VERSION_ADDED } from './index';
import { PACKAGE_VERSION_ADDED, PACKAGE_TAG_ADDED, PACKAGE_TAG_CHANGED } from './index';
import { getScopeAndName } from '../../common/PackageUtil';
import { PackageManagerService } from '../service/PackageManagerService';
import { PackageVersionFileService } from '../service/PackageVersionFileService';
Expand All @@ -25,6 +25,14 @@ class SyncPackageVersionFileEvent {
if (!packageVersion) return;
await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
}

protected async syncPackageReadmeToLatestVersion(fullname: string) {
const [ scope, name ] = getScopeAndName(fullname);
const { pkg, packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
scope, name, 'latest');
if (!pkg || !packageVersion) return;
await this.packageVersionFileService.syncPackageReadme(pkg, packageVersion);
}
}

@Event(PACKAGE_VERSION_ADDED)
Expand All @@ -33,3 +41,19 @@ export class PackageVersionAdded extends SyncPackageVersionFileEvent {
await this.syncPackageVersionFile(fullname, version);
}
}

@Event(PACKAGE_TAG_ADDED)
export class PackageTagAdded extends SyncPackageVersionFileEvent {
async handle(fullname: string, tag: string) {
if (tag !== 'latest') return;
await this.syncPackageReadmeToLatestVersion(fullname);
}
}

@Event(PACKAGE_TAG_CHANGED)
export class PackageTagChanged extends SyncPackageVersionFileEvent {
async handle(fullname: string, tag: string) {
if (tag !== 'latest') return;
await this.syncPackageReadmeToLatestVersion(fullname);
}
}
26 changes: 12 additions & 14 deletions app/core/service/PackageManagerService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,24 +480,22 @@ export class PackageManagerService extends AbstractService {
}

/**
* save package version readme and auto update package full manifests readme if the package version is latest
* save package version readme
*/
async savePackageVersionReadme(pkgVersion: PackageVersion, readmeFile: string) {
public async savePackageVersionReadme(pkgVersion: PackageVersion, readmeFile: string) {
await this.distRepository.saveDist(pkgVersion.readmeDist, readmeFile);
this.logger.info('[PackageManagerService.savePackageVersionReadme] save packageVersionId:%s readme:%s to dist:%s',
pkgVersion.packageVersionId, readmeFile, pkgVersion.readmeDist.distId);
const latestTag = await this.packageRepository.findPackageTag(pkgVersion.packageId, 'latest');
if (latestTag?.version === pkgVersion.version) {
// update package readme dist
const pkg = await this.packageRepository.findPackageByPackageId(pkgVersion.packageId);
if (!pkg || !pkg.manifestsDist) return;
const fullManifests = await this.distRepository.readDistBytesToJSON<PackageManifestType>(pkg.manifestsDist);
if (!fullManifests) return;
fullManifests.readme = await readFile(readmeFile, 'utf-8');
await this._updatePackageManifestsToDists(pkg, fullManifests, null);
this.logger.info('[PackageManagerService.savePackageVersionReadme] save packageId:%s readme, size: %s',
pkg.packageId, fullManifests.readme.length);
}
}

public async savePackageReadme(pkg: Package, readmeFile: string) {
if (!pkg.manifestsDist) return;
const fullManifests = await this.distRepository.readDistBytesToJSON<PackageManifestType>(pkg.manifestsDist);
if (!fullManifests) return;
fullManifests.readme = await readFile(readmeFile, 'utf-8');
await this._updatePackageManifestsToDists(pkg, fullManifests, null);
this.logger.info('[PackageManagerService.savePackageReadme] save packageId:%s readme, size: %s',
pkg.packageId, fullManifests.readme.length);
}

private async _removePackageVersionAndDist(pkgVersion: PackageVersion) {
Expand Down
20 changes: 10 additions & 10 deletions app/core/service/PackageSyncerService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -804,20 +804,20 @@ export class PackageSyncerService extends AbstractService {
}
}
// 3.2 shoud add latest tag
// 在同步sepcific version时如果没有同步latestTag的版本会出现latestTag丢失或指向版本不正确的情况
// 在同步 sepcific version 时如果没有同步 latestTag 的版本会出现 latestTag 丢失或指向版本不正确的情况
if (specificVersions && this.config.cnpmcore.strictSyncSpecivicVersion) {
// 不允许自动同步latest版本，从已同步版本中选出latest
let latestStabelVersion;
// 不允许自动同步 latest 版本，从已同步版本中选出 latest
let latestStableVersion: string;
const sortedVersionList = specificVersions.sort(semverRcompare);
latestStabelVersion = sortedVersionList.filter(i => !semverPrerelease(i))[0];
// 所有版本都不是稳定版本则指向非稳定版本保证latest存在
if (!latestStabelVersion) {
latestStabelVersion = sortedVersionList[0];
latestStableVersion = sortedVersionList.filter(i => !semverPrerelease(i))[0];
// 所有版本都不是稳定版本则指向非稳定版本保证 latest 存在
if (!latestStableVersion) {
latestStableVersion = sortedVersionList[0];
}
if (!existsDistTags.latest || semverRcompare(existsDistTags.latest, latestStabelVersion) === 1) {
if (!existsDistTags.latest || semverRcompare(existsDistTags.latest, latestStableVersion) === 1) {
logs.push(`[${isoNow()}] 🚧 patch latest tag from specific versions 🚧`);
changedTags.push({ action: 'change', tag: 'latest', version: latestStabelVersion });
await this.packageManagerService.savePackageTag(pkg, 'latest', latestStabelVersion);
changedTags.push({ action: 'change', tag: 'latest', version: latestStableVersion });
await this.packageManagerService.savePackageTag(pkg, 'latest', latestStableVersion);
}
}

Expand Down
97 changes: 85 additions & 12 deletions app/core/service/PackageVersionFileService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,51 @@ export class PackageVersionFileService extends AbstractService {
}
}

// 基于 latest version 同步 package readme
async syncPackageReadme(pkg: Package, latestPkgVersion: PackageVersion) {
const dirname = `unpkg_${pkg.fullname.replace('/', '_')}@${latestPkgVersion.version}_latest_readme_${randomUUID()}`;
const tmpdir = await createTempDir(this.config.dataDir, dirname);
const tarFile = `${tmpdir}.tgz`;
const readmeFilenames: string[] = [];
try {
this.logger.info('[PackageVersionFileService.syncPackageReadme:download-start] dist:%s(path:%s, size:%s) => tarFile:%s',
latestPkgVersion.tarDist.distId, latestPkgVersion.tarDist.path, latestPkgVersion.tarDist.size, tarFile);
await this.distRepository.downloadDistToFile(latestPkgVersion.tarDist, tarFile);
this.logger.info('[PackageVersionFileService.syncPackageReadme:extract-start] tmpdir:%s', tmpdir);
await tar.extract({
file: tarFile,
cwd: tmpdir,
strip: 1,
onentry: entry => {
const filename = this.#formatTarEntryFilename(entry);
if (!filename) return;
if (this.#matchReadmeFilename(filename)) {
readmeFilenames.push(filename);
}
},
});
if (readmeFilenames.length > 0) {
const readmeFilename = this.#preferMarkdownReadme(readmeFilenames);
const readmeFile = join(tmpdir, readmeFilename);
await this.packageManagerService.savePackageReadme(pkg, readmeFile);
}
} catch (err) {
this.logger.warn('[PackageVersionFileService.syncPackageReadme:error] packageVersionId: %s, readmeFilenames: %j, tmpdir: %s, error: %s',
latestPkgVersion.packageVersionId, readmeFilenames, tmpdir, err);
// ignore TAR_BAD_ARCHIVE error
if (err.code === 'TAR_BAD_ARCHIVE') return;
throw err;
} finally {
try {
await fs.rm(tarFile, { force: true });
await fs.rm(tmpdir, { recursive: true, force: true });
} catch (err) {
this.logger.warn('[PackageVersionFileService.syncPackageReadme:warn] remove tmpdir: %s, error: %s',
tmpdir, err);
}
}
}

async syncPackageVersionFiles(pkgVersion: PackageVersion) {
const files: PackageVersionFile[] = [];
const pkg = await this.packageRepository.findPackageByPackageId(pkgVersion.packageId);
Expand All @@ -62,7 +107,7 @@ export class PackageVersionFileService extends AbstractService {
const tmpdir = await createTempDir(this.config.dataDir, dirname);
const tarFile = `${tmpdir}.tgz`;
const paths: string[] = [];
let readmeFilename = '';
const readmeFilenames: string[] = [];
try {
this.logger.info('[PackageVersionFileService.syncPackageVersionFiles:download-start] dist:%s(path:%s, size:%s) => tarFile:%s',
pkgVersion.tarDist.distId, pkgVersion.tarDist.path, pkgVersion.tarDist.size, tarFile);
Expand All @@ -73,17 +118,11 @@ export class PackageVersionFileService extends AbstractService {
cwd: tmpdir,
strip: 1,
onentry: entry => {
if (entry.type !== 'File') return;
// ignore hidden dir
if (entry.path.includes('/./')) return;
// https://github.com/cnpm/cnpmcore/issues/452#issuecomment-1570077310
// strip first dir, e.g.: 'package/', 'lodash-es/'
const filename = entry.path.split('/').slice(1).join('/');
const filename = this.#formatTarEntryFilename(entry);
if (!filename) return;
paths.push('/' + filename);
if (!readmeFilename) {
if (filename === 'README.md' || filename === 'readme.md' || filename === 'Readme.md') {
readmeFilename = filename;
}
if (this.#matchReadmeFilename(filename)) {
readmeFilenames.push(filename);
}
},
});
Expand All @@ -94,7 +133,8 @@ export class PackageVersionFileService extends AbstractService {
}
this.logger.info('[PackageVersionFileService.syncPackageVersionFiles:success] packageVersionId: %s, %d paths, %d files, tmpdir: %s',
pkgVersion.packageVersionId, paths.length, files.length, tmpdir);
if (readmeFilename) {
if (readmeFilenames.length > 0) {
const readmeFilename = this.#preferMarkdownReadme(readmeFilenames);
const readmeFile = join(tmpdir, readmeFilename);
await this.packageManagerService.savePackageVersionReadme(pkgVersion, readmeFile);
}
Expand Down Expand Up @@ -158,4 +198,37 @@ export class PackageVersionFileService extends AbstractService {
name: basename(path),
};
}

#formatTarEntryFilename(entry: tar.ReadEntry) {
if (entry.type !== 'File') return;
// ignore hidden dir
if (entry.path.includes('/./')) return;
// https://github.com/cnpm/cnpmcore/issues/452#issuecomment-1570077310
// strip first dir, e.g.: 'package/', 'lodash-es/'
const filename = entry.path.split('/').slice(1).join('/');
return filename;
}

#matchReadmeFilename(filename: string) {
// support README,README.*
// https://github.com/npm/read-package-json/blob/main/lib/read-json.js#L280
return (/^README(\.\w{1,20}|$)/i.test(filename));
}

// https://github.com/npm/read-package-json/blob/main/lib/read-json.js#L280
#preferMarkdownReadme(files: string[]) {
let fallback = 0;
const markdownRE = /\.m?a?r?k?d?o?w?n?$/i;
for (let i = 0; i < files.length; i++) {
const file = files[i];
if (markdownRE.test(file)) {
return file;
} else if (file.toLowerCase() === 'README') {
fallback = i;
}
}
// prefer README.md, followed by README; otherwise, return
// the first filename (which could be README)
return files[fallback];
}
}
16 changes: 15 additions & 1 deletion test/port/controller/PackageVersionFileController/raw.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { strict as assert } from 'node:assert';
import { app, mock } from 'egg-mock/bootstrap';
import { TestUtil } from '../../../../test/TestUtil';
import { calculateIntegrity } from '../../../../app/common/PackageUtil';
import { PackageTagChanged, PackageTagAdded } from '../../../../app/core/event/SyncPackageVersionFile';

describe('test/port/controller/PackageVersionFileController/raw.test.ts', () => {
let publisher;
Expand Down Expand Up @@ -286,6 +287,17 @@ describe('test/port/controller/PackageVersionFileController/raw.test.ts', () =>
// console.log(res.body);
assert.equal(res.body.files.find(file => file.path === '/.'), undefined);
assert(res.body.files.find(file => file.path === '/dist'));
const packageTagAdded = await app.getEggObject(PackageTagAdded);
await packageTagAdded.handle(pkg.name, 'foo');
await packageTagAdded.handle(pkg.name, 'latest');
res = await app.httpRequest()
.get(`/${pkg.name}/1.0.0`);
const readme = res.body.readme;
assert.match(readme, /# bovo-ui/);
// pkg readme change to latest
res = await app.httpRequest()
.get(`/${pkg.name}`);
assert.equal(res.body.readme, readme);
});

it('should handle big tgz file', async () => {
Expand Down Expand Up @@ -341,11 +353,13 @@ describe('test/port/controller/PackageVersionFileController/raw.test.ts', () =>
.expect(200);
assert.notEqual(res.body.readme, oldReadme);
assert.match(res.body.readme, /The Javascript Database that Syncs/);
const packageTagChanged = await app.getEggObject(PackageTagChanged);
await packageTagChanged.handle(pkg.name, 'foo');
await packageTagChanged.handle(pkg.name, 'latest');
// pkg version change too
res = await app.httpRequest()
.get(`/${pkg.name}`)
.expect(200);
assert.notEqual(res.body.readme, oldReadme);
assert.match(res.body.readme, /The Javascript Database that Syncs/);

res = await app.httpRequest()
Expand Down

0 comments on commit f64e273

Please sign in to comment.