feat: add source registry name in manifest (#448)

> Add a private field, _source_registry_name in the version manifest. * 🧶 Add related types for PackageManifestType and adjust relevant unit tests. * 🤖 Update the workflow trigger. * ♻️ No compensation will be made for the _source_registry_name field in the existing packageVersion. ------- > 在 version manifest 中新增私有字段，_source_registry_name 用于标记 * 🧶 新增 PackageManifestType 相关类型，并调整相关单测 * 🤖 调整 workflow 触发时机，不限制 target 分支 * ♻️ 存量 packageVersion 内 _source_registry_name 不做补偿
cnpm · Apr 25, 2023 · f891aed · f891aed
1 parent ae191f3
commit f891aed
Show file tree

Hide file tree

Showing 18 changed files with 360 additions and 128 deletions.
diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
@@ -3,11 +3,7 @@
 
 name: Node.js CI
 
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
+on: [push, pull_request]
 
 jobs:
   test-mysql57-fs-nfs:

diff --git a/.github/workflows/sql-review.yml b/.github/workflows/sql-review.yml
@@ -2,7 +2,7 @@
 
 name: SQL Review
 
-on: [pull_request]
+on: [push, pull_request]
 
 jobs:
   sql-review:

diff --git a/app/core/service/ChangesStreamService.ts b/app/core/service/ChangesStreamService.ts
@@ -13,12 +13,11 @@ import { HOST_NAME, ChangesStreamTask, Task } from '../entity/Task';
 import { PackageSyncerService, RegistryNotMatchError } from './PackageSyncerService';
 import { TaskService } from './TaskService';
 import { RegistryManagerService } from './RegistryManagerService';
-import { RegistryType } from '../../common/enum/Registry';
 import { E500 } from 'egg-errors';
 import { Registry } from '../entity/Registry';
 import { AbstractChangeStream } from '../../common/adapter/changesStream/AbstractChangesStream';
 import { getScopeAndName } from '../../common/PackageUtil';
-import { GLOBAL_WORKER, PresetRegistryName } from '../../common/constants';
+import { GLOBAL_WORKER } from '../../common/constants';
 import { ScopeManagerService } from './ScopeManagerService';
 import { PackageRepository } from '../../repository/PackageRepository';
 
@@ -123,16 +122,7 @@ export class ChangesStreamService extends AbstractService {
       return registry;
     }
 
-    // 从配置文件默认生成
-    const { changesStreamRegistryMode, changesStreamRegistry: changesStreamHost, sourceRegistry: host } = this.config.cnpmcore;
-    const type = changesStreamRegistryMode === 'json' ? RegistryType.Cnpmcore : RegistryType.Npm;
-    const registry = await this.registryManagerService.createRegistry({
-      name: PresetRegistryName.default,
-      type,
-      userPrefix: 'npm:',
-      host,
-      changeStream: `${changesStreamHost}/_changes`,
-    });
+    const registry = await this.registryManagerService.ensureDefaultRegistry();
     task.data = {
       ...(task.data || {}),
       registryId: registry.registryId,

diff --git a/app/core/service/PackageManagerService.ts b/app/core/service/PackageManagerService.ts
diff --git a/app/core/service/PackageSyncerService.ts b/app/core/service/PackageSyncerService.ts
@@ -32,7 +32,7 @@ import { Registry } from '../entity/Registry';
 import { BadRequestError } from 'egg-errors';
 import { ScopeManagerService } from './ScopeManagerService';
 import { EventCorkAdvice } from './EventCorkerAdvice';
-import { PresetRegistryName, SyncDeleteMode } from '../../common/constants';
+import { SyncDeleteMode } from '../../common/constants';
 
 type syncDeletePkgOptions = {
   task: Task,
@@ -328,7 +328,7 @@ export class PackageSyncerService extends AbstractService {
 
     // 采用默认的 registry
     if (!registry) {
-      registry = await this.registryManagerService.findByRegistryName(PresetRegistryName.default);
+      registry = await this.registryManagerService.ensureDefaultRegistry();
     }
 
     // 更新 targetHost 地址
@@ -537,8 +537,8 @@ export class PackageSyncerService extends AbstractService {
     for (const item of versions) {
       const version: string = item.version;
       if (!version) continue;
-      let existsItem = existsVersionMap[version];
-      let existsAbbreviatedItem = abbreviatedVersionMap[version];
+      let existsItem: typeof existsVersionMap[string] | undefined = existsVersionMap[version];
+      let existsAbbreviatedItem: typeof abbreviatedVersionMap[string] | undefined = abbreviatedVersionMap[version];
       const shouldDeleteReadme = !!(existsItem && 'readme' in existsItem);
       if (pkg) {
         if (existsItem) {

diff --git a/app/core/service/RegistryManagerService.ts b/app/core/service/RegistryManagerService.ts
@@ -134,4 +134,26 @@ export class RegistryManagerService extends AbstractService {
     return newRegistry;
 
   }
+
+  async ensureDefaultRegistry(): Promise<Registry> {
+    const existRegistry = await this.registryRepository.findRegistry(PresetRegistryName.default);
+    if (existRegistry) {
+      return existRegistry;
+    }
+
+    // 从配置文件默认生成
+    const { changesStreamRegistryMode, changesStreamRegistry: changesStreamHost, sourceRegistry: host } = this.config.cnpmcore;
+    const type = changesStreamRegistryMode === 'json' ? RegistryType.Cnpmcore : RegistryType.Npm;
+    const registry = await this.createRegistry({
+      name: PresetRegistryName.default,
+      type,
+      userPrefix: 'npm:',
+      host,
+      changeStream: `${changesStreamHost}/_changes`,
+    });
+
+    return registry;
+
+  }
+
 }
diff --git a/app/port/controller/package/SavePackageVersionController.ts b/app/port/controller/package/SavePackageVersionController.ts
@@ -26,6 +26,7 @@ import {
   Description as DescriptionType,
 } from '../../typebox';
 import { RegistryManagerService } from '../../../core/service/RegistryManagerService';
+import { PackageJSONType } from '../../../repository/PackageRepository';
 
 type PackageVersion = Simplify<PackageJson.PackageJsonStandard & {
   name: 'string';
@@ -181,7 +182,7 @@ export class SavePackageVersionController extends AbstractController {
       name,
       version: packageVersion.version,
       description: packageVersion.description,
-      packageJson: packageVersion,
+      packageJson: packageVersion as PackageJSONType,
       readme,
       dist: {
         content: tarballBytes,

diff --git a/app/repository/DistRepository.ts b/app/repository/DistRepository.ts
@@ -1,6 +1,6 @@
 import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg';
 import { NFSAdapter } from '../common/adapter/NFSAdapter';
-import { PackageRepository } from './PackageRepository';
+import { PackageJSONType, PackageRepository } from './PackageRepository';
 import { Dist } from '../core/entity/Dist';
 
 @SingletonProto({
@@ -17,10 +17,12 @@ export class DistRepository {
     const packageVersion = await this.packageRepository.findPackageVersion(packageId, version);
     if (packageVersion) {
       const [ packageVersionJson, readme ] = await Promise.all([
-        this.readDistBytesToJSON(packageVersion.manifestDist),
+        this.readDistBytesToJSON<PackageJSONType>(packageVersion.manifestDist),
         this.readDistBytesToString(packageVersion.readmeDist),
       ]);
-      packageVersionJson.readme = readme;
+      if (packageVersionJson) {
+        packageVersionJson.readme = readme;
+      }
       return packageVersionJson;
     }
   }
@@ -32,10 +34,10 @@ export class DistRepository {
     }
   }
 
-  async readDistBytesToJSON(dist: Dist) {
+  async readDistBytesToJSON<T>(dist: Dist) {
     const str = await this.readDistBytesToString(dist);
     if (str) {
-      return JSON.parse(str);
+      return JSON.parse(str) as T;
     }
   }
 

diff --git a/app/repository/PackageRepository.ts b/app/repository/PackageRepository.ts
@@ -18,6 +18,134 @@ import { AbstractRepository } from './AbstractRepository';
 import { EggAppConfig } from 'egg';
 import { Bone } from 'leoric';
 
+export type PackageManifestType = Pick<PackageJSONType, PackageJSONPickKey> & {
+  _id: string;
+  _rev: string;
+  'dist-tags': Record<string, string>;
+  versions: Record<string, PackageJSONType | undefined>;
+  maintainers: AuthorType[];
+  time: {
+    created: Date;
+    modified: Date;
+    [key: string]: Date;
+  };
+} & CnpmcorePatchInfo;
+
+export type AbbreviatedPackageJSONType = Pick<PackageJSONType, AbbreviatedKey> & CnpmcorePatchInfo;
+
+export type AbbreviatedPackageManifestType = Pick<PackageManifestType, 'dist-tags' | 'name'> & {
+  modified: Date;
+  versions: Record<string, AbbreviatedPackageJSONType | undefined>;
+  time?: PackageManifestType['time'];
+} & CnpmcorePatchInfo;
+
+export type PackageJSONType = CnpmcorePatchInfo & {
+  name: string;
+  version: string;
+  readme?: string;
+  description?: string;
+  keywords?: string[];
+  homepage?: string;
+  bugs?: {
+    url?: string;
+    email?: string;
+  };
+  license?: string;
+  author?: AuthorType | string;
+  contributors?: ContributorType[] | string[];
+  maintainers?: ContributorType[] | string[];
+  files?: string[];
+  main?: string;
+  bin?: string | {
+    [key: string]: string;
+  };
+  man?: string | string[];
+  directories?: DirectoriesType;
+  repository?: RepositoryType;
+  scripts?: Record<string, string>;
+  config?: Record<string, unknown>;
+  dependencies?: DepInfo;
+  devDependencies?: DepInfo;
+  peerDependencies?: DepInfo;
+  peerDependenciesMeta?: {
+    [key: string]: {
+      optional?: boolean;
+      required?: string;
+      version?: string;
+      [key: string]: unknown;
+    };
+  };
+  bundleDependencies?: string[];
+  bundledDependencies?: string[];
+  optionalDependencies?: DepInfo;
+  engines?: {
+    node?: string;
+    npm?: string;
+    [key: string]: string | undefined;
+  };
+  os?: string[];
+  cpu?: string[];
+  preferGlobal?: boolean;
+  private?: boolean;
+  publishConfig?: {
+    access?: 'public' | 'restricted';
+    [key: string]: unknown;
+  };
+  _hasShrinkwrap?: boolean;
+  hasInstallScript?: boolean;
+  dist?: DistType;
+  workspace?: string[];
+  [key: string]: unknown;
+};
+
+type PackageJSONPickKey = 'name' | 'author' | 'bugs' | 'description' | 'homepage' | 'keywords' | 'license' | 'readme' | 'readmeFilename' | 'repository' | 'versions';
+
+type CnpmcorePatchInfo = {
+  _cnpmcore_publish_time?: Date;
+  publish_time?: number;
+  _source_registry_name?: string;
+  block?: string;
+};
+
+type AbbreviatedKey = 'name' | 'version' | 'deprecated' | 'dependencies' | 'optionalDependencies' | 'devDependencies' | 'bundleDependencies' | 'peerDependencies' | 'peerDependenciesMeta' | 'bin' | 'os' | 'cpu' | 'libc' | 'workspaces' | 'directories' | 'dist' | 'engines' | 'hasInstallScript' | 'publish_time' | 'block' | '_hasShrinkwrap';
+
+type DistType = {
+  tarball: string,
+  size: number,
+  shasum: string,
+  integrity: string,
+  [key: string]: unknown,
+};
+
+type AuthorType = {
+  name: string;
+  email?: string;
+  url?: string;
+};
+
+type ContributorType = {
+  name?: string;
+  email?: string;
+  url?: string;
+  [key: string]: unknown;
+};
+
+type DirectoriesType = {
+  lib?: string;
+  bin?: string;
+  man?: string;
+  test?: string;
+  [key: string]: string | undefined;
+};
+
+type RepositoryType = {
+  type: string;
+  url: string;
+  [key: string]: unknown;
+};
+
+type DepInfo = Record<string, string>;
+
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
 })

diff --git a/test/TestUtil.ts b/test/TestUtil.ts
@@ -9,6 +9,7 @@ import path from 'path';
 import crypto from 'crypto';
 import { getScopeAndName } from '../app/common/PackageUtil';
 import semver from 'semver';
+import { PackageJSONType } from '../app/repository/PackageRepository';
 
 type PackageOptions = {
   name?: string;
@@ -157,7 +158,7 @@ export class TestUtil {
     return JSON.parse(bytes.toString());
   }
 
-  static async getFullPackage(options?: PackageOptions): Promise<any> {
+  static async getFullPackage(options?: PackageOptions): Promise<PackageJSONType & { versions: PackageJSONType[] }> {
     const fullJSONFile = this.getFixtures('exampleFullPackage.json');
     const pkg = JSON.parse((await fs.readFile(fullJSONFile)).toString());
     if (options) {

diff --git a/test/cli/npm/install.test.ts b/test/cli/npm/install.test.ts
@@ -128,7 +128,6 @@ describe('test/cli/npm/install.test.ts', () => {
           cwd: demoDir,
         })
         .debug()
-        .expect('stdout', /added 1 package/)
         .expect('code', 0)
         .end();
 

diff --git a/test/core/service/PackageManagerService/publish.test.ts b/test/core/service/PackageManagerService/publish.test.ts
@@ -41,7 +41,7 @@ describe('test/core/service/PackageManagerService/publish.test.ts', () => {
         scope: '',
         name: 'foo',
         description: 'foo description',
-        packageJson: {},
+        packageJson: await TestUtil.getFullPackage({ name: 'foo' }),
         readme: '',
         version: '1.0.0',
         isPrivate: true,
@@ -81,7 +81,7 @@ describe('test/core/service/PackageManagerService/publish.test.ts', () => {
         scope: '',
         name: 'foo',
         description: '~'.repeat(1100 * 100),
-        packageJson: {},
+        packageJson: await TestUtil.getFullPackage({ name: 'foo' }),
         readme: '',
         version: '1.0.0',
         isPrivate: true,