feat: support sync exist mode (#275)

添加`syncMode:
exist`的支持，参考：https://github.com/cnpm/cnpmjs.org/blob/master/sync/sync_exist.js
的实现

注意事项：建议企业内部使用时，关闭 `enableChangesStream`，同时也关闭
`enableCheckRecentlyUpdated`。手工同步一些常见的包后，再启用 `syncMode:
exist`，这样包的数量可控，也能保障包的状态是最新的

app/core/service/ChangesStreamService.ts

@@ -126,6 +126,11 @@ export class ChangesStreamService extends AbstractService {
     const [ scopeName, name ] = getScopeAndName(fullname);
     const packageEntity = await this.packageRepository.findPackage(scopeName, name);
 
+    // 如果包不存在，且处在 exist 模式下，则不同步
+    if (this.config.cnpmcore.syncMode === 'exist' && !packageEntity) {
+      return false;
+    }
+
     if (packageEntity?.registryId) {
       return registry.registryId === packageEntity.registryId;
     }

app/port/controller/AbstractController.ts

@@ -42,8 +42,8 @@ export abstract class AbstractController extends MiddlewareController {
     return this.config.cnpmcore.sourceRegistry;
   }
 
-  protected get enableSyncAll() {
-    return this.config.cnpmcore.syncMode === 'all';
+  protected get enableSync() {
+    return this.config.cnpmcore.syncMode === 'all' || this.config.cnpmcore.syncMode === 'exist';
   }
 
   protected isPrivateScope(scope: string) {
@@ -57,7 +57,7 @@ export abstract class AbstractController extends MiddlewareController {
     // dont sync private scope
     if (!this.isPrivateScope(scope)) {
       // syncMode = none, redirect public package to source registry
-      if (!this.enableSyncAll) {
+      if (!this.enableSync) {
         err.redirectToSourceRegistry = this.sourceRegistry;
       }
     }

app/port/controller/PackageSyncController.ts

@@ -54,7 +54,7 @@ export class PackageSyncController extends AbstractController {
     method: HTTPMethodEnum.PUT,
   })
   async createSyncTask(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPBody() data: SyncPackageTaskType) {
-    if (!this.enableSyncAll) {
+    if (!this.enableSync) {
       throw new ForbiddenError('Not allow to sync package');
     }
     const tips = data.tips || `Sync cause by "${ctx.href}", parent traceId: ${ctx.tracer.traceId}`;

app/port/schedule/CheckRecentlyUpdatedPackages.ts

@@ -2,6 +2,8 @@ import { EggAppConfig, EggHttpClient, EggLogger } from 'egg';
 import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule';
 import { Inject } from '@eggjs/tegg';
 import { PackageSyncerService } from '../../core/service/PackageSyncerService';
+import { PackageRepository } from '../../repository/PackageRepository';
+import { getScopeAndName } from '../../common/PackageUtil';
 
 // https://github.com/cnpm/cnpmcore/issues/9
 @Schedule<IntervalParams>({
@@ -14,6 +16,8 @@ import { PackageSyncerService } from '../../core/service/PackageSyncerService';
 export class CheckRecentlyUpdatedPackages {
   @Inject()
   private readonly packageSyncerService: PackageSyncerService;
+  @Inject()
+  private readonly packageRepository: PackageRepository;
 
   @Inject()
   private readonly config: EggAppConfig;
@@ -25,7 +29,7 @@ export class CheckRecentlyUpdatedPackages {
   private readonly httpclient: EggHttpClient;
 
   async subscribe() {
-    if (this.config.cnpmcore.syncMode !== 'all' || !this.config.cnpmcore.enableCheckRecentlyUpdated) return;
+    if (this.config.cnpmcore.syncMode === 'none' || !this.config.cnpmcore.enableCheckRecentlyUpdated) return;
     const pageSize = 36;
     const pageCount = this.config.env === 'unittest' ? 2 : 5;
     for (let pageIndex = 0; pageIndex < pageCount; pageIndex++) {
@@ -59,6 +63,14 @@ export class CheckRecentlyUpdatedPackages {
           this.logger.info('[CheckRecentlyUpdatedPackages.subscribe][%s] parse %d packages on %s',
             pageIndex, packages.length, pageUrl);
           for (const pkg of packages) {
+            // skip update when package does not exist
+            if (this.config.cnpmcore.syncMode === 'exist') {
+              const [ scope, name ] = getScopeAndName(pkg.name);
+              const pkgId = await this.packageRepository.findPackageId(scope, name);
+              if (!pkgId) {
+                continue;
+              }
+            }
             const task = await this.packageSyncerService.createTask(pkg.name, {
               tips: `Sync cause by recently updated packages ${pageUrl}`,
             });

config/config.default.ts

@@ -21,6 +21,7 @@ export default (appInfo: EggAppConfig) => {
     // sync mode
     //  - none: don't sync npm package, just redirect it to sourceRegistry
     //  - all: sync all npm packages
+    //  - exist: only sync exist packages, effected when `enableCheckRecentlyUpdated` or `enableChangesStream` is enabled
     syncMode: 'none',
     hookEnable: false,
     syncPackageWorkerMaxConcurrentTasks: 10,

test/core/service/ChangesStreamService.test.ts

@@ -121,6 +121,18 @@ describe('test/core/service/ChangesStreamService.test.ts', () => {
       assert(!res);
     });
 
+    it('the package does not exist should not sync with any registry', async () => {
+      mock(app.config.cnpmcore, 'syncMode', 'exist');
+      await TestUtil.createPackage({
+        name: '@cnpm/test',
+        isPrivate: false,
+        registryId: npmRegistry.registryId,
+      });
+      let res = await changesStreamService.needSync(npmRegistry, 'banana');
+      assert(!res);
+      res = await changesStreamService.needSync(npmRegistry, '@cnpm/test');
+      assert(res);
+    });
   });
 
   describe('getInitialSince()', () => {

test/schedule/CheckRecentlyUpdatedPackages.test.ts

@@ -2,6 +2,7 @@ import assert = require('assert');
 import { app, mock } from 'egg-mock/bootstrap';
 import { Context } from 'egg';
 import { PackageSyncerService } from 'app/core/service/PackageSyncerService';
+import { TestUtil } from 'test/TestUtil';
 
 const CheckRecentlyUpdatedPackagesPath = require.resolve('../../app/port/schedule/CheckRecentlyUpdatedPackages');
 
@@ -19,19 +20,63 @@ describe('test/schedule/CheckRecentlyUpdatedPackages.test.ts', () => {
 
   it('should work', async () => {
     app.mockLog();
+
     // syncMode=none
+    mock(app.config.cnpmcore, 'syncMode', 'none');
     await app.runSchedule(CheckRecentlyUpdatedPackagesPath);
     app.notExpectLog('[CheckRecentlyUpdatedPackages.subscribe]');
 
+    // syncMode=exist
+    mock(app.config.cnpmcore, 'syncMode', 'exist');
+    await app.runSchedule(CheckRecentlyUpdatedPackagesPath);
+    app.expectLog('[CheckRecentlyUpdatedPackages.subscribe][0] request');
+    app.expectLog('[CheckRecentlyUpdatedPackages.subscribe][0] parse');
+    const executeTask = await packageSyncerService.findExecuteTask();
+    assert(!executeTask);
+
     // syncMode=all
     mock(app.config.cnpmcore, 'syncMode', 'all');
     await app.runSchedule(CheckRecentlyUpdatedPackagesPath);
     app.expectLog('[CheckRecentlyUpdatedPackages.subscribe][0] request');
     app.expectLog('[CheckRecentlyUpdatedPackages.subscribe][0] parse');
+    app.expectLog('[CheckRecentlyUpdatedPackages.subscribe:createTask]');
     const task = await packageSyncerService.findExecuteTask();
     assert(task);
   });
 
+  it('should not sync packages with exist mode', async () => {
+    await TestUtil.createPackage({
+      name: '@cnpm/foo',
+      version: '1.0.0',
+      isPrivate: false,
+    });
+    app.mockLog();
+    app.mockHttpclient(/https:\/\/www.npmjs.com\/browse\/updated/, () => {
+      const ret = {
+        context: {
+          packages: [
+            {
+              name: 'test',
+              version: '1.0.0',
+            },
+            {
+              name: '@cnpm/foo',
+              version: '1.1.0',
+            },
+          ],
+        },
+      };
+      return `window.__context__ = ${JSON.stringify(ret)}</script>`;
+    });
+    mock(app.config.cnpmcore, 'syncMode', 'exist');
+    await app.runSchedule(CheckRecentlyUpdatedPackagesPath);
+    let task = await packageSyncerService.findExecuteTask();
+    assert(task);
+    assert(task.targetName === '@cnpm/foo');
+    task = await packageSyncerService.findExecuteTask();
+    assert(!task);
+  });
+
   it('should handle pageUrl request error', async () => {
     mock(app.config.cnpmcore, 'syncMode', 'all');
     app.mockLog();