feat: read remote auth token from database (#595)

closes #586

app/common/FileUtil.ts

@@ -64,7 +64,7 @@ async function _downloadToTempfile(httpclient: EggContextHttpClient,
   try {
     // max 10 mins to download
     // FIXME: should show download progress
-    const authorization = optionalConfig?.remoteAuthToken ? `Bearer ${optionalConfig?.remoteAuthToken}` : '';
+    const authorization = optionalConfig?.remoteAuthToken ? `Bearer ${optionalConfig.remoteAuthToken}` : '';
     const { status, headers, res } = await httpclient.request(url, {
       timeout: 60000 * 10,
       headers: { authorization },

app/core/entity/Registry.ts

@@ -9,6 +9,7 @@ interface RegistryData extends EntityData {
   changeStream: string;
   userPrefix: string;
   type: RegistryType;
+  authToken?: string;
 }
 
 export type CreateRegistryData = Omit<EasyData<RegistryData, 'registryId'>, 'id'>;
@@ -20,6 +21,7 @@ export class Registry extends Entity {
   changeStream: string;
   userPrefix: string;
   type: RegistryType;
+  authToken?: string;
 
   constructor(data: RegistryData) {
     super(data);
@@ -29,10 +31,11 @@ export class Registry extends Entity {
     this.changeStream = data.changeStream;
     this.userPrefix = data.userPrefix;
     this.type = data.type;
+    this.authToken = data.authToken;
   }
 
   public static create(data: CreateRegistryData): Registry {
-    const newData = EntityUtil.defaultData(data, 'registryId');
+    const newData = EntityUtil.defaultData<RegistryData, 'registryId'>(data, 'registryId');
     return new Registry(newData);
   }
 }

app/core/entity/Task.ts

@@ -31,7 +31,6 @@ export interface TaskData<T = TaskBaseData> extends EntityData {
 export type SyncPackageTaskOptions = {
   authorId?: string;
   authorIp?: string;
-  remoteAuthToken?: string;
   tips?: string;
   skipDependencies?: boolean;
   syncDownloadData?: boolean;
@@ -52,7 +51,6 @@ export interface TriggerHookTaskData extends TaskBaseData {
 }
 
 export interface CreateSyncPackageTaskData extends TaskBaseData {
-  remoteAuthToken?: string;
   tips?: string;
   skipDependencies?: boolean;
   syncDownloadData?: boolean;
@@ -133,7 +131,6 @@ export class Task<T extends TaskBaseData = TaskBaseData> extends Entity {
       data: {
         // task execute worker
         taskWorker: '',
-        remoteAuthToken: options?.remoteAuthToken,
         tips: options?.tips,
         registryId: options?.registryId ?? '',
         skipDependencies: options?.skipDependencies,

app/core/service/PackageSyncerService.ts

@@ -121,13 +121,13 @@ export class PackageSyncerService extends AbstractService {
     const start = '2011-01-01';
     const end = this.config.cnpmcore.syncDownloadDataMaxDate;
     const registry = this.config.cnpmcore.syncDownloadDataSourceRegistry;
+    const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry);
     const logs: string[] = [];
     let downloads: { day: string; downloads: number }[];
 
     logs.push(`[${isoNow()}][DownloadData] 🚧🚧🚧🚧🚧 Syncing "${fullname}" download data "${start}:${end}" on ${registry} 🚧🚧🚧🚧🚧`);
     const failEnd = '❌❌❌❌❌ 🚮 give up 🚮 ❌❌❌❌❌';
     try {
-      const { remoteAuthToken } = task.data as SyncPackageTaskOptions;
       const { data, status, res } = await this.npmRegistry.getDownloadRanges(registry, fullname, start, end, { remoteAuthToken });
       downloads = data.downloads || [];
       logs.push(`[${isoNow()}][DownloadData] 🚧 HTTP [${status}] timing: ${JSON.stringify(res.timing)}, downloads: ${downloads.length}`);
@@ -164,7 +164,7 @@ export class PackageSyncerService extends AbstractService {
   private async syncUpstream(task: Task) {
     const registry = this.npmRegistry.registry;
     const fullname = task.targetName;
-    const { remoteAuthToken } = task.data as SyncPackageTaskOptions;
+    const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry);
     let logs: string[] = [];
     let logId = '';
     logs.push(`[${isoNow()}][UP] 🚧🚧🚧🚧🚧 Waiting sync "${fullname}" task on ${registry} 🚧🚧🚧🚧🚧`);
@@ -351,10 +351,11 @@ export class PackageSyncerService extends AbstractService {
   public async executeTask(task: Task) {
     const fullname = task.targetName;
     const [ scope, name ] = getScopeAndName(fullname);
-    const { tips, skipDependencies: originSkipDependencies, syncDownloadData, forceSyncHistory, remoteAuthToken, specificVersions } = task.data as SyncPackageTaskOptions;
+    const { tips, skipDependencies: originSkipDependencies, syncDownloadData, forceSyncHistory, specificVersions } = task.data as SyncPackageTaskOptions;
     let pkg = await this.packageRepository.findPackage(scope, name);
     const registry = await this.initSpecRegistry(task, pkg, scope);
     const registryHost = this.npmRegistry.registry;
+    const remoteAuthToken = registry.authToken;
     let logs: string[] = [];
     if (tips) {
       logs.push(`[${isoNow()}] 👉👉👉👉👉 Tips: ${tips} 👈👈👈👈👈`);
@@ -875,7 +876,6 @@ export class PackageSyncerService extends AbstractService {
         authorId: task.authorId,
         authorIp: task.authorIp,
         tips,
-        remoteAuthToken,
       });
       logs.push(`[${isoNow()}] 📦 Add dependency "${dependencyName}" sync task: ${dependencyTask.taskId}, db id: ${dependencyTask.id}`);
     }

app/core/service/RegistryManagerService.ts

@@ -14,10 +14,10 @@ import { Task } from '../entity/Task';
 import { ChangesStreamMode, PresetRegistryName } from '../../common/constants';
 import { RegistryType } from '../../common/enum/Registry';
 
-export interface CreateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name'> {
+export interface CreateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name' | 'authToken' > {
   operatorId?: string;
 }
-export interface UpdateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'userPrefix' | 'type' | 'name' | 'registryId'> {
+export interface UpdateRegistryCmd extends Pick<Registry, 'changeStream' | 'host' | 'type' | 'name' | 'authToken' > {
   operatorId?: string;
 }
 export interface RemoveRegistryCmd extends Pick<Registry, 'registryId'> {
@@ -61,23 +61,24 @@ export class RegistryManagerService extends AbstractService {
   }
 
   async createRegistry(createCmd: CreateRegistryCmd): Promise<Registry> {
-    const { name, changeStream = '', host, userPrefix = '', type, operatorId = '-' } = createCmd;
+    const { name, changeStream = '', host, userPrefix = '', type, operatorId = '-', authToken } = createCmd;
     this.logger.info('[RegistryManagerService.createRegistry:prepare] operatorId: %s, createCmd: %j', operatorId, createCmd);
     const registry = Registry.create({
       name,
       changeStream,
       host,
       userPrefix,
       type,
+      authToken,
     });
     await this.registryRepository.saveRegistry(registry);
     return registry;
   }
 
   // 更新部分 registry 信息
   // 不允许 userPrefix 字段变更
-  async updateRegistry(updateCmd: UpdateRegistryCmd) {
-    const { name, changeStream, host, type, registryId, operatorId = '-' } = updateCmd;
+  async updateRegistry(registryId: string, updateCmd: UpdateRegistryCmd) {
+    const { name, changeStream, host, type, operatorId = '-', authToken } = updateCmd;
     this.logger.info('[RegistryManagerService.updateRegistry:prepare] operatorId: %s, updateCmd: %j', operatorId, updateCmd);
     const registry = await this.registryRepository.findRegistryByRegistryId(registryId);
     if (!registry) {
@@ -88,6 +89,7 @@ export class RegistryManagerService extends AbstractService {
       changeStream,
       host,
       type,
+      authToken,
     });
     await this.registryRepository.saveRegistry(registry);
   }
@@ -105,6 +107,10 @@ export class RegistryManagerService extends AbstractService {
     return await this.registryRepository.findRegistry(registryName);
   }
 
+  async findByRegistryHost(host?: string): Promise<Registry | null> {
+    return host ? await this.registryRepository.findRegistryByRegistryHost(host) : null;
+  }
+
   // 删除 Registry 方法
   // 可选传入 operatorId 作为参数，用于记录操作人员
   // 同时删除对应的 scope 数据
@@ -156,4 +162,12 @@ export class RegistryManagerService extends AbstractService {
 
   }
 
+  async getAuthTokenByRegistryHost(host: string): Promise<string|undefined> {
+    const registry = await this.findByRegistryHost(host);
+    if (!registry) {
+      return undefined;
+    }
+    return registry.authToken;
+  }
+
 }

app/port/controller/PackageSyncController.ts

@@ -67,7 +67,6 @@ export class PackageSyncController extends AbstractController {
 
     const params = {
       fullname,
-      remoteAuthToken: data.remoteAuthToken,
       tips,
       skipDependencies: !!data.skipDependencies,
       syncDownloadData: !!data.syncDownloadData,
@@ -97,7 +96,6 @@ export class PackageSyncController extends AbstractController {
     const task = await this.packageSyncerService.createTask(params.fullname, {
       authorIp: ctx.ip,
       authorId: authorized?.user.userId,
-      remoteAuthToken: params.remoteAuthToken,
       tips: params.tips,
       skipDependencies: params.skipDependencies,
       syncDownloadData: params.syncDownloadData,

app/port/controller/RegistryController.ts

@@ -13,10 +13,10 @@ import {
 import { NotFoundError } from 'egg-errors';
 import { AbstractController } from './AbstractController';
 import { Static } from 'egg-typebox-validate/typebox';
-import { RegistryManagerService } from '../../core/service/RegistryManagerService';
+import { RegistryManagerService, UpdateRegistryCmd } from '../../core/service/RegistryManagerService';
 import { AdminAccess } from '../middleware/AdminAccess';
 import { ScopeManagerService } from '../../core/service/ScopeManagerService';
-import { RegistryCreateOptions, QueryPageOptions, RegistryCreateSyncOptions } from '../typebox';
+import { RegistryCreateOptions, QueryPageOptions, RegistryCreateSyncOptions, RegistryUpdateOptions } from '../typebox';
 
 @HTTPController()
 export class RegistryController extends AbstractController {
@@ -67,14 +67,15 @@ export class RegistryController extends AbstractController {
   async createRegistry(@Context() ctx: EggContext, @HTTPBody() registryOptions: Static<typeof RegistryCreateOptions>) {
     ctx.tValidate(RegistryCreateOptions, registryOptions);
     const authorizedUser = await this.userRoleManager.requiredAuthorizedUser(ctx, 'setting');
-    const { name, changeStream, host, userPrefix = '', type } = registryOptions;
+    const { name, changeStream, host, userPrefix = '', type, authToken } = registryOptions;
     await this.registryManagerService.createRegistry({
       name,
       changeStream,
       host,
       userPrefix,
       operatorId: authorizedUser.userId,
       type,
+      authToken,
     });
     return { ok: true };
   }
@@ -106,4 +107,29 @@ export class RegistryController extends AbstractController {
     await this.registryManagerService.remove({ registryId: id, operatorId: authorizedUser.userId });
     return { ok: true };
   }
+
+  @HTTPMethod({
+    path: '/-/registry/:id',
+    method: HTTPMethodEnum.PATCH,
+  })
+  @Middleware(AdminAccess)
+  async updateRegistry(@Context() ctx: EggContext, @HTTPParam() id: string, @HTTPBody() updateRegistryOptions: Partial<UpdateRegistryCmd>) {
+    ctx.tValidate(RegistryUpdateOptions, updateRegistryOptions);
+    const registry = await this.registryManagerService.findByRegistryId(id);
+    if (!registry) {
+      throw new NotFoundError('registry not found');
+    } else {
+      const { name, changeStream, host, type, authToken } = registry;
+      const _updateRegistryOptions = {
+        name,
+        changeStream,
+        host,
+        type,
+        authToken,
+        ...updateRegistryOptions,
+      };
+      await this.registryManagerService.updateRegistry(registry.registryId, _updateRegistryOptions);
+    }
+    return { ok: true };
+  }
 }

app/port/typebox.ts

@@ -79,12 +79,6 @@ export const TagWithVersionRule = Type.Object({
 
 export const SyncPackageTaskRule = Type.Object({
   fullname: Name,
-  remoteAuthToken: Type.Optional(
-    Type.String({
-      transform: [ 'trim' ],
-      maxLength: 200,
-    }),
-  ),
   tips: Type.String({
     transform: [ 'trim' ],
     maxLength: 1024,
@@ -210,35 +204,44 @@ export const RegistryCreateOptions = Type.Object({
     maxLength: 256,
   })),
   type: Type.Enum(RegistryType),
+  authToken: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      maxLength: 256,
+    }),
+  ),
 });
 
 export const RegistryUpdateOptions = Type.Object({
-  name: Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 256,
-  }),
-  host: Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 4096,
-  }),
-  changeStream: Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 4096,
-  }),
-  userPrefix: Type.Optional(Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 256,
-  })),
-  type: Type.Enum(RegistryType),
-  registryId: Type.String({
-    transform: [ 'trim' ],
-    minLength: 1,
-    maxLength: 256,
-  }),
+  name: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      minLength: 1,
+      maxLength: 256,
+    }),
+  ),
+  host: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      minLength: 1,
+      maxLength: 4096,
+    }),
+  ),
+  changeStream: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      minLength: 1,
+      maxLength: 4096,
+    }),
+  ),
+  type: Type.Optional(Type.Enum(RegistryType)),
+  authToken: Type.Optional(
+    Type.String({
+      transform: [ 'trim' ],
+      minLength: 1,
+      maxLength: 256,
+    }),
+  ),
 });
 
 export const ScopeCreateOptions = Type.Object({

app/repository/RegistryRepository.ts

@@ -38,6 +38,14 @@ export class RegistryRepository extends AbstractRepository {
     return null;
   }
 
+  async findRegistryByRegistryHost(host: string): Promise<RegistryEntity | null> {
+    const model = await this.Registry.findOne({ host });
+    if (model) {
+      return ModelConvertor.convertModelToEntity(model, RegistryEntity);
+    }
+    return null;
+  }
+
   async saveRegistry(registry: Registry) {
     if (registry.id) {
       const model = await this.Registry.findOne({ id: registry.id });

app/repository/model/Registry.ts

@@ -36,4 +36,7 @@ export class Registry extends Bone {
   @Attribute(DataTypes.STRING(256))
   type: RegistryType;
 
+  @Attribute(DataTypes.STRING(256), { name: 'auth_token' })
+  authToken?: string;
+
 }

sql/3.46.0.sql

@@ -0,0 +1 @@
+ALTER TABLE `registries` ADD COLUMN `auth_token` varchar(256) DEFAULT NULL COMMENT 'registry auth token';

test/core/service/PackageSyncerService/executeTask.test.ts

@@ -428,8 +428,10 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
       assert(log.includes('] 📦 Add dependency "@resvg/resvg-js-win32-x64-msvc" sync task: '));
     });
 
-    it('should bring auth token when set remoteAuthToken', async () => {
+    it('should bring auth token which in registry database.', async () => {
       const testToken = 'test-auth-token';
+      const registry = await registryManagerService.ensureDefaultRegistry();
+      await registryManagerService.updateRegistry(registry.registryId, { ...registry, authToken: testToken });
       const fullManifests = await TestUtil.readFixturesFile('registry.npmjs.org/foobar.json');
       const tgzBuffer1_0_0 = await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz');
       const tgzBuffer1_1_0 = await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.1.0.tgz');
@@ -459,7 +461,7 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
           persist: false,
         };
       });
-      await packageSyncerService.createTask('foobar', { skipDependencies: true, remoteAuthToken: testToken });
+      await packageSyncerService.createTask('foobar', { skipDependencies: true });
       const task = await packageSyncerService.findExecuteTask();
       assert(task);
       await packageSyncerService.executeTask(task);