feat: support unpkg features (#456)

WARN: include sql change

😄 Follow unpkg router
😄 Auto sync files after package version add

closes #452

DEVELOPER.md

@@ -97,14 +97,40 @@ PaaS 环境实现自己的 infra module。
 - QueueAdapter.ts
 - AuthAdapter.ts
 
+## 架构分层依赖图
+
+```txt
++--------------------------------+  +--------+  +----------+
+|            Controller          |  |        |  |          |
++----^-------------^-------------+  |        |  |          |
+     |             |                |        |  |          |
+     | inject      | inject         |        |  |          |
+     |             |                |        |  |          |
+     |  +----------+-------------+  |        |  |          |
+     |  |        Service         |  | Entity |  |          |
+     |  +-----------^------------+  |        |  |          |
+     |              |               |        |  |  Common  |
+     |              | inject        |        |  |          |
+     |              |               |        |  |          |
++----+--------------+------------+  |        |  |          |
+|          Repository            |  |        |  |          |
++-------------------^------------+  +---^----|  |          |
+                    |                   |       |          |
+                    | inject        ORM |       |          |
+                    |                   |       |          |
+        +-----------+------------+      |       |          |
+        |         Model          +<-----+       |          |
+        +------------------------+              +----------+
+```
+
 ## Controller 开发指南
 
 目前只支持 HTTP 协议的 Controller，代码在 `app/port/controller` 目录下。
 基于类继承的模式来实现，类关系大致如下：
 
 ```txt
 +----------------------+   +----------------------+   +---------------+
-| PackageController.ts |   | PackageTagController |   | XxxController |
+| PackageController    |   | PackageTagController |   | XxxController |
 +---------------+------+   +---+------------------+   +--+------------+
                 |              |                         |
                 | extends      | extends                 | extends
@@ -197,8 +223,34 @@ const { pkg } = await this.ensurePublishAccess(ctx, fullname);
 
 ## Service 开发指南
 
+Service 依赖 Repository，然后被 Controller 依赖
+
+```txt
++---------------------------+   +----------------------+   +-------------+
+| PackageVersionFileService |   | PackageSyncerService |   | XxxService  |
++---------------^-----------+   +---^------------------+   +--^----------+
+                |                   |                         |
+                | inject            | inject                  | inject
+                |                   |                         |
+            +---+-------------------+-------------------------+--+
+            |               PackageManagerService                |
+            +-----------------------^----------------------------+
+                                    |
+                                    | inject
+                                    |
+                          +---------+--------+
+                          |   XxxRepository  |
+                          +------------------+
+```
+
+### PackageManagerService 管理所有包以及版本信息
+
+它会被其他 Service 依赖
+
 ## Repository 开发指南
 
+Repository 依赖 Model，然后被 Service 和 Controller 依赖
+
 ### Repository 类方法命名规则
 
 - `findSomething` 查询一个模型数据

app/common/FileUtil.ts

@@ -1,10 +1,11 @@
-import { mkdir, rm } from 'fs/promises';
-import { createWriteStream } from 'fs';
-import { setTimeout } from 'timers/promises';
-import path from 'path';
-import url from 'url';
-import { randomBytes } from 'crypto';
+import { mkdir, rm } from 'node:fs/promises';
+import { createWriteStream } from 'node:fs';
+import { setTimeout } from 'node:timers/promises';
+import path from 'node:path';
+import url from 'node:url';
+import { randomBytes } from 'node:crypto';
 import { EggContextHttpClient, HttpClientResponse } from 'egg';
+import mime from 'mime-types';
 import dayjs from './dayjs';
 
 interface DownloadToTempfileOptionalConfig {
@@ -13,11 +14,18 @@ interface DownloadToTempfileOptionalConfig {
   remoteAuthToken?: string
 }
 
-export async function createTempfile(dataDir: string, filename: string) {
+export async function createTempDir(dataDir: string, dirname?: string) {
   // will auto clean on CleanTempDir Schedule
-  const tmpdir = path.join(dataDir, 'downloads', dayjs().format('YYYY/MM/DD'));
+  let tmpdir = path.join(dataDir, 'downloads', dayjs().format('YYYY/MM/DD'));
+  if (dirname) {
+    tmpdir = path.join(tmpdir, dirname);
+  }
   await mkdir(tmpdir, { recursive: true });
+  return tmpdir;
+}
 
+export async function createTempfile(dataDir: string, filename: string) {
+  const tmpdir = await createTempDir(dataDir);
   // The filename is a URL (from dist.tarball), which needs to be truncated, (`getconf NAME_MAX /` # max filename length: 255 bytes)
   // https://github.com/cnpm/cnpmjs.org/pull/1345
   const tmpfile = path.join(tmpdir, `${randomBytes(10).toString('hex')}-${path.basename(url.parse(filename).pathname!)}`);
@@ -84,3 +92,23 @@ async function _downloadToTempfile(httpclient: EggContextHttpClient,
     throw err;
   }
 }
+
+const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
+const WHITE_FILENAME_CONTENT_TYPES = {
+  license: 'text/plain',
+  readme: 'text/plain',
+  history: 'text/plain',
+  changelog: 'text/plain',
+  '.npmignore': 'text/plain',
+  '.jshintignore': 'text/plain',
+  '.jshintrc': 'application/json',
+  '.eslintignore': 'text/plain',
+  '.eslintrc': 'application/json',
+};
+
+export function mimeLookup(filepath: string) {
+  const filename = path.basename(filepath);
+  return mime.lookup(filename) ||
+    WHITE_FILENAME_CONTENT_TYPES[filename.toLowerCase()] ||
+    DEFAULT_CONTENT_TYPE;
+}

app/common/PackageUtil.ts

@@ -1,7 +1,8 @@
-import { createReadStream } from 'fs';
-import tar from 'tar';
-import stream from 'stream';
+import { createReadStream } from 'node:fs';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
 import * as ssri from 'ssri';
+import tar from 'tar';
 
 // /@cnpm%2ffoo
 // /@cnpm%2Ffoo
@@ -62,11 +63,11 @@ export function detectInstallScript(manifest: any) {
 
 /** 判断一个版本压缩包中是否包含 npm-shrinkwrap.json */
 export async function hasShrinkWrapInTgz(contentOrFile: Uint8Array | string): Promise<boolean> {
-  let readable: stream.Readable;
+  let readable: Readable;
   if (typeof contentOrFile === 'string') {
     readable = createReadStream(contentOrFile);
   } else {
-    readable = new stream.Readable({
+    readable = new Readable({
       read() {
         this.push(contentOrFile);
         this.push(null);
@@ -88,7 +89,7 @@ export async function hasShrinkWrapInTgz(contentOrFile: Uint8Array | string): Pr
   });
 
   try {
-    await stream.promises.pipeline(readable, parser, { signal: abortController.signal });
+    await pipeline(readable, parser, { signal: abortController.signal });
     return hasShrinkWrap;
   } catch (e) {
     if (e.code === 'ABORT_ERR') {

app/core/entity/Package.ts

@@ -84,6 +84,11 @@ export class Package extends Entity {
     return this.createDist(DIST_NAMES.ABBREVIATED_MANIFESTS, info);
   }
 
+  createPackageVersionFile(path: string, version: string, info: FileInfo) {
+    // path should starts with `/`, e.g.: '/foo/bar/index.js'
+    return this.createDist(`files${path}`, info, version);
+  }
+
   private distDir(filename: string, version?: string) {
     if (version) {
       return `/packages/${this.fullname}/${version}/${filename}`;

app/core/entity/PackageVersionFile.ts

@@ -0,0 +1,43 @@
+import { Dist } from './Dist';
+import { Entity, EntityData } from './Entity';
+import { EasyData, EntityUtil } from '../util/EntityUtil';
+
+interface PackageVersionFileData extends EntityData {
+  packageVersionFileId: string;
+  packageVersionId: string;
+  dist: Dist;
+  directory: string;
+  name: string;
+  contentType: string;
+  mtime: Date;
+}
+
+export class PackageVersionFile extends Entity {
+  packageVersionFileId: string;
+  packageVersionId: string;
+  dist: Dist;
+  directory: string;
+  name: string;
+  contentType: string;
+  mtime: Date;
+
+  constructor(data: PackageVersionFileData) {
+    super(data);
+    this.packageVersionFileId = data.packageVersionFileId;
+    this.packageVersionId = data.packageVersionId;
+    this.dist = data.dist;
+    this.directory = data.directory;
+    this.name = data.name;
+    this.contentType = data.contentType;
+    this.mtime = data.mtime;
+  }
+
+  get path() {
+    return this.directory === '/' ? `/${this.name}` : `${this.directory}/${this.name}`;
+  }
+
+  static create(data: EasyData<PackageVersionFileData, 'packageVersionFileId'>): PackageVersionFile {
+    const newData = EntityUtil.defaultData(data, 'packageVersionFileId');
+    return new PackageVersionFile(newData);
+  }
+}

app/core/event/SyncPackageVersionFile.ts

@@ -0,0 +1,33 @@
+import { Event, Inject } from '@eggjs/tegg';
+import {
+  EggAppConfig,
+} from 'egg';
+import { PACKAGE_VERSION_ADDED } from './index';
+import { getScopeAndName } from '../../common/PackageUtil';
+import { PackageManagerService } from '../service/PackageManagerService';
+import { PackageVersionFileService } from '../service/PackageVersionFileService';
+
+class SyncPackageVersionFileEvent {
+  @Inject()
+  protected readonly config: EggAppConfig;
+  @Inject()
+  private readonly packageManagerService: PackageManagerService;
+  @Inject()
+  private readonly packageVersionFileService: PackageVersionFileService;
+
+  protected async syncPackageVersionFile(fullname: string, version: string) {
+    if (!this.config.cnpmcore.enableUnpkg) return;
+    const [ scope, name ] = getScopeAndName(fullname);
+    const { packageVersion } = await this.packageManagerService.showPackageVersionByVersionOrTag(
+      scope, name, version);
+    if (!packageVersion) return;
+    await this.packageVersionFileService.syncPackageVersionFiles(packageVersion);
+  }
+}
+
+@Event(PACKAGE_VERSION_ADDED)
+export class PackageVersionAdded extends SyncPackageVersionFileEvent {
+  async handle(fullname: string, version: string) {
+    await this.syncPackageVersionFile(fullname, version);
+  }
+}

app/core/service/PackageManagerService.ts

@@ -47,7 +47,6 @@ import { BugVersion } from '../entity/BugVersion';
 import { RegistryManagerService } from './RegistryManagerService';
 import { Registry } from '../entity/Registry';
 
-
 export interface PublishPackageCmd {
   // maintainer: Maintainer;
   scope: string;
@@ -370,21 +369,16 @@ export class PackageManagerService extends AbstractService {
     return await this._listPackageFullOrAbbreviatedManifests(scope, name, false, isSync);
   }
 
-  async showPackageVersionManifest(scope: string, name: string, versionOrTag: string, isSync = false) {
-    let blockReason = '';
-    let manifest;
+  async showPackageVersionByVersionOrTag(scope: string, name: string, versionOrTag: string): Promise<{
+    blockReason?: string,
+    pkg?: Package,
+    packageVersion?: PackageVersion | null,
+  }> {
     const pkg = await this.packageRepository.findPackage(scope, name);
-    const pkgId = pkg?.packageId;
-    if (!pkg) return { manifest: null, blockReason, pkgId };
-
+    if (!pkg) return {};
     const block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
     if (block) {
-      blockReason = block.reason;
-      return {
-        blockReason,
-        manifest,
-        pkgId,
-      };
+      return { blockReason: block.reason, pkg };
     }
     let version = versionOrTag;
     if (!semver.valid(versionOrTag)) {
@@ -395,8 +389,21 @@ export class PackageManagerService extends AbstractService {
       }
     }
     const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
-    if (!packageVersion) return { manifest: null, blockReason, pkgId };
-    manifest = await this.distRepository.findPackageVersionManifest(packageVersion.packageId, version);
+    return { packageVersion, pkg };
+  }
+
+  async showPackageVersionManifest(scope: string, name: string, versionOrTag: string, isSync = false) {
+    let manifest;
+    const { blockReason, packageVersion, pkg } = await this.showPackageVersionByVersionOrTag(scope, name, versionOrTag);
+    if (blockReason) {
+      return {
+        blockReason,
+        manifest,
+        pkg,
+      };
+    }
+    if (!packageVersion) return { manifest: null, blockReason, pkg };
+    manifest = await this.distRepository.findPackageVersionManifest(packageVersion.packageId, packageVersion.version);
     let bugVersion: BugVersion | undefined;
     // sync mode response no bug version fixed
     if (!isSync) {
@@ -406,8 +413,7 @@ export class PackageManagerService extends AbstractService {
       const fullname = getFullname(scope, name);
       manifest = await this.bugVersionService.fixPackageBugVersion(bugVersion, fullname, manifest);
     }
-    return { manifest, blockReason, pkgId };
-
+    return { manifest, blockReason, pkg };
   }
 
   async downloadPackageVersionTar(packageVersion: PackageVersion) {