feat: impl fast semver search

app/core/entity/PackageVersion.ts

@@ -1,6 +1,7 @@
 import { Dist } from './Dist';
 import { Entity, EntityData } from './Entity';
 import { EasyData, EntityUtil } from '../util/EntityUtil';
+import { PaddingSemVer } from './PaddingSemVer';
 
 interface PackageVersionData extends EntityData {
   packageId: string;
@@ -22,6 +23,8 @@ export class PackageVersion extends Entity {
   tarDist: Dist;
   readmeDist: Dist;
   publishTime: Date;
+  paddingVersion: string;
+  isPreRelease: boolean;
 
   constructor(data: PackageVersionData) {
     super(data);
@@ -33,6 +36,9 @@ export class PackageVersion extends Entity {
     this.tarDist = data.tarDist;
     this.readmeDist = data.readmeDist;
     this.publishTime = data.publishTime;
+    const paddingSemVer = new PaddingSemVer(this.version);
+    this.paddingVersion = paddingSemVer.paddingVersion;
+    this.isPreRelease = paddingSemVer.isPreRelease;
   }
 
   static create(data: EasyData<PackageVersionData, 'packageVersionId'>): PackageVersion {

app/core/entity/PaddingSemVer.ts

@@ -0,0 +1,44 @@
+import { SemVer } from 'semver';
+
+export class PaddingSemVer {
+  private readonly semver: SemVer;
+  // 跳过 semver 中的 buildInfo, buildInfo 不参与版本比较
+  private _paddingVersion: string;
+  readonly isPreRelease: boolean;
+
+  constructor(semver: string | SemVer) {
+    this.semver = new SemVer(semver);
+    if ((this.semver as any).includePrerelease) {
+      this.isPreRelease = true;
+    } else if (this.semver.prerelease && this.semver.prerelease.length) {
+      this.isPreRelease = true;
+    } else {
+      this.isPreRelease = false;
+    }
+  }
+
+  get paddingVersion(): string {
+    if (!this._paddingVersion) {
+      this._paddingVersion = PaddingSemVer.paddingVersion(this.semver.major)
+        + PaddingSemVer.paddingVersion(this.semver.minor)
+        + PaddingSemVer.paddingVersion(this.semver.patch);
+    }
+    return this._paddingVersion;
+  }
+
+  // 版本信息中为纯数字, JS 中支持的最大整型为 16 位
+  // 因此填充成 16 位对齐，如果版本号超过 16 位，则抛出异常
+  static paddingVersion(v) {
+    const t = String(v);
+    if (t.length <= 16) {
+      const padding = new Array(16 - t.length).fill(0)
+        .join('');
+      return padding + t;
+    }
+    throw new Error(`v ${v} too long`);
+  }
+
+  static anyVersion() {
+    return '000000000000000000000000000000000000000000000000';
+  }
+}

app/core/entity/SqlRange.ts

@@ -0,0 +1,81 @@
+import { Range, Comparator } from 'semver';
+import { PaddingSemVer } from './PaddingSemVer';
+
+const OPERATOR_MAP = {
+  '<': '$lt',
+  '<=': '$lte',
+  '>': '$gt',
+  '>=': '$gte',
+  '': '$eq',
+};
+
+export class SqlRange {
+  private readonly range: Range;
+  private _containPreRelease: boolean;
+  readonly condition: object;
+
+  constructor(range: string | Range) {
+    this.range = new Range(range);
+    this._containPreRelease = false;
+    this.condition = this.generateWhere();
+  }
+
+  private comparatorToSql(comparator: Comparator) {
+    if (comparator.semver === (Comparator as any).ANY) {
+      return {
+        $and: [
+          {
+            isPreRelease: {
+              $lte: 0,
+            },
+          },
+          {
+            paddingVersion: {
+              $gte: PaddingSemVer.anyVersion(),
+            },
+          },
+        ],
+      };
+    }
+    const paddingSemver = new PaddingSemVer(comparator.semver);
+    const operator = OPERATOR_MAP[comparator.operator];
+    if (!operator) {
+      throw new Error(`unknown operator ${comparator.operator}`);
+    }
+    this._containPreRelease = this._containPreRelease || paddingSemver.isPreRelease;
+    return {
+      $and: [
+        {
+          isPreRelease: {
+            $lte: paddingSemver.isPreRelease ? 1 : 0,
+          },
+        },
+        {
+          paddingVersion: {
+            [operator]: paddingSemver.paddingVersion,
+          },
+        },
+      ],
+    };
+  }
+
+  private comparatorSetToSql(comparatorSet: Array<Comparator>) {
+    const condition: Array<object> = [];
+    for (const comparator of comparatorSet) {
+      condition.push(this.comparatorToSql(comparator));
+    }
+    return { $and: condition };
+  }
+
+  private generateWhere() {
+    const conditions: Array<object> = [];
+    for (const rangeSet of this.range.set) {
+      conditions.push(this.comparatorSetToSql(rangeSet as Comparator[]));
+    }
+    return { $or: conditions };
+  }
+
+  get containPreRelease(): boolean {
+    return this._containPreRelease;
+  }
+}

app/core/event/BugVersionFixHandler.ts

@@ -2,23 +2,20 @@ import { Event, Inject } from '@eggjs/tegg';
 import { EggLogger } from 'egg';
 import { PACKAGE_VERSION_ADDED } from './index';
 import { BUG_VERSIONS } from '../../common/constants';
-import { PackageManagerService } from '../service/PackageManagerService';
 import { BugVersionService } from '../service/BugVersionService';
 
 @Event(PACKAGE_VERSION_ADDED)
 export class BugVersionFixHandler {
   @Inject()
   private readonly bugVersionService: BugVersionService;
-  @Inject()
-  private readonly packageManagerService: PackageManagerService;
 
   @Inject()
   private readonly logger: EggLogger;
 
   async handle(fullname: string) {
     if (fullname !== BUG_VERSIONS) return;
     try {
-      const bugVersion = await this.packageManagerService.getBugVersion();
+      const bugVersion = await this.bugVersionService.getBugVersion();
       if (!bugVersion) return;
       await this.bugVersionService.cleanBugVersionPackageCaches(bugVersion);
     } catch (e) {

app/core/service/BugVersionService.ts

@@ -2,10 +2,12 @@ import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg';
 import { EggLogger } from 'egg';
 import pMap from 'p-map';
 import { BugVersion } from '../entity/BugVersion';
-import { PackageRepository } from '../../repository/PackageRepository';
+import { PackageJSONType, PackageRepository } from '../../repository/PackageRepository';
 import { DistRepository } from '../../repository/DistRepository';
 import { getScopeAndName } from '../../common/PackageUtil';
 import { CacheService } from './CacheService';
+import { BUG_VERSIONS, LATEST_TAG } from '../../common/constants';
+import { BugVersionStore } from '../../common/adapter/BugVersionStore';
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -23,6 +25,27 @@ export class BugVersionService {
   @Inject()
   private readonly cacheService: CacheService;
 
+  @Inject()
+  private readonly bugVersionStore: BugVersionStore;
+
+  async getBugVersion(): Promise<BugVersion | undefined> {
+    // TODO performance problem, cache bugVersion and update with schedule
+    const pkg = await this.packageRepository.findPackage('', BUG_VERSIONS);
+    if (!pkg) return;
+    /* c8 ignore next 10 */
+    const tag = await this.packageRepository.findPackageTag(pkg!.packageId, LATEST_TAG);
+    if (!tag) return;
+    let bugVersion = this.bugVersionStore.getBugVersion(tag!.version);
+    if (!bugVersion) {
+      const packageVersionJson = (await this.distRepository.findPackageVersionManifest(pkg!.packageId, tag!.version)) as PackageJSONType;
+      if (!packageVersionJson) return;
+      const data = packageVersionJson.config?.['bug-versions'];
+      bugVersion = new BugVersion(data);
+      this.bugVersionStore.setBugVersion(bugVersion, tag!.version);
+    }
+    return bugVersion;
+  }
+
   async cleanBugVersionPackageCaches(bugVersion: BugVersion) {
     const fullnames = bugVersion.listAllPackagesHasBugs();
     await pMap(fullnames, async fullname => {

app/core/service/FixNoPaddingVersionService.ts

@@ -0,0 +1,33 @@
+import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg';
+import { EggLogger } from 'egg';
+import pMap from 'p-map';
+import { PackageVersionRepository } from '../../repository/PackageVersionRepository';
+import { PaddingSemVer } from '../entity/PaddingSemVer';
+
+@SingletonProto({
+  accessLevel: AccessLevel.PUBLIC,
+})
+export class FixNoPaddingVersionService {
+  @Inject()
+  private readonly packageVersionRepository: PackageVersionRepository;
+
+  @Inject()
+  private readonly logger: EggLogger;
+
+  async fixPaddingVersion(id?: number): Promise<void> {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const packageVersions = await this.packageVersionRepository.findHaveNotPaddingVersion(id);
+      if (packageVersions.length === 0) {
+        break;
+      }
+      id = packageVersions[packageVersions.length - 1].id as unknown as number + 1;
+      this.logger.info('[FixNoPaddingVersionService] fix padding version ids %j', packageVersions.map(t => t.id));
+
+      await pMap(packageVersions, async packageVersion => {
+        const paddingSemver = new PaddingSemVer(packageVersion.version);
+        await this.packageVersionRepository.fixPaddingVersion(packageVersion.packageVersionId, paddingSemver);
+      }, { concurrency: 30 });
+    }
+  }
+}

app/core/service/PackageManagerService.ts

@@ -7,6 +7,7 @@ import {
 } from '@eggjs/tegg';
 import { ForbiddenError } from 'egg-errors';
 import { RequireAtLeastOne } from 'type-fest';
+import npa from 'npm-package-arg';
 import semver from 'semver';
 import {
   calculateIntegrity,
@@ -17,8 +18,6 @@ import {
   hasShrinkWrapInTgz,
 } from '../../common/PackageUtil';
 import { AbstractService } from '../../common/AbstractService';
-import { BugVersionStore } from '../../common/adapter/BugVersionStore';
-import { BUG_VERSIONS, LATEST_TAG } from '../../common/constants';
 import { AbbreviatedPackageJSONType, AbbreviatedPackageManifestType, PackageJSONType, PackageManifestType, PackageRepository } from '../../repository/PackageRepository';
 import { PackageVersionBlockRepository } from '../../repository/PackageVersionBlockRepository';
 import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
@@ -46,6 +45,7 @@ import { BugVersionService } from './BugVersionService';
 import { BugVersion } from '../entity/BugVersion';
 import { RegistryManagerService } from './RegistryManagerService';
 import { Registry } from '../entity/Registry';
+import { PackageVersionService } from './PackageVersionService';
 
 export interface PublishPackageCmd {
   // maintainer: Maintainer;
@@ -91,11 +91,11 @@ export class PackageManagerService extends AbstractService {
   @Inject()
   private readonly bugVersionService: BugVersionService;
   @Inject()
-  private readonly bugVersionStore: BugVersionStore;
-  @Inject()
   private readonly distRepository: DistRepository;
   @Inject()
   private readonly registryManagerService: RegistryManagerService;
+  @Inject()
+  private readonly packageVersionService: PackageVersionService;
 
   private static downloadCounters = {};
 
@@ -371,7 +371,7 @@ export class PackageManagerService extends AbstractService {
     return await this._listPackageFullOrAbbreviatedManifests(scope, name, false, isSync);
   }
 
-  async showPackageVersionByVersionOrTag(scope: string, name: string, versionOrTag: string): Promise<{
+  async showPackageVersionByVersionOrTag(scope: string, name: string, spec: string): Promise<{
     blockReason?: string,
     pkg?: Package,
     packageVersion?: PackageVersion | null,
@@ -382,40 +382,27 @@ export class PackageManagerService extends AbstractService {
     if (block) {
       return { blockReason: block.reason, pkg };
     }
-    let version = versionOrTag;
-    if (!semver.valid(versionOrTag)) {
-      // invalid version, versionOrTag is a tag
-      const packageTag = await this.packageRepository.findPackageTag(pkg.packageId, versionOrTag);
-      if (packageTag) {
-        version = packageTag.version;
-      }
+    const fullname = getFullname(scope, name);
+    const result = npa(`${fullname}@${spec}`);
+    const version = await this.packageVersionService.getVersion(result);
+    if (!version) {
+      return {};
     }
     const packageVersion = await this.packageRepository.findPackageVersion(pkg.packageId, version);
     return { packageVersion, pkg };
   }
 
-  async showPackageVersionManifest(scope: string, name: string, versionOrTag: string, isSync = false) {
-    let manifest;
-    const { blockReason, packageVersion, pkg } = await this.showPackageVersionByVersionOrTag(scope, name, versionOrTag);
-    if (blockReason) {
-      return {
-        blockReason,
-        manifest,
-        pkg,
-      };
-    }
-    if (!packageVersion) return { manifest: null, blockReason, pkg };
-    manifest = await this.distRepository.findPackageVersionManifest(packageVersion.packageId, packageVersion.version);
-    let bugVersion: BugVersion | undefined;
-    // sync mode response no bug version fixed
-    if (!isSync) {
-      bugVersion = await this.getBugVersion();
-    }
-    if (bugVersion) {
-      const fullname = getFullname(scope, name);
-      manifest = await this.bugVersionService.fixPackageBugVersion(bugVersion, fullname, manifest);
+  async showPackageVersionManifest(scope: string, name: string, spec: string, isSync = false, isFullManifests = false) {
+    const pkg = await this.packageRepository.findPackage(scope, name);
+    if (!pkg) return {};
+    const block = await this.packageVersionBlockRepository.findPackageBlock(pkg.packageId);
+    if (block) {
+      return { blockReason: block.reason, pkg };
     }
-    return { manifest, blockReason, pkg };
+    const fullname = getFullname(scope, name);
+    const result = npa(`${fullname}@${spec}`);
+    const manifest = await this.packageVersionService.readManifest(pkg.packageId, result, isFullManifests, !isSync);
+    return { manifest, blockReason: null, pkg };
   }
 
   async downloadPackageVersionTar(packageVersion: PackageVersion) {
@@ -646,24 +633,6 @@ export class PackageManagerService extends AbstractService {
     await this._updatePackageManifestsToDists(pkg, fullManifests, abbreviatedManifests);
   }
 
-  async getBugVersion(): Promise<BugVersion | undefined> {
-    // TODO performance problem, cache bugVersion and update with schedule
-    const pkg = await this.packageRepository.findPackage('', BUG_VERSIONS);
-    if (!pkg) return;
-    /* c8 ignore next 10 */
-    const tag = await this.packageRepository.findPackageTag(pkg!.packageId, LATEST_TAG);
-    if (!tag) return;
-    let bugVersion = this.bugVersionStore.getBugVersion(tag!.version);
-    if (!bugVersion) {
-      const packageVersionJson = (await this.distRepository.findPackageVersionManifest(pkg!.packageId, tag!.version)) as PackageJSONType;
-      if (!packageVersionJson) return;
-      const data = packageVersionJson.config?.['bug-versions'];
-      bugVersion = new BugVersion(data);
-      this.bugVersionStore.setBugVersion(bugVersion, tag!.version);
-    }
-    return bugVersion;
-  }
-
   async getSourceRegistry(pkg: Package): Promise<Registry | null> {
     let registry: Registry | null;
     if (pkg.registryId) {
@@ -815,7 +784,7 @@ export class PackageManagerService extends AbstractService {
     let bugVersion: BugVersion | undefined;
     // sync mode response no bug version fixed
     if (!isSync) {
-      bugVersion = await this.getBugVersion();
+      bugVersion = await this.bugVersionService.getBugVersion();
     }
     const fullname = getFullname(scope, name);