feat: sync prisma binary from R2 (#474)

closes #473

app/common/adapter/binary/AbstractBinary.ts

@@ -26,12 +26,12 @@ export abstract class AbstractBinary {
   @Inject()
   protected httpclient: EggHttpClient;
 
-  abstract init(binaryName: BinaryName): Promise<void>;
+  abstract initFetch(binaryName: BinaryName): Promise<void>;
   abstract fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined>;
 
   protected async requestXml(url: string) {
     const { status, data, headers } = await this.httpclient.request(url, {
-      timeout: 20000,
+      timeout: 30000,
       followRedirect: true,
       gzip: true,
     });
@@ -45,7 +45,7 @@ export abstract class AbstractBinary {
 
   protected async requestJSON(url: string) {
     const { status, data, headers } = await this.httpclient.request(url, {
-      timeout: 20000,
+      timeout: 30000,
       dataType: 'json',
       followRedirect: true,
       gzip: true,

app/common/adapter/binary/ApiBinary.ts

@@ -9,7 +9,7 @@ export class ApiBinary extends AbstractBinary {
   @Inject()
   private readonly config: EggAppConfig;
 
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/adapter/binary/BucketBinary.ts

@@ -7,7 +7,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Bucket)
 export class BucketBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/adapter/binary/ChromeForTestingBinary.ts

@@ -9,7 +9,7 @@ export class ChromeForTestingBinary extends AbstractBinary {
     [key: string]: BinaryItem[];
   };
 
-  async init() {
+  async initFetch() {
     this.dirItems = undefined;
   }
 

app/common/adapter/binary/CypressBinary.ts

@@ -9,7 +9,7 @@ export class CypressBinary extends AbstractBinary {
     [key: string]: BinaryItem[];
   } | null;
 
-  async init() {
+  async initFetch() {
     this.dirItems = undefined;
   }
 

app/common/adapter/binary/GithubBinary.ts

@@ -8,7 +8,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 export class GithubBinary extends AbstractBinary {
   private releases: Record<string, any[]> = {};
 
-  async init(binaryName: BinaryName) {
+  async initFetch(binaryName: BinaryName) {
     delete this.releases[binaryName];
   }
 

app/common/adapter/binary/ImageminBinary.ts

@@ -6,7 +6,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Imagemin)
 export class ImageminBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/adapter/binary/NodeBinary.ts

@@ -6,7 +6,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Node)
 export class NodeBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/adapter/binary/NodePreGypBinary.ts

@@ -7,7 +7,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.NodePreGyp)
 export class NodePreGypBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/adapter/binary/PlaywrightBinary.ts

@@ -150,7 +150,7 @@ const DOWNLOAD_PATHS = {
 @BinaryAdapter(BinaryType.Playwright)
 export class PlaywrightBinary extends AbstractBinary {
   private dirItems?: Record<string, BinaryItem[]>;
-  async init() {
+  async initFetch() {
     this.dirItems = undefined;
   }
 

app/common/adapter/binary/PrismaBinary.ts

@@ -0,0 +1,128 @@
+import path from 'node:path';
+import { SingletonProto } from '@eggjs/tegg';
+import { BinaryType } from '../../enum/Binary';
+import binaries, { BinaryName, BinaryTaskConfig } from '../../../../config/binaries';
+import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './AbstractBinary';
+
+@SingletonProto()
+@BinaryAdapter(BinaryType.Prisma)
+export class PrismaBinary extends AbstractBinary {
+  private dirItems: {
+    [key: string]: BinaryItem[];
+  } = {};
+
+  async initFetch() {
+    // https://github.com/cnpm/cnpmcore/issues/473#issuecomment-1562115738
+    const pkgUrl = 'https://registry.npmjs.com/@prisma/engines';
+    const data = await this.requestJSON(pkgUrl);
+    const modified = data.time.modified;
+    this.dirItems = {};
+    this.dirItems['/'] = [
+      {
+        name: 'all_commits/',
+        date: modified,
+        size: '-',
+        isDir: true,
+        url: '',
+      },
+    ];
+    this.dirItems['/all_commits/'] = [];
+    const commitIdMap: Record<string, boolean> = {};
+    // https://list-binaries.prisma-orm.workers.dev/?delimiter=/&prefix=all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/
+    for (const version in data.versions) {
+      const major = parseInt(version.split('.', 1)[0]);
+      // need >= 3.0.0
+      if (major < 3) continue;
+      const date = data.time[version];
+      const pkg = data.versions[version];
+      // https://registry.npmjs.com/@prisma/engines/4.14.1
+      const enginesVersion = pkg.devDependencies['@prisma/engines-version'] || '';
+      // "@prisma/engines-version": "4.14.0-67.d9a4c5988f480fa576d43970d5a23641aa77bc9c"
+      const matched = /\.(\w{30,})$/.exec(enginesVersion);
+      if (!matched) continue;
+      const commitId = matched[1];
+      if (commitIdMap[commitId]) continue;
+      commitIdMap[commitId] = true;
+      this.dirItems['/all_commits/'].push({
+        name: `${commitId}/`,
+        date,
+        size: '-',
+        isDir: true,
+        url: '',
+      });
+    }
+  }
+
+  async fetch(dir: string, binaryName: BinaryName): Promise<FetchResult | undefined> {
+    const existsItems = this.dirItems[dir];
+    if (existsItems) {
+      return { items: existsItems, nextParams: null };
+    }
+    // /foo/ => foo/
+    const binaryConfig = binaries[binaryName];
+    const subDir = dir.substring(1);
+    const url = `${binaryConfig.distUrl}?delimiter=/&prefix=${encodeURIComponent(subDir)}`;
+    const result = await this.requestJSON(url);
+    return { items: this.#parseItems(result, binaryConfig), nextParams: null };
+  }
+
+  #parseItems(result: any, binaryConfig: BinaryTaskConfig): BinaryItem[] {
+    const items: BinaryItem[] = [];
+    // objects": [
+    //   {
+    //   "uploaded": "2023-05-23T15:43:05.772Z",
+    //   "checksums": {
+    //   "md5": "d41d8cd98f00b204e9800998ecf8427e"
+    //   },
+    //   "httpEtag": "\"d41d8cd98f00b204e9800998ecf8427e\"",
+    //   "etag": "d41d8cd98f00b204e9800998ecf8427e",
+    //   "size": 0,
+    //   "version": "7e77b6b8c1d214f2c6be3c959749b5a6",
+    //   "key": "all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/darwin-arm64/.finished"
+    //   },
+    // {
+    //   "uploaded": "2023-05-23T15:41:33.861Z",
+    //   "checksums": {
+    //   "md5": "4822215a13ae372ae82afd12689fce37"
+    //   },
+    //   "httpEtag": "\"4822215a13ae372ae82afd12689fce37\"",
+    //   "etag": "4822215a13ae372ae82afd12689fce37",
+    //   "size": 96,
+    //   "version": "7e77b6ba29d4e776023e4fa62825c13a",
+    //   "key": "all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/darwin-arm64/libquery_engine.dylib.node.gz.sha256"
+    //   },
+    // https://list-binaries.prisma-orm.workers.dev/?delimiter=/&prefix=all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/darwin-arm64/
+    const objects: {
+      uploaded: string;
+      size: number;
+      key: string;
+    }[] = result.objects || [];
+    for (const o of objects) {
+      const fullname = o.key;
+      // ignore size = 0
+      if (o.size === 0) continue;
+      const name = path.basename(fullname);
+      items.push({
+        name,
+        isDir: false,
+        url: `${binaryConfig.distUrl}${fullname}`,
+        size: o.size,
+        date: o.uploaded,
+      });
+    }
+    // delimitedPrefixes: [ 'all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/darwin-arm64/' ]
+    // https://list-binaries.prisma-orm.workers.dev/?delimiter=/&prefix=all_commits/61023c35d2c8762f66f09bc4183d2f630b541d08/
+    const delimitedPrefixes: string[] = result.delimitedPrefixes || [];
+    for (const fullname of delimitedPrefixes) {
+      const name = `${path.basename(fullname)}/`;
+      items.push({
+        name,
+        isDir: true,
+        url: '',
+        size: '-',
+        date: new Date().toISOString(),
+      });
+    }
+    return items;
+  }
+}

app/common/adapter/binary/PuppeteerBinary.ts

@@ -9,7 +9,7 @@ export class PuppeteerBinary extends AbstractBinary {
     [key: string]: BinaryItem[];
   };
 
-  async init() {
+  async initFetch() {
     this.dirItems = undefined;
   }
 

app/common/adapter/binary/SqlcipherBinary.ts

@@ -5,7 +5,7 @@ import { AbstractBinary, FetchResult, BinaryItem, BinaryAdapter } from './Abstra
 @SingletonProto()
 @BinaryAdapter(BinaryType.Sqlcipher)
 export class SqlcipherBinary extends AbstractBinary {
-  async init() {
+  async initFetch() {
     // do nothing
     return;
   }

app/common/enum/Binary.ts

@@ -10,6 +10,7 @@ export enum BinaryType {
   Nwjs = 'nwjs',
   Playwright = 'playwright',
   Puppeteer = 'puppeteer',
+  Prisma = 'prisma',
   Sqlcipher = 'sqlcipher',
   ChromeForTesting = 'chromeForTesting',
 }

app/core/service/BinarySyncerService.ts

@@ -287,7 +287,7 @@ export class BinarySyncerService extends AbstractService {
     } else {
       binaryAdapter = await this.eggObjectFactory.getEggObject(AbstractBinary, binaryConfig.type);
     }
-    await binaryAdapter.init(binaryName);
+    await binaryAdapter.initFetch(binaryName);
     return binaryAdapter;
   }
 }

config/binaries.ts

@@ -220,26 +220,10 @@ const binaries = {
   },
   prisma: {
     category: 'prisma',
-    description: 'Next-generation ORM for Node.js & TypeScript | PostgreSQL, MySQL, MariaDB, SQL Server, SQLite & MongoDB (Preview) https://www.prisma.io/',
-    type: BinaryType.Bucket,
+    description: 'Next-generation Node.js and TypeScript ORM https://www.prisma.io/',
+    type: BinaryType.Prisma,
     repo: 'prisma/prisma',
-    distUrl: 'https://prisma-builds.s3-eu-west-1.amazonaws.com/',
-    ignoreDirs: [
-      // https://prisma-builds.s3-eu-west-1.amazonaws.com/?delimiter=/&prefix=
-      // https://github.com/cnpm/cnpmcore/issues/209
-      // need all_commits dir on 3.12.0
-      // '/all_commits/',
-      '/build_testruns/',
-      '/bump_engineer/',
-      '/m1_builds/',
-      '/master/',
-      '/ci/',
-      '/unreverse/',
-      '/signature_test_run/',
-      '/sql-server-char-collation-fix/',
-      '/test-ldd-output-on-release/',
-      '/windows-mysql-ci/',
-    ],
+    distUrl: 'https://list-binaries.prisma-orm.workers.dev/',
   },
   // ImageminBinary
   'jpegtran-bin': {

test/common/adapter/binary/BucketBinary.test.ts

@@ -87,19 +87,5 @@ describe('test/common/adapter/binary/BucketBinary.test.ts', () => {
         assert(item.name !== 'AWSLogs/');
       }
     });
-
-    it('should ignore build_testruns/', async () => {
-      app.mockHttpclient('https://prisma-builds.s3-eu-west-1.amazonaws.com/', 'GET', {
-        data: await TestUtil.readFixturesFile('prisma-builds.s3-eu-west-1.amazonaws.com/index.xml'),
-        persist: false,
-      });
-      const result = await binary.fetch('/', 'prisma');
-      assert(result);
-      assert(result.items.length > 0);
-      for (const item of result.items) {
-        assert(item.name !== 'build_testruns/');
-        assert(/^\d{4}\-\d{2}\-\d{2}T\d{2}:00:00Z$/.test(item.date));
-      }
-    });
   });
 });