fix: use nfs download api (#461)

cnpm · May 6, 2023 · bb16957 · bb16957
1 parent 0858efb
commit bb16957
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 7 deletions.
diff --git a/app/common/adapter/NFSAdapter.ts b/app/common/adapter/NFSAdapter.ts
@@ -49,9 +49,16 @@ export class NFSAdapter {
  await this.nfsClient.upload(file, { key: storeKey });
  }
 
+ @Pointcut(AsyncTimer)
+ async downloadFile(storeKey: string, file: string, timeout: number) {
+ this.logger.info('[%s:downloadFile] key: %s, file: %s, timeout: %s',
+ INSTANCE_NAME, storeKey, file, timeout);
+ await this.nfsClient.download(storeKey, file, { timeout });
+ }
+
  @Pointcut(AsyncTimer)
  async remove(storeKey: string) {
- this.logger.info('[%s:remove] key: %s, file: %s', INSTANCE_NAME, storeKey);
+ this.logger.info('[%s:remove] key: %s', INSTANCE_NAME, storeKey);
  await this.nfsClient.remove(storeKey);
  }
 

diff --git a/app/common/typing.ts b/app/common/typing.ts
@@ -21,6 +21,10 @@ export interface AppendOptions {
  headers?: IncomingHttpHeaders,
 }
 
+export interface DownloadOptions {
+ timeout: number;
+}
+
 export interface NFSClient {
  uploadBytes(bytes: Uint8Array, options: UploadOptions): Promise<UploadResult>;
 
@@ -34,6 +38,8 @@ export interface NFSClient {
 
  createDownloadStream(key: string): Promise<Readable | undefined>;
 
+ download(key: string, filepath: string, options: DownloadOptions): Promise<void>;
+
  url?(key: string): string;
 }
 

diff --git a/app/core/service/PackageVersionFileService.ts b/app/core/service/PackageVersionFileService.ts
@@ -1,7 +1,5 @@
 import fs from 'node:fs/promises';
-import { createWriteStream } from 'node:fs';
 import { join, dirname, basename } from 'node:path';
-import { pipeline } from 'node:stream/promises';
 import { randomUUID } from 'node:crypto';
 import tar from 'tar';
 import {
@@ -55,16 +53,17 @@ export class PackageVersionFileService extends AbstractService {
 
  async syncPackageVersionFiles(pkgVersion: PackageVersion) {
  const files: PackageVersionFile[] = [];
- const tarStream = await this.distRepository.getDistStream(pkgVersion.tarDist);
- if (!tarStream) return files;
  const pkg = await this.packageRepository.findPackageByPackageId(pkgVersion.packageId);
  if (!pkg) return files;
  const dirname = `unpkg_${pkg.fullname.replace('/', '_')}@${pkgVersion.version}_${randomUUID()}`;
  const tmpdir = await createTempDir(this.config.dataDir, dirname);
  const tarFile = `${tmpdir}.tgz`;
  const paths: string[] = [];
  try {
- await pipeline(tarStream, createWriteStream(tarFile));
+ this.logger.info('[PackageVersionFileService.syncPackageVersionFiles:download-start] dist:%s(path:%s, size:%s) => tarFile:%s',
+ pkgVersion.tarDist.distId, pkgVersion.tarDist.path, pkgVersion.tarDist.size, tarFile);
+ await this.distRepository.downloadDistToFile(pkgVersion.tarDist, tarFile);
+ this.logger.info('[PackageVersionFileService.syncPackageVersionFiles:extract-start] tmpdir:%s', tmpdir);
  await tar.extract({
  file: tarFile,
  cwd: tmpdir,

diff --git a/app/infra/NFSClientAdapter.ts b/app/infra/NFSClientAdapter.ts
@@ -8,7 +8,7 @@ import {
 } from '@eggjs/tegg';
 import { EggAppConfig, EggLogger } from 'egg';
 import FSClient from 'fs-cnpm';
-import { AppendResult, NFSClient, UploadOptions, UploadResult } from '../common/typing';
+import { AppendResult, NFSClient, UploadOptions, UploadResult, DownloadOptions } from '../common/typing';
 import { Readable } from 'stream';
 
 @SingletonProto({
@@ -79,4 +79,8 @@ export class NFSClientAdapter implements EggObjectLifecycle, NFSClient {
  }
  return await this._client.uploadBuffer(bytes, options);
  }
+
+ async download(key: string, filePath: string, options: DownloadOptions): Promise<void> {
+ return await this._client.download(key, filePath, options);
+ }
 }
diff --git a/app/repository/DistRepository.ts b/app/repository/DistRepository.ts
@@ -69,4 +69,9 @@ export class DistRepository {
  async downloadDist(dist: Dist) {
  return await this.nfsAdapter.getDownloadUrlOrStream(dist.path);
  }
+
+ async downloadDistToFile(dist: Dist, file: string) {
+ // max up to 5mins
+ return await this.nfsAdapter.downloadFile(dist.path, file, 60000 * 5);
+ }
 }