fix: download tgz file to local file before untar it (#457)

avoid "zlib: unexpected end of file"
cnpm · May 5, 2023 · 90d5046 · 90d5046
1 parent 5f9a7a8
commit 90d5046
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 14 deletions.
diff --git a/app/common/FileUtil.ts b/app/common/FileUtil.ts
@@ -94,21 +94,23 @@ async function _downloadToTempfile(httpclient: EggContextHttpClient,
 }
 
 const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
+const PLAIN_TEXT = 'text/plain';
 const WHITE_FILENAME_CONTENT_TYPES = {
-  license: 'text/plain',
-  readme: 'text/plain',
-  history: 'text/plain',
-  changelog: 'text/plain',
-  '.npmignore': 'text/plain',
-  '.jshintignore': 'text/plain',
+  license: PLAIN_TEXT,
+  readme: PLAIN_TEXT,
+  history: PLAIN_TEXT,
+  changelog: PLAIN_TEXT,
+  '.npmignore': PLAIN_TEXT,
+  '.jshintignore': PLAIN_TEXT,
+  '.eslintignore': PLAIN_TEXT,
   '.jshintrc': 'application/json',
-  '.eslintignore': 'text/plain',
   '.eslintrc': 'application/json',
 };
 
 export function mimeLookup(filepath: string) {
-  const filename = path.basename(filepath);
+  const filename = path.basename(filepath).toLowerCase();
+  if (filename.endsWith('.ts')) return PLAIN_TEXT;
   return mime.lookup(filename) ||
-    WHITE_FILENAME_CONTENT_TYPES[filename.toLowerCase()] ||
+    WHITE_FILENAME_CONTENT_TYPES[filename] ||
     DEFAULT_CONTENT_TYPE;
 }
diff --git a/app/core/service/PackageVersionFileService.ts b/app/core/service/PackageVersionFileService.ts
@@ -1,4 +1,5 @@
 import fs from 'node:fs/promises';
+import { createWriteStream } from 'node:fs';
 import { join, dirname, basename } from 'node:path';
 import { pipeline } from 'node:stream/promises';
 import { randomUUID } from 'node:crypto';
@@ -60,15 +61,18 @@ export class PackageVersionFileService extends AbstractService {
     if (!pkg) return files;
     const dirname = `unpkg_${pkg.fullname.replace('/', '_')}@${pkgVersion.version}_${randomUUID()}`;
     const tmpdir = await createTempDir(this.config.dataDir, dirname);
+    const tarFile = `${tmpdir}.tgz`;
     const paths: string[] = [];
     try {
-      await pipeline(tarStream, tar.extract({
+      await pipeline(tarStream, createWriteStream(tarFile));
+      await tar.extract({
+        file: tarFile,
         cwd: tmpdir,
         strip: 1,
         onentry: entry => {
           paths.push(entry.path.replace(/^package\//i, '/'));
         },
-      }));
+      });
       for (const path of paths) {
         const localFile = join(tmpdir, path);
         const file = await this.#savePackageVersionFile(pkg, pkgVersion, path, localFile);
@@ -78,13 +82,19 @@ export class PackageVersionFileService extends AbstractService {
         pkgVersion.packageVersionId, paths.length, files.length, tmpdir);
       return files;
     } catch (err) {
-      this.logger.warn('[PackageVersionFileService.syncPackageVersionFiles:error] packageVersionId: %s, %d paths, error: %s, tmpdir: %s',
-        pkgVersion.packageVersionId, paths.length, err, tmpdir);
+      this.logger.warn('[PackageVersionFileService.syncPackageVersionFiles:error] packageVersionId: %s, %d paths, tmpdir: %s, error: %s',
+        pkgVersion.packageVersionId, paths.length, tmpdir, err);
       // ignore TAR_BAD_ARCHIVE error
       if (err.code === 'TAR_BAD_ARCHIVE') return files;
       throw err;
     } finally {
-      await fs.rm(tmpdir, { recursive: true, force: true });
+      try {
+        await fs.rm(tarFile, { force: true });
+        await fs.rm(tmpdir, { recursive: true, force: true });
+      } catch (err) {
+        this.logger.warn('[PackageVersionFileService.syncPackageVersionFiles:warn] remove tmpdir: %s, error: %s',
+          tmpdir, err);
+      }
     }
   }
 

diff --git a/test/common/FileUtil.test.ts b/test/common/FileUtil.test.ts
@@ -19,6 +19,9 @@ describe('test/common/FileUtil.test.ts', () => {
       assert.equal(mimeLookup('/docs/static/less/pouchdb/anchors.css'), 'text/css');
       assert.equal(mimeLookup('/foo/bar.xml'), 'application/xml');
       assert.equal(mimeLookup('/favicon.ico'), 'image/vnd.microsoft.icon');
+      assert.equal(mimeLookup('/index.ts'), 'text/plain');
+      assert.equal(mimeLookup('/index.d.ts'), 'text/plain');
+      assert.equal(mimeLookup('/index.txt'), 'text/plain');
     });
   });
 });
diff --git a/test/port/controller/PackageVersionFileController/raw.test.ts b/test/port/controller/PackageVersionFileController/raw.test.ts
@@ -1,4 +1,5 @@
 import { strict as assert } from 'node:assert';
+import { setTimeout } from 'node:timers/promises';
 import { app, mock } from 'egg-mock/bootstrap';
 import { TestUtil } from 'test/TestUtil';
 import { calculateIntegrity } from 'app/common/PackageUtil';
@@ -143,6 +144,8 @@ describe('test/port/controller/PackageVersionFileController/raw.test.ts', () =>
         .set('user-agent', publisher.ua)
         .send(pkg);
       assert.equal(res.status, 201);
+      // wait for sync event finish
+      await setTimeout(3000);
       res = await app.httpRequest()
         .get(`/${pkg.name}/1.0.0/files/`);
       assert.equal(res.status, 200);