fix: ignore non-file on tar entry filter (#458)

> ENOENT: no such file or directory, stat
'/root/.cnpmcore/downloads/2023/05/06/unpkg_@iov_wallet-providers@1.0.0_0f152162-9cce-4a80-bacc-41271b7aac3f/package'

app/common/FileUtil.ts

@@ -110,6 +110,7 @@ const WHITE_FILENAME_CONTENT_TYPES = {
 export function mimeLookup(filepath: string) {
   const filename = path.basename(filepath).toLowerCase();
   if (filename.endsWith('.ts')) return PLAIN_TEXT;
+  if (filename.endsWith('.lock')) return PLAIN_TEXT;
   return mime.lookup(filename) ||
     WHITE_FILENAME_CONTENT_TYPES[filename] ||
     DEFAULT_CONTENT_TYPE;

app/core/service/PackageVersionFileService.ts

@@ -70,6 +70,8 @@ export class PackageVersionFileService extends AbstractService {
         cwd: tmpdir,
         strip: 1,
         onentry: entry => {
+          if (entry.type !== 'File') return;
+          if (!entry.path.startsWith('package/')) return;
           paths.push(entry.path.replace(/^package\//i, '/'));
         },
       });
@@ -105,7 +107,10 @@ export class PackageVersionFileService extends AbstractService {
     if (file) return file;
     const stat = await fs.stat(localFile);
     const distIntegrity = await calculateIntegrity(localFile);
-    const dist = pkg.createPackageVersionFile(path, pkgVersion.version, {
+    // make sure dist.path store to ascii, e.g. '/resource/ToOneFromχ.js' => '/resource/ToOneFrom%CF%87.js'
+    // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURI
+    const distPath = encodeURI(path);
+    const dist = pkg.createPackageVersionFile(distPath, pkgVersion.version, {
       size: stat.size,
       shasum: distIntegrity.shasum,
       integrity: distIntegrity.integrity,

sql/1.15.0.sql

@@ -5,8 +5,8 @@ CREATE TABLE IF NOT EXISTS `package_version_files` (
   `package_version_id` varchar(24) NOT NULL COMMENT 'package version id',
   `package_version_file_id` varchar(24) NOT NULL COMMENT 'package version file id',
   `dist_id` varchar(24) NOT NULL COMMENT 'file dist id',
-  `directory` varchar(500) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL COMMENT 'directory path, e.g.: /bin',
-  `name` varchar(200) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL COMMENT 'file name, e.g.: index.js',
+  `directory` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'directory path, e.g.: /bin',
+  `name` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'file name, e.g.: index.js',
   `content_type` varchar(200) NOT NULL COMMENT 'file content type, e.g.: application/javascript',
   `mtime` datetime(3) NOT NULL COMMENT 'file modified time',
   PRIMARY KEY (`id`),

test/common/FileUtil.test.ts

@@ -22,6 +22,7 @@ describe('test/common/FileUtil.test.ts', () => {
       assert.equal(mimeLookup('/index.ts'), 'text/plain');
       assert.equal(mimeLookup('/index.d.ts'), 'text/plain');
       assert.equal(mimeLookup('/index.txt'), 'text/plain');
+      assert.equal(mimeLookup('pkg-yarn.lock'), 'text/plain');
     });
   });
 });

test/port/controller/PackageVersionFileController/raw.test.ts

@@ -120,6 +120,76 @@ describe('test/port/controller/PackageVersionFileController/raw.test.ts', () =>
       assert.equal(res.body.error, `[NOT_FOUND] File ${pkg.name}@1.0.0/package2.json not found`);
     });
 
+    it('should ignore not exists file on tar onentry', async () => {
+      const tarball = await TestUtil.readFixturesFile('unpkg.com/ide-metrics-api-grpc-0.0.1-main-gha.8962.tgz');
+      const { integrity } = await calculateIntegrity(tarball);
+      const pkg = await TestUtil.getFullPackage({
+        name: '@cnpm/foo-tag-latest',
+        version: '1.0.0',
+        versionObject: {
+          description: 'foo latest description',
+        },
+        attachment: {
+          data: tarball.toString('base64'),
+          length: tarball.length,
+        },
+        dist: {
+          integrity,
+        },
+        main: './lib/index.js',
+      });
+      let res = await app.httpRequest()
+        .put(`/${pkg.name}`)
+        .set('authorization', publisher.authorization)
+        .set('user-agent', publisher.ua)
+        .send(pkg);
+      assert.equal(res.status, 201);
+      res = await app.httpRequest()
+        .get(`/${pkg.name}/1.0.0/files/`);
+      assert.equal(res.status, 200);
+    });
+
+    it('should support non-ascii file name', async () => {
+      // https://unpkg.com/browse/@ppwcode/openapi@7.3.3/resource/ToOneFrom%CF%87.js
+      const tarball = await TestUtil.readFixturesFile('unpkg.com/openapi-7.3.3.tgz');
+      const { integrity } = await calculateIntegrity(tarball);
+      const pkg = await TestUtil.getFullPackage({
+        name: '@cnpm/foo-tag-latest',
+        version: '1.0.0',
+        versionObject: {
+          description: 'foo latest description',
+        },
+        attachment: {
+          data: tarball.toString('base64'),
+          length: tarball.length,
+        },
+        dist: {
+          integrity,
+        },
+        main: './lib/index.js',
+      });
+      let res = await app.httpRequest()
+        .put(`/${pkg.name}`)
+        .set('authorization', publisher.authorization)
+        .set('user-agent', publisher.ua)
+        .send(pkg);
+      assert.equal(res.status, 201);
+      await setTimeout(1000);
+      res = await app.httpRequest()
+        .get(`/${pkg.name}/1.0.0/files/resource/`);
+      assert.equal(res.status, 200);
+      // console.log(res.body);
+      assert(res.body.files.find(file => file.path === '/resource/ToOneFromχ.js'));
+      // res = await app.httpRequest()
+      // .get(`/${pkg.name}/1.0.0/files/resource/ToOneFromχ.js`);
+      res = await app.httpRequest()
+        .get(`/${pkg.name}/1.0.0/files/resource/ToOneFrom%CF%87.js`);
+      assert.equal(res.status, 200);
+      assert.equal(res.headers['content-type'], 'application/javascript; charset=utf-8');
+      // console.log(res.text);
+      assert.match(res.text, /ToOneFromχ/);
+    });
+
     it('should handle big tgz file', async () => {
       const tarball = await TestUtil.readFixturesFile('unpkg.com/pouchdb-3.2.1.tgz');
       const { integrity } = await calculateIntegrity(tarball);
@@ -144,8 +214,7 @@ describe('test/port/controller/PackageVersionFileController/raw.test.ts', () =>
         .set('user-agent', publisher.ua)
         .send(pkg);
       assert.equal(res.status, 201);
-      // wait for sync event finish
-      await setTimeout(3000);
+      await setTimeout(5000);
       res = await app.httpRequest()
         .get(`/${pkg.name}/1.0.0/files/`);
       assert.equal(res.status, 200);