From 914b59c7efde9dac5aca5fc6b8ceef0f25a73040 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Thu, 14 Nov 2024 23:13:03 +0800 Subject: [PATCH] fix: adapter new html format on nodejs.org/dist (#728) ## Summary by CodeRabbit - **New Features** - Enhanced file fetching capabilities with updated logic to handle a broader range of file formats and attributes. - Introduced a new index HTML page for Node.js distribution version 18.15.0, listing downloadable files with metadata. - **Bug Fixes** - Improved regex for matching HTML anchor tags to accurately capture additional file types and structures. - **Tests** - Added new test cases for the `fetch()` method to verify functionality against the Node.js distribution version 18.15.0. --- app/common/adapter/binary/NodeBinary.ts | 24 ++++++-- test/common/adapter/binary/NodeBinary.test.ts | 39 +++++++++++++ .../nodejs.org/site/v18.15.0/index.html | 55 +++++++++++++++++++ 3 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 test/fixtures/nodejs.org/site/v18.15.0/index.html diff --git a/app/common/adapter/binary/NodeBinary.ts b/app/common/adapter/binary/NodeBinary.ts index be3f9eb3..73134bd0 100644 --- a/app/common/adapter/binary/NodeBinary.ts +++ b/app/common/adapter/binary/NodeBinary.ts @@ -1,3 +1,4 @@ +import { basename } from 'node:path'; import { SingletonProto } from '@eggjs/tegg'; import { BinaryType } from '../../enum/Binary'; import binaries, { BinaryName } from '../../../../config/binaries'; @@ -21,26 +22,39 @@ export class NodeBinary extends AbstractBinary { // index.tab 17-Dec-2021 23:16 136319 // node-0.0.1.tar.gz 26-Aug-2011 16:22 2846972 // node-v14.0.0-nightly20200119b318926634-linux-ar..> 19-Jan-2020 06:07 18565976 - const re = /]*?>[^<]+?<\/a>\s+?([\w\-]+? \w{2}\:\d{2})\s+?(\d+|\-)/ig; + + // new html format + // docs/ - - + // win-x64/ - - + // win-x86/ - - + // SHASUMS256.txt.asc 04-Nov-2024 17:29 3.7 KB + // SHASUMS256.txt.sig 04-Nov-2024 17:29 310 B + // SHASUMS256.txt 04-Nov-2024 17:29 3.2 KB + const re = /]*?>[^<]+?<\/a>\s+?((?:[\w\-]+? \w{2}\:\d{2})|\-)\s+?([\d\.\-\s\w]+)/ig; const matchs = html.matchAll(re); const items: BinaryItem[] = []; for (const m of matchs) { - const name = m[1]; + let name = m[1]; const isDir = name.endsWith('/'); + if (!isDir) { + // /dist/v18.15.0/SHASUMS256.txt => SHASUMS256.txt + name = basename(name); + } const fileUrl = isDir ? '' : `${url}${name}`; const date = m[2]; - const size = m[3]; + const size = m[3].trim(); if (size === '0') continue; if (binaryConfig.ignoreFiles?.includes(`${dir}${name}`)) continue; - items.push({ + const item = { name, isDir, url: fileUrl, size, date, ignoreDownloadStatuses: binaryConfig.options?.ignoreDownloadStatuses, - }); + }; + items.push(item); } return { items, nextParams: null }; } diff --git a/test/common/adapter/binary/NodeBinary.test.ts b/test/common/adapter/binary/NodeBinary.test.ts index 8b8f0088..27506a71 100644 --- a/test/common/adapter/binary/NodeBinary.test.ts +++ b/test/common/adapter/binary/NodeBinary.test.ts @@ -73,6 +73,45 @@ describe('test/common/adapter/binary/NodeBinary.test.ts', () => { assert(matchFile); }); + it('should fetch subdir: /v18.15.0/ work', async () => { + app.mockHttpclient('https://nodejs.org/dist/v18.15.0/', 'GET', { + data: await TestUtil.readFixturesFile('nodejs.org/site/v18.15.0/index.html'), + }); + const result = await binary.fetch('/v18.15.0/', 'node'); + assert(result); + assert(result.items.length > 0); + let matchDir = false; + let matchFile = false; + for (const item of result.items) { + if (item.name === 'docs/') { + assert.equal(item.date, '-'); + assert.equal(item.isDir, true); + assert.equal(item.size, '-'); + matchDir = true; + } + if (item.name === 'SHASUMS256.txt') { + assert.equal(item.date, '04-Nov-2024 17:29'); + assert.equal(item.isDir, false); + assert.equal(item.size, '3.2 KB'); + assert.equal(item.url, 'https://nodejs.org/dist/v18.15.0/SHASUMS256.txt'); + matchFile = true; + } + if (item.name === 'node-v18.15.0-win-x64.zip') { + assert.equal(item.date, '30-Oct-2024 18:04'); + assert.equal(item.isDir, false); + assert.equal(item.size, '29 MB'); + assert.equal(item.url, 'https://nodejs.org/dist/v18.15.0/node-v18.15.0-win-x64.zip'); + matchFile = true; + } + if (!item.isDir) { + assert(typeof item.size === 'string'); + assert(item.size.length > 2); + } + } + assert(matchDir); + assert(matchFile); + }); + it('should fetch subdir: /v14.0.0-nightly20200119b318926634/ work', async () => { app.mockHttpclient('https://nodejs.org/download/nightly/v14.0.0-nightly20200119b318926634/', 'GET', { data: await TestUtil.readFixturesFile('nodejs.org/download/nightly/v14.0.0-nightly20200119b318926634/index.html'), diff --git a/test/fixtures/nodejs.org/site/v18.15.0/index.html b/test/fixtures/nodejs.org/site/v18.15.0/index.html new file mode 100644 index 00000000..d4728768 --- /dev/null +++ b/test/fixtures/nodejs.org/site/v18.15.0/index.html @@ -0,0 +1,55 @@ + + + + Index of /dist/v18.15.0/ + + + +

Index of /dist/v18.15.0/

../
+docs/                                                             -                   -
+win-x64/                                                          -                   -
+win-x86/                                                          -                   -
+SHASUMS256.txt.asc                                 04-Nov-2024 17:29               3.7 KB
+SHASUMS256.txt.sig                                 04-Nov-2024 17:29                310 B
+SHASUMS256.txt                                     04-Nov-2024 17:29               3.2 KB
+node-v18.15.0-aix-ppc64.tar.gz                     30-Oct-2024 18:04                57 MB
+node-v18.15.0-darwin-arm64.tar.gz                  30-Oct-2024 18:04                40 MB
+node-v18.15.0-darwin-arm64.tar.xz                  30-Oct-2024 18:04                20 MB
+node-v18.15.0-darwin-x64.tar.gz                    30-Oct-2024 18:04                42 MB
+node-v18.15.0-darwin-x64.tar.xz                    30-Oct-2024 18:04                22 MB
+node-v18.15.0-headers.tar.gz                       30-Oct-2024 18:04               8.6 MB
+node-v18.15.0-headers.tar.xz                       04-Nov-2024 17:29               478 KB
+node-v18.15.0-linux-arm64.tar.gz                   30-Oct-2024 18:04                44 MB
+node-v18.15.0-linux-arm64.tar.xz                   30-Oct-2024 18:04                23 MB
+node-v18.15.0-linux-armv7l.tar.gz                  30-Oct-2024 18:04                41 MB
+node-v18.15.0-linux-armv7l.tar.xz                  30-Oct-2024 18:04                21 MB
+node-v18.15.0-linux-ppc64le.tar.gz                 30-Oct-2024 18:04                46 MB
+node-v18.15.0-linux-ppc64le.tar.xz                 30-Oct-2024 18:04                24 MB
+node-v18.15.0-linux-s390x.tar.gz                   30-Oct-2024 18:04                44 MB
+node-v18.15.0-linux-s390x.tar.xz                   30-Oct-2024 18:04                22 MB
+node-v18.15.0-linux-x64.tar.gz                     30-Oct-2024 18:04                44 MB
+node-v18.15.0-linux-x64.tar.xz                     30-Oct-2024 18:04                24 MB
+node-v18.15.0-win-x64.7z                           30-Oct-2024 18:04                18 MB
+node-v18.15.0-win-x64.zip                          30-Oct-2024 18:04                29 MB
+node-v18.15.0-win-x86.7z                           30-Oct-2024 18:04                17 MB
+node-v18.15.0-win-x86.zip                          30-Oct-2024 18:04                27 MB
+node-v18.15.0-x64.msi                              30-Oct-2024 18:04                32 MB
+node-v18.15.0-x86.msi                              30-Oct-2024 18:04                30 MB
+node-v18.15.0.pkg                                  30-Oct-2024 18:04                70 MB
+node-v18.15.0.tar.gz                               30-Oct-2024 18:04                85 MB
+node-v18.15.0.tar.xz                               30-Oct-2024 18:04                40 MB
+
+