Skip to content

Commit

Permalink
fix: adapter new html format on nodejs.org/dist (#728)
Browse files Browse the repository at this point in the history 
<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **New Features**
- Enhanced file fetching capabilities with updated logic to handle a
broader range of file formats and attributes.
- Introduced a new index HTML page for Node.js distribution version
18.15.0, listing downloadable files with metadata.

- **Bug Fixes**
- Improved regex for matching HTML anchor tags to accurately capture
additional file types and structures.

- **Tests**
- Added new test cases for the `fetch()` method to verify functionality
against the Node.js distribution version 18.15.0.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
  • Loading branch information
@fengmk2
fengmk2 authored Nov 14, 2024
1 parent ac4709a commit 914b59c
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 5 deletions.
24 changes: 19 additions & 5 deletions app/common/adapter/binary/NodeBinary.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { basename } from 'node:path';
import { SingletonProto } from '@eggjs/tegg';
import { BinaryType } from '../../enum/Binary';
import binaries, { BinaryName } from '../../../../config/binaries';
Expand All @@ -21,26 +22,39 @@ export class NodeBinary extends AbstractBinary {
// <a href="index.tab">index.tab</a> 17-Dec-2021 23:16 136319
// <a href="node-0.0.1.tar.gz">node-0.0.1.tar.gz</a> 26-Aug-2011 16:22 2846972
// <a href="node-v14.0.0-nightly20200119b318926634-linux-armv7l.tar.xz">node-v14.0.0-nightly20200119b318926634-linux-ar..&gt;</a> 19-Jan-2020 06:07 18565976
const re = /<a href="([^\"]+?)"[^>]*?>[^<]+?<\/a>\s+?([\w\-]+? \w{2}\:\d{2})\s+?(\d+|\-)/ig;

// new html format
// <a href="docs/">docs/</a> - -
// <a href="win-x64/">win-x64/</a> - -
// <a href="win-x86/">win-x86/</a> - -
// <a href="/dist/v18.15.0/SHASUMS256.txt.asc">SHASUMS256.txt.asc</a> 04-Nov-2024 17:29 3.7 KB
// <a href="/dist/v18.15.0/SHASUMS256.txt.sig">SHASUMS256.txt.sig</a> 04-Nov-2024 17:29 310 B
// <a href="/dist/v18.15.0/SHASUMS256.txt">SHASUMS256.txt</a> 04-Nov-2024 17:29 3.2 KB
const re = /<a href="([^\"]+?)"[^>]*?>[^<]+?<\/a>\s+?((?:[\w\-]+? \w{2}\:\d{2})|\-)\s+?([\d\.\-\s\w]+)/ig;
const matchs = html.matchAll(re);
const items: BinaryItem[] = [];
for (const m of matchs) {
const name = m[1];
let name = m[1];
const isDir = name.endsWith('/');
if (!isDir) {
// /dist/v18.15.0/SHASUMS256.txt => SHASUMS256.txt
name = basename(name);
}
const fileUrl = isDir ? '' : `${url}${name}`;
const date = m[2];
const size = m[3];
const size = m[3].trim();
if (size === '0') continue;
if (binaryConfig.ignoreFiles?.includes(`${dir}${name}`)) continue;

items.push({
const item = {
name,
isDir,
url: fileUrl,
size,
date,
ignoreDownloadStatuses: binaryConfig.options?.ignoreDownloadStatuses,
});
};
items.push(item);
}
return { items, nextParams: null };
}
Expand Down
39 changes: 39 additions & 0 deletions test/common/adapter/binary/NodeBinary.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,45 @@ describe('test/common/adapter/binary/NodeBinary.test.ts', () => {
assert(matchFile);
});

it('should fetch subdir: /v18.15.0/ work', async () => {
app.mockHttpclient('https://nodejs.org/dist/v18.15.0/', 'GET', {
data: await TestUtil.readFixturesFile('nodejs.org/site/v18.15.0/index.html'),
});
const result = await binary.fetch('/v18.15.0/', 'node');
assert(result);
assert(result.items.length > 0);
let matchDir = false;
let matchFile = false;
for (const item of result.items) {
if (item.name === 'docs/') {
assert.equal(item.date, '-');
assert.equal(item.isDir, true);
assert.equal(item.size, '-');
matchDir = true;
}
if (item.name === 'SHASUMS256.txt') {
assert.equal(item.date, '04-Nov-2024 17:29');
assert.equal(item.isDir, false);
assert.equal(item.size, '3.2 KB');
assert.equal(item.url, 'https://nodejs.org/dist/v18.15.0/SHASUMS256.txt');
matchFile = true;
}
if (item.name === 'node-v18.15.0-win-x64.zip') {
assert.equal(item.date, '30-Oct-2024 18:04');
assert.equal(item.isDir, false);
assert.equal(item.size, '29 MB');
assert.equal(item.url, 'https://nodejs.org/dist/v18.15.0/node-v18.15.0-win-x64.zip');
matchFile = true;
}
if (!item.isDir) {
assert(typeof item.size === 'string');
assert(item.size.length > 2);
}
}
assert(matchDir);
assert(matchFile);
});

it('should fetch subdir: /v14.0.0-nightly20200119b318926634/ work', async () => {
app.mockHttpclient('https://nodejs.org/download/nightly/v14.0.0-nightly20200119b318926634/', 'GET', {
data: await TestUtil.readFixturesFile('nodejs.org/download/nightly/v14.0.0-nightly20200119b318926634/index.html'),
Expand Down
55 changes: 55 additions & 0 deletions test/fixtures/nodejs.org/site/v18.15.0/index.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@

<!DOCTYPE html><html>
<head>
<title>Index of /dist/v18.15.0/</title>
<style>
@media (prefers-color-scheme: dark) {
body {
color: white;
background-color: #1c1b22;
}
a {
color: #3391ff;
}
a:visited {
color: #C63B65;
}
}
</style>
</head>
<body>
<h1>Index of /dist/v18.15.0/</h1><hr><pre><a href="../">../</a>
<a href="docs/">docs/</a> - -
<a href="win-x64/">win-x64/</a> - -
<a href="win-x86/">win-x86/</a> - -
<a href="/dist/v18.15.0/SHASUMS256.txt.asc">SHASUMS256.txt.asc</a> 04-Nov-2024 17:29 3.7 KB
<a href="/dist/v18.15.0/SHASUMS256.txt.sig">SHASUMS256.txt.sig</a> 04-Nov-2024 17:29 310 B
<a href="/dist/v18.15.0/SHASUMS256.txt">SHASUMS256.txt</a> 04-Nov-2024 17:29 3.2 KB
<a href="/dist/v18.15.0/node-v18.15.0-aix-ppc64.tar.gz">node-v18.15.0-aix-ppc64.tar.gz</a> 30-Oct-2024 18:04 57 MB
<a href="/dist/v18.15.0/node-v18.15.0-darwin-arm64.tar.gz">node-v18.15.0-darwin-arm64.tar.gz</a> 30-Oct-2024 18:04 40 MB
<a href="/dist/v18.15.0/node-v18.15.0-darwin-arm64.tar.xz">node-v18.15.0-darwin-arm64.tar.xz</a> 30-Oct-2024 18:04 20 MB
<a href="/dist/v18.15.0/node-v18.15.0-darwin-x64.tar.gz">node-v18.15.0-darwin-x64.tar.gz</a> 30-Oct-2024 18:04 42 MB
<a href="/dist/v18.15.0/node-v18.15.0-darwin-x64.tar.xz">node-v18.15.0-darwin-x64.tar.xz</a> 30-Oct-2024 18:04 22 MB
<a href="/dist/v18.15.0/node-v18.15.0-headers.tar.gz">node-v18.15.0-headers.tar.gz</a> 30-Oct-2024 18:04 8.6 MB
<a href="/dist/v18.15.0/node-v18.15.0-headers.tar.xz">node-v18.15.0-headers.tar.xz</a> 04-Nov-2024 17:29 478 KB
<a href="/dist/v18.15.0/node-v18.15.0-linux-arm64.tar.gz">node-v18.15.0-linux-arm64.tar.gz</a> 30-Oct-2024 18:04 44 MB
<a href="/dist/v18.15.0/node-v18.15.0-linux-arm64.tar.xz">node-v18.15.0-linux-arm64.tar.xz</a> 30-Oct-2024 18:04 23 MB
<a href="/dist/v18.15.0/node-v18.15.0-linux-armv7l.tar.gz">node-v18.15.0-linux-armv7l.tar.gz</a> 30-Oct-2024 18:04 41 MB
<a href="/dist/v18.15.0/node-v18.15.0-linux-armv7l.tar.xz">node-v18.15.0-linux-armv7l.tar.xz</a> 30-Oct-2024 18:04 21 MB
<a href="/dist/v18.15.0/node-v18.15.0-linux-ppc64le.tar.gz">node-v18.15.0-linux-ppc64le.tar.gz</a> 30-Oct-2024 18:04 46 MB
<a href="/dist/v18.15.0/node-v18.15.0-linux-ppc64le.tar.xz">node-v18.15.0-linux-ppc64le.tar.xz</a> 30-Oct-2024 18:04 24 MB
<a href="/dist/v18.15.0/node-v18.15.0-linux-s390x.tar.gz">node-v18.15.0-linux-s390x.tar.gz</a> 30-Oct-2024 18:04 44 MB
<a href="/dist/v18.15.0/node-v18.15.0-linux-s390x.tar.xz">node-v18.15.0-linux-s390x.tar.xz</a> 30-Oct-2024 18:04 22 MB
<a href="/dist/v18.15.0/node-v18.15.0-linux-x64.tar.gz">node-v18.15.0-linux-x64.tar.gz</a> 30-Oct-2024 18:04 44 MB
<a href="/dist/v18.15.0/node-v18.15.0-linux-x64.tar.xz">node-v18.15.0-linux-x64.tar.xz</a> 30-Oct-2024 18:04 24 MB
<a href="/dist/v18.15.0/node-v18.15.0-win-x64.7z">node-v18.15.0-win-x64.7z</a> 30-Oct-2024 18:04 18 MB
<a href="/dist/v18.15.0/node-v18.15.0-win-x64.zip">node-v18.15.0-win-x64.zip</a> 30-Oct-2024 18:04 29 MB
<a href="/dist/v18.15.0/node-v18.15.0-win-x86.7z">node-v18.15.0-win-x86.7z</a> 30-Oct-2024 18:04 17 MB
<a href="/dist/v18.15.0/node-v18.15.0-win-x86.zip">node-v18.15.0-win-x86.zip</a> 30-Oct-2024 18:04 27 MB
<a href="/dist/v18.15.0/node-v18.15.0-x64.msi">node-v18.15.0-x64.msi</a> 30-Oct-2024 18:04 32 MB
<a href="/dist/v18.15.0/node-v18.15.0-x86.msi">node-v18.15.0-x86.msi</a> 30-Oct-2024 18:04 30 MB
<a href="/dist/v18.15.0/node-v18.15.0.pkg">node-v18.15.0.pkg</a> 30-Oct-2024 18:04 70 MB
<a href="/dist/v18.15.0/node-v18.15.0.tar.gz">node-v18.15.0.tar.gz</a> 30-Oct-2024 18:04 85 MB
<a href="/dist/v18.15.0/node-v18.15.0.tar.xz">node-v18.15.0.tar.xz</a> 30-Oct-2024 18:04 40 MB
</pre><hr /></body>
</html>

0 comments on commit 914b59c

Please sign in to comment.