fix(perf): links are queued just once (#154)

src/index.ts

@@ -71,13 +71,16 @@ export class LinkChecker extends EventEmitter {
  });
 
  const results = new Array<LinkResult>();
+ const url = new URL(options.path);
+ const initCache: Set<string> = new Set();
+ initCache.add(url.href);
  queue.add(async () => {
  await this.crawl({
  url: new URL(options.path),
  crawl: true,
  checkOptions: options,
  results,
- cache: new Set(),
+ cache: initCache,
  queue,
  });
  });
@@ -116,13 +119,7 @@ export class LinkChecker extends EventEmitter {
  * @private
  * @returns A list of crawl results consisting of urls and status codes
  */
- private async crawl(opts: CrawlOptions): Promise<void> {
- // Check to see if we've already scanned this url
- if (opts.cache.has(opts.url.href)) {
- return;
- }
- opts.cache.add(opts.url.href);
-
+ async crawl(opts: CrawlOptions): Promise<void> {
  // explicitly skip non-http[s] links before making the request
  const proto = opts.url.protocol;
  if (proto !== 'http:' && proto !== 'https:') {
@@ -248,6 +245,7 @@ export class LinkChecker extends EventEmitter {
  // Ensure the url hasn't already been touched, largely to avoid a
  // very large queue length and runaway memory consumption
  if (!opts.cache.has(result.url.href)) {
+ opts.cache.add(result.url.href);
  opts.queue.add(async () => {
  await this.crawl({
  url: result.url!,

test/test.ts

@@ -4,7 +4,7 @@ import * as nock from 'nock';
 import * as sinon from 'sinon';
 import * as path from 'path';
 
-import { check, LinkState } from '../src';
+import { check, LinkState, LinkChecker } from '../src';
 
 nock.disableNetConnect();
 nock.enableNetConnect('localhost');
@@ -33,6 +33,19 @@ describe('linkinator', () => {
  scope.done();
  });
 
+ it('should only queue a link once', async () => {
+ const scope = nock('http://fake.local')
+ .head('/')
+ .reply(200);
+ const checker = new LinkChecker();
+ const checkerSpy = sinon.spy(checker, 'crawl');
+ const results = await checker.check({ path: 'test/fixtures/twice' });
+ assert.ok(results.passed);
+ assert.strictEqual(results.links.length, 2);
+ assert.strictEqual(checkerSpy.callCount, 2);
+ scope.done();
+ });
+
  it('should skip links if asked nicely', async () => {
  const results = await check({
  path: 'test/fixtures/skip',