make it go

src/cli.ts

@@ -106,7 +106,7 @@ async function main() {
   const opts: CheckOptions = {
     path: cli.input[0],
     recurse: flags.recurse,
-    concurrency: flags.concurrency
+    concurrency: flags.concurrency,
   };
   if (flags.skip) {
     if (typeof flags.skip === 'string') {

src/index.ts

@@ -2,10 +2,11 @@ import { EventEmitter } from 'events';
 import * as gaxios from 'gaxios';
 import * as http from 'http';
 import enableDestroy = require('server-destroy');
-import PQueue from 'p-queue';
+import PQueue, { Queue, DefaultAddOptions } from 'p-queue';
 
 import { getLinks } from './links';
 import { URL } from 'url';
+import PriorityQueue from 'p-queue/dist/priority-queue';
 
 const finalhandler = require('finalhandler');
 const serveStatic = require('serve-static');
@@ -41,6 +42,7 @@ interface CrawlOptions {
   results: LinkResult[];
   cache: Set<string>;
   checkOptions: CheckOptions;
+  queue: PQueue<PriorityQueue, DefaultAddOptions>;
 }
 
 /**
@@ -64,16 +66,22 @@ export class LinkChecker extends EventEmitter {
     }
 
     const queue = new PQueue({
-      concurrency: 10
+      concurrency: options.concurrency || 100,
     });
 
-    const results = await this.crawl({
-      url: options.path,
-      crawl: true,
-      checkOptions: options,
-      results: [],
-      cache: new Set(),
+    const results = new Array<LinkResult>();
+    queue.add(async () => {
+      await this.crawl({
+        url: options.path,
+        crawl: true,
+        checkOptions: options,
+        results,
+        cache: new Set(),
+        queue,
+      });
     });
+    await queue.onIdle();
+
     const result = {
       links: results,
       passed: results.filter(x => x.state === LinkState.BROKEN).length === 0,
@@ -107,10 +115,10 @@ export class LinkChecker extends EventEmitter {
    * @private
    * @returns A list of crawl results consisting of urls and status codes
    */
-  private async crawl(opts: CrawlOptions): Promise<LinkResult[]> {
+  private async crawl(opts: CrawlOptions): Promise<void> {
     // Check to see if we've already scanned this url
     if (opts.cache.has(opts.url)) {
-      return opts.results;
+      return;
     }
     opts.cache.add(opts.url);
 
@@ -120,11 +128,12 @@ export class LinkChecker extends EventEmitter {
         return new RegExp(linkToSkip).test(opts.url);
       })
       .filter(match => !!match);
+
     if (skips.length > 0) {
       const result: LinkResult = { url: opts.url, state: LinkState.SKIPPED };
       opts.results.push(result);
       this.emit('link', result);
-      return opts.results;
+      return;
     }
 
     // Perform a HEAD or GET request based on the need to crawl
@@ -179,18 +188,19 @@ export class LinkChecker extends EventEmitter {
             crawl = crawl && parsedUrl.host === pathUrl.host;
           } catch {}
         }
-        await this.crawl({
-          url,
-          crawl,
-          cache: opts.cache,
-          results: opts.results,
-          checkOptions: opts.checkOptions,
+
+        opts.queue.add(async () => {
+          await this.crawl({
+            url,
+            crawl,
+            cache: opts.cache,
+            results: opts.results,
+            checkOptions: opts.checkOptions,
+            queue: opts.queue,
+          });
         });
       }
     }
-
-    // Return the aggregate results
-    return opts.results;
   }
 }