feat: enable concurrent requests

README.md

@@ -38,6 +38,9 @@ $ linkinator LOCATION [ --arguments ]
     --config
         Path to the config file to use. Looks for `linkinator.config.json` by default.
 
+    --concurrency
+          The number of connections to make simultaneously. Defaults to 100.
+
     --recurse, -r
         Recurively follow links on the same root domain.
 
@@ -105,6 +108,7 @@ All options are optional. It should look like this:
   "format": "json",
   "recurse": true,
   "silent": true,
+  "concurrency": 100,
   "skip": "www.googleapis.com"
 }
 ```
@@ -120,6 +124,7 @@ $ linkinator --config /some/path/your-config.json
 #### linkinator.check(options)
 Asynchronous method that runs a site wide scan. Options come in the form of an object that includes:
 - `path` (string) - A fully qualified path to the url to be scanned, or the path to the directory on disk that contains files to be scanned. *required*.
+- `concurrency` (number) -  The number of connections to make simultaneously. Defaults to 100.
 - `port` (number) - When the `path` is provided as a local path on disk, the `port` on which to start the temporary web server.  Defaults to a random high range order port.
 - `recurse` (boolean) - By default, all scans are shallow.  Only the top level links on the requested page will be scanned.  By setting `recurse` to `true`, the crawler will follow all links on the page, and continue scanning links **on the same domain** for as long as it can go. Results are cached, so no worries about loops.
 - `linksToSkip` (array) - An array of regular expression strings that should be skipped during the scan.

package.json

@@ -25,6 +25,7 @@
     "gaxios": "^2.0.1",
     "jsonexport": "^2.4.1",
     "meow": "^5.0.0",
+    "p-queue": "^6.2.1",
     "serve-static": "^1.14.1",
     "server-destroy": "^1.0.1",
     "update-notifier": "^3.0.1"

src/cli.ts

@@ -25,6 +25,9 @@ const cli = meow(
       --config
           Path to the config file to use. Looks for \`linkinator.config.json\` by default.
 
+      --concurrency
+          The number of connections to make simultaneously. Defaults to 100.
+
       --recurse, -r
           Recurively follow links on the same root domain.
 
@@ -50,6 +53,7 @@ const cli = meow(
   {
     flags: {
       config: { type: 'string' },
+      concurrency: { type: 'string' },
       recurse: { type: 'boolean', alias: 'r', default: undefined },
       skip: { type: 'string', alias: 's' },
       format: { type: 'string', alias: 'f' },
@@ -73,11 +77,11 @@ async function main() {
     log(`🏊‍♂️ crawling ${cli.input}`);
   }
   const checker = new LinkChecker();
-  checker.on('pagestart', url => {
-    if (!flags.silent) {
-      log(`\n Scanning ${chalk.grey(url)}`);
-    }
-  });
+  // checker.on('pagestart', url => {
+  //   if (!flags.silent) {
+  //     log(`\n Scanning ${chalk.grey(url)}`);
+  //   }
+  // });
   checker.on('link', (link: LinkResult) => {
     if (flags.silent && link.state !== LinkState.BROKEN) {
       return;
@@ -97,9 +101,13 @@ async function main() {
       default:
         throw new Error('Invalid state.');
     }
-    log(`  ${state} ${chalk.gray(link.url)}`);
+    log(`${state} ${chalk.gray(link.url)}`);
   });
-  const opts: CheckOptions = { path: cli.input[0], recurse: flags.recurse };
+  const opts: CheckOptions = {
+    path: cli.input[0],
+    recurse: flags.recurse,
+    concurrency: Number(flags.concurrency),
+  };
   if (flags.skip) {
     if (typeof flags.skip === 'string') {
       opts.linksToSkip = flags.skip.split(' ').filter(x => !!x);
@@ -118,6 +126,39 @@ async function main() {
     const csv = await toCSV(result.links);
     console.log(csv);
     return;
+  } else {
+    const parents = result.links.reduce((acc, curr) => {
+      const parent = curr.parent || '';
+      if (!acc[parent]) {
+        acc[parent] = [];
+      }
+      acc[parent].push(curr);
+      return acc;
+    }, {} as { [index: string]: LinkResult[] });
+    Object.keys(parents).forEach(parent => {
+      const links = parents[parent];
+      log(chalk.blue(parent));
+      links.forEach(link => {
+        if (flags.silent && link.state !== LinkState.BROKEN) {
+          return;
+        }
+        let state = '';
+        switch (link.state) {
+          case LinkState.BROKEN:
+            state = `[${chalk.red(link.status!.toString())}]`;
+            break;
+          case LinkState.OK:
+            state = `[${chalk.green(link.status!.toString())}]`;
+            break;
+          case LinkState.SKIPPED:
+            state = `[${chalk.grey('SKP')}]`;
+            break;
+          default:
+            throw new Error('Invalid state.');
+        }
+        log(`  ${state} ${chalk.gray(link.url)}`);
+      });
+    });
   }
 
   const total = (Date.now() - start) / 1000;

src/config.ts

@@ -4,6 +4,7 @@ import { promisify } from 'util';
 const readFile = promisify(fs.readFile);
 
 export interface Flags {
+  concurrency?: number;
   config?: string;
   recurse?: boolean;
   skip?: string;

src/index.ts

@@ -2,14 +2,17 @@ import { EventEmitter } from 'events';
 import * as gaxios from 'gaxios';
 import * as http from 'http';
 import enableDestroy = require('server-destroy');
+import PQueue, { DefaultAddOptions } from 'p-queue';
 
 import { getLinks } from './links';
 import { URL } from 'url';
+import PriorityQueue from 'p-queue/dist/priority-queue';
 
 const finalhandler = require('finalhandler');
 const serveStatic = require('serve-static');
 
 export interface CheckOptions {
+  concurrency?: number;
   port?: number;
   path: string;
   recurse?: boolean;
@@ -26,6 +29,7 @@ export interface LinkResult {
   url: string;
   status?: number;
   state: LinkState;
+  parent?: string;
 }
 
 export interface CrawlResult {
@@ -35,10 +39,12 @@ export interface CrawlResult {
 
 interface CrawlOptions {
   url: string;
+  parent?: string;
   crawl: boolean;
   results: LinkResult[];
   cache: Set<string>;
   checkOptions: CheckOptions;
+  queue: PQueue<PriorityQueue, DefaultAddOptions>;
 }
 
 /**
@@ -60,13 +66,24 @@ export class LinkChecker extends EventEmitter {
       enableDestroy(server);
       options.path = `http://localhost:${port}`;
     }
-    const results = await this.crawl({
-      url: options.path,
-      crawl: true,
-      checkOptions: options,
-      results: [],
-      cache: new Set(),
+
+    const queue = new PQueue({
+      concurrency: options.concurrency || 100,
     });
+
+    const results = new Array<LinkResult>();
+    queue.add(async () => {
+      await this.crawl({
+        url: options.path,
+        crawl: true,
+        checkOptions: options,
+        results,
+        cache: new Set(),
+        queue,
+      });
+    });
+    await queue.onIdle();
+
     const result = {
       links: results,
       passed: results.filter(x => x.state === LinkState.BROKEN).length === 0,
@@ -100,10 +117,10 @@ export class LinkChecker extends EventEmitter {
    * @private
    * @returns A list of crawl results consisting of urls and status codes
    */
-  private async crawl(opts: CrawlOptions): Promise<LinkResult[]> {
+  private async crawl(opts: CrawlOptions): Promise<void> {
     // Check to see if we've already scanned this url
     if (opts.cache.has(opts.url)) {
-      return opts.results;
+      return;
     }
     opts.cache.add(opts.url);
 
@@ -113,11 +130,16 @@ export class LinkChecker extends EventEmitter {
         return new RegExp(linkToSkip).test(opts.url);
       })
       .filter(match => !!match);
+
     if (skips.length > 0) {
-      const result: LinkResult = { url: opts.url, state: LinkState.SKIPPED };
+      const result: LinkResult = {
+        url: opts.url,
+        state: LinkState.SKIPPED,
+        parent: opts.parent,
+      };
       opts.results.push(result);
       this.emit('link', result);
-      return opts.results;
+      return;
     }
 
     // Perform a HEAD or GET request based on the need to crawl
@@ -153,7 +175,12 @@ export class LinkChecker extends EventEmitter {
     } catch (err) {
       // request failure: invalid domain name, etc.
     }
-    const result: LinkResult = { url: opts.url, status, state };
+    const result: LinkResult = {
+      url: opts.url,
+      status,
+      state,
+      parent: opts.parent,
+    };
     opts.results.push(result);
     this.emit('link', result);
 
@@ -172,18 +199,20 @@ export class LinkChecker extends EventEmitter {
             crawl = crawl && parsedUrl.host === pathUrl.host;
           } catch {}
         }
-        await this.crawl({
-          url,
-          crawl,
-          cache: opts.cache,
-          results: opts.results,
-          checkOptions: opts.checkOptions,
+
+        opts.queue.add(async () => {
+          await this.crawl({
+            url,
+            crawl,
+            cache: opts.cache,
+            results: opts.results,
+            checkOptions: opts.checkOptions,
+            queue: opts.queue,
+            parent: opts.url,
+          });
         });
       }
     }
-
-    // Return the aggregate results
-    return opts.results;
   }
 }
 

test/fixtures/config/linkinator.config.json

@@ -2,5 +2,6 @@
   "format": "json",
   "recurse": true,
   "silent": true,
+  "concurrency": 17,
   "skip": "🌳"
 }

test/test.config.ts

@@ -10,6 +10,7 @@ describe('config', () => {
       recurse: true,
       silent: true,
       skip: '🌳',
+      concurrency: 22,
     };
     const config = await getConfig(cfg);
     assert.deepStrictEqual(config, cfg);