Reverted: wait-until option; prettified

Author: Serhii Larchenko

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 yarn-error.log
+.vscode

src/puppeteer_utils.js

@@ -6,20 +6,20 @@ const path = require("path");
 const fs = require("fs");
 const { createTracker, augmentTimeoutError } = require("./tracker");
 
-const errorToString = jsHandle =>
-  jsHandle.executionContext().evaluate(e => e.toString(), jsHandle);
+const errorToString = (jsHandle) =>
+  jsHandle.executionContext().evaluate((e) => e.toString(), jsHandle);
 
-const objectToJson = jsHandle => jsHandle.jsonValue();
+const objectToJson = (jsHandle) => jsHandle.jsonValue();
 
 /**
  * @param {{page: Page, options: {skipThirdPartyRequests: true}, basePath: string }} opt
  * @return {Promise<void>}
  */
-const skipThirdPartyRequests = async opt => {
+const skipThirdPartyRequests = async (opt) => {
   const { page, options, basePath } = opt;
   if (!options.skipThirdPartyRequests) return;
   await page.setRequestInterception(true);
-  page.on("request", request => {
+  page.on("request", (request) => {
     if (request.url().startsWith(basePath)) {
       request.continue();
     } else {
@@ -32,37 +32,37 @@ const skipThirdPartyRequests = async opt => {
  * @param {{page: Page, options: {sourceMaps: boolean}, route: string, onError: ?function }} opt
  * @return {void}
  */
-const enableLogging = opt => {
+const enableLogging = (opt) => {
   const { page, options, route, onError, sourcemapStore } = opt;
-  page.on("console", msg => {
+  page.on("console", (msg) => {
     const text = msg.text();
     if (text === "JSHandle@object") {
-      Promise.all(msg.args().map(objectToJson)).then(args =>
+      Promise.all(msg.args().map(objectToJson)).then((args) =>
         console.log(`💬  console.log at ${route}:`, ...args)
       );
     } else if (text === "JSHandle@error") {
-      Promise.all(msg.args().map(errorToString)).then(args =>
+      Promise.all(msg.args().map(errorToString)).then((args) =>
         console.log(`💬  console.log at ${route}:`, ...args)
       );
     } else {
       console.log(`️️️💬  console.log at ${route}:`, text);
     }
   });
-  page.on("error", msg => {
+  page.on("error", (msg) => {
     console.log(`🔥  error at ${route}:`, msg);
     onError && onError();
   });
-  page.on("pageerror", e => {
+  page.on("pageerror", (e) => {
     if (options.sourceMaps) {
       mapStackTrace(e.stack || e.message, {
         isChromeOrEdge: true,
-        store: sourcemapStore || {}
+        store: sourcemapStore || {},
       })
-        .then(result => {
+        .then((result) => {
           // TODO: refactor mapStackTrace: return array not a string, return first row too
           const stackRows = result.split("\n");
           const puppeteerLine =
-            stackRows.findIndex(x => x.includes("puppeteer")) ||
+            stackRows.findIndex((x) => x.includes("puppeteer")) ||
             stackRows.length - 1;
 
           console.log(
@@ -71,7 +71,7 @@ const enableLogging = opt => {
             )[0] + "\n"}${stackRows.slice(0, puppeteerLine).join("\n")}`
           );
         })
-        .catch(e2 => {
+        .catch((e2) => {
           console.log(`🔥  pageerror at ${route}:`, e);
           console.log(
             `️️️⚠️  warning at ${route} (error in source maps):`,
@@ -83,7 +83,7 @@ const enableLogging = opt => {
     }
     onError && onError();
   });
-  page.on("response", response => {
+  page.on("response", (response) => {
     if (response.status() >= 400) {
       let route = "";
       try {
@@ -105,21 +105,23 @@ const enableLogging = opt => {
  * @param {{page: Page}} opt
  * @return {Promise<Array<string>>}
  */
-const getLinks = async opt => {
+const getLinks = async (opt) => {
   const { page } = opt;
   const anchors = await page.evaluate(() =>
-    Array.from(document.querySelectorAll("a,link[rel='alternate']")).map(anchor => {
-      if (anchor.href.baseVal) {
-        const a = document.createElement("a");
-        a.href = anchor.href.baseVal;
-        return a.href;
+    Array.from(document.querySelectorAll("a,link[rel='alternate']")).map(
+      (anchor) => {
+        if (anchor.href.baseVal) {
+          const a = document.createElement("a");
+          a.href = anchor.href.baseVal;
+          return a.href;
+        }
+        return anchor.href;
       }
-      return anchor.href;
-    })
+    )
   );
 
   const iframes = await page.evaluate(() =>
-    Array.from(document.querySelectorAll("iframe")).map(iframe => iframe.src)
+    Array.from(document.querySelectorAll("iframe")).map((iframe) => iframe.src)
   );
   return anchors.concat(iframes);
 };
@@ -130,15 +132,15 @@ const getLinks = async opt => {
  * @param {{options: *, basePath: string, beforeFetch: ?(function({ page: Page, route: string }):Promise), afterFetch: ?(function({ page: Page, browser: Browser, route: string }):Promise), onEnd: ?(function():void)}} opt
  * @return {Promise}
  */
-const crawl = async opt => {
+const crawl = async (opt) => {
   const {
     options,
     basePath,
     beforeFetch,
     afterFetch,
     onEnd,
     publicPath,
-    sourceDir
+    sourceDir,
   } = opt;
   let shuttingDown = false;
   let streamClosed = false;
@@ -155,7 +157,7 @@ const crawl = async opt => {
   };
   process.on("SIGINT", onSigint);
 
-  const onUnhandledRejection = error => {
+  const onUnhandledRejection = (error) => {
     console.log("🔥  UnhandledPromiseRejectionWarning", error);
     shuttingDown = true;
   };
@@ -172,7 +174,7 @@ const crawl = async opt => {
    * @param {string} path
    * @returns {void}
    */
-  const addToQueue = newUrl => {
+  const addToQueue = (newUrl) => {
     const { hostname, search, hash, port } = url.parse(newUrl);
     newUrl = newUrl.replace(`${search || ""}${hash || ""}`, "");
 
@@ -184,7 +186,12 @@ const crawl = async opt => {
     // Port can be null, therefore we need the null check
     const isOnAppPort = port && port.toString() === options.port.toString();
 
-    if (hostname === "localhost" && isOnAppPort && !uniqueUrls.has(newUrl) && !streamClosed) {
+    if (
+      hostname === "localhost" &&
+      isOnAppPort &&
+      !uniqueUrls.has(newUrl) &&
+      !streamClosed
+    ) {
       uniqueUrls.add(newUrl);
       enqued++;
       queue.write(newUrl);
@@ -194,15 +201,18 @@ const crawl = async opt => {
     }
   };
 
-  console.log('Puppeteer is starting with following options: ', options.puppeteer);
+  console.log(
+    "Puppeteer is starting with following options: ",
+    options.puppeteer
+  );
 
   const browser = await puppeteer.launch(options.puppeteer);
 
   /**
    * @param {string} pageUrl
    * @returns {Promise<string>}
    */
-  const fetchPage = async pageUrl => {
+  const fetchPage = async (pageUrl) => {
     const route = pageUrl.replace(basePath, "");
 
     let skipExistingFile = false;
@@ -228,13 +238,16 @@ const crawl = async opt => {
           onError: () => {
             shuttingDown = true;
           },
-          sourcemapStore
+          sourcemapStore,
         });
         beforeFetch && beforeFetch({ page, route });
         await page.setUserAgent(options.userAgent);
         const tracker = createTracker(page);
         try {
-          await page.goto(pageUrl, { timeout: options.puppeteer.timeout, waitUntil: "domcontentloaded" });
+          await page.goto(pageUrl, {
+            timeout: options.puppeteer.timeout,
+            waitUntil: "networkidle0",
+          });
         } catch (e) {
           e.message = augmentTimeoutError(e.message, tracker);
           throw e;
@@ -268,12 +281,12 @@ const crawl = async opt => {
   };
 
   if (options.include) {
-    options.include.map(x => addToQueue(`${basePath}${x}`));
+    options.include.map((x) => addToQueue(`${basePath}${x}`));
   }
 
   return new Promise((resolve, reject) => {
     queue
-      .map(x => _(fetchPage(x)))
+      .map((x) => _(fetchPage(x)))
       .mergeWithLimit(options.concurrency)
       .toArray(async () => {
         process.removeListener("SIGINT", onSigint);