From 4d8b37dbe92987784d0e82b00629f54c4f177d87 Mon Sep 17 00:00:00 2001
From: Sviar <svpetrenko123@gmail.com>
Date: Fri, 25 Aug 2023 14:35:37 +0300
Subject: [PATCH 1/7] fix(browser-pool): provide hints to use the correct
 Docker base images if browser fails to launch on Apify platform

---
 .../src/playwright/playwright-plugin.ts       | 22 +++++++++++++++++--
 .../src/puppeteer/puppeteer-plugin.ts         |  5 ++++-
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.ts
index 2a3b72d0292f..d4506b7cfa6f 100644
--- a/packages/browser-pool/src/playwright/playwright-plugin.ts
+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts
@@ -63,7 +63,10 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
 
         try {
             if (useIncognitoPages) {
-                browser = await this.library.launch(launchOptions);
+                browser = await this.library.launch(launchOptions).catch((error) => {
+                    this._notifyOfFailedLaunch();
+                    throw error;
+                });
 
                 if (anonymizedProxyUrl) {
                     browser.on('disconnected', async () => {
@@ -108,7 +111,10 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
                     }
                 }
 
-                const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions);
+                const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions).catch((error) => {
+                    this._notifyOfFailedLaunch();
+                    throw error;
+                });
 
                 browserContext.once('close', () => {
                     if (userDataDir.includes('apify-playwright-firefox-taac-')) {
@@ -174,6 +180,18 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
         return browser;
     }
 
+    private _notifyOfFailedLaunch() {
+        let debugMessage = `Failed to launch browser. ${this.launchOptions?.executablePath ? 'Check whether the provided executable path is correct.' : ''} `;
+        if (process.env.APIFY_IS_AT_HOME) {
+            debugMessage += 'Make sure you used the correct Actor template for Playwright: '
+                + 'your Dockerfile has to extend `apify/actor-node-playwright-*` (with a correct browser name). Or install ';
+        } else {
+            debugMessage += 'Try installing ';
+        }
+        debugMessage += 'the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers). The original error will be displayed below.';
+        log.error(debugMessage);
+    }
+
     protected _createController(): BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
         return new PlaywrightController(this);
     }
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
index 5e0de0f1cbf5..493569f8c8d7 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
@@ -74,7 +74,10 @@ export class PuppeteerPlugin extends BrowserPlugin<
                 }
             } catch (error) {
                 await close();
-
+                if (process.env.APIFY_IS_AT_HOME) {
+                    log.error('Failed to launch browser. Make sure you used the correct Actor template for Puppeteer: your Dockerfile has to extend `apify/actor-node-puppeteer-chrome`.'
+                        + ' Or install the missing dependencies manually, referring to the Puppeteer\'s troubleshooting guide. The original error will be displayed below.');
+                }
                 throw error;
             }
         }

From 0964001ed49295a404c458d6ff3da5b8b0e3dce7 Mon Sep 17 00:00:00 2001
From: Sviar <svpetrenko123@gmail.com>
Date: Fri, 25 Aug 2023 14:43:03 +0300
Subject: [PATCH 2/7] lint: split the string into multiple lines

---
 packages/browser-pool/src/playwright/playwright-plugin.ts | 3 ++-
 packages/browser-pool/src/puppeteer/puppeteer-plugin.ts   | 6 ++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.ts
index d4506b7cfa6f..af7554e392a0 100644
--- a/packages/browser-pool/src/playwright/playwright-plugin.ts
+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts
@@ -188,7 +188,8 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
         } else {
             debugMessage += 'Try installing ';
         }
-        debugMessage += 'the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers). The original error will be displayed below.';
+        debugMessage += 'the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).'
+            + ' The original error will be displayed below.';
         log.error(debugMessage);
     }
 
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
index 493569f8c8d7..0be16706557c 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
@@ -75,8 +75,10 @@ export class PuppeteerPlugin extends BrowserPlugin<
             } catch (error) {
                 await close();
                 if (process.env.APIFY_IS_AT_HOME) {
-                    log.error('Failed to launch browser. Make sure you used the correct Actor template for Puppeteer: your Dockerfile has to extend `apify/actor-node-puppeteer-chrome`.'
-                        + ' Or install the missing dependencies manually, referring to the Puppeteer\'s troubleshooting guide. The original error will be displayed below.');
+                    log.error('Failed to launch browser. Make sure you used the correct Actor template for Puppeteer:'
+                        + ' your Dockerfile has to extend `apify/actor-node-puppeteer-chrome`.'
+                        + ' Or install the missing dependencies manually, referring to the Puppeteer\'s troubleshooting guide.'
+                        + ' The original error will be displayed below.');
                 }
                 throw error;
             }

From faecfb5c47daac687ca6a95d0f42570755e2b91e Mon Sep 17 00:00:00 2001
From: Sviar <svpetrenko123@gmail.com>
Date: Tue, 29 Aug 2023 16:06:16 +0300
Subject: [PATCH 3/7] fix(browser-pool): change logs to better inform about
 missing browser dependencies

---
 .../src/playwright/playwright-plugin.ts         | 17 +++++++++--------
 .../src/puppeteer/puppeteer-plugin.ts           | 14 ++++++++++----
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.ts
index af7554e392a0..c6c6744d666d 100644
--- a/packages/browser-pool/src/playwright/playwright-plugin.ts
+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts
@@ -64,7 +64,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
         try {
             if (useIncognitoPages) {
                 browser = await this.library.launch(launchOptions).catch((error) => {
-                    this._notifyOfFailedLaunch();
+                    this._notifyOfFailedLaunch(launchContext);
                     throw error;
                 });
 
@@ -112,7 +112,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
                 }
 
                 const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions).catch((error) => {
-                    this._notifyOfFailedLaunch();
+                    this._notifyOfFailedLaunch(launchContext);
                     throw error;
                 });
 
@@ -180,15 +180,16 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
         return browser;
     }
 
-    private _notifyOfFailedLaunch() {
-        let debugMessage = `Failed to launch browser. ${this.launchOptions?.executablePath ? 'Check whether the provided executable path is correct.' : ''} `;
+    private _notifyOfFailedLaunch(launchContext: LaunchContext<BrowserType>) {
+        let debugMessage = `Failed to launch browser.`
+        + `${launchContext.launchOptions?.executablePath
+            ? ` Check whether the provided executable path is correct: ${launchContext.launchOptions?.executablePath}.` : ''}`;
         if (process.env.APIFY_IS_AT_HOME) {
-            debugMessage += 'Make sure you used the correct Actor template for Playwright: '
-                + 'your Dockerfile has to extend `apify/actor-node-playwright-*` (with a correct browser name). Or install ';
+            debugMessage += ' Make sure your Dockerfile extends apify/actor-node-playwright-*` (with a correct browser name). Or install';
         } else {
-            debugMessage += 'Try installing ';
+            debugMessage += ' Try installing';
         }
-        debugMessage += 'the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).'
+        debugMessage += ' the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).'
             + ' The original error will be displayed below.';
         log.error(debugMessage);
     }
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
index 0be16706557c..6cf0d39c9129 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
@@ -74,12 +74,18 @@ export class PuppeteerPlugin extends BrowserPlugin<
                 }
             } catch (error) {
                 await close();
+                let debugMessage = `Failed to launch browser.`
+                    + `${launchContext.launchOptions?.executablePath
+                        ? ` Check whether the provided executable path is correct: ${launchContext.launchOptions?.executablePath}.` : ''}`;
                 if (process.env.APIFY_IS_AT_HOME) {
-                    log.error('Failed to launch browser. Make sure you used the correct Actor template for Puppeteer:'
-                        + ' your Dockerfile has to extend `apify/actor-node-puppeteer-chrome`.'
-                        + ' Or install the missing dependencies manually, referring to the Puppeteer\'s troubleshooting guide.'
-                        + ' The original error will be displayed below.');
+                    debugMessage += ' Make sure your Dockerfile extends `apify/actor-node-puppeteer-chrome. Or install';
+                } else {
+                    debugMessage += ' Try installing';
                 }
+                debugMessage += ' browser, if it\'s missing, by running `npx @puppeteer/browsers install chromium --path [path]`'
+                    + ' and pointing `executablePath` to the downloaded executable (https://playwright.dev/docs/browsers).'
+                    + ' The original error will be displayed below.';
+                log.error(debugMessage);
                 throw error;
             }
         }

From 4a95cb618ad9cf7450192d9949414bcbf738a42b Mon Sep 17 00:00:00 2001
From: Sviar <svpetrenko123@gmail.com>
Date: Wed, 30 Aug 2023 13:31:00 +0300
Subject: [PATCH 4/7] fix(browser-pool): throw `CriticalError` with the
 original error in `cause` instead of just rethrowing

---
 .../src/playwright/playwright-plugin.ts            | 13 ++++++++-----
 .../browser-pool/src/puppeteer/puppeteer-plugin.ts | 14 ++++++++------
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.ts
index c6c6744d666d..3f6b56d17e53 100644
--- a/packages/browser-pool/src/playwright/playwright-plugin.ts
+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts
@@ -3,6 +3,7 @@ import net from 'net';
 import os from 'os';
 import path from 'path';
 
+import { CriticalError } from '@crawlee/core';
 import type { Browser as PlaywrightBrowser, BrowserType } from 'playwright';
 
 import { loadFirefoxAddon } from './load-firefox-addon';
@@ -64,7 +65,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
         try {
             if (useIncognitoPages) {
                 browser = await this.library.launch(launchOptions).catch((error) => {
-                    this._notifyOfFailedLaunch(launchContext);
+                    this._throwOnFailedLaunch(launchContext, error);
                     throw error;
                 });
 
@@ -112,7 +113,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
                 }
 
                 const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions).catch((error) => {
-                    this._notifyOfFailedLaunch(launchContext);
+                    this._throwOnFailedLaunch(launchContext, error);
                     throw error;
                 });
 
@@ -180,7 +181,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
         return browser;
     }
 
-    private _notifyOfFailedLaunch(launchContext: LaunchContext<BrowserType>) {
+    private _throwOnFailedLaunch(launchContext: LaunchContext<BrowserType>, cause: unknown) {
         let debugMessage = `Failed to launch browser.`
         + `${launchContext.launchOptions?.executablePath
             ? ` Check whether the provided executable path is correct: ${launchContext.launchOptions?.executablePath}.` : ''}`;
@@ -190,8 +191,10 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
             debugMessage += ' Try installing';
         }
         debugMessage += ' the required dependencies by running `npx playwright install --with-deps` (https://playwright.dev/docs/browsers).'
-            + ' The original error will be displayed below.';
-        log.error(debugMessage);
+            + ' The original error will be displayed at the bottom as the [cause].';
+        throw new CriticalError(debugMessage, {
+            cause,
+        });
     }
 
     protected _createController(): BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
index 6cf0d39c9129..85baa8b0c569 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
@@ -1,3 +1,4 @@
+import { CriticalError } from '@crawlee/core';
 import type { Dictionary } from '@crawlee/types';
 import type Puppeteer from 'puppeteer';
 import type * as PuppeteerTypes from 'puppeteer';
@@ -72,7 +73,7 @@ export class PuppeteerPlugin extends BrowserPlugin<
                         await close();
                     });
                 }
-            } catch (error) {
+            } catch (error: any) {
                 await close();
                 let debugMessage = `Failed to launch browser.`
                     + `${launchContext.launchOptions?.executablePath
@@ -82,11 +83,12 @@ export class PuppeteerPlugin extends BrowserPlugin<
                 } else {
                     debugMessage += ' Try installing';
                 }
-                debugMessage += ' browser, if it\'s missing, by running `npx @puppeteer/browsers install chromium --path [path]`'
-                    + ' and pointing `executablePath` to the downloaded executable (https://playwright.dev/docs/browsers).'
-                    + ' The original error will be displayed below.';
-                log.error(debugMessage);
-                throw error;
+                debugMessage += ` a browser, if it's missing, by running \`npx @puppeteer/browsers install chromium --path [path]\``
+                    + ` and pointing \`executablePath\` to the downloaded executable (https://playwright.dev/docs/browsers).`
+                    + ` The original error will be displayed at the bottom as the [cause].`;
+                throw new CriticalError(debugMessage, {
+                    cause: error,
+                });
             }
         }
 

From 53095461dbaf92f4cc76e4b3a38a657bd6590a7b Mon Sep 17 00:00:00 2001
From: Sviar <svpetrenko123@gmail.com>
Date: Wed, 30 Aug 2023 15:30:09 +0300
Subject: [PATCH 5/7] fix(core, basic-crawler): avoid logging critical errors
 as they are logged by Node.js

---
 packages/basic-crawler/src/internals/basic-crawler.ts | 4 +++-
 packages/browser-pool/package.json                    | 1 +
 packages/core/src/autoscaling/autoscaled_pool.ts      | 8 +++++++-
 packages/core/src/errors.ts                           | 1 +
 yarn.lock                                             | 1 +
 5 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 80a87b20d06c..e576eb206a1e 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1123,7 +1123,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                 );
                 request.state = RequestState.DONE;
             } catch (secondaryError: any) {
-                if (!secondaryError.triggeredFromUserHandler) {
+                if (!secondaryError.triggeredFromUserHandler
+                    // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
+                    && !(secondaryError instanceof CriticalError)) {
                     const apifySpecific = process.env.APIFY_IS_AT_HOME
                         ? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.` : '';
                     this.log.exception(secondaryError as Error, 'An exception occurred during handling of failed request. '
diff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.json
index a05a8676604f..5b2720bfe72f 100644
--- a/packages/browser-pool/package.json
+++ b/packages/browser-pool/package.json
@@ -38,6 +38,7 @@
     "dependencies": {
         "@apify/log": "^2.4.0",
         "@apify/timeout": "^0.3.0",
+        "@crawlee/core": "3.5.2",
         "@crawlee/types": "3.5.2",
         "fingerprint-generator": "^2.0.6",
         "fingerprint-injector": "^2.0.5",
diff --git a/packages/core/src/autoscaling/autoscaled_pool.ts b/packages/core/src/autoscaling/autoscaled_pool.ts
index 8e1c601f6f02..376fdc992c86 100644
--- a/packages/core/src/autoscaling/autoscaled_pool.ts
+++ b/packages/core/src/autoscaling/autoscaled_pool.ts
@@ -9,6 +9,7 @@ import { Snapshotter } from './snapshotter';
 import type { SystemInfo, SystemStatusOptions } from './system_status';
 import { SystemStatus } from './system_status';
 import { Configuration } from '../configuration';
+import { CriticalError } from '../errors';
 import { log as defaultLog } from '../log';
 
 export interface AutoscaledPoolOptions {
@@ -551,7 +552,12 @@ export class AutoscaledPool {
             // We might have already rejected this promise.
             if (this.reject) {
                 // No need to log all concurrent errors.
-                this.log.exception(err, 'runTaskFunction failed.');
+                if (
+                    // avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
+                    !(e instanceof CriticalError)
+                ) {
+                    this.log.exception(err, 'runTaskFunction failed.');
+                }
                 this.reject(err);
             }
         }
diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.ts
index f8a0e1499f4c..3e55610caf62 100644
--- a/packages/core/src/errors.ts
+++ b/packages/core/src/errors.ts
@@ -5,6 +5,7 @@ export class NonRetryableError extends Error {}
 
 /**
  * Errors of `CriticalError` type will shut down the whole crawler.
+ * Error handlers catching CriticalError should avoid logging it, as it will be logged by Node.js itself at the end
  */
 export class CriticalError extends NonRetryableError {}
 
diff --git a/yarn.lock b/yarn.lock
index 6d03872ddd2d..3817b66196be 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -761,6 +761,7 @@ __metadata:
   dependencies:
     "@apify/log": ^2.4.0
     "@apify/timeout": ^0.3.0
+    "@crawlee/core": 3.5.2
     "@crawlee/types": 3.5.2
     fingerprint-generator: ^2.0.6
     fingerprint-injector: ^2.0.5

From a71586c84f8ceae085e620fc48158e71e9741e38 Mon Sep 17 00:00:00 2001
From: Sviar <svpetrenko123@gmail.com>
Date: Wed, 30 Aug 2023 15:34:37 +0300
Subject: [PATCH 6/7] chore: remove unreachable throw statement

---
 packages/browser-pool/src/playwright/playwright-plugin.ts | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/packages/browser-pool/src/playwright/playwright-plugin.ts b/packages/browser-pool/src/playwright/playwright-plugin.ts
index 3f6b56d17e53..fa1abf4e074b 100644
--- a/packages/browser-pool/src/playwright/playwright-plugin.ts
+++ b/packages/browser-pool/src/playwright/playwright-plugin.ts
@@ -65,8 +65,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
         try {
             if (useIncognitoPages) {
                 browser = await this.library.launch(launchOptions).catch((error) => {
-                    this._throwOnFailedLaunch(launchContext, error);
-                    throw error;
+                    return this._throwOnFailedLaunch(launchContext, error);
                 });
 
                 if (anonymizedProxyUrl) {
@@ -113,8 +112,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
                 }
 
                 const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions).catch((error) => {
-                    this._throwOnFailedLaunch(launchContext, error);
-                    throw error;
+                    return this._throwOnFailedLaunch(launchContext, error);
                 });
 
                 browserContext.once('close', () => {
@@ -181,7 +179,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
         return browser;
     }
 
-    private _throwOnFailedLaunch(launchContext: LaunchContext<BrowserType>, cause: unknown) {
+    private _throwOnFailedLaunch(launchContext: LaunchContext<BrowserType>, cause: unknown): never {
         let debugMessage = `Failed to launch browser.`
         + `${launchContext.launchOptions?.executablePath
             ? ` Check whether the provided executable path is correct: ${launchContext.launchOptions?.executablePath}.` : ''}`;

From 2db8fa4f3ecf95e69a9eab8d91621db53282a31f Mon Sep 17 00:00:00 2001
From: Sviar <svpetrenko123@gmail.com>
Date: Wed, 30 Aug 2023 15:38:50 +0300
Subject: [PATCH 7/7] fix(browser-pool): correct link to puppeteer browsers api
 in logs

---
 packages/browser-pool/src/puppeteer/puppeteer-plugin.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
index 85baa8b0c569..8d011e615e28 100644
--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
@@ -84,7 +84,7 @@ export class PuppeteerPlugin extends BrowserPlugin<
                     debugMessage += ' Try installing';
                 }
                 debugMessage += ` a browser, if it's missing, by running \`npx @puppeteer/browsers install chromium --path [path]\``
-                    + ` and pointing \`executablePath\` to the downloaded executable (https://playwright.dev/docs/browsers).`
+                    + ` and pointing \`executablePath\` to the downloaded executable (https://pptr.dev/browsers-api).`
                     + ` The original error will be displayed at the bottom as the [cause].`;
                 throw new CriticalError(debugMessage, {
                     cause: error,