diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index ba744364f44e..96e6734d3087 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -61,7 +61,7 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000; export type RequestHandler = (inputs: Context) => Awaitable; -export type FailedRequestHandler = (inputs: Context, error: Error) => Awaitable; +export type ErrorHandler = (inputs: Context, error: Error) => Awaitable; export interface BasicCrawlerOptions { /** @@ -144,7 +144,7 @@ export interface BasicCrawlerOptions; + errorHandler?: ErrorHandler; /** * A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times. @@ -154,7 +154,7 @@ export interface BasicCrawlerOptions; + failedRequestHandler?: ErrorHandler; /** * A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times. @@ -167,7 +167,7 @@ export interface BasicCrawlerOptions; + handleFailedRequestFunction?: ErrorHandler; /** * Indicates how many times the request is retried if {@link BasicCrawlerOptions.requestHandler|`requestHandler`} fails. @@ -335,8 +335,8 @@ export class BasicCrawler; - protected errorHandler?: FailedRequestHandler; - protected failedRequestHandler?: FailedRequestHandler; + protected errorHandler?: ErrorHandler; + protected failedRequestHandler?: ErrorHandler; protected requestHandlerTimeoutMillis!: number; protected internalTimeoutMillis: number; protected maxRequestRetries: number; diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts index 981380dc5d64..1186cbcb5343 100644 --- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts +++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts @@ -1,6 +1,6 @@ import { addTimeoutToPromise, tryCancel } from '@apify/timeout'; import { concatStreamToBuffer, readStreamToString } from '@apify/utilities'; -import type { BasicCrawlerOptions } from '@crawlee/basic'; +import type { BasicCrawlerOptions, ErrorHandler } from '@crawlee/basic'; import { BasicCrawler, BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '@crawlee/basic'; import type { CrawlingContext, EnqueueLinksOptions, ProxyConfiguration, Request, RequestQueue, Session } from '@crawlee/core'; import { CrawlerExtension, enqueueLinks, mergeCookies, Router, resolveBaseUrl, validators } from '@crawlee/core'; @@ -35,7 +35,7 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = { }, }; -export type CheerioFailedRequestHandler = (inputs: CheerioCrawlingContext, error: Error) => Awaitable; +export type CheerioErrorHandler = ErrorHandler>; export interface CheerioCrawlerOptions extends Omit>, // Overridden with cheerio context @@ -137,7 +137,7 @@ export interface CheerioCrawlerOptions extends Omit; + errorHandler?: CheerioErrorHandler; /** * A function to handle requests that failed more than `option.maxRequestRetries` times. @@ -150,7 +150,7 @@ export interface CheerioCrawlerOptions extends Omit; + failedRequestHandler?: CheerioErrorHandler; /** * A function to handle requests that failed more than `option.maxRequestRetries` times. @@ -166,7 +166,7 @@ export interface CheerioCrawlerOptions extends Omit; + handleFailedRequestFunction?: CheerioErrorHandler; /** * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 1d0d4d096f0c..8b82ef52ab24 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -4,7 +4,7 @@ import type { AddressInfo } from 'net'; import log from '@apify/log'; import type { CrawlingContext, - FailedRequestHandler, + ErrorHandler, RequestHandler } from '@crawlee/basic'; import { Request, @@ -278,13 +278,13 @@ describe('BasicCrawler', () => { throw new Error(`This is an error ${errorHandlerCalls}`); }; - const errorHandler: FailedRequestHandler = async ({ request }, error) => { + const errorHandler: ErrorHandler = async ({ request }, error) => { expect(error.message).toBe(`This is an error ${errorHandlerCalls}`); errorHandlerCalls++; request.label = `error_${errorHandlerCalls}`; }; - const failedRequestHandler: FailedRequestHandler = async ({ request }, error) => { + const failedRequestHandler: ErrorHandler = async ({ request }, error) => { failed[request.url] = { request, error }; failedRequestHandlerCalls++; }; @@ -322,7 +322,7 @@ describe('BasicCrawler', () => { processed[request.url] = request; }; - const failedRequestHandler: FailedRequestHandler = async ({ request }, error) => { + const failedRequestHandler: ErrorHandler = async ({ request }, error) => { failed[request.url] = request; errors.push(error); }; @@ -362,7 +362,7 @@ describe('BasicCrawler', () => { throw new NonRetryableError('some-error'); }; - const failedRequestHandler: FailedRequestHandler = async ({ request }, error) => { + const failedRequestHandler: ErrorHandler = async ({ request }, error) => { failed[request.url] = request; errors.push(error); }; @@ -399,7 +399,7 @@ describe('BasicCrawler', () => { throw new CriticalError('some-error'); }; - const failedRequestHandler = jest.fn() as FailedRequestHandler; + const failedRequestHandler = jest.fn() as ErrorHandler; const basicCrawler = new BasicCrawler({ requestList, @@ -421,7 +421,7 @@ describe('BasicCrawler', () => { ]; const requestList = await RequestList.open(null, sources); - const failedRequestHandler = jest.fn() as FailedRequestHandler; + const failedRequestHandler = jest.fn() as ErrorHandler; const basicCrawler = new BasicCrawler({ requestList,