Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: rename interface FailedRequestHandler to ErrorHandler #1410

Merged
merged 1 commit into from
Jul 20, 2022
Merged

fix: rename interface FailedRequestHandler to ErrorHandler #1410

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions packages/basic-crawler/src/internals/basic-crawler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;

export type RequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context) => Awaitable<void>;

export type FailedRequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context, error: Error) => Awaitable<void>;
export type ErrorHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context, error: Error) => Awaitable<void>;

export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCrawlingContext> {
/**
Expand Down Expand Up @@ -144,7 +144,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
* Second argument is the `Error` instance that
* represents the last error thrown during processing of the request.
*/
errorHandler?: FailedRequestHandler<Context>;
errorHandler?: ErrorHandler<Context>;

/**
* A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
Expand All @@ -154,7 +154,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
* Second argument is the `Error` instance that
* represents the last error thrown during processing of the request.
*/
failedRequestHandler?: FailedRequestHandler<Context>;
failedRequestHandler?: ErrorHandler<Context>;

/**
* A function to handle requests that failed more than {@link BasicCrawlerOptions.maxRequestRetries|`maxRequestRetries`} times.
Expand All @@ -167,7 +167,7 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
* @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.
* @ignore
*/
handleFailedRequestFunction?: FailedRequestHandler<Context>;
handleFailedRequestFunction?: ErrorHandler<Context>;

/**
* Indicates how many times the request is retried if {@link BasicCrawlerOptions.requestHandler|`requestHandler`} fails.
Expand Down Expand Up @@ -335,8 +335,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext

protected log: Log;
protected requestHandler!: RequestHandler<Context>;
protected errorHandler?: FailedRequestHandler<Context>;
protected failedRequestHandler?: FailedRequestHandler<Context>;
protected errorHandler?: ErrorHandler<Context>;
protected failedRequestHandler?: ErrorHandler<Context>;
protected requestHandlerTimeoutMillis!: number;
protected internalTimeoutMillis: number;
protected maxRequestRetries: number;
Expand Down
10 changes: 5 additions & 5 deletions packages/cheerio-crawler/src/internals/cheerio-crawler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
import { concatStreamToBuffer, readStreamToString } from '@apify/utilities';
import type { BasicCrawlerOptions } from '@crawlee/basic';
import type { BasicCrawlerOptions, ErrorHandler } from '@crawlee/basic';
import { BasicCrawler, BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '@crawlee/basic';
import type { CrawlingContext, EnqueueLinksOptions, ProxyConfiguration, Request, RequestQueue, Session } from '@crawlee/core';
import { CrawlerExtension, enqueueLinks, mergeCookies, Router, resolveBaseUrl, validators } from '@crawlee/core';
Expand Down Expand Up @@ -35,7 +35,7 @@ const CHEERIO_OPTIMIZED_AUTOSCALED_POOL_OPTIONS = {
},
};

export type CheerioFailedRequestHandler<JSONData = Dictionary> = (inputs: CheerioCrawlingContext<JSONData>, error: Error) => Awaitable<void>;
export type CheerioErrorHandler<JSONData = Dictionary> = ErrorHandler<CheerioCrawlingContext<JSONData>>;

export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<BasicCrawlerOptions<CheerioCrawlingContext<JSONData>>,
// Overridden with cheerio context
Expand Down Expand Up @@ -137,7 +137,7 @@ export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<Basic
* Second argument is the `Error` instance that
* represents the last error thrown during processing of the request.
*/
errorHandler?: CheerioFailedRequestHandler<JSONData>;
errorHandler?: CheerioErrorHandler<JSONData>;

/**
* A function to handle requests that failed more than `option.maxRequestRetries` times.
Expand All @@ -150,7 +150,7 @@ export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<Basic
* See [source code](https://github.com/apify/crawlee/blob/master/src/crawlers/cheerio_crawler.js#L13)
* for the default implementation of this function.
*/
failedRequestHandler?: CheerioFailedRequestHandler<JSONData>;
failedRequestHandler?: CheerioErrorHandler<JSONData>;

/**
* A function to handle requests that failed more than `option.maxRequestRetries` times.
Expand All @@ -166,7 +166,7 @@ export interface CheerioCrawlerOptions<JSONData = Dictionary> extends Omit<Basic
* @deprecated `handleFailedRequestFunction` has been renamed to `failedRequestHandler` and will be removed in a future version.
* @ignore
*/
handleFailedRequestFunction?: CheerioFailedRequestHandler<JSONData>;
handleFailedRequestFunction?: CheerioErrorHandler<JSONData>;

/**
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
Expand Down
14 changes: 7 additions & 7 deletions test/core/crawlers/basic_crawler.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import type { AddressInfo } from 'net';
import log from '@apify/log';
import type {
CrawlingContext,
FailedRequestHandler,
ErrorHandler,
RequestHandler } from '@crawlee/basic';
import {
Request,
Expand Down Expand Up @@ -278,13 +278,13 @@ describe('BasicCrawler', () => {
throw new Error(`This is an error ${errorHandlerCalls}`);
};

const errorHandler: FailedRequestHandler = async ({ request }, error) => {
const errorHandler: ErrorHandler = async ({ request }, error) => {
expect(error.message).toBe(`This is an error ${errorHandlerCalls}`);
errorHandlerCalls++;
request.label = `error_${errorHandlerCalls}`;
};

const failedRequestHandler: FailedRequestHandler = async ({ request }, error) => {
const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
failed[request.url] = { request, error };
failedRequestHandlerCalls++;
};
Expand Down Expand Up @@ -322,7 +322,7 @@ describe('BasicCrawler', () => {
processed[request.url] = request;
};

const failedRequestHandler: FailedRequestHandler = async ({ request }, error) => {
const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
failed[request.url] = request;
errors.push(error);
};
Expand Down Expand Up @@ -362,7 +362,7 @@ describe('BasicCrawler', () => {
throw new NonRetryableError('some-error');
};

const failedRequestHandler: FailedRequestHandler = async ({ request }, error) => {
const failedRequestHandler: ErrorHandler = async ({ request }, error) => {
failed[request.url] = request;
errors.push(error);
};
Expand Down Expand Up @@ -399,7 +399,7 @@ describe('BasicCrawler', () => {
throw new CriticalError('some-error');
};

const failedRequestHandler = jest.fn() as FailedRequestHandler;
const failedRequestHandler = jest.fn() as ErrorHandler;

const basicCrawler = new BasicCrawler({
requestList,
Expand All @@ -421,7 +421,7 @@ describe('BasicCrawler', () => {
];
const requestList = await RequestList.open(null, sources);

const failedRequestHandler = jest.fn() as FailedRequestHandler;
const failedRequestHandler = jest.fn() as ErrorHandler;

const basicCrawler = new BasicCrawler({
requestList,
Expand Down