Skip to content

Commit

Permalink
feat(core): add forefront option to enqueueLinks helper
Browse files Browse the repository at this point in the history
Closes #1595
  • Loading branch information
B4nan committed Oct 13, 2022
1 parent 3e90d37 commit f8755b6
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions packages/core/src/enqueue_links/enqueue_links.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ import {
createRequestOptions,
createRequests,
} from './shared';
import type { RequestQueue } from '../storages/request_queue';
import type { RequestQueue, RequestQueueOperationOptions } from '../storages/request_queue';
import type { RequestOptions } from '../request';

export interface EnqueueLinksOptions {
export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
/** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
limit?: number;

Expand Down Expand Up @@ -169,6 +169,7 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
ow(options, ow.object.exactShape({
urls: ow.array.ofType(ow.string),
requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
forefront: ow.optional.boolean,
limit: ow.optional.number,
selector: ow.optional.string,
baseUrl: ow.optional.string,
Expand Down Expand Up @@ -202,6 +203,7 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
globs,
regexps,
transformRequestFunction,
forefront,
} = options;

const urlPatternObjects: UrlPatternObject[] = [];
Expand Down Expand Up @@ -281,7 +283,7 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
let requests = createFilteredRequests();
if (limit) requests = requests.slice(0, limit);

return requestQueue.addRequests(requests);
return requestQueue.addRequests(requests, { forefront });
}

/**
Expand Down

0 comments on commit f8755b6

Please sign in to comment.