Skip to content

Commit 7a50122

Browse files
committed
feat: add tests
1 parent 1a9053e commit 7a50122

File tree

3 files changed

+1082
-9
lines changed

3 files changed

+1082
-9
lines changed

scrapegraph-js/src/crawl.js

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,15 @@ import { zodToJsonSchema } from 'zod-to-json-schema';
88
*
99
* @param {string} apiKey - Your ScrapeGraph AI API key
1010
* @param {string} url - The starting URL for the crawl
11-
* @param {string} prompt - The prompt to guide the crawl and extraction
12-
* @param {Object|ZodType} schema - JSON schema or Zod schema defining the structure of the extracted data
11+
* @param {string|null} prompt - The prompt to guide the crawl and extraction (null for markdown mode)
12+
* @param {Object|ZodType|null} schema - JSON schema or Zod schema defining the structure of the extracted data (null for markdown mode)
1313
* @param {Object} [options] - Optional crawl parameters
14+
* @param {boolean} [options.extractionMode=true] - true for AI extraction, false for markdown conversion (NO AI/LLM)
1415
* @param {boolean} [options.cacheWebsite=true] - Whether to cache the website content
1516
* @param {number} [options.depth=2] - Maximum depth of the crawl (1-10)
1617
* @param {number} [options.maxPages=2] - Maximum number of pages to crawl (1-100)
1718
* @param {boolean} [options.sameDomainOnly=true] - Whether to only crawl pages from the same domain
19+
* @param {boolean} [options.sitemap] - Whether to use sitemap for better page discovery
1820
* @param {number} [options.batchSize=1] - Batch size for processing pages (1-10)
1921
* @returns {Promise<Object>} The crawl job response
2022
* @throws {Error} Throws an error if the HTTP request fails
@@ -33,13 +35,15 @@ export async function crawl(
3335
'Content-Type': 'application/json',
3436
};
3537

36-
let schemaPayload;
37-
if (schema instanceof ZodType) {
38-
schemaPayload = zodToJsonSchema(schema);
39-
} else if (typeof schema === 'object' && schema !== null) {
40-
schemaPayload = schema;
41-
} else {
42-
throw new Error('The schema must be a Zod schema or a plain object');
38+
let schemaPayload = null;
39+
if (schema !== null && schema !== undefined) {
40+
if (schema instanceof ZodType) {
41+
schemaPayload = zodToJsonSchema(schema);
42+
} else if (typeof schema === 'object') {
43+
schemaPayload = schema;
44+
} else {
45+
throw new Error('The schema must be a Zod schema, a plain object, or null');
46+
}
4347
}
4448

4549
const {

0 commit comments

Comments
 (0)