From 36744aa2b1003b3eb54617ef38e9e31742ef8e7c Mon Sep 17 00:00:00 2001 From: ZachryMsk Date: Fri, 8 Nov 2024 13:50:26 +0800 Subject: [PATCH 1/2] feat: Add Custom api url for Self-hoseted FireCrawl endpoint --- .../components/credentials/FireCrawlApi.credential.ts | 6 ++++++ .../nodes/documentloaders/FireCrawl/FireCrawl.ts | 10 +++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/components/credentials/FireCrawlApi.credential.ts b/packages/components/credentials/FireCrawlApi.credential.ts index b3daa363ccd..b65d618b622 100644 --- a/packages/components/credentials/FireCrawlApi.credential.ts +++ b/packages/components/credentials/FireCrawlApi.credential.ts @@ -18,6 +18,12 @@ class FireCrawlApiCredential implements INodeCredential { label: 'FireCrawl API', name: 'firecrawlApiToken', type: 'password' + }, + { + label: 'FireCrawl API URL', + name: 'firecrawlApiUrl', + type: 'string', + default: 'https://api.firecrawl.dev' } ] } diff --git a/packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts b/packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts index 4669bfd24e9..71bcaddfaae 100644 --- a/packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts +++ b/packages/components/nodes/documentloaders/FireCrawl/FireCrawl.ts @@ -194,19 +194,21 @@ class FirecrawlApp { interface FirecrawlLoaderParameters { url: string apiKey?: string + apiUrl?: string mode?: 'crawl' | 'scrape' params?: Record } class FireCrawlLoader extends BaseDocumentLoader { private apiKey: string + private apiUrl: string private url: string private mode: 'crawl' | 'scrape' private params?: Record constructor(loaderParams: FirecrawlLoaderParameters) { super() - const { apiKey, url, mode = 'crawl', params } = loaderParams + const { apiKey, apiUrl, url, mode = 'crawl', params } = loaderParams if (!apiKey) { throw new Error('Firecrawl API key not set. You can set it as FIRECRAWL_API_KEY in your .env file, or pass it to Firecrawl.') } @@ -215,10 +217,11 @@ class FireCrawlLoader extends BaseDocumentLoader { this.url = url this.mode = mode this.params = params + this.apiUrl = apiUrl || 'https://api.firecrawl.dev' } public async load(): Promise { - const app = new FirecrawlApp({ apiKey: this.apiKey }) + const app = new FirecrawlApp({ apiKey: this.apiKey, apiUrl: this.apiUrl }) let firecrawlDocs: FirecrawlDocument[] if (this.mode === 'scrape') { @@ -319,6 +322,7 @@ class FireCrawl_DocumentLoaders implements INode { const onlyMainContent = nodeData.inputs?.onlyMainContent as boolean const credentialData = await getCredentialData(nodeData.credential ?? '', options) const firecrawlApiToken = getCredentialParam('firecrawlApiToken', credentialData, nodeData) + const firecrawlApiUrl = getCredentialParam('firecrawlApiUrl', credentialData, nodeData, 'https://api.firecrawl.dev') const urlPatternsExcludes = nodeData.inputs?.urlPatternsExcludes ? (nodeData.inputs.urlPatternsExcludes.split(',') as string[]) @@ -331,6 +335,7 @@ class FireCrawl_DocumentLoaders implements INode { url, mode: crawlerType as 'crawl' | 'scrape', apiKey: firecrawlApiToken, + apiUrl: firecrawlApiUrl, params: { crawlerOptions: { includes: urlPatternsIncludes, @@ -344,7 +349,6 @@ class FireCrawl_DocumentLoaders implements INode { } } } - const loader = new FireCrawlLoader(input) let docs = [] From 3a97f759fabd40ba3cb9d4fc47867772cd046f8a Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Sat, 9 Nov 2024 00:00:18 +0000 Subject: [PATCH 2/2] Update FireCrawlApi.credential.ts --- packages/components/credentials/FireCrawlApi.credential.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/components/credentials/FireCrawlApi.credential.ts b/packages/components/credentials/FireCrawlApi.credential.ts index b65d618b622..4a8ad75960e 100644 --- a/packages/components/credentials/FireCrawlApi.credential.ts +++ b/packages/components/credentials/FireCrawlApi.credential.ts @@ -10,7 +10,7 @@ class FireCrawlApiCredential implements INodeCredential { constructor() { this.label = 'FireCrawl API' this.name = 'fireCrawlApi' - this.version = 1.0 + this.version = 2.0 this.description = 'You can find the FireCrawl API token on your FireCrawl account page.' this.inputs = [