Merge pull request #551 from FlowiseAI/feature/WebCrawl

FlowiseAI · Jul 16, 2023 · 7d985db · 7d985db
2 parents bb55198 + 97eb80b
commit 7d985db
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/packages/components/src/utils.ts b/packages/components/src/utils.ts
@@ -320,7 +320,7 @@ export async function xmlScrape(currentURL: string, limit: number): Promise<stri
         }
 
         const contentType: string | null = resp.headers.get('content-type')
-        if ((contentType && !contentType.includes('application/xml')) || !contentType) {
+        if ((contentType && !contentType.includes('application/xml') && !contentType.includes('text/xml')) || !contentType) {
             if (process.env.DEBUG === 'true') console.error(`non xml response, content type: ${contentType}, on page: ${currentURL}`)
             return urls
         }