From a159da5bfe2104f6ab8cfd675f87fd5874d0d6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9my=20JOURDIN?= <2821990+JJK801@users.noreply.github.com> Date: Fri, 1 Nov 2024 23:03:53 +0100 Subject: [PATCH] [Usntructured loaders] Make Unstructured API URL optional when environment variable is present (#3414) * Make Unstructured API URL optional when environment variable is present * Fix empty apiUrl option in Unsctructured flowise loader * Add focumentation for env vars --- .../nodes/documentloaders/S3File/README.md | 13 +++++++++++++ .../nodes/documentloaders/S3File/S3File.ts | 3 ++- .../nodes/documentloaders/Unstructured/README.md | 13 +++++++++++++ .../documentloaders/Unstructured/Unstructured.ts | 12 ++++++------ .../Unstructured/UnstructuredFile.ts | 3 ++- .../Unstructured/UnstructuredFolder.ts | 3 ++- 6 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 packages/components/nodes/documentloaders/S3File/README.md create mode 100644 packages/components/nodes/documentloaders/Unstructured/README.md diff --git a/packages/components/nodes/documentloaders/S3File/README.md b/packages/components/nodes/documentloaders/S3File/README.md new file mode 100644 index 00000000000..142ed86e1b9 --- /dev/null +++ b/packages/components/nodes/documentloaders/S3File/README.md @@ -0,0 +1,13 @@ +# S3 File Loader + +DS File Loarder integration for Flowise + +## 🌱 Env Variables + +| Variable | Description | Type | Default | +| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | +| UNSTRUCTURED_API_URL | Default `unstructuredApiUrl` for S3 File Loader | String | http://localhost:8000/general/v0/general | + +## License + +Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md). \ No newline at end of file diff --git a/packages/components/nodes/documentloaders/S3File/S3File.ts b/packages/components/nodes/documentloaders/S3File/S3File.ts index 2e8fcaf1852..fc83a448ed6 100644 --- a/packages/components/nodes/documentloaders/S3File/S3File.ts +++ b/packages/components/nodes/documentloaders/S3File/S3File.ts @@ -70,7 +70,8 @@ class S3_DocumentLoaders implements INode { description: 'Your Unstructured.io URL. Read more on how to get started', type: 'string', - default: 'http://localhost:8000/general/v0/general' + placeholder: process.env.UNSTRUCTURED_API_URL || 'http://localhost:8000/general/v0/general', + optional: !!process.env.UNSTRUCTURED_API_URL }, { label: 'Unstructured API KEY', diff --git a/packages/components/nodes/documentloaders/Unstructured/README.md b/packages/components/nodes/documentloaders/Unstructured/README.md new file mode 100644 index 00000000000..0854cc67d7b --- /dev/null +++ b/packages/components/nodes/documentloaders/Unstructured/README.md @@ -0,0 +1,13 @@ +# Unstructured File/Floder Loader + +Unstructured File Loader integration for Flowise + +## 🌱 Env Variables + +| Variable | Description | Type | Default | +| ---------------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ----------------------------------- | +| UNSTRUCTURED_API_URL | Default `apiUrl` for Unstructured File/Floder Loader | String | http://localhost:8000/general/v0/general | + +## License + +Source code in this repository is made available under the [Apache License Version 2.0](https://github.com/FlowiseAI/Flowise/blob/master/LICENSE.md). \ No newline at end of file diff --git a/packages/components/nodes/documentloaders/Unstructured/Unstructured.ts b/packages/components/nodes/documentloaders/Unstructured/Unstructured.ts index 18ca94f2898..8f3e49f6258 100644 --- a/packages/components/nodes/documentloaders/Unstructured/Unstructured.ts +++ b/packages/components/nodes/documentloaders/Unstructured/Unstructured.ts @@ -29,9 +29,9 @@ type Element = { export class UnstructuredLoader extends BaseDocumentLoader { public filePath: string - private apiUrl = 'https://api.unstructuredapp.io/general/v0/general' + private apiUrl = process.env.UNSTRUCTURED_API_URL || 'https://api.unstructuredapp.io/general/v0/general' - private apiKey?: string + private apiKey: string | undefined = process.env.UNSTRUCTURED_API_KEY private strategy: StringWithAutocomplete = 'hi_res' @@ -66,10 +66,10 @@ export class UnstructuredLoader extends BaseDocumentLoader { const options = optionsOrLegacyFilePath this.apiKey = options.apiKey - this.apiUrl = options.apiUrl ?? this.apiUrl - this.strategy = options.strategy ?? this.strategy + this.apiUrl = options.apiUrl || this.apiUrl + this.strategy = options.strategy || this.strategy this.encoding = options.encoding - this.ocrLanguages = options.ocrLanguages ?? this.ocrLanguages + this.ocrLanguages = options.ocrLanguages || this.ocrLanguages this.coordinates = options.coordinates this.pdfInferTableStructure = options.pdfInferTableStructure this.xmlKeepTags = options.xmlKeepTags @@ -128,7 +128,7 @@ export class UnstructuredLoader extends BaseDocumentLoader { } const headers = { - 'UNSTRUCTURED-API-KEY': this.apiKey ?? '' + 'UNSTRUCTURED-API-KEY': this.apiKey || '' } const response = await fetch(this.apiUrl, { diff --git a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts index df680e63781..28dc793ec45 100644 --- a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts +++ b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts @@ -63,7 +63,8 @@ class UnstructuredFile_DocumentLoaders implements INode { description: 'Unstructured API URL. Read more on how to get started', type: 'string', - default: 'http://localhost:8000/general/v0/general' + placeholder: process.env.UNSTRUCTURED_API_URL || 'http://localhost:8000/general/v0/general', + optional: !!process.env.UNSTRUCTURED_API_URL }, { label: 'Strategy', diff --git a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts index 996af5ac1ed..df396590d1f 100644 --- a/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts +++ b/packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts @@ -51,7 +51,8 @@ class UnstructuredFolder_DocumentLoaders implements INode { description: 'Unstructured API URL. Read more on how to get started', type: 'string', - default: 'http://localhost:8000/general/v0/general' + placeholder: process.env.UNSTRUCTURED_API_URL || 'http://localhost:8000/general/v0/general', + optional: !!process.env.UNSTRUCTURED_API_URL }, { label: 'Strategy',