From ab9ad195be0123d11b62307c3607eb24b63ad4fc Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 13:07:56 +0700 Subject: [PATCH 01/11] bump llama_cloud and update pipeline --- helpers/python.ts | 2 +- .../vectordbs/python/llamacloud/generate.py | 54 +++-------------- .../vectordbs/python/llamacloud/index.py | 60 +++++++++++++++++-- .../vectordbs/python/llamacloud/service.py | 10 ++-- .../types/streaming/fastapi/pyproject.toml | 2 +- 5 files changed, 70 insertions(+), 58 deletions(-) diff --git a/helpers/python.ts b/helpers/python.ts index 4cd8cb75c..a59a5160d 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -103,7 +103,7 @@ const getAdditionalDependencies = ( case "llamacloud": dependencies.push({ name: "llama-index-indices-managed-llama-cloud", - version: "^0.6.0", + version: "^0.6.3", }); break; } diff --git a/templates/components/vectordbs/python/llamacloud/generate.py b/templates/components/vectordbs/python/llamacloud/generate.py index acd28777e..a084da3bd 100644 --- a/templates/components/vectordbs/python/llamacloud/generate.py +++ b/templates/components/vectordbs/python/llamacloud/generate.py @@ -1,5 +1,4 @@ # flake8: noqa: E402 -import os from dotenv import load_dotenv @@ -7,62 +6,23 @@ import logging -from app.engine.index import get_client, get_index +from llama_index.core.readers import SimpleDirectoryReader + +from app.engine.index import get_index from app.engine.service import LLamaCloudFileService # type: ignore from app.settings import init_settings -from llama_cloud import PipelineType -from llama_index.core.readers import SimpleDirectoryReader -from llama_index.core.settings import Settings logging.basicConfig(level=logging.INFO) logger = logging.getLogger() -def ensure_index(index): - project_id = index._get_project_id() - client = get_client() - pipelines = client.pipelines.search_pipelines( - project_id=project_id, - pipeline_name=index.name, - pipeline_type=PipelineType.MANAGED.value, - ) - if len(pipelines) == 0: - from llama_index.embeddings.openai import OpenAIEmbedding - - if not isinstance(Settings.embed_model, OpenAIEmbedding): - raise ValueError( - "Creating a new pipeline with a non-OpenAI embedding model is not supported." - ) - client.pipelines.upsert_pipeline( - project_id=project_id, - request={ - "name": index.name, - "embedding_config": { - "type": "OPENAI_EMBEDDING", - "component": { - "api_key": os.getenv("OPENAI_API_KEY"), # editable - "model_name": os.getenv("EMBEDDING_MODEL"), - }, - }, - "transform_config": { - "mode": "auto", - "config": { - "chunk_size": Settings.chunk_size, # editable - "chunk_overlap": Settings.chunk_overlap, # editable - }, - }, - }, - ) - - def generate_datasource(): init_settings() logger.info("Generate index for the provided data") - index = get_index() - ensure_index(index) - project_id = index._get_project_id() - pipeline_id = index._get_pipeline_id() + index = get_index(create_if_missing=True) + if index is None: + raise ValueError("Index not found and could not be created") # use SimpleDirectoryReader to retrieve the files to process reader = SimpleDirectoryReader( @@ -78,7 +38,7 @@ def generate_datasource(): f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}" ) LLamaCloudFileService.add_file_to_pipeline( - project_id, pipeline_id, f, custom_metadata={} + index.project.id, index.pipeline.id, f, custom_metadata={} ) logger.info("Finished generating the index") diff --git a/templates/components/vectordbs/python/llamacloud/index.py b/templates/components/vectordbs/python/llamacloud/index.py index f6f7e4f67..32c1d9e2c 100644 --- a/templates/components/vectordbs/python/llamacloud/index.py +++ b/templates/components/vectordbs/python/llamacloud/index.py @@ -2,10 +2,12 @@ import os from typing import Optional +from llama_cloud import PipelineType from llama_index.core.callbacks import CallbackManager from llama_index.core.ingestion.api_utils import ( get_client as llama_cloud_get_client, ) +from llama_index.core.settings import Settings from llama_index.indices.managed.llama_cloud import LlamaCloudIndex from pydantic import BaseModel, Field, field_validator @@ -82,14 +84,64 @@ def to_index_kwargs(self) -> dict: } -def get_index(config: IndexConfig = None): +def get_index( + config: IndexConfig = None, + create_if_missing: bool = False, +): if config is None: config = IndexConfig() - index = LlamaCloudIndex(**config.to_index_kwargs()) - - return index + # Check whether the index exists + try: + index = LlamaCloudIndex(**config.to_index_kwargs()) + return index + except ValueError: + if create_if_missing: + logger.info( + f"Index {config.llama_cloud_pipeline_config.pipeline} not found, creating it" + ) + _create_index(config) + return LlamaCloudIndex(**config.to_index_kwargs()) + return None def get_client(): config = LlamaCloudConfig() return llama_cloud_get_client(**config.to_client_kwargs()) + + +def _create_index( + config: IndexConfig, +): + client = get_client() + pipeline_name = config.llama_cloud_pipeline_config.pipeline + + pipelines = client.pipelines.search_pipelines( + pipeline_name=pipeline_name, + pipeline_type=PipelineType.MANAGED.value, + ) + if len(pipelines) == 0: + from llama_index.embeddings.openai import OpenAIEmbedding + + if not isinstance(Settings.embed_model, OpenAIEmbedding): + raise ValueError( + "Creating a new pipeline with a non-OpenAI embedding model is not supported." + ) + client.pipelines.upsert_pipeline( + request={ + "name": pipeline_name, + "embedding_config": { + "type": "OPENAI_EMBEDDING", + "component": { + "api_key": os.getenv("OPENAI_API_KEY"), # editable + "model_name": os.getenv("EMBEDDING_MODEL"), + }, + }, + "transform_config": { + "mode": "auto", + "config": { + "chunk_size": Settings.chunk_size, # editable + "chunk_overlap": Settings.chunk_overlap, # editable + }, + }, + }, + ) diff --git a/templates/components/vectordbs/python/llamacloud/service.py b/templates/components/vectordbs/python/llamacloud/service.py index 68216f98e..71fcd7ac1 100644 --- a/templates/components/vectordbs/python/llamacloud/service.py +++ b/templates/components/vectordbs/python/llamacloud/service.py @@ -1,18 +1,18 @@ -from io import BytesIO import logging import os import time -from typing import Any, Dict, List, Optional, Set, Tuple, Union import typing +from io import BytesIO +from typing import Any, Dict, List, Optional, Set, Tuple, Union +import requests from fastapi import BackgroundTasks from llama_cloud import ManagedIngestionStatus, PipelineFileCreateCustomMetadataValue +from llama_index.core.schema import NodeWithScore from pydantic import BaseModel -import requests + from app.api.routers.models import SourceNodes from app.engine.index import get_client -from llama_index.core.schema import NodeWithScore - logger = logging.getLogger("uvicorn") diff --git a/templates/types/streaming/fastapi/pyproject.toml b/templates/types/streaming/fastapi/pyproject.toml index 4d181fa6b..46672bc51 100644 --- a/templates/types/streaming/fastapi/pyproject.toml +++ b/templates/types/streaming/fastapi/pyproject.toml @@ -19,7 +19,7 @@ python-dotenv = "^1.0.0" pydantic = "<2.10" aiostream = "^0.5.2" cachetools = "^5.3.3" -llama-index = "^0.11.17" +llama-index = "^0.12.1" rich = "^13.9.4" [tool.poetry.group.dev.dependencies] From 621197c7fc338cc4140c92b48176fadaf0167c8b Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 13:11:22 +0700 Subject: [PATCH 02/11] add change set --- .changeset/shaggy-rats-draw.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/shaggy-rats-draw.md diff --git a/.changeset/shaggy-rats-draw.md b/.changeset/shaggy-rats-draw.md new file mode 100644 index 000000000..d579556be --- /dev/null +++ b/.changeset/shaggy-rats-draw.md @@ -0,0 +1,5 @@ +--- +"create-llama": patch +--- + +Bump the LlamaCloud library and fix breaking changes (Python). From 36f1b177aa241d7e0136f35235af84f04e85ac88 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 13:16:15 +0700 Subject: [PATCH 03/11] add missing change --- .../components/vectordbs/python/llamacloud/service.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/templates/components/vectordbs/python/llamacloud/service.py b/templates/components/vectordbs/python/llamacloud/service.py index 71fcd7ac1..2754b6a23 100644 --- a/templates/components/vectordbs/python/llamacloud/service.py +++ b/templates/components/vectordbs/python/llamacloud/service.py @@ -67,10 +67,11 @@ def add_file_to_pipeline( ) -> str: client = get_client() file = client.files.upload_file(project_id=project_id, upload_file=upload_file) + file_id = file.id files = [ { - "file_id": file.id, - "custom_metadata": {"file_id": file.id, **(custom_metadata or {})}, + "file_id": file_id, + "custom_metadata": {"file_id": file_id, **(custom_metadata or {})}, } ] files = client.pipelines.add_files_to_pipeline(pipeline_id, request=files) @@ -79,12 +80,14 @@ def add_file_to_pipeline( max_attempts = 20 attempt = 0 while attempt < max_attempts: - result = client.pipelines.get_pipeline_file_status(pipeline_id, file.id) + result = client.pipelines.get_pipeline_file_status( + file_id=file_id, pipeline_id=pipeline_id + ) if result.status == ManagedIngestionStatus.ERROR: raise Exception(f"File processing failed: {str(result)}") if result.status == ManagedIngestionStatus.SUCCESS: # File is ingested - return the file id - return file.id + return file_id attempt += 1 time.sleep(0.1) # Sleep for 100ms raise Exception( From 0ba0c8d61eee89dd6e9d7491818bce9bfea1f7bb Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 13:18:19 +0700 Subject: [PATCH 04/11] better error handling --- templates/components/vectordbs/python/llamacloud/index.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/templates/components/vectordbs/python/llamacloud/index.py b/templates/components/vectordbs/python/llamacloud/index.py index 32c1d9e2c..97261900a 100644 --- a/templates/components/vectordbs/python/llamacloud/index.py +++ b/templates/components/vectordbs/python/llamacloud/index.py @@ -95,10 +95,9 @@ def get_index( index = LlamaCloudIndex(**config.to_index_kwargs()) return index except ValueError: + logger.warning("Index not found") if create_if_missing: - logger.info( - f"Index {config.llama_cloud_pipeline_config.pipeline} not found, creating it" - ) + logger.info("Creating index") _create_index(config) return LlamaCloudIndex(**config.to_index_kwargs()) return None From ac70b6ca52dcdf39f85363001f783eceff0d9f6f Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 13:31:39 +0700 Subject: [PATCH 05/11] bump llama-index --- helpers/python.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/helpers/python.ts b/helpers/python.ts index a59a5160d..9fb3faa5d 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -167,15 +167,15 @@ const getAdditionalDependencies = ( if (templateType !== "multiagent") { dependencies.push({ name: "llama-index-llms-openai", - version: "^0.2.0", + version: "^0.3.2", }); dependencies.push({ name: "llama-index-embeddings-openai", - version: "^0.2.3", + version: "^0.3.1", }); dependencies.push({ name: "llama-index-agent-openai", - version: "^0.3.0", + version: "^0.4.0", }); } break; From 8a5543106d2b467c001953df809596f0b81bf41d Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 13:45:04 +0700 Subject: [PATCH 06/11] bump llama-index --- helpers/python.ts | 22 +++++++++++----------- helpers/tools.ts | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/helpers/python.ts b/helpers/python.ts index 9fb3faa5d..5e08008b4 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -37,21 +37,21 @@ const getAdditionalDependencies = ( case "mongo": { dependencies.push({ name: "llama-index-vector-stores-mongodb", - version: "^0.3.1", + version: "^0.6.0", }); break; } case "pg": { dependencies.push({ name: "llama-index-vector-stores-postgres", - version: "^0.2.5", + version: "^0.3.2", }); break; } case "pinecone": { dependencies.push({ name: "llama-index-vector-stores-pinecone", - version: "^0.2.1", + version: "^0.4.1", constraints: { python: ">=3.11,<3.13", }, @@ -61,7 +61,7 @@ const getAdditionalDependencies = ( case "milvus": { dependencies.push({ name: "llama-index-vector-stores-milvus", - version: "^0.2.0", + version: "^0.3.0", }); dependencies.push({ name: "pymilvus", @@ -72,14 +72,14 @@ const getAdditionalDependencies = ( case "astra": { dependencies.push({ name: "llama-index-vector-stores-astra-db", - version: "^0.2.0", + version: "^0.4.0", }); break; } case "qdrant": { dependencies.push({ name: "llama-index-vector-stores-qdrant", - version: "^0.3.0", + version: "^0.4.0", constraints: { python: ">=3.11,<3.13", }, @@ -89,14 +89,14 @@ const getAdditionalDependencies = ( case "chroma": { dependencies.push({ name: "llama-index-vector-stores-chroma", - version: "^0.2.0", + version: "^0.4.0", }); break; } case "weaviate": { dependencies.push({ name: "llama-index-vector-stores-weaviate", - version: "^1.1.1", + version: "^1.2.3", }); break; } @@ -122,13 +122,13 @@ const getAdditionalDependencies = ( case "web": dependencies.push({ name: "llama-index-readers-web", - version: "^0.2.2", + version: "^0.3.0", }); break; case "db": dependencies.push({ name: "llama-index-readers-database", - version: "^0.2.0", + version: "^0.3.0", }); dependencies.push({ name: "pymysql", @@ -524,7 +524,7 @@ export const installPythonTemplate = async ({ if (observability === "llamatrace") { addOnDependencies.push({ name: "llama-index-callbacks-arize-phoenix", - version: "^0.2.1", + version: "^0.3.0", constraints: { python: ">=3.11,<3.13", }, diff --git a/helpers/tools.ts b/helpers/tools.ts index c3a34390c..33bd96e60 100644 --- a/helpers/tools.ts +++ b/helpers/tools.ts @@ -41,7 +41,7 @@ export const supportedTools: Tool[] = [ dependencies: [ { name: "llama-index-tools-google", - version: "^0.2.0", + version: "^0.3.0", }, ], supportedFrameworks: ["fastapi"], @@ -82,7 +82,7 @@ For better results, you can specify the region parameter to get results from a s dependencies: [ { name: "llama-index-tools-wikipedia", - version: "^0.2.0", + version: "^0.3.0", }, ], supportedFrameworks: ["fastapi", "express", "nextjs"], From 1e93f93effe2fd5b1d18a3890150d4eafcbd4dd9 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 14:01:38 +0700 Subject: [PATCH 07/11] bump llama-index --- templates/types/extractor/fastapi/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/types/extractor/fastapi/pyproject.toml b/templates/types/extractor/fastapi/pyproject.toml index e9574a019..a9cad0f76 100644 --- a/templates/types/extractor/fastapi/pyproject.toml +++ b/templates/types/extractor/fastapi/pyproject.toml @@ -14,7 +14,7 @@ fastapi = "^0.109.1" uvicorn = { extras = ["standard"], version = "^0.23.2" } python-dotenv = "^1.0.0" pydantic = "<2.10" -llama-index = "^0.11.1" +llama-index = "^0.12.1" cachetools = "^5.3.3" reflex = "^0.6.2.post1" From cb4a16f99ec8ba08542d7b2ce55e1c7e36592f08 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 15:01:22 +0700 Subject: [PATCH 08/11] fix reflex --- .../types/extractor/fastapi/app/ui/components/upload.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/types/extractor/fastapi/app/ui/components/upload.py b/templates/types/extractor/fastapi/app/ui/components/upload.py index 64421feb9..e404840a1 100644 --- a/templates/types/extractor/fastapi/app/ui/components/upload.py +++ b/templates/types/extractor/fastapi/app/ui/components/upload.py @@ -2,6 +2,7 @@ from typing import List import reflex as rx + from app.engine.generate import generate_datasource @@ -78,10 +79,10 @@ def upload_component() -> rx.Component: UploadedFilesState.uploaded_files, lambda file: rx.card( rx.stack( - rx.text(file.file_name, size="sm"), + rx.text(file.file_name, size="2"), rx.button( "x", - size="sm", + size="2", on_click=UploadedFilesState.remove_file(file.file_name), ), justify="between", From 661f13d5ccc65fbe0269f2b90bb9be5c531d58be Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 15:16:38 +0700 Subject: [PATCH 09/11] add option for not waiting file upload to LlamaCloud --- .../components/vectordbs/python/llamacloud/generate.py | 6 +++++- templates/components/vectordbs/python/llamacloud/service.py | 4 ++++ templates/types/streaming/fastapi/app/services/file.py | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/templates/components/vectordbs/python/llamacloud/generate.py b/templates/components/vectordbs/python/llamacloud/generate.py index a084da3bd..5c7d4d30b 100644 --- a/templates/components/vectordbs/python/llamacloud/generate.py +++ b/templates/components/vectordbs/python/llamacloud/generate.py @@ -38,7 +38,11 @@ def generate_datasource(): f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}" ) LLamaCloudFileService.add_file_to_pipeline( - index.project.id, index.pipeline.id, f, custom_metadata={} + index.project.id, + index.pipeline.id, + f, + custom_metadata={}, + wait_for_processing=False, ) logger.info("Finished generating the index") diff --git a/templates/components/vectordbs/python/llamacloud/service.py b/templates/components/vectordbs/python/llamacloud/service.py index 2754b6a23..31b91365d 100644 --- a/templates/components/vectordbs/python/llamacloud/service.py +++ b/templates/components/vectordbs/python/llamacloud/service.py @@ -64,6 +64,7 @@ def add_file_to_pipeline( pipeline_id: str, upload_file: Union[typing.IO, Tuple[str, BytesIO]], custom_metadata: Optional[Dict[str, PipelineFileCreateCustomMetadataValue]], + wait_for_processing: bool = True, ) -> str: client = get_client() file = client.files.upload_file(project_id=project_id, upload_file=upload_file) @@ -76,6 +77,9 @@ def add_file_to_pipeline( ] files = client.pipelines.add_files_to_pipeline(pipeline_id, request=files) + if not wait_for_processing: + return file_id + # Wait 2s for the file to be processed max_attempts = 20 attempt = 0 diff --git a/templates/types/streaming/fastapi/app/services/file.py b/templates/types/streaming/fastapi/app/services/file.py index 7aa6696c3..3fc1a64f1 100644 --- a/templates/types/streaming/fastapi/app/services/file.py +++ b/templates/types/streaming/fastapi/app/services/file.py @@ -249,6 +249,7 @@ def _add_file_to_llama_cloud_index( index.pipeline.id, upload_file, custom_metadata={}, + wait_for_processing=True, ) return doc_id From 05c2d531d800be0e6780132308a824dc86230522 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 15:23:37 +0700 Subject: [PATCH 10/11] add tqdm to show upload progress --- .../components/vectordbs/python/llamacloud/generate.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/templates/components/vectordbs/python/llamacloud/generate.py b/templates/components/vectordbs/python/llamacloud/generate.py index 5c7d4d30b..371809db6 100644 --- a/templates/components/vectordbs/python/llamacloud/generate.py +++ b/templates/components/vectordbs/python/llamacloud/generate.py @@ -7,6 +7,7 @@ import logging from llama_index.core.readers import SimpleDirectoryReader +from tqdm import tqdm from app.engine.index import get_index from app.engine.service import LLamaCloudFileService # type: ignore @@ -32,9 +33,13 @@ def generate_datasource(): files_to_process = reader.input_files # add each file to the LlamaCloud pipeline - for input_file in files_to_process: + for input_file in tqdm( + files_to_process, + desc="Processing files", + unit="file", + ): with open(input_file, "rb") as f: - logger.info( + logger.debug( f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}" ) LLamaCloudFileService.add_file_to_pipeline( From 2aa94226e0e6ec4f931ca91b7d9e884b85cfa410 Mon Sep 17 00:00:00 2001 From: leehuwuj Date: Tue, 3 Dec 2024 15:26:46 +0700 Subject: [PATCH 11/11] add error handling --- .../vectordbs/python/llamacloud/generate.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/templates/components/vectordbs/python/llamacloud/generate.py b/templates/components/vectordbs/python/llamacloud/generate.py index 371809db6..3932b86ab 100644 --- a/templates/components/vectordbs/python/llamacloud/generate.py +++ b/templates/components/vectordbs/python/llamacloud/generate.py @@ -33,6 +33,7 @@ def generate_datasource(): files_to_process = reader.input_files # add each file to the LlamaCloud pipeline + error_files = [] for input_file in tqdm( files_to_process, desc="Processing files", @@ -42,13 +43,20 @@ def generate_datasource(): logger.debug( f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}" ) - LLamaCloudFileService.add_file_to_pipeline( - index.project.id, - index.pipeline.id, - f, - custom_metadata={}, - wait_for_processing=False, - ) + try: + LLamaCloudFileService.add_file_to_pipeline( + index.project.id, + index.pipeline.id, + f, + custom_metadata={}, + wait_for_processing=False, + ) + except Exception as e: + error_files.append(input_file) + logger.error(f"Error adding file {input_file}: {e}") + + if error_files: + logger.error(f"Failed to add the following files: {error_files}") logger.info("Finished generating the index")