From e5b5e9fec3b11a052234ab433c1726d2bcbb6bda Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 11:06:16 +0700 Subject: [PATCH 01/80] chore: replace Python examples with llama-deploy --- .../create-llama/helpers/env-variables.ts | 49 ++++--- packages/create-llama/helpers/index.ts | 15 +- packages/create-llama/helpers/python.ts | 46 ++++-- .../templates/components/ts-proxy/index.ts | 14 ++ .../components/ts-proxy/package.json | 18 +++ .../python/code_generator/README-template.md | 80 +++++++---- .../use-cases/python/code_generator/utils.py | 131 ++++++++++++++++++ .../python/code_generator/workflow.py | 54 ++++---- .../llamaindexserver/fastapi/app/__init__.py | 0 .../llamaindexserver/fastapi/llama_deploy.yml | 24 ++++ .../types/llamaindexserver/fastapi/main.py | 32 ----- .../llamaindexserver/fastapi/pyproject.toml | 11 +- .../fastapi/{ => src}/generate.py | 1 + .../fastapi/{app => src}/index.py | 0 .../fastapi/{app => src}/settings.py | 0 15 files changed, 356 insertions(+), 119 deletions(-) create mode 100644 packages/create-llama/templates/components/ts-proxy/index.ts create mode 100644 packages/create-llama/templates/components/ts-proxy/package.json create mode 100644 packages/create-llama/templates/components/use-cases/python/code_generator/utils.py delete mode 100644 packages/create-llama/templates/types/llamaindexserver/fastapi/app/__init__.py create mode 100644 packages/create-llama/templates/types/llamaindexserver/fastapi/llama_deploy.yml delete mode 100644 packages/create-llama/templates/types/llamaindexserver/fastapi/main.py rename packages/create-llama/templates/types/llamaindexserver/fastapi/{ => src}/generate.py (98%) rename packages/create-llama/templates/types/llamaindexserver/fastapi/{app => src}/index.py (100%) rename packages/create-llama/templates/types/llamaindexserver/fastapi/{app => src}/settings.py (100%) diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index 4f64262d9..6f9bbc155 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -228,7 +228,14 @@ Otherwise, use CHROMA_HOST and CHROMA_PORT config above`, } }; -const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => { +const getModelEnvs = ( + modelConfig: ModelConfig, + framework: TemplateFramework, + template: TemplateType, +): EnvVar[] => { + const isPythonUseCase = + framework === "fastapi" && template === "llamaindexserver"; + return [ { name: "MODEL", @@ -240,10 +247,15 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => { description: "Name of the embedding model to use.", value: modelConfig.embeddingModel, }, - { - name: "CONVERSATION_STARTERS", - description: "The questions to help users get started (multi-line).", - }, + ...(isPythonUseCase + ? [] + : [ + { + name: "CONVERSATION_STARTERS", + description: + "The questions to help users get started (multi-line).", + }, + ]), ...(modelConfig.provider === "openai" ? [ { @@ -251,14 +263,18 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => { description: "The OpenAI API key to use.", value: modelConfig.apiKey, }, - { - name: "LLM_TEMPERATURE", - description: "Temperature for sampling from the model.", - }, - { - name: "LLM_MAX_TOKENS", - description: "Maximum number of tokens to generate.", - }, + ...(isPythonUseCase + ? [] + : [ + { + name: "LLM_TEMPERATURE", + description: "Temperature for sampling from the model.", + }, + { + name: "LLM_MAX_TOKENS", + description: "Maximum number of tokens to generate.", + }, + ]), ] : []), ...(modelConfig.provider === "anthropic" @@ -367,11 +383,12 @@ const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => { const getFrameworkEnvs = ( framework: TemplateFramework, + template?: TemplateType, port?: number, ): EnvVar[] => { const sPort = port?.toString() || "8000"; const result: EnvVar[] = []; - if (framework === "fastapi") { + if (framework === "fastapi" && template !== "llamaindexserver") { result.push( ...[ { @@ -418,8 +435,8 @@ export const createBackendEnvFile = async ( ] : []), ...getVectorDBEnvs(opts.vectorDb, opts.framework, opts.template), - ...getFrameworkEnvs(opts.framework, opts.port), - ...getModelEnvs(opts.modelConfig), + ...getFrameworkEnvs(opts.framework, opts.template, opts.port), + ...getModelEnvs(opts.modelConfig, opts.framework, opts.template), ]; // Render and write env file const content = renderEnvVar(envVars); diff --git a/packages/create-llama/helpers/index.ts b/packages/create-llama/helpers/index.ts index 210cdfaf7..e02f35b39 100644 --- a/packages/create-llama/helpers/index.ts +++ b/packages/create-llama/helpers/index.ts @@ -156,6 +156,11 @@ export const installTemplate = async (props: InstallTemplateArgs) => { await installTSTemplate(props); } + const isPythonUseCase = + props.framework === "fastapi" && + props.template === "llamaindexserver" && + !!props.useCase; + // This is a backend, so we need to copy the test data and create the env file. // Copy the environment file to the target directory. @@ -183,10 +188,12 @@ export const installTemplate = async (props: InstallTemplateArgs) => { ); } - // Create outputs directory - await makeDir(path.join(props.root, "output/tools")); - await makeDir(path.join(props.root, "output/uploaded")); - await makeDir(path.join(props.root, "output/llamacloud")); + if (!isPythonUseCase) { + // Create outputs directory (python use-cases are using llama-deploy so don't need this) + await makeDir(path.join(props.root, "output/tools")); + await makeDir(path.join(props.root, "output/uploaded")); + await makeDir(path.join(props.root, "output/llamacloud")); + } }; export * from "./types"; diff --git a/packages/create-llama/helpers/python.ts b/packages/create-llama/helpers/python.ts index 4d2e2af12..40af34e6f 100644 --- a/packages/create-llama/helpers/python.ts +++ b/packages/create-llama/helpers/python.ts @@ -412,13 +412,35 @@ const installLlamaIndexServerTemplate = async ({ process.exit(1); } - await copy("*.py", path.join(root, "app"), { + /** + * Python use-cases structure: + * src/ + * ├── workflow.py + * ├── settings.py + * ├── index.py + * ├── generate.py + * ├── ... (other utility files) + * ui/ + * ├── index.ts + * └── package.json + * ├── components/*.tsx + * ├── layout/*.tsx + * llama_deploy.yaml + * pyproject.toml + * README.md + */ + + const srcDir = path.join(root, "src"); + const uiDir = path.join(root, "ui"); + + // copy workflow code to src folder + await copy("*.py", srcDir, { parents: true, cwd: path.join(templatesDir, "components", "use-cases", "python", useCase), }); - // copy model provider settings to app folder - await copy("**", path.join(root, "app"), { + // copy model provider settings to src folder + await copy("**", srcDir, { cwd: path.join( templatesDir, "components", @@ -428,20 +450,26 @@ const installLlamaIndexServerTemplate = async ({ ), }); - // Copy custom UI component code - await copy(`*`, path.join(root, "components"), { + // copy ts server to ui folder + await copy("**", uiDir, { + parents: true, + cwd: path.join(templatesDir, "components", "ts-proxy"), + }); + + // Copy custom UI components to ui/components folder + await copy(`*`, path.join(uiDir, "components"), { parents: true, cwd: path.join(templatesDir, "components", "ui", "use-cases", useCase), }); - // Copy layout components to layout folder in root - await copy("*", path.join(root, "layout"), { + // Copy layout components to ui/layout folder + await copy("*", path.join(uiDir, "layout"), { parents: true, cwd: path.join(templatesDir, "components", "ui", "layout"), }); if (useLlamaParse) { - await copy("index.py", path.join(root, "app"), { + await copy("index.py", srcDir, { parents: true, cwd: path.join( templatesDir, @@ -453,7 +481,7 @@ const installLlamaIndexServerTemplate = async ({ ), }); // TODO: Consider moving generate.py to app folder. - await copy("generate.py", path.join(root), { + await copy("generate.py", srcDir, { parents: true, cwd: path.join( templatesDir, diff --git a/packages/create-llama/templates/components/ts-proxy/index.ts b/packages/create-llama/templates/components/ts-proxy/index.ts new file mode 100644 index 000000000..4169e22e2 --- /dev/null +++ b/packages/create-llama/templates/components/ts-proxy/index.ts @@ -0,0 +1,14 @@ +import { LlamaIndexServer } from '@llamaindex/server' + +new LlamaIndexServer({ + uiConfig: { + starterQuestions: ['Generate calculator app', 'Generate todo list app'], + componentsDir: 'components', + layoutDir: 'layout', + llamaDeploy: { + deployment: 'chat', + workflow: 'workflow', + }, + }, + port: 3000, +}).start() diff --git a/packages/create-llama/templates/components/ts-proxy/package.json b/packages/create-llama/templates/components/ts-proxy/package.json new file mode 100644 index 000000000..5307666dc --- /dev/null +++ b/packages/create-llama/templates/components/ts-proxy/package.json @@ -0,0 +1,18 @@ +{ + "name": "llamaindex-server-ui", + "version": "0.0.1", + "private": true, + "scripts": { + "dev": "nodemon --exec tsx index.ts" + }, + "dependencies": { + "@llamaindex/server": "latest", + "dotenv": "^16.4.7" + }, + "devDependencies": { + "@types/node": "^20.10.3", + "nodemon": "^3.1.10", + "tsx": "4.7.2", + "typescript": "^5.3.2" + } +} diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md index 8df458fd2..7fa3dfbdc 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md @@ -1,49 +1,76 @@ -This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/). +# LlamaIndex Workflow Example -## Getting Started +This is a [LlamaIndex](https://www.llamaindex.ai/) project that using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/) deployed with [LlamaDeploy](https://github.com/run-llama/llama_deploy). -First, setup the environment with uv: +LlamaDeploy is a system for deploying and managing LlamaIndex workflows, while LlamaIndexServer provides a pre-built TypeScript server with an integrated chat UI that can connect directly to LlamaDeploy deployments. This example shows how you can quickly set up a complete chat application by combining these two technologies/ -> **_Note:_** This step is not needed if you are using the dev-container. +## Installation -```shell +Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, just run: + +```bash uv sync ``` -Then check the parameters that have been pre-configured in the `.env` file in this directory. -Make sure you have set the `OPENAI_API_KEY` for the LLM. +## Running the Deployment -Then, run the development server: +At this point we have all we need to run this deployment. Ideally, we would have the API server already running +somewhere in the cloud, but to get started let's start an instance locally. Run the following python script +from a shell: -```shell -uv run fastapi dev +``` +$ uv run -m llama_deploy.apiserver +INFO: Started server process [10842] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:4501 (Press CTRL+C to quit) ``` -Then open [http://localhost:8000](http://localhost:8000) with your browser to start the chat UI. - -To start the app optimized for **production**, run: +From another shell, use the CLI, `llamactl`, to create the deployment: ``` -uv run fastapi run +$ uv run llamactl deploy llama_deploy.yml +Deployment successful: chat ``` -## Configure LLM and Embedding Model +## UI Interface -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). +LlamaDeploy will serve the UI through the apiserver. Point the browser to [http://localhost:4501/deployments/chat/ui](http://localhost:4501/deployments/chat/ui) to interact with your deployment through a user-friendly interface. -## Use Case -AI-powered code generator that can help you generate app with a chat interface, code editor and app preview. +## API endpoints -To update the workflow, you can modify the code in [`workflow.py`](app/workflow.py). +You can find all the endpoints in the [API documentation](http://localhost:4501/docs). To get started, you can try the following endpoints: -You can start by sending an request on the [chat UI](http://localhost:8000) or you can test the `/api/chat` endpoint with the following curl request: +Create a new task: +```bash +curl -X POST 'http://localhost:4501/deployments/chat/tasks/create' \ + -H 'Content-Type: application/json' \ + -d '{ + "input": "{\"user_msg\":\"Hello\",\"chat_history\":[]}", + "service_id": "workflow" + }' ``` -curl --location 'localhost:8000/api/chat' \ ---header 'Content-Type: application/json' \ ---data '{ "messages": [{ "role": "user", "content": "Create a report comparing the finances of Apple and Tesla" }] }' + +Stream events: + +```bash +curl 'http://localhost:4501/deployments/chat/tasks/0b411be6-005d-43f0-9b6b-6a0017f08002/events?session_id=dd36442c-45ca-4eaa-8d75-b4e6dad1a83e&raw_event=true' \ + -H 'Content-Type: application/json' ``` +Note that the task_id and session_id are returned when creating a new task. + +## Use Case + +AI-powered code generator that can help you generate app with a chat interface, code editor and app preview. +To update the workflow, you can modify the code in [`workflow.py`](app/workflow.py). + +## Configure LLM and Embedding Model + +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). + + ## Customize the UI To customize the UI, you can start by modifying the [./components/ui_event.jsx](./components/ui_event.jsx) file. @@ -56,10 +83,9 @@ uv run generate_ui ## Learn More -To learn more about LlamaIndex, take a look at the following resources: - - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. - [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows. -- [LlamaIndex Server](https://pypi.org/project/llama-index-server/) +- [LlamaDeploy GitHub Repository](https://github.com/run-llama/llama_deploy) +- [Chat-UI Documentation](https://ts.llamaindex.ai/docs/chat-ui) -You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! +You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! \ No newline at end of file diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/utils.py b/packages/create-llama/templates/components/use-cases/python/code_generator/utils.py new file mode 100644 index 000000000..15eff6115 --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/utils.py @@ -0,0 +1,131 @@ +import json +import re +from typing import List, Optional, Any + +from pydantic import ValidationError +from llama_index.core.chat_ui.models.artifact import ( + Artifact, + ArtifactType, + CodeArtifactData, + DocumentArtifactData, +) +from llama_index.core.llms import ChatMessage + +INLINE_ANNOTATION_KEY = "annotation" + + +def get_inline_annotations(message: ChatMessage) -> List[Any]: + """Extract inline annotations from a chat message.""" + markdown_content = message.content + + inline_annotations: List[Any] = [] + + # Regex to match annotation code blocks + # Matches ```annotation followed by content until closing ``` + annotation_regex = re.compile( + rf"```{re.escape(INLINE_ANNOTATION_KEY)}\s*\n([\s\S]*?)\n```", re.MULTILINE + ) + + for match in annotation_regex.finditer(markdown_content): + json_content = match.group(1).strip() if match.group(1) else None + + if not json_content: + continue + + try: + # Parse the JSON content + parsed = json.loads(json_content) + + # Check for required fields in the parsed annotation + if ( + not isinstance(parsed, dict) + or "type" not in parsed + or "data" not in parsed + ): + continue + + # Extract the annotation data + inline_annotations.append(parsed) + except (json.JSONDecodeError, ValidationError) as error: + # Skip invalid annotations - they might be malformed JSON or invalid schema + print(f"Failed to parse annotation: {error}") + + return inline_annotations + + +def artifact_from_message(message: ChatMessage) -> Optional[Artifact]: + """Create an artifact from a chat message if it contains artifact annotations.""" + inline_annotations = get_inline_annotations(message) + + for annotation in inline_annotations: + if isinstance(annotation, dict) and annotation.get("type") == "artifact": + try: + # Create artifact data based on type + artifact_data = annotation.get("data") + if not artifact_data: + continue + + artifact_type = artifact_data.get("type") + + if artifact_type == "code": + # Get the nested data object that contains the actual code information + code_info = artifact_data.get("data", {}) + code_data = CodeArtifactData( + file_name=code_info.get("file_name", ""), + code=code_info.get("code", ""), + language=code_info.get("language", ""), + ) + artifact = Artifact( + created_at=artifact_data.get("created_at"), + type=ArtifactType.CODE, + data=code_data, + ) + elif artifact_type == "document": + # Get the nested data object that contains the actual document information + doc_info = artifact_data.get("data", {}) + doc_data = DocumentArtifactData( + title=doc_info.get("title", ""), + content=doc_info.get("content", ""), + type=doc_info.get("type", "markdown"), + sources=doc_info.get("sources"), + ) + artifact = Artifact( + created_at=artifact_data.get("created_at"), + type=ArtifactType.DOCUMENT, + data=doc_data, + ) + else: + continue + + return artifact + except Exception as e: + print( + f"Failed to parse artifact from annotation: {annotation}. Error: {e}" + ) + + return None + + +def get_artifacts(chat_history: List[ChatMessage]) -> List[Artifact]: + """ + Return a list of artifacts sorted by their creation time. + Artifacts without a creation time are placed at the end. + """ + artifacts = [] + + for message in chat_history: + artifact = artifact_from_message(message) + if artifact is not None: + artifacts.append(artifact) + + # Sort by creation time, with None values at the end + return sorted( + artifacts, + key=lambda a: (a.created_at is None, a.created_at), + ) + + +def get_last_artifact(chat_history: List[ChatMessage]) -> Optional[Artifact]: + """Get the last artifact from chat history.""" + artifacts = get_artifacts(chat_history) + return artifacts[-1] if len(artifacts) > 0 else None diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py b/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py index b7c478bdf..4e38acc29 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py @@ -2,11 +2,10 @@ import time from typing import Any, Literal, Optional, Union -from llama_index.core.chat_engine.types import ChatMessage -from llama_index.core.llms import LLM +from llama_index.core import Settings +from llama_index.core.llms import LLM, ChatMessage from llama_index.core.memory import ChatMemoryBuffer from llama_index.core.prompts import PromptTemplate -from llama_index.llms.openai import OpenAI from llama_index.core.workflow import ( Context, Event, @@ -15,26 +14,19 @@ Workflow, step, ) -from llama_index.server.api.models import ( +from llama_index.core.chat_ui.models.artifact import ( Artifact, - ArtifactEvent, ArtifactType, - ChatRequest, CodeArtifactData, +) +from llama_index.core.chat_ui.events import ( UIEvent, + ArtifactEvent, ) -from llama_index.server.api.utils import get_last_artifact -from pydantic import BaseModel, Field - - -def create_workflow(chat_request: ChatRequest) -> Workflow: - workflow = CodeArtifactWorkflow( - llm=OpenAI(model="gpt-4.1"), - chat_request=chat_request, - timeout=120.0, - ) - return workflow +from src.utils import get_last_artifact +from src.settings import init_settings +from pydantic import BaseModel class Requirement(BaseModel): next_step: Literal["answering", "coding"] @@ -83,8 +75,6 @@ class CodeArtifactWorkflow(Workflow): def __init__( self, - llm: LLM, - chat_request: ChatRequest, **kwargs: Any, ): """ @@ -93,9 +83,8 @@ def __init__( chat_request: The chat request from the chat app to use. """ super().__init__(**kwargs) - self.llm = llm - self.chat_request = chat_request - self.last_artifact = get_last_artifact(chat_request) + self.llm: LLM = Settings.llm + self.last_artifact: Optional[Artifact] = None @step async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> PlanEvent: @@ -103,13 +92,21 @@ async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> PlanEvent: if user_msg is None: raise ValueError("user_msg is required to run the workflow") await ctx.set("user_msg", user_msg) - chat_history = ev.chat_history or [] - chat_history.append( + + # prepare chat history from StartEvent + messages = [ ChatMessage( - role="user", - content=user_msg, + role=msg.get("role", "user"), + content=msg.get("content", ""), ) - ) + for msg in ev.get("chat_history", []) + ] + chat_history = [*messages, ChatMessage(role="user", content=user_msg)] + + # extract inline artifact from chat history + last_artifact = get_last_artifact(messages) + self.last_artifact = last_artifact + memory = ChatMemoryBuffer.from_defaults( chat_history=chat_history, llm=self.llm, @@ -373,3 +370,6 @@ async def synthesize_answer( ) ) return StopEvent(result=response_stream) + +init_settings() +workflow = CodeArtifactWorkflow() diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/app/__init__.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/app/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/llama_deploy.yml b/packages/create-llama/templates/types/llamaindexserver/fastapi/llama_deploy.yml new file mode 100644 index 000000000..3e7587282 --- /dev/null +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/llama_deploy.yml @@ -0,0 +1,24 @@ +name: chat + +control-plane: + port: 8000 + +default-service: workflow + +services: + workflow: + name: Workflow + source: + type: local + name: src + path: src/workflow:workflow + python-dependencies: + - llama-index-llms-openai>=0.4.5 + - llama-index-core>=0.12.45 + +ui: + name: My Nextjs App + port: 3000 + source: + type: local + name: ui diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/main.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/main.py deleted file mode 100644 index 35356b38d..000000000 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/main.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging - -from app.settings import init_settings -from app.workflow import create_workflow -from dotenv import load_dotenv -from llama_index.server import LlamaIndexServer, UIConfig - -logger = logging.getLogger("uvicorn") - -# A path to a directory where the customized UI code is stored -COMPONENT_DIR = "components" - - -def create_app(): - app = LlamaIndexServer( - workflow_factory=create_workflow, # A factory function that creates a new workflow for each request - ui_config=UIConfig( - component_dir=COMPONENT_DIR, - dev_mode=True, # Please disable this in production - layout_dir="layout", - ), - logger=logger, - env="dev", - ) - # You can also add custom FastAPI routes to app - app.add_api_route("/api/health", lambda: {"message": "OK"}, status_code=200) - return app - - -load_dotenv() -init_settings() -app = create_app() diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml index eb8753493..d10a79c14 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml @@ -12,9 +12,12 @@ dependencies = [ "pydantic<2.10", "aiostream>=0.5.2,<0.6.0", "llama-index-core>=0.12.28,<0.13.0", - "llama-index-server>=0.1.17,<0.2.0", + "llama-deploy", ] +[tool.uv.sources] +llama-deploy = { git = "https://github.com/run-llama/llama_deploy" } + [project.optional-dependencies] dev = [ "mypy>=1.8.0,<2.0.0", @@ -23,9 +26,9 @@ dev = [ ] [project.scripts] -generate = "generate:generate_index" -generate_index = "generate:generate_index" -generate_ui = "generate:generate_ui_for_workflow" +generate = "src.generate:generate_index" +generate_index = "src.generate:generate_index" +generate_ui = "src.generate:generate_ui_for_workflow" [tool.mypy] diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/generate.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py similarity index 98% rename from packages/create-llama/templates/types/llamaindexserver/fastapi/generate.py rename to packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py index 2b792288f..a41fc6b88 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/generate.py +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py @@ -59,6 +59,7 @@ def generate_ui_for_workflow(): except ImportError: raise ImportError("Couldn't generate UI component for the current workflow.") from llama_index.server.gen_ui import generate_event_component + # TODO: remove llama_index.server # works well with OpenAI gpt-4.1, Claude 3.7 Sonnet or Gemini Pro 2.5 code = asyncio.run( diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/app/index.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py similarity index 100% rename from packages/create-llama/templates/types/llamaindexserver/fastapi/app/index.py rename to packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/app/settings.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/settings.py similarity index 100% rename from packages/create-llama/templates/types/llamaindexserver/fastapi/app/settings.py rename to packages/create-llama/templates/types/llamaindexserver/fastapi/src/settings.py From d96d8305923d2e25876990a3ddf74633eca33060 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 11:12:55 +0700 Subject: [PATCH 02/80] use fixed version for ts-server package --- .../create-llama/templates/components/ts-proxy/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-llama/templates/components/ts-proxy/package.json b/packages/create-llama/templates/components/ts-proxy/package.json index 5307666dc..de659d601 100644 --- a/packages/create-llama/templates/components/ts-proxy/package.json +++ b/packages/create-llama/templates/components/ts-proxy/package.json @@ -6,7 +6,7 @@ "dev": "nodemon --exec tsx index.ts" }, "dependencies": { - "@llamaindex/server": "latest", + "@llamaindex/server": "0.2.9", "dotenv": "^16.4.7" }, "devDependencies": { From b6fc63bd95b86146a68cfc2cc335b8c394faf7e6 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 11:19:29 +0700 Subject: [PATCH 03/80] update readme --- .../use-cases/python/code_generator/README-template.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md index 7fa3dfbdc..e3782dc1e 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md @@ -12,6 +12,8 @@ Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, uv sync ``` +If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). + ## Running the Deployment At this point we have all we need to run this deployment. Ideally, we would have the API server already running From fad34146af1b7359a1a863d8d5663125e48c562c Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 11:24:13 +0700 Subject: [PATCH 04/80] update pyproject.toml --- .../types/llamaindexserver/fastapi/pyproject.toml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml index d10a79c14..4064484d0 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml @@ -9,7 +9,7 @@ readme = "README.md" requires-python = ">=3.11,<3.14" dependencies = [ "python-dotenv>=1.0.0,<2.0.0", - "pydantic<2.10", + "pydantic>=2.11.5", "aiostream>=0.5.2,<0.6.0", "llama-index-core>=0.12.28,<0.13.0", "llama-deploy", @@ -43,15 +43,4 @@ ignore_missing_imports = true follow_imports = "silent" implicit_optional = true strict_optional = false -disable_error_code = [ "return-value", "assignment" ] - -[[tool.mypy.overrides]] -module = "app.*" -ignore_missing_imports = false - -[tool.hatch.metadata] -allow-direct-references = true - -[build-system] -requires = [ "hatchling>=1.24" ] -build-backend = "hatchling.build" \ No newline at end of file +disable_error_code = [ "return-value", "assignment" ] \ No newline at end of file From 2a00cadc8f420894535f3cac82dda671c2a12d04 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 11:47:28 +0700 Subject: [PATCH 05/80] fix load env --- .../create-llama/helpers/env-variables.ts | 18 +++++++-- packages/create-llama/helpers/index.ts | 10 ++--- .../python/code_generator/workflow.py | 38 ++++++++++++------- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index 6f9bbc155..1d5ddd232 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -233,7 +233,7 @@ const getModelEnvs = ( framework: TemplateFramework, template: TemplateType, ): EnvVar[] => { - const isPythonUseCase = + const isPythonLlamaDeploy = framework === "fastapi" && template === "llamaindexserver"; return [ @@ -247,7 +247,7 @@ const getModelEnvs = ( description: "Name of the embedding model to use.", value: modelConfig.embeddingModel, }, - ...(isPythonUseCase + ...(isPythonLlamaDeploy ? [] : [ { @@ -263,7 +263,7 @@ const getModelEnvs = ( description: "The OpenAI API key to use.", value: modelConfig.apiKey, }, - ...(isPythonUseCase + ...(isPythonLlamaDeploy ? [] : [ { @@ -440,6 +440,16 @@ export const createBackendEnvFile = async ( ]; // Render and write env file const content = renderEnvVar(envVars); - await fs.writeFile(path.join(root, envFileName), content); + + const isPythonLlamaDeploy = + opts.framework === "fastapi" && opts.template === "llamaindexserver"; + + // llama-deploy will copy the whole src folder contains workflow file so that + // we need to put the .env file inside it for loading the env variables correctly + const envPath = isPythonLlamaDeploy + ? path.join(root, "src", envFileName) + : path.join(root, envFileName); + + await fs.writeFile(envPath, content); console.log(`Created '${envFileName}' file. Please check the settings.`); }; diff --git a/packages/create-llama/helpers/index.ts b/packages/create-llama/helpers/index.ts index e02f35b39..17e58ac8b 100644 --- a/packages/create-llama/helpers/index.ts +++ b/packages/create-llama/helpers/index.ts @@ -156,10 +156,8 @@ export const installTemplate = async (props: InstallTemplateArgs) => { await installTSTemplate(props); } - const isPythonUseCase = - props.framework === "fastapi" && - props.template === "llamaindexserver" && - !!props.useCase; + const isPythonLlamaDeploy = + props.framework === "fastapi" && props.template === "llamaindexserver"; // This is a backend, so we need to copy the test data and create the env file. @@ -188,8 +186,8 @@ export const installTemplate = async (props: InstallTemplateArgs) => { ); } - if (!isPythonUseCase) { - // Create outputs directory (python use-cases are using llama-deploy so don't need this) + if (!isPythonLlamaDeploy) { + // Create outputs directory (llama-deploy doesn't need this) await makeDir(path.join(props.root, "output/tools")); await makeDir(path.join(props.root, "output/uploaded")); await makeDir(path.join(props.root, "output/llamacloud")); diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py b/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py index 4e38acc29..a0c8c84cb 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py @@ -26,7 +26,9 @@ from src.utils import get_last_artifact from src.settings import init_settings -from pydantic import BaseModel +from pydantic import BaseModel, Field +from dotenv import load_dotenv + class Requirement(BaseModel): next_step: Literal["answering", "coding"] @@ -114,9 +116,9 @@ async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> PlanEvent: await ctx.set("memory", memory) return PlanEvent( user_msg=user_msg, - context=str(self.last_artifact.model_dump_json()) - if self.last_artifact - else "", + context=( + str(self.last_artifact.model_dump_json()) if self.last_artifact else "" + ), ) @step @@ -136,7 +138,8 @@ async def planning( ), ) ) - prompt = PromptTemplate(""" + prompt = PromptTemplate( + """ You are a product analyst responsible for analyzing the user's request and providing the next step for code or document generation. You are helping user with their code artifact. To update the code, you need to plan a coding step. @@ -190,10 +193,13 @@ async def planning( Now, plan the user's next step for this request: {user_msg} - """).format( - context="" - if event.context is None - else f"## The context is: \n{event.context}\n", + """ + ).format( + context=( + "" + if event.context is None + else f"## The context is: \n{event.context}\n" + ), user_msg=event.user_msg, ) response = await self.llm.acomplete( @@ -251,7 +257,8 @@ async def generate_artifact( ), ) ) - prompt = PromptTemplate(""" + prompt = PromptTemplate( + """ You are a skilled developer who can help user with coding. You are given a task to generate or update a code for a given requirement. @@ -298,10 +305,11 @@ async def generate_artifact( Now, i have to generate the code for the following requirement: {requirement} ``` - """).format( - previous_artifact=self.last_artifact.model_dump_json() - if self.last_artifact - else "", + """ + ).format( + previous_artifact=( + self.last_artifact.model_dump_json() if self.last_artifact else "" + ), requirement=event.requirement, ) response = await self.llm.acomplete( @@ -371,5 +379,7 @@ async def synthesize_answer( ) return StopEvent(result=response_stream) + +load_dotenv() init_settings() workflow = CodeArtifactWorkflow() From b701ca3ba0a1d4da8210f862f93a0ab785b3e6cf Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 11:53:57 +0700 Subject: [PATCH 06/80] update readme for .env --- packages/create-llama/helpers/env-variables.ts | 4 ++-- .../python/code_generator/README-template.md | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index 1d5ddd232..7140405a9 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -444,8 +444,8 @@ export const createBackendEnvFile = async ( const isPythonLlamaDeploy = opts.framework === "fastapi" && opts.template === "llamaindexserver"; - // llama-deploy will copy the whole src folder contains workflow file so that - // we need to put the .env file inside it for loading the env variables correctly + // llama-deploy only copies the src folder without copying files in root, + // so we need to put the .env file inside src/ to use env variables in the workflow file const envPath = isPythonLlamaDeploy ? path.join(root, "src", envFileName) : path.join(root, envFileName); diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md index e3782dc1e..d1d6d33a0 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md @@ -14,6 +14,12 @@ uv sync If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). +## Configure LLM and Embedding Model + +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). + +Remember to setup their respective API keys in the `src/.env` file. + ## Running the Deployment At this point we have all we need to run this deployment. Ideally, we would have the API server already running @@ -68,11 +74,6 @@ Note that the task_id and session_id are returned when creating a new task. AI-powered code generator that can help you generate app with a chat interface, code editor and app preview. To update the workflow, you can modify the code in [`workflow.py`](app/workflow.py). -## Configure LLM and Embedding Model - -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). - - ## Customize the UI To customize the UI, you can start by modifying the [./components/ui_event.jsx](./components/ui_event.jsx) file. From b991ca81f8dde18402c7eca6a1dafc6aa6d39076 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 12:19:24 +0700 Subject: [PATCH 07/80] use absolute logo --- packages/create-llama/templates/components/ui/layout/header.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-llama/templates/components/ui/layout/header.tsx b/packages/create-llama/templates/components/ui/layout/header.tsx index a54de8154..7ee30db3b 100644 --- a/packages/create-llama/templates/components/ui/layout/header.tsx +++ b/packages/create-llama/templates/components/ui/layout/header.tsx @@ -21,7 +21,7 @@ export default function Header() { Llama Logo From 1f25c9d34f59882075eb6f229b939cc7c978f906 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 13:31:12 +0700 Subject: [PATCH 08/80] update comment --- packages/create-llama/helpers/env-variables.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index 7140405a9..58de76bcb 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -444,8 +444,9 @@ export const createBackendEnvFile = async ( const isPythonLlamaDeploy = opts.framework === "fastapi" && opts.template === "llamaindexserver"; - // llama-deploy only copies the src folder without copying files in root, - // so we need to put the .env file inside src/ to use env variables in the workflow file + // each llama-deploy service will need a .env inside its directory + // this .env will be copied along with workflow code when service is deployed + // so that we need to put the .env file inside src/ instead of root const envPath = isPythonLlamaDeploy ? path.join(root, "src", envFileName) : path.join(root, envFileName); From a20dd6630952403f243754c1e8fb80aacd1364a8 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 14:52:47 +0700 Subject: [PATCH 09/80] document generator --- .../document_generator/README-template.md | 82 +++++++---- .../python/document_generator/utils.py | 131 ++++++++++++++++++ .../python/document_generator/workflow.py | 59 ++++---- 3 files changed, 217 insertions(+), 55 deletions(-) create mode 100644 packages/create-llama/templates/components/use-cases/python/document_generator/utils.py diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md index b5a972bcc..9a739114f 100644 --- a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md @@ -1,50 +1,79 @@ -This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/). +# LlamaIndex Workflow Example -## Getting Started +This is a [LlamaIndex](https://www.llamaindex.ai/) project that using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/) deployed with [LlamaDeploy](https://github.com/run-llama/llama_deploy). -First, setup the environment with uv: +LlamaDeploy is a system for deploying and managing LlamaIndex workflows, while LlamaIndexServer provides a pre-built TypeScript server with an integrated chat UI that can connect directly to LlamaDeploy deployments. This example shows how you can quickly set up a complete chat application by combining these two technologies/ -> **_Note:_** This step is not needed if you are using the dev-container. +## Installation -```shell +Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, just run: + +```bash uv sync ``` -Then check the parameters that have been pre-configured in the `.env` file in this directory. -Make sure you have set the `OPENAI_API_KEY` for the LLM. +If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). -Then, run the development server: +## Configure LLM and Embedding Model -```shell -uv run fastapi dev -``` +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). + +Remember to setup their respective API keys in the `src/.env` file. -Then open [http://localhost:8000](http://localhost:8000) with your browser to start the chat UI. +## Running the Deployment -To start the app optimized for **production**, run: +At this point we have all we need to run this deployment. Ideally, we would have the API server already running +somewhere in the cloud, but to get started let's start an instance locally. Run the following python script +from a shell: ``` -uv run fastapi run +$ uv run -m llama_deploy.apiserver +INFO: Started server process [10842] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:4501 (Press CTRL+C to quit) ``` -## Configure LLM and Embedding Model +From another shell, use the CLI, `llamactl`, to create the deployment: -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). +``` +$ uv run llamactl deploy llama_deploy.yml +Deployment successful: chat +``` -## Use Case +## UI Interface -AI-powered document generator that can help you generate documents with a chat interface and simple markdown editor. +LlamaDeploy will serve the UI through the apiserver. Point the browser to [http://localhost:4501/deployments/chat/ui](http://localhost:4501/deployments/chat/ui) to interact with your deployment through a user-friendly interface. + +## API endpoints -To update the workflow, you can modify the code in [`workflow.py`](app/workflow.py). +You can find all the endpoints in the [API documentation](http://localhost:4501/docs). To get started, you can try the following endpoints: -You can start by sending an request on the [chat UI](http://localhost:8000) or you can test the `/api/chat` endpoint with the following curl request: +Create a new task: +```bash +curl -X POST 'http://localhost:4501/deployments/chat/tasks/create' \ + -H 'Content-Type: application/json' \ + -d '{ + "input": "{\"user_msg\":\"Hello\",\"chat_history\":[]}", + "service_id": "workflow" + }' ``` -curl --location 'localhost:8000/api/chat' \ ---header 'Content-Type: application/json' \ ---data '{ "messages": [{ "role": "user", "content": "Create a report comparing the finances of Apple and Tesla" }] }' + +Stream events: + +```bash +curl 'http://localhost:4501/deployments/chat/tasks/0b411be6-005d-43f0-9b6b-6a0017f08002/events?session_id=dd36442c-45ca-4eaa-8d75-b4e6dad1a83e&raw_event=true' \ + -H 'Content-Type: application/json' ``` +Note that the task_id and session_id are returned when creating a new task. + +## Use Case + +AI-powered document generator that can help you generate documents with a chat interface and simple markdown editor. +To update the workflow, you can modify the code in [`workflow.py`](src/workflow.py). + ## Customize the UI To customize the UI, you can start by modifying the [./components/ui_event.jsx](./components/ui_event.jsx) file. @@ -57,10 +86,9 @@ uv run generate_ui ## Learn More -To learn more about LlamaIndex, take a look at the following resources: - - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. - [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows. -- [LlamaIndex Server](https://pypi.org/project/llama-index-server/) +- [LlamaDeploy GitHub Repository](https://github.com/run-llama/llama_deploy) +- [Chat-UI Documentation](https://ts.llamaindex.ai/docs/chat-ui) -You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! +You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! \ No newline at end of file diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/utils.py b/packages/create-llama/templates/components/use-cases/python/document_generator/utils.py new file mode 100644 index 000000000..15eff6115 --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/utils.py @@ -0,0 +1,131 @@ +import json +import re +from typing import List, Optional, Any + +from pydantic import ValidationError +from llama_index.core.chat_ui.models.artifact import ( + Artifact, + ArtifactType, + CodeArtifactData, + DocumentArtifactData, +) +from llama_index.core.llms import ChatMessage + +INLINE_ANNOTATION_KEY = "annotation" + + +def get_inline_annotations(message: ChatMessage) -> List[Any]: + """Extract inline annotations from a chat message.""" + markdown_content = message.content + + inline_annotations: List[Any] = [] + + # Regex to match annotation code blocks + # Matches ```annotation followed by content until closing ``` + annotation_regex = re.compile( + rf"```{re.escape(INLINE_ANNOTATION_KEY)}\s*\n([\s\S]*?)\n```", re.MULTILINE + ) + + for match in annotation_regex.finditer(markdown_content): + json_content = match.group(1).strip() if match.group(1) else None + + if not json_content: + continue + + try: + # Parse the JSON content + parsed = json.loads(json_content) + + # Check for required fields in the parsed annotation + if ( + not isinstance(parsed, dict) + or "type" not in parsed + or "data" not in parsed + ): + continue + + # Extract the annotation data + inline_annotations.append(parsed) + except (json.JSONDecodeError, ValidationError) as error: + # Skip invalid annotations - they might be malformed JSON or invalid schema + print(f"Failed to parse annotation: {error}") + + return inline_annotations + + +def artifact_from_message(message: ChatMessage) -> Optional[Artifact]: + """Create an artifact from a chat message if it contains artifact annotations.""" + inline_annotations = get_inline_annotations(message) + + for annotation in inline_annotations: + if isinstance(annotation, dict) and annotation.get("type") == "artifact": + try: + # Create artifact data based on type + artifact_data = annotation.get("data") + if not artifact_data: + continue + + artifact_type = artifact_data.get("type") + + if artifact_type == "code": + # Get the nested data object that contains the actual code information + code_info = artifact_data.get("data", {}) + code_data = CodeArtifactData( + file_name=code_info.get("file_name", ""), + code=code_info.get("code", ""), + language=code_info.get("language", ""), + ) + artifact = Artifact( + created_at=artifact_data.get("created_at"), + type=ArtifactType.CODE, + data=code_data, + ) + elif artifact_type == "document": + # Get the nested data object that contains the actual document information + doc_info = artifact_data.get("data", {}) + doc_data = DocumentArtifactData( + title=doc_info.get("title", ""), + content=doc_info.get("content", ""), + type=doc_info.get("type", "markdown"), + sources=doc_info.get("sources"), + ) + artifact = Artifact( + created_at=artifact_data.get("created_at"), + type=ArtifactType.DOCUMENT, + data=doc_data, + ) + else: + continue + + return artifact + except Exception as e: + print( + f"Failed to parse artifact from annotation: {annotation}. Error: {e}" + ) + + return None + + +def get_artifacts(chat_history: List[ChatMessage]) -> List[Artifact]: + """ + Return a list of artifacts sorted by their creation time. + Artifacts without a creation time are placed at the end. + """ + artifacts = [] + + for message in chat_history: + artifact = artifact_from_message(message) + if artifact is not None: + artifacts.append(artifact) + + # Sort by creation time, with None values at the end + return sorted( + artifacts, + key=lambda a: (a.created_at is None, a.created_at), + ) + + +def get_last_artifact(chat_history: List[ChatMessage]) -> Optional[Artifact]: + """Get the last artifact from chat history.""" + artifacts = get_artifacts(chat_history) + return artifacts[-1] if len(artifacts) > 0 else None diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py b/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py index a20886381..951500e2c 100644 --- a/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py @@ -1,10 +1,9 @@ import re import time -from typing import Any, Literal, Optional +from typing import Any, Literal, Optional, Union -from llama_index.core.chat_engine.types import ChatMessage -from llama_index.core.llms import LLM -from llama_index.llms.openai import OpenAI +from llama_index.core import Settings +from llama_index.core.llms import LLM, ChatMessage from llama_index.core.memory import ChatMemoryBuffer from llama_index.core.prompts import PromptTemplate from llama_index.core.workflow import ( @@ -15,25 +14,20 @@ Workflow, step, ) -from llama_index.server.api.models import ( +from llama_index.core.chat_ui.models.artifact import ( Artifact, - ArtifactEvent, ArtifactType, - ChatRequest, - DocumentArtifactData, + CodeArtifactData, +) +from llama_index.core.chat_ui.events import ( UIEvent, + ArtifactEvent, ) -from llama_index.server.api.utils import get_last_artifact -from pydantic import BaseModel, Field - -def create_workflow(chat_request: ChatRequest) -> Workflow: - workflow = DocumentArtifactWorkflow( - llm=OpenAI(model="gpt-4.1"), - chat_request=chat_request, - timeout=120.0, - ) - return workflow +from src.utils import get_last_artifact +from src.settings import init_settings +from pydantic import BaseModel, Field +from dotenv import load_dotenv class DocumentRequirement(BaseModel): @@ -81,8 +75,6 @@ class DocumentArtifactWorkflow(Workflow): def __init__( self, - llm: LLM, - chat_request: ChatRequest, **kwargs: Any, ): """ @@ -91,9 +83,8 @@ def __init__( chat_request: The chat request from the chat app to use. """ super().__init__(**kwargs) - self.llm = llm - self.chat_request = chat_request - self.last_artifact = get_last_artifact(chat_request) + self.llm: LLM = Settings.llm + self.last_artifact: Optional[Artifact] = None @step async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> PlanEvent: @@ -101,13 +92,21 @@ async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> PlanEvent: if user_msg is None: raise ValueError("user_msg is required to run the workflow") await ctx.set("user_msg", user_msg) - chat_history = ev.chat_history or [] - chat_history.append( + + # prepare chat history from StartEvent + messages = [ ChatMessage( - role="user", - content=user_msg, + role=msg.get("role", "user"), + content=msg.get("content", ""), ) - ) + for msg in ev.get("chat_history", []) + ] + chat_history = [*messages, ChatMessage(role="user", content=user_msg)] + + # extract inline artifact from chat history + last_artifact = get_last_artifact(messages) + self.last_artifact = last_artifact + memory = ChatMemoryBuffer.from_defaults( chat_history=chat_history, llm=self.llm, @@ -345,3 +344,7 @@ async def synthesize_answer( ) ) return StopEvent(result=response_stream) + +load_dotenv() +init_settings() +workflow = DocumentArtifactWorkflow() \ No newline at end of file From d01876a82a7f7de1d81ec8860c3331bde977b930 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 14:52:53 +0700 Subject: [PATCH 10/80] fix doc --- .../use-cases/python/code_generator/README-template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md index d1d6d33a0..a5814b568 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md @@ -72,7 +72,7 @@ Note that the task_id and session_id are returned when creating a new task. ## Use Case AI-powered code generator that can help you generate app with a chat interface, code editor and app preview. -To update the workflow, you can modify the code in [`workflow.py`](app/workflow.py). +To update the workflow, you can modify the code in [`workflow.py`](src/workflow.py). ## Customize the UI From e25b63a0eb7aae3706aa2c618f9cd0a05de57169 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 14:56:19 +0700 Subject: [PATCH 11/80] fix format --- .../templates/components/ts-proxy/index.ts | 14 ++++---- .../python/code_generator/workflow.py | 32 ++++++++----------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/packages/create-llama/templates/components/ts-proxy/index.ts b/packages/create-llama/templates/components/ts-proxy/index.ts index 4169e22e2..791fd7b1d 100644 --- a/packages/create-llama/templates/components/ts-proxy/index.ts +++ b/packages/create-llama/templates/components/ts-proxy/index.ts @@ -1,14 +1,14 @@ -import { LlamaIndexServer } from '@llamaindex/server' +import { LlamaIndexServer } from "@llamaindex/server"; new LlamaIndexServer({ uiConfig: { - starterQuestions: ['Generate calculator app', 'Generate todo list app'], - componentsDir: 'components', - layoutDir: 'layout', + starterQuestions: ["Generate calculator app", "Generate todo list app"], + componentsDir: "components", + layoutDir: "layout", llamaDeploy: { - deployment: 'chat', - workflow: 'workflow', + deployment: "chat", + workflow: "workflow", }, }, port: 3000, -}).start() +}).start(); diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py b/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py index a0c8c84cb..67028e4f3 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py @@ -116,9 +116,9 @@ async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> PlanEvent: await ctx.set("memory", memory) return PlanEvent( user_msg=user_msg, - context=( - str(self.last_artifact.model_dump_json()) if self.last_artifact else "" - ), + context=str(self.last_artifact.model_dump_json()) + if self.last_artifact + else "", ) @step @@ -138,8 +138,7 @@ async def planning( ), ) ) - prompt = PromptTemplate( - """ + prompt = PromptTemplate(""" You are a product analyst responsible for analyzing the user's request and providing the next step for code or document generation. You are helping user with their code artifact. To update the code, you need to plan a coding step. @@ -193,13 +192,10 @@ async def planning( Now, plan the user's next step for this request: {user_msg} - """ - ).format( - context=( - "" - if event.context is None - else f"## The context is: \n{event.context}\n" - ), + """).format( + context="" + if event.context is None + else f"## The context is: \n{event.context}\n", user_msg=event.user_msg, ) response = await self.llm.acomplete( @@ -257,8 +253,7 @@ async def generate_artifact( ), ) ) - prompt = PromptTemplate( - """ + prompt = PromptTemplate(""" You are a skilled developer who can help user with coding. You are given a task to generate or update a code for a given requirement. @@ -305,11 +300,10 @@ async def generate_artifact( Now, i have to generate the code for the following requirement: {requirement} ``` - """ - ).format( - previous_artifact=( - self.last_artifact.model_dump_json() if self.last_artifact else "" - ), + """).format( + previous_artifact=self.last_artifact.model_dump_json() + if self.last_artifact + else "", requirement=event.requirement, ) response = await self.llm.acomplete( From 2846364b3fb77abbe662e8abdaee913a648a99a9 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 14:58:15 +0700 Subject: [PATCH 12/80] fix doc --- .../use-cases/python/code_generator/README-template.md | 2 +- .../use-cases/python/document_generator/README-template.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md index a5814b568..99973499e 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md @@ -16,7 +16,7 @@ If you don't have uv installed, you can follow the instructions [here](https://d ## Configure LLM and Embedding Model -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](src/settings.py). Remember to setup their respective API keys in the `src/.env` file. diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md index 9a739114f..691c59d03 100644 --- a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md @@ -16,7 +16,7 @@ If you don't have uv installed, you can follow the instructions [here](https://d ## Configure LLM and Embedding Model -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](src/settings.py). Remember to setup their respective API keys in the `src/.env` file. From d5edd2c533f9fed742045547a9b644fa34a8bcee Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 15:05:34 +0700 Subject: [PATCH 13/80] fix imports --- .../python/document_generator/workflow.py | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py b/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py index 951500e2c..8b6b5df12 100644 --- a/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py @@ -17,7 +17,7 @@ from llama_index.core.chat_ui.models.artifact import ( Artifact, ArtifactType, - CodeArtifactData, + DocumentArtifactData, ) from llama_index.core.chat_ui.events import ( UIEvent, @@ -114,9 +114,9 @@ async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> PlanEvent: await ctx.set("memory", memory) return PlanEvent( user_msg=user_msg, - context=str(self.last_artifact.model_dump_json()) - if self.last_artifact - else "", + context=( + str(self.last_artifact.model_dump_json()) if self.last_artifact else "" + ), ) @step @@ -134,7 +134,8 @@ async def planning(self, ctx: Context, event: PlanEvent) -> GenerateArtifactEven ), ) ) - prompt = PromptTemplate(""" + prompt = PromptTemplate( + """ You are a documentation analyst responsible for analyzing the user's request and providing requirements for document generation or update. Follow these instructions: 1. Carefully analyze the conversation history and the user's request to determine what has been done and what the next step should be. @@ -175,10 +176,13 @@ async def planning(self, ctx: Context, event: PlanEvent) -> GenerateArtifactEven Now, please plan for the user's request: {user_msg} - """).format( - context="" - if event.context is None - else f"## The context is: \n{event.context}\n", + """ + ).format( + context=( + "" + if event.context is None + else f"## The context is: \n{event.context}\n" + ), user_msg=event.user_msg, ) response = await self.llm.acomplete( @@ -231,7 +235,8 @@ async def generate_artifact( ), ) ) - prompt = PromptTemplate(""" + prompt = PromptTemplate( + """ You are a skilled technical writer who can help users with documentation. You are given a task to generate or update a document for a given requirement. @@ -264,10 +269,11 @@ async def generate_artifact( Now, please generate the document for the following requirement: {requirement} - """).format( - previous_artifact=self.last_artifact.model_dump_json() - if self.last_artifact - else "", + """ + ).format( + previous_artifact=( + self.last_artifact.model_dump_json() if self.last_artifact else "" + ), requirement=event.requirement, ) response = await self.llm.acomplete( @@ -345,6 +351,7 @@ async def synthesize_answer( ) return StopEvent(result=response_stream) + load_dotenv() init_settings() -workflow = DocumentArtifactWorkflow() \ No newline at end of file +workflow = DocumentArtifactWorkflow() From e2812edbef8491043ed852cb550cbca6989949dc Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 15:13:14 +0700 Subject: [PATCH 14/80] update ui config --- .../templates/components/ts-proxy/index.ts | 6 +----- .../python/code_generator/README-template.md | 16 +++++++++++++--- .../python/document_generator/README-template.md | 16 +++++++++++++--- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/create-llama/templates/components/ts-proxy/index.ts b/packages/create-llama/templates/components/ts-proxy/index.ts index 791fd7b1d..276cebff0 100644 --- a/packages/create-llama/templates/components/ts-proxy/index.ts +++ b/packages/create-llama/templates/components/ts-proxy/index.ts @@ -2,13 +2,9 @@ import { LlamaIndexServer } from "@llamaindex/server"; new LlamaIndexServer({ uiConfig: { - starterQuestions: ["Generate calculator app", "Generate todo list app"], componentsDir: "components", layoutDir: "layout", - llamaDeploy: { - deployment: "chat", - workflow: "workflow", - }, + llamaDeploy: { deployment: "chat", workflow: "workflow" }, }, port: 3000, }).start(); diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md index 99973499e..e19725602 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md @@ -16,7 +16,7 @@ If you don't have uv installed, you can follow the instructions [here](https://d ## Configure LLM and Embedding Model -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](src/settings.py). +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). Remember to setup their respective API keys in the `src/.env` file. @@ -72,11 +72,21 @@ Note that the task_id and session_id are returned when creating a new task. ## Use Case AI-powered code generator that can help you generate app with a chat interface, code editor and app preview. -To update the workflow, you can modify the code in [`workflow.py`](src/workflow.py). +To update the workflow, you can modify the code in [`src/workflow.py`](src/workflow.py). ## Customize the UI -To customize the UI, you can start by modifying the [./components/ui_event.jsx](./components/ui_event.jsx) file. +The UI is served by LLamaIndexServer package, you can configure the UI by modifying the `uiConfig` in the [ui/index.ts](ui/index.ts) file. + +The following are the available options: + +- `starterQuestions`: Predefined questions for chat interface +- `componentsDir`: Directory for custom event components +- `layoutDir`: Directory for custom layout components +- `llamaCloudIndexSelector`: Enable LlamaCloud integration +- `llamaDeploy`: The LlamaDeploy configration (deployment name and workflow name that defined in the [llama_deploy.yml](llama_deploy.yml) file) + +To customize the UI, you can start by modifying the [./ui/components/ui_event.jsx](./ui/components/ui_event.jsx) file. You can also generate a new code for the workflow using LLM by running the following command: diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md index 691c59d03..3df064589 100644 --- a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md @@ -16,7 +16,7 @@ If you don't have uv installed, you can follow the instructions [here](https://d ## Configure LLM and Embedding Model -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](src/settings.py). +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). Remember to setup their respective API keys in the `src/.env` file. @@ -72,11 +72,21 @@ Note that the task_id and session_id are returned when creating a new task. ## Use Case AI-powered document generator that can help you generate documents with a chat interface and simple markdown editor. -To update the workflow, you can modify the code in [`workflow.py`](src/workflow.py). +To update the workflow, you can modify the code in [`src/workflow.py`](src/workflow.py). ## Customize the UI -To customize the UI, you can start by modifying the [./components/ui_event.jsx](./components/ui_event.jsx) file. +The UI is served by LLamaIndexServer package, you can configure the UI by modifying the `uiConfig` in the [ui/index.ts](ui/index.ts) file. + +The following are the available options: + +- `starterQuestions`: Predefined questions for chat interface +- `componentsDir`: Directory for custom event components +- `layoutDir`: Directory for custom layout components +- `llamaCloudIndexSelector`: Enable LlamaCloud integration +- `llamaDeploy`: The LlamaDeploy configration (deployment name and workflow name that defined in the [llama_deploy.yml](llama_deploy.yml) file) + +To customize the UI, you can start by modifying the [./ui/components/ui_event.jsx](./ui/components/ui_event.jsx) file. You can also generate a new code for the workflow using LLM by running the following command: From 6451777c9f47d1a4bd190098bd5de4ace2465538 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 15:34:02 +0700 Subject: [PATCH 15/80] keep create workflow contract --- .../use-cases/python/code_generator/workflow.py | 9 ++++++--- .../use-cases/python/document_generator/workflow.py | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py b/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py index 67028e4f3..b9ee66141 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/workflow.py @@ -29,6 +29,11 @@ from pydantic import BaseModel, Field from dotenv import load_dotenv +def create_workflow() -> Workflow: + load_dotenv() + init_settings() + return CodeArtifactWorkflow(timeout=120.0) + class Requirement(BaseModel): next_step: Literal["answering", "coding"] @@ -374,6 +379,4 @@ async def synthesize_answer( return StopEvent(result=response_stream) -load_dotenv() -init_settings() -workflow = CodeArtifactWorkflow() +workflow = create_workflow() diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py b/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py index 8b6b5df12..c5c86afc8 100644 --- a/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/workflow.py @@ -29,6 +29,10 @@ from pydantic import BaseModel, Field from dotenv import load_dotenv +def create_workflow() -> Workflow: + load_dotenv() + init_settings() + return DocumentArtifactWorkflow(timeout=120.0) class DocumentRequirement(BaseModel): type: Literal["markdown", "html"] @@ -352,6 +356,4 @@ async def synthesize_answer( return StopEvent(result=response_stream) -load_dotenv() -init_settings() -workflow = DocumentArtifactWorkflow() +workflow = create_workflow() From b0e4c60f217d82e6b512582c14a1aae7d6837184 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 15:53:34 +0700 Subject: [PATCH 16/80] agentic rag --- .../python/agentic_rag/README-template.md | 103 ++++++++++++----- .../use-cases/python/agentic_rag/citation.py | 106 ++++++++++++++++++ .../use-cases/python/agentic_rag/query.py | 47 ++++++++ .../use-cases/python/agentic_rag/workflow.py | 19 ++-- .../llamaindexserver/fastapi/src/index.py | 8 +- 5 files changed, 242 insertions(+), 41 deletions(-) create mode 100644 packages/create-llama/templates/components/use-cases/python/agentic_rag/citation.py create mode 100644 packages/create-llama/templates/components/use-cases/python/agentic_rag/query.py diff --git a/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md b/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md index 2268407cd..aee4d14ff 100644 --- a/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md @@ -1,59 +1,112 @@ -This is a [LlamaIndex](https://www.llamaindex.ai/) simple agentic RAG project using [Agent Workflows](https://docs.llamaindex.ai/en/stable/examples/agent/agent_workflow_basic/). +# LlamaIndex Workflow Example -## Getting Started +This is a [LlamaIndex](https://www.llamaindex.ai/) project that using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/) deployed with [LlamaDeploy](https://github.com/run-llama/llama_deploy). -First, setup the environment with uv: +LlamaDeploy is a system for deploying and managing LlamaIndex workflows, while LlamaIndexServer provides a pre-built TypeScript server with an integrated chat UI that can connect directly to LlamaDeploy deployments. This example shows how you can quickly set up a complete chat application by combining these two technologies/ -> **_Note:_** This step is not needed if you are using the dev-container. +## Installation -```shell +Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, just run: + +```bash uv sync ``` -Then check the parameters that have been pre-configured in the `.env` file in this directory. -Make sure you have set the `OPENAI_API_KEY` for the LLM. +If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). + +## Configure LLM and Embedding Model + +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). + +Remember to setup their respective API keys in the `src/.env` file. + +## Generate Index -Second, generate the embeddings of the documents in the `./data` directory: +Generate the embeddings of the documents in the `./data` directory: ```shell uv run generate ``` -Third, run the development server: +## Running the Deployment -```shell -uv run fastapi dev -``` +At this point we have all we need to run this deployment. Ideally, we would have the API server already running +somewhere in the cloud, but to get started let's start an instance locally. Run the following python script +from a shell: -Then open [http://localhost:8000](http://localhost:8000) with your browser to start the chat UI. +``` +$ uv run -m llama_deploy.apiserver +INFO: Started server process [10842] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:4501 (Press CTRL+C to quit) +``` -To start the app optimized for **production**, run: +From another shell, use the CLI, `llamactl`, to create the deployment: ``` -uv run fastapi run +$ uv run llamactl deploy llama_deploy.yml +Deployment successful: chat ``` -## Configure LLM and Embedding Model +## UI Interface + +LlamaDeploy will serve the UI through the apiserver. Point the browser to [http://localhost:4501/deployments/chat/ui](http://localhost:4501/deployments/chat/ui) to interact with your deployment through a user-friendly interface. + +## API endpoints -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). +You can find all the endpoints in the [API documentation](http://localhost:4501/docs). To get started, you can try the following endpoints: + +Create a new task: + +```bash +curl -X POST 'http://localhost:4501/deployments/chat/tasks/create' \ + -H 'Content-Type: application/json' \ + -d '{ + "input": "{\"user_msg\":\"Hello\",\"chat_history\":[]}", + "service_id": "workflow" + }' +``` + +Stream events: + +```bash +curl 'http://localhost:4501/deployments/chat/tasks/0b411be6-005d-43f0-9b6b-6a0017f08002/events?session_id=dd36442c-45ca-4eaa-8d75-b4e6dad1a83e&raw_event=true' \ + -H 'Content-Type: application/json' +``` + +Note that the task_id and session_id are returned when creating a new task. ## Use Case -We have prepared an [example workflow](./app/workflow.py) for the agentic RAG use case, where you can ask questions about the example documents in the [./data](./data) directory. +We have prepared an [example workflow](./src/workflow.py) for the agentic RAG use case, where you can ask questions about the example documents in the [./data](./data) directory. +To update the workflow, you can modify the code in [`src/workflow.py`](src/workflow.py). -You can start by sending an request on the [chat UI](http://localhost:8000) or you can test the `/api/chat` endpoint with the following curl request: +## Customize the UI + +The UI is served by LLamaIndexServer package, you can configure the UI by modifying the `uiConfig` in the [ui/index.ts](ui/index.ts) file. + +The following are the available options: + +- `starterQuestions`: Predefined questions for chat interface +- `componentsDir`: Directory for custom event components +- `layoutDir`: Directory for custom layout components +- `llamaCloudIndexSelector`: Enable LlamaCloud integration +- `llamaDeploy`: The LlamaDeploy configration (deployment name and workflow name that defined in the [llama_deploy.yml](llama_deploy.yml) file) + +To customize the UI, you can start by modifying the [./ui/components/ui_event.jsx](./ui/components/ui_event.jsx) file. + +You can also generate a new code for the workflow using LLM by running the following command: ``` -curl --location 'localhost:8000/api/chat' \ ---header 'Content-Type: application/json' \ ---data '{ "messages": [{ "role": "user", "content": "What standards for a letter exist?" }] }' +uv run generate_ui ``` ## Learn More -To learn more about LlamaIndex, take a look at the following resources: - - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. - [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows. +- [LlamaDeploy GitHub Repository](https://github.com/run-llama/llama_deploy) +- [Chat-UI Documentation](https://ts.llamaindex.ai/docs/chat-ui) -You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! +You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! \ No newline at end of file diff --git a/packages/create-llama/templates/components/use-cases/python/agentic_rag/citation.py b/packages/create-llama/templates/components/use-cases/python/agentic_rag/citation.py new file mode 100644 index 000000000..7193ab254 --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/agentic_rag/citation.py @@ -0,0 +1,106 @@ +from typing import Any, List, Optional + +from llama_index.core import QueryBundle +from llama_index.core.postprocessor.types import BaseNodePostprocessor +from llama_index.core.prompts import PromptTemplate +from llama_index.core.query_engine.retriever_query_engine import RetrieverQueryEngine +from llama_index.core.response_synthesizers import Accumulate +from llama_index.core.schema import NodeWithScore +from llama_index.core.tools.query_engine import QueryEngineTool + + +# Used as a prompt for synthesizer +# Override this prompt by setting the `CITATION_PROMPT` environment variable +CITATION_PROMPT = """ +Context information is below. +------------------ +{context_str} +------------------ +The context are multiple text chunks, each text chunk has its own citation_id at the beginning. +Use the citation_id for citation construction. + +Answer the following query with citations: +------------------ +{query_str} +------------------ + +## Citation format + +[citation:id] + +Where: +- [citation:] is a matching pattern which is required for all citations. +- `id` is the `citation_id` provided in the context or previous response. + +Example: +``` + Here is a response that uses context information [citation:90ca859f-4f32-40ca-8cd0-edfad4fb298b] + and other ideas that don't use context information [citation:17b2cc9a-27ae-4b6d-bede-5ca60fc00ff4] .\n + The citation block will be displayed automatically with useful information for the user in the UI [citation:1c606612-e75f-490e-8374-44e79f818d19] . +``` + +## Requirements: +1. Always include citations for every fact from the context information in your response. +2. Make sure that the citation_id is correct with the context, don't mix up the citation_id with other information. + +Now, you answer the query with citations: +""" + + +class NodeCitationProcessor(BaseNodePostprocessor): + """ + Add a new field `citation_id` to the metadata of the node by copying the id from the node. + Useful for citation construction. + """ + + def _postprocess_nodes( + self, + nodes: List[NodeWithScore], + query_bundle: Optional[QueryBundle] = None, + ) -> List[NodeWithScore]: + for node_score in nodes: + node_score.node.metadata["citation_id"] = node_score.node.node_id + return nodes + + +class CitationSynthesizer(Accumulate): + """ + Overload the Accumulate synthesizer to: + 1. Update prepare node metadata for citation id + 2. Update text_qa_template to include citations + """ + + def __init__(self, **kwargs: Any) -> None: + text_qa_template = kwargs.pop("text_qa_template", None) + if text_qa_template is None: + text_qa_template = PromptTemplate(template=CITATION_PROMPT) + super().__init__(text_qa_template=text_qa_template, **kwargs) + + +# Add this prompt to your agent system prompt +CITATION_SYSTEM_PROMPT = ( + "\nAnswer the user question using the response from the query tool. " + "It's important to respect the citation information in the response. " + "Don't mix up the citation_id, keep them at the correct fact." +) + + +def enable_citation(query_engine_tool: QueryEngineTool) -> QueryEngineTool: + """ + Enable citation for a query engine tool by using CitationSynthesizer and NodePostprocessor. + Note: This function will override the response synthesizer of your query engine. + """ + query_engine = query_engine_tool.query_engine + if not isinstance(query_engine, RetrieverQueryEngine): + raise ValueError( + "Citation feature requires a RetrieverQueryEngine. Your tool's query engine is a " + f"{type(query_engine)}." + ) + # Update the response synthesizer and node postprocessors + query_engine._response_synthesizer = CitationSynthesizer() + query_engine._node_postprocessors += [NodeCitationProcessor()] + query_engine_tool._query_engine = query_engine + + # Update tool metadata + query_engine_tool.metadata.description += "\nThe output will include citations with the format [citation:id] for each chunk of information in the knowledge base." + return query_engine_tool diff --git a/packages/create-llama/templates/components/use-cases/python/agentic_rag/query.py b/packages/create-llama/templates/components/use-cases/python/agentic_rag/query.py new file mode 100644 index 000000000..62c59240f --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/agentic_rag/query.py @@ -0,0 +1,47 @@ +import os +from typing import Any, Optional + +from llama_index.core.base.base_query_engine import BaseQueryEngine +from llama_index.core.indices.base import BaseIndex +from llama_index.core.tools.query_engine import QueryEngineTool + +def create_query_engine(index: BaseIndex, **kwargs: Any) -> BaseQueryEngine: + """ + Create a query engine for the given index. + + Args: + index: The index to create a query engine for. + params (optional): Additional parameters for the query engine, e.g: similarity_top_k + """ + top_k = int(os.getenv("TOP_K", 0)) + if top_k != 0 and kwargs.get("filters") is None: + kwargs["similarity_top_k"] = top_k + + return index.as_query_engine(**kwargs) + + +def get_query_engine_tool( + index: BaseIndex, + name: Optional[str] = None, + description: Optional[str] = None, + **kwargs: Any, +) -> QueryEngineTool: + """ + Get a query engine tool for the given index. + + Args: + index: The index to create a query engine for. + name (optional): The name of the tool. + description (optional): The description of the tool. + """ + if name is None: + name = "query_index" + if description is None: + description = "Use this tool to retrieve information from a knowledge base. Provide a specific query and can call the tool multiple times if necessary." + query_engine = create_query_engine(index, **kwargs) + tool = QueryEngineTool.from_defaults( + query_engine=query_engine, + name=name, + description=description, + ) + return tool diff --git a/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py b/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py index 4bea9e0a1..1f7bd1052 100644 --- a/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py @@ -1,18 +1,12 @@ -from typing import Optional - -from app.index import get_index +from src.index import get_index from llama_index.core.agent.workflow import AgentWorkflow from llama_index.core.settings import Settings -from llama_index.server.api.models import ChatRequest -from llama_index.server.tools.index import get_query_engine_tool -from llama_index.server.tools.index.citation import ( - CITATION_SYSTEM_PROMPT, - enable_citation, -) +from src.query import get_query_engine_tool +from src.citation import CITATION_SYSTEM_PROMPT, enable_citation -def create_workflow(chat_request: Optional[ChatRequest] = None) -> AgentWorkflow: - index = get_index(chat_request=chat_request) +def create_workflow() -> AgentWorkflow: + index = get_index() if index is None: raise RuntimeError( "Index not found! Please run `uv run generate` to index the data first." @@ -30,3 +24,6 @@ def create_workflow(chat_request: Optional[ChatRequest] = None) -> AgentWorkflow llm=Settings.llm, system_prompt=system_prompt, ) + + +workflow = create_workflow() diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py index 196ab1ef2..fa350c2e6 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py @@ -1,23 +1,21 @@ import logging import os -from typing import Optional from llama_index.core.indices import load_index_from_storage -from llama_index.server.api.models import ChatRequest -from llama_index.server.tools.index.utils import get_storage_context +from llama_index.core.storage import StorageContext logger = logging.getLogger("uvicorn") STORAGE_DIR = "storage" -def get_index(chat_request: Optional[ChatRequest] = None): +def get_index(): # check if storage already exists if not os.path.exists(STORAGE_DIR): return None # load the existing index logger.info(f"Loading index from {STORAGE_DIR}...") - storage_context = get_storage_context(STORAGE_DIR) + storage_context = StorageContext.from_defaults(persist_dir=STORAGE_DIR) index = load_index_from_storage(storage_context) logger.info(f"Finished loading index from {STORAGE_DIR}") return index From bb6bebc5fc5a8aaaabef36dec860900e9ca74bd8 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 16:30:36 +0700 Subject: [PATCH 17/80] fix generate --- .../llamaindexserver/fastapi/pyproject.toml | 31 +++++++++++++--- .../llamaindexserver/fastapi/src/generate.py | 36 ++----------------- .../llamaindexserver/fastapi/src/index.py | 2 +- 3 files changed, 30 insertions(+), 39 deletions(-) diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml index 4064484d0..06b81f846 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml @@ -26,9 +26,7 @@ dev = [ ] [project.scripts] -generate = "src.generate:generate_index" -generate_index = "src.generate:generate_index" -generate_ui = "src.generate:generate_ui_for_workflow" +generate = "generate:generate_index" [tool.mypy] @@ -43,4 +41,29 @@ ignore_missing_imports = true follow_imports = "silent" implicit_optional = true strict_optional = false -disable_error_code = [ "return-value", "assignment" ] \ No newline at end of file +disable_error_code = [ "return-value", "assignment" ] + +[[tool.mypy.overrides]] +module = "src.*" +ignore_missing_imports = false + +[tool.hatch.metadata] +allow-direct-references = true + +[build-system] +requires = [ "hatchling>=1.24" ] +build-backend = "hatchling.build" + +[[tool.mypy.overrides]] +module = "src.*" +ignore_missing_imports = false + +[tool.hatch.metadata] +allow-direct-references = true + +[build-system] +requires = [ "hatchling>=1.24" ] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/generate.py"] \ No newline at end of file diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py index a41fc6b88..7bc5a2fdb 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py @@ -11,8 +11,8 @@ def generate_index(): """ Index the documents in the data directory. """ - from app.index import STORAGE_DIR - from app.settings import init_settings + from index import STORAGE_DIR + from settings import init_settings from llama_index.core.indices import ( VectorStoreIndex, ) @@ -35,35 +35,3 @@ def generate_index(): # store it for later index.storage_context.persist(STORAGE_DIR) logger.info(f"Finished creating new index. Stored in {STORAGE_DIR}") - - -def generate_ui_for_workflow(): - """ - Generate UI for UIEventData event in app/workflow.py - """ - import asyncio - - from app.settings import init_settings - from llama_index.core.settings import Settings - from main import COMPONENT_DIR - - load_dotenv() - init_settings() - - # To generate UI components for additional event types, - # import the corresponding data model (e.g., MyCustomEventData) - # and run the generate_ui_for_workflow function with the imported model. - # Make sure the output filename of the generated UI component matches the event type (here `ui_event`) - try: - from app.workflow import UIEventData # type: ignore - except ImportError: - raise ImportError("Couldn't generate UI component for the current workflow.") - from llama_index.server.gen_ui import generate_event_component - # TODO: remove llama_index.server - - # works well with OpenAI gpt-4.1, Claude 3.7 Sonnet or Gemini Pro 2.5 - code = asyncio.run( - generate_event_component(event_cls=UIEventData, llm=Settings.llm) - ) - with open(f"{COMPONENT_DIR}/ui_event.jsx", "w") as f: - f.write(code) diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py index fa350c2e6..d12af23f5 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/index.py @@ -6,7 +6,7 @@ logger = logging.getLogger("uvicorn") -STORAGE_DIR = "storage" +STORAGE_DIR = "src/storage" def get_index(): From f5bc6c4c60d88c30b49a33b519d87511b1ee77e2 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 16:42:40 +0700 Subject: [PATCH 18/80] prerequisites --- .../python/agentic_rag/README-template.md | 14 ++++++++------ .../python/code_generator/README-template.md | 15 +++++++++------ .../python/document_generator/README-template.md | 14 ++++++++------ 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md b/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md index aee4d14ff..25c1d3065 100644 --- a/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md @@ -4,6 +4,14 @@ This is a [LlamaIndex](https://www.llamaindex.ai/) project that using [Workflows LlamaDeploy is a system for deploying and managing LlamaIndex workflows, while LlamaIndexServer provides a pre-built TypeScript server with an integrated chat UI that can connect directly to LlamaDeploy deployments. This example shows how you can quickly set up a complete chat application by combining these two technologies/ +## Prerequisites + +If you haven't installed uv, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/) to install it. + +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). + +Please setup their API keys in the `src/.env` file. + ## Installation Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, just run: @@ -14,12 +22,6 @@ uv sync If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). -## Configure LLM and Embedding Model - -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). - -Remember to setup their respective API keys in the `src/.env` file. - ## Generate Index Generate the embeddings of the documents in the `./data` directory: diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md index e19725602..893bb9343 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md @@ -4,6 +4,15 @@ This is a [LlamaIndex](https://www.llamaindex.ai/) project that using [Workflows LlamaDeploy is a system for deploying and managing LlamaIndex workflows, while LlamaIndexServer provides a pre-built TypeScript server with an integrated chat UI that can connect directly to LlamaDeploy deployments. This example shows how you can quickly set up a complete chat application by combining these two technologies/ +## Prerequisites + +If you haven't installed uv, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/) to install it. + +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). + +Please setup their API keys in the `src/.env` file. + + ## Installation Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, just run: @@ -14,12 +23,6 @@ uv sync If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). -## Configure LLM and Embedding Model - -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). - -Remember to setup their respective API keys in the `src/.env` file. - ## Running the Deployment At this point we have all we need to run this deployment. Ideally, we would have the API server already running diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md index 3df064589..73cdbdaa3 100644 --- a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md @@ -4,6 +4,14 @@ This is a [LlamaIndex](https://www.llamaindex.ai/) project that using [Workflows LlamaDeploy is a system for deploying and managing LlamaIndex workflows, while LlamaIndexServer provides a pre-built TypeScript server with an integrated chat UI that can connect directly to LlamaDeploy deployments. This example shows how you can quickly set up a complete chat application by combining these two technologies/ +## Prerequisites + +If you haven't installed uv, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/) to install it. + +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). + +Please setup their API keys in the `src/.env` file. + ## Installation Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, just run: @@ -14,12 +22,6 @@ uv sync If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). -## Configure LLM and Embedding Model - -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). - -Remember to setup their respective API keys in the `src/.env` file. - ## Running the Deployment At this point we have all we need to run this deployment. Ideally, we would have the API server already running From 07cea8cb0aec711bf59fcc3352f35c08b320687d Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 16:47:05 +0700 Subject: [PATCH 19/80] fix pyproject.toml --- .../llamaindexserver/fastapi/pyproject.toml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml index 06b81f846..4ffc7626b 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml @@ -52,18 +52,4 @@ allow-direct-references = true [build-system] requires = [ "hatchling>=1.24" ] -build-backend = "hatchling.build" - -[[tool.mypy.overrides]] -module = "src.*" -ignore_missing_imports = false - -[tool.hatch.metadata] -allow-direct-references = true - -[build-system] -requires = [ "hatchling>=1.24" ] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/generate.py"] \ No newline at end of file +build-backend = "hatchling.build" \ No newline at end of file From 4beefa831181741cbefec90cadfaa4c3f9462606 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 16:51:42 +0700 Subject: [PATCH 20/80] fix typo --- .../components/use-cases/python/agentic_rag/README-template.md | 2 +- .../use-cases/python/code_generator/README-template.md | 2 +- .../use-cases/python/document_generator/README-template.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md b/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md index 25c1d3065..23cd2623f 100644 --- a/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md @@ -10,7 +10,7 @@ If you haven't installed uv, you can follow the instructions [here](https://docs You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). -Please setup their API keys in the `src/.env` file. +Please setup their API keys in the `src/.env` file. ## Installation diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md index 893bb9343..dc38c7d96 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md @@ -10,7 +10,7 @@ If you haven't installed uv, you can follow the instructions [here](https://docs You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). -Please setup their API keys in the `src/.env` file. +Please setup their API keys in the `src/.env` file. ## Installation diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md index 73cdbdaa3..203020f73 100644 --- a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md @@ -10,7 +10,7 @@ If you haven't installed uv, you can follow the instructions [here](https://docs You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). -Please setup their API keys in the `src/.env` file. +Please setup their API keys in the `src/.env` file. ## Installation From c61dfe63ffa267710b1d47777783beea5668792a Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 17:00:52 +0700 Subject: [PATCH 21/80] fix: define wheel --- .../templates/types/llamaindexserver/fastapi/pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml index 4ffc7626b..4a020bbcb 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml @@ -52,4 +52,7 @@ allow-direct-references = true [build-system] requires = [ "hatchling>=1.24" ] -build-backend = "hatchling.build" \ No newline at end of file +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/app"] \ No newline at end of file From 2b202a574a57851084a6d757b693d4d99c5f7434 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 17:19:31 +0700 Subject: [PATCH 22/80] fix generate --- .../templates/types/llamaindexserver/fastapi/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml index 4a020bbcb..da9e5a5e4 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml @@ -55,4 +55,4 @@ requires = [ "hatchling>=1.24" ] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["src/app"] \ No newline at end of file +packages = [ "src/generate.py" ] \ No newline at end of file From 7a165c4004c2e63a2428b0157679a65dd0c5be48 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 3 Jul 2025 17:24:47 +0700 Subject: [PATCH 23/80] fix rag worklow --- .../components/use-cases/python/agentic_rag/workflow.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py b/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py index 1f7bd1052..1538640b7 100644 --- a/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py @@ -4,8 +4,13 @@ from src.query import get_query_engine_tool from src.citation import CITATION_SYSTEM_PROMPT, enable_citation +from dotenv import load_dotenv +from src.settings import init_settings + def create_workflow() -> AgentWorkflow: + load_dotenv() + init_settings() index = get_index() if index is None: raise RuntimeError( From 11052af6f4676a2cec36576f7ea17dd01de6f0ca Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 11:18:30 +0700 Subject: [PATCH 24/80] feat: HITL use case --- .../use-cases/python/hitl/README-template.md | 130 ++++++++++++------ .../use-cases/python/hitl/events.py | 10 +- .../use-cases/python/hitl/workflow.py | 7 +- 3 files changed, 93 insertions(+), 54 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md b/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md index 65be6f62e..f7e2f989c 100644 --- a/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md @@ -1,50 +1,90 @@ -This is a [LlamaIndex](https://www.llamaindex.ai/) project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/). +# LlamaIndex Workflow Example -## Getting Started +This is a [LlamaIndex](https://www.llamaindex.ai/) project that using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/) deployed with [LlamaDeploy](https://github.com/run-llama/llama_deploy). -First, setup the environment with uv: +LlamaDeploy is a system for deploying and managing LlamaIndex workflows, while LlamaIndexServer provides a pre-built TypeScript server with an integrated chat UI that can connect directly to LlamaDeploy deployments. This example shows how you can quickly set up a complete chat application by combining these two technologies/ -> **_Note:_** This step is not needed if you are using the dev-container. +## Prerequisites -```shell +If you haven't installed uv, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/) to install it. + +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). + +Please setup their API keys in the `src/.env` file. + +## Installation + +Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, just run: + +```bash uv sync ``` -Then check the parameters that have been pre-configured in the `.env` file in this directory. -Make sure you have set the `OPENAI_API_KEY` for the LLM. +If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). -Then, run the development server: +## Generate Index + +Generate the embeddings of the documents in the `./data` directory: ```shell -uv run fastapi dev +uv run generate ``` -Then open [http://localhost:8000](http://localhost:8000) with your browser to start the chat UI. +## Running the Deployment -To start the app optimized for **production**, run: +At this point we have all we need to run this deployment. Ideally, we would have the API server already running +somewhere in the cloud, but to get started let's start an instance locally. Run the following python script +from a shell: ``` -uv run fastapi run +$ uv run -m llama_deploy.apiserver +INFO: Started server process [10842] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:4501 (Press CTRL+C to quit) ``` -## Configure LLM and Embedding Model +From another shell, use the CLI, `llamactl`, to create the deployment: + +``` +$ uv run llamactl deploy llama_deploy.yml +Deployment successful: chat +``` -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). +## UI Interface -## Use Case +LlamaDeploy will serve the UI through the apiserver. Point the browser to [http://localhost:4501/deployments/chat/ui](http://localhost:4501/deployments/chat/ui) to interact with your deployment through a user-friendly interface. -This example shows how to use the LlamaIndexServer with a human in the loop. It allows you to start CLI commands that are reviewed by a human before execution. +## API endpoints -To update the workflow, you can modify the code in [`workflow.py`](app/workflow.py). +You can find all the endpoints in the [API documentation](http://localhost:4501/docs). To get started, you can try the following endpoints: -You can start by sending an request on the [chat UI](http://localhost:8000) or you can test the `/api/chat` endpoint with the following curl request: +Create a new task: +```bash +curl -X POST 'http://localhost:4501/deployments/chat/tasks/create' \ + -H 'Content-Type: application/json' \ + -d '{ + "input": "{\"user_msg\":\"Hello\",\"chat_history\":[]}", + "service_id": "workflow" + }' ``` -curl --location 'localhost:8000/api/chat' \ ---header 'Content-Type: application/json' \ ---data '{ "messages": [{ "role": "user", "content": "Show me the files in the current directory" }] }' + +Stream events: + +```bash +curl 'http://localhost:4501/deployments/chat/tasks/0b411be6-005d-43f0-9b6b-6a0017f08002/events?session_id=dd36442c-45ca-4eaa-8d75-b4e6dad1a83e&raw_event=true' \ + -H 'Content-Type: application/json' ``` +Note that the task_id and session_id are returned when creating a new task. + +## Use Case + +This example shows how to use the LlamaIndexServer with a human in the loop. It allows you to start CLI commands that are reviewed by a human before execution. + +To update the workflow, you can modify the code in [`src/workflow.py`](src/workflow.py). + ## How does HITL work? ### Events @@ -53,30 +93,17 @@ The human-in-the-loop approach used here is based on a simple idea: the workflow To do this, you will need to implement two custom events: -- [HumanInputEvent](https://github.com/run-llama/create-llama/blob/main/packages/server/src/utils/hitl/events.ts): This event is used to request input from the user. -- [HumanResponseEvent](https://github.com/run-llama/create-llama/blob/main/packages/server/src/utils/hitl/events.ts): This event is sent to the workflow to resume execution with input from the user. +- [HumanInputEvent](src/events.py): This event is used to request input from the user. +- [HumanResponseEvent](src/events.py): This event is sent to the workflow to resume execution with input from the user. -In this example, we have implemented these two custom events in [`events.ts`](src/app/events.ts): +In this example, we have implemented these two custom events in [`events.py`](src/events.py): - `cliHumanInputEvent` – to request input from the user for CLI command execution. - `cliHumanResponseEvent` – to resume the workflow with the response from the user. -```typescript -export const cliHumanInputEvent = humanInputEvent<{ - type: "cli_human_input"; - data: { command: string }; - response: typeof cliHumanResponseEvent; -}>(); - -export const cliHumanResponseEvent = humanResponseEvent<{ - type: "human_response"; - data: { execute: boolean; command: string }; -}>(); -``` - ### UI Component -HITL also needs a custom UI component, that is shown when the LlamaIndexServer receives the `cliHumanInputEvent`. The name of the component is defined in the `type` field of the `cliHumanInputEvent` - in our case, it is `cli_human_input`, which corresponds to the [cli_human_input.tsx](./components/cli_human_input.tsx) component. +HITL also needs a custom UI component, that is shown when the LlamaIndexServer receives the `cliHumanInputEvent`. The name of the component is defined in the `type` field of the `cliHumanInputEvent` - in our case, it is `cli_human_input`, which corresponds to the [cli_human_input.tsx](./ui/components/cli_human_input.tsx) component. The custom component must use `append` to send a message with a `human_response` annotation. The data of the annotation must be in the format of the response event `cliHumanResponseEvent`, in our case, for sending to execute the command `ls -l`, we would send: @@ -98,12 +125,31 @@ append({ This component displays the command to execute and the user can choose to execute or cancel the command execution. -## Learn More +## Customize the UI + +The UI is served by LLamaIndexServer package, you can configure the UI by modifying the `uiConfig` in the [ui/index.ts](ui/index.ts) file. -To learn more about LlamaIndex, take a look at the following resources: +The following are the available options: + +- `starterQuestions`: Predefined questions for chat interface +- `componentsDir`: Directory for custom event components +- `layoutDir`: Directory for custom layout components +- `llamaCloudIndexSelector`: Enable LlamaCloud integration +- `llamaDeploy`: The LlamaDeploy configration (deployment name and workflow name that defined in the [llama_deploy.yml](llama_deploy.yml) file) + +To customize the UI, you can start by modifying the [./ui/components/ui_event.jsx](./ui/components/ui_event.jsx) file. + +You can also generate a new code for the workflow using LLM by running the following command: + +``` +uv run generate_ui +``` + +## Learn More - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. - [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows. -- [LlamaIndex Server](https://pypi.org/project/llama-index-server/) +- [LlamaDeploy GitHub Repository](https://github.com/run-llama/llama_deploy) +- [Chat-UI Documentation](https://ts.llamaindex.ai/docs/chat-ui) -You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! +You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! \ No newline at end of file diff --git a/packages/create-llama/templates/components/use-cases/python/hitl/events.py b/packages/create-llama/templates/components/use-cases/python/hitl/events.py index bb86a5289..fbba17b91 100644 --- a/packages/create-llama/templates/components/use-cases/python/hitl/events.py +++ b/packages/create-llama/templates/components/use-cases/python/hitl/events.py @@ -1,9 +1,6 @@ -from typing import Type - from pydantic import BaseModel, Field -from llama_index.server.models import HumanInputEvent, HumanResponseEvent - +from llama_index.core.workflow.events import HumanResponseEvent, InputRequiredEvent class CLIHumanResponseEvent(HumanResponseEvent): execute: bool = Field( @@ -17,7 +14,7 @@ class CLICommand(BaseModel): # We need an event that extends from HumanInputEvent for HITL feature -class CLIHumanInputEvent(HumanInputEvent): +class CLIHumanInputEvent(InputRequiredEvent): """ CLIInputRequiredEvent is sent when the agent needs permission from the user to execute the CLI command or not. Render this event by showing the command and a boolean button to execute the command or not. @@ -26,9 +23,6 @@ class CLIHumanInputEvent(HumanInputEvent): event_type: str = ( "cli_human_input" # used by UI to render with appropriate component ) - response_event_type: Type = ( - CLIHumanResponseEvent # used by workflow to resume with the correct event - ) data: CLICommand = Field( # the data that sent to the UI for rendering description="The command to execute.", ) diff --git a/packages/create-llama/templates/components/use-cases/python/hitl/workflow.py b/packages/create-llama/templates/components/use-cases/python/hitl/workflow.py index f4c622849..23d78db28 100644 --- a/packages/create-llama/templates/components/use-cases/python/hitl/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/hitl/workflow.py @@ -15,10 +15,6 @@ ) -def create_workflow() -> Workflow: - return CLIWorkflow() - - class CLIWorkflow(Workflow): """ A workflow has ability to execute command line tool with human in the loop for confirmation. @@ -85,3 +81,6 @@ async def handle_human_response( return StopEvent(result=res.stdout or res.stderr) else: return StopEvent(result=None) + + +workflow = CLIWorkflow() \ No newline at end of file From a434e34a81fbd598dc163ad5280e95e512ff2d38 Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 11:21:13 +0700 Subject: [PATCH 25/80] update doc --- .../components/use-cases/python/hitl/README-template.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md b/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md index f7e2f989c..b82bc127c 100644 --- a/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md @@ -22,14 +22,6 @@ uv sync If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). -## Generate Index - -Generate the embeddings of the documents in the `./data` directory: - -```shell -uv run generate -``` - ## Running the Deployment At this point we have all we need to run this deployment. Ideally, we would have the API server already running From 72ea59be3b3a63693bcac7d16ee0f40610f4cc21 Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 11:22:39 +0700 Subject: [PATCH 26/80] update doc --- .../use-cases/python/agentic_rag/README-template.md | 8 -------- .../use-cases/python/code_generator/README-template.md | 8 -------- .../use-cases/python/deep_research/README-template.md | 6 ------ .../python/document_generator/README-template.md | 6 ------ .../components/use-cases/python/hitl/README-template.md | 6 ------ 5 files changed, 34 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md b/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md index 23cd2623f..211d8f543 100644 --- a/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/agentic_rag/README-template.md @@ -96,14 +96,6 @@ The following are the available options: - `llamaCloudIndexSelector`: Enable LlamaCloud integration - `llamaDeploy`: The LlamaDeploy configration (deployment name and workflow name that defined in the [llama_deploy.yml](llama_deploy.yml) file) -To customize the UI, you can start by modifying the [./ui/components/ui_event.jsx](./ui/components/ui_event.jsx) file. - -You can also generate a new code for the workflow using LLM by running the following command: - -``` -uv run generate_ui -``` - ## Learn More - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. diff --git a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md index dc38c7d96..725fbe871 100644 --- a/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/code_generator/README-template.md @@ -89,14 +89,6 @@ The following are the available options: - `llamaCloudIndexSelector`: Enable LlamaCloud integration - `llamaDeploy`: The LlamaDeploy configration (deployment name and workflow name that defined in the [llama_deploy.yml](llama_deploy.yml) file) -To customize the UI, you can start by modifying the [./ui/components/ui_event.jsx](./ui/components/ui_event.jsx) file. - -You can also generate a new code for the workflow using LLM by running the following command: - -``` -uv run generate_ui -``` - ## Learn More - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. diff --git a/packages/create-llama/templates/components/use-cases/python/deep_research/README-template.md b/packages/create-llama/templates/components/use-cases/python/deep_research/README-template.md index 8f49a4ef3..fc4b532a2 100644 --- a/packages/create-llama/templates/components/use-cases/python/deep_research/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/deep_research/README-template.md @@ -53,12 +53,6 @@ curl --location 'localhost:8000/api/chat' \ To customize the UI, you can start by modifying the [./components/ui_event.jsx](./components/ui_event.jsx) file. -You can also generate a new code for the workflow using LLM by running the following command: - -``` -uv run generate_ui -``` - ## Learn More To learn more about LlamaIndex, take a look at the following resources: diff --git a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md index 203020f73..5d90432e6 100644 --- a/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/document_generator/README-template.md @@ -90,12 +90,6 @@ The following are the available options: To customize the UI, you can start by modifying the [./ui/components/ui_event.jsx](./ui/components/ui_event.jsx) file. -You can also generate a new code for the workflow using LLM by running the following command: - -``` -uv run generate_ui -``` - ## Learn More - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. diff --git a/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md b/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md index b82bc127c..94e6829b5 100644 --- a/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md @@ -131,12 +131,6 @@ The following are the available options: To customize the UI, you can start by modifying the [./ui/components/ui_event.jsx](./ui/components/ui_event.jsx) file. -You can also generate a new code for the workflow using LLM by running the following command: - -``` -uv run generate_ui -``` - ## Learn More - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. From d881f5a38906f329c7a43ef1f965b77f65cd655f Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 11:50:21 +0700 Subject: [PATCH 27/80] add todo --- packages/server/examples/hitl/components/cli_human_input.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/examples/hitl/components/cli_human_input.tsx b/packages/server/examples/hitl/components/cli_human_input.tsx index 973f65fb4..8700d6182 100644 --- a/packages/server/examples/hitl/components/cli_human_input.tsx +++ b/packages/server/examples/hitl/components/cli_human_input.tsx @@ -10,6 +10,7 @@ const CLIInputEventSchema = z.object({ }); type CLIInputEvent = z.infer; +// TODO: this component is working well for TS server. But not for HITL in Python llama-deploy. const CLIHumanInput: FC<{ events: JSONValue[]; }> = ({ events }) => { From 97dcd8cfed0543e6283852697b47acd836629c53 Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 12:04:35 +0700 Subject: [PATCH 28/80] arrage imports --- .../components/use-cases/python/agentic_rag/workflow.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py b/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py index 1538640b7..8c415d510 100644 --- a/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/agentic_rag/workflow.py @@ -1,10 +1,11 @@ -from src.index import get_index +from dotenv import load_dotenv + from llama_index.core.agent.workflow import AgentWorkflow from llama_index.core.settings import Settings + +from src.index import get_index from src.query import get_query_engine_tool from src.citation import CITATION_SYSTEM_PROMPT, enable_citation - -from dotenv import load_dotenv from src.settings import init_settings From 00675f79eeaa00422011bedeac31aa3b174bd64a Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 12:23:25 +0700 Subject: [PATCH 29/80] feat: deep research --- .../python/deep_research/README-template.md | 97 +++++++++++++------ .../use-cases/python/deep_research/utils.py | 45 +++++++++ .../python/deep_research/workflow.py | 17 ++-- 3 files changed, 124 insertions(+), 35 deletions(-) create mode 100644 packages/create-llama/templates/components/use-cases/python/deep_research/utils.py diff --git a/packages/create-llama/templates/components/use-cases/python/deep_research/README-template.md b/packages/create-llama/templates/components/use-cases/python/deep_research/README-template.md index fc4b532a2..47065d5e1 100644 --- a/packages/create-llama/templates/components/use-cases/python/deep_research/README-template.md +++ b/packages/create-llama/templates/components/use-cases/python/deep_research/README-template.md @@ -1,63 +1,106 @@ -This is a [LlamaIndex](https://www.llamaindex.ai/) multi-agents project using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/). +# LlamaIndex Workflow Example -## Getting Started +This is a [LlamaIndex](https://www.llamaindex.ai/) project that using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/) deployed with [LlamaDeploy](https://github.com/run-llama/llama_deploy). -First, setup the environment with uv: +LlamaDeploy is a system for deploying and managing LlamaIndex workflows, while LlamaIndexServer provides a pre-built TypeScript server with an integrated chat UI that can connect directly to LlamaDeploy deployments. This example shows how you can quickly set up a complete chat application by combining these two technologies/ -> **_Note:_** This step is not needed if you are using the dev-container. +## Prerequisites -```shell +If you haven't installed uv, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/) to install it. + +You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). + +Please setup their API keys in the `src/.env` file. + +## Installation + +Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, just run: + +```bash uv sync ``` -Then check the parameters that have been pre-configured in the `.env` file in this directory. -Make sure you have set the `OPENAI_API_KEY` for the LLM. +If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). -Second, generate the embeddings of the documents in the `./data` directory: +## Generate Index + +Generate the embeddings of the documents in the `./data` directory: ```shell uv run generate ``` -Third, run the development server: +## Running the Deployment -```shell -uv run fastapi dev -``` +At this point we have all we need to run this deployment. Ideally, we would have the API server already running +somewhere in the cloud, but to get started let's start an instance locally. Run the following python script +from a shell: -Then open [http://localhost:8000](http://localhost:8000) with your browser to start the chat UI. +``` +$ uv run -m llama_deploy.apiserver +INFO: Started server process [10842] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:4501 (Press CTRL+C to quit) +``` -To start the app optimized for **production**, run: +From another shell, use the CLI, `llamactl`, to create the deployment: ``` -uv run fastapi run +$ uv run llamactl deploy llama_deploy.yml +Deployment successful: chat ``` -## Configure LLM and Embedding Model +## UI Interface -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [settings.py](app/settings.py). +LlamaDeploy will serve the UI through the apiserver. Point the browser to [http://localhost:4501/deployments/chat/ui](http://localhost:4501/deployments/chat/ui) to interact with your deployment through a user-friendly interface. -## Use Case +## API endpoints -We have prepared an [example workflow](./app/workflow.py) for the deep research use case, where you can ask questions about the example documents in the [./data](./data) directory. +You can find all the endpoints in the [API documentation](http://localhost:4501/docs). To get started, you can try the following endpoints: -You can start by sending an request on the [chat UI](http://localhost:8000) or you can test the `/api/chat` endpoint with the following curl request: +Create a new task: +```bash +curl -X POST 'http://localhost:4501/deployments/chat/tasks/create' \ + -H 'Content-Type: application/json' \ + -d '{ + "input": "{\"user_msg\":\"Hello\",\"chat_history\":[]}", + "service_id": "workflow" + }' ``` -curl --location 'localhost:8000/api/chat' \ ---header 'Content-Type: application/json' \ ---data '{ "messages": [{ "role": "user", "content": "Create a report comparing the finances of Apple and Tesla" }] }' + +Stream events: + +```bash +curl 'http://localhost:4501/deployments/chat/tasks/0b411be6-005d-43f0-9b6b-6a0017f08002/events?session_id=dd36442c-45ca-4eaa-8d75-b4e6dad1a83e&raw_event=true' \ + -H 'Content-Type: application/json' ``` +Note that the task_id and session_id are returned when creating a new task. + +## Use Case + +We have prepared an [example workflow](./app/workflow.py) for the deep research use case, where you can ask questions about the example documents in the [./data](./data) directory. +To update the workflow, you can modify the code in [`src/workflow.py`](src/workflow.py). + ## Customize the UI -To customize the UI, you can start by modifying the [./components/ui_event.jsx](./components/ui_event.jsx) file. +The UI is served by LLamaIndexServer package, you can configure the UI by modifying the `uiConfig` in the [ui/index.ts](ui/index.ts) file. -## Learn More +The following are the available options: -To learn more about LlamaIndex, take a look at the following resources: +- `starterQuestions`: Predefined questions for chat interface +- `componentsDir`: Directory for custom event components +- `layoutDir`: Directory for custom layout components +- `llamaCloudIndexSelector`: Enable LlamaCloud integration +- `llamaDeploy`: The LlamaDeploy configration (deployment name and workflow name that defined in the [llama_deploy.yml](llama_deploy.yml) file) + +## Learn More - [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. - [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows. +- [LlamaDeploy GitHub Repository](https://github.com/run-llama/llama_deploy) +- [Chat-UI Documentation](https://ts.llamaindex.ai/docs/chat-ui) -You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! +You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! \ No newline at end of file diff --git a/packages/create-llama/templates/components/use-cases/python/deep_research/utils.py b/packages/create-llama/templates/components/use-cases/python/deep_research/utils.py new file mode 100644 index 000000000..9e9010da3 --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/deep_research/utils.py @@ -0,0 +1,45 @@ +from typing import AsyncGenerator, Union +from llama_index.core.base.llms.types import ( + CompletionResponse, + CompletionResponseAsyncGen, +) +from llama_index.core.workflow import Context +from llama_index.core.agent.workflow.workflow_events import AgentStream + + +async def write_response_to_stream( + res: Union[CompletionResponse, CompletionResponseAsyncGen], + ctx: Context, + current_agent_name: str = "assistant", +) -> str: + """ + Handle both streaming and non-streaming LLM responses. + + Args: + res: The LLM response (either streaming or non-streaming) + ctx: The workflow context for writing events to stream + current_agent_name: The name of the current agent (default: "assistant") + + Returns: + The final response text as a string + """ + final_response = "" + + if isinstance(res, AsyncGenerator): + # Handle streaming response (CompletionResponseAsyncGen) + async for chunk in res: + ctx.write_event_to_stream( + AgentStream( + delta=chunk.delta or "", + response=final_response, + current_agent_name=current_agent_name, + tool_calls=[], + raw=chunk.raw or "", + ) + ) + final_response = chunk.text + else: + # Handle non-streaming response (CompletionResponse) + final_response = res.text + + return final_response diff --git a/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py b/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py index 06519086f..2d172c8d0 100644 --- a/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py @@ -3,7 +3,7 @@ import uuid from typing import List, Literal, Optional -from app.index import get_index +from src.index import get_index from llama_index.core.base.llms.types import ( CompletionResponse, CompletionResponseAsyncGen, @@ -23,18 +23,19 @@ Workflow, step, ) -from llama_index.server.api.models import ( - ArtifactEvent, - ArtifactType, - ChatRequest, - SourceNodesEvent, - UIEvent, +from llama_index.core.chat_ui.models.artifact import ( Artifact, + ArtifactType, DocumentArtifactData, DocumentArtifactSource, ) +from llama_index.core.chat_ui.events import ( + UIEvent, + ArtifactEvent, + SourceNodesEvent, +) import time -from llama_index.server.utils.stream import write_response_to_stream +from src.utils import write_response_to_stream from pydantic import BaseModel, Field logger = logging.getLogger("uvicorn") From a7c8a06aac16c2a16b11af0153fc52511b1e01a4 Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 13:35:33 +0700 Subject: [PATCH 30/80] fix: deep research --- .../components/use-cases/python/deep_research/workflow.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py b/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py index 2d172c8d0..9f041fc6c 100644 --- a/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py @@ -42,8 +42,8 @@ logger.setLevel(logging.INFO) -def create_workflow(chat_request: Optional[ChatRequest] = None) -> Workflow: - index = get_index(chat_request=chat_request) +def create_workflow() -> Workflow: + index = get_index() if index is None: raise ValueError( "Index is not found. Try run generation script to create the index first." @@ -575,3 +575,5 @@ def _get_text_node_content_for_citation(node: NodeWithScore) -> str: node_id = node.node.node_id content = f"\n{node.get_content(metadata_mode=MetadataMode.LLM)}" return content + +workflow = create_workflow() \ No newline at end of file From 07bdf6c27117ccbf844cbe36470cb6e8f240fa89 Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 13:58:39 +0700 Subject: [PATCH 31/80] update deep research --- .../use-cases/python/deep_research/workflow.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py b/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py index 9f041fc6c..a48c646c2 100644 --- a/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py @@ -1,9 +1,11 @@ import logging import os import uuid +import time from typing import List, Literal, Optional +from pydantic import BaseModel, Field +from dotenv import load_dotenv -from src.index import get_index from llama_index.core.base.llms.types import ( CompletionResponse, CompletionResponseAsyncGen, @@ -34,15 +36,19 @@ ArtifactEvent, SourceNodesEvent, ) -import time + +from src.index import get_index +from src.settings import init_settings from src.utils import write_response_to_stream -from pydantic import BaseModel, Field logger = logging.getLogger("uvicorn") logger.setLevel(logging.INFO) def create_workflow() -> Workflow: + load_dotenv() + init_settings() + # TODO: load index in StartEvent index = get_index() if index is None: raise ValueError( @@ -576,4 +582,5 @@ def _get_text_node_content_for_citation(node: NodeWithScore) -> str: content = f"\n{node.get_content(metadata_mode=MetadataMode.LLM)}" return content + workflow = create_workflow() \ No newline at end of file From eb8b01c09097475851a30759dff8040b32cb377b Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 14:57:49 +0700 Subject: [PATCH 32/80] fix ecoding when genearing index --- .../templates/types/llamaindexserver/fastapi/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml index da9e5a5e4..a7742c700 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml @@ -12,6 +12,8 @@ dependencies = [ "pydantic>=2.11.5", "aiostream>=0.5.2,<0.6.0", "llama-index-core>=0.12.28,<0.13.0", + "llama-index-readers-file>=0.4.6,<1.0.0", + "llama-index-indices-managed-llama-cloud>=0.6.3,<1.0.0", "llama-deploy", ] From 436f9f6e91152b60be90e9d6c1d3b466697b0224 Mon Sep 17 00:00:00 2001 From: thucpn Date: Fri, 4 Jul 2025 15:09:47 +0700 Subject: [PATCH 33/80] package src --- .../templates/types/llamaindexserver/fastapi/pyproject.toml | 4 ++-- .../templates/types/llamaindexserver/fastapi/src/generate.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml index a7742c700..c5b1eee6d 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/pyproject.toml @@ -28,7 +28,7 @@ dev = [ ] [project.scripts] -generate = "generate:generate_index" +generate = "src.generate:generate_index" [tool.mypy] @@ -57,4 +57,4 @@ requires = [ "hatchling>=1.24" ] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = [ "src/generate.py" ] \ No newline at end of file +packages = ["src"] \ No newline at end of file diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py index 7bc5a2fdb..906fd0bea 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py @@ -11,8 +11,8 @@ def generate_index(): """ Index the documents in the data directory. """ - from index import STORAGE_DIR - from settings import init_settings + from src.index import STORAGE_DIR + from src.settings import init_settings from llama_index.core.indices import ( VectorStoreIndex, ) From c4b2e48054c7f2a4c8e014132dab2b739e7c6c74 Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 08:26:14 +0700 Subject: [PATCH 34/80] fix: deep research prepare messages --- .../python/deep_research/workflow.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py b/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py index a48c646c2..c2fdae385 100644 --- a/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/deep_research/workflow.py @@ -147,21 +147,23 @@ async def retrieve(self, ctx: Context, ev: StartEvent) -> PlanResearchEvent: """ self.stream = ev.get("stream", True) self.user_request = ev.get("user_msg") - chat_history = ev.get("chat_history") - if chat_history is not None: - self.memory.put_messages(chat_history) + + messages = [ + ChatMessage( + role=msg.get("role", "user"), + content=msg.get("content", ""), + ) + for msg in ev.get("chat_history", []) + ] + user_message = ChatMessage(role="user", content=self.user_request) + chat_history = [*messages, user_message] + self.memory.put_messages(chat_history) await ctx.set("total_questions", 0) # Add user message to memory - self.memory.put_messages( - messages=[ - ChatMessage( - role=MessageRole.USER, - content=self.user_request, - ) - ] - ) + self.memory.put_messages(messages=[user_message]) + ctx.write_event_to_stream( UIEvent( type="ui_event", From a3e3a7a90674f6efe10f5e4d5853d9c04780f093 Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 08:41:25 +0700 Subject: [PATCH 35/80] support financial report --- .../python/financial_report/agent_tool.py | 254 ++++++++++++++++++ .../financial_report/document_generator.py | 242 +++++++++++++++++ .../python/financial_report/events.py | 25 ++ .../python/financial_report/interpreter.py | 218 +++++++++++++++ .../python/financial_report/query.py | 47 ++++ .../python/financial_report/workflow.py | 37 ++- 6 files changed, 810 insertions(+), 13 deletions(-) create mode 100644 packages/create-llama/templates/components/use-cases/python/financial_report/agent_tool.py create mode 100644 packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py create mode 100644 packages/create-llama/templates/components/use-cases/python/financial_report/events.py create mode 100644 packages/create-llama/templates/components/use-cases/python/financial_report/interpreter.py create mode 100644 packages/create-llama/templates/components/use-cases/python/financial_report/query.py diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/agent_tool.py b/packages/create-llama/templates/components/use-cases/python/financial_report/agent_tool.py new file mode 100644 index 000000000..c0daf95eb --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/agent_tool.py @@ -0,0 +1,254 @@ +import logging +import uuid +from abc import ABC, abstractmethod +from typing import Any, AsyncGenerator, Optional + +from pydantic import BaseModel, ConfigDict + +from llama_index.core.base.llms.types import ChatMessage, ChatResponse +from llama_index.core.llms.function_calling import FunctionCallingLLM +from llama_index.core.tools import ( + BaseTool, + FunctionTool, + ToolOutput, + ToolSelection, +) +from llama_index.core.workflow import Context +from llama_index.core.agent.workflow.workflow_events import ToolCall, ToolCallResult + +from src.events import AgentRunEvent, AgentRunEventType + +logger = logging.getLogger("uvicorn") + + +class ToolCallOutput(BaseModel): + tool_call_id: str + tool_output: ToolOutput + + +class ContextAwareTool(FunctionTool, ABC): + @abstractmethod + async def acall(self, ctx: Context, input: Any) -> ToolOutput: # type: ignore + pass + + +class ChatWithToolsResponse(BaseModel): + """ + A tool call response from chat_with_tools. + """ + + tool_calls: Optional[list[ToolSelection]] + tool_call_message: Optional[ChatMessage] + generator: Optional[AsyncGenerator[ChatResponse | None, None]] + + model_config = ConfigDict(arbitrary_types_allowed=True) + + def is_calling_different_tools(self) -> bool: + tool_names = {tool_call.tool_name for tool_call in self.tool_calls or []} + return len(tool_names) > 1 + + def has_tool_calls(self) -> bool: + return self.tool_calls is not None and len(self.tool_calls) > 0 + + def tool_name(self) -> str: + if not self.has_tool_calls(): + raise ValueError("No tool calls") + if self.is_calling_different_tools(): + raise ValueError("Calling different tools") + return self.tool_calls[0].tool_name # type: ignore + + async def full_response(self) -> str: + assert self.generator is not None + full_response = "" + async for chunk in self.generator: + content = chunk.delta # type: ignore + if content: + full_response += content + return full_response + + +async def chat_with_tools( # type: ignore + llm: FunctionCallingLLM, + tools: list[BaseTool], + chat_history: list[ChatMessage], +) -> ChatWithToolsResponse: + """ + Request LLM to call tools or not. + This function doesn't change the memory. + """ + generator = _tool_call_generator(llm, tools, chat_history) + is_tool_call = await generator.__anext__() + if is_tool_call: + # Last chunk is the full response + # Wait for the last chunk + full_response = None + async for chunk in generator: + full_response = chunk + assert isinstance(full_response, ChatResponse) + return ChatWithToolsResponse( + tool_calls=llm.get_tool_calls_from_response(full_response), + tool_call_message=full_response.message, + generator=None, + ) + else: + return ChatWithToolsResponse( + tool_calls=None, + tool_call_message=None, + generator=generator, # type: ignore + ) + + +async def call_tools( + ctx: Context, + agent_name: str, + tools: list[BaseTool], + tool_calls: list[ToolSelection], + emit_agent_events: bool = True, +) -> list[ToolCallOutput]: + """ + Call tools and return the tool call responses. + """ + if len(tool_calls) == 0: + return [] + tools_by_name = {tool.metadata.get_name(): tool for tool in tools} + if len(tool_calls) == 1: + if emit_agent_events: + ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=f"{tool_calls[0].tool_name}: {tool_calls[0].tool_kwargs}", + ) + ) + return [ + await call_tool(ctx, tools_by_name[tool_calls[0].tool_name], tool_calls[0]) + ] + # Multiple tool calls, show progress + tool_call_outputs: list[ToolCallOutput] = [] + + progress_id = str(uuid.uuid4()) + total_steps = len(tool_calls) + if emit_agent_events: + ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=f"Making {total_steps} tool calls", + ) + ) + for i, tool_call in enumerate(tool_calls): + tool = tools_by_name.get(tool_call.tool_name) + if not tool: + tool_call_outputs.append( + ToolCallOutput( + tool_call_id=tool_call.tool_id, + tool_output=ToolOutput( + is_error=True, + content=f"Tool {tool_call.tool_name} does not exist", + tool_name=tool_call.tool_name, + raw_input=tool_call.tool_kwargs, + raw_output={ + "error": f"Tool {tool_call.tool_name} does not exist", + }, + ), + ) + ) + continue + + tool_call_output = await call_tool( + ctx, + tool, + tool_call, + ) + if emit_agent_events: + ctx.write_event_to_stream( + AgentRunEvent( + name=agent_name, + msg=f"{tool_call.tool_name}: {tool_call.tool_kwargs}", + event_type=AgentRunEventType.PROGRESS, + data={ + "id": progress_id, + "total": total_steps, + "current": i, + }, + ) + ) + tool_call_outputs.append(tool_call_output) + return tool_call_outputs + + +async def call_tool( + ctx: Context, + tool: BaseTool, + tool_call: ToolSelection, +) -> ToolCallOutput: + ctx.write_event_to_stream( + ToolCall( + tool_name=tool_call.tool_name, + tool_id=tool_call.tool_id, + tool_kwargs=tool_call.tool_kwargs, + ) + ) + try: + if isinstance(tool, ContextAwareTool): + if ctx is None: + raise ValueError("Context is required for context aware tool") + # inject context for calling an context aware tool + output = await tool.acall(ctx=ctx, **tool_call.tool_kwargs) + else: + output = await tool.acall(**tool_call.tool_kwargs) # type: ignore + except Exception as e: + logger.error(f"Got error in tool {tool_call.tool_name}: {e!s}") + output = ToolOutput( + is_error=True, + content=f"Error: {e!s}", + tool_name=tool.metadata.get_name(), + raw_input=tool_call.tool_kwargs, + raw_output={ + "error": str(e), + }, + ) + ctx.write_event_to_stream( + ToolCallResult( + tool_name=tool_call.tool_name, + tool_kwargs=tool_call.tool_kwargs, + tool_id=tool_call.tool_id, + tool_output=output, + return_direct=False, + ) + ) + return ToolCallOutput( + tool_call_id=tool_call.tool_id, + tool_output=output, + ) + + +async def _tool_call_generator( + llm: FunctionCallingLLM, + tools: list[BaseTool], + chat_history: list[ChatMessage], +) -> AsyncGenerator[ChatResponse | bool, None]: + response_stream = await llm.astream_chat_with_tools( + tools, + chat_history=chat_history, + allow_parallel_tool_calls=False, + ) + + full_response = None + yielded_indicator = False + async for chunk in response_stream: + if "tool_calls" not in chunk.message.additional_kwargs: + # Yield a boolean to indicate whether the response is a tool call + if not yielded_indicator: + yield False + yielded_indicator = True + + # if not a tool call, yield the chunks! + yield chunk # type: ignore + elif not yielded_indicator: + # Yield the indicator for a tool call + yield True + yielded_indicator = True + + full_response = chunk + + if full_response: + yield full_response # type: ignore diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py b/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py new file mode 100644 index 000000000..4438daca9 --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py @@ -0,0 +1,242 @@ +import logging +import os +import re +from enum import Enum +from io import BytesIO + +from llama_index.core.tools.function_tool import FunctionTool + +OUTPUT_DIR = "output/tools" + + +class DocumentType(Enum): + PDF = "pdf" + HTML = "html" + + +COMMON_STYLES = """ +body { + font-family: Arial, sans-serif; + line-height: 1.3; + color: #333; +} +h1, h2, h3, h4, h5, h6 { + margin-top: 1em; + margin-bottom: 0.5em; +} +p { + margin-bottom: 0.7em; +} +code { + background-color: #f4f4f4; + padding: 2px 4px; + border-radius: 4px; +} +pre { + background-color: #f4f4f4; + padding: 10px; + border-radius: 4px; + overflow-x: auto; +} +table { + border-collapse: collapse; + width: 100%; + margin-bottom: 1em; +} +th, td { + border: 1px solid #ddd; + padding: 8px; + text-align: left; +} +th { + background-color: #f2f2f2; + font-weight: bold; +} +""" + +HTML_SPECIFIC_STYLES = """ +body { + max-width: 800px; + margin: 0 auto; + padding: 20px; +} +""" + +PDF_SPECIFIC_STYLES = """ +@page { + size: letter; + margin: 2cm; +} +body { + font-size: 11pt; +} +h1 { font-size: 18pt; } +h2 { font-size: 16pt; } +h3 { font-size: 14pt; } +h4, h5, h6 { font-size: 12pt; } +pre, code { + font-family: Courier, monospace; + font-size: 0.9em; +} +""" + +HTML_TEMPLATE = """ + + + + + + + + + {content} + + +""" + + +class DocumentGenerator: + def __init__(self, file_server_url_prefix: str): + if not file_server_url_prefix: + raise ValueError("file_server_url_prefix is required") + self.file_server_url_prefix = file_server_url_prefix + + @classmethod + def _generate_html_content(cls, original_content: str) -> str: + """ + Generate HTML content from the original markdown content. + """ + try: + import markdown # type: ignore + except ImportError: + raise ImportError( + "Failed to import required modules. Please install markdown." + ) + + # Convert markdown to HTML with fenced code and table extensions + return markdown.markdown(original_content, extensions=["fenced_code", "tables"]) + + @classmethod + def _generate_pdf(cls, html_content: str) -> BytesIO: + """ + Generate a PDF from the HTML content. + """ + try: + from xhtml2pdf import pisa + except ImportError: + raise ImportError( + "Failed to import required modules. Please install xhtml2pdf." + ) + + pdf_html = HTML_TEMPLATE.format( + common_styles=COMMON_STYLES, + specific_styles=PDF_SPECIFIC_STYLES, + content=html_content, + ) + + buffer = BytesIO() + pdf = pisa.pisaDocument( + BytesIO(pdf_html.encode("UTF-8")), buffer, encoding="UTF-8" + ) + + if pdf.err: + logging.error(f"PDF generation failed: {pdf.err}") + raise ValueError("PDF generation failed") + + buffer.seek(0) + return buffer + + @classmethod + def _generate_html(cls, html_content: str) -> str: + """ + Generate a complete HTML document with the given HTML content. + """ + return HTML_TEMPLATE.format( + common_styles=COMMON_STYLES, + specific_styles=HTML_SPECIFIC_STYLES, + content=html_content, + ) + + def generate_document( + self, original_content: str, document_type: str, file_name: str + ) -> str: + """ + To generate document as PDF or HTML file. + Parameters: + original_content: str (markdown style) + document_type: str (pdf or html) specify the type of the file format based on the use case + file_name: str (name of the document file) must be a valid file name, no extensions needed + Returns: + str (URL to the document file): A file URL ready to serve. + """ + try: + doc_type = DocumentType(document_type.lower()) + except ValueError: + raise ValueError( + f"Invalid document type: {document_type}. Must be 'pdf' or 'html'." + ) + # Always generate html content first + html_content = self._generate_html_content(original_content) + + # Based on the type of document, generate the corresponding file + if doc_type == DocumentType.PDF: + content = self._generate_pdf(html_content) + file_extension = "pdf" + elif doc_type == DocumentType.HTML: + content = BytesIO(self._generate_html(html_content).encode("utf-8")) + file_extension = "html" + else: + raise ValueError(f"Unexpected document type: {document_type}") + + file_name = self._validate_file_name(file_name) + file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}") + + self._write_to_file(content, file_path) + + return ( + f"{self.file_server_url_prefix}/{OUTPUT_DIR}/{file_name}.{file_extension}" + ) + + @staticmethod + def _write_to_file(content: BytesIO, file_path: str) -> None: + """ + Write the content to a file. + """ + try: + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "wb") as file: + file.write(content.getvalue()) + except Exception: + raise + + @staticmethod + def _validate_file_name(file_name: str) -> str: + """ + Validate the file name. + """ + # Don't allow directory traversal + if os.path.isabs(file_name): + raise ValueError("File name is not allowed.") + # Don't allow special characters + if re.match(r"^[a-zA-Z0-9_.-]+$", file_name): + return file_name + else: + raise ValueError("File name is not allowed to contain special characters.") + + @classmethod + def _validate_packages(cls) -> None: + try: + import markdown # noqa: F401 + import xhtml2pdf # noqa: F401 + except ImportError: + raise ImportError( + "Failed to import required modules. Please install markdown and xhtml2pdf " + "using `pip install markdown xhtml2pdf`" + ) + + def to_tool(self) -> FunctionTool: + self._validate_packages() + return FunctionTool.from_defaults(self.generate_document) diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/events.py b/packages/create-llama/templates/components/use-cases/python/financial_report/events.py new file mode 100644 index 000000000..0ef33840c --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/events.py @@ -0,0 +1,25 @@ +from typing import List, Optional, Enum +from llama_index.core.base.llms.types import ChatMessage +from llama_index.core.tools import ToolSelection +from llama_index.core.workflow import Event + + +class AgentRunEventType(Enum): + TEXT = "text" + PROGRESS = "progress" + + +class AgentRunEvent(Event): + name: str + msg: str + event_type: AgentRunEventType = AgentRunEventType.TEXT + data: Optional[dict] = None + + +class InputEvent(Event): + input: List[ChatMessage] + response: bool = False + + +class ResearchEvent(Event): + input: list[ToolSelection] diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/interpreter.py b/packages/create-llama/templates/components/use-cases/python/financial_report/interpreter.py new file mode 100644 index 000000000..a239bbd28 --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/interpreter.py @@ -0,0 +1,218 @@ +import base64 +import logging +import os +import uuid +from typing import Any, List, Optional + +from pydantic import BaseModel + +from llama_index.core.tools import FunctionTool +from llama_index.server.models.file import ServerFile +from llama_index.server.services.file import FileService + +logger = logging.getLogger("uvicorn") + + +class InterpreterExtraResult(BaseModel): + type: str + content: Optional[str] = None + filename: Optional[str] = None + url: Optional[str] = None + + +class E2BToolOutput(BaseModel): + is_error: bool + logs: "Logs" # type: ignore # noqa: F821 + error_message: Optional[str] = None + results: List[InterpreterExtraResult] = [] + retry_count: int = 0 + + +class E2BCodeInterpreter: + output_dir = "output/tools" + uploaded_files_dir = "output/uploaded" + interpreter: Optional["Sandbox"] = None # type: ignore # noqa: F821 + + def __init__( + self, + api_key: str, + output_dir: Optional[str] = None, + uploaded_files_dir: Optional[str] = None, + ): + """ + Args: + api_key: The API key for the E2B Code Interpreter. + output_dir: The directory for the output files. Default is `output/tools`. + uploaded_files_dir: The directory for the files to be uploaded to the sandbox. Default is `output/uploaded`. + """ + self._validate_package() + if not api_key: + raise ValueError( + "api_key is required to run code interpreter. Get it here: https://e2b.dev/docs/getting-started/api-key" + ) + self.api_key = api_key + self.output_dir = output_dir or "output/tools" + self.uploaded_files_dir = uploaded_files_dir or "output/uploaded" + + @classmethod + def _validate_package(cls) -> None: + try: + from e2b_code_interpreter import Sandbox # noqa: F401 + from e2b_code_interpreter.models import Logs # noqa: F401 + except ImportError: + raise ImportError( + "e2b_code_interpreter is not installed. Please install it using `pip install e2b-code-interpreter`." + ) + + def __del__(self) -> None: + """ + Kill the interpreter when the tool is no longer in use. + """ + if self.interpreter is not None: + self.interpreter.kill() + + def _init_interpreter(self, sandbox_files: List[str] = []) -> None: + """ + Lazily initialize the interpreter. + """ + from e2b_code_interpreter import Sandbox + + logger.info(f"Initializing interpreter with {len(sandbox_files)} files") + self.interpreter = Sandbox(api_key=self.api_key) + if len(sandbox_files) > 0: + for file_path in sandbox_files: + file_name = os.path.basename(file_path) + local_file_path = os.path.join(self.uploaded_files_dir, file_name) + with open(local_file_path, "rb") as f: + content = f.read() + if self.interpreter and self.interpreter.files: + self.interpreter.files.write(file_path, content) + logger.info(f"Uploaded {len(sandbox_files)} files to sandbox") + + def _save_to_disk(self, base64_data: str, ext: str) -> ServerFile: + buffer = base64.b64decode(base64_data) + + # Output from e2b doesn't have a name. Create a random name for it. + filename = f"e2b_file_{uuid.uuid4()}.{ext}" + + return FileService.save_file( + buffer, file_name=filename, save_dir=self.output_dir + ) + + def _parse_result(self, result: Any) -> List[InterpreterExtraResult]: + """ + The result could include multiple formats (e.g. png, svg, etc.) but encoded in base64 + We save each result to disk and return saved file metadata (extension, filename, url). + """ + if not result: + return [] + + output = [] + + try: + formats = result.formats() + results = [result[format] for format in formats] + + for ext, data in zip(formats, results): + if ext in ["png", "svg", "jpeg", "pdf"]: + document_file = self._save_to_disk(data, ext) + output.append( + InterpreterExtraResult( + type=ext, + filename=document_file.id, + url=document_file.url, + ) + ) + else: + # Try serialize data to string + try: + data = str(data) + except Exception as e: + data = f"Error when serializing data: {e}" + output.append( + InterpreterExtraResult( + type=ext, + content=data, + ) + ) + except Exception as error: + logger.exception(error, exc_info=True) + logger.error("Error when parsing output from E2b interpreter tool", error) + + return output + + def interpret( + self, + code: str, + sandbox_files: List[str] = [], + retry_count: int = 0, + ) -> E2BToolOutput: + """ + Execute Python code in a Jupyter notebook cell. The tool will return the result, stdout, stderr, display_data, and error. + If the code needs to use a file, ALWAYS pass the file path in the sandbox_files argument. + You have a maximum of 3 retries to get the code to run successfully. + + Parameters: + code (str): The Python code to be executed in a single cell. + sandbox_files (List[str]): List of local file paths to be used by the code. The tool will throw an error if a file is not found. + retry_count (int): Number of times the tool has been retried. + """ + from e2b_code_interpreter.models import Logs + + if retry_count > 2: + return E2BToolOutput( + is_error=True, + logs=Logs( + stdout="", + stderr="", + display_data="", + error="", + ), + error_message="Failed to execute the code after 3 retries. Explain the error to the user and suggest a fix.", + retry_count=retry_count, + ) + + if self.interpreter is None: + self._init_interpreter(sandbox_files) + + if self.interpreter: + logger.info( + f"\n{'=' * 50}\n> Running following AI-generated code:\n{code}\n{'=' * 50}" + ) + exec = self.interpreter.run_code(code) + + if exec.error: + error_message = f"The code failed to execute successfully. Error: {exec.error}. Try to fix the code and run again." + logger.error(error_message) + # Calling the generated code caused an error. Kill the interpreter and return the error to the LLM so it can try to fix the error + try: + self.interpreter.kill() # type: ignore + except Exception: + pass + finally: + self.interpreter = None + output = E2BToolOutput( + is_error=True, + logs=exec.logs, + results=[], + error_message=error_message, + retry_count=retry_count + 1, + ) + else: + if len(exec.results) == 0: + output = E2BToolOutput(is_error=False, logs=exec.logs, results=[]) + else: + results = self._parse_result(exec.results[0]) + output = E2BToolOutput( + is_error=False, + logs=exec.logs, + results=results, + retry_count=retry_count + 1, + ) + return output + else: + raise ValueError("Interpreter is not initialized.") + + def to_tool(self) -> FunctionTool: + self._validate_package() + return FunctionTool.from_defaults(self.interpret) diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/query.py b/packages/create-llama/templates/components/use-cases/python/financial_report/query.py new file mode 100644 index 000000000..62c59240f --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/query.py @@ -0,0 +1,47 @@ +import os +from typing import Any, Optional + +from llama_index.core.base.base_query_engine import BaseQueryEngine +from llama_index.core.indices.base import BaseIndex +from llama_index.core.tools.query_engine import QueryEngineTool + +def create_query_engine(index: BaseIndex, **kwargs: Any) -> BaseQueryEngine: + """ + Create a query engine for the given index. + + Args: + index: The index to create a query engine for. + params (optional): Additional parameters for the query engine, e.g: similarity_top_k + """ + top_k = int(os.getenv("TOP_K", 0)) + if top_k != 0 and kwargs.get("filters") is None: + kwargs["similarity_top_k"] = top_k + + return index.as_query_engine(**kwargs) + + +def get_query_engine_tool( + index: BaseIndex, + name: Optional[str] = None, + description: Optional[str] = None, + **kwargs: Any, +) -> QueryEngineTool: + """ + Get a query engine tool for the given index. + + Args: + index: The index to create a query engine for. + name (optional): The name of the tool. + description (optional): The description of the tool. + """ + if name is None: + name = "query_index" + if description is None: + description = "Use this tool to retrieve information from a knowledge base. Provide a specific query and can call the tool multiple times if necessary." + query_engine = create_query_engine(index, **kwargs) + tool = QueryEngineTool.from_defaults( + query_engine=query_engine, + name=name, + description=description, + ) + return tool diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py index 58d155aef..ba8ca0fc6 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py @@ -1,7 +1,8 @@ import os -from typing import List, Optional +from typing import List, Optional, Enum +from dotenv import load_dotenv + -from app.index import get_index from llama_index.core import Settings from llama_index.core.base.llms.types import ChatMessage, MessageRole from llama_index.core.llms.function_calling import FunctionCallingLLM @@ -15,19 +16,18 @@ Workflow, step, ) -from llama_index.server.api.models import AgentRunEvent, ChatRequest -from llama_index.server.settings import server_settings -from llama_index.server.tools.document_generator import DocumentGenerator -from llama_index.server.tools.index import get_query_engine_tool -from llama_index.server.tools.interpreter import E2BCodeInterpreter -from llama_index.server.utils.agent_tool import ( - call_tools, - chat_with_tools, -) +from src.index import get_index +from src.settings import init_settings +from src.query import get_query_engine_tool +from src.document_generator import DocumentGenerator +from src.interpreter import E2BCodeInterpreter +from src.agent_tool import AgentRunEvent, AgentRunEventType -def create_workflow(chat_request: Optional[ChatRequest] = None) -> Workflow: - index = get_index(chat_request=chat_request) +def create_workflow() -> Workflow: + load_dotenv() + init_settings() + index = get_index() if index is None: raise ValueError( "Index is not found. Try run generation script to create the index first." @@ -50,6 +50,17 @@ def create_workflow(chat_request: Optional[ChatRequest] = None) -> Workflow: timeout=180, ) +class AgentRunEventType(Enum): + TEXT = "text" + PROGRESS = "progress" + + +class AgentRunEvent(Event): + name: str + msg: str + event_type: AgentRunEventType = AgentRunEventType.TEXT + data: Optional[dict] = None + class InputEvent(Event): input: List[ChatMessage] From 643ad9868779cb4c45e5c76ecb625205de29eb3a Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 09:05:51 +0700 Subject: [PATCH 36/80] fix: implement save file --- .../python/financial_report/events.py | 3 +- .../python/financial_report/interpreter.py | 74 +++++++++++++++++-- .../python/financial_report/workflow.py | 44 ++++------- 3 files changed, 84 insertions(+), 37 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/events.py b/packages/create-llama/templates/components/use-cases/python/financial_report/events.py index 0ef33840c..dec3b5865 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/events.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/events.py @@ -1,4 +1,5 @@ -from typing import List, Optional, Enum +from typing import List, Optional +from enum import Enum from llama_index.core.base.llms.types import ChatMessage from llama_index.core.tools import ToolSelection from llama_index.core.workflow import Event diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/interpreter.py b/packages/create-llama/templates/components/use-cases/python/financial_report/interpreter.py index a239bbd28..2c1ce5a0d 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/interpreter.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/interpreter.py @@ -1,18 +1,28 @@ import base64 import logging import os +import re import uuid +from pathlib import Path from typing import Any, List, Optional from pydantic import BaseModel from llama_index.core.tools import FunctionTool -from llama_index.server.models.file import ServerFile -from llama_index.server.services.file import FileService logger = logging.getLogger("uvicorn") +class FileMetadata(BaseModel): + """Simple file metadata model""" + + id: str + type: str + size: int + url: str + path: str + + class InterpreterExtraResult(BaseModel): type: str content: Optional[str] = None @@ -89,15 +99,67 @@ def _init_interpreter(self, sandbox_files: List[str] = []) -> None: self.interpreter.files.write(file_path, content) logger.info(f"Uploaded {len(sandbox_files)} files to sandbox") - def _save_to_disk(self, base64_data: str, ext: str) -> ServerFile: + def _process_file_name(self, file_name: str) -> tuple[str, str]: + """ + Process original file name to generate a unique file id and extension. + """ + _id = str(uuid.uuid4()) + name, extension = os.path.splitext(file_name) + extension = extension.lstrip(".") + if extension == "": + raise ValueError("File name is not valid! It must have an extension.") + # sanitize the name + name = re.sub(r"[^a-zA-Z0-9.]", "_", name) + file_id = f"{name}_{_id}.{extension}" + return file_id, extension + + def _get_file_url(self, file_id: str, save_dir: str) -> str: + """ + Get the URL of a file. + """ + # Ensure the path uses forward slashes for URLs + url_path = f"{save_dir}/{file_id}".replace("\\", "/") + return f"/api/files/{url_path}" + + def _save_file(self, content: bytes, file_name: str, save_dir: str) -> FileMetadata: + file_id, extension = self._process_file_name(file_name) + file_path = os.path.join(save_dir, file_id) + + # Write the file directly + try: + os.makedirs(os.path.dirname(file_path), exist_ok=True) + with open(file_path, "wb") as f: + f.write(content) + except PermissionError as e: + logger.error(f"Permission denied when writing to file {file_path}: {e!s}") + raise + except OSError as e: + logger.error(f"IO error occurred when writing to file {file_path}: {e!s}") + raise + except Exception as e: + logger.error(f"Unexpected error when writing to file {file_path}: {e!s}") + raise + + logger.info(f"Saved file to {file_path}") + + file_size = os.path.getsize(file_path) + file_url = self._get_file_url(file_id, save_dir) + + return FileMetadata( + id=file_id, + type=extension, + size=file_size, + url=file_url, + path=file_path, + ) + + def _save_to_disk(self, base64_data: str, ext: str) -> FileMetadata: buffer = base64.b64decode(base64_data) # Output from e2b doesn't have a name. Create a random name for it. filename = f"e2b_file_{uuid.uuid4()}.{ext}" - return FileService.save_file( - buffer, file_name=filename, save_dir=self.output_dir - ) + return self._save_file(buffer, file_name=filename, save_dir=self.output_dir) def _parse_result(self, result: Any) -> List[InterpreterExtraResult]: """ diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py index ba8ca0fc6..8e831ee0d 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py @@ -1,5 +1,6 @@ import os -from typing import List, Optional, Enum +from typing import List, Optional +from enum import Enum from dotenv import load_dotenv @@ -22,7 +23,15 @@ from src.query import get_query_engine_tool from src.document_generator import DocumentGenerator from src.interpreter import E2BCodeInterpreter -from src.agent_tool import AgentRunEvent, AgentRunEventType +from src.events import ( + InputEvent, + ResearchEvent, + AnalyzeEvent, + ReportEvent, + AgentRunEvent, +) +from src.agent_tool import call_tools, chat_with_tools + def create_workflow() -> Workflow: load_dotenv() @@ -50,34 +59,6 @@ def create_workflow() -> Workflow: timeout=180, ) -class AgentRunEventType(Enum): - TEXT = "text" - PROGRESS = "progress" - - -class AgentRunEvent(Event): - name: str - msg: str - event_type: AgentRunEventType = AgentRunEventType.TEXT - data: Optional[dict] = None - - -class InputEvent(Event): - input: List[ChatMessage] - response: bool = False - - -class ResearchEvent(Event): - input: list[ToolSelection] - - -class AnalyzeEvent(Event): - input: list[ToolSelection] | ChatMessage - - -class ReportEvent(Event): - input: list[ToolSelection] - class FinancialReportWorkflow(Workflow): """ @@ -342,3 +323,6 @@ async def report(self, ctx: Context, ev: ReportEvent) -> InputEvent: ) # After the tool calls, fallback to the input with the latest chat history return InputEvent(input=self.memory.get()) + + +workflow = create_workflow() From 02e36df3b04f5c6195c07bc0edb67212d9eb0426 Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 09:47:41 +0700 Subject: [PATCH 37/80] missing e2b for finance --- .../create-llama/helpers/env-variables.ts | 19 ++++++++++++++++++- .../python/financial_report/events.py | 6 ++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index 58de76bcb..27321a764 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -5,6 +5,7 @@ import { ModelConfig, TemplateFramework, TemplateType, + TemplateUseCase, TemplateVectorDB, } from "./types"; @@ -232,9 +233,11 @@ const getModelEnvs = ( modelConfig: ModelConfig, framework: TemplateFramework, template: TemplateType, + useCase: TemplateUseCase, ): EnvVar[] => { const isPythonLlamaDeploy = framework === "fastapi" && template === "llamaindexserver"; + const needE2B = useCase === "financial_report"; return [ { @@ -256,6 +259,14 @@ const getModelEnvs = ( "The questions to help users get started (multi-line).", }, ]), + ...(needE2B + ? [ + { + name: "E2B_API_KEY", + description: "The E2B API key to use to use code interpreter tool", + }, + ] + : []), ...(modelConfig.provider === "openai" ? [ { @@ -420,6 +431,7 @@ export const createBackendEnvFile = async ( | "template" | "port" | "useLlamaParse" + | "useCase" >, ) => { // Init env values @@ -436,7 +448,12 @@ export const createBackendEnvFile = async ( : []), ...getVectorDBEnvs(opts.vectorDb, opts.framework, opts.template), ...getFrameworkEnvs(opts.framework, opts.template, opts.port), - ...getModelEnvs(opts.modelConfig, opts.framework, opts.template), + ...getModelEnvs( + opts.modelConfig, + opts.framework, + opts.template, + opts.useCase, + ), ]; // Render and write env file const content = renderEnvVar(envVars); diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/events.py b/packages/create-llama/templates/components/use-cases/python/financial_report/events.py index dec3b5865..2e5214da2 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/events.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/events.py @@ -24,3 +24,9 @@ class InputEvent(Event): class ResearchEvent(Event): input: list[ToolSelection] + +class AnalyzeEvent(Event): + input: list[ToolSelection] | ChatMessage + +class ReportEvent(Event): + input: list[ToolSelection] \ No newline at end of file From 0b5a7d91c6984899c17e55ef730749f63029859d Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 09:58:50 +0700 Subject: [PATCH 38/80] missing deps --- .../create-llama/helpers/env-variables.ts | 4 +- packages/create-llama/helpers/python.ts | 52 +++++++++++++++---- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index 27321a764..f26bfcd74 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -237,7 +237,7 @@ const getModelEnvs = ( ): EnvVar[] => { const isPythonLlamaDeploy = framework === "fastapi" && template === "llamaindexserver"; - const needE2B = useCase === "financial_report"; + const isFinancialReport = useCase === "financial_report"; return [ { @@ -259,7 +259,7 @@ const getModelEnvs = ( "The questions to help users get started (multi-line).", }, ]), - ...(needE2B + ...(isFinancialReport ? [ { name: "E2B_API_KEY", diff --git a/packages/create-llama/helpers/python.ts b/packages/create-llama/helpers/python.ts index 40af34e6f..d5dcc5add 100644 --- a/packages/create-llama/helpers/python.ts +++ b/packages/create-llama/helpers/python.ts @@ -7,12 +7,7 @@ import { isUvAvailable, tryUvSync } from "./uv"; import { assetRelocator, copy } from "./copy"; import { templatesDir } from "./dir"; -import { - InstallTemplateArgs, - ModelConfig, - TemplateDataSource, - TemplateVectorDB, -} from "./types"; +import { InstallTemplateArgs } from "./types"; interface Dependency { name: string; @@ -22,12 +17,44 @@ interface Dependency { } const getAdditionalDependencies = ( - modelConfig: ModelConfig, - vectorDb?: TemplateVectorDB, - dataSources?: TemplateDataSource[], + opts: Pick< + InstallTemplateArgs, + | "framework" + | "template" + | "useCase" + | "modelConfig" + | "vectorDb" + | "dataSources" + >, ) => { + const { framework, template, useCase, modelConfig, vectorDb, dataSources } = + opts; + const isPythonLlamaDeploy = + framework === "fastapi" && template === "llamaindexserver"; + const isPythonFinancialReport = + isPythonLlamaDeploy && useCase === "financial_report"; + const dependencies: Dependency[] = []; + if (isPythonFinancialReport) { + dependencies.push( + ...[ + { + name: "e2b-code-interpreter", + version: ">=1.1.1,<2.0.0", + }, + { + name: "markdown", + version: ">=3.7,<4.0", + }, + { + name: "xhtml2pdf", + version: ">=0.2.17,<1.0.0", + }, + ], + ); + } + // Add vector db dependencies switch (vectorDb) { case "mongo": { @@ -545,11 +572,14 @@ export const installPythonTemplate = async ({ } console.log("Adding additional dependencies"); - const addOnDependencies = getAdditionalDependencies( + const addOnDependencies = getAdditionalDependencies({ + framework, + template, + useCase, modelConfig, vectorDb, dataSources, - ); + }); await addDependencies(root, addOnDependencies); From 57914120f82ee5b4df3bba74f7be4a8c4cdba366 Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 10:01:38 +0700 Subject: [PATCH 39/80] file_server_url_prefix --- .../components/use-cases/python/financial_report/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py index 8e831ee0d..9913d54cb 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py @@ -49,7 +49,7 @@ def create_workflow() -> Workflow: ) code_interpreter_tool = E2BCodeInterpreter(api_key=e2b_api_key).to_tool() document_generator_tool = DocumentGenerator( - file_server_url_prefix=server_settings.file_server_url_prefix, + file_server_url_prefix="/", ).to_tool() return FinancialReportWorkflow( From f31452830629ed2764d8672ddf48fbe579829344 Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 16:47:38 +0700 Subject: [PATCH 40/80] add file server url when init chat workflow --- .../next/app/api/files/[...slug]/route.ts | 37 +++++++++++++++++++ packages/server/next/app/api/files/helpers.ts | 30 +++++++++++++++ .../app/components/ui/chat/chat-section.tsx | 2 + packages/server/next/app/constants.ts | 3 ++ 4 files changed, 72 insertions(+) diff --git a/packages/server/next/app/api/files/[...slug]/route.ts b/packages/server/next/app/api/files/[...slug]/route.ts index ca31e1a37..2a2f05ac6 100644 --- a/packages/server/next/app/api/files/[...slug]/route.ts +++ b/packages/server/next/app/api/files/[...slug]/route.ts @@ -1,6 +1,8 @@ import fs from "fs"; +import { LLamaCloudFileService } from "llamaindex"; import { NextRequest, NextResponse } from "next/server"; import { promisify } from "util"; +import { downloadFile } from "../helpers"; export async function GET( request: NextRequest, @@ -12,6 +14,41 @@ export async function GET( return NextResponse.json({ error: "No permission" }, { status: 400 }); } + // if llamacloud file, check if exists, if not, download it + if (filePath.startsWith("output/llamacloud")) { + const fileExists = await promisify(fs.exists)(filePath); + if (!fileExists) { + // download the file + + // get file name and pipeline id from the file path: output/llamacloud/pipeline_id$file_name + const [pipeline_id, file_name] = filePath.split("/").slice(-2) ?? []; + + if (!pipeline_id || !file_name) { + return NextResponse.json( + { + error: `Invalid LlamaCloud file path: ${filePath}`, + }, + { status: 400 }, + ); + } + + // get the file url from llama cloud + const downloadUrl = await LLamaCloudFileService.getFileUrl( + pipeline_id, + file_name, + ); + if (!downloadUrl) { + return NextResponse.json( + { + error: `Cannot find the file in LlamaCloud: pipeline_id=${pipeline_id}, file_name=${file_name}`, + }, + { status: 404 }, + ); + } + await downloadFile(downloadUrl, filePath); + } + } + const decodedFilePath = decodeURIComponent(filePath); const fileExists = await promisify(fs.exists)(decodedFilePath); diff --git a/packages/server/next/app/api/files/helpers.ts b/packages/server/next/app/api/files/helpers.ts index d21ff7e9d..ae44decb4 100644 --- a/packages/server/next/app/api/files/helpers.ts +++ b/packages/server/next/app/api/files/helpers.ts @@ -1,5 +1,6 @@ import crypto from "node:crypto"; import fs from "node:fs"; +import https from "node:https"; import path from "node:path"; import { type ServerFile } from "@llamaindex/server"; @@ -55,3 +56,32 @@ async function saveFile(filepath: string, content: string | Buffer) { function sanitizeFileName(fileName: string) { return fileName.replace(/[^a-zA-Z0-9_-]/g, "_"); } + +export async function downloadFile( + urlToDownload: string, + downloadedPath: string, +) { + try { + // Check if file already exists + if (fs.existsSync(downloadedPath)) return; + + const file = fs.createWriteStream(downloadedPath); + https + .get(urlToDownload, (response) => { + response.pipe(file); + file.on("finish", () => { + file.close(() => { + console.log("File downloaded successfully"); + }); + }); + }) + .on("error", (err) => { + fs.unlink(downloadedPath, () => { + console.error("Error downloading file:", err); + throw err; + }); + }); + } catch (error) { + throw new Error(`Error downloading file: ${error}`); + } +} diff --git a/packages/server/next/app/components/ui/chat/chat-section.tsx b/packages/server/next/app/components/ui/chat/chat-section.tsx index 75a75e850..e101aac6d 100644 --- a/packages/server/next/app/components/ui/chat/chat-section.tsx +++ b/packages/server/next/app/components/ui/chat/chat-section.tsx @@ -3,6 +3,7 @@ import { ChatSection as ChatUI, useChatWorkflow } from "@llamaindex/chat-ui"; import { useChat } from "ai/react"; import { useEffect, useMemo, useState } from "react"; +import { FILE_SERVER_URL } from "../../../constants"; import { getConfig } from "../lib/utils"; import { ResizablePanel, ResizablePanelGroup } from "../resizable"; import { ChatCanvasPanel } from "./canvas/panel"; @@ -38,6 +39,7 @@ export default function ChatSection() { }); const useChatWorkflowHandler = useChatWorkflow({ + fileServerUrl: FILE_SERVER_URL, deployment, workflow, onError: handleError, diff --git a/packages/server/next/app/constants.ts b/packages/server/next/app/constants.ts index 2650fa451..14fc71f03 100644 --- a/packages/server/next/app/constants.ts +++ b/packages/server/next/app/constants.ts @@ -9,3 +9,6 @@ const DEFAULT_SCRIPT_PATH = "./config.js"; export const SCRIPT_PATH = BASE_PATH ? `${BASE_PATH}/config.js` : DEFAULT_SCRIPT_PATH; + +// default URL for the file server +export const FILE_SERVER_URL = `${BASE_PATH}/api/files`; From 586998c0f84e2a08dac3a3b56660eb0cd4b44c30 Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 17:18:12 +0700 Subject: [PATCH 41/80] update llamacloud gen --- .../llamacloud/python/generate.py | 74 +++++---- .../llamacloud/python/index.py | 149 +++++++++++++++++- .../llamacloud/python/service.py | 74 +++++++++ 3 files changed, 261 insertions(+), 36 deletions(-) create mode 100644 packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/service.py diff --git a/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/generate.py b/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/generate.py index 67d92bf61..3c96e3184 100644 --- a/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/generate.py +++ b/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/generate.py @@ -6,12 +6,12 @@ import logging -from app.index import get_index -from app.settings import init_settings -from llama_index.server.services.llamacloud.generate import ( - load_to_llamacloud, -) +from llama_index.core.readers import SimpleDirectoryReader +from tqdm import tqdm +from src.index import get_index +from src.service import LLamaCloudFileService +from src.settings import init_settings logging.basicConfig(level=logging.INFO) logger = logging.getLogger() @@ -25,29 +25,41 @@ def generate_index(): if index is None: raise ValueError("Index not found and could not be created") - load_to_llamacloud(index, logger=logger) - - -def generate_ui_for_workflow(): - """ - Generate UI for UIEventData event in app/workflow.py - """ - import asyncio - from llama_index.llms.openai import OpenAI - from main import COMPONENT_DIR - - # To generate UI components for additional event types, - # import the corresponding data model (e.g., MyCustomEventData) - # and run the generate_ui_for_workflow function with the imported model. - # Make sure the output filename of the generated UI component matches the event type (here `ui_event`) - try: - from app.workflow import UIEventData # type: ignore - except ImportError: - raise ImportError("Couldn't generate UI component for the current workflow.") - from llama_index.server.gen_ui import generate_event_component - - # works also well with Claude 3.7 Sonnet or Gemini Pro 2.5 - llm = OpenAI(model="gpt-4.1") - code = asyncio.run(generate_event_component(event_cls=UIEventData, llm=llm)) - with open(f"{COMPONENT_DIR}/ui_event.jsx", "w") as f: - f.write(code) + # use SimpleDirectoryReader to retrieve the files to process + reader = SimpleDirectoryReader( + "data", + recursive=True, + ) + files_to_process = reader.input_files + + # add each file to the LlamaCloud pipeline + error_files = [] + for input_file in tqdm( + files_to_process, + desc="Processing files", + unit="file", + ): + with open(input_file, "rb") as f: + logger.debug( + f"Adding file {input_file} to pipeline {index.name} in project {index.project_name}" + ) + try: + LLamaCloudFileService.add_file_to_pipeline( + index.project.id, + index.pipeline.id, + f, + custom_metadata={}, + wait_for_processing=False, + ) + except Exception as e: + error_files.append(input_file) + logger.error(f"Error adding file {input_file}: {e}") + + if error_files: + logger.error(f"Failed to add the following files: {error_files}") + + logger.info("Finished generating the index") + + +if __name__ == "__main__": + generate_index() diff --git a/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/index.py b/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/index.py index 538d389ed..97261900a 100644 --- a/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/index.py +++ b/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/index.py @@ -1,7 +1,146 @@ -from llama_index.server.services.llamacloud import ( - LlamaCloudIndex, - get_client, - get_index, +import logging +import os +from typing import Optional + +from llama_cloud import PipelineType +from llama_index.core.callbacks import CallbackManager +from llama_index.core.ingestion.api_utils import ( + get_client as llama_cloud_get_client, ) +from llama_index.core.settings import Settings +from llama_index.indices.managed.llama_cloud import LlamaCloudIndex +from pydantic import BaseModel, Field, field_validator + +logger = logging.getLogger("uvicorn") + + +class LlamaCloudConfig(BaseModel): + # Private attributes + api_key: str = Field( + exclude=True, # Exclude from the model representation + ) + base_url: Optional[str] = Field( + exclude=True, + ) + organization_id: Optional[str] = Field( + exclude=True, + ) + # Configuration attributes, can be set by the user + pipeline: str = Field( + description="The name of the pipeline to use", + ) + project: str = Field( + description="The name of the LlamaCloud project", + ) + + def __init__(self, **kwargs): + if "api_key" not in kwargs: + kwargs["api_key"] = os.getenv("LLAMA_CLOUD_API_KEY") + if "base_url" not in kwargs: + kwargs["base_url"] = os.getenv("LLAMA_CLOUD_BASE_URL") + if "organization_id" not in kwargs: + kwargs["organization_id"] = os.getenv("LLAMA_CLOUD_ORGANIZATION_ID") + if "pipeline" not in kwargs: + kwargs["pipeline"] = os.getenv("LLAMA_CLOUD_INDEX_NAME") + if "project" not in kwargs: + kwargs["project"] = os.getenv("LLAMA_CLOUD_PROJECT_NAME") + super().__init__(**kwargs) + + # Validate and throw error if the env variables are not set before starting the app + @field_validator("pipeline", "project", "api_key", mode="before") + @classmethod + def validate_fields(cls, value): + if value is None: + raise ValueError( + "Please set LLAMA_CLOUD_INDEX_NAME, LLAMA_CLOUD_PROJECT_NAME and LLAMA_CLOUD_API_KEY" + " to your environment variables or config them in .env file" + ) + return value + + def to_client_kwargs(self) -> dict: + return { + "api_key": self.api_key, + "base_url": self.base_url, + } + + +class IndexConfig(BaseModel): + llama_cloud_pipeline_config: LlamaCloudConfig = Field( + default_factory=lambda: LlamaCloudConfig(), + alias="llamaCloudPipeline", + ) + callback_manager: Optional[CallbackManager] = Field( + default=None, + ) + + def to_index_kwargs(self) -> dict: + return { + "name": self.llama_cloud_pipeline_config.pipeline, + "project_name": self.llama_cloud_pipeline_config.project, + "api_key": self.llama_cloud_pipeline_config.api_key, + "base_url": self.llama_cloud_pipeline_config.base_url, + "organization_id": self.llama_cloud_pipeline_config.organization_id, + "callback_manager": self.callback_manager, + } + + +def get_index( + config: IndexConfig = None, + create_if_missing: bool = False, +): + if config is None: + config = IndexConfig() + # Check whether the index exists + try: + index = LlamaCloudIndex(**config.to_index_kwargs()) + return index + except ValueError: + logger.warning("Index not found") + if create_if_missing: + logger.info("Creating index") + _create_index(config) + return LlamaCloudIndex(**config.to_index_kwargs()) + return None + + +def get_client(): + config = LlamaCloudConfig() + return llama_cloud_get_client(**config.to_client_kwargs()) + + +def _create_index( + config: IndexConfig, +): + client = get_client() + pipeline_name = config.llama_cloud_pipeline_config.pipeline + + pipelines = client.pipelines.search_pipelines( + pipeline_name=pipeline_name, + pipeline_type=PipelineType.MANAGED.value, + ) + if len(pipelines) == 0: + from llama_index.embeddings.openai import OpenAIEmbedding -__all__ = ["LlamaCloudIndex", "get_client", "get_index"] + if not isinstance(Settings.embed_model, OpenAIEmbedding): + raise ValueError( + "Creating a new pipeline with a non-OpenAI embedding model is not supported." + ) + client.pipelines.upsert_pipeline( + request={ + "name": pipeline_name, + "embedding_config": { + "type": "OPENAI_EMBEDDING", + "component": { + "api_key": os.getenv("OPENAI_API_KEY"), # editable + "model_name": os.getenv("EMBEDDING_MODEL"), + }, + }, + "transform_config": { + "mode": "auto", + "config": { + "chunk_size": Settings.chunk_size, # editable + "chunk_overlap": Settings.chunk_overlap, # editable + }, + }, + }, + ) diff --git a/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/service.py b/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/service.py new file mode 100644 index 000000000..8d4761a70 --- /dev/null +++ b/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/service.py @@ -0,0 +1,74 @@ +import logging +import os +import time +import typing +from io import BytesIO +from typing import Any, Dict, List, Optional, Set, Tuple, Union + +from llama_cloud import ManagedIngestionStatus, PipelineFileCreateCustomMetadataValue +from pydantic import BaseModel + +from src.index import get_client + +logger = logging.getLogger("uvicorn") + + +class LlamaCloudFile(BaseModel): + file_name: str + pipeline_id: str + + def __eq__(self, other): + if not isinstance(other, LlamaCloudFile): + return NotImplemented + return ( + self.file_name == other.file_name and self.pipeline_id == other.pipeline_id + ) + + def __hash__(self): + return hash((self.file_name, self.pipeline_id)) + + +class LLamaCloudFileService: + LOCAL_STORE_PATH = "output/llamacloud" + DOWNLOAD_FILE_NAME_TPL = "{pipeline_id}${filename}" + + @classmethod + def add_file_to_pipeline( + cls, + project_id: str, + pipeline_id: str, + upload_file: Union[typing.IO, Tuple[str, BytesIO]], + custom_metadata: Optional[Dict[str, PipelineFileCreateCustomMetadataValue]], + wait_for_processing: bool = True, + ) -> str: + client = get_client() + file = client.files.upload_file(project_id=project_id, upload_file=upload_file) + file_id = file.id + files = [ + { + "file_id": file_id, + "custom_metadata": {"file_id": file_id, **(custom_metadata or {})}, + } + ] + files = client.pipelines.add_files_to_pipeline_api(pipeline_id, request=files) + + if not wait_for_processing: + return file_id + + # Wait 2s for the file to be processed + max_attempts = 20 + attempt = 0 + while attempt < max_attempts: + result = client.pipelines.get_pipeline_file_status( + file_id=file_id, pipeline_id=pipeline_id + ) + if result.status == ManagedIngestionStatus.ERROR: + raise Exception(f"File processing failed: {str(result)}") + if result.status == ManagedIngestionStatus.SUCCESS: + # File is ingested - return the file id + return file_id + attempt += 1 + time.sleep(0.1) # Sleep for 100ms + raise Exception( + f"File processing did not complete after {max_attempts} attempts." + ) From e96963133c8aca7ddd86a4d29811f27912bad4bb Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 17:22:40 +0700 Subject: [PATCH 42/80] copy llamacloud vectordbs --- packages/create-llama/helpers/python.ts | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/packages/create-llama/helpers/python.ts b/packages/create-llama/helpers/python.ts index d5dcc5add..75e8b52a9 100644 --- a/packages/create-llama/helpers/python.ts +++ b/packages/create-llama/helpers/python.ts @@ -496,19 +496,7 @@ const installLlamaIndexServerTemplate = async ({ }); if (useLlamaParse) { - await copy("index.py", srcDir, { - parents: true, - cwd: path.join( - templatesDir, - "components", - "vectordbs", - "llamaindexserver", - "llamacloud", - "python", - ), - }); - // TODO: Consider moving generate.py to app folder. - await copy("generate.py", srcDir, { + await copy("**", srcDir, { parents: true, cwd: path.join( templatesDir, From 3083fd6932485b749f9cd64f91f3435e4545352f Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 17:59:41 +0700 Subject: [PATCH 43/80] fix download --- .../server/next/app/api/files/[...slug]/route.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/server/next/app/api/files/[...slug]/route.ts b/packages/server/next/app/api/files/[...slug]/route.ts index 2a2f05ac6..4a8bf76b6 100644 --- a/packages/server/next/app/api/files/[...slug]/route.ts +++ b/packages/server/next/app/api/files/[...slug]/route.ts @@ -1,6 +1,7 @@ import fs from "fs"; import { LLamaCloudFileService } from "llamaindex"; import { NextRequest, NextResponse } from "next/server"; +import path from "node:path"; import { promisify } from "util"; import { downloadFile } from "../helpers"; @@ -18,10 +19,12 @@ export async function GET( if (filePath.startsWith("output/llamacloud")) { const fileExists = await promisify(fs.exists)(filePath); if (!fileExists) { - // download the file - // get file name and pipeline id from the file path: output/llamacloud/pipeline_id$file_name - const [pipeline_id, file_name] = filePath.split("/").slice(-2) ?? []; + const startIndex = + filePath.indexOf("output/llamacloud/") + "output/llamacloud/".length; + const fileName = filePath.slice(startIndex); // pipeline_id$file_name + const [pipeline_id, file_name] = fileName.split("$") ?? []; + console.log({ pipeline_id, file_name, filePath }); if (!pipeline_id || !file_name) { return NextResponse.json( @@ -46,6 +49,10 @@ export async function GET( ); } await downloadFile(downloadUrl, filePath); + + console.log("Current working directory: ", process.cwd()); + console.log("File downloaded successfully to: ", filePath); + console.log("Absolute path: ", path.join(process.cwd(), filePath)); } } From edf1a37f7fa8c2b4439ee12fb35f141a73ac0b33 Mon Sep 17 00:00:00 2001 From: thucpn Date: Mon, 7 Jul 2025 18:45:43 +0700 Subject: [PATCH 44/80] fix download --- .../next/app/api/files/[...slug]/route.ts | 5 --- packages/server/next/app/api/files/helpers.ts | 37 +++++++++++-------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/server/next/app/api/files/[...slug]/route.ts b/packages/server/next/app/api/files/[...slug]/route.ts index 4a8bf76b6..41ed441c4 100644 --- a/packages/server/next/app/api/files/[...slug]/route.ts +++ b/packages/server/next/app/api/files/[...slug]/route.ts @@ -1,7 +1,6 @@ import fs from "fs"; import { LLamaCloudFileService } from "llamaindex"; import { NextRequest, NextResponse } from "next/server"; -import path from "node:path"; import { promisify } from "util"; import { downloadFile } from "../helpers"; @@ -24,7 +23,6 @@ export async function GET( filePath.indexOf("output/llamacloud/") + "output/llamacloud/".length; const fileName = filePath.slice(startIndex); // pipeline_id$file_name const [pipeline_id, file_name] = fileName.split("$") ?? []; - console.log({ pipeline_id, file_name, filePath }); if (!pipeline_id || !file_name) { return NextResponse.json( @@ -49,10 +47,7 @@ export async function GET( ); } await downloadFile(downloadUrl, filePath); - - console.log("Current working directory: ", process.cwd()); console.log("File downloaded successfully to: ", filePath); - console.log("Absolute path: ", path.join(process.cwd(), filePath)); } } diff --git a/packages/server/next/app/api/files/helpers.ts b/packages/server/next/app/api/files/helpers.ts index ae44decb4..7bb003f0b 100644 --- a/packages/server/next/app/api/files/helpers.ts +++ b/packages/server/next/app/api/files/helpers.ts @@ -56,32 +56,37 @@ async function saveFile(filepath: string, content: string | Buffer) { function sanitizeFileName(fileName: string) { return fileName.replace(/[^a-zA-Z0-9_-]/g, "_"); } - export async function downloadFile( urlToDownload: string, downloadedPath: string, -) { - try { - // Check if file already exists - if (fs.existsSync(downloadedPath)) return; - +): Promise { + return new Promise((resolve, reject) => { + const dir = path.dirname(downloadedPath); + fs.mkdirSync(dir, { recursive: true }); const file = fs.createWriteStream(downloadedPath); + https .get(urlToDownload, (response) => { + if (response.statusCode !== 200) { + reject( + new Error(`Failed to download file: Status ${response.statusCode}`), + ); + return; + } + response.pipe(file); + file.on("finish", () => { - file.close(() => { - console.log("File downloaded successfully"); - }); + file.close(); + resolve(); + }); + + file.on("error", (err) => { + fs.unlink(downloadedPath, () => reject(err)); }); }) .on("error", (err) => { - fs.unlink(downloadedPath, () => { - console.error("Error downloading file:", err); - throw err; - }); + fs.unlink(downloadedPath, () => reject(err)); }); - } catch (error) { - throw new Error(`Error downloading file: ${error}`); - } + }); } From 59ddab247e438aa8f20a631fce666737a08705cd Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 09:09:05 +0700 Subject: [PATCH 45/80] remove Python HITL --- .../e2e/python/resolve_dependencies.spec.ts | 4 +- .../shared/llamaindexserver_template.spec.ts | 7 +- .../typescript/resolve_dependencies.spec.ts | 4 +- packages/create-llama/helpers/types.ts | 11 +- packages/create-llama/questions/index.ts | 30 ++-- .../use-cases/python/hitl/README-template.md | 141 ------------------ .../use-cases/python/hitl/events.py | 28 ---- .../use-cases/python/hitl/workflow.py | 86 ----------- 8 files changed, 38 insertions(+), 273 deletions(-) delete mode 100644 packages/create-llama/templates/components/use-cases/python/hitl/README-template.md delete mode 100644 packages/create-llama/templates/components/use-cases/python/hitl/events.py delete mode 100644 packages/create-llama/templates/components/use-cases/python/hitl/workflow.py diff --git a/packages/create-llama/e2e/python/resolve_dependencies.spec.ts b/packages/create-llama/e2e/python/resolve_dependencies.spec.ts index 6075cf121..ac7790d96 100644 --- a/packages/create-llama/e2e/python/resolve_dependencies.spec.ts +++ b/packages/create-llama/e2e/python/resolve_dependencies.spec.ts @@ -4,7 +4,7 @@ import fs from "fs"; import path from "path"; import util from "util"; import { - ALL_USE_CASES, + ALL_PYTHON_USE_CASES, TemplateFramework, TemplateVectorDB, } from "../../helpers/types"; @@ -21,7 +21,7 @@ test.describe("Mypy check", () => { test.describe.configure({ retries: 0 }); test.describe("LlamaIndexServer", async () => { - for (const useCase of ALL_USE_CASES) { + for (const useCase of ALL_PYTHON_USE_CASES) { test(`should pass mypy for use case: ${useCase}`, async () => { const cwd = await createTestDir(); await createAndCheckLlamaProject({ diff --git a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts index 1c6d9343d..8c9f541ae 100644 --- a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts +++ b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts @@ -3,7 +3,8 @@ import { ChildProcess } from "child_process"; import fs from "fs"; import path from "path"; import { - ALL_USE_CASES, + ALL_NEXTJS_USE_CASES, + ALL_PYTHON_USE_CASES, type TemplateFramework, type TemplateVectorDB, } from "../../helpers"; @@ -17,10 +18,12 @@ const vectorDb: TemplateVectorDB = process.env.VECTORDB : "none"; const llamaCloudProjectName = "create-llama"; const llamaCloudIndexName = "e2e-test"; +const allUseCases = + templateFramework === "nextjs" ? ALL_NEXTJS_USE_CASES : ALL_PYTHON_USE_CASES; const userMessage = "Write a blog post about physical standards for letters"; -for (const useCase of ALL_USE_CASES) { +for (const useCase of allUseCases) { test.describe(`Test use case ${useCase} ${templateFramework} ${vectorDb}`, async () => { let port: number; let cwd: string; diff --git a/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts b/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts index a19456f7a..3ca0d4a4b 100644 --- a/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts +++ b/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts @@ -4,7 +4,7 @@ import fs from "fs"; import path from "path"; import util from "util"; import { - ALL_USE_CASES, + ALL_NEXTJS_USE_CASES, TemplateFramework, TemplateUseCase, TemplateVectorDB, @@ -21,7 +21,7 @@ const vectorDb: TemplateVectorDB = process.env.VECTORDB test.describe("Test resolve TS dependencies", () => { test.describe.configure({ retries: 0 }); - for (const useCase of ALL_USE_CASES) { + for (const useCase of ALL_NEXTJS_USE_CASES) { const optionDescription = `useCase: ${useCase}, vectorDb: ${vectorDb}`; test.describe(`${optionDescription}`, () => { test(`${optionDescription}`, async () => { diff --git a/packages/create-llama/helpers/types.ts b/packages/create-llama/helpers/types.ts index 0a428ccbb..cdd3c958c 100644 --- a/packages/create-llama/helpers/types.ts +++ b/packages/create-llama/helpers/types.ts @@ -49,7 +49,7 @@ export type TemplateUseCase = | "document_generator" | "hitl"; -export const ALL_USE_CASES: TemplateUseCase[] = [ +export const ALL_NEXTJS_USE_CASES: TemplateUseCase[] = [ "agentic_rag", "deep_research", "financial_report", @@ -57,6 +57,15 @@ export const ALL_USE_CASES: TemplateUseCase[] = [ "document_generator", "hitl", ]; + +export const ALL_PYTHON_USE_CASES: TemplateUseCase[] = [ + "agentic_rag", + "deep_research", + "financial_report", + "code_generator", + "document_generator", +]; + // Config for both file and folder export type FileSourceConfig = | { diff --git a/packages/create-llama/questions/index.ts b/packages/create-llama/questions/index.ts index e7b224fc2..e62e83485 100644 --- a/packages/create-llama/questions/index.ts +++ b/packages/create-llama/questions/index.ts @@ -21,7 +21,7 @@ export const askQuestions = async ( askModels: askModelsFromArgs, } = args; - const { useCase, framework } = await prompts( + const { useCase } = await prompts( [ { type: useCaseFromArgs ? null : "select", @@ -65,20 +65,28 @@ export const askQuestions = async ( ], initial: 0, }, - { - type: frameworkFromArgs ? null : "select", - name: "framework", - message: "What language do you want to use?", - choices: [ - { title: "Python (FastAPI)", value: "fastapi" }, - { title: "Typescript (NextJS)", value: "nextjs" }, - ], - initial: 0, - }, ], questionHandlers, ); + const { framework } = await prompts( + { + type: frameworkFromArgs ? null : "select", + name: "framework", + message: "What language do you want to use?", + choices: [ + // For Python Human in the Loop use case, please refer to this chat-ui example: + // https://github.com/run-llama/chat-ui/blob/main/examples/llamadeploy/chat/src/cli_workflow.py + ...(useCase !== "hitl" + ? [{ title: "Python (FastAPI)", value: "fastapi" }] + : []), + { title: "Typescript (NextJS)", value: "nextjs" }, + ], + initial: 0, + }, + questionHandlers, + ); + const finalUseCase = (useCaseFromArgs ?? useCase) as TemplateUseCase; const finalFramework = (frameworkFromArgs ?? framework) as TemplateFramework; if (!finalUseCase) { diff --git a/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md b/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md deleted file mode 100644 index 94e6829b5..000000000 --- a/packages/create-llama/templates/components/use-cases/python/hitl/README-template.md +++ /dev/null @@ -1,141 +0,0 @@ -# LlamaIndex Workflow Example - -This is a [LlamaIndex](https://www.llamaindex.ai/) project that using [Workflows](https://docs.llamaindex.ai/en/stable/understanding/workflows/) deployed with [LlamaDeploy](https://github.com/run-llama/llama_deploy). - -LlamaDeploy is a system for deploying and managing LlamaIndex workflows, while LlamaIndexServer provides a pre-built TypeScript server with an integrated chat UI that can connect directly to LlamaDeploy deployments. This example shows how you can quickly set up a complete chat application by combining these two technologies/ - -## Prerequisites - -If you haven't installed uv, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/) to install it. - -You can configure [LLM model](https://docs.llamaindex.ai/en/stable/module_guides/models/llms) and [embedding model](https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings) in [src/settings.py](src/settings.py). - -Please setup their API keys in the `src/.env` file. - -## Installation - -Both the SDK and the CLI are part of the LlamaDeploy Python package. To install, just run: - -```bash -uv sync -``` - -If you don't have uv installed, you can follow the instructions [here](https://docs.astral.sh/uv/getting-started/installation/). - -## Running the Deployment - -At this point we have all we need to run this deployment. Ideally, we would have the API server already running -somewhere in the cloud, but to get started let's start an instance locally. Run the following python script -from a shell: - -``` -$ uv run -m llama_deploy.apiserver -INFO: Started server process [10842] -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://0.0.0.0:4501 (Press CTRL+C to quit) -``` - -From another shell, use the CLI, `llamactl`, to create the deployment: - -``` -$ uv run llamactl deploy llama_deploy.yml -Deployment successful: chat -``` - -## UI Interface - -LlamaDeploy will serve the UI through the apiserver. Point the browser to [http://localhost:4501/deployments/chat/ui](http://localhost:4501/deployments/chat/ui) to interact with your deployment through a user-friendly interface. - -## API endpoints - -You can find all the endpoints in the [API documentation](http://localhost:4501/docs). To get started, you can try the following endpoints: - -Create a new task: - -```bash -curl -X POST 'http://localhost:4501/deployments/chat/tasks/create' \ - -H 'Content-Type: application/json' \ - -d '{ - "input": "{\"user_msg\":\"Hello\",\"chat_history\":[]}", - "service_id": "workflow" - }' -``` - -Stream events: - -```bash -curl 'http://localhost:4501/deployments/chat/tasks/0b411be6-005d-43f0-9b6b-6a0017f08002/events?session_id=dd36442c-45ca-4eaa-8d75-b4e6dad1a83e&raw_event=true' \ - -H 'Content-Type: application/json' -``` - -Note that the task_id and session_id are returned when creating a new task. - -## Use Case - -This example shows how to use the LlamaIndexServer with a human in the loop. It allows you to start CLI commands that are reviewed by a human before execution. - -To update the workflow, you can modify the code in [`src/workflow.py`](src/workflow.py). - -## How does HITL work? - -### Events - -The human-in-the-loop approach used here is based on a simple idea: the workflow pauses and waits for a human response before proceeding to the next step. - -To do this, you will need to implement two custom events: - -- [HumanInputEvent](src/events.py): This event is used to request input from the user. -- [HumanResponseEvent](src/events.py): This event is sent to the workflow to resume execution with input from the user. - -In this example, we have implemented these two custom events in [`events.py`](src/events.py): - -- `cliHumanInputEvent` – to request input from the user for CLI command execution. -- `cliHumanResponseEvent` – to resume the workflow with the response from the user. - -### UI Component - -HITL also needs a custom UI component, that is shown when the LlamaIndexServer receives the `cliHumanInputEvent`. The name of the component is defined in the `type` field of the `cliHumanInputEvent` - in our case, it is `cli_human_input`, which corresponds to the [cli_human_input.tsx](./ui/components/cli_human_input.tsx) component. - -The custom component must use `append` to send a message with a `human_response` annotation. The data of the annotation must be in the format of the response event `cliHumanResponseEvent`, in our case, for sending to execute the command `ls -l`, we would send: - -```tsx -append({ - content: "Yes", - role: "user", - annotations: [ - { - type: "human_response", - data: { - execute: true, - command: "ls -l", // The command to execute - }, - }, - ], -}); -``` - -This component displays the command to execute and the user can choose to execute or cancel the command execution. - -## Customize the UI - -The UI is served by LLamaIndexServer package, you can configure the UI by modifying the `uiConfig` in the [ui/index.ts](ui/index.ts) file. - -The following are the available options: - -- `starterQuestions`: Predefined questions for chat interface -- `componentsDir`: Directory for custom event components -- `layoutDir`: Directory for custom layout components -- `llamaCloudIndexSelector`: Enable LlamaCloud integration -- `llamaDeploy`: The LlamaDeploy configration (deployment name and workflow name that defined in the [llama_deploy.yml](llama_deploy.yml) file) - -To customize the UI, you can start by modifying the [./ui/components/ui_event.jsx](./ui/components/ui_event.jsx) file. - -## Learn More - -- [LlamaIndex Documentation](https://docs.llamaindex.ai) - learn about LlamaIndex. -- [Workflows Introduction](https://docs.llamaindex.ai/en/stable/understanding/workflows/) - learn about LlamaIndex workflows. -- [LlamaDeploy GitHub Repository](https://github.com/run-llama/llama_deploy) -- [Chat-UI Documentation](https://ts.llamaindex.ai/docs/chat-ui) - -You can check out [the LlamaIndex GitHub repository](https://github.com/run-llama/llama_index) - your feedback and contributions are welcome! \ No newline at end of file diff --git a/packages/create-llama/templates/components/use-cases/python/hitl/events.py b/packages/create-llama/templates/components/use-cases/python/hitl/events.py deleted file mode 100644 index fbba17b91..000000000 --- a/packages/create-llama/templates/components/use-cases/python/hitl/events.py +++ /dev/null @@ -1,28 +0,0 @@ -from pydantic import BaseModel, Field - -from llama_index.core.workflow.events import HumanResponseEvent, InputRequiredEvent - -class CLIHumanResponseEvent(HumanResponseEvent): - execute: bool = Field( - description="True if the human wants to execute the command, False otherwise." - ) - command: str = Field(description="The command to execute.") - - -class CLICommand(BaseModel): - command: str = Field(description="The command to execute.") - - -# We need an event that extends from HumanInputEvent for HITL feature -class CLIHumanInputEvent(InputRequiredEvent): - """ - CLIInputRequiredEvent is sent when the agent needs permission from the user to execute the CLI command or not. - Render this event by showing the command and a boolean button to execute the command or not. - """ - - event_type: str = ( - "cli_human_input" # used by UI to render with appropriate component - ) - data: CLICommand = Field( # the data that sent to the UI for rendering - description="The command to execute.", - ) diff --git a/packages/create-llama/templates/components/use-cases/python/hitl/workflow.py b/packages/create-llama/templates/components/use-cases/python/hitl/workflow.py deleted file mode 100644 index 23d78db28..000000000 --- a/packages/create-llama/templates/components/use-cases/python/hitl/workflow.py +++ /dev/null @@ -1,86 +0,0 @@ -import platform -import subprocess -from typing import Any - -from app.events import CLICommand, CLIHumanInputEvent, CLIHumanResponseEvent - -from llama_index.core.prompts import PromptTemplate -from llama_index.core.settings import Settings -from llama_index.core.workflow import ( - Context, - StartEvent, - StopEvent, - Workflow, - step, -) - - -class CLIWorkflow(Workflow): - """ - A workflow has ability to execute command line tool with human in the loop for confirmation. - """ - - default_prompt = PromptTemplate( - template=""" - You are a helpful assistant who can write CLI commands to execute using {cli_language}. - Your task is to analyze the user's request and write a CLI command to execute. - - ## User Request - {user_request} - - Don't be verbose, only respond with the CLI command without any other text. - """ - ) - - def __init__(self, **kwargs: Any) -> None: - # HITL Workflow should disable timeout otherwise, we will get a timeout error from callback - kwargs["timeout"] = None - super().__init__(**kwargs) - - @step - async def start(self, ctx: Context, ev: StartEvent) -> CLIHumanInputEvent: - user_msg = ev.user_msg - if user_msg is None: - raise ValueError("Missing user_msg in StartEvent") - await ctx.set("user_msg", user_msg) - # Request LLM to generate a CLI command - os_name = platform.system() - if os_name == "Linux" or os_name == "Darwin": - cli_language = "bash" - else: - cli_language = "cmd" - prompt = self.default_prompt.format( - user_request=user_msg, cli_language=cli_language - ) - llm = Settings.llm - if llm is None: - raise ValueError("Missing LLM in Settings") - response = await llm.acomplete(prompt, formatted=True) - command = response.text.strip() - if command == "": - raise ValueError("Couldn't generate a command") - # Send the command to the user for confirmation - await ctx.set("command", command) - return CLIHumanInputEvent( # type: ignore - data=CLICommand(command=command), - response_event_type=CLIHumanResponseEvent, - ) - - @step - async def handle_human_response( - self, - ctx: Context, - ev: CLIHumanResponseEvent, # This event is sent by LlamaIndexServer when user response - ) -> StopEvent: - # If we have human response, check the confirmation and execute the command - if ev.execute: - command = ev.command or "" - if command == "": - raise ValueError("Missing command in CLIExecutionEvent") - res = subprocess.run(command, shell=True, capture_output=True, text=True) - return StopEvent(result=res.stdout or res.stderr) - else: - return StopEvent(result=None) - - -workflow = CLIWorkflow() \ No newline at end of file From eea1aed4d7cda9cb4add1fee17e63f863d65e4c3 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 11:16:47 +0700 Subject: [PATCH 46/80] write response to stream --- .../python/financial_report/utils.py | 47 +++++++++++++++++++ .../python/financial_report/workflow.py | 4 +- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 packages/create-llama/templates/components/use-cases/python/financial_report/utils.py diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/utils.py b/packages/create-llama/templates/components/use-cases/python/financial_report/utils.py new file mode 100644 index 000000000..42ad4380d --- /dev/null +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/utils.py @@ -0,0 +1,47 @@ +from typing import AsyncGenerator, Union +from llama_index.core.base.llms.types import ( + CompletionResponse, + CompletionResponseAsyncGen, +) +from llama_index.core.workflow import Context +from llama_index.core.agent.workflow.workflow_events import AgentStream + + +async def write_response_to_stream( + res: Union[CompletionResponse, CompletionResponseAsyncGen], + ctx: Context, + current_agent_name: str = "assistant", +) -> str: + """ + Handle both streaming and non-streaming LLM responses. + + Args: + res: The LLM response (either streaming or non-streaming) + ctx: The workflow context for writing events to stream + current_agent_name: The name of the current agent (default: "assistant") + + Returns: + The final response text as a string + """ + final_response = "" + + # {"__is_pydantic": true, "value": {"delta": "", "response": "", "current_agent_name": "assistant", "tool_calls": []}, "qualified_name": "llama_index.core.agent.workflow.workflow_events.AgentStream"} + + if isinstance(res, AsyncGenerator): + # Handle streaming response (CompletionResponseAsyncGen) + async for chunk in res: + ctx.write_event_to_stream( + AgentStream( + delta=chunk.delta or "", + response=final_response, + current_agent_name=current_agent_name, + tool_calls=[], + raw=chunk.raw or "", + ) + ) + final_response += chunk.delta or "" + else: + # Handle non-streaming response (CompletionResponse) + final_response = res.text + + return final_response diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py index 9913d54cb..0d1aca73e 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py @@ -31,6 +31,7 @@ AgentRunEvent, ) from src.agent_tool import call_tools, chat_with_tools +from src.utils import write_response_to_stream def create_workflow() -> Workflow: @@ -156,7 +157,8 @@ async def handle_llm_input( # type: ignore ) if not response.has_tool_calls(): if self.stream: - return StopEvent(result=response.generator) + final_response = await write_response_to_stream(response.generator, ctx) + return StopEvent(result=final_response) else: return StopEvent(result=await response.full_response()) # calling different tools at the same time is not supported at the moment From 63d94e00dc34a2af662573c659b4fc11d37fd6fd Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 11:23:40 +0700 Subject: [PATCH 47/80] fix messages --- .../use-cases/python/financial_report/workflow.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py index 0d1aca73e..73ccc2ac6 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py @@ -122,10 +122,14 @@ def __init__( async def prepare_chat_history(self, ctx: Context, ev: StartEvent) -> InputEvent: self.stream = ev.get("stream", True) user_msg = ev.get("user_msg") - chat_history = ev.get("chat_history") - - if chat_history is not None: - self.memory.put_messages(chat_history) + messages = [ + ChatMessage( + role=msg.get("role", "user"), + content=msg.get("content", ""), + ) + for msg in ev.get("chat_history", []) + ] + self.memory.put_messages(messages) # Add user message to memory self.memory.put(ChatMessage(role=MessageRole.USER, content=user_msg)) From 6662e6efcafec8846629f732e4336c8e8ffa0020 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 11:57:50 +0700 Subject: [PATCH 48/80] fix document gen --- .../use-cases/python/financial_report/document_generator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py b/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py index 4438daca9..caa4c4992 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py @@ -6,6 +6,8 @@ from llama_index.core.tools.function_tool import FunctionTool +# use nextjs for file server +FILE_SERVER_URL_PREFIX = "http://localhost:3000/deployments/chat/ui/api/files" OUTPUT_DIR = "output/tools" @@ -99,7 +101,7 @@ class DocumentType(Enum): class DocumentGenerator: - def __init__(self, file_server_url_prefix: str): + def __init__(self, file_server_url_prefix: str | None = FILE_SERVER_URL_PREFIX): if not file_server_url_prefix: raise ValueError("file_server_url_prefix is required") self.file_server_url_prefix = file_server_url_prefix @@ -192,7 +194,7 @@ def generate_document( raise ValueError(f"Unexpected document type: {document_type}") file_name = self._validate_file_name(file_name) - file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}") + file_path = os.path.join("ui", OUTPUT_DIR, f"{file_name}.{file_extension}") self._write_to_file(content, file_path) From fe6f62fab86a9fe7ac7b0cb0a7a8b0544f548509 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 12:28:11 +0700 Subject: [PATCH 49/80] data inside ui dir for python llamadeloy --- packages/create-llama/helpers/index.ts | 17 +++++++++-------- packages/create-llama/helpers/python.ts | 6 ++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/create-llama/helpers/index.ts b/packages/create-llama/helpers/index.ts index 17e58ac8b..d822608a3 100644 --- a/packages/create-llama/helpers/index.ts +++ b/packages/create-llama/helpers/index.ts @@ -117,8 +117,13 @@ const downloadFile = async (url: string, destPath: string) => { const prepareContextData = async ( root: string, dataSources: TemplateDataSource[], + isPythonLlamaDeploy: boolean, ) => { - await makeDir(path.join(root, "data")); + const dataDir = isPythonLlamaDeploy + ? path.join(root, "ui", "data") + : path.join(root, "data"); + + await makeDir(dataDir); for (const dataSource of dataSources) { const dataSourceConfig = dataSource?.config as FileSourceConfig; // If the path is URLs, download the data and save it to the data directory @@ -128,8 +133,7 @@ const prepareContextData = async ( dataSourceConfig.url.toString(), ); const destPath = path.join( - root, - "data", + dataDir, dataSourceConfig.filename ?? path.basename(dataSourceConfig.url.toString()), ); @@ -137,11 +141,7 @@ const prepareContextData = async ( } else { // Copy local data console.log("Copying data from path:", dataSourceConfig.path); - const destPath = path.join( - root, - "data", - path.basename(dataSourceConfig.path), - ); + const destPath = path.join(dataDir, path.basename(dataSourceConfig.path)); await fsExtra.copy(dataSourceConfig.path, destPath); } } @@ -167,6 +167,7 @@ export const installTemplate = async (props: InstallTemplateArgs) => { await prepareContextData( props.root, props.dataSources.filter((ds) => ds.type === "file"), + isPythonLlamaDeploy, ); if ( diff --git a/packages/create-llama/helpers/python.ts b/packages/create-llama/helpers/python.ts index 75e8b52a9..b1e43392e 100644 --- a/packages/create-llama/helpers/python.ts +++ b/packages/create-llama/helpers/python.ts @@ -448,10 +448,12 @@ const installLlamaIndexServerTemplate = async ({ * ├── generate.py * ├── ... (other utility files) * ui/ - * ├── index.ts - * └── package.json + * ├── data/*.pdf + * ├── output/*.pdf * ├── components/*.tsx * ├── layout/*.tsx + * ├── index.ts + * └── package.json * llama_deploy.yaml * pyproject.toml * README.md From 46186a97cba37568e5c05f2480f5d8f50e2bd2df Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 14:44:48 +0700 Subject: [PATCH 50/80] construct file server url --- .../next/app/api/files/[...slug]/route.ts | 71 ++++++++++--------- .../app/components/ui/chat/chat-section.tsx | 3 +- packages/server/next/app/constants.ts | 3 - packages/server/src/server.ts | 27 +++++-- packages/server/src/types.ts | 2 + 5 files changed, 64 insertions(+), 42 deletions(-) diff --git a/packages/server/next/app/api/files/[...slug]/route.ts b/packages/server/next/app/api/files/[...slug]/route.ts index 41ed441c4..7044d0c38 100644 --- a/packages/server/next/app/api/files/[...slug]/route.ts +++ b/packages/server/next/app/api/files/[...slug]/route.ts @@ -8,52 +8,45 @@ export async function GET( request: NextRequest, { params }: { params: Promise<{ slug: string[] }> }, ) { + const isUsingLlamaCloud = !!process.env.LLAMA_CLOUD_API_KEY; const filePath = (await params).slug.join("/"); if (!filePath.startsWith("output") && !filePath.startsWith("data")) { return NextResponse.json({ error: "No permission" }, { status: 400 }); } - // if llamacloud file, check if exists, if not, download it - if (filePath.startsWith("output/llamacloud")) { - const fileExists = await promisify(fs.exists)(filePath); + const decodedFilePath = decodeURIComponent(filePath); + + // if using llama cloud and file not exists, download it + if (isUsingLlamaCloud) { + const fileExists = await promisify(fs.exists)(decodedFilePath); if (!fileExists) { - // get file name and pipeline id from the file path: output/llamacloud/pipeline_id$file_name - const startIndex = - filePath.indexOf("output/llamacloud/") + "output/llamacloud/".length; - const fileName = filePath.slice(startIndex); // pipeline_id$file_name - const [pipeline_id, file_name] = fileName.split("$") ?? []; - - if (!pipeline_id || !file_name) { - return NextResponse.json( - { - error: `Invalid LlamaCloud file path: ${filePath}`, - }, - { status: 400 }, - ); - } + const { pipeline_id, file_name } = + getLlamaCloudPipelineIdAndFileName(decodedFilePath); - // get the file url from llama cloud - const downloadUrl = await LLamaCloudFileService.getFileUrl( - pipeline_id, - file_name, - ); - if (!downloadUrl) { - return NextResponse.json( - { - error: `Cannot find the file in LlamaCloud: pipeline_id=${pipeline_id}, file_name=${file_name}`, - }, - { status: 404 }, + if (pipeline_id && file_name) { + // get the file url from llama cloud + const downloadUrl = await LLamaCloudFileService.getFileUrl( + pipeline_id, + file_name, ); + if (!downloadUrl) { + return NextResponse.json( + { + error: `Cannot create LlamaCloud download url for pipeline_id=${pipeline_id}, file_name=${file_name}`, + }, + { status: 404 }, + ); + } + + // download the LlamaCloud file to local + await downloadFile(downloadUrl, decodedFilePath); + console.log("File downloaded successfully to: ", decodedFilePath); } - await downloadFile(downloadUrl, filePath); - console.log("File downloaded successfully to: ", filePath); } } - const decodedFilePath = decodeURIComponent(filePath); const fileExists = await promisify(fs.exists)(decodedFilePath); - if (fileExists) { const fileBuffer = await promisify(fs.readFile)(decodedFilePath); return new NextResponse(fileBuffer); @@ -61,3 +54,17 @@ export async function GET( return NextResponse.json({ error: "File not found" }, { status: 404 }); } } + +function getLlamaCloudPipelineIdAndFileName(filePath: string) { + const fileName = filePath.split("/").pop() ?? ""; // fileName is the last slug part (pipeline_id$file_name) + + const delimiterIndex = fileName.indexOf("$"); // delimiter is the first dollar sign in the fileName + if (delimiterIndex === -1) { + return { pipeline_id: "", file_name: "" }; + } + + const pipeline_id = fileName.slice(0, delimiterIndex); // before delimiter + const file_name = fileName.slice(delimiterIndex + 1); // after delimiter + + return { pipeline_id, file_name }; +} diff --git a/packages/server/next/app/components/ui/chat/chat-section.tsx b/packages/server/next/app/components/ui/chat/chat-section.tsx index e101aac6d..8d7d296a5 100644 --- a/packages/server/next/app/components/ui/chat/chat-section.tsx +++ b/packages/server/next/app/components/ui/chat/chat-section.tsx @@ -3,7 +3,6 @@ import { ChatSection as ChatUI, useChatWorkflow } from "@llamaindex/chat-ui"; import { useChat } from "ai/react"; import { useEffect, useMemo, useState } from "react"; -import { FILE_SERVER_URL } from "../../../constants"; import { getConfig } from "../lib/utils"; import { ResizablePanel, ResizablePanelGroup } from "../resizable"; import { ChatCanvasPanel } from "./canvas/panel"; @@ -39,7 +38,7 @@ export default function ChatSection() { }); const useChatWorkflowHandler = useChatWorkflow({ - fileServerUrl: FILE_SERVER_URL, + fileServerUrl: getConfig("FILE_SERVER_URL"), deployment, workflow, onError: handleError, diff --git a/packages/server/next/app/constants.ts b/packages/server/next/app/constants.ts index 14fc71f03..2650fa451 100644 --- a/packages/server/next/app/constants.ts +++ b/packages/server/next/app/constants.ts @@ -9,6 +9,3 @@ const DEFAULT_SCRIPT_PATH = "./config.js"; export const SCRIPT_PATH = BASE_PATH ? `${BASE_PATH}/config.js` : DEFAULT_SCRIPT_PATH; - -// default URL for the file server -export const FILE_SERVER_URL = `${BASE_PATH}/api/files`; diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index 4d78d0f5a..0dcdcadfb 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -12,7 +12,6 @@ import type { LlamaDeployConfig, LlamaIndexServerOptions } from "./types"; const nextDir = path.join(__dirname, "..", "server"); const configFile = path.join(__dirname, "..", "server", "public", "config.js"); const nextConfigFile = path.join(nextDir, "next.config.ts"); -const layoutFile = path.join(nextDir, "app", "layout.tsx"); const constantsFile = path.join(nextDir, "app", "constants.ts"); const dev = process.env.NODE_ENV !== "production"; @@ -24,6 +23,8 @@ export class LlamaIndexServer { layoutDir: string; suggestNextQuestions: boolean; llamaDeploy?: LlamaDeployConfig | undefined; + serverUrl: string; + fileServer: string; constructor(options: LlamaIndexServerOptions) { const { workflow, suggestNextQuestions, ...nextAppOptions } = options; @@ -33,7 +34,13 @@ export class LlamaIndexServer { this.componentsDir = options.uiConfig?.componentsDir; this.layoutDir = options.uiConfig?.layoutDir ?? "layout"; this.suggestNextQuestions = suggestNextQuestions ?? true; + this.llamaDeploy = options.uiConfig?.llamaDeploy; + this.serverUrl = options.uiConfig?.serverUrl || "http://localhost:3000"; + + const isUsingLlamaCloud = !!getEnv("LLAMA_CLOUD_API_KEY"); + const defaultFileServer = isUsingLlamaCloud ? "output/llamacloud" : "data"; + this.fileServer = options.fileServer ?? defaultFileServer; if (this.llamaDeploy) { if (!this.llamaDeploy.deployment || !this.llamaDeploy.workflow) { @@ -41,9 +48,13 @@ export class LlamaIndexServer { "LlamaDeploy requires deployment and workflow to be set", ); } - if (options.uiConfig?.devMode) { - // workflow file is in llama-deploy src, so we should disable devmode - throw new Error("Devmode is not supported when enabling LlamaDeploy"); + const { devMode, llamaCloudIndexSelector, enableFileUpload } = + options.uiConfig ?? {}; + + if (devMode || llamaCloudIndexSelector || enableFileUpload) { + throw new Error( + "`devMode`, `llamaCloudIndexSelector`, and `enableFileUpload` are not supported when enabling LlamaDeploy", + ); } } else { // if llamaDeploy is not set but workflowFactory is not defined, we should throw an error @@ -103,6 +114,11 @@ export default { const enableFileUpload = uiConfig?.enableFileUpload ?? false; const uploadApi = enableFileUpload ? `${basePath}/api/files` : undefined; + // construct file server url for LlamaDeploy + // eg. for Non-LlamaCloud: localhost:3000/deployments/chat/ui/api/files/data + // eg. for LlamaCloud: localhost:3000/deployments/chat/ui/api/files/output/llamacloud + const fileServerUrl = `${this.serverUrl}/${basePath}/api/files/${this.fileServer}`; + // content in javascript format const content = ` window.LLAMAINDEX = { @@ -115,7 +131,8 @@ export default { SUGGEST_NEXT_QUESTIONS: ${JSON.stringify(this.suggestNextQuestions)}, UPLOAD_API: ${JSON.stringify(uploadApi)}, DEPLOYMENT: ${JSON.stringify(this.llamaDeploy?.deployment)}, - WORKFLOW: ${JSON.stringify(this.llamaDeploy?.workflow)} + WORKFLOW: ${JSON.stringify(this.llamaDeploy?.workflow)}, + FILE_SERVER_URL: ${JSON.stringify(fileServerUrl)} } `; fs.writeFileSync(configFile, content); diff --git a/packages/server/src/types.ts b/packages/server/src/types.ts index 3dd555955..f5b68cd4a 100644 --- a/packages/server/src/types.ts +++ b/packages/server/src/types.ts @@ -25,10 +25,12 @@ export type UIConfig = { devMode?: boolean; enableFileUpload?: boolean; llamaDeploy?: LlamaDeployConfig; + serverUrl?: string; }; export type LlamaIndexServerOptions = NextAppOptions & { workflow?: WorkflowFactory; uiConfig?: UIConfig; + fileServer?: string; suggestNextQuestions?: boolean; }; From ed027bea1aa6f04b2985fec53ed6b00109fb768d Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 17:41:25 +0700 Subject: [PATCH 51/80] fix basePath --- packages/server/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index 0dcdcadfb..8b2135bd7 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -117,7 +117,7 @@ export default { // construct file server url for LlamaDeploy // eg. for Non-LlamaCloud: localhost:3000/deployments/chat/ui/api/files/data // eg. for LlamaCloud: localhost:3000/deployments/chat/ui/api/files/output/llamacloud - const fileServerUrl = `${this.serverUrl}/${basePath}/api/files/${this.fileServer}`; + const fileServerUrl = `${this.serverUrl}${basePath}/api/files/${this.fileServer}`; // content in javascript format const content = ` From 5045e77e537ce8c43e2d78891ec93e2cffc6454c Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 17:41:31 +0700 Subject: [PATCH 52/80] fix data path --- .../templates/types/llamaindexserver/fastapi/src/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py index 906fd0bea..c7a88fc2d 100644 --- a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py +++ b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/generate.py @@ -24,7 +24,7 @@ def generate_index(): logger.info("Creating new index") # load the documents and create the index reader = SimpleDirectoryReader( - os.environ.get("DATA_DIR", "data"), + os.environ.get("DATA_DIR", "ui/data"), recursive=True, ) documents = reader.load_data() From 41876514ca5a35d224a90eab6741d4e3b4d0c4c7 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 18:03:23 +0700 Subject: [PATCH 53/80] use current host if not set --- packages/server/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index 8b2135bd7..27a66b4d4 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -36,7 +36,7 @@ export class LlamaIndexServer { this.suggestNextQuestions = suggestNextQuestions ?? true; this.llamaDeploy = options.uiConfig?.llamaDeploy; - this.serverUrl = options.uiConfig?.serverUrl || "http://localhost:3000"; + this.serverUrl = options.uiConfig?.serverUrl || ""; // use current host if not set const isUsingLlamaCloud = !!getEnv("LLAMA_CLOUD_API_KEY"); const defaultFileServer = isUsingLlamaCloud ? "output/llamacloud" : "data"; From c3a56ae8ba794b01c1fb3f9ca32c83ba0bb90e48 Mon Sep 17 00:00:00 2001 From: thucpn Date: Tue, 8 Jul 2025 18:11:26 +0700 Subject: [PATCH 54/80] fix data path for llamacloud --- .../vectordbs/llamaindexserver/llamacloud/python/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/generate.py b/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/generate.py index 3c96e3184..b1333e6ed 100644 --- a/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/generate.py +++ b/packages/create-llama/templates/components/vectordbs/llamaindexserver/llamacloud/python/generate.py @@ -27,7 +27,7 @@ def generate_index(): # use SimpleDirectoryReader to retrieve the files to process reader = SimpleDirectoryReader( - "data", + "ui/data", recursive=True, ) files_to_process = reader.input_files From 637a6161bb4918b21a2df3fc0b89fe5dd0b6a1d5 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 09:29:41 +0700 Subject: [PATCH 55/80] fix fin use case --- .../components/use-cases/python/financial_report/workflow.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py index 73ccc2ac6..9c897c76a 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/workflow.py @@ -49,9 +49,7 @@ def create_workflow() -> Workflow: "E2B_API_KEY is required to use the code interpreter tool. Please check README.md to know how to get the key." ) code_interpreter_tool = E2BCodeInterpreter(api_key=e2b_api_key).to_tool() - document_generator_tool = DocumentGenerator( - file_server_url_prefix="/", - ).to_tool() + document_generator_tool = DocumentGenerator().to_tool() return FinancialReportWorkflow( query_engine_tool=query_engine_tool, From 13ce350d97eb1ab2bc03e44b5dd614de67a2e86f Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 09:30:02 +0700 Subject: [PATCH 56/80] update comment --- packages/create-llama/helpers/python.ts | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/packages/create-llama/helpers/python.ts b/packages/create-llama/helpers/python.ts index b1e43392e..d27381080 100644 --- a/packages/create-llama/helpers/python.ts +++ b/packages/create-llama/helpers/python.ts @@ -439,26 +439,6 @@ const installLlamaIndexServerTemplate = async ({ process.exit(1); } - /** - * Python use-cases structure: - * src/ - * ├── workflow.py - * ├── settings.py - * ├── index.py - * ├── generate.py - * ├── ... (other utility files) - * ui/ - * ├── data/*.pdf - * ├── output/*.pdf - * ├── components/*.tsx - * ├── layout/*.tsx - * ├── index.ts - * └── package.json - * llama_deploy.yaml - * pyproject.toml - * README.md - */ - const srcDir = path.join(root, "src"); const uiDir = path.join(root, "ui"); From b85b4b71953b33fd32b47807546c2bd3d017f746 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 10:10:50 +0700 Subject: [PATCH 57/80] remove generate and index if not needed --- packages/create-llama/helpers/python.ts | 6 ++++++ packages/create-llama/questions/index.ts | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/create-llama/helpers/python.ts b/packages/create-llama/helpers/python.ts index d27381080..d4e77f03d 100644 --- a/packages/create-llama/helpers/python.ts +++ b/packages/create-llama/helpers/python.ts @@ -496,6 +496,12 @@ const installLlamaIndexServerTemplate = async ({ cwd: path.join(templatesDir, "components", "use-cases", "python", useCase), rename: assetRelocator, }); + + // Clean up, remove generate.py and index.py for non-data use cases + if (["code_generator", "document_generator", "hitl"].includes(useCase)) { + await fs.unlink(path.join(srcDir, "generate.py")); + await fs.unlink(path.join(srcDir, "index.py")); + } }; export const installPythonTemplate = async ({ diff --git a/packages/create-llama/questions/index.ts b/packages/create-llama/questions/index.ts index e62e83485..fba133783 100644 --- a/packages/create-llama/questions/index.ts +++ b/packages/create-llama/questions/index.ts @@ -110,7 +110,12 @@ export const askQuestions = async ( // Ask for LlamaCloud let llamaCloudKey = llamaCloudKeyFromArgs ?? process.env.LLAMA_CLOUD_API_KEY; let vectorDb: TemplateVectorDB = vectorDbFromArgs ?? "none"; - if (!vectorDbFromArgs && useCaseConfig.dataSources) { + + if ( + !vectorDbFromArgs && + useCaseConfig.dataSources && + !["code_generator", "document_generator", "hitl"].includes(finalUseCase) // these use cases don't use data so no need to ask for LlamaCloud + ) { const { useLlamaCloud } = await prompts( { type: "toggle", From 4872403eb1afa9ed89d51241872a1d1c2ce2d900 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 10:42:42 +0700 Subject: [PATCH 58/80] support custom starter questions --- .../create-llama/helpers/env-variables.ts | 30 ++++++++++++++++++- .../app/components/ui/chat/chat-starter.tsx | 8 +++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index f26bfcd74..94c03ce93 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -229,6 +229,27 @@ Otherwise, use CHROMA_HOST and CHROMA_PORT config above`, } }; +const useCaseStarterQuestions: Record = { + agentic_rag: ["Letter standard in the document", "Summarize the document"], + financial_report: [ + "Compare Apple and Tesla financial performance", + "Generate a report for Tesla financial", + ], + deep_research: [ + "Research about Apple and Tesla", + "Financial performance of Tesla", + ], + code_generator: [ + "Generate a code for a simple calculator", + "Generate a code for a todo list app", + ], + document_generator: [ + "Generate a document about LlamaIndex", + "Generate a document about LLM", + ], + hitl: ["List all the files in the current directory", "Check git status"], +}; + const getModelEnvs = ( modelConfig: ModelConfig, framework: TemplateFramework, @@ -251,7 +272,14 @@ const getModelEnvs = ( value: modelConfig.embeddingModel, }, ...(isPythonLlamaDeploy - ? [] + ? [ + { + name: "NEXT_PUBLIC_STARTER_QUESTIONS", + description: + "Initial questions to display in the chat (`starterQuestions`)", + value: JSON.stringify(useCaseStarterQuestions[useCase] ?? []), + }, + ] : [ { name: "CONVERSATION_STARTERS", diff --git a/packages/server/next/app/components/ui/chat/chat-starter.tsx b/packages/server/next/app/components/ui/chat/chat-starter.tsx index 149d94253..d73dd745a 100644 --- a/packages/server/next/app/components/ui/chat/chat-starter.tsx +++ b/packages/server/next/app/components/ui/chat/chat-starter.tsx @@ -6,9 +6,13 @@ import { getConfig } from "../lib/utils"; export function ChatStarter({ className }: { className?: string }) { const { append, messages, requestData } = useChatUI(); + const starterQuestionsFromConfig = getConfig("STARTER_QUESTIONS"); + const starterQuestions = - getConfig("STARTER_QUESTIONS") ?? - JSON.parse(process.env.NEXT_PUBLIC_STARTER_QUESTIONS || "[]"); + Array.isArray(starterQuestionsFromConfig) && + starterQuestionsFromConfig?.length > 0 + ? starterQuestionsFromConfig + : JSON.parse(process.env.NEXT_PUBLIC_STARTER_QUESTIONS || "[]"); if (starterQuestions.length === 0 || messages.length > 0) return null; return ( From 80c6b648b0a3bcd6a7166e034cd051269963f48f Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 11:25:54 +0700 Subject: [PATCH 59/80] fix: generate report inside temp dir --- packages/create-llama/helpers/env-variables.ts | 2 +- .../financial_report/document_generator.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index 94c03ce93..aa5aa4e08 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -233,7 +233,7 @@ const useCaseStarterQuestions: Record = { agentic_rag: ["Letter standard in the document", "Summarize the document"], financial_report: [ "Compare Apple and Tesla financial performance", - "Generate a report for Tesla financial", + "Generate a PDF report for Tesla financial", ], deep_research: [ "Research about Apple and Tesla", diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py b/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py index caa4c4992..1a1d99b50 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/document_generator.py @@ -1,5 +1,6 @@ import logging import os +import tempfile import re from enum import Enum from io import BytesIO @@ -7,9 +8,16 @@ from llama_index.core.tools.function_tool import FunctionTool # use nextjs for file server -FILE_SERVER_URL_PREFIX = "http://localhost:3000/deployments/chat/ui/api/files" -OUTPUT_DIR = "output/tools" +WORKFLOW="chat" +# define nextjs file server url prefix +FILE_SERVER_URL_PREFIX = f"/deployments/{WORKFLOW}/ui/api/files/output/tools" + +# When deploying to llama_deploy, ui folder will be copied to deployments folder in the temp directory +# We need to save generated documents to that exact ui directory to make it accessible to the file server +# eg: /tmp/llama_deploy/deployments/chat/ui/output/tools/generated_report.pdf +LLAMA_DEPLOY_DIR = os.path.join(tempfile.gettempdir(), "llama_deploy", "deployments") +OUTPUT_DIR = os.path.join(LLAMA_DEPLOY_DIR, WORKFLOW, "ui", "output", "tools") class DocumentType(Enum): PDF = "pdf" @@ -194,12 +202,12 @@ def generate_document( raise ValueError(f"Unexpected document type: {document_type}") file_name = self._validate_file_name(file_name) - file_path = os.path.join("ui", OUTPUT_DIR, f"{file_name}.{file_extension}") + file_path = os.path.join(OUTPUT_DIR, f"{file_name}.{file_extension}") self._write_to_file(content, file_path) return ( - f"{self.file_server_url_prefix}/{OUTPUT_DIR}/{file_name}.{file_extension}" + f"{self.file_server_url_prefix}/{file_name}.{file_extension}" ) @staticmethod From 2099bf09e714ba2c52dd1ebbc53d3e2482900117 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 15:04:33 +0700 Subject: [PATCH 60/80] bump chat-ui --- packages/server/package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index c38c43dbd..9e4648edc 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -68,7 +68,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.5.12", + "@llamaindex/chat-ui": "0.5.15", "@radix-ui/react-accordion": "^1.2.3", "@radix-ui/react-alert-dialog": "^1.1.7", "@radix-ui/react-aspect-ratio": "^1.1.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb77efa29..4354875bf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -187,8 +187,8 @@ importers: specifier: 0.3.25 version: 0.3.25 '@llamaindex/chat-ui': - specifier: 0.5.12 - version: 0.5.12(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.1)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.11)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.37.1)(@lezer/highlight@1.2.1)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(yjs@13.6.27) + specifier: 0.5.15 + version: 0.5.15(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.1)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.11)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.37.1)(@lezer/highlight@1.2.1)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(yjs@13.6.27) '@llamaindex/env': specifier: ~0.1.30 version: 0.1.30 @@ -1500,8 +1500,8 @@ packages: zod: optional: true - '@llamaindex/chat-ui@0.5.12': - resolution: {integrity: sha512-Gi9MDkIakTf1S3mpeOA7vzJtBDUBgD5bcQ6KiKsoEYiEVPtUCeEcMoepTU3KoCssaGe2cvjR4tN6dnaUsiEHWQ==} + '@llamaindex/chat-ui@0.5.15': + resolution: {integrity: sha512-r2Pv2a+nkt8F2Xy7rGxZsSrTFCwOfSzNkmrgOb15gfdImVCZcSieH7/czpzhsp9JNgK/+WsQIGQxoKHkT+IuCQ==} peerDependencies: react: ^18.2.0 || ^19.0.0 || ^19.0.0-rc @@ -8546,7 +8546,7 @@ snapshots: p-retry: 6.2.1 zod: 3.25.13 - '@llamaindex/chat-ui@0.5.12(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.1)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.11)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.37.1)(@lezer/highlight@1.2.1)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(yjs@13.6.27)': + '@llamaindex/chat-ui@0.5.15(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.1)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.11)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.37.1)(@lezer/highlight@1.2.1)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(yjs@13.6.27)': dependencies: '@codemirror/lang-css': 6.3.1 '@codemirror/lang-html': 6.4.9 From e4ce71546375ba627c6416df0f0f2933b9541dbf Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 15:34:21 +0700 Subject: [PATCH 61/80] missing __init --- .../templates/types/llamaindexserver/fastapi/src/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/create-llama/templates/types/llamaindexserver/fastapi/src/__init__.py diff --git a/packages/create-llama/templates/types/llamaindexserver/fastapi/src/__init__.py b/packages/create-llama/templates/types/llamaindexserver/fastapi/src/__init__.py new file mode 100644 index 000000000..e69de29bb From 07e19decf00f7e663228d1bd6f7ec9a4168ab980 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 15:46:33 +0700 Subject: [PATCH 62/80] simplify test --- .github/workflows/e2e.yml | 165 +++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 82 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b0183c642..997be9bf3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,7 +20,8 @@ jobs: matrix: node-version: [20] python-version: ["3.11"] - os: [macos-latest, windows-latest, ubuntu-22.04] + # os: [macos-latest, windows-latest, ubuntu-22.04] + os: [macos-latest] frameworks: ["fastapi"] vectordbs: ["none", "llamacloud"] defaults: @@ -63,14 +64,14 @@ jobs: run: pnpm run pack-install working-directory: packages/create-llama - - name: Build and store server package - run: | - pnpm run build - wheel_file=$(ls dist/*.whl | head -n 1) - mkdir -p "${{ runner.temp }}" - cp "$wheel_file" "${{ runner.temp }}/" - echo "SERVER_PACKAGE_PATH=${{ runner.temp }}/$(basename "$wheel_file")" >> $GITHUB_ENV - working-directory: python/llama-index-server + # - name: Build and store server package + # run: | + # pnpm run build + # wheel_file=$(ls dist/*.whl | head -n 1) + # mkdir -p "${{ runner.temp }}" + # cp "$wheel_file" "${{ runner.temp }}/" + # echo "SERVER_PACKAGE_PATH=${{ runner.temp }}/$(basename "$wheel_file")" >> $GITHUB_ENV + # working-directory: python/llama-index-server - name: Run Playwright tests for Python run: pnpm run e2e:python @@ -92,76 +93,76 @@ jobs: overwrite: true retention-days: 30 - e2e-typescript: - name: typescript - timeout-minutes: 60 - strategy: - fail-fast: true - matrix: - node-version: [22] - os: [macos-latest, windows-latest, ubuntu-22.04] - frameworks: ["nextjs"] - vectordbs: ["none", "llamacloud"] - defaults: - run: - shell: bash - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - - uses: pnpm/action-setup@v3 - - - name: Setup Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: "pnpm" - - - name: Install dependencies - run: pnpm install - - - name: Install Playwright Browsers - run: pnpm exec playwright install --with-deps - working-directory: packages/create-llama - - - name: Build create-llama - run: pnpm run build - working-directory: packages/create-llama - - - name: Install - run: pnpm run pack-install - working-directory: packages/create-llama - - - name: Build server - run: pnpm run build - working-directory: packages/server - - - name: Pack @llamaindex/server package - run: | - pnpm pack --pack-destination "${{ runner.temp }}" - if [ "${{ runner.os }}" == "Windows" ]; then - file=$(find "${{ runner.temp }}" -name "llamaindex-server-*.tgz" | head -n 1) - mv "$file" "${{ runner.temp }}/llamaindex-server.tgz" - else - mv ${{ runner.temp }}/llamaindex-server-*.tgz ${{ runner.temp }}/llamaindex-server.tgz - fi - working-directory: packages/server - - - name: Run Playwright tests for TypeScript - run: | - pnpm run e2e:ts - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }} - FRAMEWORK: ${{ matrix.frameworks }} - VECTORDB: ${{ matrix.vectordbs }} - SERVER_PACKAGE_PATH: ${{ runner.temp }}/llamaindex-server.tgz - working-directory: packages/create-llama - - - uses: actions/upload-artifact@v4 - if: always() - with: - name: playwright-report-typescript-${{ matrix.os }}-${{ matrix.frameworks }}-${{ matrix.vectordbs}}-node${{ matrix.node-version }} - path: packages/create-llama/playwright-report/ - overwrite: true - retention-days: 30 + # e2e-typescript: + # name: typescript + # timeout-minutes: 60 + # strategy: + # fail-fast: true + # matrix: + # node-version: [22] + # os: [macos-latest, windows-latest, ubuntu-22.04] + # frameworks: ["nextjs"] + # vectordbs: ["none", "llamacloud"] + # defaults: + # run: + # shell: bash + # runs-on: ${{ matrix.os }} + # steps: + # - uses: actions/checkout@v4 + + # - uses: pnpm/action-setup@v3 + + # - name: Setup Node.js ${{ matrix.node-version }} + # uses: actions/setup-node@v4 + # with: + # node-version: ${{ matrix.node-version }} + # cache: "pnpm" + + # - name: Install dependencies + # run: pnpm install + + # - name: Install Playwright Browsers + # run: pnpm exec playwright install --with-deps + # working-directory: packages/create-llama + + # - name: Build create-llama + # run: pnpm run build + # working-directory: packages/create-llama + + # - name: Install + # run: pnpm run pack-install + # working-directory: packages/create-llama + + # - name: Build server + # run: pnpm run build + # working-directory: packages/server + + # - name: Pack @llamaindex/server package + # run: | + # pnpm pack --pack-destination "${{ runner.temp }}" + # if [ "${{ runner.os }}" == "Windows" ]; then + # file=$(find "${{ runner.temp }}" -name "llamaindex-server-*.tgz" | head -n 1) + # mv "$file" "${{ runner.temp }}/llamaindex-server.tgz" + # else + # mv ${{ runner.temp }}/llamaindex-server-*.tgz ${{ runner.temp }}/llamaindex-server.tgz + # fi + # working-directory: packages/server + + # - name: Run Playwright tests for TypeScript + # run: | + # pnpm run e2e:ts + # env: + # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }} + # FRAMEWORK: ${{ matrix.frameworks }} + # VECTORDB: ${{ matrix.vectordbs }} + # SERVER_PACKAGE_PATH: ${{ runner.temp }}/llamaindex-server.tgz + # working-directory: packages/create-llama + + # - uses: actions/upload-artifact@v4 + # if: always() + # with: + # name: playwright-report-typescript-${{ matrix.os }}-${{ matrix.frameworks }}-${{ matrix.vectordbs}}-node${{ matrix.node-version }} + # path: packages/create-llama/playwright-report/ + # overwrite: true + # retention-days: 30 From 1c8eb676da1ef8332943da10b748c54e3ba200b1 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 16:17:00 +0700 Subject: [PATCH 63/80] fix run app --- packages/create-llama/helpers/run-app.ts | 26 ++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/create-llama/helpers/run-app.ts b/packages/create-llama/helpers/run-app.ts index bde0809ca..f9333db98 100644 --- a/packages/create-llama/helpers/run-app.ts +++ b/packages/create-llama/helpers/run-app.ts @@ -47,6 +47,28 @@ export function runTSApp(appPath: string, port: number) { }); } +// TODO: how to run different port (default is 4501) +async function runPythonLlamaDeployServer( + appPath: string, + port: number = 4501, +) { + // Start the llama_deploy server + createProcess("uv", ["run", "-m", "llama_deploy.apiserver"], { + stdio: "inherit", + cwd: appPath, + env: { ...process.env, APP_PORT: `${port}` }, + }); + + // create the deployment + setTimeout(() => { + createProcess("uv", ["run", "llamactl", "deploy", "llama_deploy.yml"], { + stdio: "inherit", + cwd: appPath, + env: { ...process.env, APP_PORT: `${port}` }, + }); + }, 1000); +} + export async function runApp( appPath: string, template: TemplateType, @@ -57,6 +79,10 @@ export async function runApp( // Start the app const defaultPort = framework === "nextjs" ? 3000 : 8000; + if (template === "llamaindexserver") { + return runPythonLlamaDeployServer(appPath, port); + } + const appRunner = framework === "fastapi" ? runFastAPIApp : runTSApp; await appRunner(appPath, port || defaultPort, template); } catch (error) { From 3e66f6d3075b184d730a7e681985ddf087d0b92e Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 16:29:16 +0700 Subject: [PATCH 64/80] fix uv mypy --- .../components/use-cases/python/deep_research/utils.py | 9 +++++---- .../use-cases/python/financial_report/utils.py | 9 ++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/create-llama/templates/components/use-cases/python/deep_research/utils.py b/packages/create-llama/templates/components/use-cases/python/deep_research/utils.py index 9e9010da3..510164974 100644 --- a/packages/create-llama/templates/components/use-cases/python/deep_research/utils.py +++ b/packages/create-llama/templates/components/use-cases/python/deep_research/utils.py @@ -2,13 +2,14 @@ from llama_index.core.base.llms.types import ( CompletionResponse, CompletionResponseAsyncGen, + ChatResponse, ) from llama_index.core.workflow import Context from llama_index.core.agent.workflow.workflow_events import AgentStream async def write_response_to_stream( - res: Union[CompletionResponse, CompletionResponseAsyncGen], + res: Union[CompletionResponse, CompletionResponseAsyncGen, AsyncGenerator[ChatResponse, None]], ctx: Context, current_agent_name: str = "assistant", ) -> str: @@ -26,7 +27,7 @@ async def write_response_to_stream( final_response = "" if isinstance(res, AsyncGenerator): - # Handle streaming response (CompletionResponseAsyncGen) + # Handle streaming response (CompletionResponseAsyncGen or ChatResponse AsyncGenerator) async for chunk in res: ctx.write_event_to_stream( AgentStream( @@ -34,10 +35,10 @@ async def write_response_to_stream( response=final_response, current_agent_name=current_agent_name, tool_calls=[], - raw=chunk.raw or "", + raw=getattr(chunk, 'raw', None) or "", ) ) - final_response = chunk.text + final_response += chunk.delta or "" else: # Handle non-streaming response (CompletionResponse) final_response = res.text diff --git a/packages/create-llama/templates/components/use-cases/python/financial_report/utils.py b/packages/create-llama/templates/components/use-cases/python/financial_report/utils.py index 42ad4380d..510164974 100644 --- a/packages/create-llama/templates/components/use-cases/python/financial_report/utils.py +++ b/packages/create-llama/templates/components/use-cases/python/financial_report/utils.py @@ -2,13 +2,14 @@ from llama_index.core.base.llms.types import ( CompletionResponse, CompletionResponseAsyncGen, + ChatResponse, ) from llama_index.core.workflow import Context from llama_index.core.agent.workflow.workflow_events import AgentStream async def write_response_to_stream( - res: Union[CompletionResponse, CompletionResponseAsyncGen], + res: Union[CompletionResponse, CompletionResponseAsyncGen, AsyncGenerator[ChatResponse, None]], ctx: Context, current_agent_name: str = "assistant", ) -> str: @@ -25,10 +26,8 @@ async def write_response_to_stream( """ final_response = "" - # {"__is_pydantic": true, "value": {"delta": "", "response": "", "current_agent_name": "assistant", "tool_calls": []}, "qualified_name": "llama_index.core.agent.workflow.workflow_events.AgentStream"} - if isinstance(res, AsyncGenerator): - # Handle streaming response (CompletionResponseAsyncGen) + # Handle streaming response (CompletionResponseAsyncGen or ChatResponse AsyncGenerator) async for chunk in res: ctx.write_event_to_stream( AgentStream( @@ -36,7 +35,7 @@ async def write_response_to_stream( response=final_response, current_agent_name=current_agent_name, tool_calls=[], - raw=chunk.raw or "", + raw=getattr(chunk, 'raw', None) or "", ) ) final_response += chunk.delta or "" From deac8e342108dceac6beb443f74ad7e6b528f8f5 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 16:44:35 +0700 Subject: [PATCH 65/80] frontend url and submit chat api for testing --- .../shared/llamaindexserver_template.spec.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts index 8c9f541ae..678ff4e20 100644 --- a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts +++ b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts @@ -21,6 +21,9 @@ const llamaCloudIndexName = "e2e-test"; const allUseCases = templateFramework === "nextjs" ? ALL_NEXTJS_USE_CASES : ALL_PYTHON_USE_CASES; +const isPythonLlamaDeploy = templateFramework === "fastapi"; +const DEPLOYMENT_NAME = "chat"; + const userMessage = "Write a blog post about physical standards for letters"; for (const useCase of allUseCases) { @@ -29,6 +32,8 @@ for (const useCase of allUseCases) { let cwd: string; let name: string; let appProcess: ChildProcess; + let frontendUrl: string; + let submitChatApi: string; test.beforeAll(async () => { port = Math.floor(Math.random() * 10000) + 10000; @@ -45,6 +50,12 @@ for (const useCase of allUseCases) { }); name = result.projectName; appProcess = result.appProcess; + frontendUrl = isPythonLlamaDeploy + ? `http://localhost:${port}/deployments/${DEPLOYMENT_NAME}/ui` + : `http://localhost:${port}`; + submitChatApi = isPythonLlamaDeploy + ? `/deployments/${DEPLOYMENT_NAME}/tasks/create` + : `/api/chat`; }); test("App folder should exist", async () => { @@ -53,7 +64,7 @@ for (const useCase of allUseCases) { }); test("Frontend should have a title", async ({ page }) => { - await page.goto(`http://localhost:${port}`); + await page.goto(frontendUrl); await expect(page.getByText("Built by LlamaIndex")).toBeVisible({ timeout: 5 * 60 * 1000, }); @@ -66,11 +77,11 @@ for (const useCase of allUseCases) { useCase === "financial_report" || useCase === "deep_research", "Skip chat tests for financial report and deep research.", ); - await page.goto(`http://localhost:${port}`); + await page.goto(frontendUrl); await page.fill("form textarea", userMessage); const responsePromise = page.waitForResponse((res) => - res.url().includes("/api/chat"), + res.url().includes(submitChatApi), ); await page.click("form button[type=submit]"); From b7ffea8c31d8c7f26e3b09b1f7e0f78059242914 Mon Sep 17 00:00:00 2001 From: thucpn Date: Wed, 9 Jul 2025 17:47:27 +0700 Subject: [PATCH 66/80] custom port --- packages/create-llama/helpers/run-app.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/create-llama/helpers/run-app.ts b/packages/create-llama/helpers/run-app.ts index f9333db98..7da6181e1 100644 --- a/packages/create-llama/helpers/run-app.ts +++ b/packages/create-llama/helpers/run-app.ts @@ -47,7 +47,6 @@ export function runTSApp(appPath: string, port: number) { }); } -// TODO: how to run different port (default is 4501) async function runPythonLlamaDeployServer( appPath: string, port: number = 4501, @@ -56,7 +55,7 @@ async function runPythonLlamaDeployServer( createProcess("uv", ["run", "-m", "llama_deploy.apiserver"], { stdio: "inherit", cwd: appPath, - env: { ...process.env, APP_PORT: `${port}` }, + env: { ...process.env, LLAMA_DEPLOY_APISERVER_PORT: `${port}` }, }); // create the deployment @@ -64,7 +63,6 @@ async function runPythonLlamaDeployServer( createProcess("uv", ["run", "llamactl", "deploy", "llama_deploy.yml"], { stdio: "inherit", cwd: appPath, - env: { ...process.env, APP_PORT: `${port}` }, }); }, 1000); } From dabd0405732aead3fc536f5b1bc7275b3ec92a2e Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 08:58:17 +0700 Subject: [PATCH 67/80] specify port for running llama-deploy --- packages/create-llama/helpers/run-app.ts | 53 +++++++++++++++++++----- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/packages/create-llama/helpers/run-app.ts b/packages/create-llama/helpers/run-app.ts index 7da6181e1..b86978596 100644 --- a/packages/create-llama/helpers/run-app.ts +++ b/packages/create-llama/helpers/run-app.ts @@ -1,4 +1,5 @@ -import { SpawnOptions, spawn } from "child_process"; +import { SpawnOptions, exec, spawn } from "child_process"; +import waitPort from "wait-port"; import { TemplateFramework, TemplateType } from "./types"; const createProcess = ( @@ -51,20 +52,51 @@ async function runPythonLlamaDeployServer( appPath: string, port: number = 4501, ) { - // Start the llama_deploy server - createProcess("uv", ["run", "-m", "llama_deploy.apiserver"], { - stdio: "inherit", + console.log("Starting llama_deploy server...", port); + const serverProcess = exec("uv run -m llama_deploy.apiserver", { cwd: appPath, - env: { ...process.env, LLAMA_DEPLOY_APISERVER_PORT: `${port}` }, + env: { + ...process.env, + LLAMA_DEPLOY_APISERVER_PORT: `${port}`, + }, }); - // create the deployment - setTimeout(() => { - createProcess("uv", ["run", "llamactl", "deploy", "llama_deploy.yml"], { + // Pipe output to console + serverProcess.stdout?.pipe(process.stdout); + serverProcess.stderr?.pipe(process.stderr); + + // Wait for the server to be ready + console.log("Waiting for server to be ready..."); + await waitPort({ port, host: "localhost", timeout: 30000 }); + + // create the deployment with explicit host configuration + console.log("llama_deploy server started, creating deployment...", port); + await createProcess( + "uv", + [ + "run", + "llamactl", + "-s", + `http://localhost:${port}`, + "deploy", + "llama_deploy.yml", + ], + { stdio: "inherit", cwd: appPath, + shell: true, + }, + ); + console.log(`Deployment created successfully!`); + + // Keep the main process alive and handle cleanup + return new Promise(() => { + process.on("SIGINT", () => { + console.log("\nShutting down..."); + serverProcess.kill(); + process.exit(0); }); - }, 1000); + }); } export async function runApp( @@ -78,7 +110,8 @@ export async function runApp( const defaultPort = framework === "nextjs" ? 3000 : 8000; if (template === "llamaindexserver") { - return runPythonLlamaDeployServer(appPath, port); + await runPythonLlamaDeployServer(appPath, port); + return; } const appRunner = framework === "fastapi" ? runFastAPIApp : runTSApp; From 05c535439b086c211657c7b2ce736b2151b196eb Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 09:56:51 +0700 Subject: [PATCH 68/80] cleanup --- packages/server/examples/hitl/components/cli_human_input.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/examples/hitl/components/cli_human_input.tsx b/packages/server/examples/hitl/components/cli_human_input.tsx index 8700d6182..973f65fb4 100644 --- a/packages/server/examples/hitl/components/cli_human_input.tsx +++ b/packages/server/examples/hitl/components/cli_human_input.tsx @@ -10,7 +10,6 @@ const CLIInputEventSchema = z.object({ }); type CLIInputEvent = z.infer; -// TODO: this component is working well for TS server. But not for HITL in Python llama-deploy. const CLIHumanInput: FC<{ events: JSONValue[]; }> = ({ events }) => { From b602e7d7b4739872015a6a72d3d117496ca4465f Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 10:21:12 +0700 Subject: [PATCH 69/80] revert server --- .../next/app/api/files/[...slug]/route.ts | 48 +------------------ packages/server/next/app/api/files/helpers.ts | 35 -------------- .../app/components/ui/chat/chat-section.tsx | 1 - .../app/components/ui/chat/chat-starter.tsx | 8 +--- packages/server/package.json | 2 +- packages/server/src/server.ts | 27 ++--------- packages/server/src/types.ts | 2 - pnpm-lock.yaml | 10 ++-- 8 files changed, 14 insertions(+), 119 deletions(-) diff --git a/packages/server/next/app/api/files/[...slug]/route.ts b/packages/server/next/app/api/files/[...slug]/route.ts index 7044d0c38..ca31e1a37 100644 --- a/packages/server/next/app/api/files/[...slug]/route.ts +++ b/packages/server/next/app/api/files/[...slug]/route.ts @@ -1,14 +1,11 @@ import fs from "fs"; -import { LLamaCloudFileService } from "llamaindex"; import { NextRequest, NextResponse } from "next/server"; import { promisify } from "util"; -import { downloadFile } from "../helpers"; export async function GET( request: NextRequest, { params }: { params: Promise<{ slug: string[] }> }, ) { - const isUsingLlamaCloud = !!process.env.LLAMA_CLOUD_API_KEY; const filePath = (await params).slug.join("/"); if (!filePath.startsWith("output") && !filePath.startsWith("data")) { @@ -16,37 +13,8 @@ export async function GET( } const decodedFilePath = decodeURIComponent(filePath); - - // if using llama cloud and file not exists, download it - if (isUsingLlamaCloud) { - const fileExists = await promisify(fs.exists)(decodedFilePath); - if (!fileExists) { - const { pipeline_id, file_name } = - getLlamaCloudPipelineIdAndFileName(decodedFilePath); - - if (pipeline_id && file_name) { - // get the file url from llama cloud - const downloadUrl = await LLamaCloudFileService.getFileUrl( - pipeline_id, - file_name, - ); - if (!downloadUrl) { - return NextResponse.json( - { - error: `Cannot create LlamaCloud download url for pipeline_id=${pipeline_id}, file_name=${file_name}`, - }, - { status: 404 }, - ); - } - - // download the LlamaCloud file to local - await downloadFile(downloadUrl, decodedFilePath); - console.log("File downloaded successfully to: ", decodedFilePath); - } - } - } - const fileExists = await promisify(fs.exists)(decodedFilePath); + if (fileExists) { const fileBuffer = await promisify(fs.readFile)(decodedFilePath); return new NextResponse(fileBuffer); @@ -54,17 +22,3 @@ export async function GET( return NextResponse.json({ error: "File not found" }, { status: 404 }); } } - -function getLlamaCloudPipelineIdAndFileName(filePath: string) { - const fileName = filePath.split("/").pop() ?? ""; // fileName is the last slug part (pipeline_id$file_name) - - const delimiterIndex = fileName.indexOf("$"); // delimiter is the first dollar sign in the fileName - if (delimiterIndex === -1) { - return { pipeline_id: "", file_name: "" }; - } - - const pipeline_id = fileName.slice(0, delimiterIndex); // before delimiter - const file_name = fileName.slice(delimiterIndex + 1); // after delimiter - - return { pipeline_id, file_name }; -} diff --git a/packages/server/next/app/api/files/helpers.ts b/packages/server/next/app/api/files/helpers.ts index 7bb003f0b..d21ff7e9d 100644 --- a/packages/server/next/app/api/files/helpers.ts +++ b/packages/server/next/app/api/files/helpers.ts @@ -1,6 +1,5 @@ import crypto from "node:crypto"; import fs from "node:fs"; -import https from "node:https"; import path from "node:path"; import { type ServerFile } from "@llamaindex/server"; @@ -56,37 +55,3 @@ async function saveFile(filepath: string, content: string | Buffer) { function sanitizeFileName(fileName: string) { return fileName.replace(/[^a-zA-Z0-9_-]/g, "_"); } -export async function downloadFile( - urlToDownload: string, - downloadedPath: string, -): Promise { - return new Promise((resolve, reject) => { - const dir = path.dirname(downloadedPath); - fs.mkdirSync(dir, { recursive: true }); - const file = fs.createWriteStream(downloadedPath); - - https - .get(urlToDownload, (response) => { - if (response.statusCode !== 200) { - reject( - new Error(`Failed to download file: Status ${response.statusCode}`), - ); - return; - } - - response.pipe(file); - - file.on("finish", () => { - file.close(); - resolve(); - }); - - file.on("error", (err) => { - fs.unlink(downloadedPath, () => reject(err)); - }); - }) - .on("error", (err) => { - fs.unlink(downloadedPath, () => reject(err)); - }); - }); -} diff --git a/packages/server/next/app/components/ui/chat/chat-section.tsx b/packages/server/next/app/components/ui/chat/chat-section.tsx index 8d7d296a5..75a75e850 100644 --- a/packages/server/next/app/components/ui/chat/chat-section.tsx +++ b/packages/server/next/app/components/ui/chat/chat-section.tsx @@ -38,7 +38,6 @@ export default function ChatSection() { }); const useChatWorkflowHandler = useChatWorkflow({ - fileServerUrl: getConfig("FILE_SERVER_URL"), deployment, workflow, onError: handleError, diff --git a/packages/server/next/app/components/ui/chat/chat-starter.tsx b/packages/server/next/app/components/ui/chat/chat-starter.tsx index d73dd745a..149d94253 100644 --- a/packages/server/next/app/components/ui/chat/chat-starter.tsx +++ b/packages/server/next/app/components/ui/chat/chat-starter.tsx @@ -6,13 +6,9 @@ import { getConfig } from "../lib/utils"; export function ChatStarter({ className }: { className?: string }) { const { append, messages, requestData } = useChatUI(); - const starterQuestionsFromConfig = getConfig("STARTER_QUESTIONS"); - const starterQuestions = - Array.isArray(starterQuestionsFromConfig) && - starterQuestionsFromConfig?.length > 0 - ? starterQuestionsFromConfig - : JSON.parse(process.env.NEXT_PUBLIC_STARTER_QUESTIONS || "[]"); + getConfig("STARTER_QUESTIONS") ?? + JSON.parse(process.env.NEXT_PUBLIC_STARTER_QUESTIONS || "[]"); if (starterQuestions.length === 0 || messages.length > 0) return null; return ( diff --git a/packages/server/package.json b/packages/server/package.json index 9e4648edc..c38c43dbd 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -68,7 +68,7 @@ "@babel/traverse": "^7.27.0", "@babel/types": "^7.27.0", "@hookform/resolvers": "^5.0.1", - "@llamaindex/chat-ui": "0.5.15", + "@llamaindex/chat-ui": "0.5.12", "@radix-ui/react-accordion": "^1.2.3", "@radix-ui/react-alert-dialog": "^1.1.7", "@radix-ui/react-aspect-ratio": "^1.1.3", diff --git a/packages/server/src/server.ts b/packages/server/src/server.ts index 27a66b4d4..4d78d0f5a 100644 --- a/packages/server/src/server.ts +++ b/packages/server/src/server.ts @@ -12,6 +12,7 @@ import type { LlamaDeployConfig, LlamaIndexServerOptions } from "./types"; const nextDir = path.join(__dirname, "..", "server"); const configFile = path.join(__dirname, "..", "server", "public", "config.js"); const nextConfigFile = path.join(nextDir, "next.config.ts"); +const layoutFile = path.join(nextDir, "app", "layout.tsx"); const constantsFile = path.join(nextDir, "app", "constants.ts"); const dev = process.env.NODE_ENV !== "production"; @@ -23,8 +24,6 @@ export class LlamaIndexServer { layoutDir: string; suggestNextQuestions: boolean; llamaDeploy?: LlamaDeployConfig | undefined; - serverUrl: string; - fileServer: string; constructor(options: LlamaIndexServerOptions) { const { workflow, suggestNextQuestions, ...nextAppOptions } = options; @@ -34,13 +33,7 @@ export class LlamaIndexServer { this.componentsDir = options.uiConfig?.componentsDir; this.layoutDir = options.uiConfig?.layoutDir ?? "layout"; this.suggestNextQuestions = suggestNextQuestions ?? true; - this.llamaDeploy = options.uiConfig?.llamaDeploy; - this.serverUrl = options.uiConfig?.serverUrl || ""; // use current host if not set - - const isUsingLlamaCloud = !!getEnv("LLAMA_CLOUD_API_KEY"); - const defaultFileServer = isUsingLlamaCloud ? "output/llamacloud" : "data"; - this.fileServer = options.fileServer ?? defaultFileServer; if (this.llamaDeploy) { if (!this.llamaDeploy.deployment || !this.llamaDeploy.workflow) { @@ -48,13 +41,9 @@ export class LlamaIndexServer { "LlamaDeploy requires deployment and workflow to be set", ); } - const { devMode, llamaCloudIndexSelector, enableFileUpload } = - options.uiConfig ?? {}; - - if (devMode || llamaCloudIndexSelector || enableFileUpload) { - throw new Error( - "`devMode`, `llamaCloudIndexSelector`, and `enableFileUpload` are not supported when enabling LlamaDeploy", - ); + if (options.uiConfig?.devMode) { + // workflow file is in llama-deploy src, so we should disable devmode + throw new Error("Devmode is not supported when enabling LlamaDeploy"); } } else { // if llamaDeploy is not set but workflowFactory is not defined, we should throw an error @@ -114,11 +103,6 @@ export default { const enableFileUpload = uiConfig?.enableFileUpload ?? false; const uploadApi = enableFileUpload ? `${basePath}/api/files` : undefined; - // construct file server url for LlamaDeploy - // eg. for Non-LlamaCloud: localhost:3000/deployments/chat/ui/api/files/data - // eg. for LlamaCloud: localhost:3000/deployments/chat/ui/api/files/output/llamacloud - const fileServerUrl = `${this.serverUrl}${basePath}/api/files/${this.fileServer}`; - // content in javascript format const content = ` window.LLAMAINDEX = { @@ -131,8 +115,7 @@ export default { SUGGEST_NEXT_QUESTIONS: ${JSON.stringify(this.suggestNextQuestions)}, UPLOAD_API: ${JSON.stringify(uploadApi)}, DEPLOYMENT: ${JSON.stringify(this.llamaDeploy?.deployment)}, - WORKFLOW: ${JSON.stringify(this.llamaDeploy?.workflow)}, - FILE_SERVER_URL: ${JSON.stringify(fileServerUrl)} + WORKFLOW: ${JSON.stringify(this.llamaDeploy?.workflow)} } `; fs.writeFileSync(configFile, content); diff --git a/packages/server/src/types.ts b/packages/server/src/types.ts index f5b68cd4a..3dd555955 100644 --- a/packages/server/src/types.ts +++ b/packages/server/src/types.ts @@ -25,12 +25,10 @@ export type UIConfig = { devMode?: boolean; enableFileUpload?: boolean; llamaDeploy?: LlamaDeployConfig; - serverUrl?: string; }; export type LlamaIndexServerOptions = NextAppOptions & { workflow?: WorkflowFactory; uiConfig?: UIConfig; - fileServer?: string; suggestNextQuestions?: boolean; }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4354875bf..bb77efa29 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -187,8 +187,8 @@ importers: specifier: 0.3.25 version: 0.3.25 '@llamaindex/chat-ui': - specifier: 0.5.15 - version: 0.5.15(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.1)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.11)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.37.1)(@lezer/highlight@1.2.1)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(yjs@13.6.27) + specifier: 0.5.12 + version: 0.5.12(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.1)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.11)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.37.1)(@lezer/highlight@1.2.1)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(yjs@13.6.27) '@llamaindex/env': specifier: ~0.1.30 version: 0.1.30 @@ -1500,8 +1500,8 @@ packages: zod: optional: true - '@llamaindex/chat-ui@0.5.15': - resolution: {integrity: sha512-r2Pv2a+nkt8F2Xy7rGxZsSrTFCwOfSzNkmrgOb15gfdImVCZcSieH7/czpzhsp9JNgK/+WsQIGQxoKHkT+IuCQ==} + '@llamaindex/chat-ui@0.5.12': + resolution: {integrity: sha512-Gi9MDkIakTf1S3mpeOA7vzJtBDUBgD5bcQ6KiKsoEYiEVPtUCeEcMoepTU3KoCssaGe2cvjR4tN6dnaUsiEHWQ==} peerDependencies: react: ^18.2.0 || ^19.0.0 || ^19.0.0-rc @@ -8546,7 +8546,7 @@ snapshots: p-retry: 6.2.1 zod: 3.25.13 - '@llamaindex/chat-ui@0.5.15(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.1)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.11)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.37.1)(@lezer/highlight@1.2.1)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(yjs@13.6.27)': + '@llamaindex/chat-ui@0.5.12(@babel/runtime@7.27.0)(@codemirror/autocomplete@6.18.6)(@codemirror/language@6.11.1)(@codemirror/lint@6.8.5)(@codemirror/search@6.5.11)(@codemirror/state@6.5.2)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.37.1)(@lezer/highlight@1.2.1)(@types/react-dom@19.1.2(@types/react@19.1.2))(@types/react@19.1.2)(codemirror@6.0.1)(react-dom@19.1.0(react@19.1.0))(react@19.1.0)(yjs@13.6.27)': dependencies: '@codemirror/lang-css': 6.3.1 '@codemirror/lang-html': 6.4.9 From 92fb8c77923d55070383120e28a45de64e4bd893 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 16:37:40 +0700 Subject: [PATCH 70/80] skip test --- .../shared/llamaindexserver_template.spec.ts | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts index 678ff4e20..d18879d9d 100644 --- a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts +++ b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts @@ -22,7 +22,6 @@ const allUseCases = templateFramework === "nextjs" ? ALL_NEXTJS_USE_CASES : ALL_PYTHON_USE_CASES; const isPythonLlamaDeploy = templateFramework === "fastapi"; -const DEPLOYMENT_NAME = "chat"; const userMessage = "Write a blog post about physical standards for letters"; @@ -32,8 +31,6 @@ for (const useCase of allUseCases) { let cwd: string; let name: string; let appProcess: ChildProcess; - let frontendUrl: string; - let submitChatApi: string; test.beforeAll(async () => { port = Math.floor(Math.random() * 10000) + 10000; @@ -43,19 +40,13 @@ for (const useCase of allUseCases) { templateFramework, vectorDb, port, - postInstallAction: "runApp", + postInstallAction: isPythonLlamaDeploy ? "dependencies" : "runApp", useCase, llamaCloudProjectName, llamaCloudIndexName, }); name = result.projectName; appProcess = result.appProcess; - frontendUrl = isPythonLlamaDeploy - ? `http://localhost:${port}/deployments/${DEPLOYMENT_NAME}/ui` - : `http://localhost:${port}`; - submitChatApi = isPythonLlamaDeploy - ? `/deployments/${DEPLOYMENT_NAME}/tasks/create` - : `/api/chat`; }); test("App folder should exist", async () => { @@ -64,7 +55,12 @@ for (const useCase of allUseCases) { }); test("Frontend should have a title", async ({ page }) => { - await page.goto(frontendUrl); + test.skip( + isPythonLlamaDeploy, + "Skip frontend tests for Python LllamaDeploy", + ); + + await page.goto(`http://localhost:${port}`); await expect(page.getByText("Built by LlamaIndex")).toBeVisible({ timeout: 5 * 60 * 1000, }); @@ -74,14 +70,16 @@ for (const useCase of allUseCases) { page, }) => { test.skip( - useCase === "financial_report" || useCase === "deep_research", - "Skip chat tests for financial report and deep research.", + useCase === "financial_report" || + useCase === "deep_research" || + isPythonLlamaDeploy, + "Skip chat tests for financial report and deep research. Also skip for Python LlamaDeploy", ); - await page.goto(frontendUrl); + await page.goto(`http://localhost:${port}`); await page.fill("form textarea", userMessage); const responsePromise = page.waitForResponse((res) => - res.url().includes(submitChatApi), + res.url().includes("/api/chat"), ); await page.click("form button[type=submit]"); From a30a7fb3d0b0d18ed2ea8c5e6c1f9b0bcd654520 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 16:42:05 +0700 Subject: [PATCH 71/80] typo --- .../e2e/shared/llamaindexserver_template.spec.ts | 7 ++++--- .../e2e/typescript/resolve_dependencies.spec.ts | 4 ++-- packages/create-llama/helpers/types.ts | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts index d18879d9d..c5d1367ea 100644 --- a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts +++ b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts @@ -3,8 +3,8 @@ import { ChildProcess } from "child_process"; import fs from "fs"; import path from "path"; import { - ALL_NEXTJS_USE_CASES, ALL_PYTHON_USE_CASES, + ALL_TYPESCRIPT_USE_CASES, type TemplateFramework, type TemplateVectorDB, } from "../../helpers"; @@ -19,8 +19,9 @@ const vectorDb: TemplateVectorDB = process.env.VECTORDB const llamaCloudProjectName = "create-llama"; const llamaCloudIndexName = "e2e-test"; const allUseCases = - templateFramework === "nextjs" ? ALL_NEXTJS_USE_CASES : ALL_PYTHON_USE_CASES; - + templateFramework === "nextjs" + ? ALL_TYPESCRIPT_USE_CASES + : ALL_PYTHON_USE_CASES; const isPythonLlamaDeploy = templateFramework === "fastapi"; const userMessage = "Write a blog post about physical standards for letters"; diff --git a/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts b/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts index 3ca0d4a4b..1d768d288 100644 --- a/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts +++ b/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts @@ -4,7 +4,7 @@ import fs from "fs"; import path from "path"; import util from "util"; import { - ALL_NEXTJS_USE_CASES, + ALL_TYPESCRIPT_USE_CASES, TemplateFramework, TemplateUseCase, TemplateVectorDB, @@ -21,7 +21,7 @@ const vectorDb: TemplateVectorDB = process.env.VECTORDB test.describe("Test resolve TS dependencies", () => { test.describe.configure({ retries: 0 }); - for (const useCase of ALL_NEXTJS_USE_CASES) { + for (const useCase of ALL_TYPESCRIPT_USE_CASES) { const optionDescription = `useCase: ${useCase}, vectorDb: ${vectorDb}`; test.describe(`${optionDescription}`, () => { test(`${optionDescription}`, async () => { diff --git a/packages/create-llama/helpers/types.ts b/packages/create-llama/helpers/types.ts index cdd3c958c..149ca7487 100644 --- a/packages/create-llama/helpers/types.ts +++ b/packages/create-llama/helpers/types.ts @@ -49,7 +49,7 @@ export type TemplateUseCase = | "document_generator" | "hitl"; -export const ALL_NEXTJS_USE_CASES: TemplateUseCase[] = [ +export const ALL_TYPESCRIPT_USE_CASES: TemplateUseCase[] = [ "agentic_rag", "deep_research", "financial_report", From 5639ed839871896651719f2b9b1d5288066ca08e Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 16:48:15 +0700 Subject: [PATCH 72/80] clean up --- .../create-llama/helpers/env-variables.ts | 91 ++++++++++++------- 1 file changed, 60 insertions(+), 31 deletions(-) diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index aa5aa4e08..6457db3e1 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -17,6 +17,62 @@ export type EnvVar = { value?: string; }; +const USE_CASE_CONFIGS: Record< + TemplateUseCase, + { + starterQuestions: string[]; + additionalEnvVars: EnvVar[]; + } +> = { + agentic_rag: { + starterQuestions: [ + "Letter standard in the document", + "Summarize the document", + ], + additionalEnvVars: [ + { + name: "E2B_API_KEY", + description: "The E2B API key to use to use code interpreter tool", + }, + ], + }, + financial_report: { + starterQuestions: [ + "Compare Apple and Tesla financial performance", + "Generate a PDF report for Tesla financial", + ], + additionalEnvVars: [], + }, + deep_research: { + starterQuestions: [ + "Research about Apple and Tesla", + "Financial performance of Tesla", + ], + additionalEnvVars: [], + }, + code_generator: { + starterQuestions: [ + "Generate a code for a simple calculator", + "Generate a code for a todo list app", + ], + additionalEnvVars: [], + }, + document_generator: { + starterQuestions: [ + "Generate a document about LlamaIndex", + "Generate a document about LLM", + ], + additionalEnvVars: [], + }, + hitl: { + starterQuestions: [ + "List all the files in the current directory", + "Check git status", + ], + additionalEnvVars: [], + }, +}; + const renderEnvVar = (envVars: EnvVar[]): string => { return envVars.reduce( (prev, env) => @@ -229,27 +285,6 @@ Otherwise, use CHROMA_HOST and CHROMA_PORT config above`, } }; -const useCaseStarterQuestions: Record = { - agentic_rag: ["Letter standard in the document", "Summarize the document"], - financial_report: [ - "Compare Apple and Tesla financial performance", - "Generate a PDF report for Tesla financial", - ], - deep_research: [ - "Research about Apple and Tesla", - "Financial performance of Tesla", - ], - code_generator: [ - "Generate a code for a simple calculator", - "Generate a code for a todo list app", - ], - document_generator: [ - "Generate a document about LlamaIndex", - "Generate a document about LLM", - ], - hitl: ["List all the files in the current directory", "Check git status"], -}; - const getModelEnvs = ( modelConfig: ModelConfig, framework: TemplateFramework, @@ -258,7 +293,6 @@ const getModelEnvs = ( ): EnvVar[] => { const isPythonLlamaDeploy = framework === "fastapi" && template === "llamaindexserver"; - const isFinancialReport = useCase === "financial_report"; return [ { @@ -277,7 +311,9 @@ const getModelEnvs = ( name: "NEXT_PUBLIC_STARTER_QUESTIONS", description: "Initial questions to display in the chat (`starterQuestions`)", - value: JSON.stringify(useCaseStarterQuestions[useCase] ?? []), + value: JSON.stringify( + USE_CASE_CONFIGS[useCase]?.starterQuestions ?? [], + ), }, ] : [ @@ -287,14 +323,7 @@ const getModelEnvs = ( "The questions to help users get started (multi-line).", }, ]), - ...(isFinancialReport - ? [ - { - name: "E2B_API_KEY", - description: "The E2B API key to use to use code interpreter tool", - }, - ] - : []), + ...(USE_CASE_CONFIGS[useCase]?.additionalEnvVars ?? []), ...(modelConfig.provider === "openai" ? [ { From 0e48683427140a5ee8bb13ad0ff2bcd6617cba94 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 16:56:24 +0700 Subject: [PATCH 73/80] use case configs --- .../create-llama/helpers/env-variables.ts | 64 +----------------- packages/create-llama/helpers/python.ts | 37 +++------- packages/create-llama/helpers/types.ts | 13 ++++ packages/create-llama/helpers/use-case.ts | 67 +++++++++++++++++++ 4 files changed, 90 insertions(+), 91 deletions(-) create mode 100644 packages/create-llama/helpers/use-case.ts diff --git a/packages/create-llama/helpers/env-variables.ts b/packages/create-llama/helpers/env-variables.ts index 6457db3e1..2f88699f7 100644 --- a/packages/create-llama/helpers/env-variables.ts +++ b/packages/create-llama/helpers/env-variables.ts @@ -1,6 +1,7 @@ import fs from "fs/promises"; import path from "path"; import { + EnvVar, InstallTemplateArgs, ModelConfig, TemplateFramework, @@ -10,68 +11,7 @@ import { } from "./types"; import { TSYSTEMS_LLMHUB_API_URL } from "./providers/llmhub"; - -export type EnvVar = { - name?: string; - description?: string; - value?: string; -}; - -const USE_CASE_CONFIGS: Record< - TemplateUseCase, - { - starterQuestions: string[]; - additionalEnvVars: EnvVar[]; - } -> = { - agentic_rag: { - starterQuestions: [ - "Letter standard in the document", - "Summarize the document", - ], - additionalEnvVars: [ - { - name: "E2B_API_KEY", - description: "The E2B API key to use to use code interpreter tool", - }, - ], - }, - financial_report: { - starterQuestions: [ - "Compare Apple and Tesla financial performance", - "Generate a PDF report for Tesla financial", - ], - additionalEnvVars: [], - }, - deep_research: { - starterQuestions: [ - "Research about Apple and Tesla", - "Financial performance of Tesla", - ], - additionalEnvVars: [], - }, - code_generator: { - starterQuestions: [ - "Generate a code for a simple calculator", - "Generate a code for a todo list app", - ], - additionalEnvVars: [], - }, - document_generator: { - starterQuestions: [ - "Generate a document about LlamaIndex", - "Generate a document about LLM", - ], - additionalEnvVars: [], - }, - hitl: { - starterQuestions: [ - "List all the files in the current directory", - "Check git status", - ], - additionalEnvVars: [], - }, -}; +import { USE_CASE_CONFIGS } from "./use-case"; const renderEnvVar = (envVars: EnvVar[]): string => { return envVars.reduce( diff --git a/packages/create-llama/helpers/python.ts b/packages/create-llama/helpers/python.ts index d4e77f03d..b25c350da 100644 --- a/packages/create-llama/helpers/python.ts +++ b/packages/create-llama/helpers/python.ts @@ -7,14 +7,8 @@ import { isUvAvailable, tryUvSync } from "./uv"; import { assetRelocator, copy } from "./copy"; import { templatesDir } from "./dir"; -import { InstallTemplateArgs } from "./types"; - -interface Dependency { - name: string; - version?: string; - extras?: string[]; - constraints?: Record; -} +import { Dependency, InstallTemplateArgs } from "./types"; +import { USE_CASE_CONFIGS } from "./use-case"; const getAdditionalDependencies = ( opts: Pick< @@ -29,30 +23,15 @@ const getAdditionalDependencies = ( ) => { const { framework, template, useCase, modelConfig, vectorDb, dataSources } = opts; - const isPythonLlamaDeploy = - framework === "fastapi" && template === "llamaindexserver"; - const isPythonFinancialReport = - isPythonLlamaDeploy && useCase === "financial_report"; const dependencies: Dependency[] = []; - if (isPythonFinancialReport) { - dependencies.push( - ...[ - { - name: "e2b-code-interpreter", - version: ">=1.1.1,<2.0.0", - }, - { - name: "markdown", - version: ">=3.7,<4.0", - }, - { - name: "xhtml2pdf", - version: ">=0.2.17,<1.0.0", - }, - ], - ); + const isPythonLlamaDeploy = + framework === "fastapi" && template === "llamaindexserver"; + const useCaseDependencies = + USE_CASE_CONFIGS[useCase]?.additionalDependencies ?? []; + if (isPythonLlamaDeploy && useCaseDependencies.length > 0) { + dependencies.push(...useCaseDependencies); } // Add vector db dependencies diff --git a/packages/create-llama/helpers/types.ts b/packages/create-llama/helpers/types.ts index 149ca7487..a0fa5762c 100644 --- a/packages/create-llama/helpers/types.ts +++ b/packages/create-llama/helpers/types.ts @@ -106,3 +106,16 @@ export interface InstallTemplateArgs { postInstallAction: TemplatePostInstallAction; useCase: TemplateUseCase; } + +export type EnvVar = { + name?: string; + description?: string; + value?: string; +}; + +export interface Dependency { + name: string; + version?: string; + extras?: string[]; + constraints?: Record; +} diff --git a/packages/create-llama/helpers/use-case.ts b/packages/create-llama/helpers/use-case.ts new file mode 100644 index 000000000..7081bf69b --- /dev/null +++ b/packages/create-llama/helpers/use-case.ts @@ -0,0 +1,67 @@ +import { Dependency, EnvVar, TemplateUseCase } from "./types"; + +export const USE_CASE_CONFIGS: Record< + TemplateUseCase, + { + starterQuestions: string[]; + additionalEnvVars?: EnvVar[]; + additionalDependencies?: Dependency[]; + } +> = { + agentic_rag: { + starterQuestions: [ + "Letter standard in the document", + "Summarize the document", + ], + }, + financial_report: { + starterQuestions: [ + "Compare Apple and Tesla financial performance", + "Generate a PDF report for Tesla financial", + ], + additionalEnvVars: [ + { + name: "E2B_API_KEY", + description: "The E2B API key to use to use code interpreter tool", + }, + ], + additionalDependencies: [ + { + name: "e2b-code-interpreter", + version: ">=1.1.1,<2.0.0", + }, + { + name: "markdown", + version: ">=3.7,<4.0", + }, + { + name: "xhtml2pdf", + version: ">=0.2.17,<1.0.0", + }, + ], + }, + deep_research: { + starterQuestions: [ + "Research about Apple and Tesla", + "Financial performance of Tesla", + ], + }, + code_generator: { + starterQuestions: [ + "Generate a code for a simple calculator", + "Generate a code for a todo list app", + ], + }, + document_generator: { + starterQuestions: [ + "Generate a document about LlamaIndex", + "Generate a document about LLM", + ], + }, + hitl: { + starterQuestions: [ + "List all the files in the current directory", + "Check git status", + ], + }, +}; From 8250c402f1f424599f628b328af80e308344a10b Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 17:01:29 +0700 Subject: [PATCH 74/80] refactor usecase --- .../e2e/python/resolve_dependencies.spec.ts | 7 ++----- .../shared/llamaindexserver_template.spec.ts | 5 ++--- .../e2e/typescript/resolve_dependencies.spec.ts | 2 +- packages/create-llama/helpers/run-app.ts | 1 + packages/create-llama/helpers/types.ts | 17 ----------------- packages/create-llama/helpers/use-case.ts | 17 +++++++++++++++++ 6 files changed, 23 insertions(+), 26 deletions(-) diff --git a/packages/create-llama/e2e/python/resolve_dependencies.spec.ts b/packages/create-llama/e2e/python/resolve_dependencies.spec.ts index ac7790d96..e7efd8460 100644 --- a/packages/create-llama/e2e/python/resolve_dependencies.spec.ts +++ b/packages/create-llama/e2e/python/resolve_dependencies.spec.ts @@ -3,11 +3,8 @@ import { exec } from "child_process"; import fs from "fs"; import path from "path"; import util from "util"; -import { - ALL_PYTHON_USE_CASES, - TemplateFramework, - TemplateVectorDB, -} from "../../helpers/types"; +import { TemplateFramework, TemplateVectorDB } from "../../helpers"; +import { ALL_PYTHON_USE_CASES } from "../../helpers/use-case"; import { RunCreateLlamaOptions, createTestDir, runCreateLlama } from "../utils"; const execAsync = util.promisify(exec); diff --git a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts index c5d1367ea..69185693c 100644 --- a/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts +++ b/packages/create-llama/e2e/shared/llamaindexserver_template.spec.ts @@ -2,12 +2,11 @@ import { expect, test } from "@playwright/test"; import { ChildProcess } from "child_process"; import fs from "fs"; import path from "path"; +import { type TemplateFramework, type TemplateVectorDB } from "../../helpers"; import { ALL_PYTHON_USE_CASES, ALL_TYPESCRIPT_USE_CASES, - type TemplateFramework, - type TemplateVectorDB, -} from "../../helpers"; +} from "../../helpers/use-case"; import { createTestDir, runCreateLlama } from "../utils"; const templateFramework: TemplateFramework = process.env.FRAMEWORK diff --git a/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts b/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts index 1d768d288..2b1eca853 100644 --- a/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts +++ b/packages/create-llama/e2e/typescript/resolve_dependencies.spec.ts @@ -4,11 +4,11 @@ import fs from "fs"; import path from "path"; import util from "util"; import { - ALL_TYPESCRIPT_USE_CASES, TemplateFramework, TemplateUseCase, TemplateVectorDB, } from "../../helpers/types"; +import { ALL_TYPESCRIPT_USE_CASES } from "../../helpers/use-case"; import { createTestDir, runCreateLlama } from "../utils"; const execAsync = util.promisify(exec); diff --git a/packages/create-llama/helpers/run-app.ts b/packages/create-llama/helpers/run-app.ts index b86978596..a6b555e5e 100644 --- a/packages/create-llama/helpers/run-app.ts +++ b/packages/create-llama/helpers/run-app.ts @@ -48,6 +48,7 @@ export function runTSApp(appPath: string, port: number) { }); } +// TODO: support run multiple LlamaDeploy server in the same machine async function runPythonLlamaDeployServer( appPath: string, port: number = 4501, diff --git a/packages/create-llama/helpers/types.ts b/packages/create-llama/helpers/types.ts index a0fa5762c..cd2f54660 100644 --- a/packages/create-llama/helpers/types.ts +++ b/packages/create-llama/helpers/types.ts @@ -49,23 +49,6 @@ export type TemplateUseCase = | "document_generator" | "hitl"; -export const ALL_TYPESCRIPT_USE_CASES: TemplateUseCase[] = [ - "agentic_rag", - "deep_research", - "financial_report", - "code_generator", - "document_generator", - "hitl", -]; - -export const ALL_PYTHON_USE_CASES: TemplateUseCase[] = [ - "agentic_rag", - "deep_research", - "financial_report", - "code_generator", - "document_generator", -]; - // Config for both file and folder export type FileSourceConfig = | { diff --git a/packages/create-llama/helpers/use-case.ts b/packages/create-llama/helpers/use-case.ts index 7081bf69b..09386ebcd 100644 --- a/packages/create-llama/helpers/use-case.ts +++ b/packages/create-llama/helpers/use-case.ts @@ -1,5 +1,22 @@ import { Dependency, EnvVar, TemplateUseCase } from "./types"; +export const ALL_TYPESCRIPT_USE_CASES: TemplateUseCase[] = [ + "agentic_rag", + "deep_research", + "financial_report", + "code_generator", + "document_generator", + "hitl", +]; + +export const ALL_PYTHON_USE_CASES: TemplateUseCase[] = [ + "agentic_rag", + "deep_research", + "financial_report", + "code_generator", + "document_generator", +]; + export const USE_CASE_CONFIGS: Record< TemplateUseCase, { From 51077db8fd00da98d36c789d299ef966c3b84b03 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 17:02:40 +0700 Subject: [PATCH 75/80] remove default proxy port --- packages/create-llama/templates/components/ts-proxy/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/create-llama/templates/components/ts-proxy/index.ts b/packages/create-llama/templates/components/ts-proxy/index.ts index 276cebff0..a6a8e5109 100644 --- a/packages/create-llama/templates/components/ts-proxy/index.ts +++ b/packages/create-llama/templates/components/ts-proxy/index.ts @@ -6,5 +6,4 @@ new LlamaIndexServer({ layoutDir: "layout", llamaDeploy: { deployment: "chat", workflow: "workflow" }, }, - port: 3000, }).start(); From 6b989e9a0ffbdcb81935ea37e187ea760f9770eb Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 17:18:16 +0700 Subject: [PATCH 76/80] skip llamacloud for codegen docgen --- .../create-llama/e2e/python/resolve_dependencies.spec.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/create-llama/e2e/python/resolve_dependencies.spec.ts b/packages/create-llama/e2e/python/resolve_dependencies.spec.ts index e7efd8460..1eb1876da 100644 --- a/packages/create-llama/e2e/python/resolve_dependencies.spec.ts +++ b/packages/create-llama/e2e/python/resolve_dependencies.spec.ts @@ -3,7 +3,7 @@ import { exec } from "child_process"; import fs from "fs"; import path from "path"; import util from "util"; -import { TemplateFramework, TemplateVectorDB } from "../../helpers"; +import { TemplateFramework, TemplateUseCase, TemplateVectorDB } from "../../helpers"; import { ALL_PYTHON_USE_CASES } from "../../helpers/use-case"; import { RunCreateLlamaOptions, createTestDir, runCreateLlama } from "../utils"; @@ -14,11 +14,15 @@ const vectorDb: TemplateVectorDB = process.env.VECTORDB ? (process.env.VECTORDB as TemplateVectorDB) : "none"; +const useCases: TemplateUseCase[] = vectorDb === "llamacloud" ? [ + "agentic_rag", "deep_research", "financial_report" +] : ALL_PYTHON_USE_CASES + test.describe("Mypy check", () => { test.describe.configure({ retries: 0 }); test.describe("LlamaIndexServer", async () => { - for (const useCase of ALL_PYTHON_USE_CASES) { + for (const useCase of useCases) { test(`should pass mypy for use case: ${useCase}`, async () => { const cwd = await createTestDir(); await createAndCheckLlamaProject({ From aeaf7e14c4491067701e2e461b617921863113af Mon Sep 17 00:00:00 2001 From: Thuc Pham <51660321+thucpn@users.noreply.github.com> Date: Thu, 10 Jul 2025 17:19:46 +0700 Subject: [PATCH 77/80] Create good-avocados-try.md --- .changeset/good-avocados-try.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/good-avocados-try.md diff --git a/.changeset/good-avocados-try.md b/.changeset/good-avocados-try.md new file mode 100644 index 000000000..7e81b4664 --- /dev/null +++ b/.changeset/good-avocados-try.md @@ -0,0 +1,5 @@ +--- +"create-llama": patch +--- + +chore: replace Python examples with llama-deploy From 1bb685cfb07f6b333065159a0cc84f184714d635 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 17:28:47 +0700 Subject: [PATCH 78/80] revert e2e --- .github/workflows/e2e.yml | 158 ++++++++++++++++++-------------------- 1 file changed, 74 insertions(+), 84 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 997be9bf3..e2a36c8a7 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -20,8 +20,7 @@ jobs: matrix: node-version: [20] python-version: ["3.11"] - # os: [macos-latest, windows-latest, ubuntu-22.04] - os: [macos-latest] + os: [macos-latest, windows-latest, ubuntu-22.04] frameworks: ["fastapi"] vectordbs: ["none", "llamacloud"] defaults: @@ -64,15 +63,6 @@ jobs: run: pnpm run pack-install working-directory: packages/create-llama - # - name: Build and store server package - # run: | - # pnpm run build - # wheel_file=$(ls dist/*.whl | head -n 1) - # mkdir -p "${{ runner.temp }}" - # cp "$wheel_file" "${{ runner.temp }}/" - # echo "SERVER_PACKAGE_PATH=${{ runner.temp }}/$(basename "$wheel_file")" >> $GITHUB_ENV - # working-directory: python/llama-index-server - - name: Run Playwright tests for Python run: pnpm run e2e:python env: @@ -93,76 +83,76 @@ jobs: overwrite: true retention-days: 30 - # e2e-typescript: - # name: typescript - # timeout-minutes: 60 - # strategy: - # fail-fast: true - # matrix: - # node-version: [22] - # os: [macos-latest, windows-latest, ubuntu-22.04] - # frameworks: ["nextjs"] - # vectordbs: ["none", "llamacloud"] - # defaults: - # run: - # shell: bash - # runs-on: ${{ matrix.os }} - # steps: - # - uses: actions/checkout@v4 - - # - uses: pnpm/action-setup@v3 - - # - name: Setup Node.js ${{ matrix.node-version }} - # uses: actions/setup-node@v4 - # with: - # node-version: ${{ matrix.node-version }} - # cache: "pnpm" - - # - name: Install dependencies - # run: pnpm install - - # - name: Install Playwright Browsers - # run: pnpm exec playwright install --with-deps - # working-directory: packages/create-llama - - # - name: Build create-llama - # run: pnpm run build - # working-directory: packages/create-llama - - # - name: Install - # run: pnpm run pack-install - # working-directory: packages/create-llama - - # - name: Build server - # run: pnpm run build - # working-directory: packages/server - - # - name: Pack @llamaindex/server package - # run: | - # pnpm pack --pack-destination "${{ runner.temp }}" - # if [ "${{ runner.os }}" == "Windows" ]; then - # file=$(find "${{ runner.temp }}" -name "llamaindex-server-*.tgz" | head -n 1) - # mv "$file" "${{ runner.temp }}/llamaindex-server.tgz" - # else - # mv ${{ runner.temp }}/llamaindex-server-*.tgz ${{ runner.temp }}/llamaindex-server.tgz - # fi - # working-directory: packages/server - - # - name: Run Playwright tests for TypeScript - # run: | - # pnpm run e2e:ts - # env: - # OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - # LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }} - # FRAMEWORK: ${{ matrix.frameworks }} - # VECTORDB: ${{ matrix.vectordbs }} - # SERVER_PACKAGE_PATH: ${{ runner.temp }}/llamaindex-server.tgz - # working-directory: packages/create-llama - - # - uses: actions/upload-artifact@v4 - # if: always() - # with: - # name: playwright-report-typescript-${{ matrix.os }}-${{ matrix.frameworks }}-${{ matrix.vectordbs}}-node${{ matrix.node-version }} - # path: packages/create-llama/playwright-report/ - # overwrite: true - # retention-days: 30 + e2e-typescript: + name: typescript + timeout-minutes: 60 + strategy: + fail-fast: true + matrix: + node-version: [22] + os: [macos-latest, windows-latest, ubuntu-22.04] + frameworks: ["nextjs"] + vectordbs: ["none", "llamacloud"] + defaults: + run: + shell: bash + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v3 + + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: "pnpm" + + - name: Install dependencies + run: pnpm install + + - name: Install Playwright Browsers + run: pnpm exec playwright install --with-deps + working-directory: packages/create-llama + + - name: Build create-llama + run: pnpm run build + working-directory: packages/create-llama + + - name: Install + run: pnpm run pack-install + working-directory: packages/create-llama + + - name: Build server + run: pnpm run build + working-directory: packages/server + + - name: Pack @llamaindex/server package + run: | + pnpm pack --pack-destination "${{ runner.temp }}" + if [ "${{ runner.os }}" == "Windows" ]; then + file=$(find "${{ runner.temp }}" -name "llamaindex-server-*.tgz" | head -n 1) + mv "$file" "${{ runner.temp }}/llamaindex-server.tgz" + else + mv ${{ runner.temp }}/llamaindex-server-*.tgz ${{ runner.temp }}/llamaindex-server.tgz + fi + working-directory: packages/server + + - name: Run Playwright tests for TypeScript + run: | + pnpm run e2e:ts + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }} + FRAMEWORK: ${{ matrix.frameworks }} + VECTORDB: ${{ matrix.vectordbs }} + SERVER_PACKAGE_PATH: ${{ runner.temp }}/llamaindex-server.tgz + working-directory: packages/create-llama + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report-typescript-${{ matrix.os }}-${{ matrix.frameworks }}-${{ matrix.vectordbs}}-node${{ matrix.node-version }} + path: packages/create-llama/playwright-report/ + overwrite: true + retention-days: 30 From 7e72b2cf4d09523ee26dcbb67fd90a3d36695a70 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 17:34:38 +0700 Subject: [PATCH 79/80] bump server package in create-llama --- .../create-llama/templates/components/ts-proxy/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-llama/templates/components/ts-proxy/package.json b/packages/create-llama/templates/components/ts-proxy/package.json index de659d601..eab3be415 100644 --- a/packages/create-llama/templates/components/ts-proxy/package.json +++ b/packages/create-llama/templates/components/ts-proxy/package.json @@ -6,7 +6,7 @@ "dev": "nodemon --exec tsx index.ts" }, "dependencies": { - "@llamaindex/server": "0.2.9", + "@llamaindex/server": "0.2.10", "dotenv": "^16.4.7" }, "devDependencies": { From 62d82ed26ad926a28293124228a111d12721ca03 Mon Sep 17 00:00:00 2001 From: thucpn Date: Thu, 10 Jul 2025 17:50:16 +0700 Subject: [PATCH 80/80] fix run --- packages/create-llama/helpers/run-app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/create-llama/helpers/run-app.ts b/packages/create-llama/helpers/run-app.ts index a6b555e5e..f8367b237 100644 --- a/packages/create-llama/helpers/run-app.ts +++ b/packages/create-llama/helpers/run-app.ts @@ -110,7 +110,7 @@ export async function runApp( // Start the app const defaultPort = framework === "nextjs" ? 3000 : 8000; - if (template === "llamaindexserver") { + if (template === "llamaindexserver" && framework === "fastapi") { await runPythonLlamaDeployServer(appPath, port); return; }