diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml index 36c29379af99..67ef6e3c91df 100644 --- a/.github/workflows/aws-preview.yml +++ b/.github/workflows/aws-preview.yml @@ -4,7 +4,7 @@ on: push: branches: [ "main" ] paths: - - 'backend/core/**' + - 'backend/**' env: AWS_REGION: eu-west-3 @@ -56,7 +56,7 @@ jobs: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} IMAGE_TAG: ${{ github.sha }} with: - context: ./backend/core/ + context: ./backend/ push: true tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}, ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest, ghcr.io/stangirard/quivr:latest cache-from: type=gha diff --git a/.github/workflows/aws.yml b/.github/workflows/aws.yml index 4b439a102845..c3c26f86a349 100644 --- a/.github/workflows/aws.yml +++ b/.github/workflows/aws.yml @@ -55,7 +55,7 @@ jobs: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} IMAGE_TAG: ${{ github.sha }} with: - context: ./backend/core/ + context: ./backend/ push: true tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}, ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ github.ref_name}}, ghcr.io/stangirard/quivr:${{ github.ref_name}} cache-from: type=gha diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index 1505ed36b1bd..6da93d7aaf4d 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -4,11 +4,11 @@ on: push: branches: [main] paths: - - 'backend/core/**' + - 'backend/**' pull_request: branches: [main] paths: - - 'backend/core/**' + - 'backend/**' workflow_dispatch: inputs: logLevel: @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: ./backend/core + working-directory: ./backend environment: preview strategy: matrix: diff --git a/README.md b/README.md index 070443216975..96c880891ded 100644 --- a/README.md +++ b/README.md @@ -95,11 +95,11 @@ Additionally, you'll need a [Supabase](https://supabase.com/) account for: - **Step 2 - Bis**: Copy the `.XXXXX_env` files ```bash - cp .backend_env.example backend/core/.env + cp .backend_env.example backend/.env cp .frontend_env.example frontend/.env ``` -- **Step 3**: Update the `backend/core/.env` and `frontend/.env` file +- **Step 3**: Update the `backend/.env` and `frontend/.env` file > _Your `supabase_service_key` can be found in your Supabase dashboard under Project Settings -> API. Use the `anon` `public` key found in the `Project API keys` section._ @@ -107,9 +107,9 @@ Additionally, you'll need a [Supabase](https://supabase.com/) account for: > _The `NEXT_PUBLIC_BACKEND_URL` is set to localhost:5050 for the docker. Update it if you are running the backend on a different machine._ - > _To activate vertexAI with PaLM from GCP follow the instructions [here](https://python.langchain.com/en/latest/modules/models/llms/integrations/google_vertex_ai_palm.html) and update `backend/core/.env`- It is an advanced feature, please be expert in GCP before trying to use it_ + > _To activate vertexAI with PaLM from GCP follow the instructions [here](https://python.langchain.com/en/latest/modules/models/llms/integrations/google_vertex_ai_palm.html) and update `backend/.env`- It is an advanced feature, please be expert in GCP before trying to use it_ - - [ ] Change variables in `backend/core/.env` + - [ ] Change variables in `backend/.env` - [ ] Change variables in `frontend/.env` - **Step 4**: Use the `migration.sh` script to run the migration scripts diff --git a/backend/core/Dockerfile b/backend/Dockerfile similarity index 100% rename from backend/core/Dockerfile rename to backend/Dockerfile diff --git a/backend/core/auth/__init__.py b/backend/auth/__init__.py similarity index 100% rename from backend/core/auth/__init__.py rename to backend/auth/__init__.py diff --git a/backend/core/auth/api_key_handler.py b/backend/auth/api_key_handler.py similarity index 100% rename from backend/core/auth/api_key_handler.py rename to backend/auth/api_key_handler.py diff --git a/backend/core/auth/auth_bearer.py b/backend/auth/auth_bearer.py similarity index 100% rename from backend/core/auth/auth_bearer.py rename to backend/auth/auth_bearer.py diff --git a/backend/core/auth/jwt_token_handler.py b/backend/auth/jwt_token_handler.py similarity index 100% rename from backend/core/auth/jwt_token_handler.py rename to backend/auth/jwt_token_handler.py diff --git a/backend/core/chat_service.py b/backend/chat_service.py similarity index 100% rename from backend/core/chat_service.py rename to backend/chat_service.py diff --git a/backend/core/crawl/__init__.py b/backend/crawl/__init__.py similarity index 100% rename from backend/core/crawl/__init__.py rename to backend/crawl/__init__.py diff --git a/backend/core/crawl/crawler.py b/backend/crawl/crawler.py similarity index 100% rename from backend/core/crawl/crawler.py rename to backend/crawl/crawler.py diff --git a/backend/core/crawl_service.py b/backend/crawl_service.py similarity index 100% rename from backend/core/crawl_service.py rename to backend/crawl_service.py diff --git a/backend/core/llm/__init__.py b/backend/llm/__init__.py similarity index 100% rename from backend/core/llm/__init__.py rename to backend/llm/__init__.py diff --git a/backend/core/llm/base.py b/backend/llm/base.py similarity index 100% rename from backend/core/llm/base.py rename to backend/llm/base.py diff --git a/backend/core/llm/models/FunctionCall.py b/backend/llm/models/FunctionCall.py similarity index 100% rename from backend/core/llm/models/FunctionCall.py rename to backend/llm/models/FunctionCall.py diff --git a/backend/core/llm/models/OpenAiAnswer.py b/backend/llm/models/OpenAiAnswer.py similarity index 100% rename from backend/core/llm/models/OpenAiAnswer.py rename to backend/llm/models/OpenAiAnswer.py diff --git a/backend/core/llm/openai.py b/backend/llm/openai.py similarity index 100% rename from backend/core/llm/openai.py rename to backend/llm/openai.py diff --git a/backend/core/llm/prompts/CONDENSE_PROMPT.py b/backend/llm/prompts/CONDENSE_PROMPT.py similarity index 100% rename from backend/core/llm/prompts/CONDENSE_PROMPT.py rename to backend/llm/prompts/CONDENSE_PROMPT.py diff --git a/backend/core/llm/prompts/LANGUAGE_PROMPT.py b/backend/llm/prompts/LANGUAGE_PROMPT.py similarity index 100% rename from backend/core/llm/prompts/LANGUAGE_PROMPT.py rename to backend/llm/prompts/LANGUAGE_PROMPT.py diff --git a/backend/core/llm/qa_base.py b/backend/llm/qa_base.py similarity index 100% rename from backend/core/llm/qa_base.py rename to backend/llm/qa_base.py diff --git a/backend/core/llm/qa_headless.py b/backend/llm/qa_headless.py similarity index 100% rename from backend/core/llm/qa_headless.py rename to backend/llm/qa_headless.py diff --git a/backend/core/logger.py b/backend/logger.py similarity index 100% rename from backend/core/logger.py rename to backend/logger.py diff --git a/backend/core/main.py b/backend/main.py similarity index 100% rename from backend/core/main.py rename to backend/main.py diff --git a/backend/core/middlewares/cors.py b/backend/middlewares/cors.py similarity index 100% rename from backend/core/middlewares/cors.py rename to backend/middlewares/cors.py diff --git a/backend/core/models/__init__.py b/backend/models/__init__.py similarity index 100% rename from backend/core/models/__init__.py rename to backend/models/__init__.py diff --git a/backend/core/models/brain_entity.py b/backend/models/brain_entity.py similarity index 100% rename from backend/core/models/brain_entity.py rename to backend/models/brain_entity.py diff --git a/backend/core/models/brains.py b/backend/models/brains.py similarity index 100% rename from backend/core/models/brains.py rename to backend/models/brains.py diff --git a/backend/core/models/brains_subscription_invitations.py b/backend/models/brains_subscription_invitations.py similarity index 100% rename from backend/core/models/brains_subscription_invitations.py rename to backend/models/brains_subscription_invitations.py diff --git a/backend/core/models/chat.py b/backend/models/chat.py similarity index 100% rename from backend/core/models/chat.py rename to backend/models/chat.py diff --git a/backend/core/models/chats.py b/backend/models/chats.py similarity index 100% rename from backend/core/models/chats.py rename to backend/models/chats.py diff --git a/backend/core/models/databases/repository.py b/backend/models/databases/repository.py similarity index 100% rename from backend/core/models/databases/repository.py rename to backend/models/databases/repository.py diff --git a/backend/core/models/databases/supabase/__init__.py b/backend/models/databases/supabase/__init__.py similarity index 100% rename from backend/core/models/databases/supabase/__init__.py rename to backend/models/databases/supabase/__init__.py diff --git a/backend/core/models/databases/supabase/api_key_handler.py b/backend/models/databases/supabase/api_key_handler.py similarity index 100% rename from backend/core/models/databases/supabase/api_key_handler.py rename to backend/models/databases/supabase/api_key_handler.py diff --git a/backend/core/models/databases/supabase/brains.py b/backend/models/databases/supabase/brains.py similarity index 100% rename from backend/core/models/databases/supabase/brains.py rename to backend/models/databases/supabase/brains.py diff --git a/backend/core/models/databases/supabase/brains_subscription_invitations.py b/backend/models/databases/supabase/brains_subscription_invitations.py similarity index 100% rename from backend/core/models/databases/supabase/brains_subscription_invitations.py rename to backend/models/databases/supabase/brains_subscription_invitations.py diff --git a/backend/core/models/databases/supabase/chats.py b/backend/models/databases/supabase/chats.py similarity index 100% rename from backend/core/models/databases/supabase/chats.py rename to backend/models/databases/supabase/chats.py diff --git a/backend/core/models/databases/supabase/files.py b/backend/models/databases/supabase/files.py similarity index 100% rename from backend/core/models/databases/supabase/files.py rename to backend/models/databases/supabase/files.py diff --git a/backend/core/models/databases/supabase/prompts.py b/backend/models/databases/supabase/prompts.py similarity index 100% rename from backend/core/models/databases/supabase/prompts.py rename to backend/models/databases/supabase/prompts.py diff --git a/backend/core/models/databases/supabase/supabase.py b/backend/models/databases/supabase/supabase.py similarity index 100% rename from backend/core/models/databases/supabase/supabase.py rename to backend/models/databases/supabase/supabase.py diff --git a/backend/core/models/databases/supabase/users.py b/backend/models/databases/supabase/users.py similarity index 100% rename from backend/core/models/databases/supabase/users.py rename to backend/models/databases/supabase/users.py diff --git a/backend/core/models/databases/supabase/vectors.py b/backend/models/databases/supabase/vectors.py similarity index 100% rename from backend/core/models/databases/supabase/vectors.py rename to backend/models/databases/supabase/vectors.py diff --git a/backend/core/models/files.py b/backend/models/files.py similarity index 100% rename from backend/core/models/files.py rename to backend/models/files.py diff --git a/backend/core/models/prompt.py b/backend/models/prompt.py similarity index 100% rename from backend/core/models/prompt.py rename to backend/models/prompt.py diff --git a/backend/core/models/settings.py b/backend/models/settings.py similarity index 100% rename from backend/core/models/settings.py rename to backend/models/settings.py diff --git a/backend/core/models/sqlalchemy_repository.py b/backend/models/sqlalchemy_repository.py similarity index 100% rename from backend/core/models/sqlalchemy_repository.py rename to backend/models/sqlalchemy_repository.py diff --git a/backend/core/models/user_identity.py b/backend/models/user_identity.py similarity index 100% rename from backend/core/models/user_identity.py rename to backend/models/user_identity.py diff --git a/backend/core/models/users.py b/backend/models/users.py similarity index 100% rename from backend/core/models/users.py rename to backend/models/users.py diff --git a/backend/pandoc-3.1.6.1-1-arm64.deb b/backend/pandoc-3.1.6.1-1-arm64.deb new file mode 100644 index 000000000000..ed4a7c9ef0cd Binary files /dev/null and b/backend/pandoc-3.1.6.1-1-arm64.deb differ diff --git a/backend/core/parsers/__init__.py b/backend/parsers/__init__.py similarity index 100% rename from backend/core/parsers/__init__.py rename to backend/parsers/__init__.py diff --git a/backend/core/parsers/audio.py b/backend/parsers/audio.py similarity index 100% rename from backend/core/parsers/audio.py rename to backend/parsers/audio.py diff --git a/backend/core/parsers/common.py b/backend/parsers/common.py similarity index 100% rename from backend/core/parsers/common.py rename to backend/parsers/common.py diff --git a/backend/core/parsers/csv.py b/backend/parsers/csv.py similarity index 100% rename from backend/core/parsers/csv.py rename to backend/parsers/csv.py diff --git a/backend/core/parsers/docx.py b/backend/parsers/docx.py similarity index 100% rename from backend/core/parsers/docx.py rename to backend/parsers/docx.py diff --git a/backend/core/parsers/epub.py b/backend/parsers/epub.py similarity index 100% rename from backend/core/parsers/epub.py rename to backend/parsers/epub.py diff --git a/backend/core/parsers/github.py b/backend/parsers/github.py similarity index 100% rename from backend/core/parsers/github.py rename to backend/parsers/github.py diff --git a/backend/core/parsers/html.py b/backend/parsers/html.py similarity index 100% rename from backend/core/parsers/html.py rename to backend/parsers/html.py diff --git a/backend/core/parsers/markdown.py b/backend/parsers/markdown.py similarity index 100% rename from backend/core/parsers/markdown.py rename to backend/parsers/markdown.py diff --git a/backend/core/parsers/notebook.py b/backend/parsers/notebook.py similarity index 100% rename from backend/core/parsers/notebook.py rename to backend/parsers/notebook.py diff --git a/backend/core/parsers/odt.py b/backend/parsers/odt.py similarity index 100% rename from backend/core/parsers/odt.py rename to backend/parsers/odt.py diff --git a/backend/core/parsers/pdf.py b/backend/parsers/pdf.py similarity index 100% rename from backend/core/parsers/pdf.py rename to backend/parsers/pdf.py diff --git a/backend/core/parsers/powerpoint.py b/backend/parsers/powerpoint.py similarity index 100% rename from backend/core/parsers/powerpoint.py rename to backend/parsers/powerpoint.py diff --git a/backend/core/parsers/txt.py b/backend/parsers/txt.py similarity index 100% rename from backend/core/parsers/txt.py rename to backend/parsers/txt.py diff --git a/backend/private/Dockerfile b/backend/private/Dockerfile deleted file mode 100644 index 28b2d9dc8ce1..000000000000 --- a/backend/private/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Use the same base image as the 'backend-core' container -FROM python:3.11-bullseye - -# Install necessary packages -RUN apt-get update && apt-get install -y liblzma-dev cmake git - -# Set the working directory -WORKDIR /app - -# Copy the requirements file -COPY ./requirements.txt /app/requirements.txt - -# Install Python dependencies -RUN pip install --no-cache-dir -r /app/requirements.txt --timeout 100 - -# Copy your application's code to the Docker container -COPY . /app - -# Start the Uvicorn server on port 5051 -CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "5051"] diff --git a/backend/private/logger.py b/backend/private/logger.py deleted file mode 100644 index 21a896de2377..000000000000 --- a/backend/private/logger.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging - - -def get_logger(logger_name, log_level=logging.INFO): - logger = logging.getLogger(logger_name) - logger.setLevel(log_level) - logger.propagate = False # Prevent log propagation to avoid double logging - - formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s") - - console_handler = logging.StreamHandler() - console_handler.setFormatter(formatter) - - if not logger.handlers: - logger.addHandler(console_handler) - - return logger diff --git a/backend/private/main.py b/backend/private/main.py deleted file mode 100644 index 0a21e960c9ba..000000000000 --- a/backend/private/main.py +++ /dev/null @@ -1,30 +0,0 @@ -import os - -import sentry_sdk -from fastapi import FastAPI, HTTPException -from fastapi.responses import JSONResponse -from logger import get_logger -from routes.completions_routes import completions_router - -logger = get_logger(__name__) - -if os.getenv("SENTRY_DSN"): - sentry_sdk.init( - dsn=os.getenv("SENTRY_DSN"), - # Set traces_sample_rate to 1.0 to capture 100% - # of transactions for performance monitoring. - # We recommend adjusting this value in production, - traces_sample_rate=1.0, - ) - -app = FastAPI() - -app.include_router(completions_router) - - -@app.exception_handler(HTTPException) -async def http_exception_handler(_, exc): - return JSONResponse( - status_code=exc.status_code, - content={"detail": exc.detail}, - ) diff --git a/backend/private/models/messages.py b/backend/private/models/messages.py deleted file mode 100644 index 31b4cb739973..000000000000 --- a/backend/private/models/messages.py +++ /dev/null @@ -1,18 +0,0 @@ -from typing import List - -from pydantic import BaseModel, Field - - -class Message(BaseModel): - role: str = Field( - ..., - description="The role of the messages author. One of system, user, assistant, or function.", - ) - content: str = Field( - ..., - description="The contents of the message. content is required for all messages, and may be null for assistant messages with function calls.", - ) - - -class Messages(BaseModel): - messages: List[Message] diff --git a/backend/private/requirements.txt b/backend/private/requirements.txt deleted file mode 100644 index 40139e80c8eb..000000000000 --- a/backend/private/requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -langchain==0.0.228 -tiktoken==0.4.0 -fastapi==0.95.2 -uvicorn==0.22.0 -transformers==4.30.1 -asyncpg==0.27.0 -flake8==6.0.0 -flake8-black==0.3.6 -sentence_transformers>=2.0.0 -sentry-sdk==1.26.0 -pyright==1.1.316 \ No newline at end of file diff --git a/backend/private/routes/completions_routes.py b/backend/private/routes/completions_routes.py deleted file mode 100644 index 6b5624cdfdcd..000000000000 --- a/backend/private/routes/completions_routes.py +++ /dev/null @@ -1,18 +0,0 @@ -from fastapi import APIRouter -from logger import get_logger -from models.messages import Messages - -logger = get_logger(__name__) - -completions_router = APIRouter() - - -@completions_router.post("/chat/completions", tags=["Chat Completions"]) -async def post_chat_completions( - model: str, # make union of all models - messages: Messages, -) -> str: - logger.info( - f"Received chat completions request for {model} with messages {messages}" - ) - return f"Hello from {model}!" diff --git a/backend/private/routes/embeddings_routes.py b/backend/private/routes/embeddings_routes.py deleted file mode 100644 index f413f46e0de4..000000000000 --- a/backend/private/routes/embeddings_routes.py +++ /dev/null @@ -1,16 +0,0 @@ -from fastapi import APIRouter -from logger import getLogger - -logger = getLogger(__name__) - -embeddings_router = APIRouter() - - -@embeddings_router.post("/embeddings", tags=["Embeddings"]) -async def post_embeddings( - model: str, - input: str | list[str], -) -> str: - logger.info(f"Received embeddings request for {model} with input {input}") - - return f"Hello from embeddings {model}!" diff --git a/backend/core/pyrightconfig.json b/backend/pyrightconfig.json similarity index 100% rename from backend/core/pyrightconfig.json rename to backend/pyrightconfig.json diff --git a/backend/core/repository/__init__.py b/backend/repository/__init__.py similarity index 100% rename from backend/core/repository/__init__.py rename to backend/repository/__init__.py diff --git a/backend/core/repository/brain/create_brain.py b/backend/repository/brain/create_brain.py similarity index 100% rename from backend/core/repository/brain/create_brain.py rename to backend/repository/brain/create_brain.py diff --git a/backend/core/repository/brain/create_brain_user.py b/backend/repository/brain/create_brain_user.py similarity index 100% rename from backend/core/repository/brain/create_brain_user.py rename to backend/repository/brain/create_brain_user.py diff --git a/backend/core/repository/brain/get_brain_by_id.py b/backend/repository/brain/get_brain_by_id.py similarity index 100% rename from backend/core/repository/brain/get_brain_by_id.py rename to backend/repository/brain/get_brain_by_id.py diff --git a/backend/core/repository/brain/get_brain_details.py b/backend/repository/brain/get_brain_details.py similarity index 100% rename from backend/core/repository/brain/get_brain_details.py rename to backend/repository/brain/get_brain_details.py diff --git a/backend/core/repository/brain/get_brain_for_user.py b/backend/repository/brain/get_brain_for_user.py similarity index 100% rename from backend/core/repository/brain/get_brain_for_user.py rename to backend/repository/brain/get_brain_for_user.py diff --git a/backend/core/repository/brain/get_brain_prompt_id.py b/backend/repository/brain/get_brain_prompt_id.py similarity index 100% rename from backend/core/repository/brain/get_brain_prompt_id.py rename to backend/repository/brain/get_brain_prompt_id.py diff --git a/backend/core/repository/brain/get_default_user_brain.py b/backend/repository/brain/get_default_user_brain.py similarity index 100% rename from backend/core/repository/brain/get_default_user_brain.py rename to backend/repository/brain/get_default_user_brain.py diff --git a/backend/core/repository/brain/get_default_user_brain_or_create_new.py b/backend/repository/brain/get_default_user_brain_or_create_new.py similarity index 100% rename from backend/core/repository/brain/get_default_user_brain_or_create_new.py rename to backend/repository/brain/get_default_user_brain_or_create_new.py diff --git a/backend/core/repository/brain/get_user_brains.py b/backend/repository/brain/get_user_brains.py similarity index 100% rename from backend/core/repository/brain/get_user_brains.py rename to backend/repository/brain/get_user_brains.py diff --git a/backend/core/repository/brain/set_as_default_brain_for_user.py b/backend/repository/brain/set_as_default_brain_for_user.py similarity index 100% rename from backend/core/repository/brain/set_as_default_brain_for_user.py rename to backend/repository/brain/set_as_default_brain_for_user.py diff --git a/backend/core/repository/brain/update_brain.py b/backend/repository/brain/update_brain.py similarity index 100% rename from backend/core/repository/brain/update_brain.py rename to backend/repository/brain/update_brain.py diff --git a/backend/core/repository/brain/update_user_rights.py b/backend/repository/brain/update_user_rights.py similarity index 100% rename from backend/core/repository/brain/update_user_rights.py rename to backend/repository/brain/update_user_rights.py diff --git a/backend/core/repository/brain_subscription/__init__.py b/backend/repository/brain_subscription/__init__.py similarity index 100% rename from backend/core/repository/brain_subscription/__init__.py rename to backend/repository/brain_subscription/__init__.py diff --git a/backend/core/repository/brain_subscription/get_brain_url.py b/backend/repository/brain_subscription/get_brain_url.py similarity index 100% rename from backend/core/repository/brain_subscription/get_brain_url.py rename to backend/repository/brain_subscription/get_brain_url.py diff --git a/backend/core/repository/brain_subscription/resend_invitation_email.py b/backend/repository/brain_subscription/resend_invitation_email.py similarity index 100% rename from backend/core/repository/brain_subscription/resend_invitation_email.py rename to backend/repository/brain_subscription/resend_invitation_email.py diff --git a/backend/core/repository/brain_subscription/subscription_invitation_service.py b/backend/repository/brain_subscription/subscription_invitation_service.py similarity index 100% rename from backend/core/repository/brain_subscription/subscription_invitation_service.py rename to backend/repository/brain_subscription/subscription_invitation_service.py diff --git a/backend/core/repository/chat/__init__.py b/backend/repository/chat/__init__.py similarity index 100% rename from backend/core/repository/chat/__init__.py rename to backend/repository/chat/__init__.py diff --git a/backend/core/repository/chat/create_chat.py b/backend/repository/chat/create_chat.py similarity index 100% rename from backend/core/repository/chat/create_chat.py rename to backend/repository/chat/create_chat.py diff --git a/backend/core/repository/chat/format_chat_history.py b/backend/repository/chat/format_chat_history.py similarity index 100% rename from backend/core/repository/chat/format_chat_history.py rename to backend/repository/chat/format_chat_history.py diff --git a/backend/core/repository/chat/get_chat_by_id.py b/backend/repository/chat/get_chat_by_id.py similarity index 100% rename from backend/core/repository/chat/get_chat_by_id.py rename to backend/repository/chat/get_chat_by_id.py diff --git a/backend/core/repository/chat/get_chat_history.py b/backend/repository/chat/get_chat_history.py similarity index 100% rename from backend/core/repository/chat/get_chat_history.py rename to backend/repository/chat/get_chat_history.py diff --git a/backend/core/repository/chat/get_user_chats.py b/backend/repository/chat/get_user_chats.py similarity index 100% rename from backend/core/repository/chat/get_user_chats.py rename to backend/repository/chat/get_user_chats.py diff --git a/backend/core/repository/chat/update_chat.py b/backend/repository/chat/update_chat.py similarity index 100% rename from backend/core/repository/chat/update_chat.py rename to backend/repository/chat/update_chat.py diff --git a/backend/core/repository/chat/update_chat_history.py b/backend/repository/chat/update_chat_history.py similarity index 100% rename from backend/core/repository/chat/update_chat_history.py rename to backend/repository/chat/update_chat_history.py diff --git a/backend/core/repository/chat/update_message_by_id.py b/backend/repository/chat/update_message_by_id.py similarity index 100% rename from backend/core/repository/chat/update_message_by_id.py rename to backend/repository/chat/update_message_by_id.py diff --git a/backend/core/repository/prompt/create_prompt.py b/backend/repository/prompt/create_prompt.py similarity index 100% rename from backend/core/repository/prompt/create_prompt.py rename to backend/repository/prompt/create_prompt.py diff --git a/backend/core/repository/prompt/delete_prompt_py_id.py b/backend/repository/prompt/delete_prompt_py_id.py similarity index 100% rename from backend/core/repository/prompt/delete_prompt_py_id.py rename to backend/repository/prompt/delete_prompt_py_id.py diff --git a/backend/core/repository/prompt/get_prompt_by_id.py b/backend/repository/prompt/get_prompt_by_id.py similarity index 100% rename from backend/core/repository/prompt/get_prompt_by_id.py rename to backend/repository/prompt/get_prompt_by_id.py diff --git a/backend/core/repository/prompt/get_public_prompts.py b/backend/repository/prompt/get_public_prompts.py similarity index 100% rename from backend/core/repository/prompt/get_public_prompts.py rename to backend/repository/prompt/get_public_prompts.py diff --git a/backend/core/repository/prompt/update_prompt_by_id.py b/backend/repository/prompt/update_prompt_by_id.py similarity index 100% rename from backend/core/repository/prompt/update_prompt_by_id.py rename to backend/repository/prompt/update_prompt_by_id.py diff --git a/backend/core/repository/user/get_user_email_by_user_id.py b/backend/repository/user/get_user_email_by_user_id.py similarity index 100% rename from backend/core/repository/user/get_user_email_by_user_id.py rename to backend/repository/user/get_user_email_by_user_id.py diff --git a/backend/core/repository/user/get_user_id_by_user_email.py b/backend/repository/user/get_user_id_by_user_email.py similarity index 100% rename from backend/core/repository/user/get_user_id_by_user_email.py rename to backend/repository/user/get_user_id_by_user_email.py diff --git a/backend/core/repository/user_identity/create_user_identity.py b/backend/repository/user_identity/create_user_identity.py similarity index 100% rename from backend/core/repository/user_identity/create_user_identity.py rename to backend/repository/user_identity/create_user_identity.py diff --git a/backend/core/repository/user_identity/get_user_identity.py b/backend/repository/user_identity/get_user_identity.py similarity index 100% rename from backend/core/repository/user_identity/get_user_identity.py rename to backend/repository/user_identity/get_user_identity.py diff --git a/backend/core/repository/user_identity/update_user_identity.py b/backend/repository/user_identity/update_user_identity.py similarity index 100% rename from backend/core/repository/user_identity/update_user_identity.py rename to backend/repository/user_identity/update_user_identity.py diff --git a/backend/core/requirements.txt b/backend/requirements.txt similarity index 100% rename from backend/core/requirements.txt rename to backend/requirements.txt diff --git a/backend/core/routes/__init__.py b/backend/routes/__init__.py similarity index 100% rename from backend/core/routes/__init__.py rename to backend/routes/__init__.py diff --git a/backend/core/routes/api_key_routes.py b/backend/routes/api_key_routes.py similarity index 100% rename from backend/core/routes/api_key_routes.py rename to backend/routes/api_key_routes.py diff --git a/backend/core/routes/authorizations/__init__.py b/backend/routes/authorizations/__init__.py similarity index 100% rename from backend/core/routes/authorizations/__init__.py rename to backend/routes/authorizations/__init__.py diff --git a/backend/core/routes/authorizations/brain_authorization.py b/backend/routes/authorizations/brain_authorization.py similarity index 100% rename from backend/core/routes/authorizations/brain_authorization.py rename to backend/routes/authorizations/brain_authorization.py diff --git a/backend/core/routes/authorizations/types.py b/backend/routes/authorizations/types.py similarity index 100% rename from backend/core/routes/authorizations/types.py rename to backend/routes/authorizations/types.py diff --git a/backend/core/routes/brain_routes.py b/backend/routes/brain_routes.py similarity index 100% rename from backend/core/routes/brain_routes.py rename to backend/routes/brain_routes.py diff --git a/backend/core/routes/chat_routes.py b/backend/routes/chat_routes.py similarity index 100% rename from backend/core/routes/chat_routes.py rename to backend/routes/chat_routes.py diff --git a/backend/core/routes/crawl_routes.py b/backend/routes/crawl_routes.py similarity index 100% rename from backend/core/routes/crawl_routes.py rename to backend/routes/crawl_routes.py diff --git a/backend/core/routes/explore_routes.py b/backend/routes/explore_routes.py similarity index 100% rename from backend/core/routes/explore_routes.py rename to backend/routes/explore_routes.py diff --git a/backend/core/routes/headers/__init_.py b/backend/routes/headers/__init_.py similarity index 100% rename from backend/core/routes/headers/__init_.py rename to backend/routes/headers/__init_.py diff --git a/backend/core/routes/headers/get_origin_header.py b/backend/routes/headers/get_origin_header.py similarity index 100% rename from backend/core/routes/headers/get_origin_header.py rename to backend/routes/headers/get_origin_header.py diff --git a/backend/core/routes/misc_routes.py b/backend/routes/misc_routes.py similarity index 100% rename from backend/core/routes/misc_routes.py rename to backend/routes/misc_routes.py diff --git a/backend/core/routes/prompt_routes.py b/backend/routes/prompt_routes.py similarity index 100% rename from backend/core/routes/prompt_routes.py rename to backend/routes/prompt_routes.py diff --git a/backend/core/routes/subscription_routes.py b/backend/routes/subscription_routes.py similarity index 100% rename from backend/core/routes/subscription_routes.py rename to backend/routes/subscription_routes.py diff --git a/backend/core/routes/upload_routes.py b/backend/routes/upload_routes.py similarity index 100% rename from backend/core/routes/upload_routes.py rename to backend/routes/upload_routes.py diff --git a/backend/core/routes/user_routes.py b/backend/routes/user_routes.py similarity index 100% rename from backend/core/routes/user_routes.py rename to backend/routes/user_routes.py diff --git a/backend/core/supabase/functions/add-new-email/index.ts b/backend/supabase/functions/add-new-email/index.ts similarity index 95% rename from backend/core/supabase/functions/add-new-email/index.ts rename to backend/supabase/functions/add-new-email/index.ts index d382a98fe4c5..ec11b3857a94 100644 --- a/backend/core/supabase/functions/add-new-email/index.ts +++ b/backend/supabase/functions/add-new-email/index.ts @@ -1,5 +1,5 @@ /* Instructions: -1. in .backend/core/supabase folder, create .env file with BEEHIIV_PUBLICATION_ID and BEEHIIV_API_KEY variables +1. in .backend/supabase folder, create .env file with BEEHIIV_PUBLICATION_ID and BEEHIIV_API_KEY variables 2. cd into .backend --- for the rest of these steps you will need your supabase project id which can be found in your console url: https://supabase.com/dashboard/project/ --- 3. run `supabase secrets set --env-file ./supabase/.env` to set the environment variables diff --git a/backend/core/tests/conftest.py b/backend/tests/conftest.py similarity index 100% rename from backend/core/tests/conftest.py rename to backend/tests/conftest.py diff --git a/backend/core/tests/test_api_key.py b/backend/tests/test_api_key.py similarity index 100% rename from backend/core/tests/test_api_key.py rename to backend/tests/test_api_key.py diff --git a/backend/core/tests/test_brains.py b/backend/tests/test_brains.py similarity index 100% rename from backend/core/tests/test_brains.py rename to backend/tests/test_brains.py diff --git a/backend/core/tests/test_chats.py b/backend/tests/test_chats.py similarity index 100% rename from backend/core/tests/test_chats.py rename to backend/tests/test_chats.py diff --git a/backend/core/tests/test_explore.py b/backend/tests/test_explore.py similarity index 100% rename from backend/core/tests/test_explore.py rename to backend/tests/test_explore.py diff --git a/backend/core/tests/test_files/test.csv b/backend/tests/test_files/test.csv similarity index 100% rename from backend/core/tests/test_files/test.csv rename to backend/tests/test_files/test.csv diff --git a/backend/core/tests/test_files/test.pdf b/backend/tests/test_files/test.pdf similarity index 100% rename from backend/core/tests/test_files/test.pdf rename to backend/tests/test_files/test.pdf diff --git a/backend/core/tests/test_files/test.txt b/backend/tests/test_files/test.txt similarity index 100% rename from backend/core/tests/test_files/test.txt rename to backend/tests/test_files/test.txt diff --git a/backend/core/tests/test_upload.py b/backend/tests/test_upload.py similarity index 100% rename from backend/core/tests/test_upload.py rename to backend/tests/test_upload.py diff --git a/backend/core/tests/test_user.py b/backend/tests/test_user.py similarity index 100% rename from backend/core/tests/test_user.py rename to backend/tests/test_user.py diff --git a/backend/core/upload_service.py b/backend/upload_service.py similarity index 100% rename from backend/core/upload_service.py rename to backend/upload_service.py diff --git a/backend/core/utils/__init__.py b/backend/utils/__init__.py similarity index 100% rename from backend/core/utils/__init__.py rename to backend/utils/__init__.py diff --git a/backend/core/utils/file.py b/backend/utils/file.py similarity index 100% rename from backend/core/utils/file.py rename to backend/utils/file.py diff --git a/backend/core/utils/processors.py b/backend/utils/processors.py similarity index 100% rename from backend/core/utils/processors.py rename to backend/utils/processors.py diff --git a/backend/core/utils/vectors.py b/backend/utils/vectors.py similarity index 100% rename from backend/core/utils/vectors.py rename to backend/utils/vectors.py diff --git a/backend/core/vectorstore/__init__.py b/backend/vectorstore/__init__.py similarity index 100% rename from backend/core/vectorstore/__init__.py rename to backend/vectorstore/__init__.py diff --git a/backend/core/vectorstore/supabase.py b/backend/vectorstore/supabase.py similarity index 100% rename from backend/core/vectorstore/supabase.py rename to backend/vectorstore/supabase.py diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5fe3891d2bbd..b13f132d2c8d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -27,15 +27,15 @@ services: backend-core: env_file: - - ./backend/core/.env + - ./backend/.env build: - context: backend/core + context: backend dockerfile: Dockerfile container_name: backend-core command: uvicorn main:app --host 0.0.0.0 --port 5050 restart: always volumes: - - ./backend/core/:/code/ + - ./backend/:/code/ labels: - "traefik.enable=true" - "traefik.http.routers.backend-core.rule=PathPrefix(`/`)" @@ -44,15 +44,15 @@ services: backend-chat: env_file: - - ./backend/core/.env + - ./backend/.env build: - context: backend/core + context: backend dockerfile: Dockerfile container_name: backend-chat command: uvicorn chat_service:app --host 0.0.0.0 --port 5050 restart: always volumes: - - ./backend/core/:/code/ + - ./backend/:/code/ labels: - "traefik.enable=true" - "traefik.http.routers.backend-chat.rule=PathPrefix(`/chat`)" @@ -61,15 +61,15 @@ services: backend-crawl: env_file: - - ./backend/core/.env + - ./backend/.env build: - context: backend/core + context: backend dockerfile: Dockerfile container_name: backend-crawl command: uvicorn crawl_service:app --host 0.0.0.0 --port 5050 restart: always volumes: - - ./backend/core/:/code/ + - ./backend/:/code/ labels: - "traefik.enable=true" - "traefik.http.routers.backend-crawl.rule=PathPrefix(`/crawl`)" @@ -78,15 +78,15 @@ services: backend-upload: env_file: - - ./backend/core/.env + - ./backend/.env build: - context: backend/core + context: backend dockerfile: Dockerfile container_name: backend-upload command: uvicorn upload_service:app --host 0.0.0.0 --port 5050 restart: always volumes: - - ./backend/core/:/code/ + - ./backend/:/code/ labels: - "traefik.enable=true" - "traefik.http.routers.backend-upload.rule=PathPrefix(`/upload`)" diff --git a/docker-compose.private-dev.yml b/docker-compose.private-dev.yml deleted file mode 100644 index b21d886d72c9..000000000000 --- a/docker-compose.private-dev.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: "3" - -services: - frontend: - env_file: - - ./frontend/.env - build: - context: frontend - dockerfile: Dockerfile.dev - container_name: web - restart: always - volumes: - - ./frontend/:/app - - /app/node_modules - - /app/.next - ports: - - 3000:3000 - backend-core: - env_file: - - ./backend/core/.env - build: - context: backend/core - dockerfile: Dockerfile - container_name: backend-core - restart: always - volumes: - - ./backend/core/:/code/ - - ~/.config/gcloud:/root/.config/gcloud - ports: - - 5050:5050 - backend-private: - env_file: - - ./backend/private/.env - build: - context: backend/private - dockerfile: Dockerfile - container_name: backend-private - restart: always - volumes: - - ./backend/private/:/app/ - ports: - - 5051:5050 diff --git a/docker-compose.yml b/docker-compose.yml index 609fa753b520..7fc4a8718cd4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,15 +27,15 @@ services: backend-core: env_file: - - ./backend/core/.env + - ./backend/.env build: - context: backend/core + context: backend dockerfile: Dockerfile container_name: backend-core command: uvicorn main:app --host 0.0.0.0 --port 5050 restart: always volumes: - - ./backend/core/:/code/ + - ./backend/:/code/ labels: - "traefik.enable=true" - "traefik.http.routers.backend-core.rule=PathPrefix(`/`)" @@ -44,15 +44,15 @@ services: backend-chat: env_file: - - ./backend/core/.env + - ./backend/.env build: - context: backend/core + context: backend dockerfile: Dockerfile container_name: backend-chat command: uvicorn chat_service:app --host 0.0.0.0 --port 5050 restart: always volumes: - - ./backend/core/:/code/ + - ./backend/:/code/ labels: - "traefik.enable=true" - "traefik.http.routers.backend-chat.rule=PathPrefix(`/chat`)" @@ -61,15 +61,15 @@ services: backend-crawl: env_file: - - ./backend/core/.env + - ./backend/.env build: - context: backend/core + context: backend dockerfile: Dockerfile container_name: backend-crawl command: uvicorn crawl_service:app --host 0.0.0.0 --port 5050 restart: always volumes: - - ./backend/core/:/code/ + - ./backend/:/code/ labels: - "traefik.enable=true" - "traefik.http.routers.backend-crawl.rule=PathPrefix(`/crawl`)" @@ -78,15 +78,15 @@ services: backend-upload: env_file: - - ./backend/core/.env + - ./backend/.env build: - context: backend/core + context: backend dockerfile: Dockerfile container_name: backend-upload command: uvicorn upload_service:app --host 0.0.0.0 --port 5050 restart: always volumes: - - ./backend/core/:/code/ + - ./backend/:/code/ labels: - "traefik.enable=true" - "traefik.http.routers.backend-upload.rule=PathPrefix(`/upload`)" diff --git a/docs/docs/backend/llm/private-llm.md b/docs/docs/backend/llm/private-llm.md index 716a235b178b..f4d32ed51aa7 100644 --- a/docs/docs/backend/llm/private-llm.md +++ b/docs/docs/backend/llm/private-llm.md @@ -12,7 +12,7 @@ This means that your data never leaves the server. The LLM is downloaded to the ## How to use -Set the 'private' flag to True in the /backend/core/.env file. You can also set other model parameters in the .env file. +Set the 'private' flag to True in the /backend/.env file. You can also set other model parameters in the .env file. Download the GPT4All model from [here](https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin) and place it in the /backend/local_models folder. Or you can download any model from their ecosystem on there [website](https://gpt4all.io/index.html). diff --git a/docs/docs/get_started/intro.md b/docs/docs/get_started/intro.md index 5639fd3fb988..6acc4d4236a1 100644 --- a/docs/docs/get_started/intro.md +++ b/docs/docs/get_started/intro.md @@ -73,11 +73,11 @@ Additionally, you'll need a [Supabase](https://supabase.com/) account for: - **Step 2**: Copy the `.XXXXX_env` files ```bash -cp .backend_env.example backend/core/.env +cp .backend_env.example backend/.env cp .frontend_env.example frontend/.env ``` -- **Step 3**: Update the `backend/core/.env` and `frontend/.env` file +- **Step 3**: Update the `backend/.env` and `frontend/.env` file > _Your `supabase_service_key` can be found in your Supabase dashboard under Project Settings -> API. Use the `anon` `public` key found in the `Project API keys` section._ @@ -85,9 +85,9 @@ cp .frontend_env.example frontend/.env > _The `NEXT_PUBLIC_BACKEND_URL` is set to localhost:5050 for the docker. Update it if you are running the backend on a different machine._ -> _To activate vertexAI with PaLM from GCP follow the instructions [here](https://python.langchain.com/en/latest/modules/models/llms/integrations/google_vertex_ai_palm.html) and update `backend/core/.env`- It is an advanced feature, please be expert in GCP before trying to use it_ +> _To activate vertexAI with PaLM from GCP follow the instructions [here](https://python.langchain.com/en/latest/modules/models/llms/integrations/google_vertex_ai_palm.html) and update `backend/.env`- It is an advanced feature, please be expert in GCP before trying to use it_ -- [ ] Change variables in `backend/core/.env` +- [ ] Change variables in `backend/.env` - [ ] Change variables in `frontend/.env` - **Step 4**: Create your database tables and functions with one of these two options: diff --git a/install_helper.sh b/install_helper.sh index 313ddd253f9a..3611dc37280d 100755 --- a/install_helper.sh +++ b/install_helper.sh @@ -21,9 +21,9 @@ replace_in_file() { } # Step 2: Copy the .XXXXX_env files if they don't exist -if [ ! -f backend/core/.env ]; then +if [ ! -f backend/.env ]; then echo "Copying backend-core .env example file..." - cp .backend_env.example backend/core/.env + cp .backend_env.example backend/.env fi if [ ! -f frontend/.env ]; then @@ -34,35 +34,35 @@ fi # Step 3: Ask the user for environment variables and update .env files # only if they haven't been set. -# Update backend/core/.env -if grep -q "SUPABASE_URL=" backend/core/.env; then +# Update backend/.env +if grep -q "SUPABASE_URL=" backend/.env; then echo "SUPABASE_URL can be found in your Supabase dashboard under Settings > API." - SUPABASE_URL=$(gum input --placeholder "Enter SUPABASE_URL for backend/core") - replace_in_file backend/core/.env "SUPABASE_URL=.*" "SUPABASE_URL=${SUPABASE_URL}" + SUPABASE_URL=$(gum input --placeholder "Enter SUPABASE_URL for backend") + replace_in_file backend/.env "SUPABASE_URL=.*" "SUPABASE_URL=${SUPABASE_URL}" fi -if grep -q "SUPABASE_SERVICE_KEY=" backend/core/.env; then +if grep -q "SUPABASE_SERVICE_KEY=" backend/.env; then echo "SUPABASE_SERVICE_KEY can be found in your Supabase dashboard under Settings > API. Use the anon public key found in the Project API keys section." - SUPABASE_SERVICE_KEY=$(gum input --placeholder "Enter SUPABASE_SERVICE_KEY for backend/core") - replace_in_file backend/core/.env "SUPABASE_SERVICE_KEY=.*" "SUPABASE_SERVICE_KEY=${SUPABASE_SERVICE_KEY}" + SUPABASE_SERVICE_KEY=$(gum input --placeholder "Enter SUPABASE_SERVICE_KEY for backend") + replace_in_file backend/.env "SUPABASE_SERVICE_KEY=.*" "SUPABASE_SERVICE_KEY=${SUPABASE_SERVICE_KEY}" fi -if grep -q "PG_DATABASE_URL=" backend/core/.env; then +if grep -q "PG_DATABASE_URL=" backend/.env; then echo "PG_DATABASE_URL can be found in your Postgres provider Settings > API." PG_DATABASE_URL=$(gum input --placeholder "Enter PG_DATABASE_URL for backend") replace_in_file backend/.env "PG_DATABASE_URL=.*" "PG_DATABASE_URL=${PG_DATABASE_URL}" fi -if grep -q "OPENAI_API_KEY=" backend/core/.env; then +if grep -q "OPENAI_API_KEY=" backend/.env; then echo "OPENAI_API_KEY is the API key from OpenAI, if you are using OpenAI services." - OPENAI_API_KEY=$(gum input --placeholder "Enter OPENAI_API_KEY for backend/core") - replace_in_file backend/core/.env "OPENAI_API_KEY=.*" "OPENAI_API_KEY=${OPENAI_API_KEY}" + OPENAI_API_KEY=$(gum input --placeholder "Enter OPENAI_API_KEY for backend") + replace_in_file backend/.env "OPENAI_API_KEY=.*" "OPENAI_API_KEY=${OPENAI_API_KEY}" fi -if grep -q "JWT_SECRET_KEY=" backend/core/.env; then +if grep -q "JWT_SECRET_KEY=" backend/.env; then echo "JWT_SECRET_KEY can be found in your Supabase project dashboard under Settings > API > JWT Settings > JWT Secret." - JWT_SECRET_KEY=$(gum input --placeholder "Enter JWT_SECRET_KEY for backend/core") - replace_in_file backend/core/.env "JWT_SECRET_KEY=.*" "JWT_SECRET_KEY=${JWT_SECRET_KEY}" + JWT_SECRET_KEY=$(gum input --placeholder "Enter JWT_SECRET_KEY for backend") + replace_in_file backend/.env "JWT_SECRET_KEY=.*" "JWT_SECRET_KEY=${JWT_SECRET_KEY}" fi # Update frontend/.env using the same SUPABASE_URL and SUPABASE_SERVICE_KEY