Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: pass selfhosted embeddings to graphrag #266

Closed
wants to merge 3 commits into from
Closed

feat: pass selfhosted embeddings to graphrag #266

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bbd3170
pass selfhosted embeddings to graphrag
Sep 10, 2024
6cf9cfd
pass selfhosted embeddings to align
Sep 10, 2024
418af29
format
Sep 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions libs/ktem/ktem/index/file/graph/graph_index.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ def get_retriever_pipelines(
GraphRAGRetrieverPipeline(
file_ids=file_ids,
Index=self._resources["Index"],
index_settings=self.config,
)
]

Expand Down
25 changes: 17 additions & 8 deletions libs/ktem/ktem/index/file/graph/pipelines.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
import os
import subprocess
from pathlib import Path
from shutil import rmtree
from typing import Generator
from uuid import uuid4

import openai
import pandas as pd
import tiktoken
from ktem.db.models import engine
from ktem.embeddings.manager import embedding_models_manager
from sqlalchemy.orm import Session
from theflow.settings import settings

from kotaemon.base import Document, Param, RetrievedDocument
from kotaemon.embeddings import OpenAIEmbeddings

from ..pipelines import BaseFileIndexRetriever, IndexDocumentPipeline, IndexPipeline
from .visualize import create_knowledge_graph, visualize_graph
Expand Down Expand Up @@ -145,6 +147,7 @@ class GraphRAGRetrieverPipeline(BaseFileIndexRetriever):

Index = Param(help="The SQLAlchemy Index table")
file_ids: list[str] = []
index_settings: dict = {} # why not self.params["index_settings"]?

@classmethod
def get_user_settings(cls) -> dict:
Expand Down Expand Up @@ -227,15 +230,21 @@ def _build_graph_search(self):
# Read text units
text_unit_df = pd.read_parquet(f"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet")
text_units = read_indexer_text_units(text_unit_df)

embedding_model = os.getenv("GRAPHRAG_EMBEDDING_MODEL")
kotaemonEmbeddings: OpenAIEmbeddings = embedding_models_manager[
self.index_settings.get(
"embedding", embedding_models_manager.get_default_name()
)
]
# embedding_model = os.getenv("GRAPHRAG_EMBEDDING_MODEL")
text_embedder = OpenAIEmbedding(
api_key=os.getenv("OPENAI_API_KEY"),
api_base=None,
api_key=kotaemonEmbeddings.api_key,
api_base=kotaemonEmbeddings.base_url,
api_type=OpenaiApiType.OpenAI,
model=embedding_model,
deployment_name=embedding_model,
max_retries=20,
model=kotaemonEmbeddings.model,
deployment_name=kotaemonEmbeddings.model,
max_retries=kotaemonEmbeddings.max_retries
if kotaemonEmbeddings.max_retries is not None
else openai.DEFAULT_MAX_RETRIES,
)
token_encoder = tiktoken.get_encoding("cl100k_base")

Expand Down
Loading