add jina new models in jina connector (infiniflow#3770)

### What problem does this PR solve? add new models in jinna connector, to allow use models that support multilingual models ### Type of change - [X] Other (please describe): new connectors no breaking change
isthaison · Dec 2, 2024 · 59a5813 · 59a5813
1 parent 08c1a5e
commit 59a5813
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 2 deletions.
diff --git a/conf/llm_factories.json b/conf/llm_factories.json
@@ -525,6 +525,18 @@
                     "tags": "TEXT EMBEDDING",
                     "max_tokens": 8196,
                     "model_type": "embedding"
+                },
+                {
+                    "llm_name": "jina-reranker-v2-base-multilingual",
+                    "tags": "RE-RANK,8k",
+                    "max_tokens": 8196,
+                    "model_type": "rerank"
+                },
+                {
+                    "llm_name": "jina-embeddings-v3",
+                    "tags": "TEXT EMBEDDING",
+                    "max_tokens": 8196,
+                    "model_type": "embedding"
                 }
             ]
         },
@@ -2432,6 +2444,18 @@
                     "max_tokens": 4000,
                     "model_type": "embedding"
                 },
+                {
+                    "llm_name": "voyage-3",
+                    "tags": "TEXT EMBEDDING,32000",
+                    "max_tokens": 32000,
+                    "model_type": "embedding"
+                },
+                {
+                    "llm_name": "voyage-3-lite",
+                    "tags": "TEXT EMBEDDING,32000",
+                    "max_tokens": 32000,
+                    "model_type": "embedding"
+                },                
                 {
                     "llm_name": "rerank-1",
                     "tags": "RE-RANK, 8000",
@@ -2443,6 +2467,18 @@
                     "tags": "RE-RANK, 4000",
                     "max_tokens": 4000,
                     "model_type": "rerank"
+                },
+                {
+                    "llm_name": "rerank-2",
+                    "tags": "RE-RANK, 16000",
+                    "max_tokens": 16000,
+                    "model_type": "rerank"
+                },
+                {
+                    "llm_name": "rerank-2-lite",
+                    "tags": "RE-RANK, 8000",
+                    "max_tokens": 8000,
+                    "model_type": "rerank"
                 }
             ]
         },

diff --git a/rag/llm/embedding_model.py b/rag/llm/embedding_model.py
@@ -322,7 +322,7 @@ def encode_queries(self, text):
 
 
 class JinaEmbed(Base):
-    def __init__(self, key, model_name="jina-embeddings-v2-base-zh",
+    def __init__(self, key, model_name="jina-embeddings-v3",
                  base_url="https://api.jina.ai/v1/embeddings"):
 
         self.base_url = "https://api.jina.ai/v1/embeddings"

diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.py
@@ -92,7 +92,7 @@ def similarity(self, query: str, texts: list):
 
 
 class JinaRerank(Base):
-    def __init__(self, key, model_name="jina-reranker-v1-base-en",
+    def __init__(self, key, model_name="jina-reranker-v2-base-multilingual",
                  base_url="https://api.jina.ai/v1/rerank"):
         self.base_url = "https://api.jina.ai/v1/rerank"
         self.headers = {