[Update Success] data set now works for wikitext. Success!

rti · Jan 31, 2024 · aea16e3 · aea16e3
1 parent af2dcf2
commit aea16e3
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 7 deletions.
diff --git a/gswikichat/api.py b/gswikichat/api.py
@@ -26,9 +26,8 @@ async def api(q):
 
     result = embedder.run([query])
 
-    # print(help(retriever))
     results = retriever.run(
-        query_embedding=result['documents'][0].embedding,
+        query_embedding=list(result['documents'][0].embedding),
         filters=None,
         top_k=None,
         scale_score=None,

diff --git a/gswikichat/vector_store_interface.py b/gswikichat/vector_store_interface.py
@@ -26,11 +26,15 @@
             input_documents.append(Document(content=v, meta={"src": k}))
     elif isinstance(json_obj, list):
         for obj_ in json_obj:
-            meta = obj_['meta']
+            url = obj_['meta']
             content = obj_['content']
-            # print(f"Loading {meta}")
-            # print(content)
-            input_documents.append(Document(content=content, meta=meta))
+
+            input_documents.append(
+                Document(
+                    content=content,
+                    meta={'src': url}
+                )
+            )
 else:
     input_documents = [
         Document(
@@ -56,13 +60,14 @@
 )
 # document_store.write_documents(input_documents)
 
-
+# TODO Introduce Jina.AI from HuggingFace. Establish env-variable for trust_...
 embedder = SentenceTransformersDocumentEmbedder(
     model="sentence-transformers/all-MiniLM-L6-v2"
 )
 embedder.warm_up()
 
 documents_with_embeddings = embedder.run(input_documents)
+
 document_store.write_documents(
     documents=documents_with_embeddings['documents'],
     policy=DuplicatePolicy.OVERWRITE