fix: setDocumentHash should be async (run-llama#868)

himself65 · Jun 3, 2024 · a29d835 · a29d835
1 parent da1f025
commit a29d835
Show file tree

Hide file tree

Showing 9 changed files with 14 additions and 8 deletions.
diff --git a/.changeset/twelve-bottles-wait.md b/.changeset/twelve-bottles-wait.md
@@ -0,0 +1,6 @@
+---
+"llamaindex": minor
+"docs": minor
+---
+
+setDocumentHash should be async
diff --git a/...i18n/fr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md b/...i18n/fr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md
@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
 
 #### Parameters
 

diff --git a/...i18n/hr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md b/...i18n/hr/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md
@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
 
 #### Parameters
 

diff --git a/...zh-Hans/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md b/...zh-Hans/docusaurus-plugin-content-docs/current/api/classes/BaseDocumentStore.md
@@ -271,7 +271,7 @@ custom_edit_url: null
 
 ### setDocumentHash
 
-▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `void`
+▸ `Abstract` **setDocumentHash**(`docId`, `docHash`): `Promise`<`void`\>
 
 #### Parameters
 

diff --git a/packages/core/src/indices/BaseIndex.ts b/packages/core/src/indices/BaseIndex.ts
@@ -95,7 +95,7 @@ export abstract class BaseIndex<T> {
       [nodeParserFromSettingsOrContext(this.serviceContext)],
     );
     await this.insertNodes(nodes);
-    this.docStore.setDocumentHash(document.id_, document.hash);
+    await this.docStore.setDocumentHash(document.id_, document.hash);
   }
 
   abstract insertNodes(nodes: BaseNode[]): Promise<void>;

diff --git a/packages/core/src/indices/keyword/index.ts b/packages/core/src/indices/keyword/index.ts
@@ -281,7 +281,7 @@ export class KeywordTableIndex extends BaseIndex<KeywordTable> {
 
     await docStore.addDocuments(documents, true);
     for (const doc of documents) {
-      docStore.setDocumentHash(doc.id_, doc.hash);
+      await docStore.setDocumentHash(doc.id_, doc.hash);
     }
 
     const nodes = serviceContext.nodeParser.getNodesFromDocuments(documents);

diff --git a/packages/core/src/indices/summary/index.ts b/packages/core/src/indices/summary/index.ts
@@ -138,7 +138,7 @@ export class SummaryIndex extends BaseIndex<IndexList> {
 
     await docStore.addDocuments(documents, true);
     for (const doc of documents) {
-      docStore.setDocumentHash(doc.id_, doc.hash);
+      await docStore.setDocumentHash(doc.id_, doc.hash);
     }
 
     const nodes =

diff --git a/packages/core/src/ingestion/strategies/DuplicatesStrategy.ts b/packages/core/src/ingestion/strategies/DuplicatesStrategy.ts
@@ -19,7 +19,7 @@ export class DuplicatesStrategy implements TransformComponent {
 
     for (const node of nodes) {
       if (!(node.hash in hashes) && !currentHashes.has(node.hash)) {
-        this.docStore.setDocumentHash(node.id_, node.hash);
+        await this.docStore.setDocumentHash(node.id_, node.hash);
         nodesToRun.push(node);
         currentHashes.add(node.hash);
       }

diff --git a/packages/core/src/storage/docStore/types.ts b/packages/core/src/storage/docStore/types.ts
@@ -32,7 +32,7 @@ export abstract class BaseDocumentStore {
   abstract documentExists(docId: string): Promise<boolean>;
 
   // Hash
-  abstract setDocumentHash(docId: string, docHash: string): void;
+  abstract setDocumentHash(docId: string, docHash: string): Promise<void>;
 
   abstract getDocumentHash(docId: string): Promise<string | undefined>;