fix: metadata should not be included after sentence splitter (run-lla…

…ma#1099)
himself65 · Aug 2, 2024 · 04b2f8e · 04b2f8e
1 parent 62b874e
commit 04b2f8e
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/.changeset/real-seahorses-divide.md b/.changeset/real-seahorses-divide.md
@@ -0,0 +1,5 @@
+---
+"@llamaindex/core": patch
+---
+
+Fix issue with metadata included after sentence splitter
diff --git a/packages/core/src/node-parser/base.ts b/packages/core/src/node-parser/base.ts
@@ -140,7 +140,7 @@ export abstract class MetadataAwareTextSplitter extends TextSplitter {
     return nodes.reduce<TextNode[]>((allNodes, node) => {
       const metadataStr = this.getMetadataString(node);
       const splits = this.splitTextMetadataAware(
-        node.getContent(MetadataMode.ALL),
+        node.getContent(MetadataMode.NONE),
         metadataStr,
       );
       return allNodes.concat(buildNodeFromSplits(splits, node));