fix: handling errors in splitBySentenceTokenizer (run-llama#1087)

Co-authored-by: Alex Yang <himself65@outlook.com>

.changeset/thin-pens-deliver.md

@@ -0,0 +1,6 @@
+---
+"@llamaindex/core": patch
+"@llamaindex/core-tests": patch
+---
+
+fix: handling errors in splitBySentenceTokenizer

packages/core/src/node-parser/utils.ts

@@ -39,7 +39,11 @@ export const splitBySentenceTokenizer = (): TextSplitterFn => {
   }
   const tokenizer = sentenceTokenizer;
   return (text: string) => {
-    return tokenizer.tokenize(text);
+    try {
+      return tokenizer.tokenize(text);
+    } catch {
+      return [text];
+    }
   };
 };
 

packages/core/tests/node-parser/text-splitter.test.ts

@@ -1,7 +1,10 @@
-import { SentenceSplitter } from "@llamaindex/core/node-parser";
+import {
+  SentenceSplitter,
+  splitBySentenceTokenizer,
+} from "@llamaindex/core/node-parser";
 import { describe, expect, test } from "vitest";
 
-describe("SentenceSplitter", () => {
+describe("sentence splitter", () => {
   test("initializes", () => {
     const sentenceSplitter = new SentenceSplitter();
     expect(sentenceSplitter).toBeDefined();
@@ -105,4 +108,11 @@ describe("SentenceSplitter", () => {
       "因为他照了人类，连我都在内。",
     ]);
   });
+
+  test("issue 1087 - edge case when input with brackets", () => {
+    const text =
+      "A card must be of uniform thickness and made of unfolded and uncreased paper or cardstock of approximately the quality and weight of a stamped card (i.e., a card available from USPS).";
+    const split = splitBySentenceTokenizer();
+    expect(split(text)).toEqual([text]);
+  });
 });