perf: splitter

c121914yu · Dec 7, 2023 · 883045c · 883045c
1 parent 7f0dcc1
commit 883045c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts
@@ -18,7 +18,7 @@ export const splitText2Chunks = (props: {
   tokens: number;
   overlapRatio?: number;
 } => {
-  let { text = '', chunkLen, overlapRatio = 0.2, customReg = ['\n'] } = props;
+  let { text = '', chunkLen, overlapRatio = 0.2, customReg = [] } = props;
   const splitMarker = 'SPLIT_HERE_SPLIT_HERE';
   const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER';
   const overlapLen = Math.round(chunkLen * overlapRatio);