compact chunks on text splitter

moekiorg · Aug 23, 2023 · feb2fd8 · feb2fd8
1 parent bdf9f25
commit feb2fd8
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/lib/baran/text_splitter.rb b/lib/baran/text_splitter.rb
@@ -18,7 +18,7 @@ def chunks(text)
       cursor = 0
       chunks = []
 
-      splitted(text).each do |chunk|
+      splitted(text).compact.each do |chunk|
         chunks << { text: chunk, cursor: cursor }
         cursor += chunk.length
       end

diff --git a/test/test_recursive_character_text_splitter.rb b/test/test_recursive_character_text_splitter.rb
@@ -13,4 +13,11 @@ def test_chunks
 
     assert_equal(chunks.length, 6)
   end
+
+
+  def test_empty_chunks
+    chunks = @splitter.chunks("\n\nHello, world!\n\nHello, world!\n\nHello, world!")
+
+    assert_equal(chunks.length, 6)
+  end
 end