Simple text chunking

Gemfile.lock

@@ -2,6 +2,7 @@ PATH
   remote: .
   specs:
     langchainrb (0.5.5)
+      baran
       colorize (~> 0.8.1)
       tiktoken_ruby (~> 0.0.5)
 
@@ -32,6 +33,7 @@ GEM
     afm (0.2.2)
     ai21 (0.2.0)
     ast (2.4.2)
+    baran (0.1.6)
     builder (3.2.4)
     byebug (11.1.3)
     childprocess (4.1.0)

langchain.gemspec

@@ -32,8 +32,9 @@ Gem::Specification.new do |spec|
 
   # dependencies
   # Not sure if we should require this as it only applies to OpenAI usecase.
-  spec.add_dependency "tiktoken_ruby", "~> 0.0.5"
+  spec.add_dependency "baran"
   spec.add_dependency "colorize", "~> 0.8.1"
+  spec.add_dependency "tiktoken_ruby", "~> 0.0.5"
 
   # development dependencies
   spec.add_development_dependency "dotenv-rails", "~> 2.7.6"

lib/langchain.rb

@@ -78,6 +78,11 @@ module Agent
     autoload :SQLQueryAgent, "langchain/agent/sql_query_agent/sql_query_agent.rb"
   end
 
+  module Chunker
+    autoload :Base, "langchain/chunker/base"
+    autoload :Text, "langchain/chunker/text"
+  end
+
   module Tool
     autoload :Base, "langchain/tool/base"
     autoload :Calculator, "langchain/tool/calculator"

lib/langchain/chunker/base.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Chunker
+    class Base
+    end
+  end
+end

lib/langchain/chunker/text.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "baran"
+
+module Langchain
+  module Chunker
+    class Text < Base
+      attr_reader :text, :chunk_size, :chunk_overlap, :separator
+
+      def initialize(text, chunk_size: 1000, chunk_overlap: 200, separator: "\n\n")
+        @text = text
+        @chunk_size = chunk_size
+        @chunk_overlap = chunk_overlap
+        @separator = separator
+      end
+
+      def chunks
+        splitter = Baran::CharacterTextSplitter.new(
+          chunk_size: chunk_size,
+          chunk_overlap: chunk_overlap,
+          separator: separator
+        )
+        splitter.chunks(text)
+      end
+    end
+  end
+end

lib/langchain/data.rb

@@ -12,5 +12,9 @@ def initialize(data, options = {})
     def value
       @data
     end
+
+    def chunks(opts = {})
+      Langchain::Chunker::Text.new(@data, **opts).chunks
+    end
   end
 end