add character text splitter

app/tests/test_text_splitter.py

@@ -0,0 +1,16 @@
+import unittest
+from app.text_splitters.text_splitters import (
+ TextSplitter, character_text_splitter
+)
+from lorem_text import lorem
+
+
+class TestProductsHandler(unittest.TestCase):
+ def setUp(self):
+ self.text = lorem.paragraphs(5)
+
+ def test_character_text_splitter(self):
+ splitter = TextSplitter(character_text_splitter, self.text)
+ chunks = splitter.split_text()
+ self.assertEqual(type(chunks), list)
+ self.assertGreaterEqual(len(chunks), len(chunks))

app/text_splitters/text_splitters.py

@@ -0,0 +1,34 @@
+from typing import Callable, List
+from langchain.text_splitter import CharacterTextSplitter
+from app.util import count_words
+import os
+
+DEFAULT_CHUNK_SIZE = os.environ.get("DEFAULT_CHUNK_SIZE", 75)
+DEFAULT_CHUNK_OVERLAP = os.environ.get("DEFAULT_CHUNK_OVERLAP", 75)
+DEFAULT_SEPARATOR = os.environ.get("DEFAULT_SEPARATOR", "\n")
+
+
+class TextSplitter:
+ def __init__(self, splitter: Callable, content: str) -> None:
+ self.splitter = splitter
+ self.content = content
+
+ def split_text(self) -> Callable:
+ return self.splitter(self.content)
+
+
+def character_text_splitter(
+ content: str,
+ chunk_size: int = DEFAULT_CHUNK_SIZE,
+ chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
+ length_function: Callable = count_words,
+ separator: str = DEFAULT_SEPARATOR) -> List:
+
+ text_splitter = CharacterTextSplitter(
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ length_function=length_function,
+ separator=separator,
+ )
+ pages = text_splitter.split_text(content)
+ return pages

app/util.py

@@ -14,3 +14,7 @@ def transform_input(self, inputs: list[str], model_kwargs: dict) -> bytes:
  def transform_output(self, output: bytes) -> list[list[float]]:
  response_json = json.loads(output.read().decode("utf-8"))
  return response_json["vectors"]
+
+
+def count_words(string: str):
+ return len(string.split())

pyproject.toml

@@ -29,6 +29,7 @@ black = "^23.9.1"
 reportlab = "^4.0.7"
 xlsxwriter = "^3.1.9"
 flake8 = "^6.1.0"
+lorem-text = "^2.1"
 
 [build-system]
 requires = ["poetry-core"]