Merge pull request opensearch-project#8 from xinyual/addDefaultDelimiter

add default delimiter value
xinyual · Mar 7, 2024 · 43393d2 · 43393d2
2 parents 17e34b5 + 1fa292c
commit 43393d2
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 11 deletions.
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunker.java b/src/main/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunker.java
@@ -25,8 +25,6 @@ public void validateParameters(Map<String, Object> parameters) {
             } else if (((String) delimiter).isEmpty()) {
                 throw new IllegalArgumentException("delimiter parameters should not be empty.");
             }
-        } else {
-            throw new IllegalArgumentException("You must contain field: " + DELIMITER_FIELD + " in your parameter.");
         }
         if (parameters.containsKey(MAX_CHUNK_LIMIT_FIELD)) {
             Object maxChunkLimit = parameters.get(MAX_CHUNK_LIMIT_FIELD);
@@ -40,7 +38,7 @@ public void validateParameters(Map<String, Object> parameters) {
 
     @Override
     public List<String> chunk(String content, Map<String, Object> parameters) {
-        String delimiter = (String) parameters.get(DELIMITER_FIELD);
+        String delimiter = (String) parameters.getOrDefault(DELIMITER_FIELD, ".");
         int maxChunkingNumber = (int) parameters.getOrDefault(MAX_CHUNK_LIMIT_FIELD, -1);
         List<String> chunkResult = new ArrayList<>();
         int start = 0;

diff --git a/src/test/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunkerTests.java b/src/test/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunkerTests.java
@@ -17,14 +17,6 @@
 
 public class DelimiterChunkerTests extends OpenSearchTestCase {
 
-    public void testChunkerWithNoDelimiterField() {
-        DelimiterChunker chunker = new DelimiterChunker();
-        String content = "a\nb\nc\nd";
-        Map<String, Object> inputParameters = Map.of("", "");
-        Exception exception = assertThrows(IllegalArgumentException.class, () -> chunker.validateParameters(inputParameters));
-        Assert.assertEquals("You must contain field: " + DELIMITER_FIELD + " in your parameter.", exception.getMessage());
-    }
-
     public void testChunkerWithWrongLimitFieldList() {
         DelimiterChunker chunker = new DelimiterChunker();
         String content = "a\nb\nc\nd";
@@ -81,6 +73,14 @@ public void testChunker() {
         assertEquals(List.of("a\n", "b\n", "c\n", "d"), chunkResult);
     }
 
+    public void testChunkerWithDefaultDelimiter() {
+        DelimiterChunker chunker = new DelimiterChunker();
+        String content = "a.b.c.d";
+        Map<String, Object> inputParameters = Map.of();
+        List<String> chunkResult = chunker.chunk(content, inputParameters);
+        assertEquals(List.of("a.", "b.", "c.", "d"), chunkResult);
+    }
+
     public void testChunkerWithDelimiterEnd() {
         DelimiterChunker chunker = new DelimiterChunker();
         String content = "a\nb\nc\nd\n";