use truncate flag in huggingface tokenizer

Signed-off-by: HenryL27 <hmlindeman@yahoo.com>
HenryL27 · Dec 7, 2023 · 9024a4c · 9024a4c
1 parent 2761d7d
commit 9024a4c
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/...thms/src/main/java/org/opensearch/ml/engine/algorithms/SentenceTransformerTranslator.java b/...thms/src/main/java/org/opensearch/ml/engine/algorithms/SentenceTransformerTranslator.java
@@ -25,7 +25,12 @@ public Batchifier getBatchifier() {
     @Override
     public void prepare(TranslatorContext ctx) throws IOException {
         Path path = ctx.getModel().getModelPath();
-        tokenizer = HuggingFaceTokenizer.builder().optPadding(true).optTokenizerPath(path.resolve("tokenizer.json")).build();
+        tokenizer = HuggingFaceTokenizer
+            .builder()
+            .optPadding(true)
+            .optTruncation(true)
+            .optTokenizerPath(path.resolve("tokenizer.json"))
+            .build();
     }
 
     @Override