From e3d157b04cde1c6459a50a55140595af72260ce2 Mon Sep 17 00:00:00 2001
From: Dai Sugimori <daixque@gmail.com>
Date: Sat, 15 Apr 2023 22:53:37 +0900
Subject: [PATCH] Add BertJapaneseTokenizer support

---
 eland/ml/pytorch/transformers.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/eland/ml/pytorch/transformers.py b/eland/ml/pytorch/transformers.py
index f48f7be0..b39a16f7 100644
--- a/eland/ml/pytorch/transformers.py
+++ b/eland/ml/pytorch/transformers.py
@@ -107,6 +107,7 @@
     transformers.RobertaTokenizer,
     transformers.BartTokenizer,
     transformers.SqueezeBertTokenizer,
+    transformers.BertJapaneseTokenizer,
 )
 SUPPORTED_TOKENIZERS_NAMES = ", ".join(sorted([str(x) for x in SUPPORTED_TOKENIZERS]))