diff --git a/eland/ml/pytorch/transformers.py b/eland/ml/pytorch/transformers.py index 9b2b78ee..9d3396d1 100644 --- a/eland/ml/pytorch/transformers.py +++ b/eland/ml/pytorch/transformers.py @@ -108,6 +108,7 @@ transformers.RobertaTokenizer, transformers.BartTokenizer, transformers.SqueezeBertTokenizer, + transformers.BertJapaneseTokenizer, ) SUPPORTED_TOKENIZERS_NAMES = ", ".join(sorted([str(x) for x in SUPPORTED_TOKENIZERS]))