Update ernie and roberta tokenizer (PaddlePaddle#36)

update ernie and roberta tokenizer
wangxicoding · Feb 25, 2021 · 237a638 · 237a638
2 parents aa63a93 + f40d2d6
commit 237a638
Show file tree

Hide file tree

Showing 4 changed files with 92 additions and 518 deletions.
diff --git a/paddlenlp/datasets/__init__.py b/paddlenlp/datasets/__init__.py
@@ -25,4 +25,5 @@
 from .dureader import *
 from .cnndm import *
 from .poetry import *
-from .couplet import *
+from .couplet import *
+from .experimental import load_dataset
diff --git a/paddlenlp/datasets/experimental/dataset.py b/paddlenlp/datasets/experimental/dataset.py
@@ -124,9 +124,6 @@ def get_datasets(cls, *args, **kwargs):
     return datasets if len(datasets) > 1 else datasets[0]
 
 
-Dataset.get_datasets = get_datasets
-
-
 class MapDataset(Dataset):
     """
     Wraps a dataset-like object as a instance of Dataset, and equips it with