Revert "quick fix on concatenating text to support more datasets (hug…

…gingface#8474)" This reverts commit 677c6d9.
fabiocapsouza · Nov 15, 2020 · a3bad48 · a3bad48
1 parent 408656c
commit a3bad48
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/examples/language-modeling/run_clm.py b/examples/language-modeling/run_clm.py
@@ -254,7 +254,7 @@ def tokenize_function(examples):
         tokenize_function,
         batched=True,
         num_proc=data_args.preprocessing_num_workers,
-        remove_columns=column_names,
+        remove_columns=[text_column_name],
         load_from_cache_file=not data_args.overwrite_cache,
     )
 

diff --git a/examples/language-modeling/run_mlm.py b/examples/language-modeling/run_mlm.py
@@ -292,7 +292,7 @@ def tokenize_function(examples):
             tokenize_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
-            remove_columns=column_names,
+            remove_columns=[text_column_name],
             load_from_cache_file=not data_args.overwrite_cache,
         )
 

diff --git a/examples/language-modeling/run_plm.py b/examples/language-modeling/run_plm.py
@@ -279,7 +279,7 @@ def tokenize_function(examples):
             tokenize_function,
             batched=True,
             num_proc=data_args.preprocessing_num_workers,
-            remove_columns=column_names,
+            remove_columns=[text_column_name],
             load_from_cache_file=not data_args.overwrite_cache,
         )