Reducing training and validation steps (huggingface#8)

rraminen · Jul 22, 2021 · 8682754 · 8682754
1 parent bd12e8b
commit 8682754
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/scripts/gpt2-tf2/gpt2_train_distributed.py b/scripts/gpt2-tf2/gpt2_train_distributed.py
@@ -50,7 +50,7 @@ def get_dataset(fil):
 
 def tokenize(data, tokenizer, truncate=False):
     if truncate:
-        data = tokenizer(data[:1000], return_tensors='tf', padding=True, truncation=True)
+        data = tokenizer(data[:100], return_tensors='tf', padding=True, truncation=True)
     else:
         data = tokenizer(data, return_tensors='tf', padding=True, truncation=True)
     return tf.data.Dataset.from_tensor_slices((dict(data), data['input_ids']))