fix dummy batch (#51)

dptech-corp · Jun 8, 2024 · 8cc441a · 8cc441a
1 parent bcf247c
commit 8cc441a
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/unicore/trainer.py b/unicore/trainer.py
@@ -514,7 +514,9 @@ def get_valid_iterator(
             data_buffer_size=self.args.data_buffer_size,
             disable_iterator_cache=disable_iterator_cache,
         )
-        self.reset_dummy_batch(batch_iterator.first_batch)
+        # Using training data for dummy batch. If the following line is enabled, the dummy batch will be from validation data, 
+        # and cause OOM error for some corner case during training. So disable it. 
+        # self.reset_dummy_batch(batch_iterator.first_batch)
         return batch_iterator
 
     def begin_epoch(self, epoch):