Update huggingface.py

huggingface · Sep 11, 2024 · c7f6f51 · c7f6f51
1 parent 9142e3e
commit c7f6f51
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/src/datatrove/pipeline/writers/huggingface.py b/src/datatrove/pipeline/writers/huggingface.py
@@ -117,6 +117,8 @@ def close(self, rank: int = 0):
                     logger.info("Commit creation race condition issue. Waiting...")
                     time.sleep(BASE_DELAY * 2**retries + random.uniform(0, 2))
                     retries += 1
+                else:
+                    raise e
 
     def _on_file_switch(self, original_name, old_filename, new_filename):
         """