don't wait for first chunk to be uploaded to start all workers (#21)

huggingface · Mar 3, 2021 · 5b1c1e6 · 5b1c1e6
1 parent 446ea44
commit 5b1c1e6
Showing 1 changed file with 12 additions and 0 deletions.
diff --git a/src/huggingface_hub/commands/lfs.py b/src/huggingface_hub/commands/lfs.py
@@ -189,6 +189,18 @@ def run(self):
             chunk_size = int(header.pop("chunk_size"))
             presigned_urls: List[str] = list(header.values())
 
+            # Send a "started" progress event to allow other workers to start.
+            # Otherwise they're delayed until first "progress" event is reported,
+            # i.e. after the first 5GB by default (!)
+            write_msg(
+                {
+                    "event": "progress",
+                    "oid": oid,
+                    "bytesSoFar": 1,
+                    "bytesSinceLast": 0,
+                }
+            )
+
             parts = []
             for i, presigned_url in enumerate(presigned_urls):
                 with FileSlice(