Merge pull request #2909 from iterative/efiop-patch-1

remote: base: download: don't walk dir twice
iterative · Dec 6, 2019 · eb200db · eb200db
2 parents 9b6644a + cf171f7
commit eb200db
Showing 1 changed file with 5 additions and 8 deletions.
diff --git a/dvc/remote/base.py b/dvc/remote/base.py
@@ -588,26 +588,23 @@ def download(
     def _download_dir(
         self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
     ):
-        file_to_infos = (
-            to_info / file_to_info.relative_to(from_info)
-            for file_to_info in self.walk_files(from_info)
+        from_infos = list(self.walk_files(from_info))
+        to_infos = (
+            to_info / info.relative_to(from_info) for info in from_infos
         )
 
         with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
-            file_from_info = list(self.walk_files(from_info))
             download_files = partial(
                 self._download_file,
                 name=name,
                 no_progress_bar=True,
                 file_mode=file_mode,
                 dir_mode=dir_mode,
             )
-            futures = executor.map(
-                download_files, file_from_info, file_to_infos
-            )
+            futures = executor.map(download_files, from_infos, to_infos)
             with Tqdm(
                 futures,
-                total=len(file_from_info),
+                total=len(from_infos),
                 desc="Downloading directory",
                 unit="Files",
                 disable=no_progress_bar,