import-url: support directories

dvc/remote/base.py

@@ -31,6 +31,7 @@
 from dvc.utils.fs import move
 from dvc.utils.http import open_url
 
+
 logger = logging.getLogger(__name__)
 
 STATUS_OK = 1
@@ -547,19 +548,56 @@ def download(
         if to_info.scheme != "local":
             raise NotImplementedError
 
-        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+        makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
 
-        name = name or to_info.name
+        if self.isdir(from_info):
+            makedirs(to_info, exist_ok=True, mode=dir_mode)
+            file_to_infos = (
+                to_info / file_to_info.relative_to(from_info)
+                for file_to_info in self.walk_files(from_info)
+            )
 
+            with ThreadPoolExecutor(max_workers=self.JOBS) as executor:
+                file_from_info = list(self.walk_files(from_info))
+                with Tqdm(
+                    file_from_info,
+                    total=len(file_from_info),
+                    desc="Downloading directory",
+                ) as file_from_info:
+                    return sum(
+                        executor.map(
+                            partial(
+                                self.single_file_download,
+                                name=name,
+                                no_progress_bar=True,
+                                file_mode=file_mode,
+                                dir_mode=dir_mode,
+                            ),
+                            file_from_info,
+                            file_to_infos,
+                        )
+                    )
+        else:
+            self.single_file_download(
+                from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+            )
+
+    def single_file_download(
+        self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode
+    ):
         makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
+
+        logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
+        name = name or to_info.name
+
         tmp_file = tmp_fname(to_info)
 
         try:
             self._download(
                 from_info, tmp_file, name=name, no_progress_bar=no_progress_bar
             )
         except Exception:
-            msg = "failed to download '{}' to '{}'"
+            msg = "failed to doooooownload '{}' to '{}'"
             logger.exception(msg.format(from_info, to_info))
             return 1  # 1 fail