docker_workspace: support dockerignore to filter files from the workspace

Author: d4l3k

torchx/workspace/docker_workspace.py

@@ -4,12 +4,13 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import fnmatch
 import io
 import logging
 import posixpath
 import tarfile
 import tempfile
-from typing import IO, TYPE_CHECKING, Optional, Dict, Tuple, Mapping
+from typing import IO, TYPE_CHECKING, Optional, Dict, Tuple, Mapping, Iterable
 
 import fsspec
 import torchx
@@ -129,17 +130,41 @@ def _build_context(img: str, workspace: str) -> IO[bytes]:
     return f
 
 
-def _copy_to_tarfile(workspace: str, tf: tarfile.TarFile) -> None:
-    # TODO(d4l3k) implement docker ignore files
+def _glob_any(s: str, patterns: Iterable[str]) -> bool:
+    for pattern in patterns:
+        if fnmatch.fnmatch(s, pattern):
+            return True
+    return False
+
 
+def _copy_to_tarfile(workspace: str, tf: tarfile.TarFile) -> None:
     fs, path = fsspec.core.url_to_fs(workspace)
     assert isinstance(path, str), "path must be str"
 
+    # load dockerignore
+    # https://docs.docker.com/engine/reference/builder/#dockerignore-file
+    ignore_patterns = set()
+    ignore_path = posixpath.join(path, ".dockerignore")
+    if fs.exists(ignore_path):
+        with fs.open(ignore_path, "rt") as f:
+            lines = f.readlines()
+        for line in lines:
+            line, _, _ = line.partition("#")
+            line = line.strip()
+            if len(line) == 0:
+                continue
+            ignore_patterns.add(line)
+
     for dir, dirs, files in fs.walk(path, detail=True):
         assert isinstance(dir, str), "path must be str"
         relpath = posixpath.relpath(dir, path)
+        if _glob_any(relpath, ignore_patterns):
+            continue
         for file, info in files.items():
             with fs.open(info["name"], "rb") as f:
-                tinfo = tarfile.TarInfo(posixpath.join(relpath, file))
+                filepath = posixpath.join(relpath, file) if relpath != "." else file
+                if _glob_any(filepath, ignore_patterns):
+                    continue
+                tinfo = tarfile.TarInfo(filepath)
                 tinfo.size = info["size"]
                 tf.addfile(tinfo, f)

torchx/workspace/test/docker_workspace_test.py

@@ -4,12 +4,16 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import tarfile
 import unittest
 from unittest.mock import MagicMock
 
 import fsspec
 from torchx.specs import Role, AppDef
-from torchx.workspace.docker_workspace import DockerWorkspace
+from torchx.workspace.docker_workspace import (
+    DockerWorkspace,
+    _build_context,
+)
 
 
 def has_docker() -> bool:
@@ -114,3 +118,36 @@ def test_push_images(self) -> None:
     def test_push_images_empty(self) -> None:
         workspace = DockerWorkspace()
         workspace._push_images({})
+
+    def test_dockerignore(self) -> None:
+        fs = fsspec.filesystem("memory")
+        fs.touch("dockerignore/ignoredir/bar")
+        fs.touch("dockerignore/dir1/bar")
+        fs.touch("dockerignore/dir/ignorefileglob")
+        fs.touch("dockerignore/dir/ignorefile")
+        fs.touch("dockerignore/ignorefile")
+        fs.touch("dockerignore/dir/file")
+        fs.touch("dockerignore/foo.sh")
+        with fs.open("dockerignore/.dockerignore", "wt") as f:
+            f.write(
+                """
+                # comment
+                ignoredir
+                ignorefile
+                */ignorefileglo*
+                dir?
+            """
+            )
+
+        with _build_context("img", "memory://dockerignore") as f:
+            with tarfile.open(fileobj=f, mode="r") as tf:
+                self.assertCountEqual(
+                    tf.getnames(),
+                    {
+                        "Dockerfile",
+                        "foo.sh",
+                        ".dockerignore",
+                        "dir/ignorefile",
+                        "dir/file",
+                    },
+                )