docker_workspace: support dockerignore to filter files from the workspace (#401)

Summary:
This adds a filter when building the docker workspace context to filter files specified in the `.dockerignore` file. This reduces the size of the patch when operating in a larger repo.

Reference:
https://docs.docker.com/engine/reference/builder/#dockerignore-file

Next steps from this would be to extend this logic to be workspace agnostic so we could have a `.torchxignore` file but not sure if that makes sense or is strictly necessary yet.

Pull Request resolved: #401

Test Plan:
$  torchx run --scheduler aws_batch --wait --log dist.ddp --script foo.py  -j 1x1
$ pytest torchx/workspace/

Reviewed By: kiukchung

Differential Revision: D34486363

Pulled By: d4l3k

fbshipit-source-id: b71ac0bb2a798fdfd95f13703a700ab9850d127a

Author: d4l3k

torchx/workspace/docker_workspace.py

@@ -4,12 +4,13 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import fnmatch
 import io
 import logging
 import posixpath
 import tarfile
 import tempfile
-from typing import IO, TYPE_CHECKING, Optional, Dict, Tuple, Mapping
+from typing import IO, TYPE_CHECKING, Optional, Dict, Tuple, Mapping, Iterable
 
 import fsspec
 import torchx
@@ -129,17 +130,44 @@ def _build_context(img: str, workspace: str) -> IO[bytes]:
     return f
 
 
-def _copy_to_tarfile(workspace: str, tf: tarfile.TarFile) -> None:
-    # TODO(d4l3k) implement docker ignore files
+def _ignore(s: str, patterns: Iterable[str]) -> bool:
+    match = False
+    for pattern in patterns:
+        if pattern.startswith("!") and fnmatch.fnmatch(s, pattern[1:]):
+            match = False
+        elif fnmatch.fnmatch(s, pattern):
+            match = True
+    return match
+
 
+def _copy_to_tarfile(workspace: str, tf: tarfile.TarFile) -> None:
     fs, path = fsspec.core.url_to_fs(workspace)
     assert isinstance(path, str), "path must be str"
 
+    # load dockerignore
+    # https://docs.docker.com/engine/reference/builder/#dockerignore-file
+    ignore_patterns = []
+    ignore_path = posixpath.join(path, ".dockerignore")
+    if fs.exists(ignore_path):
+        with fs.open(ignore_path, "rt") as f:
+            lines = f.readlines()
+        for line in lines:
+            line, _, _ = line.partition("#")
+            line = line.strip()
+            if len(line) == 0 or line == ".":
+                continue
+            ignore_patterns.append(line)
+
     for dir, dirs, files in fs.walk(path, detail=True):
         assert isinstance(dir, str), "path must be str"
         relpath = posixpath.relpath(dir, path)
+        if _ignore(relpath, ignore_patterns):
+            continue
         for file, info in files.items():
             with fs.open(info["name"], "rb") as f:
-                tinfo = tarfile.TarInfo(posixpath.join(relpath, file))
+                filepath = posixpath.join(relpath, file) if relpath != "." else file
+                if _ignore(filepath, ignore_patterns):
+                    continue
+                tinfo = tarfile.TarInfo(filepath)
                 tinfo.size = info["size"]
                 tf.addfile(tinfo, f)

torchx/workspace/test/docker_workspace_test.py

@@ -4,12 +4,16 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import tarfile
 import unittest
 from unittest.mock import MagicMock
 
 import fsspec
 from torchx.specs import Role, AppDef
-from torchx.workspace.docker_workspace import DockerWorkspace
+from torchx.workspace.docker_workspace import (
+    DockerWorkspace,
+    _build_context,
+)
 
 
 def has_docker() -> bool:
@@ -114,3 +118,57 @@ def test_push_images(self) -> None:
     def test_push_images_empty(self) -> None:
         workspace = DockerWorkspace()
         workspace._push_images({})
+
+    def test_dockerignore(self) -> None:
+        fs = fsspec.filesystem("memory")
+        files = [
+            "dockerignore/ignoredir/bar",
+            "dockerignore/dir1/bar",
+            "dockerignore/dir/ignorefileglob1",
+            "dockerignore/dir/recursive/ignorefileglob2",
+            "dockerignore/dir/ignorefile",
+            "dockerignore/ignorefile",
+            "dockerignore/ignorefilesuffix",
+            "dockerignore/dir/file",
+            "dockerignore/foo.sh",
+            "dockerignore/unignore",
+        ]
+        for file in files:
+            fs.touch(file)
+        with fs.open("dockerignore/.dockerignore", "wt") as f:
+            f.write(
+                """
+                # comment
+
+                # dirs/files
+                ignoredir
+                ignorefile
+
+                # globs
+                */ignorefileglo*1
+                **/ignorefileglob2
+                dir?
+
+                # inverse patterns
+                unignore
+                !unignore
+
+                # ignore .
+                .
+            """
+            )
+
+        with _build_context("img", "memory://dockerignore") as f:
+            with tarfile.open(fileobj=f, mode="r") as tf:
+                self.assertCountEqual(
+                    tf.getnames(),
+                    {
+                        "Dockerfile",
+                        "foo.sh",
+                        ".dockerignore",
+                        "dir/ignorefile",
+                        "ignorefilesuffix",
+                        "dir/file",
+                        "unignore",
+                    },
+                )