feat: leverage RUN --mount for faster image building

We make use of the Docker build cache to install python and nodejs requirements faster in the case of repeated builds. This feature is only possible for users of BuildKit, so we detect whether `docker buildx` is available at runtime. We do not make use of `COPY --link` because the `--link` option is incompatible with `--chown=app:app`: docker/buildx#1408 For reference, see: https://www.docker.com/blog/dockerfiles-now-support-multiple-build-contexts/ https://docs.docker.com/engine/reference/commandline/buildx_build/#build-context
overhangio · May 10, 2023 · e9f0547 · e9f0547
1 parent e201263
commit e9f0547
Show file tree

Hide file tree

Showing 7 changed files with 111 additions and 46 deletions.
diff --git a/changelog.d/20230427_154822_regis_build_mount.md b/changelog.d/20230427_154822_regis_build_mount.md
@@ -0,0 +1 @@
+- [Improvement] Considerably accelerate building the "openedx" Docker image with `RUN --mount=type=cache`. This feature is only for Docker with BuildKit, so detection is performed at build-time. (by @regisb)
diff --git a/tests/commands/test_images.py b/tests/commands/test_images.py
@@ -1,7 +1,7 @@
 from unittest.mock import Mock, patch
 
 from tests.helpers import PluginsTestCase, temporary_root
-from tutor import images, plugins
+from tutor import images, plugins, utils
 from tutor.__about__ import __version__
 from tutor.commands.images import ImageNotFoundError
 
@@ -128,16 +128,29 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None:
             "service1",
         ]
         with temporary_root() as root:
-            self.invoke_in_root(root, ["config", "save"])
-            result = self.invoke_in_root(root, build_args)
+            utils.is_buildkit_enabled.cache_clear()
+            with patch.object(utils, "is_buildkit_enabled", return_value=False):
+                self.invoke_in_root(root, ["config", "save"])
+                result = self.invoke_in_root(root, build_args)
         self.assertIsNone(result.exception)
         self.assertEqual(0, result.exit_code)
         image_build.assert_called()
         self.assertIn("service1:1.0.0", image_build.call_args[0])
-        for arg in image_build.call_args[0][2:]:
-            # The only extra args are `--build-arg`
-            if arg != "--build-arg":
-                self.assertIn(arg, build_args)
+        self.assertEqual(
+            [
+                "service1:1.0.0",
+                "--no-cache",
+                "--build-arg",
+                "myarg=value",
+                "--add-host",
+                "host",
+                "--target",
+                "target",
+                "docker_args",
+                "--cache-from=type=registry,ref=service1:1.0.0-cache",
+            ],
+            list(image_build.call_args[0][1:])
+        )
 
     def test_images_push(self) -> None:
         result = self.invoke(["images", "push"])

diff --git a/tutor/commands/images.py b/tutor/commands/images.py
@@ -83,6 +83,14 @@ def images_command() -> None:
     is_flag=True,
     help="Push the build cache to the remote registry. You should only enable this option if you have push rights to the remote registry.",
 )
+@click.option(
+    "--output",
+    "docker_output",
+    # Export image to docker. This is necessary to make the image available to docker-compose.
+    # The `--load` option is a shorthand for `--output=type=docker`.
+    default="type=docker",
+    help="Same as `docker build --output=...`. This option will only be used when BuildKit is enabled.",
+)
 @click.option(
     "-a",
     "--build-arg",
@@ -114,6 +122,7 @@ def build(
     no_cache: bool,
     no_registry_cache: bool,
     cache_to_registry: bool,
+    docker_output: str,
     build_args: list[str],
     add_hosts: list[str],
     target: str,
@@ -135,9 +144,8 @@ def build(
         command_args += ["--add-host", add_host]
     if target:
         command_args += ["--target", target]
-    if utils.is_buildkit_enabled():
-        # Export image to docker.
-        command_args.append("--output=type=image")
+    if utils.is_buildkit_enabled() and docker_output:
+        command_args.append(f"--output={docker_output}")
     if docker_args:
         command_args += docker_args
     for image in image_names:

diff --git a/tutor/env.py b/tutor/env.py
@@ -54,6 +54,7 @@ def _prepare_environment() -> None:
             ("HOST_USER_ID", utils.get_user_id()),
             ("TUTOR_APP", __app__.replace("-", "_")),
             ("TUTOR_VERSION", __version__),
+            ("is_buildkit_enabled", utils.is_buildkit_enabled),
         ],
     )
 

diff --git a/tutor/images.py b/tutor/images.py
@@ -9,9 +9,10 @@ def get_tag(config: Config, name: str) -> str:
 
 def build(path: str, tag: str, *args: str) -> None:
     fmt.echo_info(f"Building image {tag}")
-    command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(
-        ["build", "-t", tag, *args, path]
-    )
+    build_command = ["build", "-t", tag, *args, path]
+    if utils.is_buildkit_enabled():
+        build_command.insert(0, "buildx")
+    command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command)
     utils.docker(*command)
 
 

diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile
@@ -1,26 +1,36 @@
+{% if is_buildkit_enabled() %}# syntax=docker/dockerfile:1.4{% endif %}
 ###### Minimal image with base system requirements for most stages
 FROM docker.io/ubuntu:20.04 as minimal
 LABEL maintainer="Overhang.io <contact@overhang.io>"
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update && \
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \
+    apt update && \
     apt install -y build-essential curl git language-pack-en
 ENV LC_ALL en_US.UTF-8
 {{ patch("openedx-dockerfile-minimal") }}
 
 ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv
 FROM minimal as python
 # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites
-RUN apt update && \
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \
     apt install -y libssl-dev zlib1g-dev libbz2-dev \
     libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
     xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
-# https://github.com/pyenv/pyenv/releases
+
+# Install pyenv
 # https://www.python.org/downloads/
+# https://github.com/pyenv/pyenv/releases
 ARG PYTHON_VERSION=3.8.15
 ENV PYENV_ROOT /opt/pyenv
 RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.17 --depth 1
+
+# Install Python
 RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION
+
+# Create virtualenv
 RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv
 
 ###### Checkout edx-platform code
@@ -45,6 +55,12 @@ RUN git config --global user.email "tutor@overhang.io" \
 {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/<GITSHA1>.patch | git am #}
 {{ patch("openedx-dockerfile-post-git-checkout") }}
 
+##### Empty layer with just the repo at the root.
+# This is useful when overriding the build context with a host repo:
+# docker build --build-context edx-platform=/path/to/edx-platform
+FROM scratch as edx-platform
+COPY --from=code /openedx/edx-platform /
+
 ###### Download extra locales to /openedx/locale/contrib/locale
 FROM minimal as locales
 ARG OPENEDX_I18N_VERSION={{ OPENEDX_COMMON_VERSION }}
@@ -59,36 +75,39 @@ RUN cd /tmp \
 FROM python as python-requirements
 ENV PATH /openedx/venv/bin:${PATH}
 ENV VIRTUAL_ENV /openedx/venv/
+ENV XDG_CACHE_HOME /openedx/.cache
 
-RUN apt update && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \
+    && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev
 
 # Install the right version of pip/setuptools
-# https://pypi.org/project/setuptools/
-# https://pypi.org/project/pip/
-# https://pypi.org/project/wheel/
-RUN pip install setuptools==67.6.1 pip==23.0.1. wheel==0.40.0
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+    # https://pypi.org/project/setuptools/
+    # https://pypi.org/project/pip/
+    # https://pypi.org/project/wheel/
+    setuptools==67.6.1 pip==23.0.1. wheel==0.40.0
 
 # Install base requirements
-COPY --from=code /openedx/edx-platform/requirements/edx/base.txt /tmp/base.txt
-RUN pip install -r /tmp/base.txt
+RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \
+    --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt
 
-# Install django-redis for using redis as a django cache
-# https://pypi.org/project/django-redis/
-RUN pip install django-redis==5.2.0
-
-# Install uwsgi
-# https://pypi.org/project/uWSGI/
-RUN pip install uwsgi==2.0.21
+# Install extra requirements
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+    # Use redis as a django cache https://pypi.org/project/django-redis/
+    django-redis==5.2.0 \
+    # uwsgi server https://pypi.org/project/uWSGI/
+    uwsgi==2.0.21
 
 {{ patch("openedx-dockerfile-post-python-requirements") }}
 
 # Install private requirements: this is useful for installing custom xblocks.
 COPY ./requirements/ /openedx/requirements
-RUN cd /openedx/requirements/ \
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}cd /openedx/requirements/ \
   && touch ./private.txt \
   && pip install -r ./private.txt
 
-{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN pip install '{{ extra_requirements }}'
+{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}'
 {% endfor %}
 
 ###### Install nodejs with nodeenv in /openedx/nodeenv
@@ -103,18 +122,18 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt
 
 # Install nodejs requirements
 ARG NPM_REGISTRY={{ NPM_REGISTRY }}
-COPY --from=code /openedx/edx-platform/package.json /openedx/edx-platform/package.json
-COPY --from=code /openedx/edx-platform/package-lock.json /openedx/edx-platform/package-lock.json
 WORKDIR /openedx/edx-platform
-RUN npm clean-install --verbose --registry=$NPM_REGISTRY
+RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \
+    --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \
+    --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --verbose --no-audit --registry=$NPM_REGISTRY
 
 ###### Production image with system and python requirements
 FROM minimal as production
 
 # Install system requirements
-RUN apt update && \
-    apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind && \
-    rm -rf /var/lib/apt/lists/*
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \
+    && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind
 
 # From then on, run as unprivileged "app" user
 # Note that this must always be different from root (APP_USER_ID=0)
@@ -124,14 +143,17 @@ RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER
 USER ${APP_USER_ID}
 
 # https://hub.docker.com/r/powerman/dockerize/tags
-COPY --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize
-COPY --chown=app:app --from=code /openedx/edx-platform /openedx/edx-platform
+COPY {% if is_buildkit_enabled() %}--link {% endif %}--from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize
+COPY --chown=app:app --from=edx-platform / /openedx/edx-platform
 COPY --chown=app:app --from=locales /openedx/locale /openedx/locale
 COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv
 COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv
 COPY --chown=app:app --from=python-requirements /openedx/requirements /openedx/requirements
 COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv
-COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/edx-platform/node_modules
+COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/node_modules
+
+# Symlink node_modules such that we can bind-mount the edx-platform repository
+RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules
 
 ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH}
 ENV VIRTUAL_ENV /openedx/venv/
@@ -215,16 +237,16 @@ FROM production as development
 
 # Install useful system requirements (as root)
 USER root
-RUN apt update && \
-    apt install -y vim iputils-ping dnsutils telnet \
-    && rm -rf /var/lib/apt/lists/*
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \
+    apt install -y vim iputils-ping dnsutils telnet
 USER app
 
 # Install dev python requirements
-RUN pip install -r requirements/edx/development.txt
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements/edx/development.txt
 # https://pypi.org/project/ipdb/
 # https://pypi.org/project/ipython
-RUN pip install ipdb==0.13.13 ipython==8.12.0
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install ipdb==0.13.13 ipython==8.12.0
 
 # Add ipdb as default PYTHONBREAKPOINT
 ENV PYTHONBREAKPOINT=ipdb.set_trace

diff --git a/tutor/utils.py b/tutor/utils.py
@@ -173,6 +173,25 @@ def docker(*command: str) -> int:
     return execute("docker", *command)
 
 
+@lru_cache(maxsize=None)
+def is_buildkit_enabled() -> bool:
+    """
+    A helper function to determine whether we can run `docker buildx` with BuildKit.
+    """
+    # First, we respect the DOCKER_BUILDKIT environment variable
+    enabled_by_env = {
+        "1": True,
+        "0": False,
+    }.get(os.environ.get("DOCKER_BUILDKIT", ""))
+    if enabled_by_env is not None:
+        return enabled_by_env
+    try:
+        subprocess.run(["docker", "buildx", "version"], capture_output=True, check=True)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
 def docker_compose(*command: str) -> int:
     return execute("docker", "compose", *command)