feat: leverage RUN --mount for faster image building

We make use of the Docker build cache to install python and nodejs
requirements faster in the case of repeated builds.

This feature is only possible for users of BuildKit, so we detect
whether `docker buildx` is available at runtime.

We do not make use of `COPY --link` because the `--link` option is
incompatible with `--chown=app:app`:
docker/buildx#1408

For reference, see:

https://www.docker.com/blog/dockerfiles-now-support-multiple-build-contexts/
https://docs.docker.com/engine/reference/commandline/buildx_build/#build-context

changelog.d/20230427_154822_regis_build_mount.md

@@ -0,0 +1 @@
+- [Improvement] Considerably accelerate building the "openedx" Docker image with `RUN --mount=type=cache`. This feature is only for Docker with BuildKit, so detection is performed at build-time. (by @regisb)

tests/commands/test_images.py

@@ -1,7 +1,7 @@
 from unittest.mock import Mock, patch
 
 from tests.helpers import PluginsTestCase, temporary_root
-from tutor import images, plugins
+from tutor import images, plugins, utils
 from tutor.__about__ import __version__
 from tutor.commands.images import ImageNotFoundError
 
@@ -128,16 +128,29 @@ def test_images_build_plugin_with_args(self, image_build: Mock) -> None:
             "service1",
         ]
         with temporary_root() as root:
-            self.invoke_in_root(root, ["config", "save"])
-            result = self.invoke_in_root(root, build_args)
+            utils.is_buildkit_enabled.cache_clear()
+            with patch.object(utils, "is_buildkit_enabled", return_value=False):
+                self.invoke_in_root(root, ["config", "save"])
+                result = self.invoke_in_root(root, build_args)
         self.assertIsNone(result.exception)
         self.assertEqual(0, result.exit_code)
         image_build.assert_called()
         self.assertIn("service1:1.0.0", image_build.call_args[0])
-        for arg in image_build.call_args[0][2:]:
-            # The only extra args are `--build-arg`
-            if arg != "--build-arg":
-                self.assertIn(arg, build_args)
+        self.assertEqual(
+            [
+                "service1:1.0.0",
+                "--no-cache",
+                "--build-arg",
+                "myarg=value",
+                "--add-host",
+                "host",
+                "--target",
+                "target",
+                "docker_args",
+                "--cache-from=type=registry,ref=service1:1.0.0-cache",
+            ],
+            list(image_build.call_args[0][1:])
+        )
 
     def test_images_push(self) -> None:
         result = self.invoke(["images", "push"])

tutor/commands/images.py

@@ -136,8 +136,9 @@ def build(
     if target:
         command_args += ["--target", target]
     if utils.is_buildkit_enabled():
-        # Export image to docker.
-        command_args.append("--output=type=image")
+        # Export image to docker. This is necessary to make the image available to docker-compose.
+        # The `--load` option is a shorthand for `--output=type=docker`.
+        command_args.append("--load")
     if docker_args:
         command_args += docker_args
     for image in image_names:

tutor/env.py

@@ -54,6 +54,7 @@ def _prepare_environment() -> None:
             ("HOST_USER_ID", utils.get_user_id()),
             ("TUTOR_APP", __app__.replace("-", "_")),
             ("TUTOR_VERSION", __version__),
+            ("is_buildkit_enabled", utils.is_buildkit_enabled),
         ],
     )
 

tutor/images.py

@@ -9,9 +9,10 @@ def get_tag(config: Config, name: str) -> str:
 
 def build(path: str, tag: str, *args: str) -> None:
     fmt.echo_info(f"Building image {tag}")
-    command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(
-        ["build", "-t", tag, *args, path]
-    )
+    build_command = ["build", "-t", tag, *args, path]
+    if utils.is_buildkit_enabled():
+        build_command.insert(0, "buildx")
+    command = hooks.Filters.DOCKER_BUILD_COMMAND.apply(build_command)
     utils.docker(*command)
 
 

tutor/templates/build/openedx/Dockerfile

@@ -3,24 +3,33 @@ FROM docker.io/ubuntu:20.04 as minimal
 LABEL maintainer="Overhang.io <contact@overhang.io>"
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update && \
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked{% endif %} \
+    apt update && \
     apt install -y build-essential curl git language-pack-en
 ENV LC_ALL en_US.UTF-8
 {{ patch("openedx-dockerfile-minimal") }}
 
 ###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv
 FROM minimal as python
 # https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites
-RUN apt update && \
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \
     apt install -y libssl-dev zlib1g-dev libbz2-dev \
     libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
     xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
-# https://github.com/pyenv/pyenv/releases
+
+# Install pyenv
 # https://www.python.org/downloads/
+# https://github.com/pyenv/pyenv/releases
 ARG PYTHON_VERSION=3.8.15
 ENV PYENV_ROOT /opt/pyenv
 RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.17 --depth 1
+
+# Install Python
 RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION
+
+# Create virtualenv
 RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv
 
 ###### Checkout edx-platform code
@@ -45,6 +54,12 @@ RUN git config --global user.email "tutor@overhang.io" \
 {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/<GITSHA1>.patch | git am #}
 {{ patch("openedx-dockerfile-post-git-checkout") }}
 
+##### Empty layer with just the repo at the root.
+# This is useful when overriding the build context with a host repo:
+# docker build --build-context edx-platform=/path/to/edx-platform
+FROM scratch as edx-platform
+COPY --from=code /openedx/edx-platform /
+
 ###### Download extra locales to /openedx/locale/contrib/locale
 FROM minimal as locales
 ARG OPENEDX_I18N_VERSION={{ OPENEDX_COMMON_VERSION }}
@@ -59,36 +74,39 @@ RUN cd /tmp \
 FROM python as python-requirements
 ENV PATH /openedx/venv/bin:${PATH}
 ENV VIRTUAL_ENV /openedx/venv/
+ENV XDG_CACHE_HOME /openedx/.cache
 
-RUN apt update && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \
+    && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev
 
 # Install the right version of pip/setuptools
-# https://pypi.org/project/setuptools/
-# https://pypi.org/project/pip/
-# https://pypi.org/project/wheel/
-RUN pip install setuptools==67.6.1 pip==23.0.1. wheel==0.40.0
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+    # https://pypi.org/project/setuptools/
+    # https://pypi.org/project/pip/
+    # https://pypi.org/project/wheel/
+    setuptools==67.6.1 pip==23.0.1. wheel==0.40.0
 
 # Install base requirements
-COPY --from=code /openedx/edx-platform/requirements/edx/base.txt /tmp/base.txt
-RUN pip install -r /tmp/base.txt
+RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \
+    --mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r /openedx/edx-platform/requirements/edx/base.txt
 
-# Install django-redis for using redis as a django cache
-# https://pypi.org/project/django-redis/
-RUN pip install django-redis==5.2.0
-
-# Install uwsgi
-# https://pypi.org/project/uWSGI/
-RUN pip install uwsgi==2.0.21
+# Install extra requirements
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install \
+    # Use redis as a django cache https://pypi.org/project/django-redis/
+    django-redis==5.2.0 \
+    # uwsgi server https://pypi.org/project/uWSGI/
+    uwsgi==2.0.21
 
 {{ patch("openedx-dockerfile-post-python-requirements") }}
 
 # Install private requirements: this is useful for installing custom xblocks.
 COPY ./requirements/ /openedx/requirements
-RUN cd /openedx/requirements/ \
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}cd /openedx/requirements/ \
   && touch ./private.txt \
   && pip install -r ./private.txt
 
-{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN pip install '{{ extra_requirements }}'
+{% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %}RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install '{{ extra_requirements }}'
 {% endfor %}
 
 ###### Install nodejs with nodeenv in /openedx/nodeenv
@@ -103,18 +121,18 @@ RUN nodeenv /openedx/nodeenv --node=16.14.0 --prebuilt
 
 # Install nodejs requirements
 ARG NPM_REGISTRY={{ NPM_REGISTRY }}
-COPY --from=code /openedx/edx-platform/package.json /openedx/edx-platform/package.json
-COPY --from=code /openedx/edx-platform/package-lock.json /openedx/edx-platform/package-lock.json
 WORKDIR /openedx/edx-platform
-RUN npm clean-install --verbose --registry=$NPM_REGISTRY
+RUN {% if is_buildkit_enabled() %}--mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \
+    --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \
+    --mount=type=cache,target=/root/.npm,sharing=shared {% endif %}npm clean-install --verbose --no-audit --registry=$NPM_REGISTRY
 
 ###### Production image with system and python requirements
 FROM minimal as production
 
 # Install system requirements
-RUN apt update && \
-    apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind && \
-    rm -rf /var/lib/apt/lists/*
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update \
+    && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind
 
 # From then on, run as unprivileged "app" user
 # Note that this must always be different from root (APP_USER_ID=0)
@@ -124,14 +142,17 @@ RUN useradd --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER
 USER ${APP_USER_ID}
 
 # https://hub.docker.com/r/powerman/dockerize/tags
-COPY --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize
-COPY --chown=app:app --from=code /openedx/edx-platform /openedx/edx-platform
+COPY {% if is_buildkit_enabled() %}--link {% endif %}--from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize
+COPY --chown=app:app --from=edx-platform / /openedx/edx-platform
 COPY --chown=app:app --from=locales /openedx/locale /openedx/locale
 COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv
 COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv
 COPY --chown=app:app --from=python-requirements /openedx/requirements /openedx/requirements
 COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv
-COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/edx-platform/node_modules
+COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/node_modules
+
+# Symlink node_modules such that we can bind-mount the edx-platform repository
+RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules
 
 ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH}
 ENV VIRTUAL_ENV /openedx/venv/
@@ -215,16 +236,16 @@ FROM production as development
 
 # Install useful system requirements (as root)
 USER root
-RUN apt update && \
-    apt install -y vim iputils-ping dnsutils telnet \
-    && rm -rf /var/lib/apt/lists/*
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked {% endif %}apt update && \
+    apt install -y vim iputils-ping dnsutils telnet
 USER app
 
 # Install dev python requirements
-RUN pip install -r requirements/edx/development.txt
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install -r requirements/edx/development.txt
 # https://pypi.org/project/ipdb/
 # https://pypi.org/project/ipython
-RUN pip install ipdb==0.13.13 ipython==8.12.0
+RUN {% if is_buildkit_enabled() %}--mount=type=cache,target=/openedx/.cache/pip,sharing=shared {% endif %}pip install ipdb==0.13.13 ipython==8.12.0
 
 # Add ipdb as default PYTHONBREAKPOINT
 ENV PYTHONBREAKPOINT=ipdb.set_trace

tutor/utils.py

@@ -173,6 +173,25 @@ def docker(*command: str) -> int:
     return execute("docker", *command)
 
 
+@lru_cache(maxsize=None)
+def is_buildkit_enabled() -> bool:
+    """
+    A helper function to determine whether we can run `docker buildx` with BuildKit.
+    """
+    # First, we respect the DOCKER_BUILDKIT environment variable
+    enabled_by_env = {
+        "1": True,
+        "0": False,
+    }.get(os.environ.get("DOCKER_BUILDKIT", ""))
+    if enabled_by_env is not None:
+        return enabled_by_env
+    try:
+        subprocess.run(["docker", "buildx", "version"], capture_output=True, check=True)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
 def docker_compose(*command: str) -> int:
     return execute("docker", "compose", *command)