diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cd3109ff6936c..efdcde4162e0d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,13 +23,14 @@ builder-image:
     DOCKER_HOST: "tcp://docker:2375"
     DOCKER_DRIVER: overlay2
     DOCKER_TLS_CERTDIR: ""
+    DOCKER_BUILDKIT: 1
   before_script:
     - echo $CI_JOB_TOKEN | docker login -u gitlab-ci-token --password-stdin $CI_REGISTRY
   script:
-    - cd contrib/containers/ci
+    - cd contrib/containers/ci/32bit
     - docker pull $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG || true
     - docker pull $CI_REGISTRY_IMAGE:builder-develop || true
-    - docker build --cache-from $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG --cache-from $CI_REGISTRY_IMAGE:builder-develop -t $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG -f Dockerfile .
+    - docker build --cache-from $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG --cache-from $CI_REGISTRY_IMAGE:builder-develop -t $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG -f Dockerfile ..
     - docker push $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
 
 .build-depends-template:
diff --git a/contrib/containers/ci/32bit/Dockerfile b/contrib/containers/ci/32bit/Dockerfile
new file mode 100644
index 0000000000000..0646f17013156
--- /dev/null
+++ b/contrib/containers/ci/32bit/Dockerfile
@@ -0,0 +1,16 @@
+# syntax = edrevo/dockerfile-plus
+
+FROM ubuntu:focal
+
+RUN dpkg --add-architecture i386
+
+INCLUDE+ common/Dockerfile
+
+RUN dpkg --add-architecture i386
+
+RUN apt-get update && apt-get install $APT_ARGS \
+    g++-9-multilib \
+    wine32  \
+    && rm -rf /var/lib/apt/lists/*
+
+INCLUDE+ common/Dockerfile.finish
diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/common/Dockerfile
similarity index 90%
rename from contrib/containers/ci/Dockerfile
rename to contrib/containers/ci/common/Dockerfile
index 2b0b73dbcfa9a..8b71885bbe8d8 100644
--- a/contrib/containers/ci/Dockerfile
+++ b/contrib/containers/ci/common/Dockerfile
@@ -6,7 +6,6 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/London"
 # Build and base stuff
 # (zlib1g-dev and libssl-dev are needed for the Qt host binary builds, but should not be used by target binaries)
 ENV APT_ARGS="-y --no-install-recommends --no-upgrade"
-RUN dpkg --add-architecture i386
 RUN apt-get update && apt-get install $APT_ARGS \
     autotools-dev \
     automake \
@@ -56,7 +55,6 @@ RUN useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /dash dash
 RUN apt-get update && apt-get install $APT_ARGS \
     bc \
     gawk \
-    g++-9-multilib \
     g++-arm-linux-gnueabihf \
     g++-mingw-w64-x86-64 \
     imagemagick \
@@ -71,7 +69,6 @@ RUN apt-get update && apt-get install $APT_ARGS \
     python3-zmq \
     parallel \
     wine-stable \
-    wine32 \
     wine64 \
     xorriso \
     && rm -rf /var/lib/apt/lists/*
@@ -98,14 +95,3 @@ RUN \
   update-alternatives --set x86_64-w64-mingw32-g++  /usr/bin/x86_64-w64-mingw32-g++-posix; \
   exit 0
 
-RUN mkdir /dash-src && \
-  mkdir -p /cache/ccache && \
-  mkdir /cache/depends && \
-  mkdir /cache/sdk-sources && \
-  chown $USER_ID:$GROUP_ID /dash-src && \
-  chown $USER_ID:$GROUP_ID /cache && \
-  chown $USER_ID:$GROUP_ID /cache -R
-
-WORKDIR /dash-src
-
-USER dash
diff --git a/contrib/containers/ci/common/Dockerfile.finish b/contrib/containers/ci/common/Dockerfile.finish
new file mode 100644
index 0000000000000..fa1a0df061e6c
--- /dev/null
+++ b/contrib/containers/ci/common/Dockerfile.finish
@@ -0,0 +1,11 @@
+RUN mkdir /dash-src && \
+  mkdir -p /cache/ccache && \
+  mkdir /cache/depends && \
+  mkdir /cache/sdk-sources && \
+  chown $USER_ID:$GROUP_ID /dash-src && \
+  chown $USER_ID:$GROUP_ID /cache && \
+  chown $USER_ID:$GROUP_ID /cache -R
+
+WORKDIR /dash-src
+
+USER dash
diff --git a/contrib/containers/develop/Dockerfile b/contrib/containers/develop/Dockerfile
index 7ad971a30dac5..e8ea6bb80e8f8 100644
--- a/contrib/containers/develop/Dockerfile
+++ b/contrib/containers/develop/Dockerfile
@@ -2,7 +2,8 @@
 
 FROM ubuntu:focal
 
-INCLUDE+ ci/Dockerfile
+INCLUDE+ ci/common/Dockerfile
+INCLUDE+ ci/common/Dockerfile.finish
 
 # The inherited Dockerfile switches to non-privileged context and we've
 # just started configuring this image, give us root access