diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
index 2b9276ae4..8a3d3a4cb 100644
--- a/.github/workflows/build-docker.yml
+++ b/.github/workflows/build-docker.yml
@@ -128,6 +128,13 @@ jobs:
                 format('{0}:{1}-{2}', env.IMAGE_NAME, matrix.tag, needs.config.outputs.version_or_latest)
               ) || format('{0}:{1}', env.IMAGE_NAME, needs.config.outputs.version_or_latest)
             ) }}
+          IMAGE_CACHE_FROM:
+            |- # Always use the `latest` buildcache. :latest-buildcache or :{tag}-latest-buildcache
+            ${{ (
+              matrix.tag != '' && (
+                format('type=registry,ref={0}:{1}-latest-buildcache', env.IMAGE_NAME, matrix.tag)
+              ) || format('type=registry,ref={0}:latest-buildcache', env.IMAGE_NAME)
+            ) }}
         with:
           context: .
           builder: ${{ steps.buildx.outputs.name }}
@@ -144,7 +151,7 @@ jobs:
           target: ${{ matrix.target }}
           push: true
           tags: ${{ env.IMAGE_TAG }}
-          cache-from: type=registry,ref=${{ env.IMAGE_TAG }}-buildcache
+          cache-from: ${{ env.IMAGE_CACHE_FROM }}
           cache-to: type=registry,ref=${{ env.IMAGE_TAG }}-buildcache,mode=max
           platforms: ${{ matrix.platforms }}