multi-arch builds (#140)

.prettierignore

@@ -8,11 +8,7 @@ packages/python/tests/cli/commands/fixtures/bad_json.json
 packages/python/tests/cli/commands/fixtures/error.yaml
 
 # generated files - do not edit them by hand.
-packages/python/tests/blueprints/multi-arch/fixtures/gh-m.yaml
-packages/python/tests/blueprints/single-arch/fixtures/gh-m.yaml
-packages/python/tests/blueprints/multi-arch/fixtures/ci/_image-names.json
-packages/python/tests/blueprints/multi-arch/fixtures/ci/_image-tags.json
-packages/python/tests/blueprints/multi-arch-buildx/fixtures/ci/_image-names.json
-packages/python/tests/blueprints/multi-arch-buildx/fixtures/ci/_image-tags.json
+packages/python/tests/blueprints/*/fixtures/gh-m.yaml
+packages/python/tests/blueprints/*/fixtures/ci/_image-names.json
+packages/python/tests/blueprints/*/fixtures/ci/_image-tags.json
 packages/python/tests/github/actions/_fixtures/*.yaml
-packages/python/tests/blueprints/multi-arch-buildx/fixtures/gh-m.yaml

CHANGELOG.md

@@ -9,6 +9,11 @@ The format of this changelog is based on
 
 ## [Unreleased]
 
+- We may provide `platforms` in the `m` configuration. This is helpful when we
+  do not have access to different architectures. In this case the architecture
+  flag is still needed but they can be build in the same runner. When platform
+  is provided we can specify `linux/arm64` in there.
+
 ## [0.33.1] <a name="0.33.1" href="#0.33.1">-</a> March 27, 2024
 
 - `use_buildx` is set to `true` by default. This flag will go away in the future

allowed_errors.json

@@ -6,6 +6,7 @@
     "WPS361": 3,
     "WPS231": 2,
     "WPS421": 2,
+    "WPS214": 1,
     "WPS226": 4
   }
 }

packages/python/m/ci/docker/config.py

@@ -33,9 +33,15 @@ class DockerConfig(BaseModel):
 
     # A map of the architectures to build. It maps say `amd64` to a Github
     # runner that will build the image for that architecture.
-    #   amd64: Ubuntu 20.04
+    #    amd64: Ubuntu 20.04
     architectures: dict[str, str | list[str]] | None
 
+    # A map of the platforms to build. It maps say `amd64` to a valid buildx
+    # supported platform. Using this allows us to build multi-arch images using
+    # buildx in an environment that may not have the necessary architecture.
+    # For instance: 'amd64: linux/amd64'
+    platforms: dict[str, str] | None = None
+
     # Freeform object to allow us to specify a container in which to run
     # the docker commands.
     # https://docs.github.com/en/actions/using-jobs/running-jobs-in-a-container
@@ -145,6 +151,7 @@ def update_github_workflow(
             global_env=global_env,
             default_runner=self.default_runner,
             architectures=self.architectures or {},
+            platforms=self.platforms,
             images=self.images,
             extra_build_steps=self.extra_build_steps,
             docker_registry=self.docker_registry,

packages/python/m/ci/docker/docker_build.py

@@ -116,3 +116,16 @@ def __str__(self) -> str:
             options=self.model_dump(exclude_none=True),
         )
         return str(cmd)
+
+    def buildx_str(self) -> str:
+        """Convert the docker build command to a string.
+
+        Returns:
+            The docker build command.
+        """
+        cmd = ShellCommand(
+            prog='docker buildx build --platform "$PLATFORM"',
+            positional=['.'],
+            options=self.model_dump(exclude_none=True),
+        )
+        return str(cmd)

packages/python/m/ci/docker/env.py

@@ -21,3 +21,6 @@ class MEnvDocker(BaseModel):
 
     # The list of all the architectures if multi-arch is enabled.
     architectures: list[str]
+
+    # Flag to indicate if buildx should be used.
+    use_buildx: bool = False

packages/python/m/ci/docker/gh_workflow_multi.py

@@ -1,3 +1,7 @@
+from textwrap import dedent
+
+from m.core import yaml
+
 from .gh_workflow_single import TemplateVars as DefaultTemplateVars
 from .gh_workflow_single import Workflow as DefaultWorkflow
 
@@ -146,7 +150,7 @@
           name: m-blueprints
           path: {ci_dir}
       - name: chmod
-        run: chmod +x {ci_dir}/*.sh
+        run: chmod +x {ci_dir}/*.sh{buildx_setup}
       - {docker_login}
       {build_steps}
 
@@ -181,6 +185,8 @@ class TemplateVars(DefaultTemplateVars):
 
     manifest_strategy_options: str
 
+    buildx_setup: str
+
 
 def _indent(text: str, num: int) -> str:
     spaces = '  ' * num
@@ -196,18 +202,37 @@ class Workflow(DefaultWorkflow):
 
     use_buildx: bool = False
 
+    platforms: dict[str, str] | None
+
     def build_architectures(self: 'Workflow') -> str:
         """Generate a github action str with the build architectures.
 
         Returns:
             A string to add to the Github workflow.
         """
         arch_strs = '\n'.join([
-            f'- arch: {arch}\n  os: {os}'
+            f'- arch: {arch}\n  os: {os}{platform}'
             for arch, os in self.architectures.items()
+            for platform in (self._add_platform(arch),)
         ])
         return _indent(f'\n{arch_strs}', 5)
 
+    def buildx_setup_str(self: 'Workflow') -> str:
+        """Generate a github action str to setup buildx.
+
+        Returns:
+            A string to add to the Github workflow.
+        """
+        if not self.platforms:
+            return ''
+        platforms = ','.join(self.platforms.values())
+        setup_obj = f"""
+            - name: buildx-setup
+              uses: docker/setup-buildx-action@v3
+              with:
+                platforms: {platforms}"""
+        return _indent(dedent(setup_obj), 3)
+
     def create_manifest_str(self: 'Workflow') -> str:
         """Generate the script to create the manifest.
 
@@ -250,6 +275,26 @@ def manifest_strategy_options_str(self: 'Workflow') -> str:
             options = f'\n      max-parallel: {self.max_parallel_manifests}'
         return options
 
+    def container_str(self: 'Workflow') -> str:
+        """Generate a string specifying a container to run on.
+
+        Returns:
+            A string to add to the Github workflow.
+        """
+        if not self.container:
+            lines = [
+                '',
+                '    env:',
+                '      ARCH: ${{ matrix.arch }}',
+            ]
+            if self.platforms:
+                lines.append('      PLATFORM: ${{ matrix.platform }}')
+            return '\n'.join(lines)
+        lines = ['\n    container:']
+        content_str = _indent(yaml.dumps(self.container), 3)
+        lines.append(f'      {content_str}')
+        return '\n'.join(lines).rstrip()
+
     def __str__(self: 'Workflow') -> str:
         """Stringify the workflow file.
 
@@ -264,6 +309,7 @@ def __str__(self: 'Workflow') -> str:
             ci_dir=self.ci_dir,
             build_architectures=self.build_architectures(),
             docker_login=self.docker_login_str(),
+            buildx_setup=self.buildx_setup_str(),
             build_steps=self.build_steps_str(),
             create_manifest=self.create_manifest_str(),
             push_manifest=self.push_manifest_str(),
@@ -272,3 +318,9 @@ def __str__(self: 'Workflow') -> str:
         )
         template = TEMPLATE_BUILDX if self.use_buildx else TEMPLATE
         return template.format(**template_vars.model_dump())
+
+    def _add_platform(self: 'Workflow', arch: str) -> str:
+        if not self.platforms:
+            return ''
+        platform = self.platforms[arch]
+        return f'\n  platform: {platform}'

packages/python/m/ci/docker/image.py

@@ -132,12 +132,17 @@ def ci_build(self: 'DockerImage', m_env: MEnvDocker) -> Res[str]:
             target=self.target_stage,
             file=docker_file,
         )
+        build_cmd_str = (
+            build_cmd.buildx_str()
+            if m_env.use_buildx
+            else str(build_cmd)
+        )
         script = [
             BASH_SHEBANG,
             'export DOCKER_BUILDKIT=1',
             SET_STRICT_BASH,
             '',
-            str(build_cmd),
+            build_cmd_str,
             '',
         ]
         return Good('\n'.join(script))

packages/python/m/ci/m_blueprints.py

@@ -30,6 +30,7 @@ def _write_blueprints(
         registry=docker_config.docker_registry,
         multi_arch=bool(docker_config.architectures),
         architectures=list((docker_config.architectures or {}).keys()),
+        use_buildx=docker_config.platforms is not None,
     )
     files = FileNames.create_instance(m_dir)
     if update_makefile:

packages/python/tests/blueprints/multi-arch-buildx-platform/Makefile

@@ -0,0 +1,2 @@
+fake-target:
+	echo 'here to make sure this file is modified'

packages/python/tests/blueprints/multi-arch-buildx-platform/fixtures/Makefile

@@ -0,0 +1,26 @@
+fake-target:
+	echo 'here to make sure this file is modified'
+
+# START: M-DOCKER-IMAGES
+define m_env
+	$(eval include packages/python/tests/blueprints/multi-arch-buildx-platform/m/.m/m_env.sh)
+	$(eval $(cut -d= -f1 packages/python/tests/blueprints/multi-arch-buildx-platform/m/.m/m_env.sh))
+endef
+
+m-env:
+	mkdir -p packages/python/tests/blueprints/multi-arch-buildx-platform/m/.m && m ci env --bashrc > packages/python/tests/blueprints/multi-arch-buildx-platform/m/.m/m_env.sh
+
+m-blueprints: m-env
+	$(call m_env)
+	m blueprints --skip-makefile --skip-workflow packages/python/tests/blueprints/multi-arch-buildx-platform/m
+	chmod +x packages/python/tests/blueprints/multi-arch-buildx-platform/m/.m/blueprints/local/*.sh
+
+dev-m-image1: m-blueprints
+	$(call m_env)
+	packages/python/tests/blueprints/multi-arch-buildx-platform/m/.m/blueprints/local/m-image1.build.sh
+
+dev-m-image2: dev-m-image1
+	$(call m_env)
+	packages/python/tests/blueprints/multi-arch-buildx-platform/m/.m/blueprints/local/m-image2.build.sh
+
+# END: M-DOCKER-IMAGES

packages/python/tests/blueprints/multi-arch-buildx-platform/fixtures/ci/_find-cache.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+imageName=$1
+pullCache() {
+  if docker pull -q "$1:$2" 2> /dev/null; then
+    docker tag "$1:$2" "staged-image:cache"
+  else
+    return 1
+  fi
+}
+findCache() {
+  pullCache "$1" master || echo "NO CACHE FOUND"
+}
+set -euxo pipefail
+findCache "ghcr.io/repo-owner/$imageName"

packages/python/tests/blueprints/multi-arch-buildx-platform/fixtures/ci/_image-names.json

@@ -0,0 +1 @@
+["m-image1","m-image2"]

packages/python/tests/blueprints/multi-arch-buildx-platform/fixtures/ci/_image-tags.json

@@ -0,0 +1 @@
+["1.1.1","latest","v1","v1.1"]

packages/python/tests/blueprints/multi-arch-buildx-platform/fixtures/ci/_push-image.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+imageName=$1
+set -euxo pipefail
+docker tag staged-image:latest "ghcr.io/repo-owner/$ARCH-$imageName:$M_TAG"
+docker push "ghcr.io/repo-owner/$ARCH-$imageName:$M_TAG"

packages/python/tests/blueprints/multi-arch-buildx-platform/fixtures/ci/m-image1.build.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+export DOCKER_BUILDKIT=1
+set -euxo pipefail
+
+docker buildx build --platform "$PLATFORM" \
+  --build-arg ARCH="$ARCH" \
+  --build-arg BUILDKIT_INLINE_CACHE=1 \
+  --build-arg M_TAG= \
+  --cache-from staged-image:cache \
+  --file packages/python/tests/blueprints/multi-arch/m/docker/Dockerfile.image1 \
+  --progress plain \
+  --secret id=GITHUB_TOKEN \
+  --tag ghcr.io/repo-owner/m-image1: \
+  --tag staged-image:latest \
+  --target first \
+  .

packages/python/tests/blueprints/multi-arch-buildx-platform/fixtures/ci/m-image1.manifest.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -euxo pipefail
+
+docker buildx imagetools create \
+  -t ghcr.io/repo-owner/m-image1:1.1.1 \
+  -t ghcr.io/repo-owner/m-image1:latest \
+  -t ghcr.io/repo-owner/m-image1:v1 \
+  -t ghcr.io/repo-owner/m-image1:v1.1 \
+  ghcr.io/repo-owner/amd64-m-image1:1.1.1 \
+  ghcr.io/repo-owner/arm64-m-image1:1.1.1

packages/python/tests/blueprints/multi-arch-buildx-platform/fixtures/ci/m-image2.build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+export DOCKER_BUILDKIT=1
+set -euxo pipefail
+
+docker buildx build --platform "$PLATFORM" \
+  --build-arg ARCH="$ARCH" \
+  --build-arg BUILDKIT_INLINE_CACHE=1 \
+  --build-arg M_TAG= \
+  --cache-from staged-image:cache \
+  --file packages/python/tests/blueprints/multi-arch/m/docker/Dockerfile.image1 \
+  --progress plain \
+  --tag ghcr.io/repo-owner/m-image2: \
+  --tag staged-image:latest \
+  --target second \
+  .

packages/python/tests/blueprints/multi-arch-buildx-platform/fixtures/ci/m-image2.manifest.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -euxo pipefail
+
+docker buildx imagetools create \
+  -t ghcr.io/repo-owner/m-image2:1.1.1 \
+  -t ghcr.io/repo-owner/m-image2:latest \
+  -t ghcr.io/repo-owner/m-image2:v1 \
+  -t ghcr.io/repo-owner/m-image2:v1.1 \
+  ghcr.io/repo-owner/amd64-m-image2:1.1.1 \
+  ghcr.io/repo-owner/arm64-m-image2:1.1.1