Add support for azlinux3

This changes some things in the mariner2 target so we can share as much
as makes sense for azlinux3.

Note: mariner was renamed to azure linux, hence mariner2 -> azlinux3.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

cmd/frontend/main.go

@@ -32,6 +32,7 @@ func main() {
  if err := grpcclient.RunFromEnvironment(ctx, mux.Handler(
  // copy/paster's beware: [frontend.WithTargetForwardingHandler] should not be set except for the root dalec frontend.
  frontend.WithBuiltinHandler(azlinux.Mariner2TargetKey, azlinux.NewMariner2Handler()),
+ frontend.WithBuiltinHandler(azlinux.AzLinux3TargetKey, azlinux.NewAzlinux3Handler()),
  frontend.WithBuiltinHandler(windows.DefaultTargetKey, windows.Handle),
  frontend.WithTargetForwardingHandler,
  )); err != nil {

docker-bake.hcl

@@ -72,7 +72,7 @@ target "runc" {
  "DALEC_DISABLE_DIFF_MERGE" = DALEC_DISABLE_DIFF_MERGE
  }
  matrix = {
- distro = ["mariner2"]
+ distro = ["mariner2", "azlinux3"]
  tgt = ["rpm", "container", "rpm/spec"]
  }
  target = "${distro}/${tgt}"
@@ -88,7 +88,7 @@ target "runc" {
 target "runc-test" {
  name = "runc-test-${distro}"
  matrix = {
- distro =["mariner2"]
+ distro =["mariner2", "azlinux3"]
  }
  contexts = {
  "dalec-runc-img" = "target:runc-${distro}-container"

frontend/azlinux/azlinux3.go

@@ -0,0 +1,56 @@
+package azlinux
+
+import (
+ "context"
+ "encoding/json"
+ "path/filepath"
+
+ "github.com/Azure/dalec"
+ "github.com/moby/buildkit/client/llb"
+ "github.com/moby/buildkit/client/llb/sourceresolver"
+ gwclient "github.com/moby/buildkit/frontend/gateway/client"
+)
+
+const (
+ AzLinux3TargetKey = "azlinux3"
+ tdnfCacheNameAzlinux3 = "azlinux3-tdnf-cache"
+
+ azlinux3Ref = "azurelinuxpreview.azurecr.io/public/azurelinux/base/core:3.0"
+ azlinux3DistrolessRef = "azurelinuxpreview.azurecr.io/public/azurelinux/distroless/base:3.0"
+)
+
+func NewAzlinux3Handler() gwclient.BuildFunc {
+ return newHandler(azlinux3{})
+}
+
+type azlinux3 struct{}
+
+func (w azlinux3) Base(client gwclient.Client, opts ...llb.ConstraintsOpt) llb.State {
+ return llb.Image(azlinux3Ref, llb.WithMetaResolver(client), dalec.WithConstraints(opts...)).Run(
+ w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential"}),
+ ).Root()
+}
+
+func (w azlinux3) Install(pkgs []string, opts ...installOpt) llb.RunOption {
+ var cfg installConfig
+ setInstallOptions(&cfg, opts)
+ return dalec.WithRunOptions(tdnfInstall(&cfg, "3.0", pkgs), w.tdnfCacheMount(cfg.root))
+}
+
+func (azlinux3) DefaultImageConfig(ctx context.Context, client gwclient.Client) (*dalec.DockerImageSpec, error) {
+ _, _, dt, err := client.ResolveImageConfig(ctx, azlinux3DistrolessRef, sourceresolver.Opt{})
+ if err != nil {
+ return nil, err
+ }
+
+ var cfg dalec.DockerImageSpec
+ if err := json.Unmarshal(dt, &cfg); err != nil {
+ return nil, err
+ }
+
+ return &cfg, nil
+}
+
+func (azlinux3) tdnfCacheMount(root string) llb.RunOption {
+ return llb.AddMount(filepath.Join(root, tdnfCacheDir), llb.Scratch(), llb.AsPersistentCacheDir(tdnfCacheNameAzlinux3, llb.CacheMountLocked))
+}

frontend/azlinux/handle_container.go

@@ -6,7 +6,6 @@ import (
  "encoding/json"
  "fmt"
  "path/filepath"
- "strings"
 
  "github.com/Azure/dalec"
  "github.com/Azure/dalec/frontend"
@@ -147,23 +146,6 @@ func specToContainerLLB(w worker, client gwclient.Client, spec *dalec.Spec, targ
 
  builderImg := w.Base(client, opts...)
 
- // TODO: This is mariner specific and should probably be moved out of this
- // package.
- mfstDir := filepath.Join(workPath, "var/lib/rpmmanifest")
- mfst1 := filepath.Join(mfstDir, "container-manifest-1")
- mfst2 := filepath.Join(mfstDir, "container-manifest-2")
- rpmdbDir := filepath.Join(workPath, "var/lib/rpm")
-
- chrootedPaths := []string{
- filepath.Join(workPath, "/usr/local/bin"),
- filepath.Join(workPath, "/usr/local/sbin"),
- filepath.Join(workPath, "/usr/bin"),
- filepath.Join(workPath, "/usr/sbin"),
- filepath.Join(workPath, "/bin"),
- filepath.Join(workPath, "/sbin"),
- }
- chrootedPathEnv := strings.Join(chrootedPaths, ":")
-
  rootfs := llb.Scratch()
  if ref := frontend.GetBaseOutputImage(spec, target, ""); ref != "" {
  rootfs = llb.Image(ref, llb.WithMetaResolver(sOpt.Resolver), dalec.WithConstraints(opts...))
@@ -177,35 +159,12 @@ func specToContainerLLB(w worker, client gwclient.Client, spec *dalec.Spec, targ
  }
 
  rootfs = builderImg.Run(
- w.Install(workPath, updated, true),
+ w.Install(updated, atRoot(workPath), noGPGCheck, withManifests),
  llb.AddMount(rpmMountDir, rpmDir, llb.SourcePath("/RPMS")),
  dalec.WithConstraints(opts...),
  ).AddMount(workPath, rootfs)
  }
 
- manifestCmd := `
-#!/usr/bin/env sh
-
-# If the rpm command is in the rootfs then we don't need to do anything
-# If not then this is a distroless image and we need to generate manifests of the installed rpms and cleanup the rpmdb.
-
-PATH="` + chrootedPathEnv + `" command -v rpm && exit 0
-
-set -e
-mkdir -p ` + mfstDir + `
-rpm --dbpath=` + rpmdbDir + ` -qa > ` + mfst1 + `
-rpm --dbpath=` + rpmdbDir + ` -qa --qf "%{NAME}\t%{VERSION}-%{RELEASE}\t%{INSTALLTIME}\t%{BUILDTIME}\t%{VENDOR}\t(none)\t%{SIZE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}\n" > ` + mfst2 + `
-rm -rf ` + rpmdbDir + `
-`
-
- manifestSh := llb.Scratch().File(llb.Mkfile("manifest.sh", 0o755, []byte(manifestCmd)), opts...)
- rootfs = builderImg.
- Run(
- shArgs("/tmp/manifest.sh"),
- llb.AddMount("/tmp/manifest.sh", manifestSh, llb.SourcePath("manifest.sh")),
- dalec.WithConstraints(opts...),
- ).AddMount(workPath, rootfs)
-
  if post := getImagePostInstall(spec, target); post != nil && len(post.Symlinks) > 0 {
  rootfs = builderImg.
  Run(dalec.WithConstraints(opts...), addImagePost(post, workPath)).

frontend/azlinux/handle_depsonly.go

@@ -2,13 +2,16 @@ package azlinux
 
 import (
  "context"
+ "encoding/json"
  "strings"
 
  "github.com/Azure/dalec"
  "github.com/Azure/dalec/frontend"
  "github.com/moby/buildkit/client/llb"
+ "github.com/moby/buildkit/client/llb/sourceresolver"
  gwclient "github.com/moby/buildkit/frontend/gateway/client"
  ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
+ "github.com/pkg/errors"
 )
 
 func handleDepsOnly(w worker) gwclient.BuildFunc {
@@ -48,10 +51,22 @@ func handleDepsOnly(w worker) gwclient.BuildFunc {
  return nil, nil, err
  }
 
- base := frontend.GetBaseOutputImage(spec, targetKey, "")
- img, err := frontend.BuildImageConfig(ctx, client, spec, targetKey, base)
- if err != nil {
- return nil, nil, err
+ var img *dalec.DockerImageSpec
+ if base := frontend.GetBaseOutputImage(spec, targetKey, ""); base != "" {
+ _, _, dt, err := client.ResolveImageConfig(ctx, base, sourceresolver.Opt{})
+ if err != nil {
+ return nil, nil, errors.Wrap(err, "error resolving base image config")
+ }
+ var cfg dalec.DockerImageSpec
+ if err := json.Unmarshal(dt, &cfg); err != nil {
+ return nil, nil, errors.Wrap(err, "error unmarshalling base image config")
+ }
+ img = &cfg
+ } else {
+ img, err = w.DefaultImageConfig(ctx, client)
+ if err != nil {
+ return nil, nil, errors.Wrap(err, "could not get default image config")
+ }
  }
 
  ref, err := res.SingleRef()

frontend/azlinux/handle_rpm.go

@@ -64,7 +64,7 @@ func installBuildDeps(w worker, spec *dalec.Spec, targetKey string, opts ...llb.
  }
  opts = append(opts, dalec.ProgressGroup("Install build deps"))
 
- return in.Run(w.Install("/", deps, false), dalec.WithConstraints(opts...)).Root()
+ return in.Run(w.Install(deps), dalec.WithConstraints(opts...)).Root()
  }
 }
 

frontend/azlinux/handler.go

@@ -2,6 +2,9 @@ package azlinux
 
 import (
  "context"
+ "fmt"
+ "path/filepath"
+ "strings"
 
  "github.com/Azure/dalec"
  "github.com/Azure/dalec/frontend"
@@ -17,7 +20,7 @@ const (
 
 type worker interface {
  Base(client gwclient.Client, opts ...llb.ConstraintsOpt) llb.State
- Install(root string, pkgs []string, skipGPG bool) llb.RunOption
+ Install(pkgs []string, opts ...installOpt) llb.RunOption
  DefaultImageConfig(context.Context, gwclient.Client) (*dalec.DockerImageSpec, error)
 }
 
@@ -43,3 +46,115 @@ func newHandler(d worker) gwclient.BuildFunc {
 
  return mux.Handle
 }
+
+type installConfig struct {
+ // Tells the installer to create the distroless rpm manifest.
+ manifest bool
+ // Disables GPG checking when installing RPMs.
+ // this is needed when installing unsigned RPMs.
+ noGPGCheck bool
+
+ // Sets the root path to install rpms too.
+ // this acts like installing to a chroot.
+ root string
+
+ constraints []llb.ConstraintsOpt
+}
+
+type installOpt func(*installConfig)
+
+func noGPGCheck(cfg *installConfig) {
+ cfg.noGPGCheck = true
+}
+
+func withManifests(cfg *installConfig) {
+ cfg.manifest = true
+}
+
+func atRoot(root string) installOpt {
+ return func(cfg *installConfig) {
+ cfg.root = root
+ }
+}
+
+func installWithConstraints(opts []llb.ConstraintsOpt) installOpt {
+ return func(cfg *installConfig) {
+ cfg.constraints = opts
+ }
+}
+
+func tdnfInstallFlags(cfg *installConfig) string {
+ var cmdOpts string
+
+ if cfg.noGPGCheck {
+ cmdOpts += " --nogpgcheck"
+ }
+
+ if cfg.root != "" {
+ cmdOpts += " --installroot=" + cfg.root
+ cmdOpts += " --setopt=reposdir=/etc/yum.repos.d"
+ }
+
+ return cmdOpts
+}
+
+func setInstallOptions(cfg *installConfig, opts []installOpt) {
+ for _, o := range opts {
+ o(cfg)
+ }
+}
+
+func manifestScript(workPath string, opts ...llb.ConstraintsOpt) llb.State {
+ mfstDir := filepath.Join(workPath, "var/lib/rpmmanifest")
+ mfst1 := filepath.Join(mfstDir, "container-manifest-1")
+ mfst2 := filepath.Join(mfstDir, "container-manifest-2")
+ rpmdbDir := filepath.Join(workPath, "var/lib/rpm")
+
+ chrootedPaths := []string{
+ filepath.Join(workPath, "/usr/local/bin"),
+ filepath.Join(workPath, "/usr/local/sbin"),
+ filepath.Join(workPath, "/usr/bin"),
+ filepath.Join(workPath, "/usr/sbin"),
+ filepath.Join(workPath, "/bin"),
+ filepath.Join(workPath, "/sbin"),
+ }
+ chrootedPathEnv := strings.Join(chrootedPaths, ":")
+
+ return llb.Scratch().File(llb.Mkfile("manifest.sh", 0o700, []byte(`
+#!/usr/bin/env sh
+
+# If the rpm command is in the rootfs then we don't need to do anything
+# If not then this is a distroless image and we need to generate manifests of the installed rpms and cleanup the rpmdb.
+
+PATH="`+chrootedPathEnv+`" command -v rpm && exit 0
+
+set -e
+
+mkdir -p `+mfstDir+`
+
+rpm --dbpath=`+rpmdbDir+` -qa > `+mfst1+`
+rpm --dbpath=`+rpmdbDir+` -qa --qf "%{NAME}\t%{VERSION}-%{RELEASE}\t%{INSTALLTIME}\t%{BUILDTIME}\t%{VENDOR}\t(none)\t%{SIZE}\t%{ARCH}\t%{EPOCHNUM}\t%{SOURCERPM}\n" > `+mfst2+`
+rm -rf `+rpmdbDir+`
+`)), opts...)
+}
+
+const manifestSh = "manifest.sh"
+
+func tdnfInstall(cfg *installConfig, relVer string, pkgs []string) llb.RunOption {
+ cmdFlags := tdnfInstallFlags(cfg)
+ cmdArgs := fmt.Sprintf("set -ex; tdnf install -y --releasever=%s %s %s", relVer, cmdFlags, strings.Join(pkgs, " "))
+
+ var runOpts []llb.RunOption
+
+ if cfg.manifest {
+ mfstScript := manifestScript(cfg.root, cfg.constraints...)
+
+ manifestPath := filepath.Join("/tmp", manifestSh)
+ runOpts = append(runOpts, llb.AddMount(manifestPath, mfstScript, llb.SourcePath(manifestSh)))
+
+ cmdArgs += "; " + manifestPath
+ }
+
+ runOpts = append(runOpts, shArgs(cmdArgs))
+ return dalec.WithRunOptions(runOpts...)
+}

frontend/azlinux/mariner2.go

@@ -3,9 +3,7 @@ package azlinux
 import (
  "context"
  "encoding/json"
- "fmt"
  "path/filepath"
- "strings"
 
  "github.com/Azure/dalec"
  "github.com/moby/buildkit/client/llb"
@@ -29,24 +27,14 @@ type mariner2 struct{}
 
 func (w mariner2) Base(client gwclient.Client, opts ...llb.ConstraintsOpt) llb.State {
  return llb.Image(mariner2Ref, llb.WithMetaResolver(client), dalec.WithConstraints(opts...)).Run(
- w.Install("/", []string{"rpm-build", "mariner-rpm-macros", "build-essential"}, false),
+ w.Install([]string{"rpm-build", "mariner-rpm-macros", "build-essential"}),
  ).Root()
 }
 
-func (w mariner2) Install(root string, pkgs []string, skipGPG bool) llb.RunOption {
- if root == "" {
- root = "/"
- }
-
- var gpgCheckFl string
- if skipGPG {
- gpgCheckFl = "--nogpgcheck"
- }
-
- cmdArgs := fmt.Sprintf("set -x; tdnf install -y %s --setopt=reposdir=/etc/yum.repos.d --installroot=%s --releasever=2.0 %s", gpgCheckFl, root, strings.Join(pkgs, " "))
- cmd := shArgs(cmdArgs)
-
- return dalec.WithRunOptions(cmd, w.tdnfCacheMount(root))
+func (w mariner2) Install(pkgs []string, opts ...installOpt) llb.RunOption {
+ var cfg installConfig
+ setInstallOptions(&cfg, opts)
+ return dalec.WithRunOptions(tdnfInstall(&cfg, "2.0", pkgs), w.tdnfCacheMount(cfg.root))
 }
 
 func (mariner2) DefaultImageConfig(ctx context.Context, client gwclient.Client) (*dalec.DockerImageSpec, error) {

test/mariner2_test.go → test/azlinux_test.go

@@ -19,6 +19,13 @@ func TestMariner2(t *testing.T) {
  testLinuxDistro(ctx, t, "mariner2/container")
 }
 
+func TestAzlinux3(t *testing.T) {
+ t.Parallel()
+
+ ctx := startTestSpan(baseCtx, t)
+ testLinuxDistro(ctx, t, "azlinux3/container")
+}
+
 func testLinuxDistro(ctx context.Context, t *testing.T, buildTarget string) {
  t.Run("Fail when non-zero exit code during build", func(t *testing.T) {
  t.Parallel()