Skip to content

Commit

Permalink
dockerfile: add tests for multi-platform exports
Browse files Browse the repository at this point in the history 
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
  • Loading branch information
@tonistiigi
tonistiigi committed Jul 25, 2018
1 parent 6be1257 commit f719aad
Show file tree
Hide file tree
Showing 3 changed files with 212 additions and 68 deletions.
94 changes: 26 additions & 68 deletions client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,9 @@ package client

import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
Expand All @@ -27,6 +24,7 @@ import (
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets/secretsprovider"
"github.com/moby/buildkit/util/testutil"
"github.com/moby/buildkit/util/testutil/httpserver"
"github.com/moby/buildkit/util/testutil/integration"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
Expand Down Expand Up @@ -486,25 +484,25 @@ func testOCIExporter(t *testing.T, sb integration.Sandbox) {
dt, err := ioutil.ReadFile(out)
require.NoError(t, err)

m, err := readTarToMap(dt, false)
m, err := testutil.ReadTarToMap(dt, false)
require.NoError(t, err)

_, ok := m["oci-layout"]
require.True(t, ok)

var index ocispec.Index
err = json.Unmarshal(m["index.json"].data, &index)
err = json.Unmarshal(m["index.json"].Data, &index)
require.NoError(t, err)
require.Equal(t, 2, index.SchemaVersion)
require.Equal(t, 1, len(index.Manifests))

var mfst ocispec.Manifest
err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].data, &mfst)
err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].Data, &mfst)
require.NoError(t, err)
require.Equal(t, 2, len(mfst.Layers))

var ociimg ocispec.Image
err = json.Unmarshal(m["blobs/sha256/"+mfst.Config.Digest.Hex()].data, &ociimg)
err = json.Unmarshal(m["blobs/sha256/"+mfst.Config.Digest.Hex()].Data, &ociimg)
require.NoError(t, err)
require.Equal(t, "layers", ociimg.RootFS.Type)
require.Equal(t, 2, len(ociimg.RootFS.DiffIDs))
Expand All @@ -523,7 +521,7 @@ func testOCIExporter(t *testing.T, sb integration.Sandbox) {
RepoTags []string
Layers []string
}
err = json.Unmarshal(m["manifest.json"].data, &dockerMfst)
err = json.Unmarshal(m["manifest.json"].Data, &dockerMfst)
require.NoError(t, err)
require.Equal(t, 1, len(dockerMfst))

Expand Down Expand Up @@ -692,45 +690,45 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
dt, err = content.ReadBlob(ctx, img.ContentStore(), ocispec.Descriptor{Digest: mfst.Layers[0].Digest})
require.NoError(t, err)

m, err := readTarToMap(dt, true)
m, err := testutil.ReadTarToMap(dt, true)
require.NoError(t, err)

item, ok := m["foo/"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
require.Equal(t, 0741, int(item.header.Mode&0777))
require.Equal(t, int32(item.Header.Typeflag), tar.TypeDir)
require.Equal(t, 0741, int(item.Header.Mode&0777))

item, ok = m["foo/sub/"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
require.Equal(t, int32(item.Header.Typeflag), tar.TypeDir)

item, ok = m["foo/sub/bar"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeReg)
require.Equal(t, []byte("first"), item.data)
require.Equal(t, int32(item.Header.Typeflag), tar.TypeReg)
require.Equal(t, []byte("first"), item.Data)

_, ok = m["foo/sub/baz"]
require.False(t, ok)

dt, err = content.ReadBlob(ctx, img.ContentStore(), ocispec.Descriptor{Digest: mfst.Layers[1].Digest})
require.NoError(t, err)

m, err = readTarToMap(dt, true)
m, err = testutil.ReadTarToMap(dt, true)
require.NoError(t, err)

item, ok = m["foo/sub/baz"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeReg)
require.Equal(t, []byte("second"), item.data)
require.Equal(t, int32(item.Header.Typeflag), tar.TypeReg)
require.Equal(t, []byte("second"), item.Data)

item, ok = m["foo/"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
require.Equal(t, 0741, int(item.header.Mode&0777))
require.Equal(t, int32(item.Header.Typeflag), tar.TypeDir)
require.Equal(t, 0741, int(item.Header.Mode&0777))

item, ok = m["foo/sub/"]
require.True(t, ok)
require.Equal(t, int32(item.header.Typeflag), tar.TypeDir)
require.Equal(t, int32(item.Header.Typeflag), tar.TypeDir)

_, ok = m["foo/sub/bar"]
require.False(t, ok)
Expand Down Expand Up @@ -978,23 +976,23 @@ func testDuplicateWhiteouts(t *testing.T, sb integration.Sandbox) {
dt, err := ioutil.ReadFile(out)
require.NoError(t, err)

m, err := readTarToMap(dt, false)
m, err := testutil.ReadTarToMap(dt, false)
require.NoError(t, err)

var index ocispec.Index
err = json.Unmarshal(m["index.json"].data, &index)
err = json.Unmarshal(m["index.json"].Data, &index)
require.NoError(t, err)

var mfst ocispec.Manifest
err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].data, &mfst)
err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].Data, &mfst)
require.NoError(t, err)

lastLayer := mfst.Layers[len(mfst.Layers)-1]

layer, ok := m["blobs/sha256/"+lastLayer.Digest.Hex()]
require.True(t, ok)

m, err = readTarToMap(layer.data, true)
m, err = testutil.ReadTarToMap(layer.Data, true)
require.NoError(t, err)

_, ok = m[".wh.d0"]
Expand Down Expand Up @@ -1045,23 +1043,23 @@ func testWhiteoutParentDir(t *testing.T, sb integration.Sandbox) {
dt, err := ioutil.ReadFile(out)
require.NoError(t, err)

m, err := readTarToMap(dt, false)
m, err := testutil.ReadTarToMap(dt, false)
require.NoError(t, err)

var index ocispec.Index
err = json.Unmarshal(m["index.json"].data, &index)
err = json.Unmarshal(m["index.json"].Data, &index)
require.NoError(t, err)

var mfst ocispec.Manifest
err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].data, &mfst)
err = json.Unmarshal(m["blobs/sha256/"+index.Manifests[0].Digest.Hex()].Data, &mfst)
require.NoError(t, err)

lastLayer := mfst.Layers[len(mfst.Layers)-1]

layer, ok := m["blobs/sha256/"+lastLayer.Digest.Hex()]
require.True(t, ok)

m, err = readTarToMap(layer.data, true)
m, err = testutil.ReadTarToMap(layer.Data, true)
require.NoError(t, err)

_, ok = m["foo/.wh.bar"]
Expand Down Expand Up @@ -1213,46 +1211,6 @@ func requiresLinux(t *testing.T) {
}
}

type tarItem struct {
header *tar.Header
data []byte
}

func readTarToMap(dt []byte, compressed bool) (map[string]*tarItem, error) {
m := map[string]*tarItem{}
var r io.Reader = bytes.NewBuffer(dt)
if compressed {
gz, err := gzip.NewReader(r)
if err != nil {
return nil, err
}
defer gz.Close()
r = gz
}
tr := tar.NewReader(r)
for {
h, err := tr.Next()
if err != nil {
if err == io.EOF {
return m, nil
}
return nil, err
}
if _, ok := m[h.Name]; ok {
return nil, errors.Errorf("duplicate entries for %s", h.Name)
}

var dt []byte
if h.Typeflag == tar.TypeReg {
dt, err = ioutil.ReadAll(tr)
if err != nil {
return nil, err
}
}
m[h.Name] = &tarItem{header: h, data: dt}
}
}

func checkAllRemoved(t *testing.T, c *Client, sb integration.Sandbox) {
retries := 0
for {
Expand Down
135 changes: 135 additions & 0 deletions frontend/dockerfile/dockerfile_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/moby/buildkit/client/llb"
"github.com/moby/buildkit/frontend/dockerfile/builder"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/util/testutil"
"github.com/moby/buildkit/util/testutil/httpserver"
"github.com/moby/buildkit/util/testutil/integration"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
Expand Down Expand Up @@ -72,9 +73,143 @@ func TestIntegration(t *testing.T) {
testContextChangeDirToFile,
testPlatformArgsImplicit,
testPlatformArgsExplicit,
testExportMultiPlatform,
})
}

func testExportMultiPlatform(t *testing.T, sb integration.Sandbox) {
t.Parallel()

dockerfile := []byte(`
FROM scratch
ARG TARGETARCH
ARG TARGETPLATFORM
LABEL target=$TARGETPLATFORM
COPY arch-$TARGETARCH whoami
`)

dir, err := tmpdir(
fstest.CreateFile("Dockerfile", dockerfile, 0600),
fstest.CreateFile("arch-arm", []byte(`i am arm`), 0600),
fstest.CreateFile("arch-amd64", []byte(`i am amd64`), 0600),
fstest.CreateFile("arch-s390x", []byte(`i am s390x`), 0600),
fstest.CreateFile("arch-ppc64le", []byte(`i am ppc64le`), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)

c, err := client.New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()

destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)

_, err = c.Solve(context.TODO(), nil, client.SolveOpt{
Frontend: "dockerfile.v0",
LocalDirs: map[string]string{
builder.LocalNameDockerfile: dir,
builder.LocalNameContext: dir,
},
FrontendAttrs: map[string]string{
"platform": "windows/amd64,linux/arm,linux/s390x",
},
Exporter: client.ExporterLocal,
ExporterOutputDir: destDir,
}, nil)
require.NoError(t, err)

dt, err := ioutil.ReadFile(filepath.Join(destDir, "windows_amd64/whoami"))
require.NoError(t, err)
require.Equal(t, "i am amd64", string(dt))

dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm_v7/whoami"))
require.NoError(t, err)
require.Equal(t, "i am arm", string(dt))

dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_s390x/whoami"))
require.NoError(t, err)
require.Equal(t, "i am s390x", string(dt))

// repeat with oci exporter

destDir, err = ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)

out := filepath.Join(destDir, "out.tar")
outW, err := os.Create(out)
require.NoError(t, err)

_, err = c.Solve(context.TODO(), nil, client.SolveOpt{
Frontend: "dockerfile.v0",
LocalDirs: map[string]string{
builder.LocalNameDockerfile: dir,
builder.LocalNameContext: dir,
},
FrontendAttrs: map[string]string{
"platform": "windows/amd64,linux/arm/v6,linux/ppc64le",
},
Exporter: client.ExporterOCI,
ExporterOutput: outW,
}, nil)
require.NoError(t, err)

dt, err = ioutil.ReadFile(filepath.Join(destDir, "out.tar"))
require.NoError(t, err)

m, err := testutil.ReadTarToMap(dt, false)
require.NoError(t, err)

var idx ocispec.Index
err = json.Unmarshal(m["index.json"].Data, &idx)
require.NoError(t, err)

mlistHex := idx.Manifests[0].Digest.Hex()

idx = ocispec.Index{}
err = json.Unmarshal(m["blobs/sha256/"+mlistHex].Data, &idx)
require.NoError(t, err)

require.Equal(t, 3, len(idx.Manifests))

for i, exp := range []struct {
p string
os string
arch string
dt string
}{
{p: "windows/amd64", os: "windows", arch: "amd64", dt: "i am amd64"},
{p: "linux/arm/v6", os: "linux", arch: "arm", dt: "i am arm"},
{p: "linux/ppc64le", os: "linux", arch: "ppc64le", dt: "i am ppc64le"},
} {
t.Run(exp.p, func(t *testing.T) {
require.Equal(t, exp.p, platforms.Format(*idx.Manifests[i].Platform))

var mfst ocispec.Manifest
err = json.Unmarshal(m["blobs/sha256/"+idx.Manifests[i].Digest.Hex()].Data, &mfst)
require.NoError(t, err)

require.Equal(t, 1, len(mfst.Layers))

m2, err := testutil.ReadTarToMap(m["blobs/sha256/"+mfst.Layers[0].Digest.Hex()].Data, true)
require.NoError(t, err)
require.Equal(t, exp.dt, string(m2["whoami"].Data))

var img ocispec.Image
err = json.Unmarshal(m["blobs/sha256/"+mfst.Config.Digest.Hex()].Data, &img)
require.NoError(t, err)

require.Equal(t, exp.os, img.OS)
require.Equal(t, exp.arch, img.Architecture)
v, ok := img.Config.Labels["target"]
require.True(t, ok)
require.Equal(t, exp.p, v)
})
}
}

// tonistiigi/fsutil#46
func testContextChangeDirToFile(t *testing.T, sb integration.Sandbox) {
t.Parallel()
Expand Down
Loading

0 comments on commit f719aad

Please sign in to comment.