diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index cf4cc471787f..5ea0eae1b797 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/containerd/containerd/platforms" + "github.com/docker/distribution/reference" "github.com/docker/go-units" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/client/llb" @@ -461,6 +462,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { } c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url)) }, + ContextByName: contextByNameFunc(c, tp), }) if err != nil { @@ -802,6 +804,100 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c return opts } +func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) { + return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) { + named, err := reference.ParseNormalizedNamed(name) + if err != nil { + return nil, nil, errors.Wrapf(err, "invalid context name %s", name) + } + name = strings.TrimSuffix(reference.FamiliarString(named), ":latest") + + if p != nil { + name := name + "::" + platforms.Format(platforms.Normalize(*p)) + st, img, err := contextByName(ctx, c, name, p) + if err != nil { + return nil, nil, err + } + if st != nil { + return st, img, nil + } + } + return contextByName(ctx, c, name, p) + } +} + +func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, error) { + opts := c.BuildOpts().Opts + v, ok := opts["context:"+name] + if !ok { + return nil, nil, nil + } + + vv := strings.SplitN(v, ":", 2) + if len(vv) != 2 { + return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) + } + switch vv[0] { + case "docker-image": + imgOpt := []llb.ImageOption{ + llb.WithCustomName("[context " + name + "] " + vv[1]), + llb.WithMetaResolver(c), + } + if platform != nil { + imgOpt = append(imgOpt, llb.Platform(*platform)) + } + st := llb.Image(strings.TrimPrefix(vv[1], "//"), imgOpt...) + return &st, nil, nil + case "git": + st, ok := detectGitContext(v, "1") + if !ok { + return nil, nil, errors.Errorf("invalid git context %s", v) + } + return st, nil, nil + case "http", "https": + st, ok := detectGitContext(v, "1") + if !ok { + httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) + st = &httpst + } + return st, nil, nil + case "local": + st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name)) + return &st, nil, nil + case "input": + inputs, err := c.Inputs(ctx) + if err != nil { + return nil, nil, err + } + st, ok := inputs[vv[1]] + if !ok { + return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) + } + md, ok := opts["input-metadata:"+vv[1]] + if ok { + m := make(map[string][]byte) + if err := json.Unmarshal([]byte(md), &m); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) + } + dt, ok := m["containerimage.config"] + if ok { + st, err = st.WithImageConfig([]byte(dt)) + if err != nil { + return nil, nil, err + } + var img dockerfile2llb.Image + if err := json.Unmarshal(dt, &img); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) + } + return &st, &img, nil + } + } + return &st, nil, nil + default: + return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) + } +} + func wrapSource(err error, sm *llb.SourceMap, ranges []parser.Range) error { if sm == nil { return err diff --git a/frontend/dockerfile/builder/caps.go b/frontend/dockerfile/builder/caps.go index 279701154eac..3c78cd56c4da 100644 --- a/frontend/dockerfile/builder/caps.go +++ b/frontend/dockerfile/builder/caps.go @@ -12,6 +12,7 @@ import ( var enabledCaps = map[string]struct{}{ "moby.buildkit.frontend.inputs": {}, "moby.buildkit.frontend.subrequests": {}, + "moby.buildkit.frontend.contexts": {}, } func validateCaps(req string) (forward bool, err error) { diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index b8f4e0e857df..f97d44315d74 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -29,7 +29,7 @@ RUN --mount=target=. --mount=type=cache,target=/root/.cache \ FROM scratch AS release LABEL moby.buildkit.frontend.network.none="true" -LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests" +LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts" COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend ENTRYPOINT ["/bin/dockerfile-frontend"] diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 0a521621ca64..218b382251aa 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -69,9 +69,20 @@ type ConvertOpt struct { SourceMap *llb.SourceMap Hostname string Warn func(short, url string, detail [][]byte, location *parser.Range) + ContextByName func(context.Context, string) (*llb.State, *Image, error) } func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) { + contextByName := opt.ContextByName + opt.ContextByName = func(ctx context.Context, name string) (*llb.State, *Image, error) { + if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") { + if contextByName != nil { + return contextByName(ctx, name) + } + } + return nil, nil, nil + } + if len(dt) == 0 { return nil, nil, errors.Errorf("the Dockerfile cannot be empty") } @@ -133,13 +144,30 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, st.BaseName = name ds := &dispatchState{ - stage: st, deps: make(map[*dispatchState]struct{}), ctxPaths: make(map[string]struct{}), stageName: st.Name, prefixPlatform: opt.PrefixPlatform, } + if st.Name != "" { + s, img, err := opt.ContextByName(ctx, st.Name) + if err != nil { + return nil, nil, err + } + if s != nil { + ds.noinit = true + ds.state = *s + if img != nil { + ds.image = *img + } + allDispatchStates.addState(ds) + continue + } + } + + ds.stage = st + if st.Name == "" { ds.stageName = fmt.Sprintf("stage-%d", i) } @@ -237,7 +265,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, for i, d := range allDispatchStates.states { reachable := isReachable(target, d) // resolve image config for every stage - if d.base == nil { + if d.base == nil && !d.noinit { if d.stage.BaseName == emptyImageName { d.state = llb.Scratch() d.image = emptyImage(platformOpt.targetPlatform) @@ -260,8 +288,23 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, platform = &platformOpt.targetPlatform } d.stage.BaseName = reference.TagNameOnly(ref).String() + var isScratch bool - if metaResolver != nil && reachable { + st, img, err := opt.ContextByName(ctx, d.stage.BaseName) + if err != nil { + return err + } + if st != nil { + if img != nil { + d.image = *img + } else { + d.image = emptyImage(platformOpt.targetPlatform) + } + d.state = *st + d.platform = platform + return nil + } + if reachable { prefix := "[" if opt.PrefixPlatform && platform != nil { prefix += platforms.Format(*platform) + " " @@ -615,6 +658,7 @@ type dispatchState struct { platform *ocispecs.Platform stage instructions.Stage base *dispatchState + noinit bool deps map[*dispatchState]struct{} buildArgs []instructions.KeyValuePairOptional commands []command diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 66856008f1c2..3aaa9f0e32c3 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -39,6 +39,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/upload/uploadprovider" "github.com/moby/buildkit/solver/errdefs" + "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/testutil" "github.com/moby/buildkit/util/testutil/httpserver" @@ -118,6 +119,10 @@ var allTests = integration.TestFuncs( testShmSize, testUlimit, testCgroupParent, + testNamedImageContext, + testNamedLocalContext, + testNamedInputContext, + testNamedMultiplatformInputContext, ) var fileOpTests = integration.TestFuncs( @@ -159,6 +164,7 @@ var securityOpts []integration.TestOpt type frontend interface { Solve(context.Context, *client.Client, client.SolveOpt, chan *client.SolveStatus) (*client.SolveResponse, error) + SolveGateway(context.Context, gateway.Client, gateway.SolveRequest) (*gateway.Result, error) DFCmdArgs(string, string) (string, string) RequiresBuildctl(t *testing.T) } @@ -5406,6 +5412,371 @@ COPY --from=base /out / require.Contains(t, strings.TrimSpace(string(dt)), `/foocgroup/buildkit/`) } +func testNamedImageContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM busybox AS base +RUN cat /etc/alpine-release > /out +FROM scratch +COPY --from=base /out / +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:busybox": "docker-image://alpine", + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.True(t, len(dt) > 0) +} + +func testNamedLocalContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM busybox AS base +RUN cat /etc/alpine-release > /out +FROM scratch +COPY --from=base /out / +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + outf := []byte(`dummy-result`) + + dir2, err := tmpdir( + fstest.CreateFile("out", outf, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir2) + + f := getFrontend(t, sb) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:base": "local:basedir", + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + "basedir": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.True(t, len(dt) > 0) +} + +func testNamedInputContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM alpine +ENV FOO=bar +RUN echo first > /out +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + dockerfile2 := []byte(` +FROM base AS build +RUN echo "foo is $FOO" > /foo +FROM scratch +COPY --from=build /foo /out / +`) + + dir2, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile2, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{}) + if err != nil { + return nil, err + } + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + st, err := ref.ToState() + if err != nil { + return nil, err + } + + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + dt, ok := res.Metadata["containerimage.config"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + + res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: map[string]string{ + "dockerfilekey": builder.DefaultLocalNameDockerfile + "2", + "context:base": "input:base", + "input-metadata:base": string(dt), + }, + FrontendInputs: map[string]*pb.Definition{ + "base": def.ToPB(), + }, + }) + if err != nil { + return nil, err + } + return res, nil + } + + product := "buildkit_test" + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = c.Build(ctx, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + builder.DefaultLocalNameDockerfile + "2": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, product, b, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.Equal(t, "first\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar\n", string(dt)) +} + +func testNamedMultiplatformInputContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM --platform=$BUILDPLATFORM alpine +ARG TARGETARCH +ENV FOO=bar-$TARGETARCH +RUN echo "foo $TARGETARCH" > /out +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + dockerfile2 := []byte(` +FROM base AS build +RUN echo "foo is $FOO" > /foo +FROM scratch +COPY --from=build /foo /out / +`) + + dir2, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile2, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: map[string]string{ + "platform": "linux/amd64,linux/arm64", + }, + }) + if err != nil { + return nil, err + } + + if len(res.Refs) != 2 { + return nil, errors.Errorf("expected 2 refs, got %d", len(res.Refs)) + } + + inputs := map[string]*pb.Definition{} + st, err := res.Refs["linux/amd64"].ToState() + if err != nil { + return nil, err + } + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + inputs["base::linux/amd64"] = def.ToPB() + + st, err = res.Refs["linux/arm64"].ToState() + if err != nil { + return nil, err + } + def, err = st.Marshal(ctx) + if err != nil { + return nil, err + } + inputs["base::linux/arm64"] = def.ToPB() + + frontendOpt := map[string]string{ + "dockerfilekey": builder.DefaultLocalNameDockerfile + "2", + "context:base::linux/amd64": "input:base::linux/amd64", + "context:base::linux/arm64": "input:base::linux/arm64", + "platform": "linux/amd64,linux/arm64", + } + + dt, ok := res.Metadata["containerimage.config/linux/amd64"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + frontendOpt["input-metadata:base::linux/amd64"] = string(dt) + + dt, ok = res.Metadata["containerimage.config/linux/arm64"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + frontendOpt["input-metadata:base::linux/arm64"] = string(dt) + + res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: frontendOpt, + FrontendInputs: inputs, + }) + if err != nil { + return nil, err + } + return res, nil + } + + product := "buildkit_test" + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = c.Build(ctx, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + builder.DefaultLocalNameDockerfile + "2": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, product, b, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/out")) + require.NoError(t, err) + require.Equal(t, "foo amd64\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar-amd64\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/out")) + require.NoError(t, err) + require.Equal(t, "foo arm64\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar-arm64\n", string(dt)) +} + func tmpdir(appliers ...fstest.Applier) (string, error) { tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") if err != nil { @@ -5542,6 +5913,11 @@ func (f *builtinFrontend) Solve(ctx context.Context, c *client.Client, opt clien return c.Solve(ctx, nil, opt, statusChan) } +func (f *builtinFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + req.Frontend = "dockerfile.v0" + return c.Solve(ctx, req) +} + func (f *builtinFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return dfCmdArgs(ctx, dockerfile, "--frontend dockerfile.v0") } @@ -5556,6 +5932,13 @@ func (f *clientFrontend) Solve(ctx context.Context, c *client.Client, opt client return c.Build(ctx, opt, "", builder.Build, statusChan) } +func (f *clientFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + if req.Frontend == "" && req.Definition == nil { + req.Frontend = "dockerfile.v0" + } + return c.Solve(ctx, req) +} + func (f *clientFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return "", "" } @@ -5578,6 +5961,15 @@ func (f *gatewayFrontend) Solve(ctx context.Context, c *client.Client, opt clien return c.Solve(ctx, nil, opt, statusChan) } +func (f *gatewayFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + req.Frontend = "gateway.v0" + if req.FrontendOpt == nil { + req.FrontendOpt = make(map[string]string) + } + req.FrontendOpt["source"] = f.gw + return c.Solve(ctx, req) +} + func (f *gatewayFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return dfCmdArgs(ctx, dockerfile, "--frontend gateway.v0 --opt=source="+f.gw) }