dockerfile: implement hooks for RUN instructions

docs/reference/buildctl.md

@@ -181,6 +181,45 @@ $ buildctl build --frontend dockerfile.v0 --local context=. --local dockerfile=.
 $ buildctl build --frontend dockerfile.v0 --local context=. --local dockerfile=. --oci-layout foo2=/home/dir/oci --opt context:alpine=oci-layout://foo2@sha256:bd04a5b26dec16579cd1d7322e949c5905c4742269663fcbc84dcb2e9f4592fb
 ```
 
+##### Instruction hooks
+<!-- TODO: s/master/v0.16/ -->
+In the master branch, the Dockerfile frontend also supports "instruction hooks".
+
+e.g.,
+
+```bash
+buildctl build \
+  --frontend dockerfile.v0 \
+  --opt hook="$(cat hook.json)"
+```
+with `hook.json` as follows:
+```json
+{
+  "RUN": {
+    "entrypoint": ["/dev/.dfhook/entrypoint"],
+    "mounts": [
+       {"from": "example.com/hook", "target": "/dev/.dfhook"},
+       {"type": "secret", "source": "something", "target": "/etc/something"}
+    ]
+  }
+}
+```
+
+This will let the frontend treat `RUN foo` as:
+```dockerfile
+RUN \
+  --mount=from=example.com/hook,target=/dev/.dfhook \
+  --mount=type=secret,source=something,target=/etc/something \
+  /dev/.dfhook/entrypoint foo
+```
+
+`docker history` will still show this as `RUN foo`.
+
+<!--
+TODO: add example hook images to show concrete use-cases
+https://github.com/moby/buildkit/issues/4576
+-->
+
 #### gateway-specific options
 
 The `gateway.v0` frontend passes all of its `--opt` options on to the OCI image that is called to convert the

frontend/dockerfile/dockerfile2llb/convert.go

@@ -30,6 +30,7 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
 	"github.com/moby/buildkit/frontend/dockerfile/shell"
 	"github.com/moby/buildkit/frontend/dockerui"
+	"github.com/moby/buildkit/frontend/dockerui/types"
 	"github.com/moby/buildkit/frontend/subrequests/lint"
 	"github.com/moby/buildkit/frontend/subrequests/outline"
 	"github.com/moby/buildkit/frontend/subrequests/targets"
@@ -155,7 +156,7 @@ func ListTargets(ctx context.Context, dt []byte) (*targets.List, error) {
 		return nil, err
 	}
 
-	stages, _, err := instructions.Parse(dockerfile.AST, nil)
+	stages, _, err := instructions.Parse(dockerfile.AST, nil, instructions.ParseOpts{})
 	if err != nil {
 		return nil, err
 	}
@@ -255,7 +256,10 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
 
 	proxyEnv := proxyEnvFromBuildArgs(opt.BuildArgs)
 
-	stages, metaArgs, err := instructions.Parse(dockerfile.AST, lint)
+	parseOpts := instructions.ParseOpts{
+		InstructionHook: opt.InstructionHook,
+	}
+	stages, metaArgs, err := instructions.Parse(dockerfile.AST, lint, parseOpts)
 	if err != nil {
 		return nil, err
 	}
@@ -670,6 +674,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
 			sourceMap:           opt.SourceMap,
 			lint:                lint,
 			dockerIgnoreMatcher: dockerIgnoreMatcher,
+			instHook:            opt.InstructionHook,
 		}
 
 		if err = dispatchOnBuildTriggers(d, d.image.Config.OnBuild, opt); err != nil {
@@ -845,6 +850,7 @@ type dispatchOpt struct {
 	sourceMap           *llb.SourceMap
 	lint                *linter.Linter
 	dockerIgnoreMatcher *patternmatcher.PatternMatcher
+	instHook            *types.InstructionHook
 }
 
 func getEnv(state llb.State) shell.EnvGetter {
@@ -1114,6 +1120,9 @@ type command struct {
 }
 
 func dispatchOnBuildTriggers(d *dispatchState, triggers []string, opt dispatchOpt) error {
+	parseOpts := instructions.ParseOpts{
+		InstructionHook: opt.instHook,
+	}
 	for _, trigger := range triggers {
 		ast, err := parser.Parse(strings.NewReader(trigger))
 		if err != nil {
@@ -1122,7 +1131,7 @@ func dispatchOnBuildTriggers(d *dispatchState, triggers []string, opt dispatchOp
 		if len(ast.AST.Children) != 1 {
 			return errors.New("onbuild trigger should be a single expression")
 		}
-		ic, err := instructions.ParseCommand(ast.AST.Children[0])
+		ic, err := instructions.ParseCommand(ast.AST.Children[0], parseOpts)
 		if err != nil {
 			return err
 		}
@@ -1219,6 +1228,12 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE
 		args = withShell(d.image, args)
 	}
 
+	argsForHistory := args
+	if dopt.instHook != nil && dopt.instHook.Run != nil {
+		args = append(dopt.instHook.Run.Entrypoint, args...)
+		// leave argsForHistory unmodified
+	}
+
 	opt = append(opt, llb.Args(args), dfCmd(c), location(dopt.sourceMap, c.Location()))
 	if d.ignoreCache {
 		opt = append(opt, llb.IgnoreCache)
@@ -1282,7 +1297,7 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE
 	}
 
 	d.state = d.state.Run(opt...).Root()
-	return commitToHistory(&d.image, "RUN "+runCommandString(args, d.buildArgs, env), true, &d.state, d.epoch)
+	return commitToHistory(&d.image, "RUN "+runCommandString(argsForHistory, d.buildArgs, env), true, &d.state, d.epoch)
 }
 
 func dispatchWorkdir(d *dispatchState, c *instructions.WorkdirCommand, commit bool, opt *dispatchOpt) error {

frontend/dockerfile/dockerfile_insthook_test.go

@@ -0,0 +1,76 @@
+package dockerfile
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/containerd/continuity/fs/fstest"
+	"github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/frontend/dockerui"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/stretchr/testify/require"
+	"github.com/tonistiigi/fsutil"
+)
+
+var instHookTests = integration.TestFuncs(
+	testInstructionHook,
+)
+
+func testInstructionHook(t *testing.T, sb integration.Sandbox) {
+	integration.SkipOnPlatform(t, "windows")
+	f := getFrontend(t, sb)
+
+	dockerfile := []byte(`
+FROM busybox AS base
+RUN echo "$FOO" >/foo
+
+FROM scratch
+COPY --from=base /foo /foo
+`)
+
+	dir := integration.Tmpdir(
+		t,
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+	)
+	destDir := t.TempDir()
+
+	c, err := client.New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	build := func(attrs map[string]string) string {
+		_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+			FrontendAttrs: attrs,
+			Exports: []client.ExportEntry{
+				{
+					Type:      client.ExporterLocal,
+					OutputDir: destDir,
+				},
+			},
+			LocalMounts: map[string]fsutil.FS{
+				dockerui.DefaultLocalNameDockerfile: dir,
+				dockerui.DefaultLocalNameContext:    dir,
+			},
+		}, nil)
+		require.NoError(t, err)
+		p := filepath.Join(destDir, "foo")
+		b, err := os.ReadFile(p)
+		require.NoError(t, err)
+		return strings.TrimSpace(string(b))
+	}
+
+	require.Equal(t, "", build(nil))
+
+	const hook = `
+{
+  "RUN": {
+    "entrypoint": ["/dev/.dfhook/bin/busybox", "env", "FOO=BAR"],
+    "mounts": [
+      {"from": "busybox:uclibc", "target": "/dev/.dfhook"}
+    ]
+  }
+}`
+	require.Equal(t, "BAR", build(map[string]string{"hook": hook}))
+}

frontend/dockerfile/dockerfile_test.go

@@ -285,6 +285,8 @@ func TestIntegration(t *testing.T) {
 			"granted": networkHostGranted,
 			"denied":  networkHostDenied,
 		}))...)
+
+	integration.Run(t, instHookTests, opts...)
 }
 
 func testEmptyStringArgInEnv(t *testing.T, sb integration.Sandbox) {

frontend/dockerfile/instructions/commands.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
+	"github.com/moby/buildkit/frontend/dockerui/types"
 	dockerspec "github.com/moby/docker-image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
@@ -352,7 +353,7 @@ type ShellDependantCmdLine struct {
 //	RUN ["echo", "hi"]  # echo hi
 type RunCommand struct {
 	withNameAndCode
-	withExternalData
+	WithInstructionHook
 	ShellDependantCmdLine
 	FlagsUsed []string
 }
@@ -563,3 +564,21 @@ func (c *withExternalData) setExternalValue(k, v interface{}) {
 	}
 	c.m[k] = v
 }
+
+type WithInstructionHook struct {
+	withExternalData
+}
+
+const instHookKey = "dockerfile/run/instruction-hook"
+
+func (c *WithInstructionHook) GetInstructionHook() *types.InstructionHook {
+	x := c.getExternalValue(instHookKey)
+	if x == nil {
+		return nil
+	}
+	return x.(*types.InstructionHook)
+}
+
+func (c *WithInstructionHook) SetInstructionHook(h *types.InstructionHook) {
+	c.setExternalValue(instHookKey, h)
+}