diff --git a/build/build.go b/build/build.go
index 2c5de6fa15a6..8f9ebc93fd9b 100644
--- a/build/build.go
+++ b/build/build.go
@@ -219,7 +219,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
 			if err != nil {
 				return nil, err
 			}
-			so, release, err := toSolveOpt(ctx, np.Node(), multiDriver, opt, gatewayOpts, configDir, w, docker)
+			so, release, err := toSolveOpt(ctx, np.Node(), multiDriver, opt, gatewayOpts, configDir, addVCSLocalDir, w, docker)
 			if err != nil {
 				return nil, err
 			}
@@ -229,9 +229,6 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
 			for k, v := range gitattrs {
 				so.FrontendAttrs[k] = v
 			}
-			if addVCSLocalDir != nil {
-				addVCSLocalDir(so)
-			}
 			defers = append(defers, release)
 			reqn = append(reqn, &reqForNode{
 				resolvedNode: np,
diff --git a/build/git.go b/build/git.go
index 3655bfa3143e..cd8a92b9f9bd 100644
--- a/build/git.go
+++ b/build/git.go
@@ -10,14 +10,13 @@ import (
 
 	"github.com/docker/buildx/util/gitutil"
 	"github.com/docker/buildx/util/osutil"
-	"github.com/moby/buildkit/client"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
 const DockerfileLabel = "com.docker.image.source.entrypoint"
 
-func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath string) (map[string]string, func(*client.SolveOpt), error) {
+func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath string) (map[string]string, func(key, dir string, so *solveOpt), error) {
 	res := make(map[string]string)
 	if contextPath == "" {
 		return nil, nil, nil
@@ -112,26 +111,21 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st
 		}
 	}
 
-	return res, func(so *client.SolveOpt) {
+	return res, func(key, dir string, so *solveOpt) {
 		if !setGitInfo || root == "" {
 			return
 		}
-		for k, dir := range so.LocalDirs {
-			dir, err = filepath.EvalSymlinks(dir)
-			if err != nil {
-				continue
-			}
-			dir, err = filepath.Abs(dir)
-			if err != nil {
-				continue
-			}
-			if lp, err := osutil.GetLongPathName(dir); err == nil {
-				dir = lp
-			}
-			dir = osutil.SanitizePath(dir)
-			if r, err := filepath.Rel(root, dir); err == nil && !strings.HasPrefix(r, "..") {
-				so.FrontendAttrs["vcs:localdir:"+k] = r
-			}
+		dir, err := filepath.Abs(dir)
+		if err != nil {
+			return
+		}
+		if lp, err := osutil.GetLongPathName(dir); err == nil {
+			dir = lp
+		}
+		dir = osutil.SanitizePath(dir)
+		if r, err := filepath.Rel(root, dir); err == nil && !strings.HasPrefix(r, "..") {
+			so.FrontendAttrs["vcs:localdir:"+key] = r
 		}
+		return
 	}, nil
 }
diff --git a/build/git_test.go b/build/git_test.go
index da954ddbb304..9f373102dd89 100644
--- a/build/git_test.go
+++ b/build/git_test.go
@@ -159,11 +159,9 @@ func TestGetGitAttributesDirty(t *testing.T) {
 func TestLocalDirs(t *testing.T) {
 	setupTest(t)
 
-	so := &client.SolveOpt{
-		FrontendAttrs: map[string]string{},
-		LocalDirs: map[string]string{
-			"context":    ".",
-			"dockerfile": ".",
+	so := &solveOpt{
+		SolveOpt: client.SolveOpt{
+			FrontendAttrs: map[string]string{},
 		},
 	}
 
@@ -171,9 +169,11 @@ func TestLocalDirs(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, addVCSLocalDir)
 
-	addVCSLocalDir(so)
+	require.NoError(t, so.setLocalMount("context", ".", addVCSLocalDir))
 	require.Contains(t, so.FrontendAttrs, "vcs:localdir:context")
 	assert.Equal(t, ".", so.FrontendAttrs["vcs:localdir:context"])
+
+	require.NoError(t, so.setLocalMount("dockerfile", ".", addVCSLocalDir))
 	require.Contains(t, so.FrontendAttrs, "vcs:localdir:dockerfile")
 	assert.Equal(t, ".", so.FrontendAttrs["vcs:localdir:dockerfile"])
 }
@@ -193,11 +193,9 @@ func TestLocalDirsSub(t *testing.T) {
 	gitutil.GitCommit(c, t, "initial commit")
 	gitutil.GitSetRemote(c, t, "origin", "git@github.com:docker/buildx.git")
 
-	so := &client.SolveOpt{
-		FrontendAttrs: map[string]string{},
-		LocalDirs: map[string]string{
-			"context":    ".",
-			"dockerfile": "app",
+	so := &solveOpt{
+		SolveOpt: client.SolveOpt{
+			FrontendAttrs: map[string]string{},
 		},
 	}
 
@@ -205,9 +203,11 @@ func TestLocalDirsSub(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, addVCSLocalDir)
 
-	addVCSLocalDir(so)
+	require.NoError(t, so.setLocalMount("context", ".", addVCSLocalDir))
 	require.Contains(t, so.FrontendAttrs, "vcs:localdir:context")
 	assert.Equal(t, ".", so.FrontendAttrs["vcs:localdir:context"])
+
+	require.NoError(t, so.setLocalMount("dockerfile", "app", addVCSLocalDir))
 	require.Contains(t, so.FrontendAttrs, "vcs:localdir:dockerfile")
 	assert.Equal(t, "app", so.FrontendAttrs["vcs:localdir:dockerfile"])
 }
diff --git a/build/opt.go b/build/opt.go
index 29114b27e63b..e4d3c68a1ff9 100644
--- a/build/opt.go
+++ b/build/opt.go
@@ -32,9 +32,33 @@ import (
 	"github.com/moby/buildkit/util/entitlements"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/tonistiigi/fsutil"
 )
 
-func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Options, bopts gateway.BuildOpts, configDir string, pw progress.Writer, docker *dockerutil.Client) (solveOpt *client.SolveOpt, release func(), err error) {
+type solveOpt struct {
+	client.SolveOpt
+}
+
+func (so *solveOpt) setLocalMount(name, root string, addVCSLocalDir func(key, dir string, so *solveOpt)) error {
+	lm, err := fsutil.NewFS(root)
+	if err != nil {
+		return err
+	}
+	root, err = filepath.EvalSymlinks(root) // keep same behavior as fsutil.NewFS
+	if err != nil {
+		return err
+	}
+	if so.LocalMounts == nil {
+		so.LocalMounts = map[string]fsutil.FS{}
+	}
+	so.LocalMounts[name] = lm
+	if addVCSLocalDir != nil {
+		addVCSLocalDir(name, root, so)
+	}
+	return nil
+}
+
+func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Options, bopts gateway.BuildOpts, configDir string, addVCSLocalDir func(key, dir string, so *solveOpt), pw progress.Writer, docker *dockerutil.Client) (_ *client.SolveOpt, release func(), err error) {
 	nodeDriver := node.Driver
 	defers := make([]func(), 0, 2)
 	releaseF := func() {
@@ -93,15 +117,17 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
 		cacheFrom = append(cacheFrom, e)
 	}
 
-	so := client.SolveOpt{
-		Ref:                 opt.Ref,
-		Frontend:            "dockerfile.v0",
-		FrontendAttrs:       map[string]string{},
-		LocalDirs:           map[string]string{},
-		CacheExports:        cacheTo,
-		CacheImports:        cacheFrom,
-		AllowedEntitlements: opt.Allow,
-		SourcePolicy:        opt.SourcePolicy,
+	so := solveOpt{
+		SolveOpt: client.SolveOpt{
+			Ref:                 opt.Ref,
+			Frontend:            "dockerfile.v0",
+			FrontendAttrs:       map[string]string{},
+			LocalMounts:         map[string]fsutil.FS{},
+			CacheExports:        cacheTo,
+			CacheImports:        cacheFrom,
+			AllowedEntitlements: opt.Allow,
+			SourcePolicy:        opt.SourcePolicy,
+		},
 	}
 
 	if so.Ref == "" {
@@ -260,7 +286,7 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
 	so.Exports = opt.Exports
 	so.Session = opt.Session
 
-	releaseLoad, err := loadInputs(ctx, nodeDriver, opt.Inputs, pw, &so)
+	releaseLoad, err := loadInputs(ctx, nodeDriver, opt.Inputs, addVCSLocalDir, pw, &so)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -347,10 +373,10 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
 		so.FrontendAttrs["ulimit"] = ulimits
 	}
 
-	return &so, releaseF, nil
+	return &so.SolveOpt, releaseF, nil
 }
 
-func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, pw progress.Writer, target *client.SolveOpt) (func(), error) {
+func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, addVCSLocalDir func(key, dir string, so *solveOpt), pw progress.Writer, target *solveOpt) (func(), error) {
 	if inp.ContextPath == "" {
 		return nil, errors.New("please specify build context (e.g. \".\" for the current directory)")
 	}
@@ -396,11 +422,15 @@ func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, pw prog
 				dockerfileReader = buf
 				inp.ContextPath, _ = os.MkdirTemp("", "empty-dir")
 				toRemove = append(toRemove, inp.ContextPath)
-				target.LocalDirs["context"] = inp.ContextPath
+				if err := target.setLocalMount("context", inp.ContextPath, addVCSLocalDir); err != nil {
+					return nil, err
+				}
 			}
 		}
 	case osutil.IsLocalDir(inp.ContextPath):
-		target.LocalDirs["context"] = inp.ContextPath
+		if err := target.setLocalMount("context", inp.ContextPath, addVCSLocalDir); err != nil {
+			return nil, err
+		}
 		switch inp.DockerfilePath {
 		case "-":
 			dockerfileReader = inp.InStream
@@ -452,7 +482,9 @@ func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, pw prog
 	}
 
 	if dockerfileDir != "" {
-		target.LocalDirs["dockerfile"] = dockerfileDir
+		if err := target.setLocalMount("dockerfile", dockerfileDir, addVCSLocalDir); err != nil {
+			return nil, err
+		}
 		dockerfileName = handleLowercaseDockerfile(dockerfileDir, dockerfileName)
 	}
 
@@ -547,7 +579,9 @@ func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, pw prog
 		if k == "context" || k == "dockerfile" {
 			localName = "_" + k // underscore to avoid collisions
 		}
-		target.LocalDirs[localName] = v.Path
+		if err := target.setLocalMount(localName, v.Path, addVCSLocalDir); err != nil {
+			return nil, err
+		}
 		target.FrontendAttrs["context:"+k] = "local:" + localName
 	}
 
diff --git a/go.mod b/go.mod
index b71467ce8ef7..75838c4784ce 100644
--- a/go.mod
+++ b/go.mod
@@ -37,6 +37,7 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.4
+	github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302
 	github.com/zclconf/go-cty v1.14.1
 	go.opentelemetry.io/otel v1.19.0
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.42.0
@@ -138,7 +139,6 @@ require (
 	github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect
 	github.com/shibumi/go-pathspec v1.3.0 // indirect
 	github.com/theupdateframework/notary v0.7.0 // indirect
-	github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302 // indirect
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect
 	github.com/tonistiigi/vt100 v0.0.0-20230623042737-f9a4f7ef6531 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect