From 6e483a232339be14d359438899d372af3b14663c Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Fri, 9 Jun 2023 16:01:06 +0100 Subject: [PATCH 1/3] tests: add test for unicode filenames Signed-off-by: Justin Chadwell (cherry picked from commit f9f2a00d34f06ba38f48566e0a414bd304fa4128) --- frontend/dockerfile/dockerfile_test.go | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 70ef31069520..847a5b15577c 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -166,6 +166,7 @@ var allTests = integration.TestFuncs( testSBOMScannerArgs, testMultiPlatformWarnings, testNilContextInSolveGateway, + testCopyUnicodePath, ) // Tests that depend on the `security.*` entitlements @@ -6732,6 +6733,46 @@ func testNilContextInSolveGateway(t *testing.T, sb integration.Sandbox) { require.ErrorContains(t, err, "invalid nil input definition to definition op") } +func testCopyUnicodePath(t *testing.T, sb integration.Sandbox) { + f := getFrontend(t, sb) + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM alpine +COPY test-äöü.txt / +`) + + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + fstest.CreateFile("test-äöü.txt", []byte("test"), 0644), + ) + require.NoError(t, err) + + destDir, err := integration.Tmpdir(t) + require.NoError(t, err) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + LocalDirs: map[string]string{ + dockerui.DefaultLocalNameDockerfile: dir, + dockerui.DefaultLocalNameContext: dir, + }, + }, nil) + require.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(destDir, "test-äöü.txt")) + require.NoError(t, err) + require.Equal(t, "test", string(dt)) +} + func runShell(dir string, cmds ...string) error { for _, args := range cmds { var cmd *exec.Cmd From 9d491e7818e8ebbdb6822fed0ee3c9bac1b92a13 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 9 Jun 2023 17:53:07 -0700 Subject: [PATCH 2/3] filesync: fix handling non-ascii in file paths Signed-off-by: Tonis Tiigi --- session/filesync/filesync.go | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/session/filesync/filesync.go b/session/filesync/filesync.go index e31354262930..9ce282dfec47 100644 --- a/session/filesync/filesync.go +++ b/session/filesync/filesync.go @@ -4,8 +4,10 @@ import ( "context" "fmt" io "io" + "net/url" "os" "strings" + "unicode" "github.com/moby/buildkit/session" "github.com/pkg/errors" @@ -82,6 +84,7 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr } opts, _ := metadata.FromIncomingContext(stream.Context()) // if no metadata continue with empty object + opts = decodeOpts(opts) dirName := "" name, ok := opts[keyDirName] @@ -209,6 +212,8 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error { var stream grpc.ClientStream + opts = encodeOpts(opts) + ctx = metadata.NewOutgoingContext(ctx, opts) switch pr.name { @@ -337,3 +342,44 @@ func (e InvalidSessionError) Error() string { func (e InvalidSessionError) Unwrap() error { return e.err } + +func encodeOpts(opts map[string][]string) map[string][]string { + md := make(map[string][]string, len(opts)) + for k, v := range opts { + out := make([]string, len(v)) + for i, s := range v { + out[i] = encodeStringForHeader(s) + } + md[k] = out + } + return md +} + +func decodeOpts(opts map[string][]string) map[string][]string { + md := make(map[string][]string, len(opts)) + for k, v := range opts { + out := make([]string, len(v)) + for i, s := range v { + out[i], _ = url.QueryUnescape(s) + } + md[k] = out + } + return md +} + +// encodeStringForHeader encodes a string value so it can be used in grpc header. This encoding +// is backwards compatible and avoids encoding ASCII characters. +func encodeStringForHeader(input string) string { + var output strings.Builder + for _, runeVal := range input { + // Only encode non-ASCII characters. + if runeVal > unicode.MaxASCII { + // Encode each non-ASCII character individually. + output.WriteString(url.QueryEscape(string(runeVal))) + } else { + // Directly append ASCII characters and '*' to the output. + output.WriteRune(runeVal) + } + } + return output.String() +} From a09e2d88ddb6fe8052aeacf23d667a0735132e11 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Wed, 5 Jul 2023 23:50:49 -0700 Subject: [PATCH 3/3] filesync: mark if options have been encoded to detect old versions Signed-off-by: Tonis Tiigi --- session/filesync/filesync.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/session/filesync/filesync.go b/session/filesync/filesync.go index 9ce282dfec47..522f453203b4 100644 --- a/session/filesync/filesync.go +++ b/session/filesync/filesync.go @@ -6,6 +6,7 @@ import ( io "io" "net/url" "os" + "strconv" "strings" "unicode" @@ -26,6 +27,7 @@ const ( keyFollowPaths = "followpaths" keyDirName = "dir-name" keyExporterMetaPrefix = "exporter-md-" + keyOptsEncoded = "opts-encoded" ) type fsSyncProvider struct { @@ -84,7 +86,17 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr } opts, _ := metadata.FromIncomingContext(stream.Context()) // if no metadata continue with empty object - opts = decodeOpts(opts) + + isDecoded := false + if v, ok := opts[keyOptsEncoded]; ok && len(v) > 0 { + if b, _ := strconv.ParseBool(v[0]); b { + isDecoded = true + } + } + + if isDecoded { + opts = decodeOpts(opts) + } dirName := "" name, ok := opts[keyDirName] @@ -212,6 +224,9 @@ func FSSync(ctx context.Context, c session.Caller, opt FSSendRequestOpt) error { var stream grpc.ClientStream + // mark that we have encoded options so older versions with raw values can be detected on client side + opts[keyOptsEncoded] = []string{"1"} + opts = encodeOpts(opts) ctx = metadata.NewOutgoingContext(ctx, opts)