diff --git a/source/git/gitsource.go b/source/git/gitsource.go index ce745e9472efb..79dfcb429eaf7 100644 --- a/source/git/gitsource.go +++ b/source/git/gitsource.go @@ -129,16 +129,22 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri } }() + git, err := newGitSession(dir, "", "", nil, auth) + if err != nil { + return "", nil, err + } + defer git.cleanup() + if initializeRepo { // Explicitly set the Git config 'init.defaultBranch' to the // implied default to suppress "hint:" output about not having a // default initial branch name set which otherwise spams unit // test logs. - if _, err := gitWithinDir(ctx, dir, "", "", "", nil, auth, "-c", "init.defaultBranch=master", "init", "--bare"); err != nil { + if _, err := git.run(ctx, "-c", "init.defaultBranch=master", "init", "--bare"); err != nil { return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir) } - if _, err := gitWithinDir(ctx, dir, "", "", "", nil, auth, "remote", "add", "origin", remote); err != nil { + if _, err := git.run(ctx, "remote", "add", "origin", remote); err != nil { return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir) } @@ -371,9 +377,15 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index return "", "", nil, false, err } + git, err := newGitSession(gitDir, sock, knownHosts, netConf, gs.auth) + if err != nil { + return "", "", nil, false, err + } + defer git.cleanup() + ref := gs.src.Ref if ref == "" { - ref, err = getDefaultBranch(ctx, gitDir, "", sock, knownHosts, netConf, gs.auth, gs.src.Remote) + ref, err = getDefaultBranch(ctx, git, gs.src.Remote) if err != nil { return "", "", nil, false, err } @@ -381,7 +393,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index // TODO: should we assume that remote tag is immutable? add a timer? - buf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, netConf, gs.auth, "ls-remote", "origin", ref) + buf, err := git.run(ctx, "ls-remote", "origin", ref) if err != nil { return "", "", nil, false, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remote)) } @@ -457,9 +469,15 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out return nil, err } + git, err := newGitSession(gitDir, sock, knownHosts, netConf, gs.auth) + if err != nil { + return nil, err + } + defer git.cleanup() + ref := gs.src.Ref if ref == "" { - ref, err = getDefaultBranch(ctx, gitDir, "", sock, knownHosts, netConf, gs.auth, gs.src.Remote) + ref, err = getDefaultBranch(ctx, git, gs.src.Remote) if err != nil { return nil, err } @@ -468,7 +486,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out doFetch := true if isCommitSHA(ref) { // skip fetch if commit already exists - if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, netConf, nil, "cat-file", "-e", ref+"^{commit}"); err == nil { + if _, err := git.run(ctx, "cat-file", "-e", ref+"^{commit}"); err == nil { doFetch = false } } @@ -492,10 +510,10 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out // in case the ref is a branch and it now points to a different commit sha // TODO: is there a better way to do this? } - if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, netConf, gs.auth, args...); err != nil { + if _, err := git.run(ctx, args...); err != nil { return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(gs.src.Remote)) } - _, err = gitWithinDir(ctx, gitDir, "", sock, knownHosts, netConf, nil, "reflog", "expire", "--all", "--expire=now") + _, err = git.run(ctx, "reflog", "expire", "--all", "--expire=now") if err != nil { return nil, errors.Wrapf(err, "failed to expire reflog for remote %s", urlutil.RedactCredentials(gs.src.Remote)) } @@ -537,19 +555,20 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out if err := os.MkdirAll(checkoutDir, 0711); err != nil { return nil, err } - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, netConf, nil, "-c", "init.defaultBranch=master", "init") + checkoutGit := git.withinDir(checkoutDirGit, "") + _, err = checkoutGit.run(ctx, "-c", "init.defaultBranch=master", "init") if err != nil { return nil, err } // Defense-in-depth: clone using the file protocol to disable local-clone // optimizations which can be abused on some versions of Git to copy unintended // host files into the build context. - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, netConf, nil, "remote", "add", "origin", "file://"+gitDir) + _, err = checkoutGit.run(ctx, "remote", "add", "origin", "file://"+gitDir) if err != nil { return nil, err } - gitCatFileBuf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, netConf, gs.auth, "cat-file", "-t", ref) + gitCatFileBuf, err := git.run(ctx, "cat-file", "-t", ref) if err != nil { return nil, err } @@ -560,26 +579,26 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out pullref += ":refs/tags/" + pullref } else if isCommitSHA(ref) { pullref = "refs/buildkit/" + identity.NewID() - _, err = gitWithinDir(ctx, gitDir, "", sock, knownHosts, netConf, gs.auth, "update-ref", pullref, ref) + _, err = git.run(ctx, "update-ref", pullref, ref) if err != nil { return nil, err } } else { pullref += ":" + pullref } - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, netConf, gs.auth, "fetch", "-u", "--depth=1", "origin", pullref) + _, err = checkoutGit.run(ctx, "fetch", "-u", "--depth=1", "origin", pullref) if err != nil { return nil, err } - _, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, sock, knownHosts, netConf, nil, "checkout", "FETCH_HEAD") + _, err = checkoutGit.run(ctx, "checkout", "FETCH_HEAD") if err != nil { return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote)) } - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, netConf, nil, "remote", "set-url", "origin", urlutil.RedactCredentials(gs.src.Remote)) + _, err = checkoutGit.run(ctx, "remote", "set-url", "origin", urlutil.RedactCredentials(gs.src.Remote)) if err != nil { return nil, errors.Wrapf(err, "failed to set remote origin to %s", urlutil.RedactCredentials(gs.src.Remote)) } - _, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, netConf, nil, "reflog", "expire", "--all", "--expire=now") + _, err = checkoutGit.run(ctx, "reflog", "expire", "--all", "--expire=now") if err != nil { return nil, errors.Wrapf(err, "failed to expire reflog for remote %s", urlutil.RedactCredentials(gs.src.Remote)) } @@ -595,7 +614,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out return nil, errors.Wrapf(err, "failed to create temporary checkout dir") } } - _, err = gitWithinDir(ctx, gitDir, cd, sock, knownHosts, netConf, nil, "checkout", ref, "--", ".") + _, err = git.withinDir(gitDir, cd).run(ctx, "checkout", ref, "--", ".") if err != nil { return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote)) } @@ -628,7 +647,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out } } - _, err = gitWithinDir(ctx, gitDir, checkoutDir, sock, knownHosts, netConf, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1") + _, err = git.withinDir(gitDir, checkoutDir).run(ctx, "submodule", "update", "--init", "--recursive", "--depth=1") if err != nil { return nil, errors.Wrapf(err, "failed to update submodules for %s", urlutil.RedactCredentials(gs.src.Remote)) } @@ -669,14 +688,6 @@ func isCommitSHA(str string) bool { return validHex.MatchString(str) } -func gitWithinDir(ctx context.Context, gitDir, workDir, sshAuthSock, knownHosts string, netConf *networks.Config, auth []string, args ...string) (*bytes.Buffer, error) { - a := append([]string{"--git-dir", gitDir}, auth...) - if workDir != "" { - a = append(a, "--work-tree", workDir) - } - return git(ctx, workDir, sshAuthSock, knownHosts, netConf, append(a, args...)...) -} - func getGitSSHCommand(knownHosts string) string { gitSSHCommand := "ssh -F /dev/null" if knownHosts != "" { @@ -687,7 +698,40 @@ func getGitSSHCommand(knownHosts string) string { return gitSSHCommand } -func git(ctx context.Context, dir, sshAuthSock, knownHosts string, netConf *networks.Config, args ...string) (_ *bytes.Buffer, err error) { +type gitSession struct { + gitDir string // --git-dir flag + sshAuthSock string + knownHosts string + auth []string + + workDir string // command cwd + + hostsPath string + resolvPath string +} + +func newGitSession(gitDir, sshAuthSock, knownHosts string, netConf *networks.Config, auth []string) (*gitSession, error) { + session := &gitSession{ + workDir: gitDir, + sshAuthSock: sshAuthSock, + knownHosts: knownHosts, + auth: auth, + } + if err := session.initConfig(netConf); err != nil { + session.cleanup() + return nil, err + } + return session, nil +} + +func (s *gitSession) withinDir(gitDir, workDir string) *gitSession { + cp := *s + cp.gitDir = gitDir + cp.workDir = workDir + return &cp +} + +func (s *gitSession) run(ctx context.Context, args ...string) (_ *bytes.Buffer, err error) { for { stdout, stderr, flush := logs.NewLogStreams(ctx, true) defer stdout.Close() @@ -697,9 +741,16 @@ func git(ctx context.Context, dir, sshAuthSock, knownHosts string, netConf *netw flush() } }() + if s.gitDir != "" { + args = append([]string{"--git-dir", s.gitDir}, args...) + } args = append([]string{"-c", "protocol.file.allow=user"}, args...) // Block sneaky repositories from using repos from the filesystem as submodules. cmd := exec.Command("git", args...) - cmd.Dir = dir // some commands like submodule require this + if s.workDir != "" { + cmd.Dir = s.workDir // some commands like submodule require this + } else { + cmd.Dir = s.gitDir // set a sane default for commands where it doesn't matter + } buf := bytes.NewBuffer(nil) errbuf := bytes.NewBuffer(nil) cmd.Stdin = nil @@ -708,22 +759,20 @@ func git(ctx context.Context, dir, sshAuthSock, knownHosts string, netConf *netw cmd.Env = []string{ "PATH=" + os.Getenv("PATH"), "GIT_TERMINAL_PROMPT=0", - "GIT_SSH_COMMAND=" + getGitSSHCommand(knownHosts), + "GIT_SSH_COMMAND=" + getGitSSHCommand(s.knownHosts), // "GIT_TRACE=1", "GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig. "HOME=/dev/null", // Disable reading from user gitconfig. "LC_ALL=C", // Ensure consistent output. } - if netConf != nil { - if len(netConf.IpHosts) > 0 { - cmd.Env = append(cmd.Env, "EXTRA_HOSTS="+netConf.ExtraHosts()) - } - if netConf.Dns != nil && len(netConf.Dns.SearchDomains) > 0 { - cmd.Env = append(cmd.Env, "SEARCH_DOMAINS="+strings.Join(netConf.Dns.SearchDomains, " ")) - } + if s.hostsPath != "" { + cmd.Env = append(cmd.Env, "HOSTS_OVERRIDE="+s.hostsPath) + } + if s.resolvPath != "" { + cmd.Env = append(cmd.Env, "RESOLV_OVERRIDE="+s.resolvPath) } - if sshAuthSock != "" { - cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+sshAuthSock) + if s.sshAuthSock != "" { + cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+s.sshAuthSock) } // remote git commands spawn helper processes that inherit FDs and don't // handle parent death signal so exec.CommandContext can't be used @@ -762,8 +811,8 @@ func tokenScope(remote string) string { } // getDefaultBranch gets the default branch of a repository using ls-remote -func getDefaultBranch(ctx context.Context, gitDir, workDir, sshAuthSock, knownHosts string, netConf *networks.Config, auth []string, remoteURL string) (string, error) { - buf, err := gitWithinDir(ctx, gitDir, workDir, sshAuthSock, knownHosts, netConf, auth, "ls-remote", "--symref", remoteURL, "HEAD") +func getDefaultBranch(ctx context.Context, git *gitSession, remoteURL string) (string, error) { + buf, err := git.run(ctx, "ls-remote", "--symref", remoteURL, "HEAD") if err != nil { return "", errors.Wrapf(err, "error fetching default branch for repository %s", urlutil.RedactCredentials(remoteURL)) } diff --git a/source/git/gitsource_test.go b/source/git/gitsource_test.go index 9eeee87a9a134..7281266eb4aec 100644 --- a/source/git/gitsource_test.go +++ b/source/git/gitsource_test.go @@ -292,13 +292,17 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated } if keepGitDir { + git, err := newGitSession(dir, "", "", nil, nil) + require.NoError(t, err) + defer git.cleanup() + if isAnnotatedTag { // get commit sha that the annotated tag points to - annotatedTagCommit, err := git(ctx, dir, "", "", nil, "", "rev-list", "-n", "1", tag) + annotatedTagCommit, err := git.run(ctx, "rev-list", "-n", "1", tag) require.NoError(t, err) // get current commit sha - headCommit, err := git(ctx, dir, "", "", nil, "", "rev-parse", "HEAD") + headCommit, err := git.run(ctx, "rev-parse", "HEAD") require.NoError(t, err) // HEAD should match the actual commit sha (and not the sha of the annotated tag, @@ -309,7 +313,7 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated // test that we checked out the correct commit // (in the case of an annotated tag, this message is of the commit the annotated tag points to // and not the message of the tag) - gitLogOutput, err := git(ctx, dir, "", "", nil, "", "log", "-n", "1", "--format=%s") + gitLogOutput, err := git.run(ctx, "log", "-n", "1", "--format=%s") require.NoError(t, err) require.True(t, strings.Contains(strings.TrimSpace(gitLogOutput.String()), expectedCommitSubject)) } diff --git a/source/git/gitsource_unix.go b/source/git/gitsource_unix.go index 4a07a44e29e4c..d18f9138b68fb 100644 --- a/source/git/gitsource_unix.go +++ b/source/git/gitsource_unix.go @@ -17,6 +17,7 @@ import ( "time" "github.com/docker/docker/pkg/reexec" + "github.com/moby/buildkit/session/networks" "github.com/moby/sys/mount" "github.com/pkg/errors" "golang.org/x/sys/unix" @@ -34,35 +35,10 @@ func gitMain() { // Need standard user umask for git process. unix.Umask(0022) - extraHosts := os.Getenv("EXTRA_HOSTS") - searchDomains := os.Getenv("SEARCH_DOMAINS") - - if extraHosts != "" || searchDomains != "" { - // Unshare the mount namespace so we can override /etc/hosts and/or - // /etc/resolv.conf. - - // Unsharing is per-thread, so we have to pin this goroutine to the - // current thread for any of this to behave predictably. - runtime.LockOSThread() - - // Create a mount namespace, which the sub-process will inherit. - syscall.Unshare(syscall.CLONE_NEWNS) - } - - if extraHosts != "" { - cleanup, err := overrideHosts(extraHosts) - if err != nil { - panic(err) - } - defer cleanup() - } - - if searchDomains != "" { - cleanup, err := overrideSearch(strings.Fields(searchDomains)) - if err != nil { - panic(err) - } - defer cleanup() + if err := overrideNetworkConfig(); err != nil { + fmt.Fprintf(os.Stderr, "failed to override network config: %v\n", err) + os.Exit(1) + return } // Reexec git command @@ -114,77 +90,133 @@ func gitMain() { os.Exit(0) } -func overrideHosts(extraHosts string) (func(), error) { - currentHosts, err := os.ReadFile("/etc/hosts") - if err != nil { - return nil, errors.Wrap(err, "read current hosts") - } +func overrideNetworkConfig() error { + hostsOverride := os.Getenv("HOSTS_OVERRIDE") + resolvOverride := os.Getenv("RESOLV_OVERRIDE") - override, err := os.CreateTemp("", "buildkit-git-extra-hosts") - if err != nil { - return nil, errors.Wrap(err, "create hosts override") + if hostsOverride != "" || resolvOverride != "" { + // Unshare the mount namespace so we can override /etc/hosts and/or + // /etc/resolv.conf. + + // Unsharing is per-thread, so we have to pin this goroutine to the + // current thread for any of this to behave predictably. + runtime.LockOSThread() + + // Create a mount namespace, which the sub-process will inherit. + syscall.Unshare(syscall.CLONE_NEWNS) } - cleanup := func() { - _ = override.Close() - _ = os.Remove(override.Name()) + if hostsOverride != "" { + if err := mount.Mount(hostsOverride, "/etc/hosts", "none", "bind,ro"); err != nil { + return errors.Wrap(err, "mount hosts override") + } } - if err := replaceHosts(override, currentHosts, extraHosts); err != nil { - cleanup() - return nil, err + if resolvOverride != "" { + if err := mount.Mount(resolvOverride, "/etc/resolv.conf", "none", "bind,ro"); err != nil { + return errors.Wrap(err, "mount hosts override") + } } - return cleanup, nil + return nil } -func replaceHosts(override *os.File, currentHosts []byte, extraHosts string) error { - if _, err := override.Write(currentHosts); err != nil { - return errors.Wrap(err, "write current hosts") +func (s *gitSession) initConfig(netConf *networks.Config) error { + if netConf == nil { + return nil } - if _, err := fmt.Fprintln(override); err != nil { - return errors.Wrap(err, "write newline") + if len(netConf.IpHosts) > 0 { + if err := s.overrideHosts(netConf.IpHosts); err != nil { + return err + } + } + + if netConf.Dns != nil { + if err := s.overrideSearch(netConf.Dns); err != nil { + return err + } } - if _, err := fmt.Fprintln(override, extraHosts); err != nil { - return errors.Wrap(err, "write extra hosts") + return nil +} + +func (s *gitSession) cleanup() { + if s.hostsPath != "" { + os.Remove(s.hostsPath) } + if s.resolvPath != "" { + os.Remove(s.resolvPath) + } +} - if err := override.Close(); err != nil { - return errors.Wrap(err, "close hosts override") +func (s *gitSession) overrideHosts(extraHosts []*networks.IPHosts) error { + src, err := os.Open("/etc/hosts") + if err != nil { + return errors.Wrap(err, "read current hosts") } + defer src.Close() - if err := mount.Mount(override.Name(), "/etc/hosts", "none", "bind,ro"); err != nil { - return errors.Wrap(err, "mount hosts override") + override, err := os.CreateTemp("", "buildkit-git-extra-hosts") + if err != nil { + return errors.Wrap(err, "create hosts override") } + defer override.Close() - return nil + s.hostsPath = override.Name() + + if err := replaceHosts(override, src, extraHosts); err != nil { + return err + } + + return override.Close() } -func overrideSearch(searchDomains []string) (func(), error) { +func (s *gitSession) overrideSearch(dns *networks.DNSConfig) error { src, err := os.Open("/etc/resolv.conf") if err != nil { - return nil, err + return err } defer src.Close() override, err := os.CreateTemp("", "buildkit-git-resolv") if err != nil { - return nil, errors.Wrap(err, "create hosts override") + return errors.Wrap(err, "create hosts override") } - cleanup := func() { - _ = override.Close() - _ = os.Remove(override.Name()) + s.resolvPath = override.Name() + + defer override.Close() + + // TODO the rest + if err := replaceSearch(override, src, dns.SearchDomains); err != nil { + return err } - if err := replaceSearch(override, src, searchDomains); err != nil { - cleanup() - return nil, err + return nil +} + +func replaceHosts(override *os.File, currentHosts io.Reader, extraHosts []*networks.IPHosts) error { + if _, err := io.Copy(override, currentHosts); err != nil { + return errors.Wrap(err, "write current hosts") } - return cleanup, nil + if _, err := fmt.Fprintln(override); err != nil { + return errors.Wrap(err, "write newline") + } + + for _, host := range extraHosts { + hosts := strings.Join(host.Hosts, " ") + if _, err := fmt.Fprintf(override, "%s\t%s\n", host.Ip, hosts); err != nil { + return errors.Wrap(err, "write extra host") + } + } + + if err := override.Close(); err != nil { + return errors.Wrap(err, "close hosts override") + } + + return nil } func runProcessGroup(ctx context.Context, cmd *exec.Cmd) error {