Skip to content

Commit

Permalink
Support cloning from non-standard git repos
Browse files Browse the repository at this point in the history 
- fixes assumption that ssh git clones must be via the `git` user.
- allows passing the SSH_AUTH_SOCK from the client to GitSource
- allows passing a known_host entry for ssh
  • Loading branch information
@alexcb
alexcb committed Nov 10, 2020
1 parent 64123a4 commit e186e4a
Show file tree
Hide file tree
Showing 6 changed files with 194 additions and 31 deletions.
25 changes: 24 additions & 1 deletion client/llb/source.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ func Git(remote, ref string, opts ...GitOption) State {
url := ""

for _, prefix := range []string{
"http://", "https://", "git://", "git@",
"http://", "https://", "git://",
} {
if strings.HasPrefix(remote, prefix) {
url = strings.Split(remote, "#")[0]
Expand Down Expand Up @@ -243,6 +243,14 @@ func Git(remote, ref string, opts ...GitOption) State {
addCap(&gi.Constraints, pb.CapSourceGitHTTPAuth)
}
}
if gi.KnownSSHHosts != "" {
attrs[pb.AttrKnownSSHHosts] = gi.KnownSSHHosts
addCap(&gi.Constraints, pb.CapSourceGitKnownSSHHosts)
}
if gi.MountSSHSock {
attrs[pb.AttrMountSSHSock] = "true"
addCap(&gi.Constraints, pb.CapSourceGitMountSSHSock)
}

addCap(&gi.Constraints, pb.CapSourceGit)

Expand All @@ -265,6 +273,8 @@ type GitInfo struct {
AuthTokenSecret string
AuthHeaderSecret string
addAuthCap bool
KnownSSHHosts string
MountSSHSock bool
}

func KeepGitDir() GitOption {
Expand All @@ -287,6 +297,19 @@ func AuthHeaderSecret(v string) GitOption {
})
}

func KnownSSHHosts(key string) GitOption {
key = strings.TrimSuffix(key, "\n")
return gitOptionFunc(func(gi *GitInfo) {
gi.KnownSSHHosts = gi.KnownSSHHosts + key + "\n"
})
}

func MountSSHSock() GitOption {
return gitOptionFunc(func(gi *GitInfo) {
gi.MountSSHSock = true
})
}

func Scratch() State {
return NewState(nil)
}
Expand Down
2 changes: 2 additions & 0 deletions solver/pb/attr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ const AttrKeepGitDir = "git.keepgitdir"
const AttrFullRemoteURL = "git.fullurl"
const AttrAuthHeaderSecret = "git.authheadersecret"
const AttrAuthTokenSecret = "git.authtokensecret"
const AttrKnownSSHHosts = "git.knownsshhosts"
const AttrMountSSHSock = "git.mountsshsock"
const AttrLocalSessionID = "local.session"
const AttrLocalUniqueID = "local.unique"
const AttrIncludePatterns = "local.includepattern"
Expand Down
22 changes: 18 additions & 4 deletions solver/pb/caps.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ const (
CapSourceLocalExcludePatterns apicaps.CapID = "source.local.excludepatterns"
CapSourceLocalSharedKeyHint apicaps.CapID = "source.local.sharedkeyhint"

CapSourceGit apicaps.CapID = "source.git"
CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir"
CapSourceGitFullURL apicaps.CapID = "source.git.fullurl"
CapSourceGitHTTPAuth apicaps.CapID = "source.git.httpauth"
CapSourceGit apicaps.CapID = "source.git"
CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir"
CapSourceGitFullURL apicaps.CapID = "source.git.fullurl"
CapSourceGitHTTPAuth apicaps.CapID = "source.git.httpauth"
CapSourceGitKnownSSHHosts apicaps.CapID = "source.git.knownsshhosts"
CapSourceGitMountSSHSock apicaps.CapID = "source.git.mountsshsock"

CapSourceHTTP apicaps.CapID = "source.http"
CapSourceHTTPChecksum apicaps.CapID = "source.http.checksum"
Expand Down Expand Up @@ -138,6 +140,18 @@ func init() {
Status: apicaps.CapStatusExperimental,
})

Caps.Init(apicaps.Cap{
ID: CapSourceGitKnownSSHHosts,
Enabled: true,
Status: apicaps.CapStatusExperimental,
})

Caps.Init(apicaps.Cap{
ID: CapSourceGitMountSSHSock,
Enabled: true,
Status: apicaps.CapStatusExperimental,
})

Caps.Init(apicaps.Cap{
ID: CapSourceHTTP,
Enabled: true,
Expand Down
147 changes: 132 additions & 15 deletions source/git/gitsource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@ import (
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"os/exec"
"os/user"
"path/filepath"
"regexp"
"strconv"
"strings"

"github.com/moby/buildkit/cache"
Expand All @@ -19,6 +22,7 @@ import (
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/secrets"
"github.com/moby/buildkit/session/sshforward"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver"
"github.com/moby/buildkit/source"
Expand All @@ -27,6 +31,8 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
bolt "go.etcd.io/bbolt"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)

var validHex = regexp.MustCompile(`^[a-f0-9]{40}$`)
Expand Down Expand Up @@ -123,11 +129,11 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri
}()

if initializeRepo {
if _, err := gitWithinDir(ctx, dir, "", auth, "init", "--bare"); err != nil {
if _, err := gitWithinDir(ctx, dir, "", "", "", auth, "init", "--bare"); err != nil {
return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir)
}

if _, err := gitWithinDir(ctx, dir, "", auth, "remote", "add", "origin", remote); err != nil {
if _, err := gitWithinDir(ctx, dir, "", "", "", auth, "remote", "add", "origin", remote); err != nil {
return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir)
}

Expand Down Expand Up @@ -232,6 +238,70 @@ func (gs *gitSourceHandler) getAuthToken(ctx context.Context, g session.Group) e
})
}

func (gs *gitSourceHandler) mountSSHAuthSock(ctx context.Context, g session.Group) (string, func() error, error) {
sshID := "default"
var caller session.Caller
err := gs.sm.Any(ctx, g, func(ctx context.Context, _ string, c session.Caller) error {
if err := sshforward.CheckSSHID(ctx, c, sshID); err != nil {
if st, ok := status.FromError(err); ok && st.Code() == codes.Unimplemented {
return errors.Errorf("no SSH key %q forwarded from the client", sshID)
}

return err
}
caller = c
return nil
})
if err != nil {
return "", nil, err
}

usr, err := user.Current()
if err != nil {
return "", nil, err
}

// best effor, default to root
uid, _ := strconv.Atoi(usr.Uid)
gid, _ := strconv.Atoi(usr.Uid)

sock, cleanup, err := sshforward.MountSSHSocket(ctx, caller, sshforward.SocketOpt{
ID: sshID,
UID: uid,
GID: gid,
Mode: 0775,
})
if err != nil {
return "", nil, err
}

return sock, cleanup, nil
}

func (gs *gitSourceHandler) mountKnownHosts(ctx context.Context) (string, func() error, error) {
if gs.src.KnownSSHHosts == "" {
return "", nil, errors.Errorf("no configured known hosts forwarded from the client")
}
knownHosts, err := ioutil.TempFile("", "")
if err != nil {
return "", nil, err
}
cleanup := func() error {
return os.Remove(knownHosts.Name())
}
_, err = knownHosts.Write([]byte(gs.src.KnownSSHHosts))
if err != nil {
cleanup()
return "", nil, err
}
err = knownHosts.Close()
if err != nil {
cleanup()
return "", nil, err
}
return knownHosts.Name(), cleanup, nil
}

func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index int) (string, solver.CacheOpts, bool, error) {
remote := gs.src.Remote
ref := gs.src.Ref
Expand All @@ -255,9 +325,29 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index
}
defer unmountGitDir()

var sock string
if gs.src.MountSSHSock {
var unmountSock func() error
sock, unmountSock, err = gs.mountSSHAuthSock(ctx, g)
if err != nil {
return "", nil, false, err
}
defer unmountSock()
}

var knownHosts string
if gs.src.KnownSSHHosts != "" {
var unmountKnownHosts func() error
knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx)
if err != nil {
return "", nil, false, err
}
defer unmountKnownHosts()
}

// TODO: should we assume that remote tag is immutable? add a timer?

buf, err := gitWithinDir(ctx, gitDir, "", gs.auth, "ls-remote", "origin", ref)
buf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "ls-remote", "origin", ref)
if err != nil {
return "", nil, false, errors.Wrapf(err, "failed to fetch remote %s", remote)
}
Expand Down Expand Up @@ -313,10 +403,30 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
}
defer unmountGitDir()

var sock string
if gs.src.MountSSHSock {
var unmountSock func() error
sock, unmountSock, err = gs.mountSSHAuthSock(ctx, g)
if err != nil {
return nil, err
}
defer unmountSock()
}

var knownHosts string
if gs.src.KnownSSHHosts != "" {
var unmountKnownHosts func() error
knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx)
if err != nil {
return nil, err
}
defer unmountKnownHosts()
}

doFetch := true
if isCommitSHA(ref) {
// skip fetch if commit already exists
if _, err := gitWithinDir(ctx, gitDir, "", nil, "cat-file", "-e", ref+"^{commit}"); err == nil {
if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, nil, "cat-file", "-e", ref+"^{commit}"); err == nil {
doFetch = false
}
}
Expand All @@ -340,7 +450,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
// in case the ref is a branch and it now points to a different commit sha
// TODO: is there a better way to do this?
}
if _, err := gitWithinDir(ctx, gitDir, "", gs.auth, args...); err != nil {
if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, args...); err != nil {
return nil, errors.Wrapf(err, "failed to fetch remote %s", gs.src.Remote)
}
}
Expand Down Expand Up @@ -376,41 +486,41 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
if err := os.MkdirAll(checkoutDir, 0711); err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", nil, "init")
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "init")
if err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", nil, "remote", "add", "origin", gitDir)
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "remote", "add", "origin", gitDir)
if err != nil {
return nil, err
}
pullref := ref
if isCommitSHA(ref) {
pullref = "refs/buildkit/" + identity.NewID()
_, err = gitWithinDir(ctx, gitDir, "", gs.auth, "update-ref", pullref, ref)
_, err = gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "update-ref", pullref, ref)
if err != nil {
return nil, err
}
} else {
pullref += ":" + pullref
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", gs.auth, "fetch", "-u", "--depth=1", "origin", pullref)
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, gs.auth, "fetch", "-u", "--depth=1", "origin", pullref)
if err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, nil, "checkout", "FETCH_HEAD")
_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, sock, knownHosts, nil, "checkout", "FETCH_HEAD")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
}
gitDir = checkoutDirGit
} else {
_, err = gitWithinDir(ctx, gitDir, checkoutDir, nil, "checkout", ref, "--", ".")
_, err = gitWithinDir(ctx, gitDir, checkoutDir, sock, knownHosts, nil, "checkout", ref, "--", ".")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", gs.src.Remote)
}
}

_, err = gitWithinDir(ctx, gitDir, checkoutDir, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1")
_, err = gitWithinDir(ctx, gitDir, checkoutDir, sock, knownHosts, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1")
if err != nil {
return nil, errors.Wrapf(err, "failed to update submodules for %s", gs.src.Remote)
}
Expand Down Expand Up @@ -459,15 +569,15 @@ func isCommitSHA(str string) bool {
return validHex.MatchString(str)
}

func gitWithinDir(ctx context.Context, gitDir, workDir string, auth []string, args ...string) (*bytes.Buffer, error) {
func gitWithinDir(ctx context.Context, gitDir, workDir, sshAuthSock, knownHosts string, auth []string, args ...string) (*bytes.Buffer, error) {
a := append([]string{"--git-dir", gitDir}, auth...)
if workDir != "" {
a = append(a, "--work-tree", workDir)
}
return git(ctx, workDir, append(a, args...)...)
return git(ctx, workDir, sshAuthSock, knownHosts, append(a, args...)...)
}

func git(ctx context.Context, dir string, args ...string) (*bytes.Buffer, error) {
func git(ctx context.Context, dir, sshAuthSock, knownHosts string, args ...string) (*bytes.Buffer, error) {
for {
stdout, stderr := logs.NewLogStreams(ctx, false)
defer stdout.Close()
Expand All @@ -484,6 +594,13 @@ func git(ctx context.Context, dir string, args ...string) (*bytes.Buffer, error)
"GIT_TERMINAL_PROMPT=0",
// "GIT_TRACE=1",
}
if sshAuthSock != "" {
cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+sshAuthSock)
}
if knownHosts != "" {
cmd.Env = append(cmd.Env, "GIT_SSH_COMMAND=ssh -o UserKnownHostsFile="+knownHosts)
}

// remote git commands spawn helper processes that inherit FDs and don't
// handle parent death signal so exec.CommandContext can't be used
err := runProcessGroup(ctx, cmd)
Expand Down
Loading

0 comments on commit e186e4a

Please sign in to comment.