Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

controllers: Git checkout strategies and in-memory known hosts db #52

Merged
merged 6 commits into from
May 3, 2020
Merged

controllers: Git checkout strategies and in-memory known hosts db #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
48fcc52
controllers: factor out GitRepository verification
hiddeco May 2, 2020
d048730
git/ssh: use in-memory known hosts database
hiddeco May 2, 2020
e42561f
git: refactor transport into strategies
hiddeco May 2, 2020
40b1369
git: add checkout strategies
hiddeco May 2, 2020
9c67baa
controllers: implement checkout strategies
hiddeco May 3, 2020
078d005
build: drop openssh-client
hiddeco May 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o source-c

FROM alpine:3.11

RUN apk add --no-cache openssh-client ca-certificates tar tini 'git>=2.12.0' socat curl bash
RUN apk add --no-cache ca-certificates tar tini 'git>=2.12.0' socat curl bash

COPY --from=builder /workspace/source-controller /usr/local/bin/

Expand Down
214 changes: 39 additions & 175 deletions controllers/gitrepository_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@ import (
"io/ioutil"
"os"

"github.com/blang/semver"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -121,34 +119,18 @@ func (r *GitRepositoryReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, o
}

func (r *GitRepositoryReconciler) sync(ctx context.Context, repository sourcev1.GitRepository) (sourcev1.GitRepository, error) {
// set defaults: master branch, no tags fetching, max two commits
branch := "master"
revision := ""
tagMode := git.NoTags
depth := 2

// determine ref
refName := plumbing.NewBranchReferenceName(branch)
if repository.Spec.Reference != nil {
if repository.Spec.Reference.Branch != "" {
branch = repository.Spec.Reference.Branch
refName = plumbing.NewBranchReferenceName(branch)
}
if repository.Spec.Reference.Commit != "" {
depth = 0
} else {
if repository.Spec.Reference.Tag != "" {
refName = plumbing.NewTagReferenceName(repository.Spec.Reference.Tag)
}
if repository.Spec.Reference.SemVer != "" {
tagMode = git.AllTags
}
}
// create tmp dir for the Git clone
tmpGit, err := ioutil.TempDir("", repository.Name)
if err != nil {
err = fmt.Errorf("tmp dir error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
}
defer os.RemoveAll(tmpGit)

// determine auth method
var auth transport.AuthMethod
if repository.Spec.SecretRef != nil {
authStrategy := intgit.AuthSecretStrategyForURL(repository.Spec.URL)
if repository.Spec.SecretRef != nil && authStrategy != nil {
name := types.NamespacedName{
Namespace: repository.GetNamespace(),
Name: repository.Spec.SecretRef.Name,
Expand All @@ -161,173 +143,32 @@ func (r *GitRepositoryReconciler) sync(ctx context.Context, repository sourcev1.
return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
}

method, cleanup, err := intgit.AuthMethodFromSecret(repository.Spec.URL, secret)
auth, err = authStrategy.Method(secret)
if err != nil {
err = fmt.Errorf("auth error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.AuthenticationFailedReason, err.Error()), err
}
if cleanup != nil {
defer cleanup()
}
auth = method
}

// create tmp dir for the Git clone
tmpGit, err := ioutil.TempDir("", repository.Name)
checkoutStrategy := intgit.CheckoutStrategyForRef(repository.Spec.Reference)
commit, revision, err := checkoutStrategy.Checkout(ctx, tmpGit, repository.Spec.URL, auth)
if err != nil {
err = fmt.Errorf("tmp dir error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err
}
defer os.RemoveAll(tmpGit)

// clone to tmp
gitCtx, cancel := context.WithTimeout(ctx, repository.GetTimeout())
repo, err := git.PlainCloneContext(gitCtx, tmpGit, false, &git.CloneOptions{
URL: repository.Spec.URL,
Auth: auth,
RemoteName: "origin",
ReferenceName: refName,
SingleBranch: true,
NoCheckout: false,
Depth: depth,
RecurseSubmodules: 0,
Progress: nil,
Tags: tagMode,
})
cancel()
if err != nil {
err = fmt.Errorf("git clone error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}

// checkout commit or tag
if repository.Spec.Reference != nil {
if commit := repository.Spec.Reference.Commit; commit != "" {
w, err := repo.Worktree()
if err != nil {
err = fmt.Errorf("git worktree error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}

err = w.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(commit),
Force: true,
})
if err != nil {
err = fmt.Errorf("git checkout '%s' for '%s' error: %w", commit, branch, err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
} else if exp := repository.Spec.Reference.SemVer; exp != "" {
rng, err := semver.ParseRange(exp)
if err != nil {
err = fmt.Errorf("semver parse range error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}

repoTags, err := repo.Tags()
if err != nil {
err = fmt.Errorf("git list tags error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}

tags := make(map[string]string)
_ = repoTags.ForEach(func(t *plumbing.Reference) error {
tags[t.Name().Short()] = t.Strings()[1]
return nil
})

svTags := make(map[string]string)
var svers []semver.Version
for tag, _ := range tags {
v, _ := semver.ParseTolerant(tag)
if rng(v) {
svers = append(svers, v)
svTags[v.String()] = tag
}
}

if len(svers) > 0 {
semver.Sort(svers)
v := svers[len(svers)-1]
t := svTags[v.String()]
commit := tags[t]
revision = fmt.Sprintf("%s/%s", t, commit)

w, err := repo.Worktree()
if err != nil {
err = fmt.Errorf("git worktree error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}

err = w.Checkout(&git.CheckoutOptions{
Hash: plumbing.NewHash(commit),
})
if err != nil {
err = fmt.Errorf("git checkout error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
} else {
err = fmt.Errorf("no match found for semver: %s", repository.Spec.Reference.SemVer)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}
}
}

// read commit hash
ref, err := repo.Head()
if err != nil {
err = fmt.Errorf("git resolve HEAD error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}

// verify PGP signature
if repository.Spec.Verification != nil {
commit, err := repo.CommitObject(ref.Hash())
if err != nil {
err = fmt.Errorf("git resolve HEAD error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.GitOperationFailedReason, err.Error()), err
}

if commit.PGPSignature == "" {
err = fmt.Errorf("PGP signature not found for commit '%s'", ref.Hash())
return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
}

name := types.NamespacedName{
Namespace: repository.GetNamespace(),
err := r.verify(ctx, types.NamespacedName{
Namespace: repository.Namespace,
Name: repository.Spec.Verification.SecretRef.Name,
}

var secret corev1.Secret
err = r.Client.Get(ctx, name, &secret)
}, commit)
if err != nil {
err = fmt.Errorf("PGP public keys secret error: %w", err)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
}

var verified bool
for _, bytes := range secret.Data {
if _, err := commit.Verify(string(bytes)); err == nil {
verified = true
break
}
}

if !verified {
err = fmt.Errorf("PGP signature of '%s' can't be verified", commit.Author)
return sourcev1.GitRepositoryNotReady(repository, sourcev1.VerificationFailedReason, err.Error()), err
}
}

if revision == "" {
revision = fmt.Sprintf("%s/%s", branch, ref.Hash().String())
if repository.Spec.Reference != nil && repository.Spec.Reference.Tag != "" {
revision = fmt.Sprintf("%s/%s", repository.Spec.Reference.Tag, ref.Hash().String())
}
}

artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(),
fmt.Sprintf("%s.tar.gz", ref.Hash().String()), revision)
fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision)

// create artifact dir
err = r.Storage.MkdirAll(artifact)
Expand Down Expand Up @@ -388,6 +229,29 @@ func (r *GitRepositoryReconciler) shouldResetStatus(repository sourcev1.GitRepos
}
}

func (r *GitRepositoryReconciler) verify(ctx context.Context, publicKeySecret types.NamespacedName, commit *object.Commit) error {
if commit.PGPSignature == "" {
return fmt.Errorf("no PGP signature found for commit: %s", commit.Hash)
}

var secret corev1.Secret
if err := r.Client.Get(ctx, publicKeySecret, &secret); err != nil {
return fmt.Errorf("PGP public keys secret error: %w", err)
}

var verified bool
for _, bytes := range secret.Data {
if _, err := commit.Verify(string(bytes)); err == nil {
verified = true
break
}
}
if !verified {
return fmt.Errorf("PGP signature '%s' of '%s' can't be verified", commit.PGPSignature, commit.Author)
}
return nil
}

// gc performs a garbage collection on all but current artifacts of
// the given repository.
func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository) error {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ require (
github.com/go-logr/logr v0.1.0
github.com/onsi/ginkgo v1.11.0
github.com/onsi/gomega v1.8.1
github.com/pkg/errors v0.9.1
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073
helm.sh/helm/v3 v3.1.2
k8s.io/api v0.17.2
k8s.io/apimachinery v0.17.2
Expand Down
Loading