Update source-controller

- Panic recovery for Git operations.
- Improved SSH connection management without use of caching.
- Enforce context timeout for managed SSH.
- Remove dependency to callback functions.
- Add support for hashed known_hosts.

Signed-off-by: Paulo Gomes <paulo.gomes@weave.works>

Dockerfile

@@ -72,6 +72,7 @@ WORKDIR /workspace
 COPY main.go main.go
 COPY controllers/ controllers/
 COPY pkg/ pkg/
+COPY internal/ internal/
 
 COPY --from=musl-tool-chain /workspace/build /workspace/build
 

controllers/imageupdateautomation_controller.go

@@ -250,31 +250,20 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr
  return failWithError(err)
  }
 
- repositoryURL := origin.Spec.URL
+ // managed GIT transport only affects the libgit2 implementation
  if managed.Enabled() {
- // At present only HTTP connections have the ability to define remote options.
- // Although this can be easily extended by ensuring that the fake URL below uses the
- // target ssh scheme, and the libgit2/managed/ssh.go pulls that information accordingly.
- //
- // This is due to the fact the key libgit2 remote callbacks do not take place for HTTP
- // whilst most still work for SSH.
- if strings.HasPrefix(repositoryURL, "http") {
- if access.auth != nil && len(access.auth.CAFile) > 0 {
- // Due to the lack of the callback feature, a fake target URL is created to allow
- // for the smart sub transport be able to pick the options specific for this
- // GitRepository object.
- // The URL should use unique information that do not collide in a multi tenant
- // deployment.
- repositoryURL = fmt.Sprintf("http://%s/%s/%d", auto.Name, auto.UID, auto.Generation)
- managed.AddTransportOptions(repositoryURL,
- managed.TransportOptions{
- TargetURL: repositoryURL,
- CABundle: access.auth.CAFile,
- })
-
- // We remove the options from memory, to avoid accumulating unused options over time.
- defer managed.RemoveTransportOptions(repositoryURL)
- }
+ // We set the TransportOptionsURL of this set of authentication options here by constructing
+ // a unique URL that won't clash in a multi tenant environment. This unique URL is used by
+ // libgit2 managed transports. This enables us to bypass the inbuilt credentials callback in
+ // libgit2, which is inflexible and unstable.
+ // NB: The Transport Options URL must be unique, therefore it must use the object under
+ // reconciliation details, instead of the repository it depends on.
+ if strings.HasPrefix(origin.Spec.URL, "http") {
+ access.auth.TransportOptionsURL = fmt.Sprintf("http://%s/%s/%d", auto.Name, auto.UID, auto.Generation)
+ } else if strings.HasPrefix(origin.Spec.URL, "ssh") {
+ access.auth.TransportOptionsURL = fmt.Sprintf("ssh://%s/%s/%d", auto.Name, auto.UID, auto.Generation)
+ } else {
+ return failWithError(fmt.Errorf("git repository URL '%s' has invalid transport type, supported types are: http, https, ssh", origin.Spec.URL))
  }
  }
 
@@ -287,6 +276,20 @@ func (r *ImageUpdateAutomationReconciler) Reconcile(ctx context.Context, req ctr
  }
  defer repo.Free()
 
+ if managed.Enabled() {
+ // Checkout removes TransportOptions before returning, therefore this
+ // must happen after cloneInto.
+ // TODO(pjbgf): Git consolidation should improve the API workflow.
+ managed.AddTransportOptions(access.auth.TransportOptionsURL, managed.TransportOptions{
+ TargetURL: origin.Spec.URL,
+ AuthOpts: access.auth,
+ ProxyOptions: &libgit2.ProxyOptions{Type: libgit2.ProxyTypeAuto},
+ Context: cloneCtx,
+ })
+
+ defer managed.RemoveTransportOptions(access.auth.TransportOptionsURL)
+ }
+
  // When there's a push spec, the pushed-to branch is where commits
  // shall be made
 
@@ -806,6 +809,11 @@ func push(ctx context.Context, path, branch string, access repoAccess) error {
  defer origin.Free()
 
  callbacks := access.remoteCallbacks(ctx)
+ if managed.Enabled() {
+ // Override callbacks with dummy ones as they are not needed within Managed Transport.
+ // However, not setting them may lead to git2go panicing.
+ callbacks = managed.RemoteCallbacks()
+ }
 
  // calling repo.Push will succeed even if a reference update is
  // rejected; to detect this case, this callback is supplied.

controllers/update_test.go

@@ -1593,7 +1593,7 @@ func createSSHIdentitySecret(kClient client.Client, name, namespace, repoURL str
  if err != nil {
  return err
  }
- knownhosts, err := ssh.ScanHostKey(url.Host, 5*time.Second)
+ knownhosts, err := ssh.ScanHostKey(url.Host, 5*time.Second, []string{}, false)
  if err != nil {
  return err
  }

go.mod

@@ -6,33 +6,33 @@ replace github.com/fluxcd/image-automation-controller/api => ./api
 
 require (
  github.com/Masterminds/sprig/v3 v3.2.2
- github.com/ProtonMail/go-crypto v0.0.0-20220407094043-a94812496cf5
+ github.com/ProtonMail/go-crypto v0.0.0-20220517143526-88bb52951d5b
  github.com/cyphar/filepath-securejoin v0.2.3
  github.com/fluxcd/image-automation-controller/api v0.22.1
  github.com/fluxcd/image-reflector-controller/api v0.18.0
  github.com/fluxcd/pkg/apis/acl v0.0.3
- github.com/fluxcd/pkg/apis/meta v0.13.0
- github.com/fluxcd/pkg/gittestserver v0.5.2
- github.com/fluxcd/pkg/runtime v0.15.1
- github.com/fluxcd/pkg/ssh v0.3.2
- github.com/fluxcd/source-controller v0.24.4
+ github.com/fluxcd/pkg/apis/meta v0.14.1
+ github.com/fluxcd/pkg/gittestserver v0.5.3
+ github.com/fluxcd/pkg/runtime v0.16.1
+ github.com/fluxcd/pkg/ssh v0.4.1
+ github.com/fluxcd/source-controller v0.24.5-0.20220527125950-978148ea7139
  github.com/fluxcd/source-controller/api v0.24.4
  github.com/go-logr/logr v1.2.3
  github.com/google/go-containerregistry v0.8.0
  github.com/libgit2/git2go/v33 v33.0.9
  github.com/onsi/gomega v1.19.0
  github.com/otiai10/copy v1.7.0
  github.com/spf13/pflag v1.0.5
- k8s.io/api v0.23.6
- k8s.io/apimachinery v0.23.6
- k8s.io/client-go v0.23.6
- k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf
+ k8s.io/api v0.24.0
+ k8s.io/apimachinery v0.24.0
+ k8s.io/client-go v0.24.0
+ k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42
  sigs.k8s.io/controller-runtime v0.11.2
  sigs.k8s.io/kustomize/kyaml v0.13.6
 )
 
 require (
- cloud.google.com/go/compute v1.6.0 // indirect
+ cloud.google.com/go/compute v1.6.1 // indirect
  github.com/Masterminds/goutils v1.1.1 // indirect
  github.com/Masterminds/semver/v3 v3.1.1 // indirect
  github.com/Microsoft/go-winio v0.5.2 // indirect
@@ -43,8 +43,10 @@ require (
  github.com/beorn7/perks v1.0.1 // indirect
  github.com/cespare/xxhash/v2 v2.1.2 // indirect
  github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/emirpasic/gods v1.12.0 // indirect
+ github.com/emicklei/go-restful v2.9.5+incompatible // indirect
+ github.com/emirpasic/gods v1.18.1 // indirect
  github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+ github.com/fluxcd/gitkit v0.5.1 // indirect
  github.com/fluxcd/pkg/gitutil v0.1.0 // indirect
  github.com/fluxcd/pkg/version v0.1.0 // indirect
  github.com/fsnotify/fsnotify v1.5.1 // indirect
@@ -60,19 +62,19 @@ require (
  github.com/gogo/protobuf v1.3.2 // indirect
  github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
  github.com/golang/protobuf v1.5.2 // indirect
- github.com/google/go-cmp v0.5.7 // indirect
+ github.com/google/gnostic v0.5.7-v3refs // indirect
+ github.com/google/go-cmp v0.5.8 // indirect
  github.com/google/gofuzz v1.2.0 // indirect
  github.com/google/uuid v1.3.0 // indirect
- github.com/googleapis/gnostic v0.5.5 // indirect
  github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
- github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
+ github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
  github.com/huandu/xstrings v1.3.2 // indirect
  github.com/imdario/mergo v0.3.12 // indirect
  github.com/inconshreveable/mousetrap v1.0.0 // indirect
  github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
  github.com/josharian/intern v1.0.0 // indirect
  github.com/json-iterator/go v1.1.12 // indirect
- github.com/kevinburke/ssh_config v1.1.0 // indirect
+ github.com/kevinburke/ssh_config v1.2.0 // indirect
  github.com/mailru/easyjson v0.7.6 // indirect
  github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
  github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -82,15 +84,15 @@ require (
  github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
  github.com/modern-go/reflect2 v1.0.2 // indirect
  github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
  github.com/pkg/errors v0.9.1 // indirect
  github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/prometheus/client_golang v1.12.1 // indirect
+ github.com/prometheus/client_golang v1.12.2 // indirect
  github.com/prometheus/client_model v0.2.0 // indirect
  github.com/prometheus/common v0.32.1 // indirect
  github.com/prometheus/procfs v0.7.3 // indirect
  github.com/sergi/go-diff v1.2.0 // indirect
  github.com/shopspring/decimal v1.2.0 // indirect
- github.com/sosedoff/gitkit v0.3.0 // indirect
  github.com/spf13/cast v1.4.1 // indirect
  github.com/spf13/cobra v1.4.0 // indirect
  github.com/stretchr/testify v1.7.1 // indirect
@@ -99,10 +101,10 @@ require (
  go.uber.org/atomic v1.7.0 // indirect
  go.uber.org/multierr v1.6.0 // indirect
  go.uber.org/zap v1.21.0 // indirect
- golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f
- golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
+ golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898
+ golang.org/x/net v0.0.0-20220524220425-1d687d428aca // indirect
  golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
- golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
+ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
  golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
  golang.org/x/text v0.3.7 // indirect
  golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
@@ -113,10 +115,11 @@ require (
  gopkg.in/warnings.v0 v0.1.2 // indirect
  gopkg.in/yaml.v2 v2.4.0 // indirect
  gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
- k8s.io/apiextensions-apiserver v0.23.5 // indirect
- k8s.io/component-base v0.23.5 // indirect
- k8s.io/klog/v2 v2.50.0 // indirect
+ k8s.io/apiextensions-apiserver v0.24.0 // indirect
+ k8s.io/component-base v0.24.0 // indirect
+ k8s.io/klog/v2 v2.60.1 // indirect
  k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
+ sigs.k8s.io/cli-utils v0.31.1 // indirect
  sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
  sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
  sigs.k8s.io/yaml v1.3.0 // indirect