From 3d276b679b3a21b66dcadd30b00c96b03c91ad34 Mon Sep 17 00:00:00 2001 From: Robert Clarke Date: Tue, 18 Jan 2022 19:35:47 +0000 Subject: [PATCH] libgit2: Configured libgit2 clone ProxyOptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This configures ProxyOptions for all libgit2 Checkout functions when cloning and configures the options based on current environment settings using the git2go.ProxyTypeAuto option. Refs: #131 Signed-off-by: Robert Clarke Co-authored-by: Aurélien GARNIER --- go.mod | 1 + go.sum | 6 +- pkg/git/libgit2/checkout.go | 4 + pkg/git/strategy/proxy/strategy_proxy_test.go | 290 ++++++++++++++++++ 4 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 pkg/git/strategy/proxy/strategy_proxy_test.go diff --git a/go.mod b/go.mod index 9db5e8b0e..1fe12d2d4 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 github.com/cyphar/filepath-securejoin v0.2.2 + github.com/elazarl/goproxy v0.0.0-20211114080932-d06c3be7c11b github.com/fluxcd/pkg/apis/meta v0.10.2 github.com/fluxcd/pkg/gittestserver v0.5.0 github.com/fluxcd/pkg/gitutil v0.1.0 diff --git a/go.sum b/go.sum index a362ff12c..69b819334 100644 --- a/go.sum +++ b/go.sum @@ -267,8 +267,11 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20211114080932-d06c3be7c11b h1:1XqENn2YoYZd6w3Awx+7oa+aR87DFIZJFLF2n1IojA0= +github.com/elazarl/goproxy v0.0.0-20211114080932-d06c3be7c11b/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= @@ -806,6 +809,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= diff --git a/pkg/git/libgit2/checkout.go b/pkg/git/libgit2/checkout.go index 09d74a10c..60b2830eb 100644 --- a/pkg/git/libgit2/checkout.go +++ b/pkg/git/libgit2/checkout.go @@ -64,6 +64,7 @@ func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *g FetchOptions: &git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsNone, RemoteCallbacks: RemoteCallbacks(ctx, opts), + ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, }, CheckoutBranch: c.Branch, }) @@ -93,6 +94,7 @@ func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git. FetchOptions: &git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsAll, RemoteCallbacks: RemoteCallbacks(ctx, opts), + ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, }, }) if err != nil { @@ -116,6 +118,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *g FetchOptions: &git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsNone, RemoteCallbacks: RemoteCallbacks(ctx, opts), + ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, }, }) if err != nil { @@ -147,6 +150,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *g FetchOptions: &git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsAll, RemoteCallbacks: RemoteCallbacks(ctx, opts), + ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, }, }) if err != nil { diff --git a/pkg/git/strategy/proxy/strategy_proxy_test.go b/pkg/git/strategy/proxy/strategy_proxy_test.go new file mode 100644 index 000000000..e61dfa921 --- /dev/null +++ b/pkg/git/strategy/proxy/strategy_proxy_test.go @@ -0,0 +1,290 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/elazarl/goproxy" + "github.com/fluxcd/pkg/gittestserver" + . "github.com/onsi/gomega" + + "github.com/fluxcd/source-controller/pkg/git" + "github.com/fluxcd/source-controller/pkg/git/gogit" + "github.com/fluxcd/source-controller/pkg/git/libgit2" + "github.com/fluxcd/source-controller/pkg/git/strategy" +) + +// These tests are run in a different _test.go file because go-git uses the ProxyFromEnvironment function of the net/http package +// which caches the Proxy settings, hence not including other tests in the same file ensures a clean proxy setup for the tests to run. +func TestCheckoutStrategyForImplementation_Proxied(t *testing.T) { + + type cleanupFunc func() + + type testCase struct { + name string + gitImpl git.Implementation + url string + branch string + setupGitProxy func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) + shortTimeout bool + wantUsedProxy bool + wantError bool + } + + g := NewWithT(t) + + // Get a free port for proxy to use. + l, err := net.Listen("tcp", ":0") + g.Expect(err).ToNot(HaveOccurred()) + proxyAddr := fmt.Sprintf("localhost:%d", l.Addr().(*net.TCPAddr).Port) + g.Expect(l.Close()).ToNot(HaveOccurred()) + + // Note there is no libgit2 HTTP_PROXY test as libgit2 doesnt support proxied HTTP requests. + cases := []testCase{ + { + name: "libgit2_HTTPS_PROXY", + gitImpl: libgit2.Implementation, + url: "https://example.com/bar/test-reponame", + branch: "main", + setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) { + // Create the git server. + gitServer, err := gittestserver.NewTempGitServer() + g.Expect(err).ToNot(HaveOccurred()) + + username := "test-user" + password := "test-password" + gitServer.Auth(username, password) + gitServer.KeyDir(gitServer.Root()) + + // Start the HTTPS server. + examplePublicKey, err := os.ReadFile("../testdata/certs/server.pem") + g.Expect(err).ToNot(HaveOccurred()) + examplePrivateKey, err := os.ReadFile("../testdata/certs/server-key.pem") + g.Expect(err).ToNot(HaveOccurred()) + exampleCA, err := os.ReadFile("../testdata/certs/ca.pem") + g.Expect(err).ToNot(HaveOccurred()) + err = gitServer.StartHTTPS(examplePublicKey, examplePrivateKey, exampleCA, "example.com") + g.Expect(err).ToNot(HaveOccurred()) + + // Initialize a git repo. + repoPath := "bar/test-reponame" + err = gitServer.InitRepo("../testdata/repo1", "main", repoPath) + g.Expect(err).ToNot(HaveOccurred()) + + u, err := url.Parse(gitServer.HTTPAddress()) + g.Expect(err).ToNot(HaveOccurred()) + + // The request is being forwarded to the local test git server in this handler. + // The certificate used here is valid for both example.com and localhost. + var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + // Check if the host matches with the git server address and the user-agent is the expected git client. + userAgent := ctx.Req.Header.Get("User-Agent") + if strings.Contains(host, "example.com") && strings.Contains(userAgent, "libgit2") { + *proxyGotRequest = true + return goproxy.OkConnect, u.Host + } + // Reject if it isn't our request. + return goproxy.RejectConnect, host + } + proxy.OnRequest().HandleConnect(proxyHandler) + + return &git.AuthOptions{ + Transport: git.HTTPS, + Username: username, + Password: password, + CAFile: exampleCA, + }, func() { + os.RemoveAll(gitServer.Root()) + gitServer.StopHTTP() + } + }, + shortTimeout: false, + wantUsedProxy: true, + wantError: false, + }, + { + name: "gogit_HTTP_PROXY", + gitImpl: gogit.Implementation, + url: "http://example.com/bar/test-reponame", + branch: "main", + setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) { + // Create the git server. + gitServer, err := gittestserver.NewTempGitServer() + g.Expect(err).ToNot(HaveOccurred()) + + username := "test-user" + password := "test-password" + gitServer.Auth(username, password) + gitServer.KeyDir(gitServer.Root()) + + g.Expect(gitServer.StartHTTP()).ToNot(HaveOccurred()) + + // Initialize a git repo. + err = gitServer.InitRepo("../testdata/repo1", "main", "bar/test-reponame") + g.Expect(err).ToNot(HaveOccurred()) + + u, err := url.Parse(gitServer.HTTPAddress()) + g.Expect(err).ToNot(HaveOccurred()) + + // The request is being forwarded to the local test git server in this handler. + var proxyHandler goproxy.FuncReqHandler = func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + userAgent := req.Header.Get("User-Agent") + if strings.Contains(req.Host, "example.com") && strings.Contains(userAgent, "git") { + *proxyGotRequest = true + req.Host = u.Host + req.URL.Host = req.Host + return req, nil + } + // Reject if it isnt our request. + return req, goproxy.NewResponse(req, goproxy.ContentTypeText, http.StatusForbidden, "") + } + proxy.OnRequest().Do(proxyHandler) + + return &git.AuthOptions{ + Transport: git.HTTP, + Username: username, + Password: password, + }, func() { + os.RemoveAll(gitServer.Root()) + gitServer.StopHTTP() + } + }, + shortTimeout: false, + wantUsedProxy: true, + wantError: false, + }, + { + name: "gogit_HTTPS_PROXY", + gitImpl: gogit.Implementation, + url: "https://github.com/git-fixtures/basic", + branch: "master", + setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) { + var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + // We don't check for user agent as this handler is only going to process CONNECT requests, and because Go's net/http + // is the one making such a request on behalf of go-git, adding a check for the go net/http user agent (Go-http-client) + // would only allow false positives from any request originating from Go's net/http. + if strings.Contains(host, "github.com") { + *proxyGotRequest = true + return goproxy.OkConnect, host + } + // Reject if it isnt our request. + return goproxy.RejectConnect, host + } + proxy.OnRequest().HandleConnect(proxyHandler) + + // go-git does not allow to use an HTTPS proxy and a custom root CA at the same time. + // See https://github.com/fluxcd/source-controller/pull/524#issuecomment-1006673163. + return nil, func() {} + }, + shortTimeout: false, + wantUsedProxy: true, + wantError: false, + }, + { + name: "gogit_NO_PROXY", + gitImpl: gogit.Implementation, + url: "https://192.0.2.1/bar/test-reponame", + branch: "main", + setupGitProxy: func(g *WithT, proxy *goproxy.ProxyHttpServer, proxyGotRequest *bool) (*git.AuthOptions, cleanupFunc) { + var proxyHandler goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + // We shouldn't hit the proxy so we just want to check for any interaction, then reject. + *proxyGotRequest = true + return goproxy.RejectConnect, host + } + proxy.OnRequest().HandleConnect(proxyHandler) + + return nil, func() {} + }, + shortTimeout: true, + wantUsedProxy: false, + wantError: true, + }, + // TODO: Add a NO_PROXY test for libgit2 once the version of libgit2 used by the source controller is updated to a version that includes + // the NO_PROXY functionality + // This PR introduces the functionality in libgit2: https://github.com/libgit2/libgit2/pull/6026 + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + // Run a proxy server. + proxy := goproxy.NewProxyHttpServer() + proxy.Verbose = true + + proxyGotRequest := false + authOpts, cleanup := tt.setupGitProxy(g, proxy, &proxyGotRequest) + defer cleanup() + + proxyServer := http.Server{ + Addr: proxyAddr, + Handler: proxy, + } + l, err := net.Listen("tcp", proxyServer.Addr) + g.Expect(err).ToNot(HaveOccurred()) + go proxyServer.Serve(l) + defer proxyServer.Close() + + // Set the proxy env vars for both HTTP and HTTPS because go-git caches them. + os.Setenv("HTTPS_PROXY", fmt.Sprintf("http://%s", proxyAddr)) + defer os.Unsetenv("HTTPS_PROXY") + + os.Setenv("HTTP_PROXY", fmt.Sprintf("http://%s", proxyAddr)) + defer os.Unsetenv("HTTP_PROXY") + + os.Setenv("NO_PROXY", "*.0.2.1") + defer os.Unsetenv("NO_PROXY") + + // Checkout the repo. + checkoutStrategy, err := strategy.CheckoutStrategyForImplementation(context.TODO(), tt.gitImpl, git.CheckoutOptions{ + Branch: tt.branch, + }) + g.Expect(err).ToNot(HaveOccurred()) + + tmpDir, err := os.MkdirTemp("", "test-checkout") + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(tmpDir) + + // for the NO_PROXY test we dont want to wait the 30s for it to timeout/fail, so shorten the timeout + checkoutCtx := context.TODO() + if tt.shortTimeout { + var cancel context.CancelFunc + checkoutCtx, cancel = context.WithTimeout(context.TODO(), 1*time.Second) + defer cancel() + } + + _, err = checkoutStrategy.Checkout(checkoutCtx, tmpDir, tt.url, authOpts) + if tt.wantError { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + + g.Expect(proxyGotRequest).To(Equal(tt.wantUsedProxy)) + + }) + } +}