From 6d35d54228caaa773ec0332cef3805fa6180979f Mon Sep 17 00:00:00 2001 From: Martin Kunc <5441732+martinkunc@users.noreply.github.com> Date: Thu, 19 Mar 2020 14:42:41 +0100 Subject: [PATCH 1/6] Support for specific http proxy for the service --- go.mod | 2 +- .../clusterauthorizer/clusterauthorizer.go | 6 +++ pkg/config/config.go | 9 ++++ pkg/config/configobserver/configobserver.go | 10 ++++ pkg/insights/insightsclient/insightsclient.go | 48 +++++++++++++++++-- 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index bee9d54b1..8944567e0 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect - golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect + golang.org/x/net v0.0.0-20200301022130-244492dfa37a golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 google.golang.org/appengine v1.6.1 // indirect k8s.io/api v0.17.1 diff --git a/pkg/authorizer/clusterauthorizer/clusterauthorizer.go b/pkg/authorizer/clusterauthorizer/clusterauthorizer.go index 5fd449d5b..662e25fdc 100644 --- a/pkg/authorizer/clusterauthorizer/clusterauthorizer.go +++ b/pkg/authorizer/clusterauthorizer/clusterauthorizer.go @@ -44,3 +44,9 @@ func (a *Authorizer) Authorize(req *http.Request) error { } return nil } + +// HTTPConfig provides the Http Proxy settings from Secret +func (a *Authorizer) HTTPConfig() config.HTTPConfig { + cfg := a.configurator.Config() + return cfg.HTTPConfig +} diff --git a/pkg/config/config.go b/pkg/config/config.go index e63d40bb0..a5c3d1e69 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -49,4 +49,13 @@ type Controller struct { Username string Password string Token string + + HTTPConfig HTTPConfig +} + +// HTTPConfig configures http proxy and exception settings if they come from config +type HTTPConfig struct { + HTTPProxy string + HTTPSProxy string + NoProxy string } diff --git a/pkg/config/configobserver/configobserver.go b/pkg/config/configobserver/configobserver.go index a929a6da7..4229558d3 100644 --- a/pkg/config/configobserver/configobserver.go +++ b/pkg/config/configobserver/configobserver.go @@ -130,6 +130,15 @@ func (c *Controller) retrieveConfig() error { if endpoint, ok := secret.Data["endpoint"]; ok { nextConfig.Endpoint = string(endpoint) } + if httpproxy, ok := secret.Data["httpProxy"]; ok { + nextConfig.HTTPConfig.HTTPProxy = string(httpproxy) + } + if httpsproxy, ok := secret.Data["httpsProxy"]; ok { + nextConfig.HTTPConfig.HTTPSProxy = string(httpsproxy) + } + if noproxy, ok := secret.Data["noProxy"]; ok { + nextConfig.HTTPConfig.NoProxy = string(noproxy) + } nextConfig.Report = len(nextConfig.Endpoint) > 0 if intervalString, ok := secret.Data["interval"]; ok { @@ -207,6 +216,7 @@ func (c *Controller) mergeConfigLocked() { if len(c.secretConfig.Endpoint) > 0 { cfg.Endpoint = c.secretConfig.Endpoint } + cfg.HTTPConfig = c.secretConfig.HTTPConfig } if c.tokenConfig != nil { cfg.Token = c.tokenConfig.Token diff --git a/pkg/insights/insightsclient/insightsclient.go b/pkg/insights/insightsclient/insightsclient.go index bb342c159..e50f52447 100644 --- a/pkg/insights/insightsclient/insightsclient.go +++ b/pkg/insights/insightsclient/insightsclient.go @@ -12,10 +12,12 @@ import ( "net" "net/http" "net/textproto" + "net/url" "os" "strconv" "time" + "golang.org/x/net/http/httpproxy" knet "k8s.io/apimachinery/pkg/util/net" "k8s.io/client-go/pkg/version" "k8s.io/client-go/transport" @@ -27,6 +29,7 @@ import ( configv1 "github.com/openshift/api/config/v1" "github.com/openshift/insights-operator/pkg/authorizer" + "github.com/openshift/insights-operator/pkg/config" ) type Client struct { @@ -40,6 +43,7 @@ type Client struct { type Authorizer interface { Authorize(req *http.Request) error + HTTPConfig() config.HTTPConfig } type ClusterVersionInfo interface { @@ -88,10 +92,11 @@ func getTrustedCABundle() (*x509.CertPool, error) { return certs, nil } -func clientTransport() http.RoundTripper { - // default transport, proxy from env +// clientTransport creates new http.Transport based on httpConfig if used, or Env +func clientTransport(httpConfig config.HTTPConfig) http.RoundTripper { + // default transport, proxy from configmap or env clientTransport := &http.Transport{ - Proxy: knet.NewProxierWithNoProxyCIDR(http.ProxyFromEnvironment), + Proxy: NewProxier(knet.NewProxierWithNoProxyCIDR(http.ProxyFromEnvironment), FromConfig(httpConfig)), DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, @@ -113,6 +118,39 @@ func clientTransport() http.RoundTripper { return transport.DebugWrappers(clientTransport) } +// ConfigProxier is creating a Proxier from proxy set in HttpConfig +func ConfigProxier(c config.HTTPConfig) func(req *http.Request) (*url.URL, error) { + proxyConfig := httpproxy.Config{ + HTTPProxy: c.HTTPProxy, + HTTPSProxy: c.HTTPSProxy, + NoProxy: c.NoProxy, + } + // The golang ProxyFunc seems to have NoProxy already built in + return func(req *http.Request) (*url.URL, error) { + return proxyConfig.ProxyFunc()(req.URL) + } +} + +// FromConfig is setting HttpProxy from HttpConfig in support secret, if it is used +func FromConfig(c config.HTTPConfig) func(req *http.Request) (*url.URL, error) { + if len(c.HTTPProxy) > 0 || len(c.HTTPSProxy) > 0 || len(c.NoProxy) > 0 { + return ConfigProxier(c) + } + return nil +} + +// NewProxier will create new http.Proxier function. +// If any of customProxiers is set, it will use it, otherwise will use system +func NewProxier(system func(req *http.Request) (*url.URL, error), customProxiers ...func(req *http.Request) (*url.URL, error)) func(req *http.Request) (*url.URL, error) { + for _, p := range customProxiers { + if p != nil { + return p + } + } + + return system +} + func (c *Client) Send(ctx context.Context, endpoint string, source Source) error { cv := c.clusterInfo.ClusterVersion() if cv == nil { @@ -158,14 +196,14 @@ func (c *Client) Send(ctx context.Context, endpoint string, source Source) error req.Body = pr // dynamically set the proxy environment - c.client.Transport = clientTransport() + c.client.Transport = clientTransport(c.authorizer.HTTPConfig()) klog.V(4).Infof("Uploading %s to %s", source.Type, req.URL.String()) resp, err := c.client.Do(req) if err != nil { klog.V(4).Infof("Unable to build a request, possible invalid token: %v", err) // if the request is not build, for example because of invalid endpoint,(maybe some problem with DNS), we want to have record about it in metrics as well. - counterRequestSend.WithLabelValues(c.metricsName, "0").Inc() + counterRequestSend.WithLabelValues(c.metricsName, "0").Inc() return fmt.Errorf("unable to build request to connect to Insights server: %v", err) } From f77afd7b34c0d2bf38fedea95156ef8e42883680 Mon Sep 17 00:00:00 2001 From: Martin Kunc <5441732+martinkunc@users.noreply.github.com> Date: Fri, 20 Mar 2020 07:39:34 +0100 Subject: [PATCH 2/6] Updating vendor folder --- .../golang.org/x/net/http/httpproxy/proxy.go | 370 ++++++++++++++++++ vendor/modules.txt | 1 + 2 files changed, 371 insertions(+) create mode 100644 vendor/golang.org/x/net/http/httpproxy/proxy.go diff --git a/vendor/golang.org/x/net/http/httpproxy/proxy.go b/vendor/golang.org/x/net/http/httpproxy/proxy.go new file mode 100644 index 000000000..163645b86 --- /dev/null +++ b/vendor/golang.org/x/net/http/httpproxy/proxy.go @@ -0,0 +1,370 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpproxy provides support for HTTP proxy determination +// based on environment variables, as provided by net/http's +// ProxyFromEnvironment function. +// +// The API is not subject to the Go 1 compatibility promise and may change at +// any time. +package httpproxy + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "unicode/utf8" + + "golang.org/x/net/idna" +) + +// Config holds configuration for HTTP proxy settings. See +// FromEnvironment for details. +type Config struct { + // HTTPProxy represents the value of the HTTP_PROXY or + // http_proxy environment variable. It will be used as the proxy + // URL for HTTP requests and HTTPS requests unless overridden by + // HTTPSProxy or NoProxy. + HTTPProxy string + + // HTTPSProxy represents the HTTPS_PROXY or https_proxy + // environment variable. It will be used as the proxy URL for + // HTTPS requests unless overridden by NoProxy. + HTTPSProxy string + + // NoProxy represents the NO_PROXY or no_proxy environment + // variable. It specifies a string that contains comma-separated values + // specifying hosts that should be excluded from proxying. Each value is + // represented by an IP address prefix (1.2.3.4), an IP address prefix in + // CIDR notation (1.2.3.4/8), a domain name, or a special DNS label (*). + // An IP address prefix and domain name can also include a literal port + // number (1.2.3.4:80). + // A domain name matches that name and all subdomains. A domain name with + // a leading "." matches subdomains only. For example "foo.com" matches + // "foo.com" and "bar.foo.com"; ".y.com" matches "x.y.com" but not "y.com". + // A single asterisk (*) indicates that no proxying should be done. + // A best effort is made to parse the string and errors are + // ignored. + NoProxy string + + // CGI holds whether the current process is running + // as a CGI handler (FromEnvironment infers this from the + // presence of a REQUEST_METHOD environment variable). + // When this is set, ProxyForURL will return an error + // when HTTPProxy applies, because a client could be + // setting HTTP_PROXY maliciously. See https://golang.org/s/cgihttpproxy. + CGI bool +} + +// config holds the parsed configuration for HTTP proxy settings. +type config struct { + // Config represents the original configuration as defined above. + Config + + // httpsProxy is the parsed URL of the HTTPSProxy if defined. + httpsProxy *url.URL + + // httpProxy is the parsed URL of the HTTPProxy if defined. + httpProxy *url.URL + + // ipMatchers represent all values in the NoProxy that are IP address + // prefixes or an IP address in CIDR notation. + ipMatchers []matcher + + // domainMatchers represent all values in the NoProxy that are a domain + // name or hostname & domain name + domainMatchers []matcher +} + +// FromEnvironment returns a Config instance populated from the +// environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the +// lowercase versions thereof). HTTPS_PROXY takes precedence over +// HTTP_PROXY for https requests. +// +// The environment values may be either a complete URL or a +// "host[:port]", in which case the "http" scheme is assumed. An error +// is returned if the value is a different form. +func FromEnvironment() *Config { + return &Config{ + HTTPProxy: getEnvAny("HTTP_PROXY", "http_proxy"), + HTTPSProxy: getEnvAny("HTTPS_PROXY", "https_proxy"), + NoProxy: getEnvAny("NO_PROXY", "no_proxy"), + CGI: os.Getenv("REQUEST_METHOD") != "", + } +} + +func getEnvAny(names ...string) string { + for _, n := range names { + if val := os.Getenv(n); val != "" { + return val + } + } + return "" +} + +// ProxyFunc returns a function that determines the proxy URL to use for +// a given request URL. Changing the contents of cfg will not affect +// proxy functions created earlier. +// +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request, as +// defined by NO_PROXY. +// +// As a special case, if req.URL.Host is "localhost" (with or without a +// port number), then a nil URL and nil error will be returned. +func (cfg *Config) ProxyFunc() func(reqURL *url.URL) (*url.URL, error) { + // Preprocess the Config settings for more efficient evaluation. + cfg1 := &config{ + Config: *cfg, + } + cfg1.init() + return cfg1.proxyForURL +} + +func (cfg *config) proxyForURL(reqURL *url.URL) (*url.URL, error) { + var proxy *url.URL + if reqURL.Scheme == "https" { + proxy = cfg.httpsProxy + } + if proxy == nil { + proxy = cfg.httpProxy + if proxy != nil && cfg.CGI { + return nil, errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") + } + } + if proxy == nil { + return nil, nil + } + if !cfg.useProxy(canonicalAddr(reqURL)) { + return nil, nil + } + + return proxy, nil +} + +func parseProxy(proxy string) (*url.URL, error) { + if proxy == "" { + return nil, nil + } + + proxyURL, err := url.Parse(proxy) + if err != nil || + (proxyURL.Scheme != "http" && + proxyURL.Scheme != "https" && + proxyURL.Scheme != "socks5") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if proxyURL, err := url.Parse("http://" + proxy); err == nil { + return proxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil +} + +// useProxy reports whether requests to addr should use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +// addr is always a canonicalAddr with a host and port. +func (cfg *config) useProxy(addr string) bool { + if len(addr) == 0 { + return true + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return false + } + ip := net.ParseIP(host) + if ip != nil { + if ip.IsLoopback() { + return false + } + } + + addr = strings.ToLower(strings.TrimSpace(host)) + + if ip != nil { + for _, m := range cfg.ipMatchers { + if m.match(addr, port, ip) { + return false + } + } + } + for _, m := range cfg.domainMatchers { + if m.match(addr, port, ip) { + return false + } + } + return true +} + +func (c *config) init() { + if parsed, err := parseProxy(c.HTTPProxy); err == nil { + c.httpProxy = parsed + } + if parsed, err := parseProxy(c.HTTPSProxy); err == nil { + c.httpsProxy = parsed + } + + for _, p := range strings.Split(c.NoProxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + + if p == "*" { + c.ipMatchers = []matcher{allMatch{}} + c.domainMatchers = []matcher{allMatch{}} + return + } + + // IPv4/CIDR, IPv6/CIDR + if _, pnet, err := net.ParseCIDR(p); err == nil { + c.ipMatchers = append(c.ipMatchers, cidrMatch{cidr: pnet}) + continue + } + + // IPv4:port, [IPv6]:port + phost, pport, err := net.SplitHostPort(p) + if err == nil { + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + if phost[0] == '[' && phost[len(phost)-1] == ']' { + phost = phost[1 : len(phost)-1] + } + } else { + phost = p + } + // IPv4, IPv6 + if pip := net.ParseIP(phost); pip != nil { + c.ipMatchers = append(c.ipMatchers, ipMatch{ip: pip, port: pport}) + continue + } + + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + + // domain.com or domain.com:80 + // foo.com matches bar.foo.com + // .domain.com or .domain.com:port + // *.domain.com or *.domain.com:port + if strings.HasPrefix(phost, "*.") { + phost = phost[1:] + } + matchHost := false + if phost[0] != '.' { + matchHost = true + phost = "." + phost + } + c.domainMatchers = append(c.domainMatchers, domainMatch{host: phost, port: pport, matchHost: matchHost}) + } +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", + "socks5": "1080", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Hostname() + if v, err := idnaASCII(addr); err == nil { + addr = v + } + port := url.Port() + if port == "" { + port = portMap[url.Scheme] + } + return net.JoinHostPort(addr, port) +} + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +func idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. + if isASCII(v) { + return v, nil + } + return idna.Lookup.ToASCII(v) +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// matcher represents the matching rule for a given value in the NO_PROXY list +type matcher interface { + // match returns true if the host and optional port or ip and optional port + // are allowed + match(host, port string, ip net.IP) bool +} + +// allMatch matches on all possible inputs +type allMatch struct{} + +func (a allMatch) match(host, port string, ip net.IP) bool { + return true +} + +type cidrMatch struct { + cidr *net.IPNet +} + +func (m cidrMatch) match(host, port string, ip net.IP) bool { + return m.cidr.Contains(ip) +} + +type ipMatch struct { + ip net.IP + port string +} + +func (m ipMatch) match(host, port string, ip net.IP) bool { + if m.ip.Equal(ip) { + return m.port == "" || m.port == port + } + return false +} + +type domainMatch struct { + host string + port string + + matchHost bool +} + +func (m domainMatch) match(host, port string, ip net.IP) bool { + if strings.HasSuffix(host, m.host) || (m.matchHost && host == m.host[1:]) { + return m.port == "" || m.port == port + } + return false +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d14316b6d..4ba6805c8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -166,6 +166,7 @@ golang.org/x/crypto/ssh/terminal golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/http/httpguts +golang.org/x/net/http/httpproxy golang.org/x/net/http2 golang.org/x/net/http2/hpack golang.org/x/net/idna From c503b45389ba43e944fac8b32c7498918530b541 Mon Sep 17 00:00:00 2001 From: Martin Kunc <5441732+martinkunc@users.noreply.github.com> Date: Fri, 20 Mar 2020 13:31:00 +0100 Subject: [PATCH 3/6] Unit tests for specific proxy settings --- pkg/insights/insightsclient/insightsclient.go | 6 +- .../insightsclient/insightsclient_test.go | 143 ++++++++++++++++++ 2 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 pkg/insights/insightsclient/insightsclient_test.go diff --git a/pkg/insights/insightsclient/insightsclient.go b/pkg/insights/insightsclient/insightsclient.go index e50f52447..d2fee858f 100644 --- a/pkg/insights/insightsclient/insightsclient.go +++ b/pkg/insights/insightsclient/insightsclient.go @@ -92,11 +92,15 @@ func getTrustedCABundle() (*x509.CertPool, error) { return certs, nil } +func proxyFromEnvironment() func(*http.Request) (*url.URL, error) { + return http.ProxyFromEnvironment +} + // clientTransport creates new http.Transport based on httpConfig if used, or Env func clientTransport(httpConfig config.HTTPConfig) http.RoundTripper { // default transport, proxy from configmap or env clientTransport := &http.Transport{ - Proxy: NewProxier(knet.NewProxierWithNoProxyCIDR(http.ProxyFromEnvironment), FromConfig(httpConfig)), + Proxy: NewProxier(knet.NewProxierWithNoProxyCIDR(proxyFromEnvironment()), FromConfig(httpConfig)), DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, diff --git a/pkg/insights/insightsclient/insightsclient_test.go b/pkg/insights/insightsclient/insightsclient_test.go new file mode 100644 index 000000000..459da1763 --- /dev/null +++ b/pkg/insights/insightsclient/insightsclient_test.go @@ -0,0 +1,143 @@ +package insightsclient + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + + "github.com/openshift/insights-operator/pkg/config" + "golang.org/x/net/http/httpproxy" + knet "k8s.io/apimachinery/pkg/util/net" +) + +// nonCachedProxyFromEnvironment creates Proxier if Proxy is set. It uses always fresh Env +func nonCachedProxyFromEnvironment() func(*http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + return httpproxy.FromEnvironment().ProxyFunc()(req.URL) + } +} + +func TestProxy(tt *testing.T) { + testCases := []struct { + Name string + EnvValues map[string]interface{} + RequestURL string + HttpConfig config.HTTPConfig + ProxyURL string + }{ + { + Name: "No env set, no specific proxy", + EnvValues: map[string]interface{}{"HTTP_PROXY": nil}, + RequestURL: "http://google.com", + ProxyURL: "", + }, + { + Name: "Env set, no specific proxy", + EnvValues: map[string]interface{}{"HTTP_PROXY": "proxy.to"}, + RequestURL: "http://google.com", + ProxyURL: "http://proxy.to", + }, + { + Name: "Env set with HTTPS, no specific proxy", + EnvValues: map[string]interface{}{"HTTPS_PROXY": "secproxy.to"}, + RequestURL: "https://google.com", + ProxyURL: "http://secproxy.to", + }, + { + Name: "Env not set, specific proxy set", + EnvValues: map[string]interface{}{"HTTP_PROXY": nil}, + RequestURL: "http://google.com", + HttpConfig: config.HTTPConfig{HTTPProxy: "specproxy.to"}, + ProxyURL: "http://specproxy.to", + }, + { + Name: "Env set, specific proxy set http", + EnvValues: map[string]interface{}{"HTTP_PROXY": "envproxy.to"}, + RequestURL: "http://google.com", + HttpConfig: config.HTTPConfig{HTTPProxy: "specproxy.to"}, + ProxyURL: "http://specproxy.to", + }, + { + Name: "Env set, specific proxy set https", + EnvValues: map[string]interface{}{"HTTPS_PROXY": "envsecproxy.to"}, + RequestURL: "https://google.com", + HttpConfig: config.HTTPConfig{HTTPProxy: "specsecproxy.to"}, + ProxyURL: "http://specsecproxy.to", + }, + { + Name: "Env set, specific proxy set noproxy, request without noproxy", + EnvValues: map[string]interface{}{"HTTPS_PROXY": "envsecproxy.to", "NO_PROXY": "envnoproxy.to"}, + RequestURL: "https://google.com", + HttpConfig: config.HTTPConfig{HTTPProxy: "specsecproxy.to", NoProxy: "specnoproxy.to"}, + ProxyURL: "http://specsecproxy.to", + }, + { + Name: "Env set, specific proxy set noproxy, request with noproxy", + EnvValues: map[string]interface{}{"HTTPS_PROXY": "envsecproxy.to", "NO_PROXY": "envnoproxy.to"}, + RequestURL: "https://specnoproxy.to", + HttpConfig: config.HTTPConfig{HTTPProxy: "specsecproxy.to", NoProxy: "specnoproxy.to"}, + ProxyURL: "", + }, + } + for _, tcase := range testCases { + tc := tcase + tt.Run(tc.Name, func(t *testing.T) { + for k, v := range tc.EnvValues { + defer SafeRestoreEnv(k)() + // nil will indicate the need to unset Env + if v != nil { + vv := v.(string) + os.Setenv(k, vv) + } else { + os.Unsetenv(k) + } + } + p := NewProxier(knet.NewProxierWithNoProxyCIDR(nonCachedProxyFromEnvironment()), FromConfig(tc.HttpConfig)) + req := httptest.NewRequest("GET", tc.RequestURL, nil) + url, err := p(req) + + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if (tc.ProxyURL == "" && url != nil) || + (len(tc.ProxyURL) > 0 && (url == nil || tc.ProxyURL != url.String())) { + t.Fatalf("Unexpected value of Proxy Url. Test %s Expected Url %s Received Url %s", tc.Name, tc.ProxyURL, url) + } + }) + } +} + +// func TestEnvNoSpecProxy(t *testing.T) { +// proxy := "HTTP_PROXY" +// defer SafeRestoreEnv(proxy) +// exp := "proxy.to" +// os.Setenv(proxy, exp) +// httpConfig := config.HTTPConfig{} +// p := NewProxier(knet.NewProxierWithNoProxyCIDR(http.ProxyFromEnvironment), FromConfig(httpConfig)) +// req := httptest.NewRequest("GET", "http://google.com", nil) +// url, err := p(req) + +// t.Logf("url %s err %s", url, err) +// if err != nil { +// t.Fatalf("unexpected err %s", err) +// } +// if nil != url { +// t.Fatalf("no HTTP_PROXY env should return no proxy url.") +// } +// } + +func SafeRestoreEnv(key string) func() { + originalVal, wasSet := os.LookupEnv(key) + return func() { + if !wasSet { + fmt.Printf("unsetting key %s", key) + os.Unsetenv(key) + } else { + fmt.Printf("restoring key %s", key) + os.Setenv(key, originalVal) + } + } +} From 147dca32c3769569c6505f4c04f884ba2a687ae0 Mon Sep 17 00:00:00 2001 From: Martin Kunc <5441732+martinkunc@users.noreply.github.com> Date: Tue, 24 Mar 2020 13:55:05 +0100 Subject: [PATCH 4/6] Remove commented code block - moved above. --- .../insightsclient/insightsclient_test.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pkg/insights/insightsclient/insightsclient_test.go b/pkg/insights/insightsclient/insightsclient_test.go index 459da1763..442779144 100644 --- a/pkg/insights/insightsclient/insightsclient_test.go +++ b/pkg/insights/insightsclient/insightsclient_test.go @@ -110,25 +110,6 @@ func TestProxy(tt *testing.T) { } } -// func TestEnvNoSpecProxy(t *testing.T) { -// proxy := "HTTP_PROXY" -// defer SafeRestoreEnv(proxy) -// exp := "proxy.to" -// os.Setenv(proxy, exp) -// httpConfig := config.HTTPConfig{} -// p := NewProxier(knet.NewProxierWithNoProxyCIDR(http.ProxyFromEnvironment), FromConfig(httpConfig)) -// req := httptest.NewRequest("GET", "http://google.com", nil) -// url, err := p(req) - -// t.Logf("url %s err %s", url, err) -// if err != nil { -// t.Fatalf("unexpected err %s", err) -// } -// if nil != url { -// t.Fatalf("no HTTP_PROXY env should return no proxy url.") -// } -// } - func SafeRestoreEnv(key string) func() { originalVal, wasSet := os.LookupEnv(key) return func() { From d19d8d83f1e289413d4305fda2da16b1acd35586 Mon Sep 17 00:00:00 2001 From: Martin Kunc <5441732+martinkunc@users.noreply.github.com> Date: Fri, 27 Mar 2020 16:58:34 +0100 Subject: [PATCH 5/6] Moving specific proxy to authorizer --- .../clusterauthorizer/clusterauthorizer.go | 29 +++++++++-- .../clusterauthorizer_test.go} | 16 ++++-- pkg/insights/insightsclient/insightsclient.go | 51 ++----------------- 3 files changed, 42 insertions(+), 54 deletions(-) rename pkg/{insights/insightsclient/insightsclient_test.go => authorizer/clusterauthorizer/clusterauthorizer_test.go} (90%) diff --git a/pkg/authorizer/clusterauthorizer/clusterauthorizer.go b/pkg/authorizer/clusterauthorizer/clusterauthorizer.go index 662e25fdc..a33f29c7c 100644 --- a/pkg/authorizer/clusterauthorizer/clusterauthorizer.go +++ b/pkg/authorizer/clusterauthorizer/clusterauthorizer.go @@ -3,9 +3,12 @@ package clusterauthorizer import ( "fmt" "net/http" + "net/url" "strings" "github.com/openshift/insights-operator/pkg/config" + "golang.org/x/net/http/httpproxy" + knet "k8s.io/apimachinery/pkg/util/net" ) type Configurator interface { @@ -14,11 +17,14 @@ type Configurator interface { type Authorizer struct { configurator Configurator + // exposed for tests + proxyFromEnvironment func(*http.Request) (*url.URL, error) } func New(configurator Configurator) *Authorizer { return &Authorizer{ - configurator: configurator, + configurator: configurator, + proxyFromEnvironment: http.ProxyFromEnvironment, } } @@ -45,8 +51,21 @@ func (a *Authorizer) Authorize(req *http.Request) error { return nil } -// HTTPConfig provides the Http Proxy settings from Secret -func (a *Authorizer) HTTPConfig() config.HTTPConfig { - cfg := a.configurator.Config() - return cfg.HTTPConfig +func (a *Authorizer) NewSystemOrConfiguredProxy() func(*http.Request) (*url.URL, error) { + // using specific proxy settings + if c := a.configurator.Config(); c != nil { + if len(c.HTTPConfig.HTTPProxy) > 0 || len(c.HTTPConfig.HTTPSProxy) > 0 || len(c.HTTPConfig.NoProxy) > 0 { + proxyConfig := httpproxy.Config{ + HTTPProxy: c.HTTPConfig.HTTPProxy, + HTTPSProxy: c.HTTPConfig.HTTPSProxy, + NoProxy: c.HTTPConfig.NoProxy, + } + // The golang ProxyFunc seems to have NoProxy already built in + return func(req *http.Request) (*url.URL, error) { + return proxyConfig.ProxyFunc()(req.URL) + } + } + } + // defautl system proxy + return knet.NewProxierWithNoProxyCIDR(a.proxyFromEnvironment) } diff --git a/pkg/insights/insightsclient/insightsclient_test.go b/pkg/authorizer/clusterauthorizer/clusterauthorizer_test.go similarity index 90% rename from pkg/insights/insightsclient/insightsclient_test.go rename to pkg/authorizer/clusterauthorizer/clusterauthorizer_test.go index 442779144..8a76c8a34 100644 --- a/pkg/insights/insightsclient/insightsclient_test.go +++ b/pkg/authorizer/clusterauthorizer/clusterauthorizer_test.go @@ -1,4 +1,4 @@ -package insightsclient +package clusterauthorizer import ( "fmt" @@ -10,7 +10,6 @@ import ( "github.com/openshift/insights-operator/pkg/config" "golang.org/x/net/http/httpproxy" - knet "k8s.io/apimachinery/pkg/util/net" ) // nonCachedProxyFromEnvironment creates Proxier if Proxy is set. It uses always fresh Env @@ -95,7 +94,10 @@ func TestProxy(tt *testing.T) { os.Unsetenv(k) } } - p := NewProxier(knet.NewProxierWithNoProxyCIDR(nonCachedProxyFromEnvironment()), FromConfig(tc.HttpConfig)) + + co2 := &testConfig{config: &config.Controller{HTTPConfig: tc.HttpConfig}} + a := Authorizer{proxyFromEnvironment: nonCachedProxyFromEnvironment(), configurator: co2} + p := a.NewSystemOrConfiguredProxy() req := httptest.NewRequest("GET", tc.RequestURL, nil) url, err := p(req) @@ -110,6 +112,14 @@ func TestProxy(tt *testing.T) { } } +type testConfig struct { + config *config.Controller +} + +func (t *testConfig) Config() *config.Controller { + return t.config +} + func SafeRestoreEnv(key string) func() { originalVal, wasSet := os.LookupEnv(key) return func() { diff --git a/pkg/insights/insightsclient/insightsclient.go b/pkg/insights/insightsclient/insightsclient.go index d2fee858f..bd70706f6 100644 --- a/pkg/insights/insightsclient/insightsclient.go +++ b/pkg/insights/insightsclient/insightsclient.go @@ -17,8 +17,6 @@ import ( "strconv" "time" - "golang.org/x/net/http/httpproxy" - knet "k8s.io/apimachinery/pkg/util/net" "k8s.io/client-go/pkg/version" "k8s.io/client-go/transport" "k8s.io/component-base/metrics" @@ -29,7 +27,6 @@ import ( configv1 "github.com/openshift/api/config/v1" "github.com/openshift/insights-operator/pkg/authorizer" - "github.com/openshift/insights-operator/pkg/config" ) type Client struct { @@ -43,7 +40,7 @@ type Client struct { type Authorizer interface { Authorize(req *http.Request) error - HTTPConfig() config.HTTPConfig + NewSystemOrConfiguredProxy() func(*http.Request) (*url.URL, error) } type ClusterVersionInfo interface { @@ -92,15 +89,10 @@ func getTrustedCABundle() (*x509.CertPool, error) { return certs, nil } -func proxyFromEnvironment() func(*http.Request) (*url.URL, error) { - return http.ProxyFromEnvironment -} - -// clientTransport creates new http.Transport based on httpConfig if used, or Env -func clientTransport(httpConfig config.HTTPConfig) http.RoundTripper { - // default transport, proxy from configmap or env +// clientTransport creates new http.Transport with either system or configured Proxy +func clientTransport(authorizer Authorizer) http.RoundTripper { clientTransport := &http.Transport{ - Proxy: NewProxier(knet.NewProxierWithNoProxyCIDR(proxyFromEnvironment()), FromConfig(httpConfig)), + Proxy: authorizer.NewSystemOrConfiguredProxy(), DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, @@ -122,39 +114,6 @@ func clientTransport(httpConfig config.HTTPConfig) http.RoundTripper { return transport.DebugWrappers(clientTransport) } -// ConfigProxier is creating a Proxier from proxy set in HttpConfig -func ConfigProxier(c config.HTTPConfig) func(req *http.Request) (*url.URL, error) { - proxyConfig := httpproxy.Config{ - HTTPProxy: c.HTTPProxy, - HTTPSProxy: c.HTTPSProxy, - NoProxy: c.NoProxy, - } - // The golang ProxyFunc seems to have NoProxy already built in - return func(req *http.Request) (*url.URL, error) { - return proxyConfig.ProxyFunc()(req.URL) - } -} - -// FromConfig is setting HttpProxy from HttpConfig in support secret, if it is used -func FromConfig(c config.HTTPConfig) func(req *http.Request) (*url.URL, error) { - if len(c.HTTPProxy) > 0 || len(c.HTTPSProxy) > 0 || len(c.NoProxy) > 0 { - return ConfigProxier(c) - } - return nil -} - -// NewProxier will create new http.Proxier function. -// If any of customProxiers is set, it will use it, otherwise will use system -func NewProxier(system func(req *http.Request) (*url.URL, error), customProxiers ...func(req *http.Request) (*url.URL, error)) func(req *http.Request) (*url.URL, error) { - for _, p := range customProxiers { - if p != nil { - return p - } - } - - return system -} - func (c *Client) Send(ctx context.Context, endpoint string, source Source) error { cv := c.clusterInfo.ClusterVersion() if cv == nil { @@ -200,7 +159,7 @@ func (c *Client) Send(ctx context.Context, endpoint string, source Source) error req.Body = pr // dynamically set the proxy environment - c.client.Transport = clientTransport(c.authorizer.HTTPConfig()) + c.client.Transport = clientTransport(c.authorizer) klog.V(4).Infof("Uploading %s to %s", source.Type, req.URL.String()) resp, err := c.client.Do(req) From 65e218343c05ed4ba98664232f893469f7583757 Mon Sep 17 00:00:00 2001 From: Martin Kunc <5441732+martinkunc@users.noreply.github.com> Date: Thu, 19 Mar 2020 14:42:41 +0100 Subject: [PATCH 6/6] Support for specific http proxy for the service --- go.mod | 2 +- .../clusterauthorizer/clusterauthorizer.go | 27 +- .../clusterauthorizer_test.go | 134 +++++++ pkg/config/config.go | 9 + pkg/config/configobserver/configobserver.go | 10 + pkg/insights/insightsclient/insightsclient.go | 13 +- .../golang.org/x/net/http/httpproxy/proxy.go | 370 ++++++++++++++++++ vendor/modules.txt | 1 + 8 files changed, 558 insertions(+), 8 deletions(-) create mode 100644 pkg/authorizer/clusterauthorizer/clusterauthorizer_test.go create mode 100644 vendor/golang.org/x/net/http/httpproxy/proxy.go diff --git a/go.mod b/go.mod index bee9d54b1..8944567e0 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect - golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect + golang.org/x/net v0.0.0-20200301022130-244492dfa37a golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 google.golang.org/appengine v1.6.1 // indirect k8s.io/api v0.17.1 diff --git a/pkg/authorizer/clusterauthorizer/clusterauthorizer.go b/pkg/authorizer/clusterauthorizer/clusterauthorizer.go index 5fd449d5b..a33f29c7c 100644 --- a/pkg/authorizer/clusterauthorizer/clusterauthorizer.go +++ b/pkg/authorizer/clusterauthorizer/clusterauthorizer.go @@ -3,9 +3,12 @@ package clusterauthorizer import ( "fmt" "net/http" + "net/url" "strings" "github.com/openshift/insights-operator/pkg/config" + "golang.org/x/net/http/httpproxy" + knet "k8s.io/apimachinery/pkg/util/net" ) type Configurator interface { @@ -14,11 +17,14 @@ type Configurator interface { type Authorizer struct { configurator Configurator + // exposed for tests + proxyFromEnvironment func(*http.Request) (*url.URL, error) } func New(configurator Configurator) *Authorizer { return &Authorizer{ - configurator: configurator, + configurator: configurator, + proxyFromEnvironment: http.ProxyFromEnvironment, } } @@ -44,3 +50,22 @@ func (a *Authorizer) Authorize(req *http.Request) error { } return nil } + +func (a *Authorizer) NewSystemOrConfiguredProxy() func(*http.Request) (*url.URL, error) { + // using specific proxy settings + if c := a.configurator.Config(); c != nil { + if len(c.HTTPConfig.HTTPProxy) > 0 || len(c.HTTPConfig.HTTPSProxy) > 0 || len(c.HTTPConfig.NoProxy) > 0 { + proxyConfig := httpproxy.Config{ + HTTPProxy: c.HTTPConfig.HTTPProxy, + HTTPSProxy: c.HTTPConfig.HTTPSProxy, + NoProxy: c.HTTPConfig.NoProxy, + } + // The golang ProxyFunc seems to have NoProxy already built in + return func(req *http.Request) (*url.URL, error) { + return proxyConfig.ProxyFunc()(req.URL) + } + } + } + // defautl system proxy + return knet.NewProxierWithNoProxyCIDR(a.proxyFromEnvironment) +} diff --git a/pkg/authorizer/clusterauthorizer/clusterauthorizer_test.go b/pkg/authorizer/clusterauthorizer/clusterauthorizer_test.go new file mode 100644 index 000000000..8a76c8a34 --- /dev/null +++ b/pkg/authorizer/clusterauthorizer/clusterauthorizer_test.go @@ -0,0 +1,134 @@ +package clusterauthorizer + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "testing" + + "github.com/openshift/insights-operator/pkg/config" + "golang.org/x/net/http/httpproxy" +) + +// nonCachedProxyFromEnvironment creates Proxier if Proxy is set. It uses always fresh Env +func nonCachedProxyFromEnvironment() func(*http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + return httpproxy.FromEnvironment().ProxyFunc()(req.URL) + } +} + +func TestProxy(tt *testing.T) { + testCases := []struct { + Name string + EnvValues map[string]interface{} + RequestURL string + HttpConfig config.HTTPConfig + ProxyURL string + }{ + { + Name: "No env set, no specific proxy", + EnvValues: map[string]interface{}{"HTTP_PROXY": nil}, + RequestURL: "http://google.com", + ProxyURL: "", + }, + { + Name: "Env set, no specific proxy", + EnvValues: map[string]interface{}{"HTTP_PROXY": "proxy.to"}, + RequestURL: "http://google.com", + ProxyURL: "http://proxy.to", + }, + { + Name: "Env set with HTTPS, no specific proxy", + EnvValues: map[string]interface{}{"HTTPS_PROXY": "secproxy.to"}, + RequestURL: "https://google.com", + ProxyURL: "http://secproxy.to", + }, + { + Name: "Env not set, specific proxy set", + EnvValues: map[string]interface{}{"HTTP_PROXY": nil}, + RequestURL: "http://google.com", + HttpConfig: config.HTTPConfig{HTTPProxy: "specproxy.to"}, + ProxyURL: "http://specproxy.to", + }, + { + Name: "Env set, specific proxy set http", + EnvValues: map[string]interface{}{"HTTP_PROXY": "envproxy.to"}, + RequestURL: "http://google.com", + HttpConfig: config.HTTPConfig{HTTPProxy: "specproxy.to"}, + ProxyURL: "http://specproxy.to", + }, + { + Name: "Env set, specific proxy set https", + EnvValues: map[string]interface{}{"HTTPS_PROXY": "envsecproxy.to"}, + RequestURL: "https://google.com", + HttpConfig: config.HTTPConfig{HTTPProxy: "specsecproxy.to"}, + ProxyURL: "http://specsecproxy.to", + }, + { + Name: "Env set, specific proxy set noproxy, request without noproxy", + EnvValues: map[string]interface{}{"HTTPS_PROXY": "envsecproxy.to", "NO_PROXY": "envnoproxy.to"}, + RequestURL: "https://google.com", + HttpConfig: config.HTTPConfig{HTTPProxy: "specsecproxy.to", NoProxy: "specnoproxy.to"}, + ProxyURL: "http://specsecproxy.to", + }, + { + Name: "Env set, specific proxy set noproxy, request with noproxy", + EnvValues: map[string]interface{}{"HTTPS_PROXY": "envsecproxy.to", "NO_PROXY": "envnoproxy.to"}, + RequestURL: "https://specnoproxy.to", + HttpConfig: config.HTTPConfig{HTTPProxy: "specsecproxy.to", NoProxy: "specnoproxy.to"}, + ProxyURL: "", + }, + } + for _, tcase := range testCases { + tc := tcase + tt.Run(tc.Name, func(t *testing.T) { + for k, v := range tc.EnvValues { + defer SafeRestoreEnv(k)() + // nil will indicate the need to unset Env + if v != nil { + vv := v.(string) + os.Setenv(k, vv) + } else { + os.Unsetenv(k) + } + } + + co2 := &testConfig{config: &config.Controller{HTTPConfig: tc.HttpConfig}} + a := Authorizer{proxyFromEnvironment: nonCachedProxyFromEnvironment(), configurator: co2} + p := a.NewSystemOrConfiguredProxy() + req := httptest.NewRequest("GET", tc.RequestURL, nil) + url, err := p(req) + + if err != nil { + t.Fatalf("unexpected err %s", err) + } + if (tc.ProxyURL == "" && url != nil) || + (len(tc.ProxyURL) > 0 && (url == nil || tc.ProxyURL != url.String())) { + t.Fatalf("Unexpected value of Proxy Url. Test %s Expected Url %s Received Url %s", tc.Name, tc.ProxyURL, url) + } + }) + } +} + +type testConfig struct { + config *config.Controller +} + +func (t *testConfig) Config() *config.Controller { + return t.config +} + +func SafeRestoreEnv(key string) func() { + originalVal, wasSet := os.LookupEnv(key) + return func() { + if !wasSet { + fmt.Printf("unsetting key %s", key) + os.Unsetenv(key) + } else { + fmt.Printf("restoring key %s", key) + os.Setenv(key, originalVal) + } + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index e63d40bb0..a5c3d1e69 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -49,4 +49,13 @@ type Controller struct { Username string Password string Token string + + HTTPConfig HTTPConfig +} + +// HTTPConfig configures http proxy and exception settings if they come from config +type HTTPConfig struct { + HTTPProxy string + HTTPSProxy string + NoProxy string } diff --git a/pkg/config/configobserver/configobserver.go b/pkg/config/configobserver/configobserver.go index a929a6da7..4229558d3 100644 --- a/pkg/config/configobserver/configobserver.go +++ b/pkg/config/configobserver/configobserver.go @@ -130,6 +130,15 @@ func (c *Controller) retrieveConfig() error { if endpoint, ok := secret.Data["endpoint"]; ok { nextConfig.Endpoint = string(endpoint) } + if httpproxy, ok := secret.Data["httpProxy"]; ok { + nextConfig.HTTPConfig.HTTPProxy = string(httpproxy) + } + if httpsproxy, ok := secret.Data["httpsProxy"]; ok { + nextConfig.HTTPConfig.HTTPSProxy = string(httpsproxy) + } + if noproxy, ok := secret.Data["noProxy"]; ok { + nextConfig.HTTPConfig.NoProxy = string(noproxy) + } nextConfig.Report = len(nextConfig.Endpoint) > 0 if intervalString, ok := secret.Data["interval"]; ok { @@ -207,6 +216,7 @@ func (c *Controller) mergeConfigLocked() { if len(c.secretConfig.Endpoint) > 0 { cfg.Endpoint = c.secretConfig.Endpoint } + cfg.HTTPConfig = c.secretConfig.HTTPConfig } if c.tokenConfig != nil { cfg.Token = c.tokenConfig.Token diff --git a/pkg/insights/insightsclient/insightsclient.go b/pkg/insights/insightsclient/insightsclient.go index bb342c159..bd70706f6 100644 --- a/pkg/insights/insightsclient/insightsclient.go +++ b/pkg/insights/insightsclient/insightsclient.go @@ -12,11 +12,11 @@ import ( "net" "net/http" "net/textproto" + "net/url" "os" "strconv" "time" - knet "k8s.io/apimachinery/pkg/util/net" "k8s.io/client-go/pkg/version" "k8s.io/client-go/transport" "k8s.io/component-base/metrics" @@ -40,6 +40,7 @@ type Client struct { type Authorizer interface { Authorize(req *http.Request) error + NewSystemOrConfiguredProxy() func(*http.Request) (*url.URL, error) } type ClusterVersionInfo interface { @@ -88,10 +89,10 @@ func getTrustedCABundle() (*x509.CertPool, error) { return certs, nil } -func clientTransport() http.RoundTripper { - // default transport, proxy from env +// clientTransport creates new http.Transport with either system or configured Proxy +func clientTransport(authorizer Authorizer) http.RoundTripper { clientTransport := &http.Transport{ - Proxy: knet.NewProxierWithNoProxyCIDR(http.ProxyFromEnvironment), + Proxy: authorizer.NewSystemOrConfiguredProxy(), DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, @@ -158,14 +159,14 @@ func (c *Client) Send(ctx context.Context, endpoint string, source Source) error req.Body = pr // dynamically set the proxy environment - c.client.Transport = clientTransport() + c.client.Transport = clientTransport(c.authorizer) klog.V(4).Infof("Uploading %s to %s", source.Type, req.URL.String()) resp, err := c.client.Do(req) if err != nil { klog.V(4).Infof("Unable to build a request, possible invalid token: %v", err) // if the request is not build, for example because of invalid endpoint,(maybe some problem with DNS), we want to have record about it in metrics as well. - counterRequestSend.WithLabelValues(c.metricsName, "0").Inc() + counterRequestSend.WithLabelValues(c.metricsName, "0").Inc() return fmt.Errorf("unable to build request to connect to Insights server: %v", err) } diff --git a/vendor/golang.org/x/net/http/httpproxy/proxy.go b/vendor/golang.org/x/net/http/httpproxy/proxy.go new file mode 100644 index 000000000..163645b86 --- /dev/null +++ b/vendor/golang.org/x/net/http/httpproxy/proxy.go @@ -0,0 +1,370 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpproxy provides support for HTTP proxy determination +// based on environment variables, as provided by net/http's +// ProxyFromEnvironment function. +// +// The API is not subject to the Go 1 compatibility promise and may change at +// any time. +package httpproxy + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "unicode/utf8" + + "golang.org/x/net/idna" +) + +// Config holds configuration for HTTP proxy settings. See +// FromEnvironment for details. +type Config struct { + // HTTPProxy represents the value of the HTTP_PROXY or + // http_proxy environment variable. It will be used as the proxy + // URL for HTTP requests and HTTPS requests unless overridden by + // HTTPSProxy or NoProxy. + HTTPProxy string + + // HTTPSProxy represents the HTTPS_PROXY or https_proxy + // environment variable. It will be used as the proxy URL for + // HTTPS requests unless overridden by NoProxy. + HTTPSProxy string + + // NoProxy represents the NO_PROXY or no_proxy environment + // variable. It specifies a string that contains comma-separated values + // specifying hosts that should be excluded from proxying. Each value is + // represented by an IP address prefix (1.2.3.4), an IP address prefix in + // CIDR notation (1.2.3.4/8), a domain name, or a special DNS label (*). + // An IP address prefix and domain name can also include a literal port + // number (1.2.3.4:80). + // A domain name matches that name and all subdomains. A domain name with + // a leading "." matches subdomains only. For example "foo.com" matches + // "foo.com" and "bar.foo.com"; ".y.com" matches "x.y.com" but not "y.com". + // A single asterisk (*) indicates that no proxying should be done. + // A best effort is made to parse the string and errors are + // ignored. + NoProxy string + + // CGI holds whether the current process is running + // as a CGI handler (FromEnvironment infers this from the + // presence of a REQUEST_METHOD environment variable). + // When this is set, ProxyForURL will return an error + // when HTTPProxy applies, because a client could be + // setting HTTP_PROXY maliciously. See https://golang.org/s/cgihttpproxy. + CGI bool +} + +// config holds the parsed configuration for HTTP proxy settings. +type config struct { + // Config represents the original configuration as defined above. + Config + + // httpsProxy is the parsed URL of the HTTPSProxy if defined. + httpsProxy *url.URL + + // httpProxy is the parsed URL of the HTTPProxy if defined. + httpProxy *url.URL + + // ipMatchers represent all values in the NoProxy that are IP address + // prefixes or an IP address in CIDR notation. + ipMatchers []matcher + + // domainMatchers represent all values in the NoProxy that are a domain + // name or hostname & domain name + domainMatchers []matcher +} + +// FromEnvironment returns a Config instance populated from the +// environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the +// lowercase versions thereof). HTTPS_PROXY takes precedence over +// HTTP_PROXY for https requests. +// +// The environment values may be either a complete URL or a +// "host[:port]", in which case the "http" scheme is assumed. An error +// is returned if the value is a different form. +func FromEnvironment() *Config { + return &Config{ + HTTPProxy: getEnvAny("HTTP_PROXY", "http_proxy"), + HTTPSProxy: getEnvAny("HTTPS_PROXY", "https_proxy"), + NoProxy: getEnvAny("NO_PROXY", "no_proxy"), + CGI: os.Getenv("REQUEST_METHOD") != "", + } +} + +func getEnvAny(names ...string) string { + for _, n := range names { + if val := os.Getenv(n); val != "" { + return val + } + } + return "" +} + +// ProxyFunc returns a function that determines the proxy URL to use for +// a given request URL. Changing the contents of cfg will not affect +// proxy functions created earlier. +// +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request, as +// defined by NO_PROXY. +// +// As a special case, if req.URL.Host is "localhost" (with or without a +// port number), then a nil URL and nil error will be returned. +func (cfg *Config) ProxyFunc() func(reqURL *url.URL) (*url.URL, error) { + // Preprocess the Config settings for more efficient evaluation. + cfg1 := &config{ + Config: *cfg, + } + cfg1.init() + return cfg1.proxyForURL +} + +func (cfg *config) proxyForURL(reqURL *url.URL) (*url.URL, error) { + var proxy *url.URL + if reqURL.Scheme == "https" { + proxy = cfg.httpsProxy + } + if proxy == nil { + proxy = cfg.httpProxy + if proxy != nil && cfg.CGI { + return nil, errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") + } + } + if proxy == nil { + return nil, nil + } + if !cfg.useProxy(canonicalAddr(reqURL)) { + return nil, nil + } + + return proxy, nil +} + +func parseProxy(proxy string) (*url.URL, error) { + if proxy == "" { + return nil, nil + } + + proxyURL, err := url.Parse(proxy) + if err != nil || + (proxyURL.Scheme != "http" && + proxyURL.Scheme != "https" && + proxyURL.Scheme != "socks5") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if proxyURL, err := url.Parse("http://" + proxy); err == nil { + return proxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil +} + +// useProxy reports whether requests to addr should use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +// addr is always a canonicalAddr with a host and port. +func (cfg *config) useProxy(addr string) bool { + if len(addr) == 0 { + return true + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return false + } + ip := net.ParseIP(host) + if ip != nil { + if ip.IsLoopback() { + return false + } + } + + addr = strings.ToLower(strings.TrimSpace(host)) + + if ip != nil { + for _, m := range cfg.ipMatchers { + if m.match(addr, port, ip) { + return false + } + } + } + for _, m := range cfg.domainMatchers { + if m.match(addr, port, ip) { + return false + } + } + return true +} + +func (c *config) init() { + if parsed, err := parseProxy(c.HTTPProxy); err == nil { + c.httpProxy = parsed + } + if parsed, err := parseProxy(c.HTTPSProxy); err == nil { + c.httpsProxy = parsed + } + + for _, p := range strings.Split(c.NoProxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + + if p == "*" { + c.ipMatchers = []matcher{allMatch{}} + c.domainMatchers = []matcher{allMatch{}} + return + } + + // IPv4/CIDR, IPv6/CIDR + if _, pnet, err := net.ParseCIDR(p); err == nil { + c.ipMatchers = append(c.ipMatchers, cidrMatch{cidr: pnet}) + continue + } + + // IPv4:port, [IPv6]:port + phost, pport, err := net.SplitHostPort(p) + if err == nil { + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + if phost[0] == '[' && phost[len(phost)-1] == ']' { + phost = phost[1 : len(phost)-1] + } + } else { + phost = p + } + // IPv4, IPv6 + if pip := net.ParseIP(phost); pip != nil { + c.ipMatchers = append(c.ipMatchers, ipMatch{ip: pip, port: pport}) + continue + } + + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + + // domain.com or domain.com:80 + // foo.com matches bar.foo.com + // .domain.com or .domain.com:port + // *.domain.com or *.domain.com:port + if strings.HasPrefix(phost, "*.") { + phost = phost[1:] + } + matchHost := false + if phost[0] != '.' { + matchHost = true + phost = "." + phost + } + c.domainMatchers = append(c.domainMatchers, domainMatch{host: phost, port: pport, matchHost: matchHost}) + } +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", + "socks5": "1080", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Hostname() + if v, err := idnaASCII(addr); err == nil { + addr = v + } + port := url.Port() + if port == "" { + port = portMap[url.Scheme] + } + return net.JoinHostPort(addr, port) +} + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +func idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. + if isASCII(v) { + return v, nil + } + return idna.Lookup.ToASCII(v) +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// matcher represents the matching rule for a given value in the NO_PROXY list +type matcher interface { + // match returns true if the host and optional port or ip and optional port + // are allowed + match(host, port string, ip net.IP) bool +} + +// allMatch matches on all possible inputs +type allMatch struct{} + +func (a allMatch) match(host, port string, ip net.IP) bool { + return true +} + +type cidrMatch struct { + cidr *net.IPNet +} + +func (m cidrMatch) match(host, port string, ip net.IP) bool { + return m.cidr.Contains(ip) +} + +type ipMatch struct { + ip net.IP + port string +} + +func (m ipMatch) match(host, port string, ip net.IP) bool { + if m.ip.Equal(ip) { + return m.port == "" || m.port == port + } + return false +} + +type domainMatch struct { + host string + port string + + matchHost bool +} + +func (m domainMatch) match(host, port string, ip net.IP) bool { + if strings.HasSuffix(host, m.host) || (m.matchHost && host == m.host[1:]) { + return m.port == "" || m.port == port + } + return false +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d14316b6d..4ba6805c8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -166,6 +166,7 @@ golang.org/x/crypto/ssh/terminal golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/http/httpguts +golang.org/x/net/http/httpproxy golang.org/x/net/http2 golang.org/x/net/http2/hpack golang.org/x/net/idna