OCPBUGS-21672: http2: trim connetions and buffers

.github/workflows/build.yml

@@ -4,8 +4,8 @@ on: [push, pull_request]
 
 env:
   QUAY_PATH: quay.io/brancz/kube-rbac-proxy
-  go-version: '1.20.4'
-  kind-version: 'v0.16.0'
+  go-version: '1.21.3'
+  kind-version: 'v0.20.0'
 
 jobs:
   check-license:
@@ -71,7 +71,7 @@ jobs:
         go-version: ${{ env.go-version }}
     - name: Create container & run tests
       run: |
-        VERSION=local make container
+        VERSION=local VERSION_SEMVER=$(cat ./VERSION) make container
         kind load docker-image ${QUAY_PATH}:local
         until docker exec $(kind get nodes) crictl images | grep "${QUAY_PATH}"; do
           echo "no kube-rbac-proxy image"

CHANGELOG.md

@@ -1,6 +1,24 @@
+## 0.15.0 / 2023-10-20
+
+- [ENHANCEMENT] bump opentelemetry to fix CVE
+- [ENHANCEMENT] add option to disable HTTP/2: `--http2-disable` (default: `false`)
+- [ENHANCEMENT] add option to fine-tune HTTP/2:
+    - `--http2-max-size` (default: 256kb)
+    - `--http2-max-concurrent-streams` (default: 100)
+
+## 0.14.4 / 2023-10-16
+
+- [ENHANCEMENT] bump golang and x/net
+
+## 0.14.3 / 2023-09-07
+
+- [BUGFIX] `--version` returns now the proper kube-rbac-proxy version
+- [ENHANCEMENT] bump dependencies
+
 ## 0.14.2 / 2023-06-05
 
 - [ENHANCEMENT] bump dependencies
+- [ENHANCEMENT] Makefile, Dockerfile to work better with different architectures
 
 ## 0.14.1 / 2023-04-06
 

Dockerfile.ocp

@@ -4,7 +4,7 @@ COPY . .
 ENV GO111MODULE=on
 # GOFLAGS is needed to build image without accessing external sources, mostly to comply with ART policies
 ENV GOFLAGS="-mod=vendor"
-RUN make build && \
+RUN GITHUB_URL=github.com/openshift/kube-rbac-proxy VERSION=$(cat VERSION) make build && \
     cp _output/kube-rbac-proxy-$(go env GOOS)-$(go env GOARCH) _output/kube-rbac-proxy
 
 FROM registry.ci.openshift.org/ocp/4.15:base

Makefile

@@ -3,20 +3,21 @@ all: check-license build generate test
 GO111MODULE=on
 export GO111MODULE
 
+PROGRAM_NAME?=kube-rbac-proxy
 GITHUB_URL=github.com/brancz/kube-rbac-proxy
 GOOS?=$(shell uname -s | tr A-Z a-z)
 GOARCH?=$(shell go env GOARCH)
 OUT_DIR=_output
-BIN?=kube-rbac-proxy
 VERSION?=$(shell cat VERSION)-$(shell git rev-parse --short HEAD)
+VERSION_SEMVER?=$(shell echo $(VERSION) | grep -o 'v[0-9]\+\.[0-9]\+\.[0-9]\+')
 PKGS=$(shell go list ./... | grep -v /test/e2e)
 DOCKER_REPO?=quay.io/brancz/kube-rbac-proxy
 KUBECONFIG?=$(HOME)/.kube/config
 CONTAINER_NAME?=$(DOCKER_REPO):$(VERSION)
 
 ALL_ARCH=amd64 arm arm64 ppc64le s390x
 ALL_PLATFORMS=$(addprefix linux/,$(ALL_ARCH))
-ALL_BINARIES ?= $(addprefix $(OUT_DIR)/$(BIN)-, \
+ALL_BINARIES ?= $(addprefix $(OUT_DIR)/$(PROGRAM_NAME)-, \
 				$(addprefix linux-,$(ALL_ARCH)) \
 				darwin-amd64 \
 				windows-amd64.exe)
@@ -33,29 +34,29 @@ check-license:
 
 crossbuild: $(ALL_BINARIES)
 
-$(OUT_DIR)/$(BIN): $(OUT_DIR)/$(BIN)-$(GOOS)-$(GOARCH)
-	cp $(OUT_DIR)/$(BIN)-$(GOOS)-$(GOARCH) $(OUT_DIR)/$(BIN)
+$(OUT_DIR)/$(PROGRAM_NAME): $(OUT_DIR)/$(PROGRAM_NAME)-$(GOOS)-$(GOARCH)
+	cp $(OUT_DIR)/$(PROGRAM_NAME)-$(GOOS)-$(GOARCH) $(OUT_DIR)/$(PROGRAM_NAME)
 
-$(OUT_DIR)/$(BIN)-%:
-	@echo ">> building for $(GOOS)/$(GOARCH) to $(OUT_DIR)/$(BIN)-$*"
+$(OUT_DIR)/$(PROGRAM_NAME)-%:
+	@echo ">> building for $(GOOS)/$(GOARCH) to $(OUT_DIR)/$(PROGRAM_NAME)-$*"
 	GOARCH=$(word 2,$(subst -, ,$(*:.exe=))) \
 	GOOS=$(word 1,$(subst -, ,$(*:.exe=))) \
 	CGO_ENABLED=0 \
-	go build --installsuffix cgo -o $(OUT_DIR)/$(BIN)-$* $(GITHUB_URL)/cmd/kube-rbac-proxy
+	go build --installsuffix cgo -ldflags="-X k8s.io/component-base/version.gitVersion=$(VERSION_SEMVER) -X k8s.io/component-base/version.gitCommit=$(shell git rev-parse HEAD) -X k8s.io/component-base/version/verflag.programName=$(PROGRAM_NAME)" -o $(OUT_DIR)/$(PROGRAM_NAME)-$* $(GITHUB_URL)/cmd/kube-rbac-proxy
 
 clean:
 	-rm -r $(OUT_DIR)
 
-build: clean $(OUT_DIR)/$(BIN)
+build: clean $(OUT_DIR)/$(PROGRAM_NAME)
 
 update-go-deps:
 	@for m in $$(go list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
 		go get -d $$m; \
 	done
 	go mod tidy
 
-container: $(OUT_DIR)/$(BIN)-$(GOOS)-$(GOARCH) Dockerfile
-	docker build --build-arg BINARY=$(BIN)-$(GOOS)-$(GOARCH) --build-arg GOARCH=$(GOARCH) -t $(CONTAINER_NAME)-$(GOARCH) .
+container: $(OUT_DIR)/$(PROGRAM_NAME)-$(GOOS)-$(GOARCH) Dockerfile
+	docker build --build-arg BINARY=$(PROGRAM_NAME)-$(GOOS)-$(GOARCH) --build-arg GOARCH=$(GOARCH) -t $(CONTAINER_NAME)-$(GOARCH) .
 ifeq ($(GOARCH), amd64)
 	docker tag $(DOCKER_REPO):$(VERSION)-$(GOARCH) $(CONTAINER_NAME)
 endif
@@ -94,20 +95,17 @@ test-unit:
 test-e2e:
 	go test -timeout 55m -v ./test/e2e/ $(TEST_RUN_ARGS) --kubeconfig=$(KUBECONFIG)
 
-test-local-setup: clean $(OUT_DIR)/$(BIN)-$(GOOS)-$(GOARCH) Dockerfile
-	docker build --build-arg BINARY=$(BIN)-$(GOOS)-$(GOARCH) \
-	  --build-arg GOOS=$(GOOS) --build-arg GOARCH=$(GOARCH) \
-	  -t $(CONTAINER_NAME)-$(GOARCH) .
-	docker tag $(DOCKER_REPO):$(VERSION)-$(GOARCH) $(DOCKER_REPO):local
-
-test-local: test-local-setup kind-create-cluster test
+test-local-setup: VERSION = local
+test-local-setup: VERSION_SEMVER = $(shell cat VERSION)
+test-local-setup: clean container kind-create-cluster
+test-local: test-local-setup test
 
 kind-delete-cluster:
 	kind delete cluster
 
 kind-create-cluster: kind-delete-cluster
 	kind create cluster --config ./test/e2e/kind-config/kind-config.yaml
-	kind load docker-image $(DOCKER_REPO):local
+	kind load docker-image $(CONTAINER_NAME)
 
 generate: build $(EMBEDMD_BINARY)
 	@echo ">> generating examples"

README.md

@@ -62,6 +62,9 @@ Kube-rbac-proxy flags:
       --auth-token-audiences strings                Comma-separated list of token audiences to accept. By default a token does not have to have any specific audience. It is recommended to set a specific audience.
       --client-ca-file string                       If set, any request presenting a client certificate signed by one of the authorities in the client-ca-file is authenticated with an identity corresponding to the CommonName of the client certificate.
       --config-file string                          Configuration file to configure kube-rbac-proxy.
+      --http2-disable                               Disable HTTP/2 support
+      --http2-max-concurrent-streams uint32         The maximum number of concurrent streams per HTTP/2 connection. (default 100)
+      --http2-max-size uint32                       The maximum number of bytes that the server will accept for frame size and buffer per stream in a HTTP/2 request. (default 262144)
       --ignore-paths strings                        Comma-separated list of paths against which kube-rbac-proxy pattern-matches the incoming request. If the requst matches, it will proxy the request without performing an authentication or authorization check. Cannot be used with --allow-paths.
       --insecure-listen-address string              [DEPRECATED] The address the kube-rbac-proxy HTTP server should listen on.
       --kubeconfig string                           Path to a kubeconfig file, specifying how to connect to the API server. If unset, in-cluster configuration will be used

RELEASE.md

@@ -7,9 +7,12 @@ Release shepherds are chosen on a voluntary basis.
 
 | release series | date of release (year-month-day) | release shepherd                            |
 |----------------|----------------------------------|---------------------------------------------|
-| v0.15.0        | TBD                              | Krzysztof Ostrowski (GitHub: @ibihim)       |
-| v0.14.2        | 2022-06-05                       | Krzysztof Ostrowski (GitHub: @ibihim)       |
-| v0.14.1        | 2022-04-06                       | Krzysztof Ostrowski (GitHub: @ibihim)       |
+| v0.16.0        | TBD                              | Krzysztof Ostrowski (GitHub: @ibihim)       |
+| v0.15.0        | 2023-10-20                       | Krzysztof Ostrowski (GitHub: @ibihim)       |
+| v0.14.4        | 2023-10-16                       | Krzysztof Ostrowski (GitHub: @ibihim)       |
+| v0.14.3        | 2023-09-07                       | Krzysztof Ostrowski (GitHub: @ibihim)       |
+| v0.14.2        | 2023-06-05                       | Krzysztof Ostrowski (GitHub: @ibihim)       |
+| v0.14.1        | 2023-04-06                       | Krzysztof Ostrowski (GitHub: @ibihim)       |
 | v0.14.0        | 2022-12-15                       | Krzysztof Ostrowski (GitHub: @ibihim)       |
 | v0.13.1        | 2022-10-04                       | Krzysztof Ostrowski (GitHub: @ibihim)       |
 | v0.13.0        | 2022-06-29                       | Krzysztof Ostrowski (GitHub: @ibihim)       |

VERSION

@@ -1 +1 @@
-v0.14.2
+v0.15.0

cmd/kube-rbac-proxy/app/kube-rbac-proxy.go

@@ -135,6 +135,9 @@ type completedProxyRunOptions struct {
 	upstreamForceH2C bool
 	upstreamCABundle *x509.CertPool
 
+	http2Disable bool
+	http2Options *http2.Server
+
 	auth *proxy.Config
 	tls  *options.TLSConfig
 
@@ -251,6 +254,15 @@ func Complete(o *options.ProxyRunOptions) (*completedProxyRunOptions, error) {
 		return nil, fmt.Errorf("failed to instantiate Kubernetes client: %w", err)
 	}
 
+	completed.http2Disable = o.HTTP2Disable
+	completed.http2Options = &http2.Server{
+		IdleTimeout:                  90 * time.Second,
+		MaxConcurrentStreams:         o.HTTP2MaxConcurrentStreams,
+		MaxReadFrameSize:             o.HTTP2MaxSize,
+		MaxUploadBufferPerStream:     int32(o.HTTP2MaxSize),
+		MaxUploadBufferPerConnection: int32(o.HTTP2MaxSize) * int32(o.HTTP2MaxConcurrentStreams),
+	}
+
 	return completed, nil
 }
 
@@ -362,7 +374,10 @@ func Run(cfg *completedProxyRunOptions) error {
 	var gr run.Group
 	{
 		if cfg.secureListenAddress != "" {
-			srv := &http.Server{Handler: mux, TLSConfig: &tls.Config{}}
+			srv := &http.Server{
+				Handler:   mux,
+				TLSConfig: &tls.Config{},
+			}
 
 			if cfg.tls.CertFile == "" && cfg.tls.KeyFile == "" {
 				klog.Info("Generating self signed cert as no cert is provided")
@@ -411,8 +426,21 @@ func Run(cfg *completedProxyRunOptions) error {
 			srv.TLSConfig.MinVersion = version
 			srv.TLSConfig.ClientAuth = tls.RequestClientCert
 
-			if err := http2.ConfigureServer(srv, nil); err != nil {
-				return fmt.Errorf("failed to configure http2 server: %w", err)
+			if cfg.http2Disable {
+				// HTTP/2 is temporarily disabled due to CVE-2023-44487
+				// Programs that must disable HTTP/2 can do so by setting
+				// Transport.TLSNextProto (for clients) or Server.TLSNextProto
+				// (for servers) to a non-nil, empty map.
+				// https://pkg.go.dev/net/http
+				srv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+
+				// For reference:
+				// https://github.com/kubernetes/kubernetes/blob/de054fbf9422d778568946de21a48c7330a6c1b7/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go#L55-L59
+				srv.TLSConfig.NextProtos = []string{"http/1.1"}
+			} else {
+				if err := http2.ConfigureServer(srv, cfg.http2Options); err != nil {
+					return fmt.Errorf("failed to configure http2 server: %w", err)
+				}
 			}
 
 			gr.Add(func() error {
@@ -441,8 +469,20 @@ func Run(cfg *completedProxyRunOptions) error {
 					TLSConfig: srv.TLSConfig.Clone(),
 				}
 
-				if err := http2.ConfigureServer(proxyEndpointsSrv, nil); err != nil {
-					return fmt.Errorf("failed to configure http2 server: %w", err)
+				if cfg.http2Disable {
+					// HTTP/2 is temporarily disabled due to CVE-2023-44487
+					// Programs that must disable HTTP/2 can do so by setting
+					// Transport.TLSNextProto (for clients) or Server.TLSNextProto
+					// (for servers) to a non-nil, empty map.
+					// https://pkg.go.dev/net/http
+					srv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+					// For reference:
+					// https://github.com/kubernetes/kubernetes/blob/de054fbf9422d778568946de21a48c7330a6c1b7/staging/src/k8s.io/apiserver/pkg/server/secure_serving.go#L55-L59
+					srv.TLSConfig.NextProtos = []string{"http/1.1"}
+				} else {
+					if err := http2.ConfigureServer(proxyEndpointsSrv, cfg.http2Options); err != nil {
+						return fmt.Errorf("failed to configure http2 server: %w", err)
+					}
 				}
 
 				gr.Add(func() error {
@@ -472,7 +512,12 @@ func Run(cfg *completedProxyRunOptions) error {
 	}
 	{
 		if cfg.insecureListenAddress != "" {
-			srv := &http.Server{Handler: h2c.NewHandler(mux, &http2.Server{})}
+			srv := &http.Server{}
+			if cfg.http2Disable {
+				srv.Handler = mux
+			} else {
+				srv.Handler = h2c.NewHandler(mux, cfg.http2Options)
+			}
 
 			l, err := net.Listen("tcp", cfg.insecureListenAddress)
 			if err != nil {

cmd/kube-rbac-proxy/app/options/options.go

@@ -41,6 +41,10 @@ type ProxyRunOptions struct {
 	KubeconfigLocation string
 	AllowPaths         []string
 	IgnorePaths        []string
+
+	HTTP2Disable              bool
+	HTTP2MaxConcurrentStreams uint32
+	HTTP2MaxSize              uint32
 }
 
 type TLSConfig struct {
@@ -113,5 +117,10 @@ func (o *ProxyRunOptions) Flags() k8sapiflag.NamedFlagSets {
 	//Kubeconfig flag
 	flagset.StringVar(&o.KubeconfigLocation, "kubeconfig", "", "Path to a kubeconfig file, specifying how to connect to the API server. If unset, in-cluster configuration will be used")
 
+	// HTTP2 flags
+	flagset.BoolVar(&o.HTTP2Disable, "http2-disable", false, "Disable HTTP/2 support")
+	flagset.Uint32Var(&o.HTTP2MaxConcurrentStreams, "http2-max-concurrent-streams", 100, "The maximum number of concurrent streams per HTTP/2 connection.")
+	flagset.Uint32Var(&o.HTTP2MaxSize, "http2-max-size", 256*1024, "The maximum number of bytes that the server will accept for frame size and buffer per stream in a HTTP/2 request.")
+
 	return namedFlagSets
 }

examples/non-resource-url-token-request/README.md

@@ -78,7 +78,7 @@ spec:
       serviceAccountName: kube-rbac-proxy
       containers:
       - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.14.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8081/"

examples/non-resource-url-token-request/deployment.yaml

@@ -63,7 +63,7 @@ spec:
       serviceAccountName: kube-rbac-proxy
       containers:
       - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.14.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8081/"

examples/non-resource-url/README.md

@@ -81,7 +81,7 @@ spec:
       serviceAccountName: kube-rbac-proxy
       containers:
       - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.14.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8081/"

examples/non-resource-url/deployment.yaml

@@ -63,7 +63,7 @@ spec:
       serviceAccountName: kube-rbac-proxy
       containers:
       - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.14.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8081/"

examples/oidc/deployment.yaml

@@ -66,7 +66,7 @@ spec:
       serviceAccountName: kube-rbac-proxy
       containers:
       - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.14.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         args:
         - "--insecure-listen-address=0.0.0.0:8444"
         - "--upstream=http://127.0.0.1:8081/"

examples/resource-attributes/README.md

@@ -95,7 +95,7 @@ spec:
       serviceAccountName: kube-rbac-proxy
       containers:
       - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.14.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8081/"

examples/resource-attributes/deployment.yaml

@@ -77,7 +77,7 @@ spec:
       serviceAccountName: kube-rbac-proxy
       containers:
       - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.14.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8081/"

examples/rewrites/README.md

@@ -97,7 +97,7 @@ spec:
       serviceAccountName: kube-rbac-proxy
       containers:
       - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.14.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8081/"

examples/rewrites/deployment.yaml

@@ -79,7 +79,7 @@ spec:
       serviceAccountName: kube-rbac-proxy
       containers:
       - name: kube-rbac-proxy
-        image: quay.io/brancz/kube-rbac-proxy:v0.14.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.15.0
         args:
         - "--secure-listen-address=0.0.0.0:8443"
         - "--upstream=http://127.0.0.1:8081/"