Merge pull request kubernetes-csi#26 from jsafrane/rebase-v3.1.0

Bug 1912237: Rebase to v3.1.0 for OCP 4.7

Dockerfile.openshift

@@ -3,6 +3,6 @@ WORKDIR /go/src/github.com/kubernetes-csi/external-attacher
 COPY . .
 RUN make build
 
-FROM registry.svc.ci.openshift.org/openshift/origin-v4.6:base
+FROM registry.svc.ci.openshift.org/openshift/origin-v4.7:base
 COPY --from=builder /go/src/github.com/kubernetes-csi/external-attacher/bin/csi-attacher /usr/bin/
 ENTRYPOINT ["/usr/bin/csi-attacher"]

README.md

@@ -68,7 +68,7 @@ Note that the external-attacher does not scale with more replicas. Only one exte
 
 * `--retry-interval-max`: The exponential backoff maximum value. See [CSI error and timeout handling](#csi-error-and-timeout-handling) for details. 5 minutes is used by default.
 
-* `--metrics-address`: The TCP network address where the prometheus metrics endpoint will run (example: `:8080` which corresponds to port 8080 on local host). The default is empty string, which means metrics endpoint is disabled.
+* `--http-endpoint`: The TCP network address where the HTTP server for diagnostics, including metrics and leader election health check, will listen (example: `:8080` which corresponds to port 8080 on local host). The default is empty string, which means the server is disabled.
 
 * `--metrics-path`: The HTTP path where prometheus metrics will be exposed. Default is `/metrics`.
 
@@ -78,6 +78,8 @@ Note that the external-attacher does not scale with more replicas. Only one exte
 
 * `--kubeconfig <path>`: Path to Kubernetes client configuration that the external-attacher uses to connect to Kubernetes API server. When omitted, default token provided by Kubernetes will be used. This option is useful only when the external-attacher does not run as a Kubernetes pod, e.g. for debugging.
 
+* `--metrics-address`: (deprecated) The TCP network address where the prometheus metrics endpoint and leader election health check will run (example: `:8080` which corresponds to port 8080 on local host). The default is empty string, which means metrics and leader election check endpoint is disabled.
+
 * `--resync <duration>`: Internal resync interval when the external-attacher re-evaluates all existing `VolumeAttachment` instances and tries to fulfill them, i.e. attach / detach corresponding volumes. It does not affect re-tries of failed CSI calls! It should be used only when there is a bug in Kubernetes watch logic.
 
 * `--version`: Prints current external-attacher version and quits.
@@ -99,6 +101,13 @@ Correct timeout value depends on the storage backend and how quickly it is able
 
 When CSI driver supports `LIST_VOLUMES` and `LIST_VOLUMES_PUBLISHED_NODES` capabilities, the external attacher periodically syncs volume attachments requested by Kubernetes with the actual state reported by CSI driver. Volumes detached by any 3rd party, but still required to be attached by Kubernetes, will be re-attached back. Frequency of this re-sync is controlled by `--reconcile-sync` command line parameter.
 
+### HTTP endpoint
+
+The external-attacher optionally exposes an HTTP endpoint at address:port specified by `--http-endpoint` argument. When set, these two paths are exposed:
+
+* Metrics path, as set by `--metrics-path` argument (default is `/metrics`).
+* Leader election health check at `/healthz/leader-election`. It is recommended to run a liveness probe against this endpoint when leader election is used to kill external-attacher leader that fails to connect to the API server to renew its leadership. See https://github.com/kubernetes-csi/csi-lib-utils/issues/66 for details.
+
 ## Community, discussion, contribution, and support
 
 Learn how to engage with the Kubernetes community on the [community page](http://kubernetes.io/community/).

cmd/csi-attacher/main.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 	"time"
 
@@ -66,7 +67,8 @@ var (
 
 	reconcileSync = flag.Duration("reconcile-sync", 1*time.Minute, "Resync interval of the VolumeAttachment reconciler.")
 
-	metricsAddress = flag.String("metrics-address", "", "The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means metrics endpoint is disabled.")
+	metricsAddress = flag.String("metrics-address", "", "(deprecated) The TCP network address where the prometheus metrics endpoint will listen (example: `:8080`). The default is empty string, which means metrics endpoint is disabled. Only one of `--metrics-address` and `--http-endpoint` can be set.")
+	httpEndpoint   = flag.String("http-endpoint", "", "The TCP network address where the HTTP server for diagnostics, including metrics and leader election health check, will listen (example: `:8080`). The default is empty string, which means the server is disabled. Only one of `--metrics-address` and `--http-endpoint` can be set.")
 	metricsPath    = flag.String("metrics-path", "/metrics", "The HTTP path where prometheus metrics will be exposed. Default is `/metrics`.")
 )
 
@@ -90,6 +92,15 @@ func main() {
 	}
 	klog.Infof("Version: %s", version)
 
+	if *metricsAddress != "" && *httpEndpoint != "" {
+		klog.Error("only one of `--metrics-address` and `--http-endpoint` can be set.")
+		os.Exit(1)
+	}
+	addr := *metricsAddress
+	if addr == "" {
+		addr = *httpEndpoint
+	}
+
 	// Create the client config. Use kubeconfig if given, otherwise assume in-cluster.
 	config, err := buildConfig(*kubeconfig)
 	if err != nil {
@@ -134,8 +145,20 @@ func main() {
 		os.Exit(1)
 	}
 	klog.V(2).Infof("CSI driver name: %q", csiAttacher)
-	metricsManager.SetDriverName(csiAttacher)
-	metricsManager.StartMetricsEndpoint(*metricsAddress, *metricsPath)
+
+	// Prepare http endpoint for metrics + leader election healthz
+	mux := http.NewServeMux()
+	if addr != "" {
+		metricsManager.RegisterToServer(mux, *metricsPath)
+		metricsManager.SetDriverName(csiAttacher)
+		go func() {
+			klog.Infof("ServeMux listening at %q", addr)
+			err := http.ListenAndServe(addr, mux)
+			if err != nil {
+				klog.Fatalf("Failed to start HTTP server at specified address (%q) and metrics path (%q): %s", addr, *metricsPath, err)
+			}
+		}()
+	}
 
 	supportsService, err := supportsPluginControllerService(ctx, csiConn)
 	if err != nil {
@@ -207,6 +230,9 @@ func main() {
 		// Name of config map with leader election lock
 		lockName := "external-attacher-leader-" + csiAttacher
 		le := leaderelection.NewLeaderElection(leClientset, lockName, run)
+		if *httpEndpoint != "" {
+			le.PrepareHealthCheck(mux, leaderelection.DefaultHealthCheckTimeout)
+		}
 
 		if *leaderElectionNamespace != "" {
 			le.WithNamespace(*leaderElectionNamespace)

deploy/kubernetes/deployment.yaml

@@ -33,11 +33,12 @@ spec:
       serviceAccount: csi-attacher
       containers:
         - name: csi-attacher
-          image: quay.io/k8scsi/csi-attacher:canary
+          image: gcr.io/k8s-staging-sig-storage/csi-attacher:canary
           args:
             - "--v=5"
             - "--csi-address=$(ADDRESS)"
             - "--leader-election"
+            - "--http-endpoint=:8080"
           env:
             - name: MY_NAME
               valueFrom:
@@ -49,7 +50,18 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
-
+          ports:
+            - containerPort: 8080
+              name: http-endpoint
+              protocol: TCP
+          livenessProbe:
+            failureThreshold: 1
+            httpGet:
+              path: /healthz/leader-election
+              port: http-endpoint
+            initialDelaySeconds: 10
+            timeoutSeconds: 10
+            periodSeconds: 20
         - name: mock-driver
           image: quay.io/k8scsi/mock-driver:canary
           imagePullPolicy: "IfNotPresent"

go.mod

@@ -3,58 +3,89 @@ module github.com/kubernetes-csi/external-attacher
 go 1.15
 
 require (
-	github.com/container-storage-interface/spec v1.2.0
+	github.com/container-storage-interface/spec v1.3.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/evanphx/json-patch v4.9.0+incompatible
-	github.com/golang/mock v1.4.3
-	github.com/golang/protobuf v1.4.2
-	github.com/imdario/mergo v0.3.9 // indirect
-	github.com/kubernetes-csi/csi-lib-utils v0.7.0
-	github.com/kubernetes-csi/csi-test/v3 v3.1.0
-	google.golang.org/grpc v1.28.0
-	k8s.io/api v0.19.0
-	k8s.io/apimachinery v0.19.0
-	k8s.io/client-go v0.19.0
-	k8s.io/csi-translation-lib v0.19.0
-	k8s.io/klog/v2 v2.2.0
+	github.com/go-logr/logr v0.3.0 // indirect
+	github.com/golang/mock v1.4.4
+	github.com/golang/protobuf v1.4.3
+	github.com/google/go-cmp v0.5.4 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/googleapis/gnostic v0.5.3 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/imdario/mergo v0.3.11 // indirect
+	github.com/kubernetes-csi/csi-lib-utils v0.9.0
+	github.com/kubernetes-csi/csi-test/v4 v4.0.2
+	github.com/prometheus/client_golang v1.8.0 // indirect
+	github.com/prometheus/common v0.15.0 // indirect
+	golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9 // indirect
+	golang.org/x/net v0.0.0-20201216054612-986b41b23924 // indirect
+	golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5 // indirect
+	golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e // indirect
+	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf // indirect
+	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d // indirect
+	google.golang.org/grpc v1.34.0
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/api v0.20.0
+	k8s.io/apimachinery v0.20.0
+	k8s.io/client-go v0.20.0
+	k8s.io/component-base v0.20.0 // indirect
+	k8s.io/csi-translation-lib v0.20.0
+	k8s.io/klog/v2 v2.4.0
 )
 
-replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.17.0
+replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.0
 
-replace k8s.io/apiserver => k8s.io/apiserver v0.17.0
+replace k8s.io/apiserver => k8s.io/apiserver v0.20.0
 
-replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.17.0
+replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.0
 
-replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.17.0
+replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.0
 
-replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.17.0
+replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.0
 
-replace k8s.io/code-generator => k8s.io/code-generator v0.17.1-beta.0
+replace k8s.io/code-generator => k8s.io/code-generator v0.20.0
 
-replace k8s.io/component-base => k8s.io/component-base v0.17.0
+replace k8s.io/component-base => k8s.io/component-base v0.20.0
 
-replace k8s.io/cri-api => k8s.io/cri-api v0.17.1-beta.0
+replace k8s.io/cri-api => k8s.io/cri-api v0.20.0
 
-replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.0
+replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.0
 
-replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.0
+replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.0
 
-replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.0
+replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.0
 
-replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.0
+replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.0
 
-replace k8s.io/kubectl => k8s.io/kubectl v0.17.0
+replace k8s.io/kubectl => k8s.io/kubectl v0.20.0
 
-replace k8s.io/kubelet => k8s.io/kubelet v0.17.0
+replace k8s.io/kubelet => k8s.io/kubelet v0.20.0
 
-replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.17.0
+replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.0
 
-replace k8s.io/metrics => k8s.io/metrics v0.17.0
+replace k8s.io/metrics => k8s.io/metrics v0.20.0
 
-replace k8s.io/node-api => k8s.io/node-api v0.17.0
+replace k8s.io/node-api => k8s.io/node-api v0.20.0
 
-replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.17.0
+replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.0
 
-replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.17.0
+replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.20.0
 
-replace k8s.io/sample-controller => k8s.io/sample-controller v0.17.0
+replace k8s.io/sample-controller => k8s.io/sample-controller v0.20.0
+
+replace k8s.io/api => k8s.io/api v0.20.0
+
+replace k8s.io/apimachinery => k8s.io/apimachinery v0.20.0
+
+replace k8s.io/client-go => k8s.io/client-go v0.20.0
+
+replace k8s.io/component-helpers => k8s.io/component-helpers v0.20.0
+
+replace k8s.io/controller-manager => k8s.io/controller-manager v0.20.0
+
+replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.0
+
+replace k8s.io/mount-utils => k8s.io/mount-utils v0.20.0