From e822c4c98a5a7beb5af58cfd6163b79b2639c2ea Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 12 Aug 2019 17:45:56 -0700 Subject: [PATCH 01/24] Add more access modes to volume capabilities --- pkg/nfs/nfs.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index d637b0d28..4a63a64f1 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -31,7 +31,7 @@ type nfsDriver struct { //ids *identityServer ns *nodeServer - cap []*csi.VolumeCapability_AccessMode + cap map[csi.VolumeCapability_AccessMode_Mode]bool cscap []*csi.ControllerServiceCapability } @@ -51,9 +51,18 @@ func NewNFSdriver(nodeID, endpoint string) *nfsDriver { version: version, nodeID: nodeID, endpoint: endpoint, + cap: map[csi.VolumeCapability_AccessMode_Mode]bool{}, } - n.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) + vcam := []csi.VolumeCapability_AccessMode_Mode{ + csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY, + csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY, + csi.VolumeCapability_AccessMode_MULTI_NODE_SINGLE_WRITER, + csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER, + } + n.AddVolumeCapabilityAccessModes(vcam) + // NFS plugin does not support ControllerServiceCapability now. // If support is added, it should set to appropriate // ControllerServiceCapability RPC types. @@ -86,8 +95,8 @@ func (n *nfsDriver) AddVolumeCapabilityAccessModes(vc []csi.VolumeCapability_Acc for _, c := range vc { glog.Infof("Enabling volume access mode: %v", c.String()) vca = append(vca, &csi.VolumeCapability_AccessMode{Mode: c}) + n.cap[c] = true } - n.cap = vca return vca } From e6c1dce07aaf4e1ff40916533592ba2f4329d476 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 14 Feb 2020 09:38:16 +0100 Subject: [PATCH 02/24] prow.sh: use 1.3.0 hostpath driver for testing The final 1.3.0 release of the hostpath driver really uses the 1.3.0 driver image in its deployment, in contrast to the previous -rc candidates which still used 1.2.0. --- release-tools/prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 0331ac3f8..d54dea518 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -193,7 +193,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # If the deployment script is called with CSI_PROW_TEST_DRIVER= as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.3.0-rc4" "CSI driver version" +configvar CSI_PROW_DRIVER_VERSION "v1.3.0" "CSI driver version" configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" From 310bc7c46dffe42c6e6201387ec0c342daaa3ca9 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Fri, 28 Feb 2020 15:08:25 +0100 Subject: [PATCH 03/24] Fix deprecated API group for StatefulSet --- deploy/kubernetes/csi-attacher-nfsplugin.yaml | 5 ++++- deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/csi-attacher-nfsplugin.yaml b/deploy/kubernetes/csi-attacher-nfsplugin.yaml index cb390e216..d94c861c2 100644 --- a/deploy/kubernetes/csi-attacher-nfsplugin.yaml +++ b/deploy/kubernetes/csi-attacher-nfsplugin.yaml @@ -16,12 +16,15 @@ spec: --- kind: StatefulSet -apiVersion: apps/v1beta1 +apiVersion: apps/v1 metadata: name: csi-attacher-nfsplugin spec: serviceName: "csi-attacher" replicas: 1 + selector: + matchLabels: + app: csi-attacher-nfsplugin template: metadata: labels: diff --git a/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml b/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml index c259cd1f9..7c53507eb 100644 --- a/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml +++ b/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml @@ -1,7 +1,7 @@ # This YAML file contains driver-registrar & csi driver nodeplugin API objects # that are necessary to run CSI nodeplugin for nfs kind: DaemonSet -apiVersion: apps/v1beta2 +apiVersion: apps/v1 metadata: name: csi-nodeplugin-nfsplugin spec: From 2959d3760f73eb2c0ad82c7e2e35185987fba5a4 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 4 Mar 2020 11:19:09 +0100 Subject: [PATCH 04/24] build for multiple platforms only in CI, add s390x Developers should not be forced to build for all platforms by default. We also don't want to copy-and-paste the go invocation for each new platform. To address both, the target platform(s) are now configurable via BUILD_PLATFORMS and additional platforms are only enabled in the Prow CI. For now this serves as a test that the source actually compiles for multiple platforms. Building images for different target platforms is a different problem. --- release-tools/build.make | 21 ++++++++++++++------- release-tools/prow.sh | 4 +++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/release-tools/build.make b/release-tools/build.make index 3bf3391c2..5c9793c2b 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -60,18 +60,25 @@ else TESTARGS = endif -ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH)) - # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below. +# BUILD_PLATFORMS contains a set of triplets, +# separated by semicolon. An empty variable or empty entry (= just a +# semicolon) builds for the default platform of the current Go +# toolchain. +BUILD_PLATFORMS = + +# This builds each command (= the sub-directories of ./cmd) for the target platform(s) +# defined by BUILD_PLATFORMS. build-%: check-go-version-go mkdir -p bin - CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* - if [ "$$ARCH" = "amd64" ]; then \ - CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ - CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \ - fi + echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ + if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \ + echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \ + exit 1; \ + fi; \ + done container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . diff --git a/release-tools/prow.sh b/release-tools/prow.sh index d54dea518..30cb38428 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -85,6 +85,8 @@ get_versioned_variable () { echo "$value" } +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" + # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because # setting it globally can break other go usages (like "go get " @@ -1026,7 +1028,7 @@ main () { images= if ${CSI_PROW_BUILD_JOB}; then # A successful build is required for testing. - run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make all' failed" + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" "BUILD_PLATFORMS=${CSI_PROW_BUILD_PLATFORMS}" || die "'make all' failed" # We don't want test failures to prevent E2E testing below, because the failure # might have been minor or unavoidable, for example when experimenting with # changes in "release-tools" in a PR (that fails the "is release-tools unmodified" From 848599cfd6482fc8eb77eccde1881426202ba699 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Thu, 5 Mar 2020 16:47:34 +0100 Subject: [PATCH 05/24] Remove csi-attacher --- deploy/kubernetes/csi-attacher-nfsplugin.yaml | 66 ------------------- deploy/kubernetes/csi-attacher-rbac.yaml | 37 ----------- deploy/kubernetes/csi-nfs-driverinfo.yaml | 9 +++ 3 files changed, 9 insertions(+), 103 deletions(-) delete mode 100644 deploy/kubernetes/csi-attacher-nfsplugin.yaml delete mode 100644 deploy/kubernetes/csi-attacher-rbac.yaml create mode 100644 deploy/kubernetes/csi-nfs-driverinfo.yaml diff --git a/deploy/kubernetes/csi-attacher-nfsplugin.yaml b/deploy/kubernetes/csi-attacher-nfsplugin.yaml deleted file mode 100644 index d94c861c2..000000000 --- a/deploy/kubernetes/csi-attacher-nfsplugin.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# This YAML file contains attacher & csi driver API objects that are necessary -# to run external CSI attacher for nfs - -kind: Service -apiVersion: v1 -metadata: - name: csi-attacher-nfsplugin - labels: - app: csi-attacher-nfsplugin -spec: - selector: - app: csi-attacher-nfsplugin - ports: - - name: dummy - port: 12345 - ---- -kind: StatefulSet -apiVersion: apps/v1 -metadata: - name: csi-attacher-nfsplugin -spec: - serviceName: "csi-attacher" - replicas: 1 - selector: - matchLabels: - app: csi-attacher-nfsplugin - template: - metadata: - labels: - app: csi-attacher-nfsplugin - spec: - serviceAccount: csi-attacher - containers: - - name: csi-attacher - image: quay.io/k8scsi/csi-attacher:v1.0.1 - args: - - "--v=5" - - "--csi-address=$(ADDRESS)" - env: - - name: ADDRESS - value: /csi/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /csi - - - name: nfs - image: quay.io/k8scsi/nfsplugin:v1.0.0 - args : - - "--nodeid=$(NODE_ID)" - - "--endpoint=$(CSI_ENDPOINT)" - env: - - name: NODE_ID - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix://plugin/csi.sock - imagePullPolicy: "IfNotPresent" - volumeMounts: - - name: socket-dir - mountPath: /plugin - volumes: - - name: socket-dir - emptyDir: diff --git a/deploy/kubernetes/csi-attacher-rbac.yaml b/deploy/kubernetes/csi-attacher-rbac.yaml deleted file mode 100644 index 975fdd67f..000000000 --- a/deploy/kubernetes/csi-attacher-rbac.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# This YAML file contains RBAC API objects that are necessary to run external -# CSI attacher for nfs flex adapter - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-attacher - ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: external-attacher-runner -rules: - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update"] - ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: csi-attacher-role -subjects: - - kind: ServiceAccount - name: csi-attacher - namespace: default -roleRef: - kind: ClusterRole - name: external-attacher-runner - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/csi-nfs-driverinfo.yaml b/deploy/kubernetes/csi-nfs-driverinfo.yaml new file mode 100644 index 000000000..ac5cf0ce6 --- /dev/null +++ b/deploy/kubernetes/csi-nfs-driverinfo.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: CSIDriver +metadata: + name: csi-nfsplugin +spec: + attachRequired: false + volumeLifecycleModes: + - Persistent + podInfoOnMount: true From 4b3176d4ceda600e396054cfdeab8415648556ba Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Fri, 6 Mar 2020 12:41:34 +0100 Subject: [PATCH 06/24] Update driver version --- pkg/nfs/nfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index 4a63a64f1..a39074a29 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -40,7 +40,7 @@ const ( ) var ( - version = "1.0.0-rc2" + version = "1.0.0" ) func NewNFSdriver(nodeID, endpoint string) *nfsDriver { From deea989920e3eecefe71fd168ef19f4392bcc6e8 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Wed, 11 Mar 2020 10:48:03 -0700 Subject: [PATCH 07/24] Update snapshotter to version 2.0.1 Signed-off-by: Grant Griffiths --- release-tools/prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 30cb38428..86b2bf058 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -342,7 +342,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kub configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0' "external-snapshotter version tag" +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply From a9b0c0658b30050159f8dd51f1fd67c990464c47 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Sun, 15 Mar 2020 13:01:35 +0100 Subject: [PATCH 08/24] Rename driver to follow CSI plugin naming conventions --- deploy/kubernetes/csi-nfs-driverinfo.yaml | 2 +- examples/kubernetes/nginx.yaml | 2 +- pkg/nfs/nfs.go | 2 +- test/nfs-testdriver.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/kubernetes/csi-nfs-driverinfo.yaml b/deploy/kubernetes/csi-nfs-driverinfo.yaml index ac5cf0ce6..7b7c0e5b3 100644 --- a/deploy/kubernetes/csi-nfs-driverinfo.yaml +++ b/deploy/kubernetes/csi-nfs-driverinfo.yaml @@ -1,7 +1,7 @@ apiVersion: storage.k8s.io/v1beta1 kind: CSIDriver metadata: - name: csi-nfsplugin + name: nfs.csi.k8s.io spec: attachRequired: false volumeLifecycleModes: diff --git a/examples/kubernetes/nginx.yaml b/examples/kubernetes/nginx.yaml index 8048e9cd6..2fd0c1aac 100644 --- a/examples/kubernetes/nginx.yaml +++ b/examples/kubernetes/nginx.yaml @@ -10,7 +10,7 @@ spec: capacity: storage: 100Gi csi: - driver: csi-nfsplugin + driver: nfs.csi.k8s.io volumeHandle: data-id volumeAttributes: server: 127.0.0.1 diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index a39074a29..5422fd29a 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -36,7 +36,7 @@ type nfsDriver struct { } const ( - driverName = "csi-nfsplugin" + driverName = "nfs.csi.k8s.io" ) var ( diff --git a/test/nfs-testdriver.go b/test/nfs-testdriver.go index 290f54c62..22a7d6ae6 100644 --- a/test/nfs-testdriver.go +++ b/test/nfs-testdriver.go @@ -54,7 +54,7 @@ func initNFSDriver(name string, manifests ...string) testsuites.TestDriver { func InitNFSDriver() testsuites.TestDriver { - return initNFSDriver("csi-nfsplugin", + return initNFSDriver("nfs.csi.k8s.io", "csi-attacher-nfsplugin.yaml", "csi-attacher-rbac.yaml", "csi-nodeplugin-nfsplugin.yaml", From 9d1c777a5d3dfe81e34c9af5ea84d42d41f947f1 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Sun, 15 Mar 2020 15:20:57 +0100 Subject: [PATCH 09/24] Change base image to centos:latest --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 13f5d2485..489fbc26b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM centos:7.4.1708 +FROM centos:latest # Copy nfsplugin from build _output directory COPY bin/nfsplugin /nfsplugin -RUN yum -y install nfs-utils && yum -y install epel-release && yum -y install jq && yum clean all +RUN yum -y install nfs-utils epel-release jq && yum clean all ENTRYPOINT ["/nfsplugin"] From 61140cb6da18c0e6c5d09dd1d6350da28661c1e1 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 16 Mar 2020 17:49:51 -0700 Subject: [PATCH 10/24] Add .prow.sh to enable prow jobs Change-Id: I1142fc76b939c03ba05070179b5ab9574c096571 --- .prow.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 .prow.sh diff --git a/.prow.sh b/.prow.sh new file mode 100755 index 000000000..c9947b93c --- /dev/null +++ b/.prow.sh @@ -0,0 +1,11 @@ +#! /bin/bash + +# A Prow job can override these defaults, but this shouldn't be necessary. + +# Only these tests make sense for csi-driver-nfs until we can integrate k/k +# e2es. +: ${CSI_PROW_TESTS:="unit"} + +. release-tools/prow.sh + +main From 4f97fc8e9e1594cdc5aec691df3fbd8e0f1394d7 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Wed, 25 Mar 2020 10:01:05 +0100 Subject: [PATCH 11/24] Prepare for v2.0.0 release --- deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml | 2 +- pkg/nfs/nfs.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml b/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml index 7c53507eb..bb36931bf 100644 --- a/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml +++ b/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml @@ -42,7 +42,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: quay.io/k8scsi/nfsplugin:v1.0.0 + image: quay.io/k8scsi/nfsplugin:v2.0.0 args : - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index 5422fd29a..728132c17 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -40,7 +40,7 @@ const ( ) var ( - version = "1.0.0" + version = "2.0.0" ) func NewNFSdriver(nodeID, endpoint string) *nfsDriver { From 89a7eebd481886575b127acf98cf92707d41223e Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Sun, 15 Mar 2020 14:33:39 +0100 Subject: [PATCH 12/24] Add info about plugin compatibility and features --- README.md | 94 +++++++++++++++++++++++++------------------------------ 1 file changed, 43 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 4d3e76473..d8025ab80 100644 --- a/README.md +++ b/README.md @@ -1,79 +1,71 @@ # CSI NFS driver -## Kubernetes -### Requirements +## Overview -The folllowing feature gates and runtime config have to be enabled to deploy the driver +This is a repository for [NFS](https://en.wikipedia.org/wiki/Network_File_System) [CSI](https://kubernetes-csi.github.io/docs/) Driver. +Currently it implements bare minimum of the [CSI spec](https://github.com/container-storage-interface/spec) and is in the alpha state +of the development. -``` -FEATURE_GATES=CSIPersistentVolume=true,MountPropagation=true -RUNTIME_CONFIG="storage.k8s.io/v1alpha1=true" -``` +#### CSI Feature matrix -Mountprogpation requries support for privileged containers. So, make sure privileged containers are enabled in the cluster. +| **nfs.csi.k8s.io** | K8s version compatibility | CSI versions compatibility | Dynamic Provisioning | Resize | Snapshots | Raw Block | AccessModes | Status | +|--------------------|---------------------------|----------------------------|----------------------|--------|-----------|-----------|--------------------------|------------------------------------------------------------------------------| +|master | 1.14 + | v1.0 + | no | no | no | no | Read/Write Multiple Pods | Alpha | +|v2.0.0 | 1.14 + | v1.0 + | no | no | no | no | Read/Write Multiple Pods | Alpha | +|v1.0.0 | 1.9 - 1.15 | v1.0 | no | no | no | no | Read/Write Multiple Pods | [deprecated](https://github.com/kubernetes-csi/drivers/tree/master/pkg/nfs) | -### Example local-up-cluster.sh +## Requirements -```ALLOW_PRIVILEGED=true FEATURE_GATES=CSIPersistentVolume=true,MountPropagation=true RUNTIME_CONFIG="storage.k8s.io/v1alpha1=true" LOG_LEVEL=5 hack/local-up-cluster.sh``` +The CSI NFS driver requires Kubernetes cluster of version 1.14 or newer and +preexisting NFS server, whether it is deployed on cluster or provisioned +independently. The plugin itself provides only a communication layer between +resources in the cluser and the NFS server. -### Deploy +## Example -```kubectl -f deploy/kubernetes create``` +There are multiple ways to create a kubernetes cluster, the NFS CSI plugin +should work invariantly of your cluster setup. Very simple way of getting +a local environment for testing can be achieved using for example +[kind](https://github.com/kubernetes-sigs/kind). -### Example Nginx application -Please update the NFS Server & share information in nginx.yaml file. +There are also multiple different NFS servers you can use for testing of +the plugin, the major versions of the protocol v2, v3 and v4 should be supported +by the current implementation. -```kubectl -f examples/kubernetes/nginx.yaml create``` +The example assumes you have your cluster created (e.g. `kind create cluster`) +and working NFS server (e.g. https://github.com/rootfs/nfs-ganesha-docker) -## Using CSC tool +#### Deploy -### Build nfsplugin +Deploy the NFS plugin along with the `CSIDriver` info. ``` -$ make nfs +kubectl -f deploy/kubernetes create ``` -### Start NFS driver -``` -$ sudo ./_output/nfsplugin --endpoint tcp://127.0.0.1:10000 --nodeid CSINode -v=5 -``` +#### Example Nginx application -## Test -Get ```csc``` tool from https://github.com/rexray/gocsi/tree/master/csc +The [/examples/kubernetes/nginx.yaml](/examples/kubernetes/nginx.yaml) contains a `PersistentVolume`, +`PersistentVolumeClaim` and an nginx `Pod` mounting the NFS volume under `/var/www`. -#### Get plugin info -``` -$ csc identity plugin-info --endpoint tcp://127.0.0.1:10000 -"NFS" "0.1.0" -``` +You will need to update the NFS Server IP and the share information under +`volumeAttributes` inside `PersistentVolume` in `nginx.yaml` file to match your +NFS server public end point and configuration. You can also provide additional +`mountOptions`, such as protocol version, in the `PersistentVolume` `spec` +relevant for your NFS Server. -#### NodePublish a volume ``` -$ export NFS_SERVER="Your Server IP (Ex: 10.10.10.10)" -$ export NFS_SHARE="Your NFS share" -$ csc node publish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/nfs --attrib server=$NFS_SERVER --attrib share=$NFS_SHARE nfstestvol -nfstestvol +kubectl -f examples/kubernetes/nginx.yaml create ``` -#### NodeUnpublish a volume -``` -$ csc node unpublish --endpoint tcp://127.0.0.1:10000 --target-path /mnt/nfs nfstestvol -nfstestvol -``` - -#### Get NodeID -``` -$ csc node get-id --endpoint tcp://127.0.0.1:10000 -CSINode -``` ## Running Kubernetes End To End tests on an NFS Driver First, stand up a local cluster `ALLOW_PRIVILEGED=1 hack/local-up-cluster.sh` (from your Kubernetes repo) For Fedora/RHEL clusters, the following might be required: - ``` - sudo chown -R $USER:$USER /var/run/kubernetes/ - sudo chown -R $USER:$USER /var/lib/kubelet - sudo chcon -R -t svirt_sandbox_file_t /var/lib/kubelet - ``` +``` +sudo chown -R $USER:$USER /var/run/kubernetes/ +sudo chown -R $USER:$USER /var/lib/kubelet +sudo chcon -R -t svirt_sandbox_file_t /var/lib/kubelet +``` If you are plannig to test using your own private image, you could either install your nfs driver using your own set of YAML files, or edit the existing YAML files to use that private image. When using the [existing set of YAML files](https://github.com/kubernetes-csi/csi-driver-nfs/tree/master/deploy/kubernetes), you would edit the [csi-attacher-nfsplugin.yaml](https://github.com/kubernetes-csi/csi-driver-nfs/blob/master/deploy/kubernetes/csi-attacher-nfsplugin.yaml#L46) and [csi-nodeplugin-nfsplugin.yaml](https://github.com/kubernetes-csi/csi-driver-nfs/blob/master/deploy/kubernetes/csi-nodeplugin-nfsplugin.yaml#L45) files to include your private image instead of the default one. After editing these files, skip to step 3 of the following steps. @@ -81,7 +73,7 @@ When using the [existing set of YAML files](https://github.com/kubernetes-csi/cs If you already have a driver installed, skip to step 4 of the following steps. 1) Build the nfs driver by running `make` -2) Create NFS Driver Image, where the image tag would be whatever that is required by your YAML deployment files `docker build -t quay.io/k8scsi/nfsplugin:v1.0.0 .` +2) Create NFS Driver Image, where the image tag would be whatever that is required by your YAML deployment files `docker build -t quay.io/k8scsi/nfsplugin:v2.0.0 .` 3) Install the Driver: `kubectl create -f deploy/kubernetes` 4) Build E2E test binary: `make build-tests` 5) Run E2E Tests using the following command: `./bin/tests --ginkgo.v --ginkgo.progress --kubeconfig=/var/run/kubernetes/admin.kubeconfig` From 8d2f882603816dc388bd26f2fe70fd9013954071 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 16 Apr 2020 18:38:40 -0700 Subject: [PATCH 13/24] Add 2.0 changelog Change-Id: I369683a10673e9da15998ddd9056890db6154b69 --- CHANGELOG/CHANGELOG-2.0.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 CHANGELOG/CHANGELOG-2.0.md diff --git a/CHANGELOG/CHANGELOG-2.0.md b/CHANGELOG/CHANGELOG-2.0.md new file mode 100644 index 000000000..a50bf8a86 --- /dev/null +++ b/CHANGELOG/CHANGELOG-2.0.md @@ -0,0 +1,22 @@ +# v2.0.0 + +## Breaking Changes + +- Changing name of the driver from "csi-nfsplugin" to "nfs.csi.k8s.io" ([#26](https://github.com/kubernetes-csi/csi-driver-nfs/pull/26), [@wozniakjan](https://github.com/wozniakjan)) + +## New Features + +- Add support for CSI spec 1.0. +- Remove external-attacher and update deployment specs to apps/v1. + ([#24](https://github.com/kubernetes-csi/csi-driver-nfs/pull/24), + [@wozniakjan](https://github.com/wozniakjan)) + +## Bug Fixes + +- Adds support for all access modes. ([#15](https://github.com/kubernetes-csi/csi-driver-nfs/pull/15), [@msau42](https://github.com/msau42)) + +## Other Notable Changes + +- Update base image to centos8. + ([#28](https://github.com/kubernetes-csi/csi-driver-nfs/pull/28), [@wozniakjan](https://github.com/wozniakjan)) +- Switch to go mod and update dependencies. ([#22](https://github.com/kubernetes-csi/csi-driver-nfs/pull/22), [@wozniakjan](https://github.com/wozniakjan)) From c8f1238a513599e48a33d63629d95138ffdf8479 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Fri, 17 Apr 2020 09:02:07 -0700 Subject: [PATCH 14/24] update release tools instructions --- release-tools/SIDECAR_RELEASE_PROCESS.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/release-tools/SIDECAR_RELEASE_PROCESS.md b/release-tools/SIDECAR_RELEASE_PROCESS.md index 91a1e1b82..c8aaa4e8b 100644 --- a/release-tools/SIDECAR_RELEASE_PROCESS.md +++ b/release-tools/SIDECAR_RELEASE_PROCESS.md @@ -54,14 +54,21 @@ naming convention `-on-`. generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) 1. Generate release notes for the release. Replace arguments with the relevant information. - ``` - GITHUB_TOKEN= ./release-notes --start-sha=0ed6978fd199e3ca10326b82b4b8b8e916211c9b --end-sha=3cb3d2f18ed8cb40371c6d8886edcabd1f27e7b9 \ - --github-org=kubernetes-csi --github-repo=external-attacher -branch=master -output out.md - ``` - * `--start-sha` should point to the last release from the same branch. For - example: - * `1.X-1.0` tag when releasing `1.X.0` - * `1.X.Y-1` tag when releasing `1.X.Y` + * For new minor releases on master: + ``` + GITHUB_TOKEN= release-notes --discover=mergebase-to-latest + --github-org=kubernetes-csi --github-repo=external-provisioner + --required-author="" --output out.md + ``` + * For new patch releases on a release branch: + ``` + GITHUB_TOKEN= release-notes --branch=release-1.1 + --start-rev=v1.1.1 --end-sha=f0a9219b29cc9053047c39d149ce9b22bc7b918b + --github-org=kubernetes-csi --github-repo=external-provisioner + --required-author="" --output out.md + ``` + * `--start-rev` should point to the last patch release from the release branch. + * `--end-sha` should point to the latest commit from the release branch. 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. 1. Reword release notes as needed. Make sure to check notes for breaking From 3a09d7dc822ae97c0000ae5d0bfebef63eaf515b Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Wed, 15 Apr 2020 19:11:33 -0700 Subject: [PATCH 15/24] Support local snapshot RBAC for pull jobs Signed-off-by: Grant Griffiths --- release-tools/prow.sh | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 86b2bf058..0b962f044 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -1064,18 +1064,24 @@ main () { # always pulling the image # (https://github.com/kubernetes-sigs/kind/issues/328). docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed" - done - if [ -e deploy/kubernetes/rbac.yaml ]; then - # This is one of those components which has its own RBAC rules (like external-provisioner). - # We are testing a locally built image and also want to test with the the current, - # potentially modified RBAC rules. - if [ "$(echo "$cmds" | wc -w)" != 1 ]; then - die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds" + # For components with multiple cmds, the RBAC file should be in the following format: + # rbac-$cmd.yaml + # If this file cannot be found, we can default to the standard location: + # deploy/kubernetes/rbac.yaml + rbac_file_path=$(find . -type f -name "rbac-$i.yaml") + if [ "$rbac_file_path" == "" ]; then + rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml" fi - e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _) - images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml" - fi + + if [ -e "$rbac_file_path" ]; then + # This is one of those components which has its own RBAC rules (like external-provisioner). + # We are testing a locally built image and also want to test with the the current, + # potentially modified RBAC rules. + e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) + images="$images ${e}_RBAC=$rbac_file_path" + fi + done fi if tests_need_non_alpha_cluster; then From cba5fbb8ce6a942ba027a7aba95903e2b83cf8a7 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 23 Apr 2020 10:21:49 -0700 Subject: [PATCH 16/24] Update patch release notes generation command --- release-tools/SIDECAR_RELEASE_PROCESS.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/release-tools/SIDECAR_RELEASE_PROCESS.md b/release-tools/SIDECAR_RELEASE_PROCESS.md index c8aaa4e8b..6960607ef 100644 --- a/release-tools/SIDECAR_RELEASE_PROCESS.md +++ b/release-tools/SIDECAR_RELEASE_PROCESS.md @@ -50,7 +50,7 @@ naming convention `-on-`. ## Release Process 1. Identify all issues and ongoing PRs that should go into the release, and drive them to resolution. -1. Download [K8s release notes +1. Download v2.8+ [K8s release notes generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) 1. Generate release notes for the release. Replace arguments with the relevant information. @@ -62,13 +62,10 @@ naming convention `-on-`. ``` * For new patch releases on a release branch: ``` - GITHUB_TOKEN= release-notes --branch=release-1.1 - --start-rev=v1.1.1 --end-sha=f0a9219b29cc9053047c39d149ce9b22bc7b918b + GITHUB_TOKEN= release-notes --discover=patch-to-latest --branch=release-1.1 --github-org=kubernetes-csi --github-repo=external-provisioner --required-author="" --output out.md ``` - * `--start-rev` should point to the last patch release from the release branch. - * `--end-sha` should point to the latest commit from the release branch. 1. Compare the generated output to the new commits for the release to check if any notable change missed a release note. 1. Reword release notes as needed. Make sure to check notes for breaking From 5b0bab063885e1f590fb176bda746dfd9d965ab4 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 28 May 2020 10:22:20 +0200 Subject: [PATCH 17/24] cloud build: initial set of shared files The approach taken here extends the existing support for cross-compiling binaries on the build host and specifying the Go compiler: Go is installed if needed (as in Prow testing), binaries are build on the host, then one image is created for each platform, and finally those are combined into a single multi-architecture image. --- cloudbuild.sh | 6 ++++ cloudbuild.yaml | 44 ++++++++++++++++++++++++++++ release-tools/build.make | 63 ++++++++++++++++++++++++++++++++++++++++ release-tools/prow.sh | 14 +++++++++ 4 files changed, 127 insertions(+) create mode 100755 cloudbuild.sh create mode 100644 cloudbuild.yaml diff --git a/cloudbuild.sh b/cloudbuild.sh new file mode 100755 index 000000000..3ba11ecad --- /dev/null +++ b/cloudbuild.sh @@ -0,0 +1,6 @@ +#! /bin/bash + +# shellcheck disable=SC1091 +. release-tools/prow.sh + +gcr_cloud_build diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 000000000..1def14994 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,44 @@ +# A configuration file for multi-arch image building with the Google cloud build service. +# +# Repos using this file must: +# - import csi-release-tools +# - add a symlink cloudbuild.yaml -> release-tools/cloudbuild.yaml +# - add a .cloudbuild.sh which can be a custom file or a symlink +# to release-tools/cloudbuild.sh +# - accept "binary" as build argument in their Dockerfile(s) (see +# https://github.com/pohly/node-driver-registrar/blob/3018101987b0bb6da2a2657de607174d6e3728f7/Dockerfile#L4-L6) +# because binaries will get built for different architectures and then +# get copied from the built host into the container image +# +# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md +# for more details on image pushing process in Kubernetes. + +# This must be specified in seconds. If omitted, defaults to 600s (10 mins). +timeout: 1200s +# This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF, +# or any new substitutions added in the future. +options: + substitution_option: ALLOW_LOOSE +steps: + # The image must contain bash and curl. Ideally it should also contain + # the desired version of Go (currently defined in release-tools/travis.yml), + # but that just speeds up the build and is not required. + - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8' + entrypoint: ./.cloudbuild.sh + env: + - GIT_TAG=${_GIT_TAG} + - PULL_BASE_REF=${_PULL_BASE_REF} + - REGISTRY_NAME=gcr.io/${_STAGING_PROJECT} + - HOME=/root +substitutions: + # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and + # can be used as a substitution. + _GIT_TAG: '12345' + # _PULL_BASE_REF will contain the ref that was pushed to trigger this build - + # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'. + _PULL_BASE_REF: 'master' + # The default gcr.io staging project for Kubernetes-CSI + # (=> https://console.cloud.google.com/gcr/images/k8s-staging-csi/GLOBAL). + # Might be overridden in the Prow build job for a repo which wants + # images elsewhere. + _STAGING_PROJECT: 'k8s-staging-csi' diff --git a/release-tools/build.make b/release-tools/build.make index 5c9793c2b..e3a44f96f 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -105,6 +105,69 @@ build: $(CMDS:%=build-%) container: $(CMDS:%=container-%) push: $(CMDS:%=push-%) +# Additional parameters are needed when pushing to a local registry, +# see https://github.com/docker/buildx/issues/94. +# However, that then runs into https://github.com/docker/cli/issues/2396. +# +# What works for local testing is: +# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME= BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" +DOCKER_BUILDX_CREATE_ARGS ?= + +# This target builds a multiarch image for one command using Moby BuildKit builder toolkit. +# Docker Buildx is included in Docker 19.03. +# +# ./cmd//Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows]. +# BUILD_PLATFORMS determines which individual images are included in the multiarch image. +# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines +# the tag for the resulting multiarch image. +push-multiarch-%: check-pull-base-ref build-% + set -ex; \ + DOCKER_CLI_EXPERIMENTAL=enabled; \ + export DOCKER_CLI_EXPERIMENTAL; \ + docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \ + trap "docker buildx rm multiarchimage-buildertest" EXIT; \ + dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ + dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ + if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ + pushMultiArch () { \ + tag=$$1; \ + echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ + docker buildx build --push \ + --tag $(IMAGE_NAME):$$arch-$$os-$$tag \ + --platform=$$os/$$arch \ + --file $$(eval echo \$${dockerfile_$$os}) \ + --build-arg binary=./bin/$*$$suffix \ + --label revision=$(REV) \ + .; \ + done; \ + images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \ + docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \ + docker manifest push -p $(IMAGE_NAME):$$tag; \ + }; \ + if [ $(PULL_BASE_REF) = "master" ]; then \ + : "creating or overwriting canary image"; \ + pushMultiArch canary; \ + elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \ + : "creating or overwriting canary image for release branch"; \ + release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \ + pushMultiArch $$release_canary_tag; \ + elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \ + : "creating release image"; \ + pushMultiArch $(PULL_BASE_REF); \ + else \ + : "ERROR: release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists: a new tag is required!"; \ + exit 1; \ + fi + +.PHONY: check-pull-base-ref +check-pull-base-ref: + if ! [ "$(PULL_BASE_REF)" ]; then \ + echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \ + exit 1; \ + fi + +push-multiarch: $(CMDS:%=push-multiarch-%) + clean: -rm -rf bin diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 0b962f044..93964e18e 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -1189,3 +1189,17 @@ main () { return "$ret" } + +# This function can be called by a repo's top-level cloudbuild.sh: +# it handles environment set up in the GCR cloud build and then +# invokes "make push-multiarch" to do the actual image building. +gcr_cloud_build () { + # Register gcloud as a Docker credential helper. + # Required for "docker buildx build --push". + gcloud auth configure-docker + + # Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value. + REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v') + + run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make push-multiarch REV="${REV}" REGISTRY_NAME="${REGISTRY_NAME}" BUILD_PLATFORMS="${CSI_PROW_BUILD_PLATFORMS}" +} From def9d8173e723d504cb8f1ce8f73914cf922674e Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Jun 2020 11:03:47 +0200 Subject: [PATCH 18/24] build.make: fix push-multiarch ambiguity "make push-multiarch" matched both push-multiarch and push-%. This seems to be none-deterministic and in at least one repo (external-provisioner), make picked the wildcard rule which then failed because there is no "multiarch" command. This ambiguity gets resolved by instantiating the wildcard rules only for existing commands. The advantage also is that "make push-no-such-command" will fail with an obvious "No rule to make target 'push-no-such-command'" instead of attempting to build the command. --- release-tools/build.make | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release-tools/build.make b/release-tools/build.make index e3a44f96f..0e4a9baaf 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -71,7 +71,7 @@ BUILD_PLATFORMS = # This builds each command (= the sub-directories of ./cmd) for the target platform(s) # defined by BUILD_PLATFORMS. -build-%: check-go-version-go +$(CMDS:%=build-%): build-%: check-go-version-go mkdir -p bin echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \ if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \ @@ -80,10 +80,10 @@ build-%: check-go-version-go fi; \ done -container-%: build-% +$(CMDS:%=container-%): container-%: build-% docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) . -push-%: container-% +$(CMDS:%=push-%): push-%: container-% set -ex; \ push_image () { \ docker tag $*:latest $(IMAGE_NAME):$$tag; \ @@ -120,7 +120,7 @@ DOCKER_BUILDX_CREATE_ARGS ?= # BUILD_PLATFORMS determines which individual images are included in the multiarch image. # PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines # the tag for the resulting multiarch image. -push-multiarch-%: check-pull-base-ref build-% +$(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% set -ex; \ DOCKER_CLI_EXPERIMENTAL=enabled; \ export DOCKER_CLI_EXPERIMENTAL; \ From 67b4ef4a74fa3f6329a3fdf9a7dfc7a25e2b21c4 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Jun 2020 11:04:29 +0200 Subject: [PATCH 19/24] build.make: properly declare push-multiarch It's not a real file and thus should better be marked as phony. --- release-tools/build.make | 1 + 1 file changed, 1 insertion(+) diff --git a/release-tools/build.make b/release-tools/build.make index 0e4a9baaf..83d903e08 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -166,6 +166,7 @@ check-pull-base-ref: exit 1; \ fi +.PHONY: push-multiarch push-multiarch: $(CMDS:%=push-multiarch-%) clean: From 1a8f5d42eba620d6f7407ce14a4174833a5e6ea0 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 3 Jun 2020 11:05:05 +0200 Subject: [PATCH 20/24] build.make: optional inclusion of Windows in multiarch images Most repos inherit the default BUILD_PLATFORMS, which includes Windows, but don't have the necessary Dockerfile.Windows yet. To simplify the rollout of multiarch image builds, Windows binary building continues to be tested (i.e. BUILD_PLATFORMS remains unchanged), but push-multiarch skips Windows if the Dockerfile.Windows is missing. --- release-tools/build.make | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/release-tools/build.make b/release-tools/build.make index 83d903e08..c17a1c194 100644 --- a/release-tools/build.make +++ b/release-tools/build.make @@ -117,6 +117,10 @@ DOCKER_BUILDX_CREATE_ARGS ?= # Docker Buildx is included in Docker 19.03. # # ./cmd//Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows]. +# It is currently optional: if no such file exists, Windows images are not included, +# even when Windows is listed in BUILD_PLATFORMS. That way, projects can test that +# Windows binaries can be built before adding a Dockerfile for it. +# # BUILD_PLATFORMS determines which individual images are included in the multiarch image. # PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines # the tag for the resulting multiarch image. @@ -129,6 +133,9 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-% dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \ dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \ if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \ + if ! [ -f "$$dockerfile_windows" ]; then \ + build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \ + fi; \ pushMultiArch () { \ tag=$$1; \ echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \ From 23fe0ff0a8fe4967f38b1c8969baf801b90bbd6c Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 4 Jun 2020 10:52:04 +0200 Subject: [PATCH 21/24] cloud build: initialize support for running commands in Dockerfile If the Dockerfile needs to run some command, that step fails unless QEMU is set up properly first: failed to solve: rpc error: code = Unknown desc = failed to load LLB: runtime execution on platform linux/ppc64le not supported --- release-tools/prow.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 93964e18e..32ec871a3 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -1198,6 +1198,12 @@ gcr_cloud_build () { # Required for "docker buildx build --push". gcloud auth configure-docker + if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then + # Needed for "RUN" steps on non-linux/amd64 platforms. + # See https://github.com/multiarch/qemu-user-static#getting-started + (set -x; docker run --rm --privileged multiarch/qemu-user-static --reset -p yes) + fi + # Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value. REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v') From a2d39091fdad0561695262f7a01933da13501950 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 12 Jun 2020 15:48:26 +0200 Subject: [PATCH 22/24] cloud build: k8s-staging-sig-storage As discussed in https://github.com/kubernetes/k8s.io/pull/943, we want to consolidate under k8s-staging-sig-storage. --- cloudbuild.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 1def14994..a22c0e7ac 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -12,6 +12,8 @@ # # See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md # for more details on image pushing process in Kubernetes. +# +# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage. # This must be specified in seconds. If omitted, defaults to 600s (10 mins). timeout: 1200s @@ -38,7 +40,7 @@ substitutions: # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'. _PULL_BASE_REF: 'master' # The default gcr.io staging project for Kubernetes-CSI - # (=> https://console.cloud.google.com/gcr/images/k8s-staging-csi/GLOBAL). + # (=> https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL). # Might be overridden in the Prow build job for a repo which wants # images elsewhere. - _STAGING_PROJECT: 'k8s-staging-csi' + _STAGING_PROJECT: 'k8s-staging-sig-storage' From 93ef622d9888024352b103d9c7fd2f2076ca7927 Mon Sep 17 00:00:00 2001 From: Yibo Cai Date: Tue, 16 Jun 2020 10:45:09 +0800 Subject: [PATCH 23/24] prow.sh: enable building arm64 image --- release-tools/prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 32ec871a3..9778635f9 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -85,7 +85,7 @@ get_versioned_variable () { echo "$value" } -configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" +configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries" # If we have a vendor directory, then use it. We must be careful to only # use this for "make" invocations inside the project's repo itself because From b3452516735d177bef64e26b066cacb93bcf8442 Mon Sep 17 00:00:00 2001 From: Mike Fedosin Date: Wed, 17 Jun 2020 14:11:34 +0200 Subject: [PATCH 24/24] Allow to set custom permissions for the mounted folder For RWX volume, kubelet does not perform recursive ownership/permission change. The heuristics that kubelet uses is being modified via - https://github.com/kubernetes/enhancements/issues/1682 Having said that, for RWX volumes which are made available via NFS protocol, using fsGroup is not recommended because if there are 2 pods that are trying to use same volume but with different fsGroup then one pod may lock out the other pod. To avoid this, we must be able to set the folder permissions to 777. This commit adds a cli option --mount-permissions, that allows to define custom permissions. If the value is not specified, then default permissions will be kept. Cherry-picked from: https://github.com/kubernetes-csi/csi-driver-nfs/pull/36 --- cmd/nfsplugin/main.go | 18 +++++++++++++++++- pkg/nfs/nfs.go | 5 ++++- pkg/nfs/nodeserver.go | 6 ++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/cmd/nfsplugin/main.go b/cmd/nfsplugin/main.go index f757df0b0..d1e86fe41 100644 --- a/cmd/nfsplugin/main.go +++ b/cmd/nfsplugin/main.go @@ -20,6 +20,7 @@ import ( "flag" "fmt" "os" + "strconv" "github.com/spf13/cobra" @@ -29,6 +30,7 @@ import ( var ( endpoint string nodeID string + perm string ) func init() { @@ -55,6 +57,8 @@ func main() { cmd.PersistentFlags().StringVar(&endpoint, "endpoint", "", "CSI endpoint") cmd.MarkPersistentFlagRequired("endpoint") + cmd.PersistentFlags().StringVar(&perm, "mount-permissions", "", "mounted folder permissions") + cmd.ParseFlags(os.Args[1:]) if err := cmd.Execute(); err != nil { fmt.Fprintf(os.Stderr, "%s", err.Error()) @@ -65,6 +69,18 @@ func main() { } func handle() { - d := nfs.NewNFSdriver(nodeID, endpoint) + // Converting string permission representation to *uint32 + var parsedPerm *uint32 + if perm != "" { + permu64, err := strconv.ParseUint(perm, 8, 32) + if err != nil { + fmt.Fprintf(os.Stderr, "Incorrect mount-permissions value: %q", perm) + os.Exit(1) + } + permu32 := uint32(permu64) + parsedPerm = &permu32 + } + + d := nfs.NewNFSdriver(nodeID, endpoint, parsedPerm) d.Run() } diff --git a/pkg/nfs/nfs.go b/pkg/nfs/nfs.go index 728132c17..bde90bdde 100644 --- a/pkg/nfs/nfs.go +++ b/pkg/nfs/nfs.go @@ -29,6 +29,8 @@ type nfsDriver struct { endpoint string + perm *uint32 + //ids *identityServer ns *nodeServer cap map[csi.VolumeCapability_AccessMode_Mode]bool @@ -43,7 +45,7 @@ var ( version = "2.0.0" ) -func NewNFSdriver(nodeID, endpoint string) *nfsDriver { +func NewNFSdriver(nodeID, endpoint string, perm *uint32) *nfsDriver { glog.Infof("Driver: %v version: %v", driverName, version) n := &nfsDriver{ @@ -52,6 +54,7 @@ func NewNFSdriver(nodeID, endpoint string) *nfsDriver { nodeID: nodeID, endpoint: endpoint, cap: map[csi.VolumeCapability_AccessMode_Mode]bool{}, + perm: perm, } vcam := []csi.VolumeCapability_AccessMode_Mode{ diff --git a/pkg/nfs/nodeserver.go b/pkg/nfs/nodeserver.go index 439d3c8bc..5360f6fac 100644 --- a/pkg/nfs/nodeserver.go +++ b/pkg/nfs/nodeserver.go @@ -73,6 +73,12 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return nil, status.Error(codes.Internal, err.Error()) } + if ns.Driver.perm != nil { + if err := os.Chmod(targetPath, os.FileMode(*ns.Driver.perm)); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } + return &csi.NodePublishVolumeResponse{}, nil }