diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..3e67581 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,62 @@ +name: Documentation + +on: + push: + paths: + - docs/** + - .github/workflows/docs.yml + - tasks/docs.py + - mkdocs.yml + - tox.ini + pull_request: + paths: + - docs/** + - .github/workflows/docs.yml + - tasks/docs.py + - mkdocs.yml + - tox.ini + +jobs: + build: + name: Build Docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: "3.8" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + python -m pip install --upgrade -r requirements.txt + + - name: Build docs + run: | + invoke docs.build + + - uses: actions/upload-artifact@v1 + with: + name: Documentation + path: site + + publish: + name: Publish Docs + runs-on: ubuntu-latest + # Only publish master + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + needs: + - build + steps: + - uses: actions/download-artifact@v1 + with: + name: Documentation + path: site + + - name: Publish generated content to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./site diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 0000000..ed9ae48 --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,142 @@ +name: Image + +on: [push, pull_request] + +jobs: + build_driver: + name: Build Driver + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 50 + + - name: Fetch Tags + run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* + + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.13.6 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: "3.8" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + python -m pip install --upgrade -r requirements.txt + + - uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Build Driver + run: | + invoke build --release + + - uses: actions/upload-artifact@v1 + with: + name: Driver + path: bin + + build_container: + name: Build Docker Container + runs-on: ubuntu-latest + needs: + - build_driver + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 50 + + - name: Fetch Tags + run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* + + - name: Install Go + uses: actions/setup-go@v1 + with: + go-version: 1.13.3 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: "3.8" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + python -m pip install --upgrade -r requirements.txt + + - uses: actions/download-artifact@v1 + with: + name: Driver + path: bin + + - name: Make Driver executable + run: "chmod +x bin/*" + + - name: Build Docker Container + run: | + invoke image --release --compress + + - name: Package Container + run: | + TAG=$(git describe --long --tags --match='v*' --dirty) + docker save -o ./container.tar.gz ofekmeister/csi-gcs:$TAG + + - uses: actions/upload-artifact@v1 + with: + name: Container + path: ./container.tar.gz + + publish_container: + name: Publish Container + runs-on: ubuntu-latest + needs: + - build_container + # Only Publish Images from our repository, not for MR + if: github.event_name == 'push' + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 50 + + - name: Fetch Tags + run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: "3.8" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + python -m pip install --upgrade -r requirements.txt + + - uses: actions/download-artifact@v1 + with: + name: Container + path: . + + - name: Import Container + run: | + TAG=$(git describe --long --tags --match='v*' --dirty) + docker import ./container.tar.gz ofekmeister/csi-gcs:$TAG + + - name: Login to Docker.io + run: | + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + env: + DOCKER_USERNAME: ${{ secrets.DockerUsername }} + DOCKER_PASSWORD: ${{ secrets.DockerPassword }} + + - name: Publish Docker Container + run: | + invoke image.deploy diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index be71e61..0000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Publish documentation - -on: - push: - paths: - - docs/** - - .github/workflows/publish.yml - - tasks/docs.py - - mkdocs.yml - - tox.ini - branches: - - master - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: "3.8" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - python -m pip install --upgrade -r requirements.txt - - - name: Build docs - run: | - invoke docs.build - - - name: Publish docs - run: | - invoke docs.publish - env: - GH_USER: "${{ secrets.GH_USER }}" - GH_EMAIL: "${{ secrets.GH_EMAIL }}" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5fdda9a..f55e5af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,46 +1,18 @@ name: Test -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: - build: - name: Build Docker Image + test_unit: + name: Unit Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 + - name: Install Go + uses: actions/setup-go@v2 with: - python-version: "3.8" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - python -m pip install --upgrade -r requirements.txt - - - name: Install Test Secrets - shell: bash - env: - TEST_SECRETS: ${{ secrets.TestSecrets }} - run: | - echo "$TEST_SECRETS" > test/secret.yaml - - - name: Build image - run: | - invoke image - - docs: - name: Generate Docs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 + go-version: 1.13.6 - name: Set up Python 3.8 uses: actions/setup-python@v1 @@ -52,21 +24,31 @@ jobs: python -m pip install --upgrade pip setuptools python -m pip install --upgrade -r requirements.txt - - name: Build docs - run: | - invoke docs.build - - - uses: actions/upload-artifact@v1 + - uses: actions/cache@v1 with: - name: Documentation - path: site + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-test_unit-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-test_unit- + ${{ runner.os }}-go- + + - name: Run Tests + run: | + invoke test.unit test_sanity: - name: Sanity Test + name: Test Sanity runs-on: ubuntu-latest + # Secrets are only available for Push + if: github.event_name == 'push' steps: - uses: actions/checkout@v2 + - name: Install Go + uses: actions/setup-go@v1 + with: + go-version: 1.13.3 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: @@ -77,6 +59,14 @@ jobs: python -m pip install --upgrade pip setuptools python -m pip install --upgrade -r requirements.txt + - uses: actions/cache@v1 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-test_sanity-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-test_sanity- + ${{ runner.os }}-go- + - name: Install Test Secrets shell: bash env: @@ -84,8 +74,9 @@ jobs: run: | echo "$TEST_SECRETS" > test/secret.yaml - - name: Sanity Test + - name: Run Tests # TODO: Remove when Sanity Tests are fixed + # https://github.com/ofek/csi-gcs/issues/15 continue-on-error: true run: | - invoke test.sanity + invoke docker -c "invoke test.sanity" diff --git a/.gitignore b/.gitignore index c63471b..b8f7423 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,7 @@ # Test Service Account /service-account.json /test/secret.yaml + +# Artifacts +/bin/* +!/bin/.gitkeep \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9813c1e..54ca175 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,7 @@ -FROM golang:1.13.6-alpine3.11 AS build +FROM golang:1.13.6-alpine3.11 AS build-gcsfuse -ARG version ARG gcsfuse_version ARG global_ldflags -ARG upx_flags - -ENV DRIVER ${GOPATH}/src/github.com/ofek/csi-gcs/ -ENV UPX ${upx_flags} RUN apk add --update --no-cache fuse fuse-dev git upx @@ -21,37 +16,33 @@ RUN go install github.com/googlecloudplatform/gcsfuse/tools/build_gcsfuse RUN mkdir /tmp/gcsfuse RUN build_gcsfuse ${GOPATH}/src/github.com/googlecloudplatform/gcsfuse /tmp/gcsfuse ${gcsfuse_version} -ldflags "all=${global_ldflags}" -ldflags "-X main.gcsfuseVersion=${gcsfuse_version} ${global_ldflags}" -# We don't need mount(8) compatibility as we call the binary directly, so only copy that -RUN mv /tmp/gcsfuse/bin/gcsfuse /tmp/bin/gcsfuse +FROM golang:1.13.6-alpine3.11 AS compress-gcfsuse -RUN mkdir -p ${DRIVER} -WORKDIR ${DRIVER} +ARG upx_flags +ENV UPX ${upx_flags} -COPY go.mod go.sum ${DRIVER} -COPY cmd ${DRIVER}/cmd -COPY pkg ${DRIVER}/pkg +RUN apk add --update --no-cache upx -# Build the driver -RUN go build -o /tmp/bin/driver -ldflags "all=${global_ldflags}" -ldflags "-X github.com/ofek/csi-gcs/pkg/driver.driverVersion=${version} ${global_ldflags}" ./cmd +COPY --from=build-gcsfuse /tmp/gcsfuse/bin/gcsfuse /tmp/bin/gcsfuse # Compress the binaries RUN if [ "${UPX}" != "" ]; then \ - upx /tmp/bin/driver \ - && upx /tmp/bin/gcsfuse; \ + upx /tmp/bin/gcsfuse; \ fi -FROM golang:1.13.6-alpine3.11 AS test +FROM golang:1.13.6-alpine3.11 AS compress-csi-gcs -RUN apk add build-base fuse fuse-dev git upx --update --no-cache +ARG upx_flags +ENV UPX ${upx_flags} -ENV DRIVER ${GOPATH}/src/github.com/ofek/csi-gcs/ +RUN apk add --update --no-cache upx -RUN mkdir -p ${DRIVER} -WORKDIR ${DRIVER} +COPY bin/driver /tmp/bin/driver -COPY --from=build /tmp/bin/* /usr/local/bin/ -COPY --from=build ${GOPATH} ${GOPATH} -COPY test ${DRIVER}/test +# Compress the binaries +RUN if [ "${UPX}" != "" ]; then \ + upx /tmp/bin/driver; \ + fi FROM alpine:3.11 @@ -75,4 +66,5 @@ WORKDIR / ENTRYPOINT ["/usr/local/bin/driver"] # Copy the binaries -COPY --from=build /tmp/bin/* /usr/local/bin/ +COPY --from=compress-gcfsuse /tmp/bin/* /usr/local/bin/ +COPY --from=compress-csi-gcs /tmp/bin/* /usr/local/bin/ diff --git a/README.md b/README.md index f794a83..cd3661b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # csi-gcs -[![CI](https://github.com/ofek/csi-gcs/workflows/Test/badge.svg)](https://github.com/ofek/csi-gcs/actions?query=workflow%3ATest) +[![CI - Image](https://github.com/ofek/csi-gcs/workflows/Image/badge.svg)](https://github.com/ofek/csi-gcs/actions?query=workflow%3AImage) +[![CI - Test](https://github.com/ofek/csi-gcs/workflows/Test/badge.svg)](https://github.com/ofek/csi-gcs/actions?query=workflow%3ATest) +[![CI - Documentation](https://github.com/ofek/csi-gcs/workflows/Documentation/badge.svg)](https://github.com/ofek/csi-gcs/actions?query=workflow%3ADocumentation) [![Docker - Pulls](https://img.shields.io/docker/pulls/ofekmeister/csi-gcs.svg)](https://hub.docker.com/r/ofekmeister/csi-gcs) [![License - MIT/Apache-2.0](https://img.shields.io/badge/license-MIT%2FApache--2.0-9400d3.svg)](https://choosealicense.com/licenses) [![Say Thanks](https://img.shields.io/badge/say-thanks-ff69b4.svg)](https://saythanks.io/to/ofekmeister%40gmail.com) diff --git a/bin/.gitkeep b/bin/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/cmd/main.go b/cmd/main.go index 30d001a..3cc1b41 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -12,10 +12,11 @@ import ( ) var ( - nodeName = flag.String("node-name", "", "Node identifier") - driverName = flag.String("driver-name", driver.CSIDriverName, "CSI driver name") - endpoint = flag.String("csi-endpoint", "unix:///csi/csi.sock", "CSI endpoint") - version = flag.Bool("version", false, "Print the version and exit") + version = "development" + nodeNameFlag = flag.String("node-name", "", "Node identifier") + driverNameFlag = flag.String("driver-name", driver.CSIDriverName, "CSI driver name") + endpointFlag = flag.String("csi-endpoint", "unix:///csi/csi.sock", "CSI endpoint") + versionFlag = flag.Bool("version", false, "Print the version and exit") ) func main() { @@ -24,7 +25,7 @@ func main() { setEnvVarFlags() flag.Parse() - if *version { + if *versionFlag { versionJSON, err := driver.GetVersionJSON() if err != nil { klog.Error(err.Error()) @@ -34,7 +35,7 @@ func main() { os.Exit(0) } - d, err := driver.NewGCSDriver(*driverName, *nodeName, *endpoint) + d, err := driver.NewGCSDriver(*driverNameFlag, *nodeNameFlag, *endpointFlag, version) if err != nil { klog.Error(err.Error()) os.Exit(1) diff --git a/dev-env.Dockerfile b/dev-env.Dockerfile new file mode 100644 index 0000000..f222343 --- /dev/null +++ b/dev-env.Dockerfile @@ -0,0 +1,33 @@ +FROM golang:1.13.6-alpine3.11 AS build-gcsfuse + +ARG gcsfuse_version + +RUN apk add --update --no-cache fuse fuse-dev git upx + +WORKDIR ${GOPATH} + +# Create Tmp Bin Dir +RUN mkdir /tmp/bin + +# Install gcsfuse using the specified version or commit hash +RUN go get -u github.com/googlecloudplatform/gcsfuse +RUN go install github.com/googlecloudplatform/gcsfuse/tools/build_gcsfuse +RUN mkdir /tmp/gcsfuse +RUN build_gcsfuse ${GOPATH}/src/github.com/googlecloudplatform/gcsfuse /tmp/gcsfuse ${gcsfuse_version} -ldflags "-X main.gcsfuseVersion=${gcsfuse_version}" + +FROM golang:1.13.6-alpine3.11 + +RUN apk add --update --no-cache fuse fuse-dev git upx python3 python3-dev py3-pip bash build-base docker + +COPY --from=build-gcsfuse /tmp/gcsfuse/bin/gcsfuse /usr/local/bin/gcsfuse + +RUN python3 -m pip install --upgrade pip setuptools + +RUN mkdir /driver +WORKDIR /driver + +COPY requirements.txt /tmp/requirements.txt + +RUN python3 -m pip install --upgrade -r /tmp/requirements.txt + +RUN rm /tmp/requirements.txt diff --git a/docs/contributing/setup.md b/docs/contributing/setup.md index 3cc592d..27e34fb 100644 --- a/docs/contributing/setup.md +++ b/docs/contributing/setup.md @@ -30,14 +30,72 @@ ## Rebuild & Test Manually in Minikube -* Build Docker Image `invoke image` -* Delete Pod +```console +# Build Binary +invoke build -## Build docs +# Build Container +invoke image +``` -* `invoke docs` +Afterwards kill the currently running pod. + +## Documentation + +```console +# Build +invoke docs.build + +# Server +invoke docs.serve +``` ## Sanity Tests -* `invoke test.sanity` +Needs root privileges and `gcsfuse` installed, execution via docker recommended. + +```console +# Local +invoke test.sanity + +# Docker +invoke docker -c "invoke test.sanity" +``` + +Additionally the file `./test/secret.yaml` has to be created with the following content: + +```yml +CreateVolumeSecret: + projectId: [Google Cloud Project ID] + key: | + [Storage Admin Key JSON] +DeleteVolumeSecret: + projectId: [Google Cloud Project ID] + key: | + [Storage Admin Key JSON] +ControllerPublishVolumeSecret: + projectId: [Google Cloud Project ID] + key: | + [Storage Admin Key JSON] +ControllerUnpublishVolumeSecret: + projectId: [Google Cloud Project ID] + key: | + [Storage Admin Key JSON] +NodeStageVolumeSecret: + projectId: [Google Cloud Project ID] + key: | + [Storage Object Admin Key JSON] +NodePublishVolumeSecret: + projectId: [Google Cloud Project ID] + key: | + [Storage Object Admin Key JSON] +ControllerValidateVolumeCapabilitiesSecret: + projectId: [Google Cloud Project ID] + key: | + [Storage Admin Key JSON] +``` + +## Develop inside Docker + +Run all `invoke` commands through `invoke docker -c "[CMD]"`. \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 127db5e..89a0f41 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,8 @@ # csi-gcs -[![CI](https://github.com/ofek/csi-gcs/workflows/Test/badge.svg)](https://github.com/ofek/csi-gcs/actions?query=workflow%3ATest) +[![CI - Image](https://github.com/ofek/csi-gcs/workflows/Image/badge.svg)](https://github.com/ofek/csi-gcs/actions?query=workflow%3AImage) +[![CI - Test](https://github.com/ofek/csi-gcs/workflows/Test/badge.svg)](https://github.com/ofek/csi-gcs/actions?query=workflow%3ATest) +[![CI - Documentation](https://github.com/ofek/csi-gcs/workflows/Documentation/badge.svg)](https://github.com/ofek/csi-gcs/actions?query=workflow%3ADocumentation) [![Docker - Pulls](https://img.shields.io/docker/pulls/ofekmeister/csi-gcs.svg)](https://hub.docker.com/r/ofekmeister/csi-gcs) [![License - MIT/Apache-2.0](https://img.shields.io/badge/license-MIT%2FApache--2.0-9400d3.svg)](https://choosealicense.com/licenses) [![Say Thanks](https://img.shields.io/badge/say-thanks-ff69b4.svg)](https://saythanks.io/to/ofekmeister%40gmail.com) diff --git a/go.mod b/go.mod index 4fb1608..b34b064 100644 --- a/go.mod +++ b/go.mod @@ -5,17 +5,14 @@ go 1.13 require ( cloud.google.com/go v0.38.0 github.com/container-storage-interface/spec v1.2.0 - github.com/ianlancetaylor/demangle v0.0.0-20200414190113-039b1ae3a340 // indirect github.com/kubernetes-csi/csi-lib-utils v0.7.0 + github.com/kubernetes-csi/csi-test v2.2.0+incompatible // indirect github.com/kubernetes-csi/csi-test/v3 v3.1.0 github.com/mitchellh/go-ps v1.0.0 github.com/onsi/ginkgo v1.10.3 github.com/onsi/gomega v1.7.1 - golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect - golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 // indirect google.golang.org/api v0.4.0 google.golang.org/grpc v1.26.0 - honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect k8s.io/apimachinery v0.17.1-beta.0 k8s.io/client-go v0.17.0 k8s.io/klog v1.0.0 diff --git a/go.sum b/go.sum index 75b2da6..92c9857 100644 --- a/go.sum +++ b/go.sum @@ -82,8 +82,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googlecloudplatform/gcsfuse v0.29.0 h1:dkKshEH24gTOMt/iByChEUGxBCzdzK90HjbV7EQ5f30= -github.com/googlecloudplatform/gcsfuse v0.29.0/go.mod h1:A5iNYAEqoneu2NdcxFTVaGrjckAdTfnXdmLywMmxqQI= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -91,8 +89,6 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20200414190113-039b1ae3a340 h1:S1+yTUaFPXuDZnPDbO+TrDFIjPzQraYH8/CwSlu9Fac= -github.com/ianlancetaylor/demangle v0.0.0-20200414190113-039b1ae3a340/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -112,6 +108,9 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kubernetes-csi/csi-lib-utils v0.7.0 h1:t1cS7HTD7z5D7h9iAdjWuHtMxJPb9s1fIv34rxytzqs= github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= +github.com/kubernetes-csi/csi-test v1.1.1 h1:L4RPre34ICeoQW7ez4X5t0PnFKaKs8K5q0c1XOrvXEM= +github.com/kubernetes-csi/csi-test v2.2.0+incompatible h1:ksIV60Q+4mY0Fg8LKvBssjEcvbyxo7nz0eAD6ZLMux0= +github.com/kubernetes-csi/csi-test v2.2.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= github.com/kubernetes-csi/csi-test/v3 v3.1.0 h1:XPiXQgEhDV9y+6vHAklH273PKA/ocqQdC8R+WcIKZKw= github.com/kubernetes-csi/csi-test/v3 v3.1.0/go.mod h1:UWxYP5cDlD6iSNVKEiLFqfJnJinuhtI7MLt61rQQOfI= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -171,8 +170,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -golang.org/x/arch v0.0.0-20200312215426-ff8b605520f4 h1:cZG+Ns0n5bdEEsURGnDinFswSebRNMqspbLvxrLZoIc= -golang.org/x/arch v0.0.0-20200312215426-ff8b605520f4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -284,7 +281,6 @@ k8s.io/apimachinery v0.17.1-beta.0 h1:0Wl/KpAiFOMe9to5h8x2Y6JnjV+BEWJiTcUk1Vx7zd k8s.io/apimachinery v0.17.1-beta.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/client-go v0.17.0 h1:8QOGvUGdqDMFrm9sD6IUFl256BcffynGoe80sxgTEDg= k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= -k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -292,10 +288,8 @@ k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kubernetes v1.18.2 h1:37sJPq6p+gx5hEHQSwCWXIiXDc9AajzV1A5UrswnDq0= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f h1:GiPwtSzdP43eI1hpPCbROQCCIgCuiMMNF8YUVLF3vJo= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index d45b5ce..c192e55 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -21,13 +21,13 @@ type GCSDriver struct { server *grpc.Server } -func NewGCSDriver(name, node, endpoint string) (*GCSDriver, error) { +func NewGCSDriver(name, node, endpoint string, version string) (*GCSDriver, error) { return &GCSDriver{ name: name, nodeName: node, endpoint: endpoint, mountPoint: BucketMountPath, - version: driverVersion, + version: version, }, nil } diff --git a/tasks/__init__.py b/tasks/__init__.py index dbc43f5..99611f4 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -3,11 +3,15 @@ from . import docs from . import image from . import test +from . import env +from . import build from .utils import set_root ns = Collection() ns.add_collection(Collection.from_module(image)) ns.add_collection(Collection.from_module(docs)) ns.add_collection(Collection.from_module(test)) +ns.add_collection(Collection.from_module(env)) +ns.add_collection(Collection.from_module(build)) set_root() diff --git a/tasks/build.py b/tasks/build.py new file mode 100644 index 0000000..6e222d2 --- /dev/null +++ b/tasks/build.py @@ -0,0 +1,26 @@ +from invoke import task + +from .utils import EnvVars, get_version + + +@task( + help={ + 'release': 'Build a release image', + }, + default=True, +) +def build(ctx, release=False): + if release: + global_ldflags = '-s -w' + else: + global_ldflags = '' + + with EnvVars({'CGO_ENABLED': '0', 'GOOS': 'linux', 'GOARCH': 'amd64'}): + ctx.run( + f'go build ' + f'-o bin/driver ' + f'-ldflags "all={global_ldflags}" ' + f'-ldflags "-X github.com/ofek/csi-gcs/pkg/driver.driverVersion={get_version()} {global_ldflags}" ' + f'./cmd', + echo=True, + ) diff --git a/tasks/constants.py b/tasks/constants.py index 64fcd6c..2d6f047 100644 --- a/tasks/constants.py +++ b/tasks/constants.py @@ -1,8 +1,5 @@ -VERSION = '0.3.0' GCSFUSE_VERSION = '0.27.0' REPO = 'ofekmeister' IMAGE = 'csi-gcs' DRIVER_NAME = f'{REPO}/{IMAGE}' -IMAGE_LATEST = f'{DRIVER_NAME}:latest' -IMAGE_DEV = f'{DRIVER_NAME}:dev' diff --git a/tasks/docs.py b/tasks/docs.py index b117277..a8fbec8 100644 --- a/tasks/docs.py +++ b/tasks/docs.py @@ -46,66 +46,11 @@ def build(ctx, verbose=False): ) def serve(ctx, no_open=False, verbose=False): """Serve and view documentation in a web browser""" - address = 'localhost:8765' - command = ['tox', '-e', 'docs', '--', 'serve', '--livereload', '--dev-addr', address] + command = ['tox', '-e', 'docs', '--', 'serve', '--livereload', '--dev-addr', '0.0.0.0:8765'] insert_verbosity_flag(command, verbose) if not no_open: - webbrowser.open_new_tab(f'http://{address}') + webbrowser.open_new_tab(f'http://localhost:8765') ctx.run(' '.join(command)) - - -@task -def publish(ctx): - """Publish documentation on GitHub Pages""" - github_token = os.getenv('GITHUB_TOKEN') - if not github_token: - raise Exit('No `GITHUB_TOKEN` has been set') - - site_dir = os.path.abspath('site') - if not os.path.isdir(site_dir): - raise Exit('Site directory does not exist, build docs by running `inv docs.build`') - - print('Reading current Git configuration...') - git_user = get_git_user(ctx) - git_email = get_git_email(ctx) - latest_commit_hash = get_latest_commit_hash(ctx) - - if 'GITHUB_ACTIONS' in os.environ: - remote = f'https://{os.getenv("GITHUB_ACTOR")}:{github_token}@github.com/ofek/csi-gcs.git' - else: - remote = f'https://{github_token}@github.com/ofek/csi-gcs.git' - - print('Copying site to a temporary directory...') - with TemporaryDirectory() as d: - temp_repo_dir = shutil.copytree(site_dir, os.path.join(os.path.realpath(d), 'site')) - - # https://help.github.com/en/github/working-with-github-pages/about-github-pages#static-site-generators - # https://github.com/mkdocs/mkdocs/pull/2060 - print('Writing .nojekyll at the root...') - create_file(os.path.join(temp_repo_dir, '.nojekyll')) - - origin = os.getcwd() - try: - os.chdir(temp_repo_dir) - print('Configuring the temporary Git repository...') - ctx.run('git init', hide=True) - ctx.run(f'git config user.name "{git_user}"', hide=True) - ctx.run(f'git config user.email "{git_email}"', hide=True) - ctx.run(f'git remote add upstream {remote}', hide=True) - - print('Discovering remote...') - ctx.run('git fetch --depth 1 upstream', hide=True) - - upstream_check = ctx.run(f'git ls-remote --heads {remote} gh-pages', hide=True) - if upstream_check.stdout.strip(): - ctx.run('git reset upstream/gh-pages', hide=True) - - print('Committing site contents to branch gh-pages...') - ctx.run('git add --all', hide=True) - ctx.run(f'git commit --allow-empty -m "build docs at {latest_commit_hash}"', hide=True) - ctx.run('git push upstream HEAD:gh-pages', hide=True) - finally: - os.chdir(origin) diff --git a/tasks/env.py b/tasks/env.py new file mode 100644 index 0000000..e6677f7 --- /dev/null +++ b/tasks/env.py @@ -0,0 +1,52 @@ +from invoke import task + +from .constants import DRIVER_NAME, GCSFUSE_VERSION +from .utils import get_root + + +@task +def create(ctx): + ctx.run( + 'docker volume create csi-gcs-go', + echo=True, + ) + ctx.run( + f'docker build . ' + f'--tag {DRIVER_NAME}-env ' + f'-f dev-env.Dockerfile ' + f'--build-arg gcsfuse_version="{GCSFUSE_VERSION}"', + echo=True, + ) + + +@task +def delete(ctx): + ctx.run( + 'docker volume rm csi-gcs-go', + echo=True, + ) + ctx.run( + f'docker image rm {DRIVER_NAME}-env', + echo=True, + ) + + +@task( + pre=[create], + default=True, +) +def run(ctx, command='echo Provide Command'): + ctx.run( + f'docker run ' + f'--rm ' + f'-v {get_root()}:/driver ' + f'-v csi-gcs-go:/go ' + f'--cap-add SYS_ADMIN ' + f'--device /dev/fuse ' + f'--privileged ' + f'-v /tmp/csi:/tmp/csi:rw ' + f'-v /var/run/docker.sock:/var/run/docker.sock ' + f'{DRIVER_NAME}-env ' + f'{command}', + echo=True, + ) diff --git a/tasks/image.py b/tasks/image.py index dad21e4..5bd8a7b 100644 --- a/tasks/image.py +++ b/tasks/image.py @@ -1,38 +1,29 @@ from invoke import task -from .constants import DRIVER_NAME, GCSFUSE_VERSION, IMAGE_DEV, IMAGE_LATEST, VERSION - - -def image_name(version): - return f'{DRIVER_NAME}:v{version}' - +from .constants import DRIVER_NAME, GCSFUSE_VERSION +from .utils import image_name, image_tags @task( help={ 'release': 'Build a release image', 'compress': 'Minimize image size', - 'version': f'The desired version (default: {VERSION})', 'gcsfuse': f'The version or commit hash of gcsfuse (default: {GCSFUSE_VERSION})', }, default=True, ) -def build(ctx, release=False, compress=False, version=VERSION, gcsfuse=GCSFUSE_VERSION): +def build(ctx, release=False, compress=False, gcsfuse=GCSFUSE_VERSION): if release: - static_image = IMAGE_LATEST global_ldflags = '-s -w' docker_build_args = '--no-cache' else: - static_image = IMAGE_DEV global_ldflags = '' - version += '-rc' docker_build_args = '' upx_flags = '--best --ultra-brute' if compress else '' - image = image_name(version) + image = image_name() ctx.run( f'docker build . --tag {image} ' - f'--build-arg version={version} ' f'--build-arg global_ldflags="{global_ldflags}" ' f'--build-arg gcsfuse_version="{gcsfuse}" ' f'--build-arg upx_flags="{upx_flags}" ' @@ -40,14 +31,13 @@ def build(ctx, release=False, compress=False, version=VERSION, gcsfuse=GCSFUSE_V echo=True, ) - ctx.run(f'docker tag {image} {static_image}', echo=True) - + for tag in image_tags(): + ctx.run(f'docker tag {image} {image_name(tag)}', echo=True) -@task( - help={ - 'version': f'The desired version (default: {VERSION})', - } -) -def deploy(ctx, version=VERSION): - ctx.run(f'docker push {image_name(version)}', echo=True) - ctx.run(f'docker push {IMAGE_LATEST}', echo=True) +@task +def deploy(ctx): + ctx.run(f'docker push {image_name()}', echo=True) + for tag in image_tags(): + if tag != 'dev': + ctx.run(f'docker tag {image_name()} {image_name(tag)}', echo=True) + ctx.run(f'docker push {image_name(tag)}', echo=True) diff --git a/tasks/test.py b/tasks/test.py index 5f7a4bf..cac1990 100644 --- a/tasks/test.py +++ b/tasks/test.py @@ -1,106 +1,27 @@ from invoke import task -from .constants import DRIVER_NAME, GCSFUSE_VERSION, IMAGE_DEV, IMAGE_LATEST, VERSION -from .image import image_name +@task +def sanity(ctx): + ctx.run(f'go test ./test', echo=True) -@task( - help={ - 'version': f'The desired version (default: {VERSION})', - 'gcsfuse': f'The version or commit hash of gcsfuse (default: {GCSFUSE_VERSION})', - }, - default=True, -) -def all(ctx, version=VERSION, gcsfuse=GCSFUSE_VERSION): - unit(ctx, version, gcsfuse) - sanity(ctx, version, gcsfuse) - -def test_image_name(version=VERSION): - image = image_name(version) - image += '-builder' - return image - -def build(ctx, version=VERSION, gcsfuse=GCSFUSE_VERSION): - global_ldflags = '' - version += '-rc' - - image = test_image_name(version) - - ctx.run( - f'docker build . --tag {image} ' - f'--build-arg version={version} ' - f'--build-arg global_ldflags="{global_ldflags}" ' - f'--build-arg gcsfuse_version="{gcsfuse}" ' - f'--target test', - echo=True, - ) - -@task( - help={ - 'version': f'The desired version (default: {VERSION})', - 'gcsfuse': f'The version or commit hash of gcsfuse (default: {GCSFUSE_VERSION})', - } -) -def sanity(ctx, version=VERSION, gcsfuse=GCSFUSE_VERSION): - build(ctx, version, gcsfuse) +@task +def unit_driver(ctx): + ctx.run(f'go test ./pkg/driver', echo=True) - version += '-rc' +@task +def unit_flags(ctx): + ctx.run(f'go test ./pkg/flags', echo=True) - image = test_image_name(version) +@task +def unit_util(ctx): + ctx.run(f'go test ./pkg/util', echo=True) - ctx.run( - f'docker run ' - f'--rm ' - f'--cap-add SYS_ADMIN ' - f'--device /dev/fuse ' - f'--privileged ' - f'-t {image} ' - f'go test ./test', - echo=True - ) +@task(pre=[unit_flags, unit_driver, unit_util]) +def unit(ctx): pass @task( - help={ - 'version': f'The desired version (default: {VERSION})', - 'gcsfuse': f'The version or commit hash of gcsfuse (default: {GCSFUSE_VERSION})', - } + pre=[unit, sanity], + default=True, ) -def unit(ctx, version=VERSION, gcsfuse=GCSFUSE_VERSION): - build(ctx, version, gcsfuse) - - version += '-rc' - - image = test_image_name(version) - - ctx.run( - f'docker run ' - f'--rm ' - f'--cap-add SYS_ADMIN ' - f'--device /dev/fuse ' - f'--privileged ' - f'-t {image} ' - f'go test ./pkg/driver', - echo=True - ) - - ctx.run( - f'docker run ' - f'--rm ' - f'--cap-add SYS_ADMIN ' - f'--device /dev/fuse ' - f'--privileged ' - f'-t {image} ' - f'go test ./pkg/flags', - echo=True - ) - - ctx.run( - f'docker run ' - f'--rm ' - f'--cap-add SYS_ADMIN ' - f'--device /dev/fuse ' - f'--privileged ' - f'-t {image} ' - f'go test ./pkg/util', - echo=True - ) \ No newline at end of file +def all(ctx): pass \ No newline at end of file diff --git a/tasks/utils.py b/tasks/utils.py index 305d16a..720c664 100644 --- a/tasks/utils.py +++ b/tasks/utils.py @@ -1,15 +1,17 @@ import os +import subprocess +from .constants import DRIVER_NAME +def get_root(): + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) def set_root(): - os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - + os.chdir(get_root()) def get_latest_commit_hash(ctx): result = ctx.run('git rev-parse HEAD', hide=True) return result.stdout.strip() - def get_git_user(ctx): user = os.getenv('GH_USER') if user is not None: @@ -18,7 +20,6 @@ def get_git_user(ctx): result = ctx.run('git config --get user.name', hide=True) return result.stdout.strip() - def get_git_email(ctx): email = os.getenv('GH_EMAIL') if email is not None: @@ -27,7 +28,54 @@ def get_git_email(ctx): result = ctx.run('git config --get user.email', hide=True) return result.stdout.strip() - def create_file(f): with open(f, 'a'): os.utime(f, None) + +def get_version(): + version = subprocess.run(['git', 'describe', '--long', '--tags', '--match=v*', '--dirty'], stdout=subprocess.PIPE, cwd=get_root()) + if version.returncode == 0: + return version.stdout.decode('utf-8').strip() + + current_ref = subprocess.run(['git', 'rev-list', '-n1', 'HEAD'], stdout=subprocess.PIPE, cwd=get_root()) + return current_ref.stdout.decode('utf-8').strip() + +def image_name(version=False): + if not version: + version = get_version() + return f'{DRIVER_NAME}:{version}' + +def image_tags(): + last_tag = subprocess.run(['git', 'describe', '--tags', '--match=v*', '--abbrev=0'], stdout=subprocess.PIPE, cwd=get_root()) + + if last_tag.returncode != 0: + return ['dev'] + + current_ref = subprocess.run(['git', 'rev-list', '-n1', 'HEAD'], stdout=subprocess.PIPE, cwd=get_root()) + last_tag_ref = subprocess.run(['git', 'rev-list', '-n1', last_tag.stdout.decode('utf-8').strip()], stdout=subprocess.PIPE, cwd=get_root()) + + if last_tag_ref.stdout.decode('utf-8').strip() == current_ref.stdout.decode('utf-8').strip(): + return [last_tag.stdout.decode('utf-8').strip(), 'latest'] + + return ['dev'] + + +class EnvVars(dict): + def __init__(self, env_vars=None, ignore=None): + super(EnvVars, self).__init__(os.environ) + self.old_env = dict(self) + + if env_vars is not None: + self.update(env_vars) + + if ignore is not None: + for env_var in ignore: + self.pop(env_var, None) + + def __enter__(self): + os.environ.clear() + os.environ.update(self) + + def __exit__(self, exc_type, exc_value, traceback): + os.environ.clear() + os.environ.update(self.old_env) diff --git a/test/sanity_test.go b/test/sanity_test.go index 97578f4..f04a6bf 100644 --- a/test/sanity_test.go +++ b/test/sanity_test.go @@ -35,7 +35,7 @@ func TestCsiGcs(t *testing.T) { var endpoint = "unix://" endpoint += endpointFile.Name() - d, err := driver.NewGCSDriver(driver.CSIDriverName, "test-node", endpoint) + d, err := driver.NewGCSDriver(driver.CSIDriverName, "test-node", endpoint, "development") if err != nil { klog.Error(err.Error()) os.Exit(1)