Merge pull request #2 from stone-z/mholt-archiver-351-main

Merge from main

.circleci/config/commands/@caches.yml

@@ -0,0 +1,56 @@
+restore_yarn_cache:
+  steps:
+    - restore_cache:
+        name: Restore yarn cache
+        key: &YARN_LOCK_CACHE_KEY yarn-lock-v6-{{ checksum "ui/yarn.lock" }}
+save_yarn_cache:
+  steps:
+    - save_cache:
+        name: Save yarn cache
+        key: *YARN_LOCK_CACHE_KEY
+        paths:
+          - ui/node_modules
+# allows restoring go mod caches by incomplete prefix. This is useful when re-generating
+# cache, but not when running builds and tests that require an exact match.
+restore_go_mod_cache_permissive:
+  steps:
+    - restore_cache:
+        name: Restore closest matching go modules cache
+        keys:
+          - &gocachekey v1.3-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}-{{checksum "api/go.sum"}}
+          -             v1.3-{{checksum "go.sum"}}-{{checksum "sdk/go.sum"}}
+          -             v1.3-{{checksum "go.sum"}}
+restore_go_mod_cache:
+  steps:
+    - restore_cache:
+        name: Restore exact go modules cache
+        keys:
+          - *gocachekey
+save_go_mod_cache:
+  steps:
+    - save_cache:
+        name: Save go modules cache
+        key: *gocachekey
+        paths:
+          - /go/pkg/mod
+refresh_go_mod_cache:
+  steps:
+    - restore_go_mod_cache_permissive
+    - run:
+        name: go mod download
+        command: |
+          # go list ./... forces downloading some additional versions of modules that 'go mod
+          # download' misses. We need this because we make use of go list itself during
+          # code generation in later builds that rely on this module cache.
+          go list ./...
+          go mod download -json
+          ( cd sdk && go mod download -json; )
+          ( cd api && go mod download -json; )
+    - run:
+        name: Verify downloading modules did not modify any files
+        command: |
+          git --no-pager diff --exit-code || {
+            echo "ERROR: Files modified by go mod download, see above."
+            exit 1
+          }
+    - save_go_mod_cache

.circleci/config/commands/configure-git.yml

@@ -0,0 +1,6 @@
+steps:
+  - add_ssh_keys:
+      fingerprints:
+        - "0e:03:77:f4:e2:c3:56:c2:53:6a:03:e1:31:91:2f:06"
+  - run: |
+      git config --global url."git@github.com:".insteadOf https://github.com/

.circleci/config/commands/exit-if-branch-does-not-need-test-ui.yml

@@ -0,0 +1,17 @@
+description: >
+  Check if branch name starts with ui/ or docs/ and if so, exit.
+steps:
+  - run:
+      working_directory: ~/
+      name: Check branch name
+      command: |
+        case "$CIRCLE_BRANCH" in
+        main|ui/*|release/*|merge*) ;;
+        *) # If the branch being tested doesn't match one of the above patterns,
+           # we don't need to run test-ui and can abort the job.
+           circleci-agent step halt
+           ;;
+        esac
+
+        # exit with success either way
+        exit 0

.circleci/config/commands/go_test.yml

@@ -14,30 +14,36 @@ parameters:
     default: false
   go_image:
     type: string
-    default: "docker.mirror.hashicorp.services/circleci/golang:1.16.2-buster"
+    default: "docker.mirror.hashicorp.services/circleci/golang:1.17.2-buster"
   use_docker:
     type: boolean
     default: false
   arch:
     type: string
     # Only supported for use_docker=false, and only other value allowed is 386
-    default: amd64
+    default: amd64 # must be 386 or amd64
 steps:
+  - configure-git
   - run:
       name: Compute test cache key
       command: |
         TZ=GMT date '+%Y%m%d' > /tmp/go-cache-key
   - restore_cache:
       keys:
         - go-test-cache-date-v1-{{ checksum "/tmp/go-cache-key" }}
+  - restore_go_mod_cache
   - run:
       name: Run Go tests
       no_output_timeout: 60m
+      environment:
+        GOPRIVATE: 'github.com/hashicorp/*'
       command: |
-        set -x
+        set -exo pipefail
 
+        EXTRA_TAGS=
         case "<< parameters.extra_flags >>" in
         *-race*) export VAULT_CI_GO_TEST_RACE=1;;
+        *)       EXTRA_TAGS=deadlock;;
         esac
 
         # Install CircleCI CLI
@@ -52,21 +58,36 @@ steps:
         USE_DOCKER=1
         <</ parameters.use_docker >>
 
-        # Split Go tests by prior test times.  If use_docker is true, only run
-        # tests that depend on docker, otherwise only those that don't.
-        if [ $USE_DOCKER == 1 ]; then
-          package_names=$(go list -test -json ./... |
-            jq -r 'select(.Deps != null) |
-              select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) |
-              .ForTest | select(. != null)' |
-              sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname)
-        else
-          package_names=$(go list -test -json ./... |
-            jq -r 'select(.Deps != null) |
-              select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) |
-              .ForTest | select(. != null)' |
-              sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname)
-        fi
+        # Check all directories with a go.mod file
+        modules=("." "api" "sdk")
+        all_package_names=""
+
+        for dir in "${modules[@]}"
+        do
+          pushd "$dir"
+          # On its own line so that -e will fail the tests if we detect errors here.
+          go list -test -json ./... > test-list.json
+          # Split Go tests by prior test times.  If use_docker is true, only run
+          # tests that depend on docker, otherwise only those that don't.
+          # The appended true condition ensures the command will succeed if no packages are found
+          if [ $USE_DOCKER == 1 ]; then
+            package_names=$(< test-list.json jq -r 'select(.Deps != null) |
+                select(any(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker"))) |
+                .ForTest | select(. != null)' |
+                sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
+          else
+            package_names=$(< test-list.json jq -r 'select(.Deps != null) |
+                select(all(.Deps[] ; contains("github.com/hashicorp/vault/helper/testhelpers/docker")|not)) |
+                .ForTest | select(. != null)' |
+                sort -u | grep -v vault/integ | circleci tests split --split-by=timings --timings-type=classname || true)
+          fi
+          # Move back into root directory
+          popd
+          # Append the test packages into the global list, if any are found
+          if [ -n "$package_names" ]; then
+            all_package_names+=" ${package_names}"
+          fi
+        done
 
         # After running tests split step, we are now running the following steps
         # in multiple different containers, each getting a different subset of
@@ -100,9 +121,17 @@ steps:
             TEST_DOCKER_NETWORK_ID=$(docker network create vaulttest)
           fi
 
+
+
           # Start a docker testcontainer to run the tests in
-          docker run -d -e TEST_DOCKER_NETWORK_ID \
-            -e DOCKER_CERT_PATH -e DOCKER_HOST -e DOCKER_MACHINE_NAME -e DOCKER_TLS_VERIFY -e NO_PROXY \
+          docker run -d \
+            -e TEST_DOCKER_NETWORK_ID \
+            -e GOPRIVATE \
+            -e DOCKER_CERT_PATH \
+            -e DOCKER_HOST \
+            -e DOCKER_MACHINE_NAME \
+            -e DOCKER_TLS_VERIFY \
+            -e NO_PROXY \
             -e VAULT_TEST_LOG_DIR=<< parameters.log_dir >> \
             --network vaulttest --name \
             testcontainer << parameters.go_image >> \
@@ -114,31 +143,39 @@ steps:
           docker cp . testcontainer:/go/src/github.com/hashicorp/vault/
           docker cp $DOCKER_CERT_PATH/ testcontainer:$DOCKER_CERT_PATH
 
+          # Copy the downloaded modules inside the container.
+          docker exec testcontainer sh -c 'mkdir -p /go/pkg'
+          docker cp "$(go env GOPATH)/pkg/mod" testcontainer:/go/pkg/mod
+
           docker exec -w /go/src/github.com/hashicorp/vault/ \
-            -e GO111MODULE -e CIRCLECI -e GOCACHE=/tmp/gocache -e VAULT_CI_GO_TEST_RACE \
+            -e CIRCLECI -e VAULT_CI_GO_TEST_RACE \
+            -e GOCACHE=/tmp/gocache \
+            -e GO_TAGS \
+            -e GOPROXY="off" \
             -e VAULT_LICENSE_CI \
+            -e GOARCH=<< parameters.arch >> \
             testcontainer \
-            gotestsum --format=short-verbose \
-              --junitfile test-results/go-test/results.xml \
-              --jsonfile test-results/go-test/results.json \
-              -- \
-              -tags "${GO_TAGS}" \
-              -timeout=60m \
-              -parallel=20 \
-              << parameters.extra_flags >> \
-              ${package_names}
+              gotestsum --format=short-verbose \
+                --junitfile test-results/go-test/results.xml \
+                --jsonfile test-results/go-test/results.json \
+                -- \
+              -tags "${GO_TAGS} ${EXTRA_TAGS}" \
+                -timeout=60m \
+                -parallel=20 \
+                << parameters.extra_flags >> \
+                ${all_package_names}
         else
           GOARCH=<< parameters.arch >> \
             GOCACHE=<< parameters.cache_dir >> \
             gotestsum --format=short-verbose \
               --junitfile test-results/go-test/results.xml \
               --jsonfile test-results/go-test/results.json \
               -- \
-              -tags "${GO_TAGS}" \
+              -tags "${GO_TAGS} ${EXTRA_TAGS}" \
               -timeout=60m \
               -parallel=20 \
               << parameters.extra_flags >> \
-              ${package_names}
+              ${all_package_names}
         fi
 
   - when:

.circleci/config/commands/setup-go.yml

@@ -1,6 +1,16 @@
 ---
 description: >
   Ensure the right version of Go is installed and set GOPATH to $HOME/go.
+parameters:
+  GOPROXY:
+    description: >
+      Set GOPROXY. By default this is set to "off" meaning you have to have all modules pre-downloaded.
+    type: string
+    default: "off"
+  GOPRIVATE:
+    description: Set GOPRIVATE, defaults to github.com/hashicorp/*
+    type: string
+    default: github.com/hashicorp/*
 steps:
   - run:
       working_directory: ~/
@@ -16,6 +26,8 @@ steps:
         mkdir $GOPATH 2>/dev/null || { sudo mkdir $GOPATH && sudo chmod 777 $GOPATH; }
         echo "export GOPATH='$GOPATH'" >> "$BASH_ENV"
         echo "export PATH='$PATH:$GOPATH/bin:/usr/local/go/bin'" >> "$BASH_ENV"
+        echo "export GOPROXY=<<parameters.GOPROXY>>" >> "$BASH_ENV"
+        echo "export GOPRIVATE=<<parameters.GOPRIVATE>>" >> "$BASH_ENV"
 
         echo "$ go version"
         go version

.circleci/config/executors/@executors.yml

@@ -2,15 +2,14 @@ go-machine:
   machine: true
   shell: /usr/bin/env bash -euo pipefail -c
   environment:
-    GO111MODULE: "off"
     CIRCLECI_CLI_VERSION: 0.1.5546  # Pin CircleCI CLI to patch version (ex: 1.2.3)
-    GO_VERSION: 1.16.2  # Pin Go to patch version (ex: 1.2.3)
+    GO_VERSION: 1.17.2  # Pin Go to patch version (ex: 1.2.3)
     GOTESTSUM_VERSION: 0.5.2  # Pin gotestsum to patch version (ex: 1.2.3)
     GO_TAGS: ""
   working_directory: /go/src/github.com/hashicorp/vault
 node:
   docker:
-    - image: docker.mirror.hashicorp.services/node:10-buster
+    - image: docker.mirror.hashicorp.services/node:14-buster
   shell: /usr/bin/env bash -euo pipefail -c
   working_directory: /go/src/github.com/hashicorp/vault
 python:
@@ -26,27 +25,24 @@ alpine:
 docker-env-go-test-remote-docker:
   resource_class: medium
   docker:
-    - image: "docker.mirror.hashicorp.services/circleci/golang:1.16.2-buster"
+    - image: "docker.mirror.hashicorp.services/circleci/golang:1.17.2-buster"
   environment:
-    GO111MODULE: "off"
     CIRCLECI_CLI_VERSION: 0.1.5546  # Pin CircleCI CLI to patch version (ex: 1.2.3)
     GO_TAGS: ""
   working_directory: /go/src/github.com/hashicorp/vault
 docker-env-go-test:
   resource_class: large
   docker:
-    - image: "docker.mirror.hashicorp.services/circleci/golang:1.16.2-buster"
+    - image: "docker.mirror.hashicorp.services/circleci/golang:1.17.2-buster"
   environment:
-    GO111MODULE: "off"
     CIRCLECI_CLI_VERSION: 0.1.5546  # Pin CircleCI CLI to patch version (ex: 1.2.3)
     GO_TAGS: ""
   working_directory: /go/src/github.com/hashicorp/vault
 docker-env-go-test-race:
   resource_class: xlarge
   docker:
-    - image: "docker.mirror.hashicorp.services/circleci/golang:1.16.2-buster"
+    - image: "docker.mirror.hashicorp.services/circleci/golang:1.17.2-buster"
   environment:
-    GO111MODULE: "off"
     CIRCLECI_CLI_VERSION: 0.1.5546  # Pin CircleCI CLI to patch version (ex: 1.2.3)
     GO_TAGS: ""
   working_directory: /go/src/github.com/hashicorp/vault

.circleci/config/jobs/build-go-dev.yml

@@ -2,6 +2,7 @@ executor: go-machine
 steps:
   - setup-go
   - checkout
+  - restore_go_mod_cache
   - attach_workspace:
       at: .
   - run:

.circleci/config/jobs/pre-flight-checks.yml

@@ -1,21 +1,20 @@
-description: Ensures nothing obvious is broken for faster failures.
-docker:
-  - image: circleci/buildpack-deps
-shell: /usr/bin/env bash -euo pipefail
-environment:
-  CCI_VERSION: 0.1.5691
+description: Ensure nothing obvious is broken, and pre-cache Go modules.
+executor: go-machine
 steps:
+  # Setup Go enabling the proxy for downloading modules.
+  - setup-go:
+      GOPROXY: https://proxy.golang.org,direct
   - checkout
   - run:
       name: Install CircleCI CLI
       environment:
         ARCH: linux_amd64
         BASE: https://github.com/CircleCI-Public/circleci-cli/releases/download
       command: |
-        export CCI_PATH=/tmp/circleci-cli/$CCI_VERSION
+        export CCI_PATH=/tmp/circleci-cli/$CIRCLECI_CLI_VERSION
         mkdir -p $CCI_PATH
-        NAME=circleci-cli_${CCI_VERSION}_${ARCH}
-        URL=$BASE/v${CCI_VERSION}/${NAME}.tar.gz
+        NAME=circleci-cli_${CIRCLECI_CLI_VERSION}_${ARCH}
+        URL=$BASE/v${CIRCLECI_CLI_VERSION}/${NAME}.tar.gz
         curl -sSL $URL \
           | tar --overwrite --strip-components=1 -xz -C $CCI_PATH "${NAME}/circleci"
         # Add circleci to the path for subsequent steps.
@@ -26,3 +25,5 @@ steps:
         which circleci
         circleci version
   - run: make ci-verify
+  - configure-git
+  - refresh_go_mod_cache

.circleci/config/jobs/test-go-nightly.yml

@@ -1,8 +1,8 @@
 executor: go-machine
 steps:
-  - check-branch-name
   - setup-go
   - checkout
+  - restore_go_mod_cache
   - go_test:
       log_dir: "/tmp/testlogs"
       save_cache: true