Skip to content

Commit

Permalink
Use a dedicated runner for the binary-based tests. (hashicorp#20377)
Browse files Browse the repository at this point in the history
  • Loading branch information
ncabatoff authored Apr 27, 2023
1 parent 299fe69 commit 7012ce2
Show file tree
Hide file tree
Showing 11 changed files with 321 additions and 348 deletions.
8 changes: 8 additions & 0 deletions .github/scripts/generate-test-package-lists.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
# solution. It distributes the entire set of test packages into 16 sublists,
# which should roughly take an equal amount of time to complete.

set -e

test_packages=()

base="github.com/hashicorp/vault"
Expand Down Expand Up @@ -282,3 +284,9 @@ if [ "${ENTERPRISE:+x}" == "x" ] ; then
test_packages[16]+=" $base/vault/external_tests/replicationext"
test_packages[16]+=" $base/vault/external_tests/sealext"
fi

for i in $(cd $(git rev-parse --show-toplevel) && go list -test -json ./... |
jq -r '.ForTest | select(.!=null) | select(.|test("_binary$"))');
do
test_packages[17]+=" $i"
done
5 changes: 4 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,10 @@ jobs:
!startsWith(github.head_ref, 'backport/docs/')
uses: ./.github/workflows/test-go.yml
with:
total-runners: 16
# The regular Go tests use an extra runner to execute the
# binary-dependent tests. We isolate them there so that the
# other tests aren't slowed down waiting for a binary build.
total-runners: 17
go-arch: amd64
go-build-tags: '${{ needs.setup.outputs.go-build-tags }},deadlock'
runs-on: ${{ needs.setup.outputs.compute-larger }}
Expand Down
103 changes: 10 additions & 93 deletions .github/workflows/test-go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,91 +60,13 @@ jobs:
INDEX_LIST="$(seq 1 ${{ inputs.total-runners }})"
INDEX_JSON="$(jq --null-input --compact-output '. |= [inputs]' <<< "${INDEX_LIST}")"
echo "indexes=${INDEX_JSON}" >> "${GITHUB_OUTPUT}"
build-vault:
permissions:
id-token: write # Note: this permission is explicitly required for Vault auth
contents: read
runs-on: ${{ fromJSON(inputs.runs-on) }}
name: Build Vault dev binary
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
if: |
! contains(inputs.extra-flags, '-race') &&
! contains(inputs.go-build-tags, 'fips') &&
github.repository != 'hashicorp/vault-enterprise'
- uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613
if: |
! contains(inputs.extra-flags, '-race') &&
! contains(inputs.go-build-tags, 'fips') &&
github.repository != 'hashicorp/vault-enterprise'
with:
go-version-file: ./.go-version
cache: true
- name: Authenticate to Vault
id: vault-auth
if: github.repository == 'hashicorp/vault-enterprise'
run: vault-auth
- name: Fetch Secrets
id: secrets
if: github.repository == 'hashicorp/vault-enterprise'
uses: hashicorp/vault-action@130d1f5f4fe645bb6c83e4225c04d64cfb62de6e
with:
url: ${{ steps.vault-auth.outputs.addr }}
caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
token: ${{ steps.vault-auth.outputs.token }}
secrets: |
kv/data/github/${{ github.repository }}/datadog-ci DATADOG_API_KEY;
kv/data/github/${{ github.repository }}/github-token username-and-token | github-token;
kv/data/github/${{ github.repository }}/license license_1 | VAULT_LICENSE_CI;
kv/data/github/${{ github.repository }}/license license_2 | VAULT_LICENSE_2;
kv/data/github/${{ github.repository }}/hcp-link HCP_API_ADDRESS;
kv/data/github/${{ github.repository }}/hcp-link HCP_AUTH_URL;
kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_ID;
kv/data/github/${{ github.repository }}/hcp-link HCP_CLIENT_SECRET;
kv/data/github/${{ github.repository }}/hcp-link HCP_RESOURCE_ID;
- id: setup-git-private
name: Setup Git configuration (private)
if: github.repository == 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ steps.secrets.outputs.github-token }}@github.com".insteadOf https://github.com
- id: setup-git-public
name: Setup Git configuration (public)
if: github.repository != 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
- id: go-mod-download
if: |
! contains(inputs.extra-flags, '-race') &&
! contains(inputs.go-build-tags, 'fips') &&
github.repository != 'hashicorp/vault-enterprise'
env:
GOPRIVATE: github.com/hashicorp/*
run: time go mod download -x
- id: build
if: |
! contains(inputs.extra-flags, '-race') &&
! contains(inputs.go-build-tags, 'fips') &&
github.repository != 'hashicorp/vault-enterprise'
env:
GOPRIVATE: github.com/hashicorp/*
run: time make ci-bootstrap dev
- name: Save dev binary
if: |
! contains(inputs.extra-flags, '-race') &&
! contains(inputs.go-build-tags, 'fips') &&
github.repository != 'hashicorp/vault-enterprise'
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
with:
name: vault-dev
path: bin/vault
test-go:
permissions:
id-token: write # Note: this permission is explicitly required for Vault auth
contents: read
name: "${{ matrix.runner-index }}"
needs:
- runner-indexes
- build-vault
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
Expand Down Expand Up @@ -195,21 +117,16 @@ jobs:
if: github.repository != 'hashicorp/vault-enterprise'
run: |
git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN}}@github.com".insteadOf https://github.com
- name: Retrieve vault dev binary
if: |
! contains(inputs.extra-flags, '-race') &&
! contains(inputs.go-build-tags, 'fips') &&
github.repository != 'hashicorp/vault-enterprise'
uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
with:
name: vault-dev
path: bin
- name: Make vault dev binary executable
if: |
! contains(inputs.extra-flags, '-race') &&
! contains(inputs.go-build-tags, 'fips') &&
github.repository != 'hashicorp/vault-enterprise'
run: chmod a+x bin/vault
- id: go-mod-download
if: matrix.runner-index > 16
env:
GOPRIVATE: github.com/hashicorp/*
run: time go mod download -x
- id: build
if: matrix.runner-index > 16
env:
GOPRIVATE: github.com/hashicorp/*
run: time make ci-bootstrap dev
- id: run-go-tests
name: Run Go tests
timeout-minutes: ${{ fromJSON(env.TIMEOUT_IN_MINUTES) }}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package pkiext
package pkiext_binary

import (
"context"
Expand All @@ -10,6 +10,7 @@ import (
"testing"

"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/builtin/logical/pkiext"
hDocker "github.com/hashicorp/vault/sdk/helper/docker"
"github.com/hashicorp/vault/sdk/helper/testcluster"
tcDocker "github.com/hashicorp/vault/sdk/helper/testcluster/docker"
Expand All @@ -21,8 +22,8 @@ func CheckCertBot(t *testing.T, vaultNetwork string, vaultNodeID string, directo
t.Logf(s)
}

logStdout := &LogConsumerWriter{logConsumer}
logStderr := &LogConsumerWriter{logConsumer}
logStdout := &pkiext.LogConsumerWriter{logConsumer}
logStderr := &pkiext.LogConsumerWriter{logConsumer}

t.Logf("creating on network: %v", vaultNetwork)
runner, err := hDocker.NewServiceRunner(hDocker.RunOptions{
Expand Down
4 changes: 2 additions & 2 deletions builtin/logical/pkiext/test_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func parseKey(t *testing.T, pemKey string) crypto.Signer {
}

type LogConsumerWriter struct {
consumer func(string)
Consumer func(string)
}

func (l LogConsumerWriter) Write(p []byte) (n int, err error) {
Expand All @@ -80,7 +80,7 @@ func (l LogConsumerWriter) Write(p []byte) (n int, err error) {
scanner := bufio.NewScanner(bytes.NewReader(p))
scanner.Buffer(make([]byte, 64*1024), bufio.MaxScanTokenSize)
for scanner.Scan() {
l.consumer(scanner.Text())
l.Consumer(scanner.Text())
}
return len(p), nil
}
140 changes: 140 additions & 0 deletions vault/external_tests/pprof/pprof.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package pprof

import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
"testing"

"github.com/hashicorp/go-cleanhttp"
"github.com/hashicorp/vault/api"
"github.com/hashicorp/vault/sdk/helper/testcluster"
"github.com/stretchr/testify/require"
"golang.org/x/net/http2"
)

func SysPprof_Test(t *testing.T, cluster testcluster.VaultCluster) {
nodes := cluster.Nodes()
client := nodes[0].APIClient()

transport := cleanhttp.DefaultPooledTransport()
transport.TLSClientConfig = nodes[0].TLSConfig()
if err := http2.ConfigureTransport(transport); err != nil {
t.Fatal(err)
}
httpClient := &http.Client{
Transport: transport,
}

cases := []struct {
name string
path string
seconds string
}{
{
"index",
"/v1/sys/pprof/",
"",
},
{
"cmdline",
"/v1/sys/pprof/cmdline",
"",
},
{
"goroutine",
"/v1/sys/pprof/goroutine",
"",
},
{
"heap",
"/v1/sys/pprof/heap",
"",
},
{
"profile",
"/v1/sys/pprof/profile",
"1",
},
{
"symbol",
"/v1/sys/pprof/symbol",
"",
},
{
"trace",
"/v1/sys/pprof/trace",
"1",
},
}

pprofRequest := func(t *testing.T, path string, seconds string) {
req := client.NewRequest("GET", path)
if seconds != "" {
req.Params.Set("seconds", seconds)
}
httpReq, err := req.ToHTTP()
if err != nil {
t.Fatal(err)
}
resp, err := httpClient.Do(httpReq)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()

httpRespBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}

httpResp := make(map[string]interface{})

// Skip this error check since some endpoints return binary blobs, we
// only care about the ok check right after as an existence check.
_ = json.Unmarshal(httpRespBody, &httpResp)

// Make sure that we don't get a error response
if _, ok := httpResp["errors"]; ok {
t.Fatalf("unexpected error response: %v", httpResp["errors"])
}

if len(httpRespBody) == 0 {
t.Fatal("no pprof index returned")
}
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
pprofRequest(t, tc.path, tc.seconds)
})
}
}

func SysPprof_Standby_Test(t *testing.T, cluster testcluster.VaultCluster) {
pprof := func(client *api.Client) (string, error) {
req := client.NewRequest("GET", "/v1/sys/pprof/cmdline")
resp, err := client.RawRequestWithContext(context.Background(), req)
if err != nil {
return "", err
}
defer resp.Body.Close()

data, err := ioutil.ReadAll(resp.Body)
return string(data), err
}

cmdline, err := pprof(cluster.Nodes()[0].APIClient())
require.Nil(t, err)
require.NotEmpty(t, cmdline)
t.Log(cmdline)

cmdline, err = pprof(cluster.Nodes()[1].APIClient())
require.Nil(t, err)
require.NotEmpty(t, cmdline)
t.Log(cmdline)
}
Loading

0 comments on commit 7012ce2

Please sign in to comment.