From 56690543ab816aa0eb6745c6e417ec3e099d754c Mon Sep 17 00:00:00 2001 From: Colleen Murphy Date: Wed, 7 Feb 2024 14:34:00 -0800 Subject: [PATCH] Make E2E tests hermetic (#3499) * Set rekor URL for online and offline tests Some tests were setting the REKOR_URL environment variable to try to test offline verification. This variable is no longer read so it was not doing anything. This change removes the variable and instead sets RekorURL in the command to either the local rekor instance (so that the public instance is not used) or to a bad url with Offline set to true so that offline verification is truly tested. This change also removes the COSIGN_EXPERIMENTAL variable which is no longer used, and replaces os.Setenv with testing.Setenv which localizes the environment setting to the scope of the test and removes the need for a cleanup function. Signed-off-by: Colleen Murphy * Fix cleanup in E2E script Calling trap multiple times replaces the last signal handler rather than appending to it. This change ensures that the most recent trap includes all previous traps so that all cleanups are executed. Signed-off-by: Colleen Murphy * Move verify tests from shell script to Go suite Move the `cosign dockerfile verify` and `cosign manifest verify` tests out of the shell script and into the e2e Go test suite file with all the other tests. This makes them consistent to manage. The initialization of fulcio roots in other tests pollutes the trust root in the new tests, so a reset is added to the fulcioroots package for testing only. Signed-off-by: Colleen Murphy * Use local services for verify tests Update TestDockerfileVerify and TestManifestVerify to sign ephemeral images within the tests so that the signatures can be created with and verified from the locally running Fulcio and Rekor instances instead of verifying images with the public Rekor instance, so that the tests no longer depend on external services. The images are signed using --identity-token to avoid changing the nature of the verification tests, which were originally written to be keyless. A mock OIDC server is provisioned to provide the token and enable verification. Signed-off-by: Colleen Murphy * Set rekor env variable in Go test suite Move the setting of SIGSTORE_REKOR_PUBLIC_KEY from the e2e shell script to the Go test suite, so that only the tests that need it have it set and the shell script is doing less setup. Also remove unnecessary instances of os.RemoveAll for temporary directories that the Go testing framework will automatically clean up. Signed-off-by: Colleen Murphy --------- Signed-off-by: Colleen Murphy Signed-off-by: Noah Kreiger --- .../cosign/fulcio/fulcioroots/fulcioroots.go | 7 + test/e2e_test.go | 418 ++++++++++++++++-- test/e2e_test.sh | 48 +- test/fakeoidc/go.mod | 11 + test/fakeoidc/go.sum | 14 + test/fakeoidc/main.go | 118 +++++ test/testdata/fancy_from.Dockerfile | 17 - test/testdata/signed_manifest.yaml | 23 - test/testdata/single_stage.Dockerfile | 17 - test/testdata/unsigned_build_stage.Dockerfile | 24 - test/testdata/unsigned_manifest.yaml | 23 - test/testdata/with_arg.Dockerfile | 17 - test/testdata/with_lowercase.Dockerfile | 15 - 13 files changed, 554 insertions(+), 198 deletions(-) create mode 100644 test/fakeoidc/go.mod create mode 100644 test/fakeoidc/go.sum create mode 100644 test/fakeoidc/main.go delete mode 100644 test/testdata/fancy_from.Dockerfile delete mode 100644 test/testdata/signed_manifest.yaml delete mode 100644 test/testdata/single_stage.Dockerfile delete mode 100644 test/testdata/unsigned_build_stage.Dockerfile delete mode 100644 test/testdata/unsigned_manifest.yaml delete mode 100644 test/testdata/with_arg.Dockerfile delete mode 100644 test/testdata/with_lowercase.Dockerfile diff --git a/internal/pkg/cosign/fulcio/fulcioroots/fulcioroots.go b/internal/pkg/cosign/fulcio/fulcioroots/fulcioroots.go index 3d86353c35f5..3b44da884201 100644 --- a/internal/pkg/cosign/fulcio/fulcioroots/fulcioroots.go +++ b/internal/pkg/cosign/fulcio/fulcioroots/fulcioroots.go @@ -56,6 +56,13 @@ func GetIntermediates() (*x509.CertPool, error) { return intermediates, singletonRootErr } +// ReInit reinitializes the global roots and intermediates, overriding the sync.Once lock. +// This is only to be used for tests, where the trust root environment variables may change after the roots are initialized in the module. +func ReInit() error { + roots, intermediates, singletonRootErr = initRoots() + return singletonRootErr +} + func initRoots() (*x509.CertPool, *x509.CertPool, error) { rootPool := x509.NewCertPool() // intermediatePool should be nil if no intermediates are found diff --git a/test/e2e_test.go b/test/e2e_test.go index 09c397eb1767..929d0946e819 100644 --- a/test/e2e_test.go +++ b/test/e2e_test.go @@ -29,6 +29,8 @@ import ( "encoding/json" "encoding/pem" "fmt" + "io" + "net/http" "net/http/httptest" "net/url" "os" @@ -54,12 +56,15 @@ import ( "github.com/sigstore/cosign/v2/cmd/cosign/cli" "github.com/sigstore/cosign/v2/cmd/cosign/cli/attach" "github.com/sigstore/cosign/v2/cmd/cosign/cli/attest" + "github.com/sigstore/cosign/v2/cmd/cosign/cli/dockerfile" "github.com/sigstore/cosign/v2/cmd/cosign/cli/download" "github.com/sigstore/cosign/v2/cmd/cosign/cli/generate" + "github.com/sigstore/cosign/v2/cmd/cosign/cli/manifest" "github.com/sigstore/cosign/v2/cmd/cosign/cli/options" "github.com/sigstore/cosign/v2/cmd/cosign/cli/publickey" "github.com/sigstore/cosign/v2/cmd/cosign/cli/sign" cliverify "github.com/sigstore/cosign/v2/cmd/cosign/cli/verify" + "github.com/sigstore/cosign/v2/internal/pkg/cosign/fulcio/fulcioroots" "github.com/sigstore/cosign/v2/internal/pkg/cosign/tsa" "github.com/sigstore/cosign/v2/internal/pkg/cosign/tsa/client" "github.com/sigstore/cosign/v2/pkg/cosign" @@ -76,9 +81,9 @@ import ( ) const ( - serverEnv = "REKOR_SERVER" rekorURL = "http://127.0.0.1:3000" fulcioURL = "http://127.0.0.1:5555" + certID = "foo@bar.com" ) var keyPass = []byte("hello") @@ -107,6 +112,7 @@ var verify = func(keyRef, imageRef string, checkClaims bool, annotations map[str var verifyTSA = func(keyRef, imageRef string, checkClaims bool, annotations map[string]interface{}, attachment, tsaCertChain string, skipTlogVerify bool) error { cmd := cliverify.VerifyCommand{ KeyRef: keyRef, + RekorURL: rekorURL, CheckClaims: checkClaims, Annotations: sigs.AnnotationsMap{Annotations: annotations}, Attachment: attachment, @@ -127,6 +133,7 @@ var verifyKeylessTSA = func(imageRef string, tsaCertChain string, skipSCT bool, CertOidcIssuerRegexp: ".*", CertIdentityRegexp: ".*", }, + RekorURL: rekorURL, HashAlgorithm: crypto.SHA256, TSACertChainPath: tsaCertChain, IgnoreSCT: skipSCT, @@ -143,6 +150,7 @@ var verifyKeylessTSA = func(imageRef string, tsaCertChain string, skipSCT bool, var verifyLocal = func(keyRef, path string, checkClaims bool, annotations map[string]interface{}, attachment string) error { cmd := cliverify.VerifyCommand{ KeyRef: keyRef, + RekorURL: rekorURL, CheckClaims: checkClaims, Annotations: sigs.AnnotationsMap{Annotations: annotations}, Attachment: attachment, @@ -157,6 +165,24 @@ var verifyLocal = func(keyRef, path string, checkClaims bool, annotations map[st return cmd.Exec(context.Background(), args) } +var verifyOffline = func(keyRef, imageRef string, checkClaims bool, annotations map[string]interface{}, attachment string) error { + cmd := cliverify.VerifyCommand{ + KeyRef: keyRef, + RekorURL: "notreal", + Offline: true, + CheckClaims: checkClaims, + Annotations: sigs.AnnotationsMap{Annotations: annotations}, + Attachment: attachment, + HashAlgorithm: crypto.SHA256, + IgnoreTlog: true, + MaxWorkers: 10, + } + + args := []string{imageRef} + + return cmd.Exec(context.Background(), args) +} + var ro = &options.RootOptions{Timeout: options.DefaultTimeout} func TestSignVerify(t *testing.T) { @@ -999,9 +1025,6 @@ func TestAttachWithRekorBundle(t *testing.T) { } func TestRekorBundle(t *testing.T) { - // turn on the tlog - defer setenv(t, env.VariableExperimental.String(), "1")() - repo, stop := reg(t) defer stop() td := t.TempDir() @@ -1028,9 +1051,7 @@ func TestRekorBundle(t *testing.T) { must(verify(pubKeyPath, imgName, true, nil, ""), t) // Make sure offline verification works with bundling - // use rekor prod since we have hardcoded the public key - os.Setenv(serverEnv, "notreal") - must(verify(pubKeyPath, imgName, true, nil, ""), t) + must(verifyOffline(pubKeyPath, imgName, true, nil, ""), t) } func TestRekorOutput(t *testing.T) { @@ -1070,9 +1091,7 @@ func TestRekorOutput(t *testing.T) { } } // Make sure offline verification works with bundling - // use rekor prod since we have hardcoded the public key - os.Setenv(serverEnv, "notreal") - must(verify(pubKeyPath, imgName, true, nil, ""), t) + must(verifyOffline(pubKeyPath, imgName, true, nil, ""), t) } func TestFulcioBundle(t *testing.T) { @@ -1105,8 +1124,7 @@ func TestFulcioBundle(t *testing.T) { // Make sure offline verification works with bundling // use rekor prod since we have hardcoded the public key - os.Setenv(serverEnv, "notreal") - must(verify(pubKeyPath, imgName, true, nil, ""), t) + must(verifyOffline(pubKeyPath, imgName, true, nil, ""), t) } func TestRFC3161Timestamp(t *testing.T) { @@ -1165,6 +1183,12 @@ func TestRFC3161Timestamp(t *testing.T) { } func TestRekorBundleAndRFC3161Timestamp(t *testing.T) { + td := t.TempDir() + err := downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), td) + if err != nil { + t.Fatal(err) + } + // TSA server needed to create timestamp viper.Set("timestamp-signer", "memory") viper.Set("timestamp-signer-hash", "sha256") @@ -1194,7 +1218,6 @@ func TestRekorBundleAndRFC3161Timestamp(t *testing.T) { repo, stop := reg(t) defer stop() - td := t.TempDir() imgName := path.Join(repo, "cosign-e2e") @@ -1276,7 +1299,7 @@ func TestKeyURLVerify(t *testing.T) { } func TestGenerateKeyPairEnvVar(t *testing.T) { - defer setenv(t, "COSIGN_PASSWORD", "foo")() + t.Setenv("COSIGN_PASSWORD", "foo") keys, err := cosign.GenerateKeyPair(generate.GetPass) if err != nil { t.Fatal(err) @@ -1299,7 +1322,7 @@ func TestGenerateKeyPairK8s(t *testing.T) { os.Chdir(wd) }() password := "foo" - defer setenv(t, "COSIGN_PASSWORD", password)() + t.Setenv("COSIGN_PASSWORD", password) ctx := context.Background() name := "cosign-secret" namespace := "default" @@ -1370,13 +1393,14 @@ func TestMultipleSignatures(t *testing.T) { } func TestSignBlob(t *testing.T) { + td := t.TempDir() + err := downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), td) + if err != nil { + t.Fatal(err) + } blob := "someblob" td1 := t.TempDir() td2 := t.TempDir() - t.Cleanup(func() { - os.RemoveAll(td1) - os.RemoveAll(td2) - }) bp := filepath.Join(td1, blob) if err := os.WriteFile(bp, []byte(blob), 0644); err != nil { @@ -1427,9 +1451,6 @@ func TestSignBlob(t *testing.T) { func TestSignBlobBundle(t *testing.T) { blob := "someblob" td1 := t.TempDir() - t.Cleanup(func() { - os.RemoveAll(td1) - }) bp := filepath.Join(td1, blob) bundlePath := filepath.Join(td1, "bundle.sig") @@ -1437,6 +1458,11 @@ func TestSignBlobBundle(t *testing.T) { t.Fatal(err) } + err := downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), td1) + if err != nil { + t.Fatal(err) + } + _, privKeyPath1, pubKeyPath1 := keypair(t, td1) ctx := context.Background() @@ -1472,12 +1498,17 @@ func TestSignBlobBundle(t *testing.T) { } // Point to a fake rekor server to make sure offline verification of the tlog entry works - os.Setenv(serverEnv, "notreal") + verifyBlobCmd.RekorURL = "notreal" verifyBlobCmd.IgnoreTlog = false must(verifyBlobCmd.Exec(ctx, bp), t) } func TestSignBlobRFC3161TimestampBundle(t *testing.T) { + td := t.TempDir() + err := downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), td) + if err != nil { + t.Fatal(err) + } // TSA server needed to create timestamp viper.Set("timestamp-signer", "memory") viper.Set("timestamp-signer-hash", "sha256") @@ -1486,13 +1517,9 @@ func TestSignBlobRFC3161TimestampBundle(t *testing.T) { t.Cleanup(server.Close) blob := "someblob" - td1 := t.TempDir() - t.Cleanup(func() { - os.RemoveAll(td1) - }) - bp := filepath.Join(td1, blob) - bundlePath := filepath.Join(td1, "bundle.sig") - tsPath := filepath.Join(td1, "rfc3161Timestamp.json") + bp := filepath.Join(td, blob) + bundlePath := filepath.Join(td, "bundle.sig") + tsPath := filepath.Join(td, "rfc3161Timestamp.json") if err := os.WriteFile(bp, []byte(blob), 0644); err != nil { t.Fatal(err) @@ -1518,7 +1545,7 @@ func TestSignBlobRFC3161TimestampBundle(t *testing.T) { t.Fatalf("error writing chain payload to temp file: %v", err) } - _, privKeyPath1, pubKeyPath1 := keypair(t, td1) + _, privKeyPath1, pubKeyPath1 := keypair(t, td) ctx := context.Background() @@ -1556,6 +1583,7 @@ func TestSignBlobRFC3161TimestampBundle(t *testing.T) { t.Fatal(err) } // Point to a fake rekor server to make sure offline verification of the tlog entry works + verifyBlobCmd.RekorURL = "notreal" verifyBlobCmd.IgnoreTlog = false must(verifyBlobCmd.Exec(ctx, bp), t) } @@ -2047,15 +2075,6 @@ func TestAttachSBOM_bom_flag(t *testing.T) { } } -func setenv(t *testing.T, k, v string) func() { - if err := os.Setenv(k, v); err != nil { - t.Fatalf("error setting env: %v", err) - } - return func() { - os.Unsetenv(k) - } -} - func TestTlog(t *testing.T) { repo, stop := reg(t) defer stop() @@ -2183,6 +2202,16 @@ func mkfile(contents, td string, t *testing.T) string { return f.Name() } +func mkfileWithExt(contents, td, ext string, t *testing.T) string { + f := mkfile(contents, td, t) + newName := f + ext + err := os.Rename(f, newName) + if err != nil { + t.Fatal(err) + } + return newName +} + func mkimage(t *testing.T, n string) (name.Reference, *remote.Descriptor, func()) { ref, err := name.ParseReference(n, name.WeakValidation) if err != nil { @@ -2333,7 +2362,6 @@ func TestInvalidBundle(t *testing.T) { // Now, we move on to image2 // Sign image2 and DO NOT store the entry in rekor - defer setenv(t, env.VariableExperimental.String(), "0")() img2 := path.Join(regName, "unrelated") imgRef2, _, cleanup := mkimage(t, img2) defer cleanup() @@ -2458,9 +2486,14 @@ func TestAttestBlobSignVerify(t *testing.T) { } func TestOffline(t *testing.T) { + td := t.TempDir() + err := downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), td) + if err != nil { + t.Fatal(err) + } + regName, stop := reg(t) defer stop() - td := t.TempDir() img1 := path.Join(regName, "cosign-e2e") @@ -2483,6 +2516,7 @@ func TestOffline(t *testing.T) { must(verify(pubKeyPath, img1, true, nil, ""), t) verifyCmd := &cliverify.VerifyCommand{ KeyRef: pubKeyPath, + RekorURL: "notreal", Offline: true, CheckClaims: true, MaxWorkers: 10, @@ -2524,3 +2558,303 @@ func TestOffline(t *testing.T) { // Confirm offline verification fails mustErr(verifyCmd.Exec(ctx, []string{img1}), t) } + +func TestDockerfileVerify(t *testing.T) { + td := t.TempDir() + + // set up SIGSTORE_ variables to point to keys for the local instances + err := setLocalEnv(t, td) + if err != nil { + t.Fatal(err) + } + + // unset the roots that were generated for timestamp signing, they won't work here + err = fulcioroots.ReInit() + if err != nil { + t.Fatal(err) + } + + identityToken, err := getOIDCToken() + if err != nil { + t.Fatal(err) + } + + // create some images + repo, stop := reg(t) + defer stop() + signedImg1 := path.Join(repo, "cosign-e2e-dockerfile-signed1") + _, _, cleanup1 := mkimage(t, signedImg1) + defer cleanup1() + signedImg2 := path.Join(repo, "cosign-e2e-dockerfile-signed2") + _, _, cleanup2 := mkimage(t, signedImg2) + defer cleanup2() + unsignedImg := path.Join(repo, "cosign-e2e-dockerfile-unsigned") + _, _, cleanupUnsigned := mkimage(t, unsignedImg) + defer cleanupUnsigned() + + // sign the images using --identity-token + ko := options.KeyOpts{ + FulcioURL: fulcioURL, + RekorURL: rekorURL, + IDToken: identityToken, + SkipConfirmation: true, + } + so := options.SignOptions{ + Upload: true, + TlogUpload: true, + SkipConfirmation: true, + } + ctx := context.Background() + must(sign.SignCmd(ro, ko, so, []string{signedImg1}), t) + must(sign.SignCmd(ro, ko, so, []string{signedImg2}), t) + + // create the dockerfiles + singleStageDockerfileContents := fmt.Sprintf(` +FROM %s +`, signedImg1) + singleStageDockerfile := mkfile(singleStageDockerfileContents, td, t) + + unsignedBuildStageDockerfileContents := fmt.Sprintf(` +FROM %s + +FROM %s + +FROM %s +`, signedImg1, unsignedImg, signedImg2) + unsignedBuildStageDockerfile := mkfile(unsignedBuildStageDockerfileContents, td, t) + + fromAsDockerfileContents := fmt.Sprintf(` +FROM --platform=linux/amd64 %s AS base +`, signedImg1) + fromAsDockerfile := mkfile(fromAsDockerfileContents, td, t) + + withArgDockerfileContents := ` +ARG test_image + +FROM ${test_image} +` + withArgDockerfile := mkfile(withArgDockerfileContents, td, t) + + withLowercaseDockerfileContents := fmt.Sprintf(` +from %s +`, signedImg1) + withLowercaseDockerfile := mkfile(withLowercaseDockerfileContents, td, t) + + issuer := os.Getenv("OIDC_URL") + + tests := []struct { + name string + dockerfile string + baseOnly bool + env map[string]string + wantErr bool + }{ + { + name: "verify single stage", + dockerfile: singleStageDockerfile, + }, + { + name: "verify unsigned build stage", + dockerfile: unsignedBuildStageDockerfile, + wantErr: true, + }, + { + name: "verify base image only", + dockerfile: unsignedBuildStageDockerfile, + baseOnly: true, + }, + { + name: "verify from as", + dockerfile: fromAsDockerfile, + }, + { + name: "verify with arg", + dockerfile: withArgDockerfile, + env: map[string]string{"test_image": signedImg1}, + }, + { + name: "verify image exists but is unsigned", + dockerfile: withArgDockerfile, + env: map[string]string{"test_image": unsignedImg}, + wantErr: true, + }, + { + name: "verify with lowercase", + dockerfile: withLowercaseDockerfile, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := dockerfile.VerifyDockerfileCommand{ + VerifyCommand: cliverify.VerifyCommand{ + CertVerifyOptions: options.CertVerifyOptions{ + CertOidcIssuer: issuer, + CertIdentity: certID, + }, + RekorURL: rekorURL, + }, + BaseOnly: test.baseOnly, + } + args := []string{test.dockerfile} + for k, v := range test.env { + t.Setenv(k, v) + } + if test.wantErr { + mustErr(cmd.Exec(ctx, args), t) + } else { + must(cmd.Exec(ctx, args), t) + } + }) + } +} + +func TestManifestVerify(t *testing.T) { + td := t.TempDir() + + // set up SIGSTORE_ variables to point to keys for the local instances + err := setLocalEnv(t, td) + if err != nil { + t.Fatal(err) + } + + // unset the roots that were generated for timestamp signing, they won't work here + err = fulcioroots.ReInit() + if err != nil { + t.Fatal(err) + } + + identityToken, err := getOIDCToken() + if err != nil { + t.Fatal(err) + } + + // create some images + repo, stop := reg(t) + defer stop() + signedImg := path.Join(repo, "cosign-e2e-manifest-signed") + _, _, cleanup := mkimage(t, signedImg) + defer cleanup() + unsignedImg := path.Join(repo, "cosign-e2e-manifest-unsigned") + _, _, cleanupUnsigned := mkimage(t, unsignedImg) + defer cleanupUnsigned() + + // sign the images using --identity-token + ko := options.KeyOpts{ + FulcioURL: fulcioURL, + RekorURL: rekorURL, + IDToken: identityToken, + SkipConfirmation: true, + } + so := options.SignOptions{ + Upload: true, + TlogUpload: true, + SkipConfirmation: true, + } + ctx := context.Background() + must(sign.SignCmd(ro, ko, so, []string{signedImg}), t) + + // create the manifests + manifestTemplate := ` +apiVersion: v1 +kind: Pod +metadata: + name: single-pod +spec: + containers: + - name: %s + image: %s +` + signedManifestContents := fmt.Sprintf(manifestTemplate, "signed-img", signedImg) + signedManifest := mkfileWithExt(signedManifestContents, td, ".yaml", t) + unsignedManifestContents := fmt.Sprintf(manifestTemplate, "unsigned-img", unsignedImg) + unsignedManifest := mkfileWithExt(unsignedManifestContents, td, ".yaml", t) + + issuer := os.Getenv("OIDC_URL") + + tests := []struct { + name string + manifest string + wantErr bool + }{ + { + name: "signed manifest", + manifest: signedManifest, + }, + { + name: "unsigned manifest", + manifest: unsignedManifest, + wantErr: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := manifest.VerifyManifestCommand{ + VerifyCommand: cliverify.VerifyCommand{ + CertVerifyOptions: options.CertVerifyOptions{ + CertOidcIssuer: issuer, + CertIdentity: certID, + }, + RekorURL: rekorURL, + }, + } + args := []string{test.manifest} + if test.wantErr { + mustErr(cmd.Exec(ctx, args), t) + } else { + must(cmd.Exec(ctx, args), t) + } + }) + } +} + +// getOIDCToken gets an OIDC token from the mock OIDC server. +func getOIDCToken() (string, error) { + issuer := os.Getenv("OIDC_URL") + resp, err := http.Get(issuer + "/token") + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +func setLocalEnv(t *testing.T, dir string) error { + // fulcio repo is downloaded to the user's home directory by e2e_test.sh + home, err := os.UserHomeDir() + if err != nil { + fmt.Errorf("error getting home directory: %w", err) + } + t.Setenv(env.VariableSigstoreCTLogPublicKeyFile.String(), path.Join(home, "fulcio/config/ctfe/pubkey.pem")) + err = downloadAndSetEnv(t, fulcioURL+"/api/v1/rootCert", env.VariableSigstoreRootFile.String(), dir) + if err != nil { + return fmt.Errorf("error setting %s env var: %w", env.VariableSigstoreRootFile.String(), err) + } + err = downloadAndSetEnv(t, rekorURL+"/api/v1/log/publicKey", env.VariableSigstoreRekorPublicKey.String(), dir) + if err != nil { + return fmt.Errorf("error setting %s env var: %w", env.VariableSigstoreRekorPublicKey.String(), err) + } + return nil +} + +func downloadAndSetEnv(t *testing.T, url, envVar, dir string) error { + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("error downloading file: %w", err) + } + defer resp.Body.Close() + f, err := os.CreateTemp(dir, "") + if err != nil { + return fmt.Errorf("error creating temp file: %w", err) + } + defer f.Close() + _, err = io.Copy(f, resp.Body) + if err != nil { + return fmt.Errorf("error writing to file: %w", err) + } + t.Setenv(envVar, f.Name()) + return nil +} diff --git a/test/e2e_test.sh b/test/e2e_test.sh index 7fb9a1310df0..3c60c76da5e0 100755 --- a/test/e2e_test.sh +++ b/test/e2e_test.sh @@ -16,6 +16,31 @@ set -ex +echo "setting up OIDC provider" +pushd ./test/fakeoidc +oidcimg=$(ko build main.go --local) +docker network ls | grep fulcio_default || docker network create fulcio_default +docker run -d --rm -p 8080:8080 --network fulcio_default --name fakeoidc $oidcimg +cleanup_oidc() { + echo "cleaning up oidc" + docker stop fakeoidc +} +trap cleanup_oidc EXIT +oidc_ip=$(docker inspect fakeoidc | jq -r '.[0].NetworkSettings.Networks.fulcio_default.IPAddress') +export OIDC_URL="http://${oidc_ip}:8080" +cat < /tmp/fulcio-config.json +{ + "OIDCIssuers": { + "$OIDC_URL": { + "IssuerURL": "$OIDC_URL", + "ClientID": "sigstore", + "Type": "email" + } + } +} +EOF +popd + pushd $HOME echo "downloading service repos" @@ -31,6 +56,7 @@ done echo "starting services" export FULCIO_METRICS_PORT=2113 +export FULCIO_CONFIG=/tmp/fulcio-config.json for repo in rekor fulcio; do pushd $repo docker-compose up -d @@ -51,6 +77,7 @@ for repo in rekor fulcio; do done cleanup_services() { echo "cleaning up" + cleanup_oidc for repo in rekor fulcio; do pushd $HOME/$repo docker-compose down @@ -59,19 +86,16 @@ cleanup_services() { } trap cleanup_services EXIT -curl http://127.0.0.1:3000/api/v1/log/publicKey > rekor.pub -export SIGSTORE_REKOR_PUBLIC_KEY=$(pwd)/rekor.pub - echo echo "running tests" popd -go build -o cosign ./cmd/cosign go test -tags=e2e -v -race ./test/... # Test on a private registry echo "testing sign/verify/clean on private registry" cleanup() { + cleanup_services docker rm -f registry } trap cleanup EXIT @@ -79,22 +103,6 @@ docker run -d -p 5000:5000 --restart always -e REGISTRY_STORAGE_DELETE_ENABLED=t export COSIGN_TEST_REPO=localhost:5000 go test -tags=e2e -v ./test/... -run TestSignVerifyClean -# Use the public instance to verify existing images and manifests -unset SIGSTORE_REKOR_PUBLIC_KEY -# Test `cosign dockerfile verify` -./cosign dockerfile verify ./test/testdata/single_stage.Dockerfile --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com -if (./cosign dockerfile verify ./test/testdata/unsigned_build_stage.Dockerfile --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com); then false; fi -./cosign dockerfile verify --base-image-only ./test/testdata/unsigned_build_stage.Dockerfile --certificate-identity https://github.com/distroless/static/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com -./cosign dockerfile verify ./test/testdata/fancy_from.Dockerfile --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com -test_image="ghcr.io/distroless/alpine-base" ./cosign dockerfile verify ./test/testdata/with_arg.Dockerfile --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com -# Image exists, but is unsigned -if (test_image="ubuntu" ./cosign dockerfile verify ./test/testdata/with_arg.Dockerfile --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com); then false; fi -./cosign dockerfile verify ./test/testdata/with_lowercase.Dockerfile --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com - -# Test `cosign manifest verify` -./cosign manifest verify ./test/testdata/signed_manifest.yaml --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com -if (./cosign manifest verify ./test/testdata/unsigned_manifest.yaml --certificate-identity https://github.com/distroless/alpine-base/.github/workflows/release.yaml@refs/heads/main --certificate-oidc-issuer https://token.actions.githubusercontent.com); then false; fi - # Run the built container to make sure it doesn't crash make ko-local img="ko.local/cosign:$(git rev-parse HEAD)" diff --git a/test/fakeoidc/go.mod b/test/fakeoidc/go.mod new file mode 100644 index 000000000000..08534a0bc5da --- /dev/null +++ b/test/fakeoidc/go.mod @@ -0,0 +1,11 @@ +module github.com/sigstore/cosign/test/fakeoidc + +go 1.21.5 + +require gopkg.in/square/go-jose.v2 v2.6.0 + +require ( + github.com/google/go-cmp v0.6.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect + golang.org/x/crypto v0.18.0 // indirect +) diff --git a/test/fakeoidc/go.sum b/test/fakeoidc/go.sum new file mode 100644 index 000000000000..f3ea650903c3 --- /dev/null +++ b/test/fakeoidc/go.sum @@ -0,0 +1,14 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= +gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/test/fakeoidc/main.go b/test/fakeoidc/main.go new file mode 100644 index 000000000000..3220afb839cd --- /dev/null +++ b/test/fakeoidc/main.go @@ -0,0 +1,118 @@ +// +// Copyright 2024 The Sigstore Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Mock OIDC server, based on https://github.com/sigstore/fulcio/blob/4a80ee645c69bac2128b03197e04b4d285c0b81e/pkg/server/grpc_server_test.go#L1626 +package main + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/json" + "fmt" + "log" + "net/http" + "time" + + "gopkg.in/square/go-jose.v2" + "gopkg.in/square/go-jose.v2/jwt" +) + +var ( + signer jose.Signer + jwk jose.JSONWebKey +) + +type config struct { + Issuer string `json:"issuer"` + JWKSURI string `json:"jwks_uri"` +} + +type customClaims struct { + Email string `json:"email"` + EmailVerified bool `json:"email_verified"` +} + +func init() { + pk, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + log.Fatal(err) + } + jwk = jose.JSONWebKey{ + Algorithm: string(jose.RS256), + Key: pk, + } + signer, err = jose.NewSigner(jose.SigningKey{ + Algorithm: jose.RS256, + Key: jwk.Key, + }, nil) + if err != nil { + log.Fatal(err) + } +} + +func token(w http.ResponseWriter, r *http.Request) { + log.Print("handling token") + token, err := jwt.Signed(signer).Claims(jwt.Claims{ + Issuer: fmt.Sprintf("http://%s", r.Host), + IssuedAt: jwt.NewNumericDate(time.Now()), + Expiry: jwt.NewNumericDate(time.Now().Add(30 * time.Minute)), + Subject: "foo@bar.com", + Audience: jwt.Audience{"sigstore"}, + }).Claims(customClaims{ + Email: "foo@bar.com", + EmailVerified: true, + }).CompactSerialize() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + w.Write([]byte(token)) +} + +func keys(w http.ResponseWriter, r *http.Request) { + log.Print("handling keys") + keys, err := json.Marshal(jose.JSONWebKeySet{ + Keys: []jose.JSONWebKey{ + jwk.Public(), + }, + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + w.Header().Add("Content-type", "application/json") + w.Write(keys) +} + +func wellKnown(w http.ResponseWriter, r *http.Request) { + log.Print("handling discovery") + issuer := fmt.Sprintf("http://%s", r.Host) + cfg, err := json.Marshal(config{ + Issuer: issuer, + JWKSURI: issuer + "/keys", + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + w.Header().Add("Content-type", "application/json") + w.Write(cfg) +} + +func main() { + http.HandleFunc("/token", token) + http.HandleFunc("/keys", keys) + http.HandleFunc("/.well-known/openid-configuration", wellKnown) + if err := http.ListenAndServe(":8080", nil); err != nil { + log.Fatal(err) + } +} diff --git a/test/testdata/fancy_from.Dockerfile b/test/testdata/fancy_from.Dockerfile deleted file mode 100644 index 7025bf411207..000000000000 --- a/test/testdata/fancy_from.Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2021 The Sigstore Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM --platform=linux/amd64 ghcr.io/distroless/alpine-base AS base - -# blah blah diff --git a/test/testdata/signed_manifest.yaml b/test/testdata/signed_manifest.yaml deleted file mode 100644 index d90b17ad3c16..000000000000 --- a/test/testdata/signed_manifest.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 The Sigstore Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: single-pod -spec: - restartPolicy: Never - containers: - - name: distroless - image: ghcr.io/distroless/alpine-base diff --git a/test/testdata/single_stage.Dockerfile b/test/testdata/single_stage.Dockerfile deleted file mode 100644 index 9206f2cee56f..000000000000 --- a/test/testdata/single_stage.Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2021 The Sigstore Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ghcr.io/distroless/alpine-base - -# blah blah diff --git a/test/testdata/unsigned_build_stage.Dockerfile b/test/testdata/unsigned_build_stage.Dockerfile deleted file mode 100644 index 1e04890e1989..000000000000 --- a/test/testdata/unsigned_build_stage.Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2021 The Sigstore Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM ghcr.io/distroless/alpine-base - -# blah blah - -# an un(co)signed image -FROM ubuntu - -# blah blah - -FROM ghcr.io/distroless/static diff --git a/test/testdata/unsigned_manifest.yaml b/test/testdata/unsigned_manifest.yaml deleted file mode 100644 index c0a5ac049a22..000000000000 --- a/test/testdata/unsigned_manifest.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 The Sigstore Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: Pod -metadata: - name: single-pod -spec: - restartPolicy: Never - containers: - - name: nginx-container - image: nginx \ No newline at end of file diff --git a/test/testdata/with_arg.Dockerfile b/test/testdata/with_arg.Dockerfile deleted file mode 100644 index 580333082500..000000000000 --- a/test/testdata/with_arg.Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2021 The Sigstore Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -ARG test_image - -FROM ${test_image} \ No newline at end of file diff --git a/test/testdata/with_lowercase.Dockerfile b/test/testdata/with_lowercase.Dockerfile deleted file mode 100644 index d1df114231a2..000000000000 --- a/test/testdata/with_lowercase.Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2021 The Sigstore Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ghcr.io/distroless/alpine-base