Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: Use Pebble as the ACME server in integration tests #339

Merged
merged 17 commits into from
Nov 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ require (
github.com/gardener/gardener v1.107.0
github.com/go-acme/lego/v4 v4.19.2
github.com/go-logr/logr v1.4.2
github.com/letsencrypt/pebble/v2 v2.6.0
github.com/miekg/dns v1.1.62
github.com/onsi/ginkgo/v2 v2.21.0
github.com/onsi/gomega v1.35.0
Expand Down Expand Up @@ -89,6 +90,7 @@ require (
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 // indirect
github.com/letsencrypt/challtestsrv v1.3.2 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
Expand Down
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,10 @@ github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyP
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/letsencrypt/challtestsrv v1.3.2 h1:pIDLBCLXR3B1DLmOmkkqg29qVa7DDozBnsOpL9PxmAY=
github.com/letsencrypt/challtestsrv v1.3.2/go.mod h1:Ur4e4FvELUXLGhkMztHOsPIsvGxD/kzSJninOrkM+zc=
github.com/letsencrypt/pebble/v2 v2.6.0 h1:7xetaJ4YaesUnWWeRGSs3UHOwyfX4I4sfOfDrkvnhNw=
github.com/letsencrypt/pebble/v2 v2.6.0/go.mod h1:SID2E75Cx6sQ9AXFkdzhLdQ6S1zhRUbw08Cgu7GJLSk=
github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
Expand All @@ -284,6 +288,7 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
Expand Down Expand Up @@ -483,6 +488,7 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
Expand All @@ -504,6 +510,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
36 changes: 30 additions & 6 deletions test/integration/controller/issuer/issuer_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"context"
"encoding/base64"
"fmt"
"io"
"os"
"path/filepath"
"testing"
Expand All @@ -16,7 +17,7 @@ import (
certclient "github.com/gardener/cert-management/pkg/cert/client"
ctrl "github.com/gardener/cert-management/pkg/controller"
_ "github.com/gardener/cert-management/pkg/controller/issuer"

testutils "github.com/gardener/cert-management/test/utils"
"github.com/gardener/controller-manager-library/pkg/controllermanager"
"github.com/gardener/controller-manager-library/pkg/controllermanager/cluster"
"github.com/gardener/controller-manager-library/pkg/controllermanager/controller/mappings"
Expand Down Expand Up @@ -47,19 +48,39 @@ var (
ctx context.Context
log logr.Logger

restConfig *rest.Config
testEnv *envtest.Environment
testClient client.Client
kubeconfigFile string
restConfig *rest.Config
testEnv *envtest.Environment
testClient client.Client
acmeDirectoryAddress string
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

acmeDirectoryAddress has been added. This variable holds the directory address of the local Pebble ACME server that will be used to create Issuer resources in tests.
The test asserts that this variable has been set before the test starts.

kubeconfigFile string

scheme *runtime.Scheme
)

var _ = BeforeSuite(func() {
var (
certificatePath string
pebbleHTTPServer io.Closer
err error
)

logf.SetLogger(logger.MustNewZapLogger(logger.DebugLevel, logger.FormatJSON, zap.WriteTo(GinkgoWriter)))
log = logf.Log.WithName(testID)

By("Start Pebble ACME server")
pebbleHTTPServer, certificatePath, acmeDirectoryAddress, err = testutils.RunPebble(log.WithName("pebble"))
Expect(err).NotTo(HaveOccurred())

// The go-acme/lego library needs to trust the TLS certificate of the Pebble ACME server.
// See: https://github.com/go-acme/lego/blob/f2f5550d3a55ec1118f73346cce7a984b4d530f6/lego/client_config.go#L19-L24
Expect(os.Setenv("LEGO_CA_CERTIFICATES", certificatePath)).To(Succeed())

// Starting the Pebble TLS server is a blocking function call that runs in a separate goroutine.
// As the ACME directory endpoint might not be available immediately, we wait until it is reachable.
Eventually(func() error {
return testutils.CheckPebbleAvailability(certificatePath, acmeDirectoryAddress)
}).Should(Succeed())

By("Start test environment")
testEnv = &envtest.Environment{
CRDInstallOptions: envtest.CRDInstallOptions{
Expand All @@ -73,7 +94,6 @@ var _ = BeforeSuite(func() {
ErrorIfCRDPathMissing: true,
}

var err error
restConfig, err = testEnv.Start()
Expect(err).NotTo(HaveOccurred())
Expect(restConfig).NotTo(BeNil())
Expand All @@ -86,7 +106,11 @@ var _ = BeforeSuite(func() {
DeferCleanup(func() {
By("Stop test environment")
Expect(testEnv.Stop()).To(Succeed())
_ = os.RemoveAll(filepath.Dir(certificatePath))
Copy link
Member Author

@marc1404 marc1404 Nov 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The code responsible for starting the Pebble server generates a certificate and private key on the fly in a temporary OS directory.
Instead of deleting the two files and their directory individually, we can clear them in one sweep.

_ = os.Remove(kubeconfigFile)
if pebbleHTTPServer != nil {
_ = pebbleHTTPServer.Close()
}
})

By("Create test client")
Expand Down
4 changes: 3 additions & 1 deletion test/integration/controller/issuer/issuer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ var _ = Describe("Issuer controller tests", func() {
)

BeforeEach(func() {
Expect(acmeDirectoryAddress).NotTo(BeEmpty())
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

During development, I messed up initializing the package level variable in the test suite. Tests should assert early on that the variable has been set.


ctxLocal := context.Background()
ctx0 := ctxutil.CancelContext(ctxutil.WaitGroupContext(context.Background(), "main"))
ctx = ctxutil.TickContext(ctx0, controllermanager.DeletionActivity)
Expand Down Expand Up @@ -78,7 +80,7 @@ var _ = Describe("Issuer controller tests", func() {
Spec: v1alpha1.IssuerSpec{
ACME: &v1alpha1.ACMESpec{
Email: "foo@somewhere-foo-123456.com",
Server: "https://acme-staging-v02.api.letsencrypt.org/directory",
Server: acmeDirectoryAddress,
AutoRegistration: true,
},
},
Expand Down
218 changes: 218 additions & 0 deletions test/utils/generate_cert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
// Adapted from the Go standard library generate_cert.go
// Source: https://github.com/golang/go/blob/master/src/crypto/tls/generate_cert.go

// Generate a self-signed X.509 certificate for a TLS server. Outputs to
// 'cert.pem' and 'key.pem' and will overwrite existing files.

package testutils

import (
"crypto/ecdsa"
"crypto/ed25519"
"crypto/elliptic"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"log"
"math/big"
"net"
"os"
"path/filepath"
"strings"
"time"
)

type certFlags struct {
// Comma-separated hostnames and IPs to generate a certificate for
host *string

//Creation date formatted as Jan 1 15:04:05 2011
validFrom *string

// Duration that certificate is valid for
validFor *time.Duration

// Whether this cert should be its own Certificate Authority
isCA *bool

// Size of RSA key to generate. Ignored if ecdsaCurve is set
rsaBits *int

// ECDSA curve to use to generate a key. Valid values are P224, P256 (recommended), P384, P521
ecdsaCurve *string

// Generate an Ed25519 key
ed25519Key *bool
}

func generateCert(certFlags certFlags, certPath, keyPath string) error {
setDefaults(&certFlags)

if len(*certFlags.host) == 0 {
log.Fatalf("Missing required --host parameter")
}

var priv any
var err error
switch *certFlags.ecdsaCurve {
case "":
if *certFlags.ed25519Key {
_, priv, err = ed25519.GenerateKey(rand.Reader)
} else {
priv, err = rsa.GenerateKey(rand.Reader, *certFlags.rsaBits)
}
case "P224":
priv, err = ecdsa.GenerateKey(elliptic.P224(), rand.Reader)
case "P256":
priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
case "P384":
priv, err = ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
case "P521":
priv, err = ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
default:
log.Fatalf("Unrecognized elliptic curve: %q", *certFlags.ecdsaCurve)
}
if err != nil {
log.Fatalf("Failed to generate private key: %v", err)
}

// ECDSA, ED25519 and RSA subject keys should have the DigitalSignature
// KeyUsage bits set in the x509.Certificate template
keyUsage := x509.KeyUsageDigitalSignature
// Only RSA subject keys should have the KeyEncipherment KeyUsage bits set. In
// the context of TLS this KeyUsage is particular to RSA key exchange and
// authentication.
if _, isRSA := priv.(*rsa.PrivateKey); isRSA {
keyUsage |= x509.KeyUsageKeyEncipherment
}

var notBefore time.Time
if len(*certFlags.validFrom) == 0 {
notBefore = time.Now()
} else {
notBefore, err = time.Parse("Jan 2 15:04:05 2006", *certFlags.validFrom)
if err != nil {
return fmt.Errorf("failed to parse creation date: %v", err)
}
}

notAfter := notBefore.Add(*certFlags.validFor)

serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
if err != nil {
return fmt.Errorf("failed to generate serial number: %v", err)
}

template := x509.Certificate{
SerialNumber: serialNumber,
Subject: pkix.Name{
Organization: []string{"Acme Co"},
},
NotBefore: notBefore,
NotAfter: notAfter,

KeyUsage: keyUsage,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}

hosts := strings.Split(*certFlags.host, ",")
for _, h := range hosts {
if ip := net.ParseIP(h); ip != nil {
template.IPAddresses = append(template.IPAddresses, ip)
} else {
template.DNSNames = append(template.DNSNames, h)
}
}

if *certFlags.isCA {
template.IsCA = true
template.KeyUsage |= x509.KeyUsageCertSign
}

derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv)
if err != nil {
return fmt.Errorf("failed to create certificate: %v", err)
}

certOut, err := os.Create(filepath.Clean(certPath))
if err != nil {
return fmt.Errorf("failed to open cert.pem for writing: %v", err)
}
if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
return fmt.Errorf("failed to write data to cert.pem: %v", err)
}
if err := certOut.Close(); err != nil {
return fmt.Errorf("error closing cert.pem: %v", err)
}

keyOut, err := os.OpenFile(filepath.Clean(keyPath), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("failed to open key.pem for writing: %v", err)
}
privBytes, err := x509.MarshalPKCS8PrivateKey(priv)
if err != nil {
return fmt.Errorf("unable to marshal private key: %v", err)
}
if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
return fmt.Errorf("failed to write data to key.pem: %v", err)
}
if err := keyOut.Close(); err != nil {
return fmt.Errorf("error closing key.pem: %v", err)
}
return nil
}

func setDefaults(certFlags *certFlags) {
if certFlags.host == nil {
certFlags.host = new(string)
*certFlags.host = ""
}

if certFlags.validFrom == nil {
certFlags.validFrom = new(string)
*certFlags.validFrom = ""
}

if certFlags.validFor == nil {
certFlags.validFor = new(time.Duration)
*certFlags.validFor = 365 * 24 * time.Hour
}

if certFlags.isCA == nil {
certFlags.isCA = new(bool)
*certFlags.isCA = false
}

if certFlags.rsaBits == nil {
certFlags.rsaBits = new(int)
*certFlags.rsaBits = 2048
}

if certFlags.ecdsaCurve == nil {
certFlags.ecdsaCurve = new(string)
*certFlags.ecdsaCurve = ""
}

if certFlags.ed25519Key == nil {
certFlags.ed25519Key = new(bool)
*certFlags.ed25519Key = false
}
}

func publicKey(priv any) any {
switch k := priv.(type) {
case *rsa.PrivateKey:
return &k.PublicKey
case *ecdsa.PrivateKey:
return &k.PublicKey
case ed25519.PrivateKey:
return k.Public().(ed25519.PublicKey)
default:
return nil
}
}
27 changes: 27 additions & 0 deletions test/utils/logbridge.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package testutils

import (
"log"
"strings"

"github.com/go-logr/logr"
)

type logBridge struct {
logr logr.Logger
}

func (logBridge *logBridge) Write(p []byte) (n int, err error) {
message := strings.TrimSpace(string(p))

logBridge.logr.Info(message)

return len(p), nil
}

// NewLogBridge creates a new log.Logger that forwards all log messages to the given logr.Logger.
func NewLogBridge(logr logr.Logger) *log.Logger {
writer := &logBridge{logr}

return log.New(writer, "", 0)
}
Loading