Skip to content

Commit

Permalink
Bugfix reload CA cert in operator-ca-tls secret (#1716)
Browse files Browse the repository at this point in the history 
* Bugfix reload CA cert in secret:
When the CA certificate in the secret `operator-ca-tls` changes, Operator recreates this secret on the Tenants namespace, but  is not using the newly provided cert for healthcheck, this will force reload the certificates on secret update

Signed-off-by: pjuarezd <pjuarezd@users.noreply.github.com>
  • Loading branch information
@pjuarezd
pjuarezd authored Aug 14, 2023
1 parent 1557ba0 commit af02d1b
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 42 deletions.
9 changes: 9 additions & 0 deletions pkg/common/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,15 @@ const (
OperatorRuntimeOpenshift Runtime = "OPENSHIFT"
// OperatorRuntimeRancher is the Rancher runtime flag
OperatorRuntimeRancher Runtime = "RANCHER"

// TLSCRT is name of the field containing tls certificate in secret
TLSCRT = "tls.crt"

// CACRT name of the field containing ca certificate in secret
CACRT = "ca.crt"

// PublicCRT name of the field containing public certificate in secret
PublicCRT = "public.crt"
)

// Runtimes is a map of the supported Kubernetes runtimes
Expand Down
65 changes: 49 additions & 16 deletions pkg/controller/minio.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@ import (
"fmt"
"math/big"
"net"
"slices"
"time"

"github.com/minio/operator/pkg/common"

"k8s.io/apimachinery/pkg/runtime/schema"

"github.com/minio/operator/pkg/controller/certificates"
Expand Down Expand Up @@ -110,23 +113,25 @@ func (c *Controller) getTLSSecret(ctx context.Context, nsName string, secretName
return c.kubeClientSet.CoreV1().Secrets(nsName).Get(ctx, secretName, metav1.GetOptions{})
}

func getOperatorCACert(secretData map[string][]byte) ([]byte, error) {
for _, key := range []string{
"tls.crt",
"ca.crt",
"public.crt",
} {
func getOperatorCertFromSecret(secretData map[string][]byte, key string) ([]byte, error) {
keys := []string{
common.TLSCRT,
common.CACRT,
common.PublicCRT,
}
if slices.Contains(keys, key) {
data, ok := secretData[key]
if ok {
return data, nil
}
}
return nil, fmt.Errorf("missing 'public.crt' in %s/%s secret", miniov2.GetNSFromFile(), OperatorCATLSSecretName)
return nil, fmt.Errorf("missing '%s' in %s/%s secret", key, miniov2.GetNSFromFile(), OperatorCATLSSecretName)
}

// checkOperatorCaForTenant create or updates the operator-ca-tls secret for tenant if need it
func (c *Controller) checkOperatorCaForTenant(ctx context.Context, tenant *miniov2.Tenant) (operatorCATLSExists bool, err error) {
var tenantCaCert []byte
var certsData map[string][]byte

// get operator-ca-tls in minio-operator namespace
operatorCaSecret, err := c.kubeClientSet.CoreV1().Secrets(miniov2.GetNSFromFile()).Get(ctx, OperatorCATLSSecretName, metav1.GetOptions{})
if err != nil {
Expand All @@ -137,11 +142,24 @@ func (c *Controller) checkOperatorCaForTenant(ctx context.Context, tenant *minio
return false, err
}

operatorCaCert, err := getOperatorCACert(operatorCaSecret.Data)
operatorPublicCert, err := getOperatorCertFromSecret(operatorCaSecret.Data, common.PublicCRT)
if err != nil {
// If no public.crt is present we error, other certs are optional
return false, err
}

certsData[common.PublicCRT] = operatorPublicCert

operatorTLSCert, err := getOperatorCertFromSecret(operatorCaSecret.Data, common.TLSCRT)
if err == nil {
certsData[common.TLSCRT] = operatorTLSCert
}

operatorCACert, err := getOperatorCertFromSecret(operatorCaSecret.Data, common.CACRT)
if err == nil {
certsData[common.CACRT] = operatorCACert
}

var tenantCaSecret *corev1.Secret

createTenantCASecret := func() error {
Expand All @@ -160,9 +178,7 @@ func (c *Controller) checkOperatorCaForTenant(ctx context.Context, tenant *minio
}),
},
},
Data: map[string][]byte{
"public.crt": operatorCaCert,
},
Data: certsData,
}
_, err = c.kubeClientSet.CoreV1().Secrets(tenant.Namespace).Create(ctx, tenantCaSecret, metav1.CreateOptions{})
return err
Expand All @@ -179,14 +195,31 @@ func (c *Controller) checkOperatorCaForTenant(ctx context.Context, tenant *minio
}
}

tenantCaCert = tenantCaSecret.Data["public.crt"]
if !bytes.Equal(tenantCaCert, operatorCaCert) {
tenantCaSecret.Data["public.crt"] = operatorCaCert
update := false

if publicCert, ok := tenantCaSecret.Data[common.PublicCRT]; ok && !bytes.Equal(publicCert, operatorPublicCert) {
tenantCaSecret.Data[common.PublicCRT] = operatorPublicCert
update = true
}

if tlsCert, ok := tenantCaSecret.Data[common.TLSCRT]; ok && !bytes.Equal(tlsCert, operatorTLSCert) {
tenantCaSecret.Data[common.TLSCRT] = tlsCert
update = true
}

if caCert, ok := tenantCaSecret.Data[common.CACRT]; ok && !bytes.Equal(caCert, operatorCACert) {
tenantCaSecret.Data[common.CACRT] = caCert
update = true
}

if update {
_, err = c.kubeClientSet.CoreV1().Secrets(tenant.Namespace).Update(ctx, tenantCaSecret, metav1.UpdateOptions{})
if err != nil {
return false, err
}
return false, fmt.Errorf("'public.crt' in '%s/%s' secret changed, updating '%s/%s' secret", miniov2.GetNSFromFile(), OperatorCATLSSecretName, tenant.Namespace, OperatorCATLSSecretName)
// Reload certificates
c.createTransport()
return false, fmt.Errorf("'%s/%s' secret changed, updating '%s/%s' secret", miniov2.GetNSFromFile(), OperatorCATLSSecretName, tenant.Namespace, OperatorCATLSSecretName)
}

return true, nil
Expand Down
67 changes: 41 additions & 26 deletions pkg/controller/operator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package controller
import (
"context"
"crypto/tls"
"crypto/x509"
"net"
"net/http"
"time"
Expand Down Expand Up @@ -89,10 +90,49 @@ func (c *Controller) fetchUserCredentials(ctx context.Context, tenant *miniov2.T
return userCredentials
}

// getTransport returns a *http.Transport with the collection of the trusted CA certificates
// returns a cached transport if already available
func (c *Controller) getTransport() *http.Transport {
if c.transport != nil {
return c.transport
}
c.transport = c.createTransport()
return c.transport
}

// createTransport returns a *http.Transport with the collection of the trusted CA certificates
func (c *Controller) createTransport() *http.Transport {
rootCAs := c.fetchTransportCACertificates()
dialer := &net.Dialer{
Timeout: 15 * time.Second,
KeepAlive: 15 * time.Second,
}
c.transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: dialer.DialContext,
MaxIdleConnsPerHost: 1024,
IdleConnTimeout: 15 * time.Second,
ResponseHeaderTimeout: 15 * time.Minute,
TLSHandshakeTimeout: 15 * time.Second,
ExpectContinueTimeout: 15 * time.Second,
// Go net/http automatically unzip if content-type is
// gzip disable this feature, as we are always interested
// in raw stream.
DisableCompression: true,
TLSClientConfig: &tls.Config{
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
RootCAs: rootCAs,
},
}

return c.transport
}

// fetchTransportCACertificates retrieves a *x509.CertPool with all CA that operator will trust
func (c *Controller) fetchTransportCACertificates() (pool *x509.CertPool) {
rootCAs := miniov2.MustGetSystemCertPool()
// Default kubernetes CA certificate
rootCAs.AppendCertsFromPEM(miniov2.GetPodCAFromFile())
Expand Down Expand Up @@ -140,32 +180,7 @@ func (c *Controller) getTransport() *http.Transport {
}
}
}
dialer := &net.Dialer{
Timeout: 15 * time.Second,
KeepAlive: 15 * time.Second,
}
c.transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: dialer.DialContext,
MaxIdleConnsPerHost: 1024,
IdleConnTimeout: 15 * time.Second,
ResponseHeaderTimeout: 15 * time.Minute,
TLSHandshakeTimeout: 15 * time.Second,
ExpectContinueTimeout: 15 * time.Second,
// Go net/http automatically unzip if content-type is
// gzip disable this feature, as we are always interested
// in raw stream.
DisableCompression: true,
TLSClientConfig: &tls.Config{
// Can't use SSLv3 because of POODLE and BEAST
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
// Can't use TLSv1.1 because of RC4 cipher usage
MinVersion: tls.VersionTLS12,
RootCAs: rootCAs,
},
}

return c.transport
return rootCAs
}

func (c *Controller) createUsers(ctx context.Context, tenant *miniov2.Tenant, tenantConfiguration map[string][]byte) (err error) {
Expand Down

0 comments on commit af02d1b

Please sign in to comment.