Skip to content

Commit

Permalink
Return CSR to use api v1beta1 (#825)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dvaldivia
dvaldivia authored Sep 12, 2021
1 parent 9a0a414 commit 1533fbd
Show file tree
Hide file tree
Showing 6 changed files with 268 additions and 93 deletions.
215 changes: 141 additions & 74 deletions pkg/controller/cluster/csr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,13 @@ import (
"syscall"
"time"

certificatesV1 "k8s.io/api/certificates/v1"

"k8s.io/klog/v2"

miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"

certificates "k8s.io/api/certificates/v1"
certificatesV1beta1 "k8s.io/api/certificates/v1beta1"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -68,73 +70,118 @@ func isEqual(a, b []string) bool {
}

// createCertificateSigningRequest is equivalent to kubectl create <csr-name> and kubectl approve csr <csr-name>
func (c *Controller) createCertificateSigningRequest(ctx context.Context, labels map[string]string, name, namespace string, csrBytes []byte, owner metav1.Object, usage string) error {
csrSignerName := "kubernetes.io/kubelet-serving"
csrKeyUsage := []certificates.KeyUsage{
certificates.UsageDigitalSignature,
certificates.UsageKeyEncipherment,
certificates.UsageServerAuth,
}
if usage == "client" {
csrSignerName = "kubernetes.io/kube-apiserver-client"
csrKeyUsage = []certificates.KeyUsage{
certificates.UsageDigitalSignature,
certificates.UsageKeyEncipherment,
certificates.UsageClientAuth,
}
}
func (c *Controller) createCertificateSigningRequest(ctx context.Context, labels map[string]string, name, namespace string, csrBytes []byte, usage string) error {
encodedBytes := pem.EncodeToMemory(&pem.Block{Type: csrType, Bytes: csrBytes})
kubeCSR := &certificates.CertificateSigningRequest{
TypeMeta: v1.TypeMeta{
APIVersion: "certificates.k8s.io/v1",
Kind: "CertificateSigningRequest",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Labels: labels,
Namespace: namespace,
OwnerReferences: []metav1.OwnerReference{
*metav1.NewControllerRef(owner, schema.GroupVersionKind{
Group: miniov2.SchemeGroupVersion.Group,
Version: miniov2.SchemeGroupVersion.Version,
Kind: miniov2.MinIOCRDResourceKind,
}),
// for the right set of csr configurations regarding CSR signers and Key usages please read:
// https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/#kubernetes-signers
if useCertificatesV1API {
csrSignerName := certificatesV1.KubeletServingSignerName
csrKeyUsage := []certificatesV1.KeyUsage{
certificatesV1.UsageDigitalSignature,
certificatesV1.UsageKeyEncipherment,
certificatesV1.UsageServerAuth,
}
if usage == "client" {
csrSignerName = certificatesV1.KubeAPIServerClientSignerName
csrKeyUsage = []certificatesV1.KeyUsage{
certificatesV1.UsageDigitalSignature,
certificatesV1.UsageKeyEncipherment,
certificatesV1.UsageClientAuth,
}
}
kubeCSR := &certificatesV1.CertificateSigningRequest{
TypeMeta: v1.TypeMeta{
APIVersion: "certificates.k8s.io/v1",
Kind: "CertificateSigningRequest",
},
},
Spec: certificates.CertificateSigningRequestSpec{
SignerName: csrSignerName,
Request: encodedBytes,
Groups: []string{"system:authenticated", "system:nodes"},
Usages: csrKeyUsage,
},
}
ObjectMeta: v1.ObjectMeta{
Name: name,
Labels: labels,
Namespace: namespace,
},
Spec: certificatesV1.CertificateSigningRequestSpec{
SignerName: csrSignerName,
Request: encodedBytes,
Groups: []string{"system:authenticated", "system:nodes"},
Usages: csrKeyUsage,
},
}
ks, err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Create(ctx, kubeCSR, metav1.CreateOptions{})
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
// Return if certificate already exists.
if k8serrors.IsAlreadyExists(err) {
return nil
}
// Update the CSR to be approved automatically
ks.Status = certificatesV1.CertificateSigningRequestStatus{
Conditions: []certificatesV1.CertificateSigningRequestCondition{
{
Type: certificatesV1.CertificateApproved,
Reason: "MinIOOperatorAutoApproval",
Message: "Automatically approved by MinIO Operator",
LastUpdateTime: metav1.NewTime(time.Now()),
Status: "True",
},
},
}
_, err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, name, ks, metav1.UpdateOptions{})
if err != nil {
return err
}
} else {
csrSignerName := certificatesV1beta1.LegacyUnknownSignerName
csrKeyUsage := []certificatesV1beta1.KeyUsage{
certificatesV1beta1.UsageDigitalSignature,
certificatesV1beta1.UsageKeyEncipherment,
certificatesV1beta1.UsageServerAuth,
certificatesV1beta1.UsageClientAuth,
}
kubeCSR := &certificatesV1beta1.CertificateSigningRequest{
TypeMeta: v1.TypeMeta{
APIVersion: "certificates.k8s.io/v1beta1",
Kind: "CertificateSigningRequest",
},
ObjectMeta: v1.ObjectMeta{
Name: name,
Labels: labels,
Namespace: namespace,
},
Spec: certificatesV1beta1.CertificateSigningRequestSpec{
SignerName: &csrSignerName,
Request: encodedBytes,
Usages: csrKeyUsage,
},
}

ks, err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Create(ctx, kubeCSR, metav1.CreateOptions{})
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}
ks, err := c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Create(ctx, kubeCSR, metav1.CreateOptions{})
if err != nil && !k8serrors.IsAlreadyExists(err) {
return err
}

// Return if certificate already exists.
if k8serrors.IsAlreadyExists(err) {
return nil
}
// Return if certificate already exists.
if k8serrors.IsAlreadyExists(err) {
return nil
}

// Update the CSR to be approved automatically
ks.Status = certificates.CertificateSigningRequestStatus{
Conditions: []certificates.CertificateSigningRequestCondition{
{
Type: certificates.CertificateApproved,
Reason: "MinIOOperatorAutoApproval",
Message: "Automatically approved by MinIO Operator",
LastUpdateTime: metav1.NewTime(time.Now()),
Status: "True",
// Update the CSR to be approved automatically
ks.Status = certificatesV1beta1.CertificateSigningRequestStatus{
Conditions: []certificatesV1beta1.CertificateSigningRequestCondition{
{
Type: certificatesV1beta1.CertificateApproved,
Reason: "MinIOOperatorAutoApproval",
Message: "Automatically approved by MinIO Operator",
LastUpdateTime: metav1.NewTime(time.Now()),
Status: "True",
},
},
},
}
}

_, err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, name, ks, metav1.UpdateOptions{})
if err != nil {
return err
_, err = c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().UpdateApproval(ctx, ks, metav1.UpdateOptions{})
if err != nil {
return err
}
}

return nil
Expand All @@ -161,22 +208,42 @@ func (c *Controller) fetchCertificate(ctx context.Context, csrName string) ([]by
return nil, fmt.Errorf("%s", s.String())

case <-tick.C:
r, err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, v1.GetOptions{})
if err != nil {
klog.Errorf("Unexpected error during certificate fetching of csr/%s: %s", csrName, err)
return nil, err
}
if r.Status.Certificate != nil {
klog.V(0).Infof("Certificate successfully fetched, creating secret with Private key and Certificate")
return r.Status.Certificate, nil
}
for _, c := range r.Status.Conditions {
if c.Type == certificates.CertificateDenied {
err := fmt.Errorf("csr/%s uid: %s is %q: %s", r.Name, r.UID, c.Type, c.String())
klog.Errorf("Unexpected error during fetch: %v", err)
if useCertificatesV1API {
r, err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, v1.GetOptions{})
if err != nil {
klog.Errorf("Unexpected error during certificate fetching of csr/%s V1: %s", csrName, err)
return nil, err
}
if r.Status.Certificate != nil {
klog.V(0).Infof("Certificate successfully fetched, creating secret with Private key and Certificate")
return r.Status.Certificate, nil
}
for _, c := range r.Status.Conditions {
if c.Type == certificatesV1.CertificateDenied {
err := fmt.Errorf("csr/%s V1 uid: %s is %q: %s", r.Name, r.UID, c.Type, c.String())
klog.Errorf("Unexpected error during fetch: %v", err)
return nil, err
}
}
} else {
r, err := c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, csrName, v1.GetOptions{})
if err != nil {
klog.Errorf("Unexpected error during certificate fetching of csr/%s V1beta1: %s", csrName, err)
return nil, err
}
if r.Status.Certificate != nil {
klog.V(0).Infof("Certificate successfully fetched, creating secret with Private key and Certificate")
return r.Status.Certificate, nil
}
for _, c := range r.Status.Conditions {
if c.Type == certificatesV1beta1.CertificateDenied {
err := fmt.Errorf("csr/%s V1beta1 uid: %s is %q: %s", r.Name, r.UID, c.Type, c.String())
klog.Errorf("Unexpected error during fetch: %v", err)
return nil, err
}
}
}

klog.V(1).Infof("Certificate of csr/%s still not available, next try in %d", csrName, miniov2.DefaultQueryInterval)

case <-timeout.C:
Expand Down
38 changes: 31 additions & 7 deletions pkg/controller/cluster/kes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func (c *Controller) createKESCSR(ctx context.Context, tenant *miniov2.Tenant) e
return err
}

err = c.createCertificateSigningRequest(ctx, tenant.KESPodLabels(), tenant.KESCSRName(), tenant.Namespace, csrBytes, tenant, "server")
err = c.createCertificateSigningRequest(ctx, tenant.KESPodLabels(), tenant.KESCSRName(), tenant.Namespace, csrBytes, "server")
if err != nil {
klog.Errorf("Unexpected error during the creation of the csr/%s: %v", tenant.KESCSRName(), err)
return err
Expand Down Expand Up @@ -154,8 +154,14 @@ func (c *Controller) checkKESCertificatesStatus(ctx context.Context, tenant *min
return err
}
// TLS secret not found, delete CSR if exists and start certificate generation process again
if err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Delete(ctx, tenant.MinIOClientCSRName(), metav1.DeleteOptions{}); err != nil {
return err
if useCertificatesV1API {
if err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Delete(ctx, tenant.MinIOClientCSRName(), metav1.DeleteOptions{}); err != nil {
return err
}
} else {
if err = c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, tenant.MinIOClientCSRName(), metav1.DeleteOptions{}); err != nil {
return err
}
}
}
}
Expand All @@ -169,8 +175,14 @@ func (c *Controller) checkKESCertificatesStatus(ctx context.Context, tenant *min
return err
}
// TLS secret not found, delete CSR if exists and start certificate generation process again
if err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Delete(ctx, tenant.KESCSRName(), metav1.DeleteOptions{}); err != nil {
return err
if useCertificatesV1API {
if err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Delete(ctx, tenant.KESCSRName(), metav1.DeleteOptions{}); err != nil {
return err
}
} else {
if err = c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, tenant.KESCSRName(), metav1.DeleteOptions{}); err != nil {
return err
}
}
}
}
Expand Down Expand Up @@ -265,7 +277,13 @@ func (c *Controller) checkKESStatus(ctx context.Context, tenant *miniov2.Tenant,
}

func (c *Controller) checkAndCreateMinIOClientCSR(ctx context.Context, nsName types.NamespacedName, tenant *miniov2.Tenant) error {
if _, err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, tenant.MinIOClientCSRName(), metav1.GetOptions{}); err != nil {
var err error
if useCertificatesV1API {
_, err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, tenant.MinIOClientCSRName(), metav1.GetOptions{})
} else {
_, err = c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, tenant.MinIOClientCSRName(), metav1.GetOptions{})
}
if err != nil {
if k8serrors.IsNotFound(err) {
if tenant, err = c.updateTenantStatus(ctx, tenant, StatusWaitingMinIOClientCert, 0); err != nil {
return err
Expand All @@ -284,7 +302,13 @@ func (c *Controller) checkAndCreateMinIOClientCSR(ctx context.Context, nsName ty
}

func (c *Controller) checkAndCreateKESCSR(ctx context.Context, nsName types.NamespacedName, tenant *miniov2.Tenant) error {
if _, err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, tenant.KESCSRName(), metav1.GetOptions{}); err != nil {
var err error
if useCertificatesV1API {
_, err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, tenant.KESCSRName(), metav1.GetOptions{})
} else {
_, err = c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, tenant.KESCSRName(), metav1.GetOptions{})
}
if err != nil {
if k8serrors.IsNotFound(err) {
if tenant, err = c.updateTenantStatus(ctx, tenant, StatusWaitingKESCert, 0); err != nil {
return err
Expand Down
17 changes: 14 additions & 3 deletions pkg/controller/cluster/main-controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,10 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error {
apiWillStart := make(chan interface{})

go func() {

// Request kubernetes version from Kube ApiServer
c.getKubeAPIServerVersion()

if isOperatorTLS() {
publicCertPath, publicKeyPath := c.generateTLSCert()
klog.Infof("Starting HTTPS api server")
Expand Down Expand Up @@ -539,9 +543,16 @@ func (c *Controller) syncHandler(key string) error {
if tenant.Spec.RequestAutoCert == nil && tenant.APIVersion != "" {
// If we get certificate signing requests for MinIO is safe to assume the Tenant v1 was deployed using AutoCert
// otherwise AutoCert will be false
tenantCSR, err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, tenant.MinIOCSRName(), metav1.GetOptions{})
if err != nil || tenantCSR == nil {
autoCertEnabled = false
if useCertificatesV1API {
tenantCSR, err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, tenant.MinIOCSRName(), metav1.GetOptions{})
if err != nil || tenantCSR == nil {
autoCertEnabled = false
}
} else {
tenantCSR, err := c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, tenant.MinIOCSRName(), metav1.GetOptions{})
if err != nil || tenantCSR == nil {
autoCertEnabled = false
}
}
} else {
autoCertEnabled = tenant.AutoCert()
Expand Down
Loading

0 comments on commit 1533fbd

Please sign in to comment.