Added renewal of deployment TLS CA certificate

pkg/apis/deployment/v1alpha/plan.go

@@ -47,6 +47,8 @@ const (
  ActionTypeWaitForMemberUp ActionType = "WaitForMemberUp"
  // ActionTypeRenewTLSCertificate causes the TLS certificate of a member to be renewed.
  ActionTypeRenewTLSCertificate ActionType = "RenewTLSCertificate"
+ // ActionTypeRenewTLSCACertificate causes the TLS CA certificate of the entire deployment to be renewed.
+ ActionTypeRenewTLSCACertificate ActionType = "RenewTLSCACertificate"
 )
 
 // Action represents a single action to be taken to update a deployment.

pkg/deployment/access_package.go

@@ -108,7 +108,7 @@ func (d *Deployment) ensureAccessPackage(apSecretName string) error {
 
  // Fetch client authentication CA
  clientAuthSecretName := spec.Sync.Authentication.GetClientCASecretName()
- clientAuthCert, clientAuthKey, err := k8sutil.GetCASecret(d.deps.KubeCli.CoreV1(), clientAuthSecretName, ns)
+ clientAuthCert, clientAuthKey, _, err := k8sutil.GetCASecret(d.deps.KubeCli.CoreV1(), clientAuthSecretName, ns, nil)
  if err != nil {
  log.Debug().Err(err).Msg("Failed to get client-auth CA secret")
  return maskAny(err)

pkg/deployment/context_impl.go

@@ -237,6 +237,25 @@ func (d *Deployment) CleanupPod(p v1.Pod) error {
  return nil
 }
 
+// RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace
+// of the deployment. If the pod does not exist, the error is ignored.
+func (d *Deployment) RemovePodFinalizers(podName string) error {
+ log := d.deps.Log
+ ns := d.GetNamespace()
+ kubecli := d.deps.KubeCli
+ p, err := kubecli.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
+ if err != nil {
+ if k8sutil.IsNotFound(err) {
+ return nil
+ }
+ return maskAny(err)
+ }
+ if err := k8sutil.RemovePodFinalizers(log, d.deps.KubeCli, p, p.GetFinalizers(), true); err != nil {
+ return maskAny(err)
+ }
+ return nil
+}
+
 // DeletePvc deletes a persistent volume claim with given name in the namespace
 // of the deployment. If the pvc does not exist, the error is ignored.
 func (d *Deployment) DeletePvc(pvcName string) error {
@@ -307,3 +326,26 @@ func (d *Deployment) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberSt
  }
  return nil
 }
+
+// GetTLSCA returns the TLS CA certificate in the secret with given name.
+// Returns: publicKey, privateKey, ownerByDeployment, error
+func (d *Deployment) GetTLSCA(secretName string) (string, string, bool, error) {
+ ns := d.apiObject.GetNamespace()
+ owner := d.apiObject.AsOwner()
+ cert, priv, isOwned, err := k8sutil.GetCASecret(d.deps.KubeCli.CoreV1(), secretName, ns, &owner)
+ if err != nil {
+ return "", "", false, maskAny(err)
+ }
+ return cert, priv, isOwned, nil
+
+}
+
+// DeleteSecret removes the Secret with given name.
+// If the secret does not exist, the error is ignored.
+func (d *Deployment) DeleteSecret(secretName string) error {
+ ns := d.apiObject.GetNamespace()
+ if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) {
+ return maskAny(err)
+ }
+ return nil
+}

pkg/deployment/reconcile/action_context.go

@@ -70,9 +70,14 @@ type ActionContext interface {
  // DeletePvc deletes a persistent volume claim with given name in the namespace
  // of the deployment. If the pvc does not exist, the error is ignored.
  DeletePvc(pvcName string) error
+ // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace
+ // of the deployment. If the pod does not exist, the error is ignored.
+ RemovePodFinalizers(podName string) error
  // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member.
  // If the secret does not exist, the error is ignored.
  DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error
+ // DeleteTLSCASecret removes the Secret containing the TLS CA certificate.
+ DeleteTLSCASecret() error
 }
 
 // newActionContext creates a new ActionContext implementation.
@@ -212,6 +217,15 @@ func (ac *actionContext) DeletePvc(pvcName string) error {
  return nil
 }
 
+// RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace
+// of the deployment. If the pod does not exist, the error is ignored.
+func (ac *actionContext) RemovePodFinalizers(podName string) error {
+ if err := ac.context.RemovePodFinalizers(podName); err != nil {
+ return maskAny(err)
+ }
+ return nil
+}
+
 // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member.
 // If the secret does not exist, the error is ignored.
 func (ac *actionContext) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error {
@@ -220,3 +234,28 @@ func (ac *actionContext) DeleteTLSKeyfile(group api.ServerGroup, member api.Memb
  }
  return nil
 }
+
+// DeleteTLSCASecret removes the Secret containing the TLS CA certificate.
+func (ac *actionContext) DeleteTLSCASecret() error {
+ spec := ac.context.GetSpec().TLS
+ if !spec.IsSecure() {
+ return nil
+ }
+ secretName := spec.GetCASecretName()
+ if secretName == "" {
+ return nil
+ }
+ // Remove secret hash, since it is going to change
+ status, lastVersion := ac.context.GetStatus()
+ if status.SecretHashes != nil {
+ status.SecretHashes.TLSCA = ""
+ if err := ac.context.UpdateStatus(status, lastVersion); err != nil {
+ return maskAny(err)
+ }
+ }
+ // Do delete the secret
+ if err := ac.context.DeleteSecret(secretName); err != nil {
+ return maskAny(err)
+ }
+ return nil
+}

pkg/deployment/reconcile/action_renew_tls_ca_certificate.go

@@ -0,0 +1,71 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package reconcile
+
+import (
+ "context"
+ "time"
+
+ api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
+ "github.com/rs/zerolog"
+)
+
+// NewRenewTLSCACertificateAction creates a new Action that implements the given
+// planned RenewTLSCACertificate action.
+func NewRenewTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action {
+ return &renewTLSCACertificateAction{
+ log: log,
+ action: action,
+ actionCtx: actionCtx,
+ }
+}
+
+// renewTLSCACertificateAction implements a RenewTLSCACertificate action.
+type renewTLSCACertificateAction struct {
+ log zerolog.Logger
+ action api.Action
+ actionCtx ActionContext
+}
+
+// Start performs the start of the action.
+// Returns true if the action is completely finished, false in case
+// the start time needs to be recorded and a ready condition needs to be checked.
+func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) {
+ // Just delete the secret.
+ // It will be re-created.
+ if err := a.actionCtx.DeleteTLSCASecret(); err != nil {
+ return false, maskAny(err)
+ }
+ return true, nil
+}
+
+// CheckProgress checks the progress of the action.
+// Returns true if the action is completely finished, false otherwise.
+func (a *renewTLSCACertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) {
+ return true, false, nil
+}
+
+// Timeout returns the amount of time after which this action will timeout.
+func (a *renewTLSCACertificateAction) Timeout() time.Duration {
+ return renewTLSCACertificateTimeout
+}

pkg/deployment/reconcile/action_rotate_member.go

@@ -57,6 +57,10 @@ func (a *actionRotateMember) Start(ctx context.Context) (bool, error) {
  if !ok {
  log.Error().Msg("No such member")
  }
+ // Remove finalizers, so Kubernetes will quickly terminate the pod
+ if err := a.actionCtx.RemovePodFinalizers(m.PodName); err != nil {
+ return false, maskAny(err)
+ }
  if group.IsArangod() {
  // Invoke shutdown endpoint
  c, err := a.actionCtx.GetServerClient(ctx, group, a.action.MemberID)

pkg/deployment/reconcile/context.go

@@ -69,6 +69,9 @@ type Context interface {
  // DeletePvc deletes a persistent volume claim with given name in the namespace
  // of the deployment. If the pvc does not exist, the error is ignored.
  DeletePvc(pvcName string) error
+ // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace
+ // of the deployment. If the pod does not exist, the error is ignored.
+ RemovePodFinalizers(podName string) error
  // GetOwnedPods returns a list of all pods owned by the deployment.
  GetOwnedPods() ([]v1.Pod, error)
  // GetTLSKeyfile returns the keyfile encoded TLS certificate+key for
@@ -77,4 +80,10 @@ type Context interface {
  // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member.
  // If the secret does not exist, the error is ignored.
  DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error
+ // GetTLSCA returns the TLS CA certificate in the secret with given name.
+ // Returns: publicKey, privateKey, ownerByDeployment, error
+ GetTLSCA(secretName string) (string, string, bool, error)
+ // DeleteSecret removes the Secret with given name.
+ // If the secret does not exist, the error is ignored.
+ DeleteSecret(secretName string) error
 }

pkg/deployment/reconcile/plan_builder.go

@@ -23,10 +23,6 @@
 package reconcile
 
 import (
- "crypto/x509"
- "encoding/pem"
- "time"
-
  "github.com/rs/zerolog"
  "github.com/rs/zerolog/log"
  "k8s.io/api/core/v1"
@@ -58,7 +54,7 @@ func (d *Reconciler) CreatePlan() error {
  apiObject := d.context.GetAPIObject()
  spec := d.context.GetSpec()
  status, lastVersion := d.context.GetStatus()
- newPlan, changed := createPlan(d.log, apiObject, status.Plan, spec, status, pods, d.context.GetTLSKeyfile)
+ newPlan, changed := createPlan(d.log, apiObject, status.Plan, spec, status, pods, d.context.GetTLSKeyfile, d.context.GetTLSCA)
 
  // If not change, we're done
  if !changed {
@@ -83,7 +79,8 @@ func (d *Reconciler) CreatePlan() error {
 func createPlan(log zerolog.Logger, apiObject metav1.Object,
  currentPlan api.Plan, spec api.DeploymentSpec,
  status api.DeploymentStatus, pods []v1.Pod,
- getTLSKeyfile func(group api.ServerGroup, member api.MemberStatus) (string, error)) (api.Plan, bool) {
+ getTLSKeyfile func(group api.ServerGroup, member api.MemberStatus) (string, error),
+ getTLSCA func(string) (string, string, bool, error)) (api.Plan, bool) {
  if len(currentPlan) > 0 {
  // Plan already exists, complete that first
  return currentPlan, false
@@ -178,41 +175,14 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object,
  })
  }
 
+ // Check for the need to rotate TLS CA certificate and all members
+ if len(plan) == 0 {
+ plan = createRotateTLSCAPlan(log, spec, status, getTLSCA)
+ }
+
  // Check for the need to rotate TLS certificate of a members
- if len(plan) == 0 && spec.TLS.IsSecure() {
- status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error {
- for _, m := range members {
- if len(plan) > 0 {
- // Only 1 change at a time
- continue
- }
- if m.Phase != api.MemberPhaseCreated {
- // Only make changes when phase is created
- continue
- }
- if group == api.ServerGroupSyncWorkers {
- // SyncWorkers have no externally created TLS keyfile
- continue
- }
- // Load keyfile
- keyfile, err := getTLSKeyfile(group, m)
- if err != nil {
- log.Warn().Err(err).
- Str("role", group.AsRole()).
- Str("id", m.ID).
- Msg("Failed to get TLS secret")
- continue
- }
- renewalNeeded := tlsKeyfileNeedsRenewal(log, keyfile)
- if renewalNeeded {
- plan = append(append(plan,
- api.NewAction(api.ActionTypeRenewTLSCertificate, group, m.ID)),
- createRotateMemberPlan(log, m, group, "TLS certificate renewal")...,
- )
- }
- }
- return nil
- })
+ if len(plan) == 0 {
+ plan = createRotateTLSServerCertificatePlan(log, spec, status, getTLSKeyfile)
  }
 
  // Return plan
@@ -304,44 +274,6 @@ func normalizeServiceAccountName(name string) string {
  return ""
 }
 
-// tlsKeyfileNeedsRenewal decides if the certificate in the given keyfile
-// should be renewed.
-func tlsKeyfileNeedsRenewal(log zerolog.Logger, keyfile string) bool {
- raw := []byte(keyfile)
- for {
- var derBlock *pem.Block
- derBlock, raw = pem.Decode(raw)
- if derBlock == nil {
- break
- }
- if derBlock.Type == "CERTIFICATE" {
- cert, err := x509.ParseCertificate(derBlock.Bytes)
- if err != nil {
- // We do not understand the certificate, let's renew it
- log.Warn().Err(err).Msg("Failed to parse x509 certificate. Renewing it")
- return true
- }
- if cert.IsCA {
- // Only look at the server certificate, not CA or intermediate
- continue
- }
- // Check expiration date. Renewal at 2/3 of lifetime.
- ttl := cert.NotAfter.Sub(cert.NotBefore)
- expirationDate := cert.NotBefore.Add((ttl / 3) * 2)
- if expirationDate.Before(time.Now()) {
- // We should renew now
- log.Debug().
- Str("not-before", cert.NotBefore.String()).
- Str("not-after", cert.NotAfter.String()).
- Str("expiration-date", expirationDate.String()).
- Msg("TLS certificate renewal needed")
- return true
- }
- }
- }
- return false
-}
-
 // createScalePlan creates a scaling plan for a single server group
 func createScalePlan(log zerolog.Logger, members api.MemberStatusList, group api.ServerGroup, count int) api.Plan {
  var plan api.Plan