Improve agent certificate rotation

Pods now mount a secret containing both old and new certificates,
with file names being the hash of the certificate. When it is time
to rotate the certificate, the operator updates the automation
config with a new path (including the hash) to the certificate.
This eliminates the need to manually restart Pods during
certificate rotation.

Author: m1kola

api/v1/mdb/mongodb_types.go

@@ -195,7 +195,7 @@ func (m *MongoDB) GetSecretsMountedIntoDBPod() []string {
 			secrets = append(secrets, tls)
 		}
 	}
-	agentCerts := m.GetSecurity().AgentClientCertificateSecretName(m.Name).Name
+	agentCerts := m.GetSecurity().AgentClientCertificateSecretName(m.Name)
 	if agentCerts != "" {
 		secrets = append(secrets, agentCerts)
 	}
@@ -852,7 +852,7 @@ func (s *Security) ShouldUseX509(currentAgentAuthMode string) bool {
 // AgentClientCertificateSecretName returns the name of the Secret that holds the agent
 // client TLS certificates.
 // If no custom name has been defined, it returns the default one.
-func (s Security) AgentClientCertificateSecretName(resourceName string) corev1.SecretKeySelector {
+func (s Security) AgentClientCertificateSecretName(resourceName string) string {
 	secretName := util.AgentSecretName
 
 	if s.CertificatesSecretsPrefix != "" {
@@ -862,10 +862,7 @@ func (s Security) AgentClientCertificateSecretName(resourceName string) corev1.S
 		secretName = s.Authentication.Agents.ClientCertificateSecretRefWrap.ClientCertificateSecretRef.Name
 	}
 
-	return corev1.SecretKeySelector{
-		Key:                  util.AutomationAgentPemSecretKey,
-		LocalObjectReference: corev1.LocalObjectReference{Name: secretName},
-	}
+	return secretName
 }
 
 // The customer has set ClientCertificateSecretRef. This signals that client certs are required,

api/v1/mdb/mongodb_types_test.go

@@ -382,15 +382,15 @@ func TestAgentClientCertificateSecretName(t *testing.T) {
 	rs := NewReplicaSetBuilder().SetSecurityTLSEnabled().EnableAuth([]AuthMode{util.X509}).Build()
 
 	// Default is the hardcoded "agent-certs"
-	assert.Equal(t, util.AgentSecretName, rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name)
+	assert.Equal(t, util.AgentSecretName, rs.GetSecurity().AgentClientCertificateSecretName(rs.Name))
 
 	// If the top-level prefix is there, we use it
 	rs.Spec.Security.CertificatesSecretsPrefix = "prefix"
-	assert.Equal(t, fmt.Sprintf("prefix-%s-%s", rs.Name, util.AgentSecretName), rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name)
+	assert.Equal(t, fmt.Sprintf("prefix-%s-%s", rs.Name, util.AgentSecretName), rs.GetSecurity().AgentClientCertificateSecretName(rs.Name))
 
 	// If the name is provided (deprecated) we return it
 	rs.GetSecurity().Authentication.Agents.ClientCertificateSecretRefWrap.ClientCertificateSecretRef.Name = "foo"
-	assert.Equal(t, "foo", rs.GetSecurity().AgentClientCertificateSecretName(rs.Name).Name)
+	assert.Equal(t, "foo", rs.GetSecurity().AgentClientCertificateSecretName(rs.Name))
 }
 
 func TestInternalClusterAuthSecretName(t *testing.T) {

changelog/20250909_feature_improved_agent_certificate_rotation.md

@@ -0,0 +1,7 @@
+---
+title: Improved agent certificate rotation
+kind: feature
+date: 2025-09-09
+---
+
+* Improve automation agent certificate rotation: the agent now restarts automatically when its certificate is renewed, ensuring smooth operation without manual intervention and allowing seamless certificate updates without requiring manual Pod restarts.

controllers/om/automation_config_test.go

@@ -365,7 +365,7 @@ func TestCanResetAgentSSL(t *testing.T) {
 	ac.AgentSSL = &AgentSSL{
 		ClientCertificateMode: util.OptionalClientCertficates,
 		CAFilePath:            util.CAFilePathInContainer,
-		AutoPEMKeyFilePath:    util.AutomationAgentPemFilePath,
+		AutoPEMKeyFilePath:    "/fake/path/to/pem",
 	}
 
 	if err := ac.Apply(); err != nil {
@@ -374,7 +374,7 @@ func TestCanResetAgentSSL(t *testing.T) {
 
 	tls := cast.ToStringMap(ac.Deployment["tls"])
 	assert.Equal(t, tls["clientCertificateMode"], util.OptionalClientCertficates)
-	assert.Equal(t, tls["autoPEMKeyFilePath"], util.AutomationAgentPemFilePath)
+	assert.Equal(t, tls["autoPEMKeyFilePath"], "/fake/path/to/pem")
 	assert.Equal(t, tls["CAFilePath"], util.CAFilePathInContainer)
 
 	ac.AgentSSL = &AgentSSL{

controllers/om/backup_agent_config.go

@@ -49,8 +49,8 @@ func (bac *BackupAgentConfig) UnsetAgentPassword() {
 	bac.BackupAgentTemplate.Password = util.MergoDelete
 }
 
-func (bac *BackupAgentConfig) EnableX509Authentication(backupAgentSubject string) {
-	bac.BackupAgentTemplate.SSLPemKeyFile = util.AutomationAgentPemFilePath
+func (bac *BackupAgentConfig) EnableX509Authentication(backupAgentSubject, automationAgentPemFilePath string) {
+	bac.BackupAgentTemplate.SSLPemKeyFile = automationAgentPemFilePath
 	bac.SetAgentUserName(backupAgentSubject)
 }
 

controllers/om/backup_agent_test.go

@@ -32,7 +32,7 @@ func TestFieldsAreUpdatedBackupConfig(t *testing.T) {
 
 func TestBackupFieldsAreNotLost(t *testing.T) {
 	config := getTestBackupConfig()
-	config.EnableX509Authentication("namespace")
+	config.EnableX509Authentication("namespace", "/fake/path/to/pem")
 
 	assert.Contains(t, config.BackingMap, "logPath")
 	assert.Contains(t, config.BackingMap, "logRotate")
@@ -48,7 +48,7 @@ func TestBackupFieldsAreNotLost(t *testing.T) {
 func TestNestedFieldsAreNotLost(t *testing.T) {
 	config := getTestBackupConfig()
 
-	config.EnableX509Authentication("namespace")
+	config.EnableX509Authentication("namespace", "/fake/path/to/pem")
 
 	_ = config.Apply()
 

controllers/om/monitoring_agent_config.go

@@ -45,9 +45,9 @@ func (m *MonitoringAgentConfig) UnsetAgentPassword() {
 	m.MonitoringAgentTemplate.Password = util.MergoDelete
 }
 
-func (m *MonitoringAgentConfig) EnableX509Authentication(MonitoringAgentSubject string) {
-	m.MonitoringAgentTemplate.SSLPemKeyFile = util.AutomationAgentPemFilePath
-	m.SetAgentUserName(MonitoringAgentSubject)
+func (m *MonitoringAgentConfig) EnableX509Authentication(monitoringAgentSubject, automationAgentPemFilePath string) {
+	m.MonitoringAgentTemplate.SSLPemKeyFile = automationAgentPemFilePath
+	m.SetAgentUserName(monitoringAgentSubject)
 }
 
 func (m *MonitoringAgentConfig) DisableX509Authentication() {

controllers/operator/appdbreplicaset_controller.go

@@ -538,7 +538,15 @@ func (r *ReconcileAppDbReplicaSet) ReconcileAppDB(ctx context.Context, opsManage
 		return result.OK()
 	}
 
-	podVars, err := r.tryConfigureMonitoringInOpsManager(ctx, opsManager, opsManagerUserPassword, log)
+	var appdbSecretPath string
+	if r.VaultClient != nil {
+		appdbSecretPath = r.VaultClient.AppDBSecretPath()
+	}
+
+	agentCertSecretName := opsManager.Spec.AppDB.GetSecurity().AgentClientCertificateSecretName(opsManager.Spec.AppDB.GetName())
+	_, agentCertPath := r.agentCertHashAndPath(ctx, log, opsManager.Namespace, agentCertSecretName, appdbSecretPath)
+
+	podVars, err := r.tryConfigureMonitoringInOpsManager(ctx, opsManager, opsManagerUserPassword, agentCertPath, log)
 	// it's possible that Ops Manager will not be available when we attempt to configure AppDB monitoring
 	// in Ops Manager. This is not a blocker to continue with the rest of the reconciliation.
 	if err != nil {
@@ -613,10 +621,6 @@ func (r *ReconcileAppDbReplicaSet) ReconcileAppDB(ctx context.Context, opsManage
 		return r.updateStatus(ctx, opsManager, workflowStatus, log, appDbStatusOption)
 	}
 
-	var appdbSecretPath string
-	if r.VaultClient != nil {
-		appdbSecretPath = r.VaultClient.AppDBSecretPath()
-	}
 	tlsSecretName := opsManager.Spec.AppDB.GetSecurity().MemberCertificateSecretName(opsManager.Spec.AppDB.Name())
 	certHash := enterprisepem.ReadHashFromSecret(ctx, r.SecretClient, opsManager.Namespace, tlsSecretName, appdbSecretPath, log)
 
@@ -1621,7 +1625,7 @@ func (r *ReconcileAppDbReplicaSet) ensureAppDbAgentApiKey(ctx context.Context, o
 
 // tryConfigureMonitoringInOpsManager attempts to configure monitoring in Ops Manager. This might not be possible if Ops Manager
 // has not been created yet, if that is the case, an empty PodVars will be returned.
-func (r *ReconcileAppDbReplicaSet) tryConfigureMonitoringInOpsManager(ctx context.Context, opsManager *omv1.MongoDBOpsManager, opsManagerUserPassword string, log *zap.SugaredLogger) (env.PodEnvVars, error) {
+func (r *ReconcileAppDbReplicaSet) tryConfigureMonitoringInOpsManager(ctx context.Context, opsManager *omv1.MongoDBOpsManager, opsManagerUserPassword string, agentCertPath string, log *zap.SugaredLogger) (env.PodEnvVars, error) {
 	var operatorVaultSecretPath string
 	if r.VaultClient != nil {
 		operatorVaultSecretPath = r.VaultClient.OperatorSecretPath()
@@ -1660,6 +1664,7 @@ func (r *ReconcileAppDbReplicaSet) tryConfigureMonitoringInOpsManager(ctx contex
 		Mechanisms:         []string{util.SCRAM},
 		ClientCertificates: util.OptionalClientCertficates,
 		AutoUser:           util.AutomationAgentUserName,
+		AutoPEMKeyFilePath: agentCertPath,
 		CAFilePath:         util.CAFilePathInContainer,
 	}
 	err = authentication.Configure(conn, opts, false, log)

controllers/operator/appdbreplicaset_controller_test.go

@@ -377,7 +377,7 @@ func TestTryConfigureMonitoringInOpsManager(t *testing.T) {
 	require.NoError(t, err)
 
 	// attempt configuring monitoring when there is no api key secret
-	podVars, err := reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", zap.S())
+	podVars, err := reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", "/fake/agent-cert/path", zap.S())
 	assert.NoError(t, err)
 
 	assert.Empty(t, podVars.ProjectID)
@@ -408,7 +408,7 @@ func TestTryConfigureMonitoringInOpsManager(t *testing.T) {
 	assert.NoError(t, err)
 
 	// once the secret exists, monitoring should be fully configured
-	podVars, err = reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", zap.S())
+	podVars, err = reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", "/fake/agent-cert/path", zap.S())
 	assert.NoError(t, err)
 
 	assert.Equal(t, om.TestGroupID, podVars.ProjectID)
@@ -522,7 +522,7 @@ func TestTryConfigureMonitoringInOpsManagerWithExternalDomains(t *testing.T) {
 	require.NoError(t, err)
 
 	// attempt configuring monitoring when there is no api key secret
-	podVars, err := reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", zap.S())
+	podVars, err := reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", "/fake/agent-cert/path", zap.S())
 	assert.NoError(t, err)
 
 	assert.Empty(t, podVars.ProjectID)
@@ -553,7 +553,7 @@ func TestTryConfigureMonitoringInOpsManagerWithExternalDomains(t *testing.T) {
 	assert.NoError(t, err)
 
 	// once the secret exists, monitoring should be fully configured
-	podVars, err = reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", zap.S())
+	podVars, err = reconciler.tryConfigureMonitoringInOpsManager(ctx, opsManager, "password", "/fake/agent-cert/path", zap.S())
 	assert.NoError(t, err)
 
 	assert.Equal(t, om.TestGroupID, podVars.ProjectID)

controllers/operator/authentication/authentication.go

@@ -43,6 +43,8 @@ type Options struct {
 	// so it is possible to use other auth mechanisms without needing to provide client certs.
 	ClientCertificates string
 
+	AutoPEMKeyFilePath string
+
 	CAFilePath string
 
 	// Use Agent Client Auth
@@ -348,7 +350,7 @@ func addOrRemoveAgentClientCertificate(conn om.Connection, opts Options, log *za
 
 		if opts.AgentsShouldUseClientAuthentication {
 			ac.AgentSSL = &om.AgentSSL{
-				AutoPEMKeyFilePath:    util.AutomationAgentPemFilePath,
+				AutoPEMKeyFilePath:    opts.AutoPEMKeyFilePath,
 				CAFilePath:            opts.CAFilePath,
 				ClientCertificateMode: opts.ClientCertificates,
 			}