spire-agent: re-attest without restarting

pkg/agent/manager/manager.go

@@ -192,28 +192,34 @@ func (m *manager) Initialize(ctx context.Context) error {
 func (m *manager) Run(ctx context.Context) error {
 	defer m.client.Release()
 
-	err := util.RunTasks(ctx,
-		m.runSynchronizer,
-		m.runSyncSVIDs,
-		m.runSVIDObserver,
-		m.runBundleObserver,
-		m.svid.Run)
+	for {
+		err := util.RunTasks(ctx,
+			m.runSynchronizer,
+			m.runSyncSVIDs,
+			m.runSVIDObserver,
+			m.runBundleObserver,
+			m.svid.Run)
 
-	switch {
-	case err == nil || errors.Is(err, context.Canceled):
-		m.c.Log.Info("Cache manager stopped")
-		return nil
-	case nodeutil.ShouldAgentReattest(err):
-		m.c.Log.WithError(err).Warn("Agent needs to re-attest; removing SVID and shutting down")
-		m.deleteSVID()
-		return err
-	case nodeutil.ShouldAgentShutdown(err):
-		m.c.Log.WithError(err).Warn("Agent is banned: removing SVID and shutting down")
-		m.deleteSVID()
-		return err
-	default:
-		m.c.Log.WithError(err).Error("Cache manager crashed")
-		return err
+		switch {
+		case err == nil || errors.Is(err, context.Canceled):
+			m.c.Log.Info("Cache manager stopped")
+			return nil
+		case nodeutil.ShouldAgentReattest(err):
+			m.c.Log.WithError(err).Warn("Agent needs to re-attest; will attempt to re-attest")
+			reattestError := m.svid.Reattest(ctx)
+			if reattestError != nil {
+				m.c.Log.WithError(reattestError).Error("Agent failed re-attestation; removing SVID and shutting down")
+				m.deleteSVID()
+				return err
+			}
+		case nodeutil.ShouldAgentShutdown(err):
+			m.c.Log.WithError(err).Warn("Agent is banned: removing SVID and shutting down")
+			m.deleteSVID()
+			return err
+		default:
+			m.c.Log.WithError(err).Error("Cache manager crashed")
+			return err
+		}
 	}
 }
 

pkg/agent/svid/rotator.go

@@ -26,6 +26,7 @@ import (
 
 type Rotator interface {
 	Run(ctx context.Context) error
+	Reattest(ctx context.Context) error
 
 	State() State
 	Subscribe() observer.Stream
@@ -137,6 +138,29 @@ func (r *rotator) SetRotationFinishedHook(f func()) {
 	r.rotationFinishedHook = f
 }
 
+func (r *rotator) Reattest(ctx context.Context) (err error) {
+	state, ok := r.state.Value().(State)
+	if !ok {
+		return fmt.Errorf("unexpected value type: %T", r.state.Value())
+	}
+
+	if state.Reattestable {
+		if !r.c.DisableReattestToRenew {
+			err = r.reattest(ctx)
+		} else {
+			return errors.New("re-attestation is disabled")
+		}
+	} else {
+		return errors.New("attestation method is not re-attestable")
+	}
+
+	if err == nil && r.rotationFinishedHook != nil {
+		r.rotationFinishedHook()
+	}
+
+	return err
+}
+
 func (r *rotator) rotateSVIDIfNeeded(ctx context.Context) (err error) {
 	state, ok := r.state.Value().(State)
 	if !ok {

pkg/agent/svid/rotator_test.go

@@ -62,10 +62,11 @@ func TestRotator(t *testing.T) {
 	}
 
 	for _, tt := range []struct {
-		name         string
-		notAfter     time.Duration
-		shouldRotate bool
-		reattest     bool
+		name          string
+		notAfter      time.Duration
+		shouldRotate  bool
+		reattest      bool
+		forceRotation bool
 	}{
 		{
 			name:         "not expired at startup",
@@ -94,6 +95,13 @@ func TestRotator(t *testing.T) {
 			shouldRotate: true,
 			reattest:     true,
 		},
+		{
+			name:          "reattest when requested",
+			notAfter:      time.Minute,
+			shouldRotate:  false,
+			reattest:      true,
+			forceRotation: true,
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			svidKM := keymanager.ForSVID(fakeagentkeymanager.New(t, ""))
@@ -200,6 +208,9 @@ func TestRotator(t *testing.T) {
 					}
 					t.Fatal("timed out waiting for rotation check to finish")
 				}
+			} else if tt.forceRotation {
+				err := rotator.Reattest(context.Background())
+				require.NoError(t, err)
 			}
 
 			// Shut down the rotator
@@ -213,7 +224,7 @@ func TestRotator(t *testing.T) {
 
 			// If rotation was supposed to happen, wait for the SVID changes
 			// on the state stream.
-			if tt.shouldRotate {
+			if tt.shouldRotate || tt.forceRotation {
 				require.True(t, stream.HasNext(), "SVID stream should have changes")
 				stream.Next()
 			} else {
@@ -224,7 +235,7 @@ func TestRotator(t *testing.T) {
 			// the appropriate number of times.
 			state := stream.Value().(State)
 			require.Len(t, state.SVID, 1)
-			if tt.shouldRotate {
+			if tt.shouldRotate || tt.forceRotation {
 				assert.NotEqual(t, svid, state.SVID)
 				assert.NotEqual(t, svidKey, state.Key)
 				assert.Equal(t, 2, mockClient.releaseCount, "client might not released after rotation")

test/integration/suites/evict-agent/02-bootstrap-agent

@@ -1,5 +1,15 @@
 #!/bin/bash
 
 log-debug "bootstrapping agent..."
-docker-compose exec -T spire-server \
-    /opt/spire/bin/spire-server bundle show > conf/agent/bootstrap.crt
+
+MAXCHECKS=30
+CHECKINTERVAL=1
+for ((i=1;i<=MAXCHECKS;i++)); do
+    log-info "trying to bootstrap agent ($i of $MAXCHECKS max)..."
+    docker-compose logs spire-agent
+    if docker-compose exec -T spire-server \
+        /opt/spire/bin/spire-server bundle show > conf/agent/bootstrap.crt; then
+	    exit 0
+    fi
+    sleep "${CHECKINTERVAL}"
+done

test/integration/suites/evict-agent/07-start-agent

@@ -1,4 +1,17 @@
 #!/bin/bash
 
 log-debug "starting agent again..."
+
 docker-up spire-agent
+
+# Check at most 30 times (with one second in between) that the agent is back up
+MAXCHECKS=30
+CHECKINTERVAL=1
+for ((i=1;i<=MAXCHECKS;i++)); do
+    log-info "checking that the agent is back up ($i of $MAXCHECKS max)..."
+    docker-compose logs spire-agent
+    if docker-compose logs spire-agent | grep "Starting Workload and SDS APIs"; then
+	    exit 0
+    fi
+    sleep "${CHECKINTERVAL}"
+done

test/integration/suites/evict-agent/08-evict-agent

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+log-debug "deleting agent..."
+
+# Check at most 30 times (with one second in between) that we can evict the agent, it may take a while for it to start up
+MAXCHECKS=30
+CHECKINTERVAL=1
+for ((i=1;i<=MAXCHECKS;i++)); do
+    log-info "attempting to evict agent ($i of $MAXCHECKS max)..."
+    if docker-compose exec -T spire-server \
+        /opt/spire/bin/spire-server agent evict \
+        -spiffeID "spiffe://domain.test/spire/agent/x509pop/$(fingerprint conf/agent/agent.crt.pem)"; then
+	    exit 0
+    fi
+    sleep "${CHECKINTERVAL}"
+done
+
+
+# Check at most 30 times (with one second in between) that the agent has to re-attest
+MAXCHECKS=30
+CHECKINTERVAL=1
+for ((i=1;i<=MAXCHECKS;i++)); do
+    log-info "checking for agent to get notification and try to reattest ($i of $MAXCHECKS max)..."
+    docker-compose logs spire-agent
+    if docker-compose logs spire-agent | grep "Agent needs to re-attest; will attempt to re-attest"; then
+	    exit 0
+    fi
+    sleep "${CHECKINTERVAL}"
+done
+
+# Check at most 30 times (with one second in between) that the agent has re-attested
+MAXCHECKS=30
+CHECKINTERVAL=1
+for ((i=1;i<=MAXCHECKS;i++)); do
+    log-info "checking for agent to get notification and try to reattest ($i of $MAXCHECKS max)..."
+    docker-compose logs spire-agent
+    if docker-compose logs spire-agent | grep "Successfully reattested node"; then
+	    exit 0
+    fi
+    sleep "${CHECKINTERVAL}"
+done
+
+fail-now "timed out waiting for agent to shut down"
+

test/integration/suites/node-re-attestation/03-evict-agents

@@ -12,4 +12,7 @@ docker-compose exec -T spire-server \
     /opt/spire/bin/spire-server agent evict -spiffeID $AGENT_B_SPIFFE_ID || fail-now "failed to evict agent b."
 
 check-evict-agents $AGENT_A_SPIFFE_ID $AGENT_B_SPIFFE_ID
-check-attested-agents
+
+# spire-agent-a will re-attest but spire-agent-b won't because join_token implements trust on first use model.
+AGENT_A_SPIFFE_ID_PATH="/spire/agent/x509pop/$(fingerprint conf/agent/agent.crt.pem)"
+check-attested-agents $AGENT_A_SPIFFE_ID_PATH