k8srollout: improve pod tracking for existing pods (#5598)

fixes #4566

internal/engine/k8srollout/podmonitor.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/jonboulle/clockwork"
 	v1 "k8s.io/api/core/v1"
 
 	"github.com/tilt-dev/tilt/internal/k8s"
@@ -25,12 +26,14 @@ type podManifest struct {
 type PodMonitor struct {
 	pods            map[podManifest]podStatus
 	trackingStarted map[podManifest]bool
+	startTime       time.Time
 }
 
-func NewPodMonitor() *PodMonitor {
+func NewPodMonitor(clock clockwork.Clock) *PodMonitor {
 	return &PodMonitor{
 		pods:            make(map[podManifest]podStatus),
 		trackingStarted: make(map[podManifest]bool),
+		startTime:       clock.Now(),
 	}
 }
 
@@ -81,9 +84,15 @@ func (m *PodMonitor) OnChange(ctx context.Context, st store.RStore, _ store.Chan
 
 func (m *PodMonitor) print(ctx context.Context, update podStatus) {
 	key := podManifest{pod: update.podID, manifest: update.manifestName}
+
 	if !m.trackingStarted[key] {
-		logger.Get(ctx).Infof("\nTracking new pod rollout (%s):", update.podID)
 		m.trackingStarted[key] = true
+
+		if m.startTime.After(update.startTime) {
+			logger.Get(ctx).Infof("\nAttaching to existing pod (%s). Only new logs will be streamed.", update.podID)
+			return
+		}
+		logger.Get(ctx).Infof("\nTracking new pod rollout (%s):", update.podID)
 	}
 
 	m.printCondition(ctx, "Scheduled", update.scheduled, update.startTime)

internal/engine/k8srollout/podmonitor_test.go

@@ -9,6 +9,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/jonboulle/clockwork"
+
 	"github.com/tilt-dev/tilt/pkg/apis"
 
 	"github.com/tilt-dev/tilt/pkg/apis/core/v1alpha1"
@@ -31,7 +33,7 @@ var PodmonitorWriteGoldenMaster = "0"
 func TestMonitorReady(t *testing.T) {
 	f := newPMFixture(t)
 
-	start := time.Now()
+	start := f.clock.Now()
 	p := v1alpha1.Pod{
 		Name:      "pod-id",
 		CreatedAt: apis.NewTime(start),
@@ -64,11 +66,47 @@ func TestMonitorReady(t *testing.T) {
 	assertSnapshot(t, f.out.String())
 }
 
+func TestAttachExisting(t *testing.T) {
+	f := newPMFixture(t)
+
+	start := f.clock.Now()
+	p := v1alpha1.Pod{
+		Name:      "pod-id",
+		CreatedAt: apis.NewTime(start.Add(-10 * time.Second)),
+		Conditions: []v1alpha1.PodCondition{
+			{
+				Type:               string(v1.PodScheduled),
+				Status:             string(v1.ConditionTrue),
+				LastTransitionTime: apis.NewTime(start.Add(-10 * time.Second)),
+			},
+			{
+				Type:               string(v1.PodInitialized),
+				Status:             string(v1.ConditionTrue),
+				LastTransitionTime: apis.NewTime(start.Add(-10 * time.Second)),
+			},
+			{
+				Type:               string(v1.PodReady),
+				Status:             string(v1.ConditionTrue),
+				LastTransitionTime: apis.NewTime(start.Add(-10 * time.Second)),
+			},
+		},
+	}
+
+	state := store.NewState()
+	state.UpsertManifestTarget(manifestutils.NewManifestTargetWithPod(
+		model.Manifest{Name: "server"}, p))
+	f.store.SetState(*state)
+
+	_ = f.pm.OnChange(f.ctx, f.store, store.LegacyChangeSummary())
+
+	assertSnapshot(t, f.out.String())
+}
+
 // https://github.com/tilt-dev/tilt/issues/3513
 func TestJobCompleted(t *testing.T) {
 	f := newPMFixture(t)
 
-	start := time.Now()
+	start := f.clock.Now()
 	p := v1alpha1.Pod{
 		Name:      "pod-id",
 		CreatedAt: apis.NewTime(start),
@@ -107,7 +145,7 @@ func TestJobCompleted(t *testing.T) {
 func TestJobCompletedAfterReady(t *testing.T) {
 	f := newPMFixture(t)
 
-	start := time.Now()
+	start := f.clock.Now()
 	p := v1alpha1.Pod{
 		Name:      "pod-id",
 		CreatedAt: apis.NewTime(start),
@@ -158,14 +196,16 @@ type pmFixture struct {
 	cancel func()
 	out    *bufsync.ThreadSafeBuffer
 	store  *testStore
+	clock  clockwork.FakeClock
 }
 
 func newPMFixture(t *testing.T) *pmFixture {
 	f := tempdir.NewTempDirFixture(t)
 
 	out := bufsync.NewThreadSafeBuffer()
 	st := NewTestingStore(out)
-	pm := NewPodMonitor()
+	clock := clockwork.NewFakeClock()
+	pm := NewPodMonitor(clock)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	ctx = logger.WithLogger(ctx, logger.NewTestLogger(out))
@@ -177,7 +217,9 @@ func newPMFixture(t *testing.T) *pmFixture {
 		cancel:         cancel,
 		out:            out,
 		store:          st,
+		clock:          clock,
 	}
+	clock.Advance(time.Second)
 
 	t.Cleanup(ret.TearDown)
 

internal/engine/k8srollout/testdata/TestAttachExisting_master

@@ -0,0 +1,2 @@
+
+Attaching to existing pod (pod-id). Only new logs will be streamed.

internal/engine/upper_test.go

@@ -3598,7 +3598,7 @@ func newTestFixture(t *testing.T, options ...fixtureOptions) *testFixture {
 	ret.disableEnvAnalyticsOpt()
 
 	tc := telemetry.NewController(clock, tracer.NewSpanCollector(ctx))
-	podm := k8srollout.NewPodMonitor()
+	podm := k8srollout.NewPodMonitor(clock)
 
 	uss := uisession.NewSubscriber(cdc)
 	urs := uiresource.NewSubscriber(cdc)