diff --git a/packages/runtime/container-runtime/src/runningSummarizer.ts b/packages/runtime/container-runtime/src/runningSummarizer.ts
index 42c26596aeb3..bf96ed024011 100644
--- a/packages/runtime/container-runtime/src/runningSummarizer.ts
+++ b/packages/runtime/container-runtime/src/runningSummarizer.ts
@@ -258,9 +258,10 @@ export class RunningSummarizer implements IDisposable {
         this.summaryCollection.unsetPendingAckTimerTimeoutCallback();
 
         if (waitStartResult.result === "done" && waitStartResult.value !== undefined) {
-            this.heuristicData.initialize({
+            this.heuristicData.updateWithLastSummaryAckInfo({
                 refSequenceNumber: waitStartResult.value.summaryOp.referenceSequenceNumber,
-                summaryTime: waitStartResult.value.summaryOp.timestamp,
+                // This will be the Summarizer starting point so only use timestamps from client's machine.
+                summaryTime: Date.now(),
                 summarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,
             });
         }
diff --git a/packages/runtime/container-runtime/src/summarizerHeuristics.ts b/packages/runtime/container-runtime/src/summarizerHeuristics.ts
index 24a8ecff83eb..e5979a492057 100644
--- a/packages/runtime/container-runtime/src/summarizerHeuristics.ts
+++ b/packages/runtime/container-runtime/src/summarizerHeuristics.ts
@@ -29,7 +29,7 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
         this._lastSuccessfulSummary = { ...attemptBaseline };
     }
 
-    public initialize(lastSummary: Readonly<ISummarizeAttempt>) {
+    public updateWithLastSummaryAckInfo(lastSummary: Readonly<ISummarizeAttempt>) {
         this._lastAttempt = lastSummary;
         this._lastSuccessfulSummary = { ...lastSummary };
     }
diff --git a/packages/runtime/container-runtime/src/summarizerTypes.ts b/packages/runtime/container-runtime/src/summarizerTypes.ts
index c7c0831ce1c3..6c663009e641 100644
--- a/packages/runtime/container-runtime/src/summarizerTypes.ts
+++ b/packages/runtime/container-runtime/src/summarizerTypes.ts
@@ -349,10 +349,10 @@ export interface ISummarizeHeuristicData {
     readonly lastSuccessfulSummary: Readonly<ISummarizeAttempt>;
 
     /**
-     * Initializes lastAttempt and lastSuccessfulAttempt based on the last summary.
+     * Updates lastAttempt and lastSuccessfulAttempt based on the last summary.
      * @param lastSummary - last ack summary
      */
-    initialize(lastSummary: ISummarizeAttempt): void;
+    updateWithLastSummaryAckInfo(lastSummary: ISummarizeAttempt): void;
 
     /**
      * Records a summary attempt. If the attempt was successfully sent,
diff --git a/packages/runtime/container-runtime/src/summaryGenerator.ts b/packages/runtime/container-runtime/src/summaryGenerator.ts
index 55247652c123..ace72995d6ee 100644
--- a/packages/runtime/container-runtime/src/summaryGenerator.ts
+++ b/packages/runtime/container-runtime/src/summaryGenerator.ts
@@ -237,6 +237,8 @@ export class SummaryGenerator {
         const { refreshLatestAck, fullTree } = options;
         const logger = ChildLogger.create(this.logger, undefined, { all: summarizeProps });
 
+        // Note: timeSinceLastAttempt and timeSinceLastSummary for the
+        // first summary are basically the time since the summarizer was loaded.
         const timeSinceLastAttempt = Date.now() - this.heuristicData.lastAttempt.summaryTime;
         const timeSinceLastSummary = Date.now() - this.heuristicData.lastSuccessfulSummary.summaryTime;
         let summarizeTelemetryProps: SummaryGeneratorTelemetry = {