From 1d038cd14c06d69d598e09f38adcfd7baf761944 Mon Sep 17 00:00:00 2001
From: Peter Waldschmidt <pewaldsc@microsoft.com>
Date: Mon, 10 Mar 2025 16:16:46 -0400
Subject: [PATCH 1/9] [AI Evaluation] Add versioning to  reporting and caching

---
 .../CSharp/Defaults.cs                               | 12 ++++++++++++
 .../CSharp/ResponseCachingChatClient.cs              |  2 +-
 .../CSharp/ScenarioRunResult.cs                      |  9 ++++++++-
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
index 777d723b790..1c76766c402 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
@@ -30,4 +30,16 @@ public static class Defaults
     /// in the <see cref="IResponseCacheProvider"/>'s cache before they are considered expired and evicted.
     /// </summary>
     public static TimeSpan DefaultTimeToLiveForCacheEntries { get; } = TimeSpan.FromDays(14);
+
+    /// <summary>
+    /// Defines the version number for the reporting format. If and when the serialized format undergoes breaking changes, this number
+    /// will be incremented to indicate the version a report file was created with compared to the version this library will produce.
+    /// </summary>
+    public const int ReportingFormatVersion = 1;
+
+    /// <summary>
+    /// Defines an version number for the cache format. Changing this value will break the cache key and force requests
+    /// to retrieve fresh data from the source until the version is updated again.
+    /// </summary>
+    public const int CacheVersion = 1;
 }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs
index 848b6583ced..5614fa80d87 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs
@@ -37,6 +37,6 @@ public ResponseCachingChatClient(
 
     /// <inheritdoc/>
     protected override string GetCacheKey(params ReadOnlySpan<object?> values)
-        => base.GetCacheKey([.. values, .. _cachingKeys]);
+        => base.GetCacheKey([Defaults.CacheVersion, .. values, .. _cachingKeys]);
 
 }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
index e1a4102e42c..b02bb7882e1 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
@@ -37,6 +37,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting;
 /// The <see cref="Evaluation.EvaluationResult"/> for the <see cref="ScenarioRun"/> corresponding to the
 /// <see cref="ScenarioRunResult"/> being constructed.
 /// </param>
+/// <param name="formatVersion">The version of the format used to persist this scenario run.</param>
 [method: JsonConstructor]
 public sealed class ScenarioRunResult(
     string scenarioName,
@@ -45,7 +46,8 @@ public sealed class ScenarioRunResult(
     DateTime creationTime,
     IList<ChatMessage> messages,
     ChatResponse modelResponse,
-    EvaluationResult evaluationResult)
+    EvaluationResult evaluationResult,
+    int formatVersion = Defaults.ReportingFormatVersion)
 {
     /// <summary>
     /// Initializes a new instance of the <see cref="ScenarioRunResult"/> class.
@@ -81,6 +83,11 @@ public ScenarioRunResult(
     {
     }
 
+    /// <summary>
+    /// Gets or sets the Version of the format used to persist this scenario run. You should not modify this value from the default.
+    /// </summary>
+    public int FormatVersion { get; set; } = formatVersion;
+
     /// <summary>
     /// Gets or sets the <see cref="ScenarioRun.ScenarioName"/>.
     /// </summary>

From f3bbe082b1202a6d36b89c03885438b419822036 Mon Sep 17 00:00:00 2001
From: Peter Waldschmidt <pewaldsc@microsoft.com>
Date: Mon, 10 Mar 2025 16:31:17 -0400
Subject: [PATCH 2/9] Move CacheVersion key down into the
 DistributedCachingChatClient

---
 .../CSharp/Defaults.cs                                   | 5 -----
 .../CSharp/ResponseCachingChatClient.cs                  | 2 +-
 .../ChatCompletion/DistributedCachingChatClient.cs       | 9 +++++++--
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
index 1c76766c402..457c76f8f34 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
@@ -37,9 +37,4 @@ public static class Defaults
     /// </summary>
     public const int ReportingFormatVersion = 1;
 
-    /// <summary>
-    /// Defines an version number for the cache format. Changing this value will break the cache key and force requests
-    /// to retrieve fresh data from the source until the version is updated again.
-    /// </summary>
-    public const int CacheVersion = 1;
 }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs
index 5614fa80d87..848b6583ced 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ResponseCachingChatClient.cs
@@ -37,6 +37,6 @@ public ResponseCachingChatClient(
 
     /// <inheritdoc/>
     protected override string GetCacheKey(params ReadOnlySpan<object?> values)
-        => base.GetCacheKey([Defaults.CacheVersion, .. values, .. _cachingKeys]);
+        => base.GetCacheKey([.. values, .. _cachingKeys]);
 
 }
diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs
index 67bd64cfa54..20bc6555fc9 100644
--- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs
+++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs
@@ -107,6 +107,11 @@ protected override async Task WriteCacheStreamingAsync(string key, IReadOnlyList
     /// The generated cache key is not guaranteed to be stable across releases of the library.
     /// </para>
     /// </remarks>
-    protected override string GetCacheKey(params ReadOnlySpan<object?> values) =>
-        AIJsonUtilities.HashDataToString(values, _jsonSerializerOptions);
+    protected override string GetCacheKey(params ReadOnlySpan<object?> values) 
+    {
+        // Bump the cache version to invalidate existing caches if the serialization format changes in a breaking way.
+        const int CacheVersion = 1;
+
+        return AIJsonUtilities.HashDataToString([CacheVersion, .. values], _jsonSerializerOptions);
+    }
 }

From 3a06929c87c9fb832c71d2b98791dd3a963350cc Mon Sep 17 00:00:00 2001
From: Stephen Toub <stoub@microsoft.com>
Date: Mon, 10 Mar 2025 21:18:51 -0400
Subject: [PATCH 3/9] Update
 src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs

---
 .../ChatCompletion/DistributedCachingChatClient.cs              | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs
index 20bc6555fc9..c59c78c9cd9 100644
--- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs
+++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/DistributedCachingChatClient.cs
@@ -107,7 +107,7 @@ protected override async Task WriteCacheStreamingAsync(string key, IReadOnlyList
     /// The generated cache key is not guaranteed to be stable across releases of the library.
     /// </para>
     /// </remarks>
-    protected override string GetCacheKey(params ReadOnlySpan<object?> values) 
+    protected override string GetCacheKey(params ReadOnlySpan<object?> values)
     {
         // Bump the cache version to invalidate existing caches if the serialization format changes in a breaking way.
         const int CacheVersion = 1;

From f332da2dd75c12937a980e2ca543fce351ecf5bd Mon Sep 17 00:00:00 2001
From: Peter Waldschmidt <pewaldsc@microsoft.com>
Date: Tue, 11 Mar 2025 18:29:51 -0400
Subject: [PATCH 4/9] Update
 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs

Co-authored-by: Igor Velikorossov <RussKie@users.noreply.github.com>
---
 .../CSharp/ScenarioRunResult.cs                                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
index b02bb7882e1..c5e08919edf 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
@@ -86,7 +86,7 @@ public ScenarioRunResult(
     /// <summary>
     /// Gets or sets the Version of the format used to persist this scenario run. You should not modify this value from the default.
     /// </summary>
-    public int FormatVersion { get; set; } = formatVersion;
+    public int FormatVersion { get; init; } = formatVersion;
 
     /// <summary>
     /// Gets or sets the <see cref="ScenarioRun.ScenarioName"/>.

From 0883cc0f31c6e077876016763c7569fbebdf2375 Mon Sep 17 00:00:00 2001
From: Peter Waldschmidt <pewaldsc@microsoft.com>
Date: Tue, 11 Mar 2025 18:30:04 -0400
Subject: [PATCH 5/9] Update
 src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs

Co-authored-by: Igor Velikorossov <RussKie@users.noreply.github.com>
---
 .../CSharp/Defaults.cs                                           | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
index 457c76f8f34..3e5be8309c1 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
@@ -36,5 +36,4 @@ public static class Defaults
     /// will be incremented to indicate the version a report file was created with compared to the version this library will produce.
     /// </summary>
     public const int ReportingFormatVersion = 1;
-
 }

From 39230450ba565636be37de3418355832a7ba1819 Mon Sep 17 00:00:00 2001
From: Peter Waldschmidt <pewaldsc@microsoft.com>
Date: Tue, 11 Mar 2025 19:50:28 -0400
Subject: [PATCH 6/9] Rollback formatVersion init change

---
 .../CSharp/ScenarioRunResult.cs                                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
index c5e08919edf..b02bb7882e1 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
@@ -86,7 +86,7 @@ public ScenarioRunResult(
     /// <summary>
     /// Gets or sets the Version of the format used to persist this scenario run. You should not modify this value from the default.
     /// </summary>
-    public int FormatVersion { get; init; } = formatVersion;
+    public int FormatVersion { get; set; } = formatVersion;
 
     /// <summary>
     /// Gets or sets the <see cref="ScenarioRun.ScenarioName"/>.

From 4e74bf3126c2d5bd1deb90ddfbd7084cadcda972 Mon Sep 17 00:00:00 2001
From: Peter Waldschmidt <pewaldsc@microsoft.com>
Date: Thu, 13 Mar 2025 10:40:02 -0400
Subject: [PATCH 7/9] Make version const internal, update comments.

---
 .../CSharp/Defaults.cs                                    | 6 +++---
 .../CSharp/ScenarioRunResult.cs                           | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
index 3e5be8309c1..380b109471e 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs
@@ -32,8 +32,8 @@ public static class Defaults
     public static TimeSpan DefaultTimeToLiveForCacheEntries { get; } = TimeSpan.FromDays(14);
 
     /// <summary>
-    /// Defines the version number for the reporting format. If and when the serialized format undergoes breaking changes, this number
-    /// will be incremented to indicate the version a report file was created with compared to the version this library will produce.
+    /// Defines the version number for the reporting format. If and when the serialized format undergoes
+    /// breaking changes, this number will be incremented.
     /// </summary>
-    public const int ReportingFormatVersion = 1;
+    internal const int ReportingFormatVersion = 1;
 }
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
index b02bb7882e1..26986f8b672 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
@@ -37,7 +37,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting;
 /// The <see cref="Evaluation.EvaluationResult"/> for the <see cref="ScenarioRun"/> corresponding to the
 /// <see cref="ScenarioRunResult"/> being constructed.
 /// </param>
-/// <param name="formatVersion">The version of the format used to persist this scenario run.</param>
+/// <param name="formatVersion">The version of the format used to persist the current <see cref="ScenarioRunResult"/>.</param>
 [method: JsonConstructor]
 public sealed class ScenarioRunResult(
     string scenarioName,
@@ -47,7 +47,7 @@ public sealed class ScenarioRunResult(
     IList<ChatMessage> messages,
     ChatResponse modelResponse,
     EvaluationResult evaluationResult,
-    int formatVersion = Defaults.ReportingFormatVersion)
+    int? formatVersion = null)
 {
     /// <summary>
     /// Initializes a new instance of the <see cref="ScenarioRunResult"/> class.
@@ -84,9 +84,9 @@ public ScenarioRunResult(
     }
 
     /// <summary>
-    /// Gets or sets the Version of the format used to persist this scenario run. You should not modify this value from the default.
+    /// Gets or sets the version of the format used to persist the current <see cref="ScenarioRunResult"/>.
     /// </summary>
-    public int FormatVersion { get; set; } = formatVersion;
+    public int? FormatVersion { get; set; } = formatVersion ?? Defaults.ReportingFormatVersion;
 
     /// <summary>
     /// Gets or sets the <see cref="ScenarioRun.ScenarioName"/>.

From 61617ff9a129680fb673a41596fe40da8db6e6f3 Mon Sep 17 00:00:00 2001
From: Peter Waldschmidt <pewaldsc@microsoft.com>
Date: Thu, 13 Mar 2025 10:46:17 -0400
Subject: [PATCH 8/9] Add assertion for report version

---
 .../ScenarioRunResultTests.cs                                   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ScenarioRunResultTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ScenarioRunResultTests.cs
index 9418a5db359..b6f173abd5a 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ScenarioRunResultTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ScenarioRunResultTests.cs
@@ -38,6 +38,7 @@ public void SerializeScenarioRunResult()
             messages: [new ChatMessage(ChatRole.User, "prompt")],
             modelResponse: new ChatResponse(new ChatMessage(ChatRole.Assistant, "response")),
             evaluationResult: new EvaluationResult(booleanMetric, numericMetric, stringMetric, metricWithNoValue));
+        Assert.Equal(Defaults.ReportingFormatVersion, entry.FormatVersion);
 
         string json = JsonSerializer.Serialize(entry, SerializerContext.Default.ScenarioRunResult);
         ScenarioRunResult? deserialized = JsonSerializer.Deserialize<ScenarioRunResult>(json, SerializerContext.Default.ScenarioRunResult);
@@ -49,6 +50,7 @@ public void SerializeScenarioRunResult()
         Assert.Equal(entry.CreationTime, deserialized.CreationTime);
         Assert.True(entry.Messages.SequenceEqual(deserialized.Messages, ChatMessageComparer.Instance));
         Assert.Equal(entry.ModelResponse, deserialized.ModelResponse, ChatResponseComparer.Instance);
+        Assert.Equal(entry.FormatVersion, deserialized.FormatVersion);
 
         ValidateEquivalence(entry.EvaluationResult, deserialized.EvaluationResult);
     }

From 30fb6769c9ce1039fb2c2a0b8671f391f8c811bf Mon Sep 17 00:00:00 2001
From: Peter Waldschmidt <pewaldsc@microsoft.com>
Date: Thu, 13 Mar 2025 12:48:26 -0400
Subject: [PATCH 9/9] Make ReportVersion property readonly

---
 .../CSharp/ScenarioRunResult.cs                               | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
index 26986f8b672..e4b9401d926 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunResult.cs
@@ -84,9 +84,9 @@ public ScenarioRunResult(
     }
 
     /// <summary>
-    /// Gets or sets the version of the format used to persist the current <see cref="ScenarioRunResult"/>.
+    /// Gets the version of the format used to persist the current <see cref="ScenarioRunResult"/>.
     /// </summary>
-    public int? FormatVersion { get; set; } = formatVersion ?? Defaults.ReportingFormatVersion;
+    public int? FormatVersion { get; } = formatVersion ?? Defaults.ReportingFormatVersion;
 
     /// <summary>
     /// Gets or sets the <see cref="ScenarioRun.ScenarioName"/>.