-
-
Prompt
-
-
+ const [isExpanded, setIsExpanded] = useState(true);
-
- {renderPrompt ?
{history} :
{history}}
-
+ const isUserSide = (role: string) => role.toLowerCase() === 'user' || role.toLowerCase() === 'system';
-
-
Response
-
-
-
- {renderResponse ?
{response} :
{response}}
+ return (
+
+
setIsExpanded(!isExpanded)}>
+ {isExpanded ? : }
+
Conversation
+
+
+ {isExpanded && (
+
+ {messages.map((message, index) => {
+ const isFromUserSide = isUserSide(message.role);
+ const messageRowClass = mergeClasses(
+ classes.messageRow,
+ isFromUserSide ? classes.userMessageRow : classes.assistantMessageRow
+ );
+
+ return (
+
+
{message.participantName}
+
+ {renderMarkdown ?
+
{message.content} :
+
{message.content}
+ }
+
+
+ );
+ })}
+
+ )}
-
);
+ );
};
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts
index 8cef12ce4f1..f49db7a2a60 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts
@@ -65,7 +65,6 @@ export class ScoreNode {
}
aggregate() {
- // Reset node to defaults before recalculating
this.failed = false;
this.numPassingIterations = 0;
this.numFailingIterations = 0;
@@ -84,7 +83,15 @@ export class ScoreNode {
this.numPassingIterations = this.failed ? 0 : 1;
this.numFailingIterations = this.failed ? 1 : 0;
const lastMessage = this.scenario?.messages[this.scenario?.messages.length - 1];
- const {history} = getPromptDetails(lastMessage ? [lastMessage] : [], this.scenario?.modelResponse);
+
+ const { messages } = getPromptDetails(lastMessage ? [lastMessage] : [], this.scenario?.modelResponse);
+ let history = "";
+ if (messages.length === 1) {
+ history = messages[0].content;
+ } else if (messages.length > 1) {
+ history = messages.map(m => `[${m.participantName}] ${m.content}`).join("\n\n");
+ }
+
this.shortenedPrompt = shortenPrompt(history);
} else {
for (const child of this.childNodes) {
@@ -104,7 +111,22 @@ export class ScoreNode {
}
}
-
+ collapseSingleChildNodes() {
+ if (this.isLeafNode) {
+ return;
+ }
+
+ while (this.childNodes.length === 1) {
+ const onlyChild = this.childNodes[0];
+ this.name += ` / ${onlyChild.name}`;
+ this.children = onlyChild.children;
+ this.scenario = onlyChild.scenario;
+ }
+
+ for (const child of this.childNodes) {
+ child.collapseSingleChildNodes();
+ }
+ }
};
export const DefaultRootNodeName = "All Evaluations";
@@ -145,25 +167,42 @@ const isTextContent = (content: AIContent): content is TextContent => {
return (content as TextContent).text !== undefined;
};
-export const getPromptDetails = (messages: ChatMessage[], modelResponse?: ChatResponse): {history:string, response: string}=> {
- let history: string = "";
- if (messages.length === 1) {
- history = messages[0].contents.map(c => (c as TextContent).text).join("\n");
- } else if (messages.length > 1) {
- const historyItems: string[] = [];
- for (const m of messages) {
+export type ChatMessageDisplay = {
+ role: string;
+ participantName: string;
+ content: string;
+};
+
+export const getPromptDetails = (messages: ChatMessage[], modelResponse?: ChatResponse): { messages: ChatMessageDisplay[] } => {
+ const chatMessages: ChatMessageDisplay[] = [];
+
+ for (const m of messages) {
+ for (const c of m.contents) {
+ if (isTextContent(c)) {
+ const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role;
+ chatMessages.push({
+ role: m.role,
+ participantName: participantName,
+ content: c.text
+ });
+ }
+ }
+ }
+
+ if (modelResponse?.messages) {
+ for (const m of modelResponse.messages) {
for (const c of m.contents) {
if (isTextContent(c)) {
- const historyItem = m.authorName
- ? `[${m.authorName} (${m.role})] ${c.text}` : `[${m.role}] ${c.text}`;
- historyItems.push(historyItem);
+ const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role || 'Assistant';
+ chatMessages.push({
+ role: m.role,
+ participantName: participantName,
+ content: c.text
+ });
}
}
}
- history = historyItems.join("\n\n");
}
- const response: string = modelResponse?.messages.map(m => m.contents.map(c => (c as TextContent).text).join("\n") ?? "").join("\n") ?? "";
-
- return { history, response };
-};
\ No newline at end of file
+ return { messages: chatMessages };
+};
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/BooleanMetric.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/BooleanMetric.cs
index bc71408ffa2..0edb9f8b0b4 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/BooleanMetric.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/BooleanMetric.cs
@@ -9,4 +9,9 @@ namespace Microsoft.Extensions.AI.Evaluation;
///
///
The name of the
.
///
The value of the
.
-public sealed class BooleanMetric(string name, bool? value = null) : EvaluationMetric
(name, value);
+///
+/// An optional string that can be used to provide some commentary around the result represented by this
+/// .
+///
+public sealed class BooleanMetric(string name, bool? value = null, string? reason = null)
+ : EvaluationMetric(name, value, reason);
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetric.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetric.cs
index 78bb6831486..038599963af 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetric.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetric.cs
@@ -15,17 +15,27 @@ namespace Microsoft.Extensions.AI.Evaluation;
/// A base class that represents the result of an evaluation.
///
/// The name of the .
+///
+/// An optional string that can be used to provide some commentary around the result represented by this
+/// .
+///
[JsonDerivedType(typeof(NumericMetric), "numeric")]
[JsonDerivedType(typeof(BooleanMetric), "boolean")]
[JsonDerivedType(typeof(StringMetric), "string")]
[JsonDerivedType(typeof(EvaluationMetric), "none")]
-public class EvaluationMetric(string name)
+public class EvaluationMetric(string name, string? reason = null)
{
///
/// Gets or sets the name of the .
///
public string Name { get; set; } = name;
+ ///
+ /// Gets or sets a string that can optionally be used to provide some commentary around the result represented by
+ /// this .
+ ///
+ public string? Reason { get; set; } = reason;
+
///
/// Gets or sets an that identifies whether the result of the
/// evaluation represented by the current is considered good or bad, passed or
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetric{T}.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetric{T}.cs
index f0d6eea9d10..d2745069bc5 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetric{T}.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationMetric{T}.cs
@@ -25,8 +25,12 @@ public class EvaluationMetric : EvaluationMetric
///
/// The name of the .
/// The value of the .
- protected EvaluationMetric(string name, T? value)
- : base(name)
+ ///
+ /// An optional string that can be used to provide some commentary around the result represented by this
+ /// .
+ ///
+ protected EvaluationMetric(string name, T? value, string? reason = null)
+ : base(name, reason)
{
Value = value;
}
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/NumericMetric.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/NumericMetric.cs
index 35dec86ca63..2a0a07c2193 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/NumericMetric.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/NumericMetric.cs
@@ -20,4 +20,9 @@ namespace Microsoft.Extensions.AI.Evaluation;
///
/// The name of the .
/// The value of the .
-public sealed class NumericMetric(string name, double? value = null) : EvaluationMetric(name, value);
+///
+/// An optional string that can be used to provide some commentary around the result represented by this
+/// .
+///
+public sealed class NumericMetric(string name, double? value = null, string? reason = null)
+ : EvaluationMetric(name, value, reason);
diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/StringMetric.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/StringMetric.cs
index b80c16fbbd8..97fd10921bc 100644
--- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/StringMetric.cs
+++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/StringMetric.cs
@@ -12,4 +12,9 @@ namespace Microsoft.Extensions.AI.Evaluation;
///
/// The name of the .
/// The value of the .
-public sealed class StringMetric(string name, string? value = null) : EvaluationMetric(name, value);
+///
+/// An optional string that can be used to provide some commentary around the result represented by this
+/// .
+///
+public sealed class StringMetric(string name, string? value = null, string? reason = null)
+ : EvaluationMetric(name, value, reason);
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/EndToEndTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/EndToEndTests.cs
index dbfdebc529c..8307dc38591 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/EndToEndTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/EndToEndTests.cs
@@ -33,8 +33,7 @@ static EndToEndTests()
if (Settings.Current.Configured)
{
- var options = new RelevanceTruthAndCompletenessEvaluatorOptions(includeReasoning: true);
- IEvaluator rtcEvaluator = new RelevanceTruthAndCompletenessEvaluator(options);
+ IEvaluator rtcEvaluator = new RelevanceTruthAndCompletenessEvaluator();
IEvaluator coherenceEvaluator = new CoherenceEvaluator();
IEvaluator fluencyEvaluator = new FluencyEvaluator();
@@ -82,9 +81,9 @@ await _reportingConfiguration.CreateScenarioRunAsync(
NumericMetric truth = result.Get(RelevanceTruthAndCompletenessEvaluator.TruthMetricName);
NumericMetric completeness = result.Get(RelevanceTruthAndCompletenessEvaluator.CompletenessMetricName);
- Assert.True(relevance.Value >= 4, string.Format("Relevance - Reasoning: {0}", relevance.Diagnostics.Single().Message));
- Assert.True(truth.Value >= 4, string.Format("Truth - Reasoning: {0}", truth.Diagnostics.Single().Message));
- Assert.True(completeness.Value >= 4, string.Format("Completeness - Reasoning: {0}", completeness.Diagnostics.Single().Message));
+ Assert.True(relevance.Value >= 4, string.Format("Relevance - Reasoning: {0}", relevance.Reason));
+ Assert.True(truth.Value >= 4, string.Format("Truth - Reasoning: {0}", truth.Reason));
+ Assert.True(completeness.Value >= 4, string.Format("Completeness - Reasoning: {0}", completeness.Reason));
NumericMetric coherence = result.Get(CoherenceEvaluator.CoherenceMetricName);
Assert.True(coherence.Value >= 4);
@@ -133,9 +132,9 @@ await _reportingConfiguration.CreateScenarioRunAsync(
NumericMetric truth = result.Get(RelevanceTruthAndCompletenessEvaluator.TruthMetricName);
NumericMetric completeness = result.Get(RelevanceTruthAndCompletenessEvaluator.CompletenessMetricName);
- Assert.True(relevance.Value >= 4, string.Format("Relevance - Reasoning: {0}", relevance.Diagnostics.Single().Message));
- Assert.True(truth.Value >= 4, string.Format("Truth - Reasoning: {0}", truth.Diagnostics.Single().Message));
- Assert.True(completeness.Value >= 4, string.Format("Completeness - Reasoning: {0}", completeness.Diagnostics.Single().Message));
+ Assert.True(relevance.Value >= 4, string.Format("Relevance - Reasoning: {0}", relevance.Reason));
+ Assert.True(truth.Value >= 4, string.Format("Truth - Reasoning: {0}", truth.Reason));
+ Assert.True(completeness.Value >= 4, string.Format("Completeness - Reasoning: {0}", completeness.Reason));
NumericMetric coherence = result.Get(CoherenceEvaluator.CoherenceMetricName);
Assert.True(coherence.Value >= 4);
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/RelevanceTruthAndCompletenessEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/RelevanceTruthAndCompletenessEvaluatorTests.cs
deleted file mode 100644
index 8b479ea57cf..00000000000
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/RelevanceTruthAndCompletenessEvaluatorTests.cs
+++ /dev/null
@@ -1,133 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-
-using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.Extensions.AI.Evaluation.Quality;
-using Microsoft.Extensions.AI.Evaluation.Reporting;
-using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
-using Microsoft.TestUtilities;
-using Xunit;
-
-namespace Microsoft.Extensions.AI.Evaluation.Integration.Tests;
-
-public class RelevanceTruthAndCompletenessEvaluatorTests
-{
- private static readonly ChatOptions _chatOptions;
- private static readonly ReportingConfiguration? _reportingConfigurationWithoutReasoning;
- private static readonly ReportingConfiguration? _reportingConfigurationWithReasoning;
-
- static RelevanceTruthAndCompletenessEvaluatorTests()
- {
- _chatOptions =
- new ChatOptions
- {
- Temperature = 0.0f,
- ResponseFormat = ChatResponseFormat.Text
- };
-
- if (Settings.Current.Configured)
- {
- IEvaluator rtcEvaluatorWithoutReasoning = new RelevanceTruthAndCompletenessEvaluator();
-
- _reportingConfigurationWithoutReasoning =
- DiskBasedReportingConfiguration.Create(
- storageRootPath: Settings.Current.StorageRootPath,
- evaluators: [rtcEvaluatorWithoutReasoning],
- chatConfiguration: Setup.CreateChatConfiguration(),
- executionName: Constants.Version);
-
- var options = new RelevanceTruthAndCompletenessEvaluatorOptions(includeReasoning: true);
- IEvaluator rtcEvaluatorWithReasoning = new RelevanceTruthAndCompletenessEvaluator(options);
-
- _reportingConfigurationWithReasoning =
- DiskBasedReportingConfiguration.Create(
- storageRootPath: Settings.Current.StorageRootPath,
- evaluators: [rtcEvaluatorWithReasoning],
- chatConfiguration: Setup.CreateChatConfiguration(),
- executionName: Constants.Version);
- }
- }
-
- [ConditionalFact]
- public async Task WithoutReasoning()
- {
- SkipIfNotConfigured();
-
- await using ScenarioRun scenarioRun =
- await _reportingConfigurationWithoutReasoning.CreateScenarioRunAsync(
- scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(RelevanceTruthAndCompletenessEvaluatorTests)}.{nameof(WithoutReasoning)}");
-
- IChatClient chatClient = scenarioRun.ChatConfiguration!.ChatClient;
-
- var messages = new List();
- string prompt = @"What is the molecular formula of ammonia?";
- ChatMessage promptMessage = prompt.ToUserMessage();
- messages.Add(promptMessage);
-
- ChatResponse response = await chatClient.GetResponseAsync(messages, _chatOptions);
- ChatMessage responseMessage = response.Messages.Single();
- Assert.NotNull(responseMessage.Text);
-
- EvaluationResult result = await scenarioRun.EvaluateAsync(promptMessage, responseMessage);
-
- Assert.False(result.ContainsDiagnostics(d => d.Severity >= EvaluationDiagnosticSeverity.Informational));
-
- NumericMetric relevance = result.Get(RelevanceTruthAndCompletenessEvaluator.RelevanceMetricName);
- NumericMetric truth = result.Get(RelevanceTruthAndCompletenessEvaluator.TruthMetricName);
- NumericMetric completeness = result.Get(RelevanceTruthAndCompletenessEvaluator.CompletenessMetricName);
-
- Assert.True(relevance.Value >= 4);
- Assert.True(truth.Value >= 4);
- Assert.True(completeness.Value >= 4);
- }
-
- [ConditionalFact]
- public async Task WithReasoning()
- {
- SkipIfNotConfigured();
-
- await using ScenarioRun scenarioRun =
- await _reportingConfigurationWithReasoning.CreateScenarioRunAsync(
- scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(RelevanceTruthAndCompletenessEvaluatorTests)}.{nameof(WithReasoning)}");
-
- IChatClient chatClient = scenarioRun.ChatConfiguration!.ChatClient;
-
- var messages = new List();
- string prompt = @"What is the molecular formula of glucose?";
- ChatMessage promptMessage = prompt.ToUserMessage();
- messages.Add(promptMessage);
-
- ChatResponse response = await chatClient.GetResponseAsync(messages, _chatOptions);
- ChatMessage responseMessage = response.Messages.Single();
- Assert.NotNull(responseMessage.Text);
-
- EvaluationResult result = await scenarioRun.EvaluateAsync(promptMessage, responseMessage);
-
- Assert.True(result.ContainsDiagnostics(d => d.Severity == EvaluationDiagnosticSeverity.Informational));
- Assert.False(result.ContainsDiagnostics(d => d.Severity >= EvaluationDiagnosticSeverity.Warning));
-
- NumericMetric relevance = result.Get(RelevanceTruthAndCompletenessEvaluator.RelevanceMetricName);
- NumericMetric truth = result.Get(RelevanceTruthAndCompletenessEvaluator.TruthMetricName);
- NumericMetric completeness = result.Get(RelevanceTruthAndCompletenessEvaluator.CompletenessMetricName);
-
- Assert.True(relevance.Value >= 4, string.Format("Relevance - Reasoning: {0}", relevance.Diagnostics.Single().Message));
- Assert.True(truth.Value >= 4, string.Format("Truth - Reasoning: {0}", truth.Diagnostics.Single().Message));
- Assert.True(completeness.Value >= 4, string.Format("Completeness - Reasoning: {0}", completeness.Diagnostics.Single().Message));
- }
-
- [MemberNotNull(nameof(_reportingConfigurationWithReasoning))]
- [MemberNotNull(nameof(_reportingConfigurationWithoutReasoning))]
- private static void SkipIfNotConfigured()
- {
- if (!Settings.Current.Configured)
- {
- throw new SkipTestException("Test is not configured");
- }
-
- Assert.NotNull(_reportingConfigurationWithReasoning);
- Assert.NotNull(_reportingConfigurationWithoutReasoning);
- }
-}
diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/ResultsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/ResultsTests.cs
index 338532e5a3d..01241b5760b 100644
--- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/ResultsTests.cs
+++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/ResultsTests.cs
@@ -151,7 +151,8 @@ public async Task ResultWithBooleanMetric()
var metricA = new BooleanMetric("Metric with value false", false);
var metricB = new BooleanMetric("Metric with value true", true);
var metricC = new BooleanMetric("Metric without value");
- evaluator.TestMetrics = [metricA, metricB, metricC];
+ var metricD = new BooleanMetric("Metric with reason", false, reason: "The reason");
+ evaluator.TestMetrics = [metricA, metricB, metricC, metricD];
await using ScenarioRun scenarioRun =
await reportingConfiguration.CreateScenarioRunAsync(
@@ -163,6 +164,7 @@ await reportingConfiguration.CreateScenarioRunAsync(
Assert.Null(metricA.Interpretation);
Assert.Null(metricB.Interpretation);
Assert.Null(metricC.Interpretation);
+ Assert.Null(metricD.Interpretation);
Assert.False(result.ContainsDiagnostics());
}
@@ -176,7 +178,8 @@ public async Task ResultWithBooleanMetricAndInterpretation()
var metricA = new BooleanMetric("Metric with value false", false);
var metricB = new BooleanMetric("Metric with value true", true);
var metricC = new BooleanMetric("Metric without value");
- evaluator.TestMetrics = [metricA, metricB, metricC];
+ var metricD = new BooleanMetric("Metric with reason", false, reason: "The reason");
+ evaluator.TestMetrics = [metricA, metricB, metricC, metricD];
await using ScenarioRun scenarioRun =
await reportingConfiguration.CreateScenarioRunAsync(
@@ -190,6 +193,8 @@ await reportingConfiguration.CreateScenarioRunAsync(
Assert.NotNull(metricB.Interpretation);
Assert.True(metricB.Interpretation!.Failed);
Assert.Null(metricC.Interpretation);
+ Assert.NotNull(metricD.Interpretation);
+ Assert.False(metricD.Interpretation!.Failed);
Assert.False(result.ContainsDiagnostics());
}
@@ -221,9 +226,9 @@ public async Task ResultWithStringMetric()
var metricF = new StringMetric("Measurement System: Nautical", "Nautical");
var metricG = new StringMetric("Measurement System: Astronomical", "Astronomical");
var metricH = new StringMetric("Measurement System: Multiple", "Multiple");
- var metricI = new StringMetric("Measurement System: Blah", "Blah");
- var metricJ = new StringMetric("Measurement System: Empty", "");
- var metricK = new StringMetric("Measurement System: Null");
+ var metricI = new StringMetric("Measurement System: Blah", "Blah", reason: "Value was unexpected");
+ var metricJ = new StringMetric("Measurement System: Empty", "", reason: "Value was empty");
+ var metricK = new StringMetric("Measurement System: Null", reason: "Value was null");
evaluator.TestMetrics =
[metricA, metricB, metricC, metricD, metricE, metricF, metricG, metricH, metricI, metricJ, metricK];
@@ -276,9 +281,9 @@ public async Task ResultWithStringMetricAndInterpretation()
var metricF = new StringMetric("Measurement System: Nautical", "Nautical");
var metricG = new StringMetric("Measurement System: Astronomical", "Astronomical");
var metricH = new StringMetric("Measurement System: Multiple", "Multiple");
- var metricI = new StringMetric("Measurement System: Blah", "Blah");
- var metricJ = new StringMetric("Measurement System: Empty", "");
- var metricK = new StringMetric("Measurement System: Null");
+ var metricI = new StringMetric("Measurement System: Blah", "Blah", reason: "Value was unexpected");
+ var metricJ = new StringMetric("Measurement System: Empty", "", reason: "Value was empty");
+ var metricK = new StringMetric("Measurement System: Null", reason: "Value was null");
evaluator.TestMetrics =
[metricA, metricB, metricC, metricD, metricE, metricF, metricG, metricH, metricI, metricJ, metricK];
@@ -322,14 +327,14 @@ public async Task ResultWithNumericMetrics()
var evaluator = new TestEvaluator();
ReportingConfiguration reportingConfiguration = CreateReportingConfiguration(evaluator);
- var metricA = new NumericMetric("Metric with value 0", 0);
- var metricB = new NumericMetric("Metric with value 1", 1);
- var metricC = new NumericMetric("Metric with value 2", 2);
- var metricD = new NumericMetric("Metric with value 3", 3);
- var metricE = new NumericMetric("Metric with value 4", 4);
- var metricF = new NumericMetric("Metric with value 5", 5);
- var metricG = new NumericMetric("Metric with value 6", 6);
- var metricH = new NumericMetric("Metric with no value");
+ var metricA = new NumericMetric("Metric with value 0", 0, reason: "Because of reason A");
+ var metricB = new NumericMetric("Metric with value 1", 1, reason: "Because of reason B");
+ var metricC = new NumericMetric("Metric with value 2", 2, reason: "Because of reason C");
+ var metricD = new NumericMetric("Metric with value 3", 3, reason: "Because of reason D");
+ var metricE = new NumericMetric("Metric with value 4", 4, reason: "Because of reason E");
+ var metricF = new NumericMetric("Metric with value 5", 5, reason: "Because of reason F");
+ var metricG = new NumericMetric("Metric with value 6", 6, reason: "Because of reason G");
+ var metricH = new NumericMetric("Metric with no value", reason: "Because of reason H");
evaluator.TestMetrics = [metricA, metricB, metricC, metricD, metricE, metricF, metricG, metricH];
await using ScenarioRun scenarioRun =
@@ -357,14 +362,14 @@ public async Task ResultWithNumericMetricsAndInterpretation()
var evaluator = new TestEvaluator();
ReportingConfiguration reportingConfiguration = CreateReportingConfiguration(evaluator);
- var metricA = new NumericMetric("Metric with value 0", 0);
- var metricB = new NumericMetric("Metric with value 1", 1);
- var metricC = new NumericMetric("Metric with value 2", 2);
- var metricD = new NumericMetric("Metric with value 3", 3);
- var metricE = new NumericMetric("Metric with value 4", 4);
- var metricF = new NumericMetric("Metric with value 5", 5);
- var metricG = new NumericMetric("Metric with value 6", 6);
- var metricH = new NumericMetric("Metric with no value");
+ var metricA = new NumericMetric("Metric with value 0", 0, reason: "Because of reason A");
+ var metricB = new NumericMetric("Metric with value 1", 1, reason: "Because of reason B");
+ var metricC = new NumericMetric("Metric with value 2", 2, reason: "Because of reason C");
+ var metricD = new NumericMetric("Metric with value 3", 3, reason: "Because of reason D");
+ var metricE = new NumericMetric("Metric with value 4", 4, reason: "Because of reason E");
+ var metricF = new NumericMetric("Metric with value 5", 5, reason: "Because of reason F");
+ var metricG = new NumericMetric("Metric with value 6", 6, reason: "Because of reason G");
+ var metricH = new NumericMetric("Metric with no value", reason: "Because of reason H");
evaluator.TestMetrics = [metricA, metricB, metricC, metricD, metricE, metricF, metricG, metricH];
await using ScenarioRun scenarioRun =
@@ -405,23 +410,28 @@ public async Task ResultWithDiagnosticsOnUninterpretedMetrics()
metric1.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 1"));
metric1.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 1"));
metric1.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 2"));
+ metric1.Reason = "Reason for metric 1";
var metric2 = new BooleanMetric("Metric with warning and informational diagnostics");
metric2.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 1"));
metric2.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 2"));
metric2.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 2"));
+ metric2.Reason = "Reason for metric 2";
var metric3 = new EvaluationMetric("Metric with error diagnostics only");
metric3.AddDiagnostic(EvaluationDiagnostic.Error("Error 1"));
metric3.AddDiagnostic(EvaluationDiagnostic.Error("Error 2"));
+ metric3.Reason = "Reason for metric 3";
HashSet allowedValues = ["A", "B", "C"];
var metric4 = new StringMetric("Metric with warning diagnostics only");
metric4.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 1"));
metric4.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 2"));
+ metric4.Reason = "Reason for metric 4";
var metric5 = new NumericMetric("Metric with informational diagnostics only");
metric5.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 1"));
+ metric5.Reason = "Reason for metric 5";
evaluator.TestMetrics = [metric1, metric2, metric3, metric4, metric5];
@@ -452,23 +462,28 @@ public async Task ResultWithDiagnosticsOnFailingMetrics()
metric1.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 1"));
metric1.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 1"));
metric1.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 2"));
+ metric1.Reason = "Reason for metric 1";
var metric2 = new BooleanMetric("Metric with warning and informational diagnostics");
metric2.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 1"));
metric2.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 2"));
metric2.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 2"));
+ metric2.Reason = "Reason for metric 2";
var metric3 = new EvaluationMetric("Metric with error diagnostics only");
metric3.AddDiagnostic(EvaluationDiagnostic.Error("Error 1"));
metric3.AddDiagnostic(EvaluationDiagnostic.Error("Error 2"));
+ metric3.Reason = "Reason for metric 3";
HashSet allowedValues = ["A", "B", "C"];
var metric4 = new StringMetric("Metric with warning diagnostics only");
metric4.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 1"));
metric4.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 2"));
+ metric4.Reason = "Reason for metric 4";
var metric5 = new NumericMetric("Metric with informational diagnostics only");
metric5.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 1"));
+ metric5.Reason = "Reason for metric 5";
evaluator.TestMetrics = [metric1, metric2, metric3, metric4, metric5];
@@ -505,23 +520,28 @@ public async Task ResultWithDiagnosticsOnPassingMetrics()
metric1.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 1"));
metric1.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 1"));
metric1.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 2"));
+ metric1.Reason = "Reason for metric 1";
var metric2 = new BooleanMetric("Metric with warning and informational diagnostics", value: true);
metric2.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 1"));
metric2.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 2"));
metric2.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 2"));
+ metric2.Reason = "Reason for metric 2";
var metric3 = new NumericMetric("Metric with error diagnostics only", value: 5);
metric3.AddDiagnostic(EvaluationDiagnostic.Error("Error 1"));
metric3.AddDiagnostic(EvaluationDiagnostic.Error("Error 2"));
+ metric3.Reason = "Reason for metric 3";
HashSet allowedValues = ["A", "B", "C"];
var metric4 = new StringMetric("Metric with warning diagnostics only", value: "A");
metric4.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 1"));
metric4.AddDiagnostic(EvaluationDiagnostic.Warning("Warning 2"));
+ metric4.Reason = "Reason for metric 4";
var metric5 = new NumericMetric("Metric with informational diagnostics only", value: 4);
metric5.AddDiagnostic(EvaluationDiagnostic.Informational("Informational 1"));
+ metric5.Reason = "Reason for metric 5";
evaluator.TestMetrics = [metric1, metric2, metric3, metric4, metric5];