Skip to content

Commit 513ea38

Browse files
shyamnamboodiripadshyamnamboodiripad
committed
Introduce telemetry in aieval dotnet tool
1 parent 476844d commit 513ea38

38 files changed

+1464
-259
lines changed

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs‎

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,46 +2,66 @@
22
// The .NET Foundation licenses this file to you under the MIT license.
33

44
using System;
5+
using System.Collections.Generic;
56
using System.IO;
67
using System.Threading;
78
using System.Threading.Tasks;
89
using Azure.Identity;
910
using Azure.Storage.Files.DataLake;
11+
using Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
1012
using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
1113
using Microsoft.Extensions.AI.Evaluation.Reporting;
1214
using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
1315
using Microsoft.Extensions.Logging;
16+
using static Microsoft.Extensions.AI.Evaluation.Console.Telemetry.TelemetryConstants;
1417

1518
namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
1619

17-
internal sealed class CleanCacheCommand(ILogger logger)
20+
internal sealed class CleanCacheCommand(ILogger logger, TelemetryHelper telemetryHelper)
1821
{
19-
internal async Task<int> InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default)
22+
internal async Task<int> InvokeAsync(
23+
DirectoryInfo? storageRootDir,
24+
Uri? endpointUri,
25+
CancellationToken cancellationToken = default)
2026
{
21-
IEvaluationResponseCacheProvider cacheProvider;
22-
23-
if (storageRootDir is not null)
24-
{
25-
string storageRootPath = storageRootDir.FullName;
26-
logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
27-
logger.LogInformation("Deleting expired cache entries...");
28-
29-
cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
30-
}
31-
else if (endpointUri is not null)
32-
{
33-
logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
34-
35-
var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
36-
cacheProvider = new AzureStorageResponseCacheProvider(fsClient);
37-
}
38-
else
39-
{
40-
throw new InvalidOperationException("Either --path or --endpoint must be specified");
41-
}
27+
var telemetryProperties = new Dictionary<string, string>();
4228

4329
await logger.ExecuteWithCatchAsync(
44-
() => cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken)).ConfigureAwait(false);
30+
operation: () =>
31+
telemetryHelper.ReportOperationAsync(
32+
operationName: EventNames.CleanCacheCommand,
33+
operation: () =>
34+
{
35+
IEvaluationResponseCacheProvider cacheProvider;
36+
37+
if (storageRootDir is not null)
38+
{
39+
string storageRootPath = storageRootDir.FullName;
40+
logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
41+
logger.LogInformation("Deleting expired cache entries...");
42+
43+
cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
44+
45+
telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeDisk;
46+
}
47+
else if (endpointUri is not null)
48+
{
49+
logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
50+
51+
var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
52+
cacheProvider = new AzureStorageResponseCacheProvider(fsClient);
53+
54+
telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeAzure;
55+
}
56+
else
57+
{
58+
throw new InvalidOperationException("Either --path or --endpoint must be specified");
59+
}
60+
61+
return cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken);
62+
},
63+
properties: telemetryProperties,
64+
logger: logger)).ConfigureAwait(false);
4565

4666
return 0;
4767
}

‎src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs‎

Lines changed: 67 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -8,76 +8,98 @@
88
using System.Threading.Tasks;
99
using Azure.Identity;
1010
using Azure.Storage.Files.DataLake;
11+
using Microsoft.Extensions.AI.Evaluation.Console.Telemetry;
1112
using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
1213
using Microsoft.Extensions.AI.Evaluation.Reporting;
1314
using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
1415
using Microsoft.Extensions.Logging;
16+
using static Microsoft.Extensions.AI.Evaluation.Console.Telemetry.TelemetryConstants;
1517

1618
namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
1719

18-
internal sealed class CleanResultsCommand(ILogger logger)
20+
internal sealed class CleanResultsCommand(ILogger logger, TelemetryHelper telemetryHelper)
1921
{
2022
internal async Task<int> InvokeAsync(
2123
DirectoryInfo? storageRootDir,
2224
Uri? endpointUri,
2325
int lastN,
2426
CancellationToken cancellationToken = default)
2527
{
26-
IEvaluationResultStore resultStore;
27-
28-
if (storageRootDir is not null)
29-
{
30-
string storageRootPath = storageRootDir.FullName;
31-
logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
28+
var telemetryProperties =
29+
new Dictionary<string, string>
30+
{
31+
[PropertyNames.LastN] = lastN.ToTelemetryPropertyValue()
32+
};
3233

33-
resultStore = new DiskBasedResultStore(storageRootPath);
34-
}
35-
else if (endpointUri is not null)
36-
{
37-
logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
34+
await logger.ExecuteWithCatchAsync(
35+
operation: () =>
36+
telemetryHelper.ReportOperationAsync(
37+
operationName: EventNames.CleanResultsCommand,
38+
operation: async ValueTask () =>
39+
{
40+
IEvaluationResultStore resultStore;
3841

39-
var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
40-
resultStore = new AzureStorageResultStore(fsClient);
41-
}
42-
else
43-
{
44-
throw new InvalidOperationException("Either --path or --endpoint must be specified");
45-
}
42+
if (storageRootDir is not null)
43+
{
44+
string storageRootPath = storageRootDir.FullName;
45+
logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
4646

47-
await logger.ExecuteWithCatchAsync(
48-
async ValueTask () =>
49-
{
50-
if (lastN is 0)
51-
{
52-
logger.LogInformation("Deleting all results...");
47+
resultStore = new DiskBasedResultStore(storageRootPath);
5348

54-
await resultStore.DeleteResultsAsync(cancellationToken: cancellationToken).ConfigureAwait(false);
55-
}
56-
else
57-
{
58-
logger.LogInformation("Deleting all results except the {lastN} most recent ones...", lastN);
49+
telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeDisk;
50+
}
51+
else if (endpointUri is not null)
52+
{
53+
logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
5954

60-
HashSet<string> toPreserve = [];
55+
var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
56+
resultStore = new AzureStorageResultStore(fsClient);
6157

62-
await foreach (string executionName in
63-
resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false))
64-
{
65-
_ = toPreserve.Add(executionName);
66-
}
58+
telemetryProperties[PropertyNames.StorageType] = PropertyValues.StorageTypeAzure;
59+
}
60+
else
61+
{
62+
throw new InvalidOperationException("Either --path or --endpoint must be specified");
63+
}
6764

68-
await foreach (string executionName in
69-
resultStore.GetLatestExecutionNamesAsync(
70-
cancellationToken: cancellationToken).ConfigureAwait(false))
71-
{
72-
if (!toPreserve.Contains(executionName))
65+
if (lastN is 0)
7366
{
67+
logger.LogInformation("Deleting all results...");
68+
7469
await resultStore.DeleteResultsAsync(
75-
executionName,
7670
cancellationToken: cancellationToken).ConfigureAwait(false);
7771
}
78-
}
79-
}
80-
}).ConfigureAwait(false);
72+
else
73+
{
74+
logger.LogInformation(
75+
"Deleting all results except the {lastN} most recent ones...",
76+
lastN);
77+
78+
HashSet<string> toPreserve = [];
79+
80+
await foreach (string executionName in
81+
resultStore.GetLatestExecutionNamesAsync(
82+
lastN,
83+
cancellationToken).ConfigureAwait(false))
84+
{
85+
_ = toPreserve.Add(executionName);
86+
}
87+
88+
await foreach (string executionName in
89+
resultStore.GetLatestExecutionNamesAsync(
90+
cancellationToken: cancellationToken).ConfigureAwait(false))
91+
{
92+
if (!toPreserve.Contains(executionName))
93+
{
94+
await resultStore.DeleteResultsAsync(
95+
executionName,
96+
cancellationToken: cancellationToken).ConfigureAwait(false);
97+
}
98+
}
99+
}
100+
},
101+
properties: telemetryProperties,
102+
logger: logger)).ConfigureAwait(false);
81103

82104
return 0;
83105
}

0 commit comments

Comments
 (0)