Skip to content

Commit dfa14d7

Browse files
CopilotbaronfelMichaelSimons
authored
Add LLM environment detection telemetry data point with rule-based detection system and multiple environment support (#50663)
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: baronfel <573979+baronfel@users.noreply.github.com> Co-authored-by: Chet Husk <baronfel@users.noreply.github.com> Co-authored-by: Michael Simons <msimons@microsoft.com>
1 parent 73c9295 commit dfa14d7

File tree

7 files changed

+210
-60
lines changed

7 files changed

+210
-60
lines changed

‎src/Cli/dotnet/Telemetry/CIEnvironmentDetectorForTelemetry.cs‎

Lines changed: 33 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,73 +1,50 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4-
#nullable disable
4+
using System;
5+
using System.Linq;
56

67
namespace Microsoft.DotNet.Cli.Telemetry;
78

89
internal class CIEnvironmentDetectorForTelemetry : ICIEnvironmentDetector
910
{
10-
// Systems that provide boolean values only, so we can simply parse and check for true
11-
private static readonly string[] _booleanVariables = [
12-
// Azure Pipelines - https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables#system-variables-devops-services
13-
"TF_BUILD",
14-
// GitHub Actions - https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
15-
"GITHUB_ACTIONS",
16-
// AppVeyor - https://www.appveyor.com/docs/environment-variables/
17-
"APPVEYOR",
18-
// A general-use flag - Many of the major players support this: AzDo, GitHub, GitLab, AppVeyor, Travis CI, CircleCI.
19-
// Given this, we could potentially remove all of these other options?
20-
"CI",
21-
// Travis CI - https://docs.travis-ci.com/user/environment-variables/#default-environment-variables
22-
"TRAVIS",
23-
// CircleCI - https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
24-
"CIRCLECI",
25-
];
26-
27-
// Systems where every variable must be present and not-null before returning true
28-
private static readonly string[][] _allNotNullVariables = [
11+
private static readonly EnvironmentDetectionRule[] _detectionRules = [
12+
// Systems that provide boolean values only, so we can simply parse and check for true
13+
new BooleanEnvironmentRule(
14+
// Azure Pipelines - https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables#system-variables-devops-services
15+
"TF_BUILD",
16+
// GitHub Actions - https://docs.github.com/en/actions/learn-github-actions/environment-variables#default-environment-variables
17+
"GITHUB_ACTIONS",
18+
// AppVeyor - https://www.appveyor.com/docs/environment-variables/
19+
"APPVEYOR",
20+
// A general-use flag - Many of the major players support this: AzDo, GitHub, GitLab, AppVeyor, Travis CI, CircleCI.
21+
// Given this, we could potentially remove all of these other options?
22+
"CI",
23+
// Travis CI - https://docs.travis-ci.com/user/environment-variables/#default-environment-variables
24+
"TRAVIS",
25+
// CircleCI - https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
26+
"CIRCLECI"
27+
),
28+
29+
// Systems where every variable must be present and not-null before returning true
2930
// AWS CodeBuild - https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
30-
["CODEBUILD_BUILD_ID", "AWS_REGION"],
31+
new AllPresentEnvironmentRule("CODEBUILD_BUILD_ID", "AWS_REGION"),
3132
// Jenkins - https://github.com/jenkinsci/jenkins/blob/master/core/src/main/resources/jenkins/model/CoreEnvironmentContributor/buildEnv.groovy
32-
["BUILD_ID", "BUILD_URL"],
33+
new AllPresentEnvironmentRule("BUILD_ID", "BUILD_URL"),
3334
// Google Cloud Build - https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values#using_default_substitutions
34-
["BUILD_ID", "PROJECT_ID"]
35-
];
36-
37-
// Systems where the variable must be present and not-null
38-
private static readonly string[] _ifNonNullVariables = [
39-
// TeamCity - https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#Predefined+Server+Build+Parameters
40-
"TEAMCITY_VERSION",
41-
// JetBrains Space - https://www.jetbrains.com/help/space/automation-environment-variables.html#general
42-
"JB_SPACE_API_URL"
35+
new AllPresentEnvironmentRule("BUILD_ID", "PROJECT_ID"),
36+
37+
// Systems where the variable must be present and not-null
38+
new AnyPresentEnvironmentRule(
39+
// TeamCity - https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#Predefined+Server+Build+Parameters
40+
"TEAMCITY_VERSION",
41+
// JetBrains Space - https://www.jetbrains.com/help/space/automation-environment-variables.html#general
42+
"JB_SPACE_API_URL"
43+
)
4344
];
4445

4546
public bool IsCIEnvironment()
4647
{
47-
foreach (var booleanVariable in _booleanVariables)
48-
{
49-
if (bool.TryParse(Environment.GetEnvironmentVariable(booleanVariable), out bool envVar) && envVar)
50-
{
51-
return true;
52-
}
53-
}
54-
55-
foreach (var variables in _allNotNullVariables)
56-
{
57-
if (variables.All((variable) => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable))))
58-
{
59-
return true;
60-
}
61-
}
62-
63-
foreach (var variable in _ifNonNullVariables)
64-
{
65-
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable)))
66-
{
67-
return true;
68-
}
69-
}
70-
71-
return false;
48+
return _detectionRules.Any(rule => rule.IsMatch());
7249
}
7350
}

‎src/Cli/dotnet/Telemetry/EnvironmentDetectionRule.cs‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
8+
namespace Microsoft.DotNet.Cli.Telemetry;
9+
10+
/// <summary>
11+
/// Base class for environment detection rules that can be evaluated against environment variables.
12+
/// </summary>
13+
internal abstract class EnvironmentDetectionRule
14+
{
15+
/// <summary>
16+
/// Evaluates the rule against the current environment.
17+
/// </summary>
18+
/// <returns>True if the rule matches the current environment; otherwise, false.</returns>
19+
public abstract bool IsMatch();
20+
}
21+
22+
/// <summary>
23+
/// Rule that matches when any of the specified environment variables is set to "true".
24+
/// </summary>
25+
internal class BooleanEnvironmentRule : EnvironmentDetectionRule
26+
{
27+
private readonly string[] _variables;
28+
29+
public BooleanEnvironmentRule(params string[] variables)
30+
{
31+
_variables = variables ?? throw new ArgumentNullException(nameof(variables));
32+
}
33+
34+
public override bool IsMatch()
35+
{
36+
return _variables.Any(variable =>
37+
bool.TryParse(Environment.GetEnvironmentVariable(variable), out bool value) && value);
38+
}
39+
}
40+
41+
/// <summary>
42+
/// Rule that matches when all specified environment variables are present and not null/empty.
43+
/// </summary>
44+
internal class AllPresentEnvironmentRule : EnvironmentDetectionRule
45+
{
46+
private readonly string[] _variables;
47+
48+
public AllPresentEnvironmentRule(params string[] variables)
49+
{
50+
_variables = variables ?? throw new ArgumentNullException(nameof(variables));
51+
}
52+
53+
public override bool IsMatch()
54+
{
55+
return _variables.All(variable => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable)));
56+
}
57+
}
58+
59+
/// <summary>
60+
/// Rule that matches when any of the specified environment variables is present and not null/empty.
61+
/// </summary>
62+
internal class AnyPresentEnvironmentRule : EnvironmentDetectionRule
63+
{
64+
private readonly string[] _variables;
65+
66+
public AnyPresentEnvironmentRule(params string[] variables)
67+
{
68+
_variables = variables ?? throw new ArgumentNullException(nameof(variables));
69+
}
70+
71+
public override bool IsMatch()
72+
{
73+
return _variables.Any(variable => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable)));
74+
}
75+
}
76+
77+
/// <summary>
78+
/// Rule that matches when any of the specified environment variables is present and not null/empty,
79+
/// and returns the associated result value.
80+
/// </summary>
81+
/// <typeparam name="T">The type of the result value.</typeparam>
82+
internal class EnvironmentDetectionRuleWithResult<T> where T : class
83+
{
84+
private readonly string[] _variables;
85+
private readonly T _result;
86+
87+
public EnvironmentDetectionRuleWithResult(T result, params string[] variables)
88+
{
89+
_variables = variables ?? throw new ArgumentNullException(nameof(variables));
90+
_result = result ?? throw new ArgumentNullException(nameof(result));
91+
}
92+
93+
/// <summary>
94+
/// Evaluates the rule and returns the result if matched.
95+
/// </summary>
96+
/// <returns>The result value if the rule matches; otherwise, null.</returns>
97+
public T? GetResult()
98+
{
99+
return _variables.Any(variable => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(variable)))
100+
? _result
101+
: null;
102+
}
103+
}

‎src/Cli/dotnet/Telemetry/ICIEnvironmentDetector.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
1-
// Licensed to the .NET Foundation under one or more agreements.
1+
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4-
#nullable disable
5-
64
namespace Microsoft.DotNet.Cli.Telemetry;
75

86
internal interface ICIEnvironmentDetector

‎src/Cli/dotnet/Telemetry/ILLMEnvironmentDetector.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
namespace Microsoft.DotNet.Cli.Telemetry;
5+
6+
internal interface ILLMEnvironmentDetector
7+
{
8+
string? GetLLMEnvironment();
9+
}

‎src/Cli/dotnet/Telemetry/LLMEnvironmentDetectorForTelemetry.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Linq;
6+
7+
namespace Microsoft.DotNet.Cli.Telemetry;
8+
9+
internal class LLMEnvironmentDetectorForTelemetry : ILLMEnvironmentDetector
10+
{
11+
private static readonly EnvironmentDetectionRuleWithResult<string>[] _detectionRules = [
12+
// Claude Code
13+
new EnvironmentDetectionRuleWithResult<string>("claude", "CLAUDECODE"),
14+
// Cursor AI
15+
new EnvironmentDetectionRuleWithResult<string>("cursor", "CURSOR_EDITOR")
16+
];
17+
18+
public string? GetLLMEnvironment()
19+
{
20+
var results = _detectionRules.Select(r => r.GetResult()).Where(r => r != null).ToArray();
21+
return results.Length > 0 ? string.Join(", ", results) : null;
22+
}
23+
}

‎src/Cli/dotnet/Telemetry/TelemetryCommonProperties.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,12 @@ internal class TelemetryCommonProperties(
1818
Func<string> getDeviceId = null,
1919
IDockerContainerDetector dockerContainerDetector = null,
2020
IUserLevelCacheWriter userLevelCacheWriter = null,
21-
ICIEnvironmentDetector ciEnvironmentDetector = null)
21+
ICIEnvironmentDetector ciEnvironmentDetector = null,
22+
ILLMEnvironmentDetector llmEnvironmentDetector = null)
2223
{
2324
private readonly IDockerContainerDetector _dockerContainerDetector = dockerContainerDetector ?? new DockerContainerDetectorForTelemetry();
2425
private readonly ICIEnvironmentDetector _ciEnvironmentDetector = ciEnvironmentDetector ?? new CIEnvironmentDetectorForTelemetry();
26+
private readonly ILLMEnvironmentDetector _llmEnvironmentDetector = llmEnvironmentDetector ?? new LLMEnvironmentDetectorForTelemetry();
2527
private readonly Func<string> _getCurrentDirectory = getCurrentDirectory ?? Directory.GetCurrentDirectory;
2628
private readonly Func<string, string> _hasher = hasher ?? Sha256Hasher.Hash;
2729
private readonly Func<string> _getMACAddress = getMACAddress ?? MacAddressGetter.GetMacAddress;
@@ -47,6 +49,7 @@ internal class TelemetryCommonProperties(
4749
private const string SessionId = "SessionId";
4850

4951
private const string CI = "Continuous Integration";
52+
private const string LLM = "llm";
5053

5154
private const string TelemetryProfileEnvironmentVariable = "DOTNET_CLI_TELEMETRY_PROFILE";
5255
private const string CannotFindMacAddress = "Unknown";
@@ -67,6 +70,7 @@ public FrozenDictionary<string, string> GetTelemetryCommonProperties(string curr
6770
{TelemetryProfile, Environment.GetEnvironmentVariable(TelemetryProfileEnvironmentVariable)},
6871
{DockerContainer, _userLevelCacheWriter.RunWithCache(IsDockerContainerCacheKey, () => _dockerContainerDetector.IsDockerContainer().ToString("G") )},
6972
{CI, _ciEnvironmentDetector.IsCIEnvironment().ToString() },
73+
{LLM, _llmEnvironmentDetector.GetLLMEnvironment() },
7074
{CurrentPathHash, _hasher(_getCurrentDirectory())},
7175
{MachineIdOld, _userLevelCacheWriter.RunWithCache(MachineIdCacheKey, GetMachineId)},
7276
// we don't want to recalcuate a new id for every new SDK version. Reuse the same path across versions.

‎test/dotnet.Tests/TelemetryCommonPropertiesTests.cs‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,13 @@ public void TelemetryCommonPropertiesShouldContainLibcReleaseAndVersion()
163163
}
164164
}
165165

166+
[Fact]
167+
public void TelemetryCommonPropertiesShouldReturnIsLLMDetection()
168+
{
169+
var unitUnderTest = new TelemetryCommonProperties(getMACAddress: () => null, userLevelCacheWriter: new NothingCache());
170+
unitUnderTest.GetTelemetryCommonProperties("dummySessionId")["llm"].Should().BeOneOf("claude", null);
171+
}
172+
166173
[Theory]
167174
[MemberData(nameof(CITelemetryTestCases))]
168175
public void CanDetectCIStatusForEnvVars(Dictionary<string, string> envVars, bool expected)
@@ -184,6 +191,27 @@ public void CanDetectCIStatusForEnvVars(Dictionary<string, string> envVars, bool
184191
}
185192
}
186193

194+
[Theory]
195+
[MemberData(nameof(LLMTelemetryTestCases))]
196+
public void CanDetectLLMStatusForEnvVars(Dictionary<string, string> envVars, string expected)
197+
{
198+
try
199+
{
200+
foreach (var (key, value) in envVars)
201+
{
202+
Environment.SetEnvironmentVariable(key, value);
203+
}
204+
new LLMEnvironmentDetectorForTelemetry().GetLLMEnvironment().Should().Be(expected);
205+
}
206+
finally
207+
{
208+
foreach (var (key, value) in envVars)
209+
{
210+
Environment.SetEnvironmentVariable(key, null);
211+
}
212+
}
213+
}
214+
187215
[Theory]
188216
[InlineData("dummySessionId")]
189217
[InlineData(null)]
@@ -196,6 +224,14 @@ public void TelemetryCommonPropertiesShouldContainSessionId(string sessionId)
196224
commonProperties["SessionId"].Should().Be(sessionId);
197225
}
198226

227+
228+
public static IEnumerable<object[]> LLMTelemetryTestCases => new List<object[]>{
229+
new object[] { new Dictionary<string, string> { { "CLAUDECODE", "1" } }, "claude" },
230+
new object[] { new Dictionary<string, string> { { "CURSOR_EDITOR", "1" } }, "cursor" },
231+
new object[] { new Dictionary<string, string> { { "CLAUDECODE", "1" }, { "CURSOR_EDITOR", "1" } }, "claude, cursor" },
232+
new object[] { new Dictionary<string, string>(), null },
233+
};
234+
199235
public static IEnumerable<object[]> CITelemetryTestCases => new List<object[]>{
200236
new object[] { new Dictionary<string, string> { { "TF_BUILD", "true" } }, true },
201237
new object[] { new Dictionary<string, string> { { "GITHUB_ACTIONS", "true" } }, true },

0 commit comments

Comments
 (0)