Skip to content

Commit c3d4094

Browse files
ViktorHoferViktorHofer
committed
Make SmokeTests use repo infrastructure
This allows the test project to just be invoked with `dotnet test` or `dotnet build /t:Test` after the VMR is built. 1. Stop depending on environment variables for state instead use RuntimeConfiguration values same as UnifiedBuild test project. Allow facts and theories to be conditioned based on boolean values instead of env vars. 2. Update the README with the new msbuild property switch names 3. Delete dead-code and the checked-in .sln file
1 parent e6a2c0f commit c3d4094

24 files changed

+220
-342
lines changed

‎src/SourceBuild/content/.gitignore

-3
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,4 @@
33
/artifacts
44
/prereqs/packages
55
/src/nuget-client/NuGet.config
6-
/test/Microsoft.DotNet.SourceBuild.SmokeTests/bin
7-
/test/Microsoft.DotNet.SourceBuild.SmokeTests/obj
8-
/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults
96
*.binlog

‎src/SourceBuild/content/Directory.Build.props

+1
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@
213213
<PoisonMarkerFile>.prebuilt.xml</PoisonMarkerFile>
214214
<PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>
215215
<PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
216+
<PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
216217
</PropertyGroup>
217218

218219
<Import Project="$(GitInfoAllRepoPropsFile)" />

‎src/SourceBuild/content/Directory.Packages.props

+2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
<PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="$(MicrosoftExtensionsFileSystemGlobbingVersion)" />
1717
<PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
1818
<PackageVersion Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" />
19+
<!-- External dependencies -->
20+
<PackageVersion Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
1921
</ItemGroup>
2022

2123
</Project>

‎src/SourceBuild/content/build.proj

+5
Original file line numberDiff line numberDiff line change
@@ -52,4 +52,9 @@
5252
<Message Importance="high" Text="Shipping assets are located in '$(ArtifactsAssetsDir)'." />
5353
</Target>
5454

55+
<!-- TODO: Condition on Tests=true property when the test script entry points are added. -->
56+
<Target Name="RunSmokeTests">
57+
<MSBuild Projects="test\Microsoft.DotNet.SourceBuild.SmokeTests\Microsoft.DotNet.SourceBuild.SmokeTests.csproj" />
58+
</Target>
59+
5560
</Project>

‎src/SourceBuild/content/eng/Versions.props

+6-4
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,15 @@
2525
-->
2626
<PrivateSourceBuiltSdkVersion>9.0.100-preview.4.24179.1</PrivateSourceBuiltSdkVersion>
2727
<PrivateSourceBuiltArtifactsVersion>9.0.100-preview.4.24179.1</PrivateSourceBuiltArtifactsVersion>
28-
<!-- msbuild -->
28+
<!-- command-line-api dependencies -->
29+
<SystemCommandLineVersion>2.0.0-beta4.24126.1</SystemCommandLineVersion>
30+
<!-- msbuild dependencies -->
2931
<MicrosoftBuildVersion>17.8.3</MicrosoftBuildVersion>
30-
<!-- runtime -->
32+
<!-- runtime dependencies -->
3133
<MicrosoftExtensionsFileSystemGlobbingVersion>9.0.0-preview.2.24128.5</MicrosoftExtensionsFileSystemGlobbingVersion>
3234
<MicrosoftExtensionsLoggingConsoleVersion>9.0.0-preview.2.24128.5</MicrosoftExtensionsLoggingConsoleVersion>
3335
<MicrosoftExtensionsLoggingVersion>9.0.0-preview.2.24128.5</MicrosoftExtensionsLoggingVersion>
34-
<!-- command-line-api -->
35-
<SystemCommandLineVersion>2.0.0-beta4.24126.1</SystemCommandLineVersion>
36+
<!-- external dependencies -->
37+
<NewtonsoftJsonVersion>13.0.3</NewtonsoftJsonVersion>
3638
</PropertyGroup>
3739
</Project>

‎src/SourceBuild/content/eng/build.sourcebuild.targets

-72
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,5 @@
11
<Project>
22

3-
<PropertyGroup>
4-
<SmokeTestsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'test', 'Microsoft.DotNet.SourceBuild.SmokeTests'))</SmokeTestsDir>
5-
</PropertyGroup>
6-
7-
<PropertyGroup Condition="'$(EnablePoison)' == 'true'">
8-
<PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
9-
</PropertyGroup>
10-
113
<!-- After building, generate a prebuilt usage report. -->
124
<Target Name="ReportPrebuiltUsage"
135
AfterTargets="Build"
@@ -121,70 +113,6 @@
121113
</Touch>
122114
</Target>
123115

124-
<Target Name="RunSmokeTest">
125-
<ItemGroup>
126-
<SdkTarballItem Include="$(ArtifactsAssetsDir)dotnet-sdk*$(ArchiveExtension)" />
127-
<SourceBuiltArtifactsItem Include="$(ArtifactsAssetsDir)$(SourceBuiltArtifactsTarballName).*$(ArchiveExtension)" />
128-
</ItemGroup>
129-
130-
<PropertyGroup>
131-
<CustomTestEnvVars Condition="'$(EnablePoison)' == 'true'">SMOKE_TESTS_POISON_REPORT_PATH=$(PoisonUsageReportFile);</CustomTestEnvVars>
132-
<SdkTarballPath>%(SdkTarballItem.Identity)</SdkTarballPath>
133-
<SourceBuiltArtifactsPath>%(SourceBuiltArtifactsItem.Identity)</SourceBuiltArtifactsPath>
134-
<SmokeTestConsoleVerbosity Condition="'$(SmokeTestConsoleVerbosity)' == ''">normal</SmokeTestConsoleVerbosity>
135-
</PropertyGroup>
136-
137-
<!-- Multiple loggers are specified so that results are captured in trx and pipelines can fail with AzDO pipeline warnings -->
138-
<!-- Workaround https://github.com/dotnet/source-build/issues/4003 by disabling VSTestUseMSBuildOutput -->
139-
<Exec Command="$(DotnetTool) test $(SmokeTestsDir) --logger:trx --logger:'console;verbosity=$(SmokeTestConsoleVerbosity)' -c $(Configuration) -p:VSTestUseMSBuildOutput=false"
140-
IgnoreStandardErrorWarningFormat="true"
141-
EnvironmentVariables="
142-
SMOKE_TESTS_SDK_TARBALL_PATH=$(SdkTarballPath);
143-
SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH=$(SourceBuiltArtifactsPath);
144-
SMOKE_TESTS_TARGET_RID=$(TargetRid);
145-
SMOKE_TESTS_PORTABLE_RID=$(PortableRid);
146-
SMOKE_TESTS_CUSTOM_PACKAGES_PATH=$(CustomSourceBuiltPackagesPath);
147-
$(CustomTestEnvVars)" />
148-
</Target>
149-
150-
<Target Name="CreateSmokeTestPrereqsTarball"
151-
AfterTargets="RunSmokeTest"
152-
Condition="'$(SkipSmokeTestPrereqsTarballCreation)' != 'true'"
153-
DependsOnTargets="
154-
CheckIfCreateSmokeTestPrereqsExistToPack;
155-
CreateSmokeTestPrereqsTarballIfPrereqsExist"/>
156-
157-
<Target Name="CheckIfCreateSmokeTestPrereqsExistToPack">
158-
<PropertyGroup>
159-
<SmokeTestsArtifactsDir>$(SmokeTestsDir)bin/$(Configuration)/$(NetCurrent)/</SmokeTestsArtifactsDir>
160-
<SmokeTestsPackagesDir>$(SmokeTestsArtifactsDir)packages/</SmokeTestsPackagesDir>
161-
</PropertyGroup>
162-
163-
<ItemGroup>
164-
<SmokeTestsPrereqs Include="$(SmokeTestsPackagesDir)**/*.nupkg" />
165-
</ItemGroup>
166-
167-
<Message Text="Found @(SmokeTestsPrereqs->Count()) prereqs in '$(SmokeTestsPackagesDir)'." Importance="High" />
168-
</Target>
169-
170-
<Target Name="CreateSmokeTestPrereqsTarballIfPrereqsExist"
171-
DependsOnTargets="DetermineSourceBuiltSdkVersion"
172-
Condition="'@(SmokeTestsPrereqs->Count())' != '0'">
173-
<PropertyGroup>
174-
<SmokeTestPrereqsTarball>$(ArtifactsAssetsDir)dotnet-smoke-test-prereqs.$(SourceBuiltSdkVersion).$(TargetRid)$(ArchiveExtension)</SmokeTestPrereqsTarball>
175-
<SmokeTestsPrereqPackagesDir>$(SmokeTestsArtifactsDir)prereq-packages/</SmokeTestsPrereqPackagesDir>
176-
</PropertyGroup>
177-
178-
<Copy SourceFiles="@(SmokeTestsPrereqs)"
179-
DestinationFolder="$(SmokeTestsPrereqPackagesDir)" />
180-
181-
<MakeDir Directories="$([System.IO.Path]::GetDirectoryName('$(SmokeTestPrereqsTarball)'))" />
182-
<Exec Command="tar --numeric-owner -czf $(SmokeTestPrereqsTarball) ."
183-
WorkingDirectory="$(SmokeTestsPrereqPackagesDir)"/>
184-
185-
<Message Importance="High" Text="Packaged smoke-test prereqs in '$(SmokeTestPrereqsTarball)'" />
186-
</Target>
187-
188116
<Target Name="CreatePrebuiltsTarball"
189117
AfterTargets="Build"
190118
DependsOnTargets="

‎src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ArtifactsSizeTest.cs

+1-5
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,9 @@ public ArtifactsSizeTest(ITestOutputHelper outputHelper) : base(outputHelper)
4242
}
4343
}
4444

45-
[SkippableFact(Config.IncludeArtifactsSizeEnv, skipOnFalseEnv: true)]
45+
[ConditionalFact(typeof(Config), nameOf(IncludeArtifactsSize))]
4646
public void CompareArtifactsToBaseline()
4747
{
48-
Utilities.ValidateNotNullOrWhiteSpace(Config.SourceBuiltArtifactsPath, Config.SourceBuiltArtifactsPathEnv);
49-
Utilities.ValidateNotNullOrWhiteSpace(Config.SdkTarballPath, Config.SdkTarballPathEnv);
50-
Utilities.ValidateNotNullOrWhiteSpace(Config.TargetRid, Config.TargetRidEnv);
51-
5248
var tarEntries = ProcessSdkAndArtifactsTarballs();
5349
ScanForDifferences(tarEntries);
5450
UpdateBaselineFile();

‎src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalFactAttribute.cs

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System.Reflection;
6+
using Xunit;
7+
8+
namespace Microsoft.DotNet.SourceBuild.SmokeTests;
9+
10+
/// <summary>
11+
/// A Fact that conditionally runs based on a type's boolean property member
12+
/// </summary>
13+
internal sealed class ConditionalFactAttribute : FactAttribute
14+
{
15+
public ConditionalFactAttribute(Type calleeType, string memberName, string? reason = null)
16+
{
17+
EvaluateSkip(calleeType, memberName, reason, (skip) => Skip = skip);
18+
}
19+
20+
internal static void EvaluateSkip(Type calleeType, string memberName, string? reason = null, Action<string> setSkip)
21+
{
22+
TypeInfo typeInfo = calleeType.GetTypeInfo();
23+
bool shouldRun = (bool?)typeInfo.GetProperty(memberName)?.GetValue(null) ?? false;
24+
if (!shouldRun)
25+
{
26+
setSkip(reason ?? "Skipped");
27+
}
28+
}
29+
}

‎src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ConditionalTheoryAttribute.cs

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using Xunit;
6+
7+
namespace Microsoft.DotNet.SourceBuild.SmokeTests;
8+
9+
/// <summary>
10+
/// A Theory that conditionally runs based on a type's boolean property member.
11+
/// </summary>
12+
internal sealed class ConditionalTheoryAttribute : TheoryAttribute
13+
{
14+
public ConditionalTheoryAttribute(Type calleeType, string memberName, string? reason = null)
15+
{
16+
ConditionalFactAttribute.EvaluateSkip(calleeType, memberName, reason, (skip) => Skip = skip);
17+
}
18+
}

‎src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs

+21-36
Original file line numberDiff line numberDiff line change
@@ -9,43 +9,28 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
99

1010
internal static class Config
1111
{
12-
public const string DotNetDirectoryEnv = "SMOKE_TESTS_DOTNET_DIR";
13-
public const string ExcludeOmniSharpEnv = "SMOKE_TESTS_EXCLUDE_OMNISHARP";
14-
public const string IncludeArtifactsSizeEnv = "SMOKE_TESTS_INCLUDE_ARTIFACTSSIZE";
15-
public const string MsftSdkTarballPathEnv = "SMOKE_TESTS_MSFT_SDK_TARBALL_PATH";
16-
public const string PoisonReportPathEnv = "SMOKE_TESTS_POISON_REPORT_PATH";
17-
public const string PortableRidEnv = "SMOKE_TESTS_PORTABLE_RID";
18-
public const string PrereqsPathEnv = "SMOKE_TESTS_PREREQS_PATH";
19-
public const string CustomPackagesPathEnv = "SMOKE_TESTS_CUSTOM_PACKAGES_PATH";
20-
public const string SdkTarballPathEnv = "SMOKE_TESTS_SDK_TARBALL_PATH";
21-
public const string SourceBuiltArtifactsPathEnv = "SMOKE_TESTS_SOURCEBUILT_ARTIFACTS_PATH";
22-
public const string TargetRidEnv = "SMOKE_TESTS_TARGET_RID";
23-
public const string WarnSdkContentDiffsEnv = "SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS";
24-
public const string WarnLicenseScanDiffsEnv = "SMOKE_TESTS_WARN_LICENSE_SCAN_DIFFS";
25-
public const string RunningInCIEnv = "SMOKE_TESTS_RUNNING_IN_CI";
26-
public const string LicenseScanPathEnv = "SMOKE_TESTS_LICENSE_SCAN_PATH";
12+
const string ConfigSwitchPrefix = "Microsoft.DotNet.SourceBuild.SmokeTests.";
2713

28-
public static string DotNetDirectory { get; } =
29-
Environment.GetEnvironmentVariable(DotNetDirectoryEnv) ?? Path.Combine(Directory.GetCurrentDirectory(), ".dotnet");
30-
public static string? MsftSdkTarballPath { get; } = Environment.GetEnvironmentVariable(MsftSdkTarballPathEnv);
31-
public static string? PoisonReportPath { get; } = Environment.GetEnvironmentVariable(PoisonReportPathEnv);
32-
public static string PortableRid { get; } = Environment.GetEnvironmentVariable(PortableRidEnv) ??
33-
throw new InvalidOperationException($"'{Config.PortableRidEnv}' must be specified");
34-
public static string? PrereqsPath { get; } = Environment.GetEnvironmentVariable(PrereqsPathEnv);
35-
public static string? CustomPackagesPath { get; } = Environment.GetEnvironmentVariable(CustomPackagesPathEnv);
36-
public static string? SdkTarballPath { get; } = Environment.GetEnvironmentVariable(SdkTarballPathEnv);
37-
public static string? SourceBuiltArtifactsPath { get; } = Environment.GetEnvironmentVariable(SourceBuiltArtifactsPathEnv);
38-
public static string TargetRid { get; } = Environment.GetEnvironmentVariable(TargetRidEnv) ??
39-
throw new InvalidOperationException($"'{Config.TargetRidEnv}' must be specified");
40-
public static string TargetArchitecture { get; } = TargetRid.Split('-')[1];
41-
public static bool WarnOnSdkContentDiffs { get; } =
42-
bool.TryParse(Environment.GetEnvironmentVariable(WarnSdkContentDiffsEnv), out bool warnOnSdkContentDiffs) && warnOnSdkContentDiffs;
43-
public static bool WarnOnLicenseScanDiffs { get; } =
44-
bool.TryParse(Environment.GetEnvironmentVariable(WarnLicenseScanDiffsEnv), out bool warnOnLicenseScanDiffs) && warnOnLicenseScanDiffs;
14+
public static string DotNetDirectory => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(DotNetDirectory)) ?? throw new InvalidOperationException("DotNetDirectory must be specified");
15+
public static string PortableRid => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PortableRid)) ?? throw new InvalidOperationException("Portable RID must be specified");
16+
public static string TargetRid => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(TargetRid)) ?? throw new InvalidOperationException("Target RID must be specified");
17+
18+
public static string? CustomPackagesPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(CustomPackagesPath));
19+
public static bool ExcludeOmniSharpTests => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(ExcludeOmniSharpTests)), out bool excludeOmniSharpTests) && excludeOmniSharpTests;
20+
public static bool IncludeArtifactsSize => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(IncludeArtifactsSize)), out bool includeArtifactsSize) && includeArtifactsSize;
21+
public static string? LicenseScanPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(LicenseScanPath));
22+
public static string? MsftSdkTarballPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(MsftSdkTarballPath));
23+
public static string? PoisonReportPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PoisonReportPath));
24+
public static string? PrereqsPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(PrereqsPath));
25+
public static string? SdkTarballPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(SdkTarballPath));
26+
public static string? SourceBuiltArtifactsPath => (string)AppContext.GetData(ConfigSwitchPrefix + nameof(SourceBuiltArtifactsPath));
27+
public static bool WarnOnLicenseScanDiffs => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(WarnOnLicenseScanDiffs)), out bool warnOnLicenseScanDiffs) && warnOnLicenseScanDiffs;
28+
public static bool WarnOnSdkContentDiffs => bool.TryParse((string)AppContext.GetData(ConfigSwitchPrefix + nameof(WarnOnSdkContentDiffs)), out bool warnOnSdkContentDiffs) && warnOnSdkContentDiffs;
4529

4630
// Indicates whether the tests are being run in the context of a CI pipeline
47-
public static bool RunningInCI { get; } =
48-
bool.TryParse(Environment.GetEnvironmentVariable(RunningInCIEnv), out bool runningInCI) && runningInCI;
49-
50-
public static string? LicenseScanPath { get; } = Environment.GetEnvironmentVariable(LicenseScanPathEnv);
31+
public static bool RunningInCI => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("DOTNET_CI")) ||
32+
!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT")) ||
33+
!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS"));
34+
35+
public static string TargetArchitecture => TargetRid.Split('-')[1];
5136
}

‎src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Directory.Build.props

-10
This file was deleted.

‎src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ private static void InitNugetConfig()
7272
if (!Directory.Exists(Config.PrereqsPath))
7373
{
7474
throw new InvalidOperationException(
75-
$"Prereqs path '{Config.PrereqsPath}' specified in {Config.PrereqsPathEnv} does not exist.");
75+
$"Prereqs path '{Config.PrereqsPath}' specified via /p:SmokeTestsPrereqsPath='...' does not exist.");
7676
}
7777

7878
string nugetConfig = File.ReadAllText(nugetConfigPath);

‎src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/LicenseScanTests.cs

+3-1
Original file line numberDiff line numberDiff line change
@@ -112,13 +112,15 @@ public class LicenseScanTests : TestBase
112112

113113
private readonly string _targetRepo;
114114

115+
public bool ShouldScanForLicenses => !string.IsNullOrWhiteSpace(Config.LicenseScanPath);
116+
115117
public LicenseScanTests(ITestOutputHelper outputHelper) : base(outputHelper)
116118
{
117119
Assert.NotNull(Config.LicenseScanPath);
118120
_targetRepo = new DirectoryInfo(Config.LicenseScanPath).Name;
119121
}
120122

121-
[SkippableFact(Config.LicenseScanPathEnv, skipOnNullOrWhiteSpaceEnv: true)]
123+
[ConditionalFact(typeof(typeof(LicenseScanTests), nameof(ShouldScanForLicenses))]
122124
public void ScanForLicenses()
123125
{
124126
Assert.NotNull(Config.LicenseScanPath);

0 commit comments

Comments
 (0)