-
Notifications
You must be signed in to change notification settings - Fork 261
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
chore: xUnit-based lit test runner #680
Changes from 193 commits
80052b6
bfade77
3902ade
9eeea1f
8eea713
396db40
42db438
57f6708
358fb6c
d852a10
caf685d
ac8a5c5
eb29654
262dd34
7b09ac6
b6a589a
866d2ec
59269cd
bb01970
fd27e50
56ae641
a5874e6
88ac6e4
07634c3
810e1b4
d2f7e3c
24dc118
766ce71
c9f3625
1f521e4
1e313c7
e3e8cea
e983891
656775d
a70ff92
0157e2b
0a7ab31
6d97886
ac824df
c462c00
926471c
daf71e9
9a5880e
e1a39dd
0fa9b7a
c6622b3
6c60719
5219755
80c1027
7988dbc
45b6152
55b7d76
b6bdb52
1d9f6de
350ac23
d0bdb27
06530bc
e098b39
e132a82
16b17b9
bcdff33
96477c3
01653f7
e6deb41
56a1e4b
77cc5ba
3995823
0bdfdda
55096e9
b6424aa
abcfc8e
a6627c9
fe15a29
fa68a97
6dd4597
b98cc63
970379a
0141c50
55530f4
f786ec3
6b01bfd
65aec14
2f5e650
12e1e9c
8e85a2f
596196a
03658d7
a70ebb8
ce0a8ab
60f8f8d
2da02cc
703e3b0
8ba6c5d
4402f53
a0a24bd
89b160f
081f367
2d1a00b
0894cd2
5f9ea8e
b935ecb
7af490a
3d37243
ec84d25
153d830
b119135
0056b2d
147b481
d0f3502
0eaba59
42c6781
4afdd35
94ce613
62056b9
7bb72c2
ebf71da
2c9d3a7
07216c9
2416327
7613ba5
f7033ee
789c900
123937a
d274ed1
b7172b5
b968ec1
0f7a204
7f8c23a
ecb61c5
0e48658
206ce46
70ba1dd
b63cc99
0cc5e48
1c521bd
86d05c4
0ca1b42
6bd6250
5003bbd
79d28ac
347d1dd
efe3d9b
737ccde
8b851c8
fa385be
e2c5838
c8be4f6
9ca99a9
6a91afa
b273512
64d8b20
ddb84b7
3627445
901dcb4
a17ea03
255152a
82e3779
316be19
2e1f03e
0906cb1
deae7f5
3e30b03
b19782f
6e3e148
ca9c15e
b1d24c8
642ece0
cdf6a00
33dd949
04b7daa
210aca7
b41e4d1
cb01db4
d03b1be
bb88828
eaa7364
dce7168
c851425
52c9f89
dd21ef1
6fa230e
9465485
c085376
ec81556
008e028
062871d
c027ae9
16668f9
87852d8
59543ac
71ee992
5766472
ec03867
9b4e75b
123b5c9
9b58828
3aaed58
4f6bd1e
1597031
e473dd5
90d0318
f6e1a01
ebd302b
b5da098
9d4aef0
c99017b
65827fa
25a65a9
3e169cb
30cc008
167d2c1
36b880c
0ca7272
7023212
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -30,6 +30,7 @@ Package/ | |
|
||
Source/*/bin/ | ||
Source/*/obj/ | ||
Source/*/bin_core/ | ||
Source/*/obj_core/ | ||
Source/packages | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
<Project Sdk="Microsoft.NET.Sdk"> | ||
|
||
<PropertyGroup> | ||
<TargetFramework>net5.0</TargetFramework> | ||
<GenerateEmbeddedFilesManifest>true</GenerateEmbeddedFilesManifest> | ||
<IsPackable>false</IsPackable> | ||
</PropertyGroup> | ||
|
||
<ItemGroup> | ||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.4" /> | ||
<PackageReference Include="xunit" Version="2.4.1" /> | ||
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3"> | ||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> | ||
<PrivateAssets>all</PrivateAssets> | ||
</PackageReference> | ||
<PackageReference Include="coverlet.collector" Version="3.0.2"> | ||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> | ||
<PrivateAssets>all</PrivateAssets> | ||
</PackageReference> | ||
</ItemGroup> | ||
|
||
<ItemGroup> | ||
<ProjectReference Include="..\DafnyDriver\DafnyDriver.csproj" /> | ||
<ProjectReference Include="..\DafnyServer\DafnyServer.csproj" /> | ||
<ProjectReference Include="..\XUnitExtensions\XUnitExtensions.csproj" /> | ||
</ItemGroup> | ||
|
||
<ItemGroup> | ||
<Content Include="..\..\Binaries\z3\**\*.*" LinkBase="z3"> | ||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
</Content> | ||
<Content Include="..\..\Test\**\*.*" LinkBase="TestFiles\LitTests\LitTest"> | ||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
</Content> | ||
<Content Include="..\..\docs\DafnyRef\examples\**\*.*" LinkBase="TestFiles\LitTests\LitTest\refman\examples"> | ||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
</Content> | ||
<Content Include="..\..\node_modules\**\*.*" LinkBase="TestFiles\LitTests\LitTest\node_modules"> | ||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
</Content> | ||
</ItemGroup> | ||
|
||
</Project> |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.IO; | ||
using System.Linq; | ||
using System.Reflection; | ||
using System.Runtime.InteropServices; | ||
using Microsoft.Dafny; | ||
using Xunit; | ||
using Xunit.Abstractions; | ||
using XUnitExtensions; | ||
using XUnitExtensions.Lit; | ||
|
||
[assembly: TestCollectionOrderer("XUnitExtensions.TestCollectionShardFilter", "XUnitExtensions")] | ||
|
||
namespace IntegrationTests { | ||
public class LitTests { | ||
|
||
private static readonly Assembly dafnyDriverAssembly = typeof(DafnyDriver).Assembly; | ||
private static readonly Assembly dafnyServerAssembly = typeof(Server).Assembly; | ||
|
||
private static readonly string[] defaultDafnyArguments = new[] { | ||
"/countVerificationErrors:0", | ||
|
||
// We do not want absolute or relative paths in error messages, just the basename of the file | ||
"/useBaseNameForFileName", | ||
|
||
// We do not want output such as "Compiled program written to Foo.cs" | ||
// from the compilers, since that changes with the target language | ||
"/compileVerbose:0", | ||
|
||
// Hide Boogie execution traces since they are meaningless for Dafny programs | ||
"/errorTrace:0", | ||
|
||
// Set a default time limit, to catch cases where verification time runs off the rails | ||
"/timeLimit:300" | ||
}; | ||
|
||
private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable<string> arguments, LitTestConfiguration config, bool invokeDirectly) { | ||
return MainMethodLitCommand.Parse(assembly, arguments, config, invokeDirectly); | ||
} | ||
|
||
private static readonly LitTestConfiguration CONFIG = new() { | ||
Commands = new Dictionary<string, Func<IEnumerable<string>, LitTestConfiguration, ILitCommand>> { | ||
{ "%baredafny", (args, config) => | ||
MainWithArguments(dafnyDriverAssembly, args, config, false) }, | ||
{ "%dafny", (args, config) => | ||
MainWithArguments(dafnyDriverAssembly, defaultDafnyArguments.Concat(args), config, false) }, | ||
{ "%server", (args, config) => | ||
MainWithArguments(dafnyServerAssembly, args, config, false) }, | ||
}, | ||
|
||
Substitions = new Dictionary<string, string> { | ||
{ "%diff", "diff" }, | ||
{ "%binaryDir", "." }, | ||
{ "%z3", Path.Join("z3", "bin", "z3") }, | ||
{ "%refmanexamples", Path.Join("TestFiles", "LitTests", "LitTest", "refman", "examples") } | ||
}, | ||
|
||
PassthroughEnvironmentVariables = new[] { "PATH", "HOME" }, | ||
}; | ||
|
||
static LitTests() { | ||
var dafnyReleaseDir = Environment.GetEnvironmentVariable("DAFNY_RELEASE"); | ||
if (dafnyReleaseDir != null) { | ||
CONFIG.Commands["%baredafny"] = (args, config) => | ||
new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "dafny"), args, config.PassthroughEnvironmentVariables); | ||
CONFIG.Commands["%dafny"] = (args, config) => | ||
new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "dafny"), defaultDafnyArguments.Concat(args), config.PassthroughEnvironmentVariables); | ||
CONFIG.Commands["%server"] = (args, config) => | ||
new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "DafnyServer"), args, config.PassthroughEnvironmentVariables); | ||
CONFIG.Substitions["%z3"] = Path.Join(dafnyReleaseDir, "z3", "bin", "z3"); | ||
} | ||
|
||
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { | ||
CONFIG.Features = new[] { "ubuntu", "posix" }; | ||
} else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { | ||
CONFIG.Features = new[] { "windows" }; | ||
} else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { | ||
CONFIG.Features = new[] { "macosx", "posix" }; | ||
} else { | ||
throw new Exception($"Unsupported OS: {RuntimeInformation.OSDescription}"); | ||
} | ||
} | ||
|
||
private readonly ITestOutputHelper output; | ||
|
||
public LitTests(ITestOutputHelper output) { | ||
this.output = output; | ||
} | ||
|
||
[FileTheory] | ||
[FileData(Includes = new[] { "**/*.dfy", "**/*.transcript" }, | ||
Excludes = new[] { "**/Inputs/**/*", "refman/examples/**/*" })] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I had a few output files from a previous run that were accidentally picked up as Lit tests, for example this one:
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Right, lit automatically filters out Output files internally. I've addressed this by adding another Ideally it would be automatic, but another improvement I want to make after this PR is to run each test in a fresh temporary directory instead of a fixed and shared |
||
public void LitTest(string path) { | ||
LitTestCase.Run(path, CONFIG, output); | ||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If this is not working, try
Expand-Archive -LiteralPath z3.zip -DestinationPath Binaries/z3
https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.archive/expand-archive?view=powershell-7.1
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Thanks for the suggestion @MikaelMayer! I did eventually get it working in the later commits, but there's probably still a better approach so let me know if I can improve it.