From 80052b67f8a2900832ef58838bf9fe5cb6f0afd4 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 31 May 2020 09:17:05 -0700 Subject: [PATCH 001/192] Checkpoint --- Test/DafnyTests/DafnyTests.cs | 46 +++++++++++++++++++++++++++++++ Test/DafnyTests/DafnyTests.csproj | 3 ++ 2 files changed, 49 insertions(+) create mode 100644 Test/DafnyTests/DafnyTests.cs diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs new file mode 100644 index 00000000000..558e1b2f6d8 --- /dev/null +++ b/Test/DafnyTests/DafnyTests.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using Xunit; + +namespace DafnyTests { + + public class TestData : IEnumerable { + public IEnumerator GetEnumerator() { + yield return new object[] { "comp/Hello.dfy", "cs" }; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + public class DafnyTests { + private string RunDafny(string filePath, params string[] otherArguments) { + using (Process dafnyProcess = new Process()) { + dafnyProcess.StartInfo.FileName = "dafny"; + dafnyProcess.StartInfo.Arguments = "/Users/salkeldr/Documents/GitHub/dafny/Test/" + filePath; + dafnyProcess.StartInfo.Arguments += " /compile:3"; + foreach (var otherArgument in otherArguments) { + dafnyProcess.StartInfo.Arguments += " " + otherArgument; + } + dafnyProcess.StartInfo.UseShellExecute = false; + dafnyProcess.StartInfo.RedirectStandardOutput = true; + dafnyProcess.StartInfo.RedirectStandardError = true; + dafnyProcess.StartInfo.CreateNoWindow = true; + dafnyProcess.Start(); + dafnyProcess.WaitForExit(); + if (dafnyProcess.ExitCode != 0) { + Assert.True(false, dafnyProcess.StandardOutput.ReadToEnd()); + } + return dafnyProcess.StandardOutput.ReadToEnd(); + } + } + + [SkippableTheory()] + [ClassData(typeof(TestData))] + public void ValidProgramOutput(String inputPath, String targetLanguage) { + string output = RunDafny(inputPath, "/compileTarget:" + targetLanguage); + Console.Out.Write(output); + } + } +} \ No newline at end of file diff --git a/Test/DafnyTests/DafnyTests.csproj b/Test/DafnyTests/DafnyTests.csproj index 708e4f782b8..4693a32d2b4 100644 --- a/Test/DafnyTests/DafnyTests.csproj +++ b/Test/DafnyTests/DafnyTests.csproj @@ -46,6 +46,9 @@ + + + From bfade7717dd2cc5ecd7ea14dcac7e617102ad7ae Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 1 Jun 2020 15:27:19 -0700 Subject: [PATCH 002/192] Working version with relative paths --- Test/DafnyTests/DafnyTests.cs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 558e1b2f6d8..cb5fe55151f 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -2,23 +2,34 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using Xunit; namespace DafnyTests { public class TestData : IEnumerable { public IEnumerator GetEnumerator() { - yield return new object[] { "comp/Hello.dfy", "cs" }; + var filePaths = new string[] {"comp/Hello.dfy"}; + var languages = new string[] {"cs", "java", "go", "js"}; + foreach (var filePath in filePaths) { + foreach (var language in languages) { + yield return new object[] { filePath, language }; + } + } } IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } public class DafnyTests { - private string RunDafny(string filePath, params string[] otherArguments) { + + private static string DAFNY_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()).Parent.Parent.Parent.Parent.FullName; + private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test"); + + private string RunDafnyProgram(string filePath, params string[] otherArguments) { using (Process dafnyProcess = new Process()) { dafnyProcess.StartInfo.FileName = "dafny"; - dafnyProcess.StartInfo.Arguments = "/Users/salkeldr/Documents/GitHub/dafny/Test/" + filePath; + dafnyProcess.StartInfo.Arguments = Path.Combine(TEST_ROOT, filePath); dafnyProcess.StartInfo.Arguments += " /compile:3"; foreach (var otherArgument in otherArguments) { dafnyProcess.StartInfo.Arguments += " " + otherArgument; @@ -27,10 +38,11 @@ private string RunDafny(string filePath, params string[] otherArguments) { dafnyProcess.StartInfo.RedirectStandardOutput = true; dafnyProcess.StartInfo.RedirectStandardError = true; dafnyProcess.StartInfo.CreateNoWindow = true; + dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; dafnyProcess.Start(); dafnyProcess.WaitForExit(); if (dafnyProcess.ExitCode != 0) { - Assert.True(false, dafnyProcess.StandardOutput.ReadToEnd()); + Assert.True(false, dafnyProcess.StandardError.ReadToEnd()); } return dafnyProcess.StandardOutput.ReadToEnd(); } @@ -39,7 +51,7 @@ private string RunDafny(string filePath, params string[] otherArguments) { [SkippableTheory()] [ClassData(typeof(TestData))] public void ValidProgramOutput(String inputPath, String targetLanguage) { - string output = RunDafny(inputPath, "/compileTarget:" + targetLanguage); + string output = RunDafnyProgram(inputPath, "/compileTarget:" + targetLanguage); Console.Out.Write(output); } } From 3902adee99f57d360811fcb4b547bd07dfb69263 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 1 Jun 2020 15:35:11 -0700 Subject: [PATCH 003/192] Add expected arguments --- Test/DafnyTests/DafnyTests.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index cb5fe55151f..9424b1be325 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -31,6 +31,16 @@ private string RunDafnyProgram(string filePath, params string[] otherArguments) dafnyProcess.StartInfo.FileName = "dafny"; dafnyProcess.StartInfo.Arguments = Path.Combine(TEST_ROOT, filePath); dafnyProcess.StartInfo.Arguments += " /compile:3"; + // Expected output does not contain logo + dafnyProcess.StartInfo.Arguments += " -nologo -countVerificationErrors:0"; + + // We do not want absolute or relative paths in error messages, just the basename of the file + dafnyProcess.StartInfo.Arguments += " -useBaseNameForFileName"; + + // We do not want output such as "Compiled program written to Foo.cs" + // from the compilers, since that changes with the target language + dafnyProcess.StartInfo.Arguments += " -compileVerbose:0"; + foreach (var otherArgument in otherArguments) { dafnyProcess.StartInfo.Arguments += " " + otherArgument; } @@ -52,7 +62,8 @@ private string RunDafnyProgram(string filePath, params string[] otherArguments) [ClassData(typeof(TestData))] public void ValidProgramOutput(String inputPath, String targetLanguage) { string output = RunDafnyProgram(inputPath, "/compileTarget:" + targetLanguage); - Console.Out.Write(output); + string expectedOutput = File.ReadAllText(Path.Combine(TEST_ROOT, inputPath + ".expect")); + Assert.Equal(expectedOutput, output); } } } \ No newline at end of file From 9eeea1fdd579c9028ec0cfd361d0508006ed286d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 2 Jun 2020 17:36:34 -0700 Subject: [PATCH 004/192] XUnit workaround for parallel theories --- Test/DafnyTests/DafnyTests.cs | 40 +++++++------ Test/DafnyTests/DafnyTests.csproj | 4 +- ...lectionPerTestCaseTestCollectionFactory.cs | 18 ++++++ .../CollectionPerTestCaseTheoryDiscoverer.cs | 26 ++++++++ .../XUnitExtensions/FanOutDiscoverer.cs | 24 ++++++++ .../XUnitExtensions/FanOutExecutor.cs | 30 ++++++++++ .../FanOutTestAssemblyRunner.cs | 31 ++++++++++ .../ParallelTheoryAttribute.cs | 7 +++ .../XUnitExtensions/TestCaseWithCollection.cs | 59 +++++++++++++++++++ 9 files changed, 219 insertions(+), 20 deletions(-) create mode 100644 Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTestCollectionFactory.cs create mode 100644 Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs create mode 100644 Test/DafnyTests/XUnitExtensions/FanOutDiscoverer.cs create mode 100644 Test/DafnyTests/XUnitExtensions/FanOutExecutor.cs create mode 100644 Test/DafnyTests/XUnitExtensions/FanOutTestAssemblyRunner.cs create mode 100644 Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs create mode 100644 Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 9424b1be325..3675ad805fd 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -3,24 +3,14 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using FluentAssertions; using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; +using Assert = Xunit.Assert; namespace DafnyTests { - public class TestData : IEnumerable { - public IEnumerator GetEnumerator() { - var filePaths = new string[] {"comp/Hello.dfy"}; - var languages = new string[] {"cs", "java", "go", "js"}; - foreach (var filePath in filePaths) { - foreach (var language in languages) { - yield return new object[] { filePath, language }; - } - } - } - - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - } - public class DafnyTests { private static string DAFNY_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()).Parent.Parent.Parent.Parent.FullName; @@ -51,19 +41,31 @@ private string RunDafnyProgram(string filePath, params string[] otherArguments) dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; dafnyProcess.Start(); dafnyProcess.WaitForExit(); + string output = dafnyProcess.StandardOutput.ReadToEnd(); + string error = dafnyProcess.StandardError.ReadToEnd(); if (dafnyProcess.ExitCode != 0) { - Assert.True(false, dafnyProcess.StandardError.ReadToEnd()); + Assert.True(false, output); } - return dafnyProcess.StandardOutput.ReadToEnd(); + return output; } } - [SkippableTheory()] - [ClassData(typeof(TestData))] + public static IEnumerable TestData() { + var filePaths = new string[] {"comp/Hello.dfy"}; + var languages = new string[] {"cs", "java", "go", "js"}; + foreach (var filePath in filePaths) { + foreach (var language in languages) { + yield return new object[] { filePath, language }; + } + } + } + + [ParallelTheory] + [MemberData(nameof(TestData))] public void ValidProgramOutput(String inputPath, String targetLanguage) { string output = RunDafnyProgram(inputPath, "/compileTarget:" + targetLanguage); string expectedOutput = File.ReadAllText(Path.Combine(TEST_ROOT, inputPath + ".expect")); - Assert.Equal(expectedOutput, output); + output.Should().Be(expectedOutput); } } } \ No newline at end of file diff --git a/Test/DafnyTests/DafnyTests.csproj b/Test/DafnyTests/DafnyTests.csproj index 4693a32d2b4..1ff5a839e0e 100644 --- a/Test/DafnyTests/DafnyTests.csproj +++ b/Test/DafnyTests/DafnyTests.csproj @@ -41,6 +41,7 @@ + @@ -48,6 +49,7 @@ + @@ -65,4 +67,4 @@ - + \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTestCollectionFactory.cs b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTestCollectionFactory.cs new file mode 100644 index 00000000000..ac0fbd4cdde --- /dev/null +++ b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTestCollectionFactory.cs @@ -0,0 +1,18 @@ +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace DefaultNamespace { + public class CollectionPerTestCaseTestCollectionFactory : IXunitTestCollectionFactory { + public ITestCollection Get(ITypeInfo testClass) { + throw new System.NotImplementedException(); + } + + public string DisplayName + { + get + { + return "collection-per-testcase"; + } + } + } +} \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs new file mode 100644 index 00000000000..9868731e5d4 --- /dev/null +++ b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; +using System.Linq; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace XUnitExtensions { + public class CollectionPerTestCaseTheoryDiscoverer : IXunitTestCaseDiscoverer { + + readonly TheoryDiscoverer theoryDiscoverer; + + public CollectionPerTestCaseTheoryDiscoverer(IMessageSink diagnosticMessageSink) + { + theoryDiscoverer = new TheoryDiscoverer(diagnosticMessageSink); + } + + private TestCollection testCollectionForTestCase(IXunitTestCase testCase) { + return new TestCollection(testCase.TestMethod.TestClass.TestCollection.TestAssembly, + (ITypeInfo) null, "Test collection for " + testCase.DisplayName); + } + + public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { + return theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute) + .Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); + } + } +} \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/FanOutDiscoverer.cs b/Test/DafnyTests/XUnitExtensions/FanOutDiscoverer.cs new file mode 100644 index 00000000000..6e564750408 --- /dev/null +++ b/Test/DafnyTests/XUnitExtensions/FanOutDiscoverer.cs @@ -0,0 +1,24 @@ +using System; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace DefaultNamespace { + public class FanOutDiscoverer : TestFrameworkDiscoverer { + readonly CollectionPerClassTestCollectionFactory testCollectionFactory; + + public FanOutDiscoverer(IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, IMessageSink diagnosticMessageSink) : base(assemblyInfo, sourceProvider, diagnosticMessageSink) { + var testAssembly = new TestAssembly(assemblyInfo, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile); + testCollectionFactory = new CollectionPerClassTestCollectionFactory(testAssembly, diagnosticMessageSink); + } + + protected override ITestClass CreateTestClass(ITypeInfo @class) { + return new TestClass(testCollectionFactory.Get(@class), @class); + } + + protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, IMessageBus messageBus, + ITestFrameworkDiscoveryOptions discoveryOptions) + { + throw new System.NotImplementedException(); + } + } +} \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/FanOutExecutor.cs b/Test/DafnyTests/XUnitExtensions/FanOutExecutor.cs new file mode 100644 index 00000000000..5d492a6ef69 --- /dev/null +++ b/Test/DafnyTests/XUnitExtensions/FanOutExecutor.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace DefaultNamespace { + public class FanOutExecutor : XunitTestFrameworkExecutor { + public FanOutExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink) + : base(assemblyName, sourceInformationProvider, diagnosticMessageSink) + { + this.TestAssembly = new TestAssembly(this.AssemblyInfo, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, assemblyName.Version); + } + + protected override ITestFrameworkDiscoverer CreateDiscoverer() { + throw new System.NotImplementedException(); + } + + protected override async void RunTestCases( + IEnumerable testCases, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) + { + using (FanOutTestAssemblyRunner assemblyRunner = new FanOutTestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions)) + { + RunSummary runSummary = await assemblyRunner.RunAsync(); + } + } + } +} \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/FanOutTestAssemblyRunner.cs b/Test/DafnyTests/XUnitExtensions/FanOutTestAssemblyRunner.cs new file mode 100644 index 00000000000..119557907b2 --- /dev/null +++ b/Test/DafnyTests/XUnitExtensions/FanOutTestAssemblyRunner.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.Design; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace DefaultNamespace { + public class FanOutTestAssemblyRunner : TestAssemblyRunner { + public FanOutTestAssemblyRunner(ITestAssembly testAssembly, + IEnumerable testCases, + IMessageSink diagnosticMessageSink, + IMessageSink executionMessageSink, + ITestFrameworkExecutionOptions executionOptions) + : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) { + } + + protected override string GetTestFrameworkDisplayName() { + return "Fan-Out Framework"; + } + + protected override Task RunTestCollectionAsync(IMessageBus messageBus, + ITestCollection testCollection, IEnumerable testCases, + CancellationTokenSource cancellationTokenSource) { + return new XunitTestCollectionRunner(testCollection, testCases, this.DiagnosticMessageSink, messageBus, + this.TestCaseOrderer, new ExceptionAggregator(this.Aggregator), cancellationTokenSource).RunAsync(); + } + } +} \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs b/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs new file mode 100644 index 00000000000..c7299727f86 --- /dev/null +++ b/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs @@ -0,0 +1,7 @@ +using System; +using Xunit; +using Xunit.Sdk; + +[AttributeUsage(AttributeTargets.Method)] +[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "DafnyTests")] +public class ParallelTheoryAttribute : TheoryAttribute { } \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs b/Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs new file mode 100644 index 00000000000..8df40bc6a71 --- /dev/null +++ b/Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Threading; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace XUnitExtensions { + public class TestCaseWithCollection : LongLivedMarshalByRefObject, IXunitTestCase { + + private IXunitTestCase testCase; + private ITestMethod testMethod; + + public TestCaseWithCollection(IXunitTestCase testCase, ITestCollection collection) + { + this.testCase = testCase; + + var testClassWithCollection = new TestClass(collection, testCase.TestMethod.TestClass.Class); + this.testMethod = new TestMethod(testClassWithCollection, testCase.TestMethod.Method); + } + + [EditorBrowsable(EditorBrowsableState.Never)] + [Obsolete("Called by the de-serializer", error: true)] + public TestCaseWithCollection() { } + + public void Deserialize(IXunitSerializationInfo info) { + testCase = info.GetValue("InnerTestCase"); + testMethod = info.GetValue("TestMethod"); + } + + public void Serialize(IXunitSerializationInfo info) { + info.AddValue("InnerTestCase", testCase); + info.AddValue("TestMethod", testMethod); + } + public string DisplayName { get { return testCase.DisplayName; } } + public string SkipReason { get { return testCase.SkipReason; } } + public ISourceInformation SourceInformation + { + get { return testCase.SourceInformation; } + set { testCase.SourceInformation = value; } + } + public ITestMethod TestMethod { get { return testMethod; } } + + public object[] TestMethodArguments { get { return testCase.TestMethodArguments; } } + public Dictionary> Traits { get { return testCase.Traits; } } + public string UniqueID { get { return testCase.UniqueID; } } + + public Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, + ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { + return testCase.RunAsync(diagnosticMessageSink, messageBus, constructorArguments, aggregator, cancellationTokenSource); + } + + public Exception InitializationException { get; } + public IMethodInfo Method { get; } + public int Timeout { get; } + } +} \ No newline at end of file From 8eea7139280647304ce3304767ab9f4e70876b65 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 Jun 2020 14:28:55 -0700 Subject: [PATCH 005/192] Swtich to NetCore, refactor most of comp/ --- Test/DafnyTests-NetCore.sln | 16 + Test/DafnyTests/DafnyTests-NetCore.csproj | 15 + Test/DafnyTests/DafnyTests.cs | 99 +++++-- Test/DafnyTests/DafnyTests.csproj | 1 + ...lectionPerTestCaseTestCollectionFactory.cs | 18 -- .../XUnitExtensions/FanOutDiscoverer.cs | 24 -- .../XUnitExtensions/FanOutExecutor.cs | 30 -- .../FanOutTestAssemblyRunner.cs | 31 -- .../ParallelTheoryAttribute.cs | 2 +- Test/comp/Arrays.dfy.expect | 54 ---- Test/comp/Calls.dfy | 6 - Test/comp/Calls.dfy.expect | 9 - Test/comp/Class.dfy | 6 - Test/comp/Class.dfy.expect | 39 --- Test/comp/Collections.dfy | 6 - Test/comp/Collections.dfy.expect | 207 ------------- Test/comp/Comprehensions.dfy | 6 - Test/comp/Comprehensions.dfy.expect | 66 ----- Test/comp/CovariantCollections.dfy | 6 - Test/comp/CovariantCollections.dfy.expect | 25 -- Test/comp/CovariantCollections.dfy.go.expect | 6 + .../comp/CovariantCollections.dfy.java.expect | 5 + Test/comp/Dt.dfy | 6 - Test/comp/Dt.dfy.expect | 57 ---- Test/comp/Forall.dfy | 6 - Test/comp/Forall.dfy.expect | 132 --------- Test/comp/Ghosts.dfy | 6 - Test/comp/Ghosts.dfy.expect | 36 --- Test/comp/Hello.dfy.expect | 15 - Test/comp/Iterators.dfy | 5 - Test/comp/Iterators.dfy.expect | 12 - Test/comp/Let.dfy | 6 - Test/comp/Let.dfy.expect | 21 -- Test/comp/Module.dfy | 6 - Test/comp/Module.dfy.expect | 9 - Test/comp/NativeNumbers.dfy.expect | 168 ----------- Test/comp/NativeNumbers.dfy.js.expect | 9 + Test/comp/Numbers.dfy | 6 - Test/comp/Numbers.dfy.expect | 276 ------------------ Test/comp/Poly.dfy | 4 - Test/comp/TailRecursion.dfy | 6 - Test/comp/TailRecursion.dfy.expect | 48 --- Test/comp/TypeParams.dfy | 6 - Test/comp/TypeParams.dfy.cs.expect | 20 ++ Test/comp/TypeParams.dfy.expect | 80 ----- Test/comp/TypeParams.dfy.go.expect | 20 ++ Test/comp/TypeParams.dfy.java.expect | 20 ++ Test/comp/TypeParams.dfy.js.expect | 20 ++ 48 files changed, 210 insertions(+), 1467 deletions(-) create mode 100644 Test/DafnyTests-NetCore.sln create mode 100644 Test/DafnyTests/DafnyTests-NetCore.csproj delete mode 100644 Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTestCollectionFactory.cs delete mode 100644 Test/DafnyTests/XUnitExtensions/FanOutDiscoverer.cs delete mode 100644 Test/DafnyTests/XUnitExtensions/FanOutExecutor.cs delete mode 100644 Test/DafnyTests/XUnitExtensions/FanOutTestAssemblyRunner.cs create mode 100644 Test/comp/CovariantCollections.dfy.go.expect create mode 100644 Test/comp/CovariantCollections.dfy.java.expect create mode 100644 Test/comp/NativeNumbers.dfy.js.expect create mode 100644 Test/comp/TypeParams.dfy.cs.expect delete mode 100644 Test/comp/TypeParams.dfy.expect create mode 100644 Test/comp/TypeParams.dfy.go.expect create mode 100644 Test/comp/TypeParams.dfy.java.expect create mode 100644 Test/comp/TypeParams.dfy.js.expect diff --git a/Test/DafnyTests-NetCore.sln b/Test/DafnyTests-NetCore.sln new file mode 100644 index 00000000000..6e5e05bcbb0 --- /dev/null +++ b/Test/DafnyTests-NetCore.sln @@ -0,0 +1,16 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests", "DafnyTests\DafnyTests-NetCore.csproj", "{45AE3DDD-D0EA-4A27-943B-AC66A581BE03}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|Any CPU.Build.0 = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Any CPU.ActiveCfg = Release|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/Test/DafnyTests/DafnyTests-NetCore.csproj b/Test/DafnyTests/DafnyTests-NetCore.csproj new file mode 100644 index 00000000000..323172ccf3d --- /dev/null +++ b/Test/DafnyTests/DafnyTests-NetCore.csproj @@ -0,0 +1,15 @@ + + + + netcoreapp2.1 + + + + + + + + + + + diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 3675ad805fd..50750b9c7aa 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -1,9 +1,12 @@ using System; +using System.CodeDom.Compiler; using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.IO; -using FluentAssertions; +using System.Linq; +using DiffMatchPatch; +using Microsoft.Extensions.DependencyModel; using Xunit; using Xunit.Abstractions; using Xunit.Sdk; @@ -13,45 +16,67 @@ namespace DafnyTests { public class DafnyTests { - private static string DAFNY_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()).Parent.Parent.Parent.Parent.FullName; + private static string DAFNY_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()).Parent.Parent.Parent.Parent.Parent.FullName; + private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test"); + private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; + + + private string RunDafnyProgram(string filePath, params string[] arguments) { + List dafnyArguments = new List { - private string RunDafnyProgram(string filePath, params string[] otherArguments) { - using (Process dafnyProcess = new Process()) { - dafnyProcess.StartInfo.FileName = "dafny"; - dafnyProcess.StartInfo.Arguments = Path.Combine(TEST_ROOT, filePath); - dafnyProcess.StartInfo.Arguments += " /compile:3"; + filePath, + "/compile:3", + // Expected output does not contain logo - dafnyProcess.StartInfo.Arguments += " -nologo -countVerificationErrors:0"; + "-nologo", + "-countVerificationErrors:0", // We do not want absolute or relative paths in error messages, just the basename of the file - dafnyProcess.StartInfo.Arguments += " -useBaseNameForFileName"; + "-useBaseNameForFileName", // We do not want output such as "Compiled program written to Foo.cs" // from the compilers, since that changes with the target language - dafnyProcess.StartInfo.Arguments += " -compileVerbose:0"; - - foreach (var otherArgument in otherArguments) { - dafnyProcess.StartInfo.Arguments += " " + otherArgument; + "-compileVerbose:0" + }; + dafnyArguments.AddRange(arguments); + return RunDafny(dafnyArguments); + } + + public static string RunDafny(IEnumerable arguments) { + using (Process dafnyProcess = new Process()) { + dafnyProcess.StartInfo.FileName = "mono"; + dafnyProcess.StartInfo.ArgumentList.Add(DAFNY_EXE); + foreach (var argument in arguments) { + dafnyProcess.StartInfo.ArgumentList.Add(argument); } + dafnyProcess.StartInfo.UseShellExecute = false; dafnyProcess.StartInfo.RedirectStandardOutput = true; dafnyProcess.StartInfo.RedirectStandardError = true; dafnyProcess.StartInfo.CreateNoWindow = true; - dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; + dafnyProcess.StartInfo.WorkingDirectory = Path.Combine(DAFNY_ROOT, "Binaries"); + + // Only preserve specific whitelisted environment variables + dafnyProcess.StartInfo.EnvironmentVariables.Clear(); + dafnyProcess.StartInfo.EnvironmentVariables.Add("PATH", System.Environment.GetEnvironmentVariable("PATH")); + dafnyProcess.Start(); dafnyProcess.WaitForExit(); string output = dafnyProcess.StandardOutput.ReadToEnd(); - string error = dafnyProcess.StandardError.ReadToEnd(); if (dafnyProcess.ExitCode != 0) { - Assert.True(false, output); + string error = dafnyProcess.StandardError.ReadToEnd(); + Assert.True(false, output + "\n" + error); } + return output; } } - + public static IEnumerable TestData() { - var filePaths = new string[] {"comp/Hello.dfy"}; + var filePaths = Directory.GetFiles(COMP_DIR, "*.dfy") + .Select(path => GetRelativePath(COMP_DIR, path)); +// var filePaths = new string[] {"NativeNumbers.dfy"}; var languages = new string[] {"cs", "java", "go", "js"}; foreach (var filePath in filePaths) { foreach (var language in languages) { @@ -60,12 +85,42 @@ public static IEnumerable TestData() { } } + // TODO-RS: Replace with Path.GetRelativePath() if we move to .NET Core 3.1 + private static string GetRelativePath(string relativeTo, string path) { + var fullRelativeTo = Path.GetFullPath(relativeTo); + var fullPath = Path.GetFullPath(path); + Assert.StartsWith(fullRelativeTo, fullPath); + return fullPath.Substring(fullRelativeTo.Length); + } + + private static void AssertEqualWithDiff(string expected, string actual) { + if (expected != actual) { + DiffMatchPatch.DiffMatchPatch dmp = DiffMatchPatchModule.Default; + List diff = dmp.DiffMain(expected, actual); + dmp.DiffCleanupSemantic(diff); + string patch = DiffText(diff); + throw new AssertActualExpectedException(expected, actual, patch); + } + } + + private static string DiffText(List diffs) { + return ""; + } + [ParallelTheory] [MemberData(nameof(TestData))] - public void ValidProgramOutput(String inputPath, String targetLanguage) { - string output = RunDafnyProgram(inputPath, "/compileTarget:" + targetLanguage); - string expectedOutput = File.ReadAllText(Path.Combine(TEST_ROOT, inputPath + ".expect")); - output.Should().Be(expectedOutput); + public void ValidProgramOutput(String program, String language) { + string fullInputPath = Path.Combine(COMP_DIR, program); + + string expectedOutputPath = fullInputPath + ".expect"; + string expectedLangOutputPath = fullInputPath + "." + language + ".expect"; + bool langSpecific = File.Exists(expectedLangOutputPath); + string expectedOutput = langSpecific ? File.ReadAllText(expectedLangOutputPath) : File.ReadAllText(expectedOutputPath); + + string output = RunDafnyProgram(fullInputPath, "/compileTarget:" + language); + + AssertEqualWithDiff(expectedOutput, output); + Skip.If(langSpecific, "Confirmed language-specific behavior"); } } } \ No newline at end of file diff --git a/Test/DafnyTests/DafnyTests.csproj b/Test/DafnyTests/DafnyTests.csproj index 1ff5a839e0e..85db0cb37ea 100644 --- a/Test/DafnyTests/DafnyTests.csproj +++ b/Test/DafnyTests/DafnyTests.csproj @@ -35,6 +35,7 @@ 4 + diff --git a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTestCollectionFactory.cs b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTestCollectionFactory.cs deleted file mode 100644 index ac0fbd4cdde..00000000000 --- a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTestCollectionFactory.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace DefaultNamespace { - public class CollectionPerTestCaseTestCollectionFactory : IXunitTestCollectionFactory { - public ITestCollection Get(ITypeInfo testClass) { - throw new System.NotImplementedException(); - } - - public string DisplayName - { - get - { - return "collection-per-testcase"; - } - } - } -} \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/FanOutDiscoverer.cs b/Test/DafnyTests/XUnitExtensions/FanOutDiscoverer.cs deleted file mode 100644 index 6e564750408..00000000000 --- a/Test/DafnyTests/XUnitExtensions/FanOutDiscoverer.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace DefaultNamespace { - public class FanOutDiscoverer : TestFrameworkDiscoverer { - readonly CollectionPerClassTestCollectionFactory testCollectionFactory; - - public FanOutDiscoverer(IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, IMessageSink diagnosticMessageSink) : base(assemblyInfo, sourceProvider, diagnosticMessageSink) { - var testAssembly = new TestAssembly(assemblyInfo, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile); - testCollectionFactory = new CollectionPerClassTestCollectionFactory(testAssembly, diagnosticMessageSink); - } - - protected override ITestClass CreateTestClass(ITypeInfo @class) { - return new TestClass(testCollectionFactory.Get(@class), @class); - } - - protected override bool FindTestsForType(ITestClass testClass, bool includeSourceInformation, IMessageBus messageBus, - ITestFrameworkDiscoveryOptions discoveryOptions) - { - throw new System.NotImplementedException(); - } - } -} \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/FanOutExecutor.cs b/Test/DafnyTests/XUnitExtensions/FanOutExecutor.cs deleted file mode 100644 index 5d492a6ef69..00000000000 --- a/Test/DafnyTests/XUnitExtensions/FanOutExecutor.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Reflection; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace DefaultNamespace { - public class FanOutExecutor : XunitTestFrameworkExecutor { - public FanOutExecutor(AssemblyName assemblyName, ISourceInformationProvider sourceInformationProvider, IMessageSink diagnosticMessageSink) - : base(assemblyName, sourceInformationProvider, diagnosticMessageSink) - { - this.TestAssembly = new TestAssembly(this.AssemblyInfo, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, assemblyName.Version); - } - - protected override ITestFrameworkDiscoverer CreateDiscoverer() { - throw new System.NotImplementedException(); - } - - protected override async void RunTestCases( - IEnumerable testCases, - IMessageSink executionMessageSink, - ITestFrameworkExecutionOptions executionOptions) - { - using (FanOutTestAssemblyRunner assemblyRunner = new FanOutTestAssemblyRunner(TestAssembly, testCases, DiagnosticMessageSink, executionMessageSink, executionOptions)) - { - RunSummary runSummary = await assemblyRunner.RunAsync(); - } - } - } -} \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/FanOutTestAssemblyRunner.cs b/Test/DafnyTests/XUnitExtensions/FanOutTestAssemblyRunner.cs deleted file mode 100644 index 119557907b2..00000000000 --- a/Test/DafnyTests/XUnitExtensions/FanOutTestAssemblyRunner.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel.Design; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace DefaultNamespace { - public class FanOutTestAssemblyRunner : TestAssemblyRunner { - public FanOutTestAssemblyRunner(ITestAssembly testAssembly, - IEnumerable testCases, - IMessageSink diagnosticMessageSink, - IMessageSink executionMessageSink, - ITestFrameworkExecutionOptions executionOptions) - : base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions) { - } - - protected override string GetTestFrameworkDisplayName() { - return "Fan-Out Framework"; - } - - protected override Task RunTestCollectionAsync(IMessageBus messageBus, - ITestCollection testCollection, IEnumerable testCases, - CancellationTokenSource cancellationTokenSource) { - return new XunitTestCollectionRunner(testCollection, testCases, this.DiagnosticMessageSink, messageBus, - this.TestCaseOrderer, new ExceptionAggregator(this.Aggregator), cancellationTokenSource).RunAsync(); - } - } -} \ No newline at end of file diff --git a/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs b/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs index c7299727f86..6ae3e119d2f 100644 --- a/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs +++ b/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs @@ -3,5 +3,5 @@ using Xunit.Sdk; [AttributeUsage(AttributeTargets.Method)] -[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "DafnyTests")] +[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "DafnyTests-NetCore")] public class ParallelTheoryAttribute : TheoryAttribute { } \ No newline at end of file diff --git a/Test/comp/Arrays.dfy.expect b/Test/comp/Arrays.dfy.expect index 6ab0a210ceb..3507348bea1 100644 --- a/Test/comp/Arrays.dfy.expect +++ b/Test/comp/Arrays.dfy.expect @@ -16,57 +16,3 @@ h e l l o 0 0 1 0 0 cube dims: 3 0 4 It's null - -Dafny program verifier finished with 9 verified, 0 errors -0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 -17 -[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] -[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] -[20, 21, 22] -[0, 1, 2, 3, 4, 5, 6, 7] -[0, 1, 2, 3, 4, 5, 6, 7] -d d d -h e l l o -0 0 0 0 0 0 0 0 -0 0 0 0 0 0 0 0 -1 0 0 0 0 -0 1 0 0 0 -0 0 1 0 0 -cube dims: 3 0 4 -It's null - -Dafny program verifier finished with 9 verified, 0 errors -0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 -17 -[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] -[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] -[20, 21, 22] -[0, 1, 2, 3, 4, 5, 6, 7] -[0, 1, 2, 3, 4, 5, 6, 7] -d d d -h e l l o -0 0 0 0 0 0 0 0 -0 0 0 0 0 0 0 0 -1 0 0 0 0 -0 1 0 0 0 -0 0 1 0 0 -cube dims: 3 0 4 -It's null - -Dafny program verifier finished with 9 verified, 0 errors -0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 -17 -[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] -[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] -[20, 21, 22] -[0, 1, 2, 3, 4, 5, 6, 7] -[0, 1, 2, 3, 4, 5, 6, 7] -d d d -h e l l o -0 0 0 0 0 0 0 0 -0 0 0 0 0 0 0 0 -1 0 0 0 0 -0 1 0 0 0 -0 0 1 0 0 -cube dims: 3 0 4 -It's null diff --git a/Test/comp/Calls.dfy b/Test/comp/Calls.dfy index 3de2be7831d..4af874ff109 100644 --- a/Test/comp/Calls.dfy +++ b/Test/comp/Calls.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - function method F(x: int, y: bool): int { x + if y then 2 else 3 } diff --git a/Test/comp/Calls.dfy.expect b/Test/comp/Calls.dfy.expect index ad5fc0bdb5e..c9d9c3e9319 100644 --- a/Test/comp/Calls.dfy.expect +++ b/Test/comp/Calls.dfy.expect @@ -1,12 +1,3 @@ Dafny program verifier finished with 1 verified, 0 errors 5 0 0 false 0 false 0 - -Dafny program verifier finished with 1 verified, 0 errors -5 0 0 false 0 false 0 - -Dafny program verifier finished with 1 verified, 0 errors -5 0 0 false 0 false 0 - -Dafny program verifier finished with 1 verified, 0 errors -5 0 0 false 0 false 0 diff --git a/Test/comp/Class.dfy b/Test/comp/Class.dfy index 644de2bb66b..7f8d90cac95 100644 --- a/Test/comp/Class.dfy +++ b/Test/comp/Class.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - class MyClass { var a: int const b: int diff --git a/Test/comp/Class.dfy.expect b/Test/comp/Class.dfy.expect index a75cb43552a..d60d6ef7059 100644 --- a/Test/comp/Class.dfy.expect +++ b/Test/comp/Class.dfy.expect @@ -11,42 +11,3 @@ true true false 70 hi later 21 4 42 5 1 - -Dafny program verifier finished with 13 verified, 0 errors -true true false -103 103 103 106 106 106 -203 17 0 18 8 9 69 70 -203 17 0 18 8 9 69 70 -203 17 0 18 8 9 69 70 -0 18 9 70 -0 18 9 70 -0 18 9 70 -70 -hi later -21 4 42 5 1 - -Dafny program verifier finished with 13 verified, 0 errors -true true false -103 103 103 106 106 106 -203 17 0 18 8 9 69 70 -203 17 0 18 8 9 69 70 -203 17 0 18 8 9 69 70 -0 18 9 70 -0 18 9 70 -0 18 9 70 -70 -hi later -21 4 42 5 1 - -Dafny program verifier finished with 13 verified, 0 errors -true true false -103 103 103 106 106 106 -203 17 0 18 8 9 69 70 -203 17 0 18 8 9 69 70 -203 17 0 18 8 9 69 70 -0 18 9 70 -0 18 9 70 -0 18 9 70 -70 -hi later -21 4 42 5 1 diff --git a/Test/comp/Collections.dfy b/Test/comp/Collections.dfy index 415a664224c..8ed0b14e6fd 100644 --- a/Test/comp/Collections.dfy +++ b/Test/comp/Collections.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method Main() { Sets(); SubSets(); diff --git a/Test/comp/Collections.dfy.expect b/Test/comp/Collections.dfy.expect index 7f0bcaddfa1..53a0cea6ebd 100644 --- a/Test/comp/Collections.dfy.expect +++ b/Test/comp/Collections.dfy.expect @@ -67,210 +67,3 @@ There are 0 occurrences of 58 in the multiset There are 536870912 occurrences of 58 in the multiset There are 633825300114114700748351602688 occurrences of 58 in the multiset There are 633825300114114700748351602688 occurrences of null in the multiset - -Dafny program verifier finished with 17 verified, 0 errors -Sets: {} {17, 82} {12, 17} - cardinality: 0 2 2 - union: {17, 82} {17, 82, 12} - intersection: {} {17} - difference: {} {82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true -false true -|s|=4 |S|=16 -{{}, {a}, {b}, {b, a}, {c}, {c, a}, {c, b}, {c, b, a}, {d}, {d, a}, {d, b}, {d, b, a}, {d, c}, {d, c, a}, {d, c, b}, {d, c, b, a}} -Multisets: multiset{} multiset{17, 17, 82, 82} multiset{12, 17} - cardinality: 0 4 2 - union: multiset{17, 17, 82, 82} multiset{17, 17, 17, 82, 82, 12} - intersection: multiset{} multiset{17} - difference: multiset{} multiset{17, 82, 82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true - update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{12, 17, 17} - multiplicity: 0 2 1 -Sequences: [] [17, 82, 17, 82] [12, 17] - cardinality: 0 4 2 - update: [42, 82, 17, 82] [42, 17] - index: 17 12 - subsequence ([lo..hi]): [82, 17] [17] - subsequence ([lo..]): [82, 17, 82] [17] - subsequence ([..hi]): [17, 82, 17][12] - subsequence ([..]): [] [17, 82, 17, 82] [12, 17] - concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] - prefix: true false true - proper prefix: true false false - membership: false true true -Bound Bound Bound Bound Bound Bound -ed ed ed ed ed ed -e e e e e e -hello -hEllo -[2, 4, 6, 8, 10] -[2, 0, 6, 8, 10] -Strings: uRuR gu - cardinality: 0 4 2 - concatenation: uRuR uRuRgu - prefix: true false true - proper prefix: true false false - membership: false true true - constructed as sequence: guru - mix: hello-d ddd-hello -Maps: map[] map[17 := 2, 82 := 2] map[17 := 0, 12 := 26] - cardinality: 0 2 2 - keys: {} {17, 82} {17, 12} - values: {} {2} {0, 26} - items: {} {(17, 2), (82, 2)} {(17, 0), (12, 26)} - update: map[17 := 6] map[17 := 6, 82 := 2] map[17 := 6, 12 := 26] - lookup: false 2 0 -m: map[Color.Blue := 30, Color.Yellow := 21] -keys: {Color.Blue, Color.Yellow} -values: {30, 21} -items: {(Color.Blue, 30), (Color.Yellow, 21)} -2: 0 1 1 -3: 0 1 2 -There are 0 occurrences of 58 in the multiset -There are 536870912 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of null in the multiset - -Dafny program verifier finished with 17 verified, 0 errors -Sets: {} {17, 82} {12, 17} - cardinality: 0 2 2 - union: {17, 82} {17, 82, 12} - intersection: {} {17} - difference: {} {82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true -false true -|s|=4 |S|=16 -{{}, {a}, {b}, {b, a}, {c}, {c, a}, {c, b}, {c, b, a}, {d}, {d, a}, {d, b}, {d, b, a}, {d, c}, {d, c, a}, {d, c, b}, {d, c, b, a}} -Multisets: multiset{} multiset{17, 17, 82, 82} multiset{12, 17} - cardinality: 0 4 2 - union: multiset{17, 17, 82, 82} multiset{17, 17, 17, 82, 82, 12} - intersection: multiset{} multiset{17} - difference: multiset{} multiset{17, 82, 82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true - update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{12, 17, 17} - multiplicity: 0 2 1 -Sequences: [] [17, 82, 17, 82] [12, 17] - cardinality: 0 4 2 - update: [42, 82, 17, 82] [42, 17] - index: 17 12 - subsequence ([lo..hi]): [82, 17] [17] - subsequence ([lo..]): [82, 17, 82] [17] - subsequence ([..hi]): [17, 82, 17][12] - subsequence ([..]): [] [17, 82, 17, 82] [12, 17] - concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] - prefix: true false true - proper prefix: true false false - membership: false true true -Bound Bound Bound Bound Bound Bound -ed ed ed ed ed ed -e e e e e e -hello -hEllo -[2, 4, 6, 8, 10] -[2, 0, 6, 8, 10] -Strings: uRuR gu - cardinality: 0 4 2 - concatenation: uRuR uRuRgu - prefix: true false true - proper prefix: true false false - membership: false true true - constructed as sequence: guru - mix: hello-d ddd-hello -Maps: map[] map[17 := 2, 82 := 2] map[17 := 0, 12 := 26] - cardinality: 0 2 2 - keys: {} {17, 82} {17, 12} - values: {} {2} {0, 26} - items: {} {(17, 2), (82, 2)} {(17, 0), (12, 26)} - update: map[17 := 6] map[17 := 6, 82 := 2] map[17 := 6, 12 := 26] - lookup: false 2 0 -m: map[Color.Blue := 30, Color.Yellow := 21] -keys: {Color.Blue, Color.Yellow} -values: {30, 21} -items: {(Color.Blue, 30), (Color.Yellow, 21)} -2: 0 1 1 -3: 0 1 2 -There are 0 occurrences of 58 in the multiset -There are 536870912 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of null in the multiset - -Dafny program verifier finished with 17 verified, 0 errors -Sets: {} {17, 82} {17, 12} - cardinality: 0 2 2 - union: {17, 82} {17, 82, 12} - intersection: {} {17} - difference: {} {82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true -false true -|s|=4 |S|=16 -{{}, {a}, {b}, {a, b}, {c}, {a, c}, {d}, {b, c}, {a, d}, {a, b, c}, {b, d}, {a, b, d}, {c, d}, {a, c, d}, {b, c, d}, {a, b, c, d}} -Multisets: multiset{} multiset{17, 17, 82, 82} multiset{17, 12} - cardinality: 0 4 2 - union: multiset{17, 17, 82, 82} multiset{17, 17, 17, 82, 82, 12} - intersection: multiset{} multiset{17} - difference: multiset{} multiset{17, 82, 82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true - update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{17, 17, 12} - multiplicity: 0 2 1 -Sequences: [] [17, 82, 17, 82] [12, 17] - cardinality: 0 4 2 - update: [42, 82, 17, 82] [42, 17] - index: 17 12 - subsequence ([lo..hi]): [82, 17] [17] - subsequence ([lo..]): [82, 17, 82] [17] - subsequence ([..hi]): [17, 82, 17][12] - subsequence ([..]): [] [17, 82, 17, 82] [12, 17] - concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] - prefix: true false true - proper prefix: true false false - membership: false true true -[B, o, u, n, d] [B, o, u, n, d] [B, o, u, n, d] [B, o, u, n, d] [B, o, u, n, d] [B, o, u, n, d] -[e, d] [e, d] [e, d] [e, d] [e, d] [e, d] -e e e e e e -hello -hEllo -[2, 4, 6, 8, 10] -[2, 0, 6, 8, 10] -Strings: [] [u, R, u, R] [g, u] - cardinality: 0 4 2 - concatenation: [u, R, u, R] [u, R, u, R, g, u] - prefix: true false true - proper prefix: true false false - membership: false true true - constructed as sequence: [g, u, r, u] - mix: hello-d ddd-hello -Maps: map[] map[17 := 2, 82 := 2] map[17 := 0, 12 := 26] - cardinality: 0 2 2 - keys: {} {17, 82} {17, 12} - values: {} {2} {0, 26} - items: {} {(17, 2), (82, 2)} {(17, 0), (12, 26)} - update: map[17 := 6] map[17 := 6, 82 := 2] map[12 := 26, 17 := 6] - lookup: false 2 0 -m: map[Color.Yellow := 21, Color.Blue := 30] -keys: {Color.Yellow, Color.Blue} -values: {21, 30} -items: {(Color.Yellow, 21), (Color.Blue, 30)} -2: 0 1 1 -3: 0 1 2 -There are 0 occurrences of 58 in the multiset -There are 536870912 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of null in the multiset diff --git a/Test/comp/Comprehensions.dfy b/Test/comp/Comprehensions.dfy index a76077f1453..540643f3dad 100644 --- a/Test/comp/Comprehensions.dfy +++ b/Test/comp/Comprehensions.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method Main() { AssignSuchThat(); LetSuchThat(); diff --git a/Test/comp/Comprehensions.dfy.expect b/Test/comp/Comprehensions.dfy.expect index 6bfa826b70c..a6b302c2f34 100644 --- a/Test/comp/Comprehensions.dfy.expect +++ b/Test/comp/Comprehensions.dfy.expect @@ -20,69 +20,3 @@ XM returned: 16 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] [0, 1, 4, 9, 16, 25, 36, 49] [0, 1, 2, 3, 4, 5, 6, 7] - -Dafny program verifier finished with 16 verified, 0 errors -x=13 y=14 -x=13 y=14 b=yes -p=(13, 14) -q=(13, 14, yes) -true false -true false -map[12 := 6, 13 := 6, 14 := 7] -map[16 := 12, 17 := 13, 18 := 14] -XP returned: 0 false -after: 0 before: 0 -after: 2 before: 2 -XM returned: 16 -0 12 3 8 -0 12 3 8 -0 12 3 8 -0 12 3 8 -[1, 1, 1, 1] -[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] -[0, 1, 4, 9, 16, 25, 36, 49] -[0, 1, 2, 3, 4, 5, 6, 7] - -Dafny program verifier finished with 16 verified, 0 errors -x=13 y=14 -x=13 y=14 b=yes -p=(13, 14) -q=(13, 14, yes) -true false -true false -map[12 := 6, 13 := 6, 14 := 7] -map[16 := 12, 17 := 13, 18 := 14] -XP returned: 0 false -after: 0 before: 0 -after: 2 before: 2 -XM returned: 16 -0 12 3 8 -0 12 3 8 -0 12 3 8 -0 12 3 8 -[1, 1, 1, 1] -[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] -[0, 1, 4, 9, 16, 25, 36, 49] -[0, 1, 2, 3, 4, 5, 6, 7] - -Dafny program verifier finished with 16 verified, 0 errors -x=13 y=14 -x=13 y=14 b=[y, e, s] -p=(13, 14) -q=(13, 14, [y, e, s]) -true false -true false -map[12 := 6, 13 := 6, 14 := 7] -map[16 := 12, 17 := 13, 18 := 14] -XP returned: 0 false -after: 0 before: 0 -after: 2 before: 2 -XM returned: 16 -0 12 3 8 -0 12 3 8 -0 12 3 8 -0 12 3 8 -[1, 1, 1, 1] -[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] -[0, 1, 4, 9, 16, 25, 36, 49] -[0, 1, 2, 3, 4, 5, 6, 7] diff --git a/Test/comp/CovariantCollections.dfy b/Test/comp/CovariantCollections.dfy index 9913fbbad82..47f25bc8992 100644 --- a/Test/comp/CovariantCollections.dfy +++ b/Test/comp/CovariantCollections.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method Main() { // TODO-RS: Just testing the recent support for covariance in sequneces in C# for now. // These also work in Javascript, but hit runtime type errors in Go and still don't diff --git a/Test/comp/CovariantCollections.dfy.expect b/Test/comp/CovariantCollections.dfy.expect index 26c65f19d6a..f504f38e3af 100644 --- a/Test/comp/CovariantCollections.dfy.expect +++ b/Test/comp/CovariantCollections.dfy.expect @@ -12,28 +12,3 @@ Sequences: [] [_module.Integer, _module.Integer, _module.Integer, _module.Intege prefix: true false true proper prefix: true false false membership: false true true - -Dafny program verifier finished with 2 verified, 0 errors -Sequences: [] [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer] - cardinality: 0 4 2 - update: [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer] - index: _module.Integer _module.Integer - subsequence ([lo..hi]): [_module.Integer, _module.Integer] [_module.Integer] - subsequence ([lo..]): [_module.Integer, _module.Integer, _module.Integer] [_module.Integer] - subsequence ([..hi]): [_module.Integer, _module.Integer, _module.Integer][_module.Integer] - subsequence ([..]): [] [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer] - concatenation: [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer, _module.Integer, _module.Integer, _module.Integer, _module.Integer] - prefix: true false true - proper prefix: true false false - membership: false true true - -Dafny program verifier finished with 2 verified, 0 errors -Sequences: [] [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer] - cardinality: 0 4 2 - update: [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer] - index: Program halted: interface conversion: interface {} is *main.Integer, not *main.Number - -Dafny program verifier finished with 2 verified, 0 errors -CovariantCollections.dfy(34,6): Error: compilation of seq is not supported; consider introducing a ghost -CovariantCollections.dfy(35,6): Error: compilation of seq is not supported; consider introducing a ghost -File CovariantCollections/CovariantCollections.java contains the partially compiled program diff --git a/Test/comp/CovariantCollections.dfy.go.expect b/Test/comp/CovariantCollections.dfy.go.expect new file mode 100644 index 00000000000..14ac2bc410c --- /dev/null +++ b/Test/comp/CovariantCollections.dfy.go.expect @@ -0,0 +1,6 @@ + +Dafny program verifier finished with 2 verified, 0 errors +Sequences: [] [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer] + cardinality: 0 4 2 + update: [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer] + index: Program halted: interface conversion: interface {} is *main.Integer, not *main.Number diff --git a/Test/comp/CovariantCollections.dfy.java.expect b/Test/comp/CovariantCollections.dfy.java.expect new file mode 100644 index 00000000000..f6e772177e6 --- /dev/null +++ b/Test/comp/CovariantCollections.dfy.java.expect @@ -0,0 +1,5 @@ + +Dafny program verifier finished with 2 verified, 0 errors +CovariantCollections.dfy(34,6): Error: compilation of seq is not supported; consider introducing a ghost +CovariantCollections.dfy(35,6): Error: compilation of seq is not supported; consider introducing a ghost +File CovariantCollections/CovariantCollections.java contains the partially compiled program diff --git a/Test/comp/Dt.dfy b/Test/comp/Dt.dfy index aced23c719f..d30e30ae8b8 100644 --- a/Test/comp/Dt.dfy +++ b/Test/comp/Dt.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - datatype List = Nil | Cons(head: int, tail: List) method Main() { diff --git a/Test/comp/Dt.dfy.expect b/Test/comp/Dt.dfy.expect index 2a037c60b5d..01f48244ae3 100644 --- a/Test/comp/Dt.dfy.expect +++ b/Test/comp/Dt.dfy.expect @@ -17,60 +17,3 @@ CoBerry.Hjortron true true false false 42 q hello 1701 - -Dafny program verifier finished with 12 verified, 0 errors -List.Nil -List.Cons(5, List.Nil) -(List.Nil, List.Cons(5, List.Nil)) -(List.Cons(5, List.Nil), List.Nil) -() -5 List.Nil -List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Cons(4, List.Cons(5, List.Cons(6, List.Nil))))))) -0 + 1 + 2 + 3 + 4 + 5 + 6 == 21 (once more, that's 21) -List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Nil)))) -0 List.Cons(1, List.Cons(2, List.Cons(3, List.Nil))) -Stream.Next -Stream.Next -{Berry.Smultron, Berry.Jordgubb, Berry.Hallon} -CoBerry.Hjortron true true false -false -42 q hello -1701 - -Dafny program verifier finished with 12 verified, 0 errors -List.Nil -List.Cons(5, List.Nil) -(List.Nil, List.Cons(5, List.Nil)) -(List.Cons(5, List.Nil), List.Nil) -() -5 List.Nil -List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Cons(4, List.Cons(5, List.Cons(6, List.Nil))))))) -0 + 1 + 2 + 3 + 4 + 5 + 6 == 21 (once more, that's 21) -List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Nil)))) -0 List.Cons(1, List.Cons(2, List.Cons(3, List.Nil))) -Stream.Next -Stream.Next -{Berry.Smultron, Berry.Jordgubb, Berry.Hallon} -CoBerry.Hjortron true true false -false -42 q hello -1701 - -Dafny program verifier finished with 12 verified, 0 errors -List.Nil -List.Cons(5, List.Nil) -(List.Nil, List.Cons(5, List.Nil)) -(List.Cons(5, List.Nil), List.Nil) -() -5 List.Nil -List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Cons(4, List.Cons(5, List.Cons(6, List.Nil))))))) -0 + 1 + 2 + 3 + 4 + 5 + 6 == 21 (once more, that's 21) -List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Nil)))) -0 List.Cons(1, List.Cons(2, List.Cons(3, List.Nil))) -Stream.Next -Stream.Next -{Berry.Jordgubb, Berry.Smultron, Berry.Hallon} -CoBerry.Hjortron true true false -false -42 q hello -1701 diff --git a/Test/comp/Forall.dfy b/Test/comp/Forall.dfy index 4c54ba101ad..6c858e3591a 100644 --- a/Test/comp/Forall.dfy +++ b/Test/comp/Forall.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method Main() { var arrayTests := new ArrayTests(); arrayTests.Run(); diff --git a/Test/comp/Forall.dfy.expect b/Test/comp/Forall.dfy.expect index b79a345bbc6..c595c63df8e 100644 --- a/Test/comp/Forall.dfy.expect +++ b/Test/comp/Forall.dfy.expect @@ -42,135 +42,3 @@ Objects: Functions [(2, 1.0), (4, 2.0), (6, 3.0)] [(2, 1.0), (3, 2.0), (3, 3.0)] [(3, 1.0), (4, 2.0), (5, 3.0)] - -Dafny program verifier finished with 25 verified, 0 errors -Arrays: Basic cases -[0, 1, 2, 3, 4] -[0, 1, 2, 3, 4] -[0, 1, 4, 3, 8] - -Arrays: Wrong index -[0, 0, 1, 2, 3] -[0, 0, 1, 2, 3] -[1, 2, 3, 4, 5] - -Arrays: Sequence conversion -[2, 1, 4, 5, 6] -[0, 3, 3, 3, 4] - -Arrays: Index collection -[1, 1, 2, 3, 4] -[1, 1, 2, 3, 4] - -Arrays: Functions -[1, 1, 3, 4, 5] -[1, 1, 3, 4, 5] -[1, 2, 3, 3, 5] -[1, 2, 3, 4, 5] - -Multi-dimensional arrays -[[0, 2, 4], [1, 3, 5], [2, 4, 6]] -[[0, 1, 2], [2, 3, 4], [4, 5, 6]] - -Objects: Basic cases -[(0, 1.0), (0, 2.0), (0, 3.0)] -[(2, 1.0), (2, 2.0), (4, 3.0)] -[(2, 1.0), (3, 2.0), (3, 3.0)] - -Objects: Bad field accesses -[(2, 1.0), (3, 2.0), (4, 3.0)] -[(2, 1.0), (2, 2.0), (2, 3.0)] -[(2, 1.0), (3, 2.0), (1, 3.0)] - -Objects: Functions -[(2, 1.0), (4, 2.0), (6, 3.0)] -[(2, 1.0), (3, 2.0), (3, 3.0)] -[(3, 1.0), (4, 2.0), (5, 3.0)] - -Dafny program verifier finished with 25 verified, 0 errors -Arrays: Basic cases -[0, 1, 2, 3, 4] -[0, 1, 2, 3, 4] -[0, 1, 4, 3, 8] - -Arrays: Wrong index -[0, 0, 1, 2, 3] -[0, 0, 1, 2, 3] -[1, 2, 3, 4, 5] - -Arrays: Sequence conversion -[2, 1, 4, 5, 6] -[0, 3, 3, 3, 4] - -Arrays: Index collection -[1, 1, 2, 3, 4] -[1, 1, 2, 3, 4] - -Arrays: Functions -[1, 1, 3, 4, 5] -[1, 1, 3, 4, 5] -[1, 2, 3, 3, 5] -[1, 2, 3, 4, 5] - -Multi-dimensional arrays -[[0, 2, 4], [1, 3, 5], [2, 4, 6]] -[[0, 1, 2], [2, 3, 4], [4, 5, 6]] - -Objects: Basic cases -[(0, 1.0), (0, 2.0), (0, 3.0)] -[(2, 1.0), (2, 2.0), (4, 3.0)] -[(2, 1.0), (3, 2.0), (3, 3.0)] - -Objects: Bad field accesses -[(2, 1.0), (3, 2.0), (4, 3.0)] -[(2, 1.0), (2, 2.0), (2, 3.0)] -[(2, 1.0), (3, 2.0), (1, 3.0)] - -Objects: Functions -[(2, 1.0), (4, 2.0), (6, 3.0)] -[(2, 1.0), (3, 2.0), (3, 3.0)] -[(3, 1.0), (4, 2.0), (5, 3.0)] - -Dafny program verifier finished with 25 verified, 0 errors -Arrays: Basic cases -[0, 1, 2, 3, 4] -[0, 1, 2, 3, 4] -[0, 1, 4, 3, 8] - -Arrays: Wrong index -[0, 0, 1, 2, 3] -[0, 0, 1, 2, 3] -[1, 2, 3, 4, 5] - -Arrays: Sequence conversion -[2, 1, 4, 5, 6] -[0, 3, 3, 3, 4] - -Arrays: Index collection -[1, 1, 2, 3, 4] -[1, 1, 2, 3, 4] - -Arrays: Functions -[1, 1, 3, 4, 5] -[1, 1, 3, 4, 5] -[1, 2, 3, 3, 5] -[1, 2, 3, 4, 5] - -Multi-dimensional arrays -[[0, 2, 4], [1, 3, 5], [2, 4, 6]] -[[0, 1, 2], [2, 3, 4], [4, 5, 6]] - -Objects: Basic cases -[(0, 1.0), (0, 2.0), (0, 3.0)] -[(2, 1.0), (2, 2.0), (4, 3.0)] -[(2, 1.0), (3, 2.0), (3, 3.0)] - -Objects: Bad field accesses -[(2, 1.0), (3, 2.0), (4, 3.0)] -[(2, 1.0), (2, 2.0), (2, 3.0)] -[(2, 1.0), (3, 2.0), (1, 3.0)] - -Objects: Functions -[(2, 1.0), (4, 2.0), (6, 3.0)] -[(2, 1.0), (3, 2.0), (3, 3.0)] -[(3, 1.0), (4, 2.0), (5, 3.0)] diff --git a/Test/comp/Ghosts.dfy b/Test/comp/Ghosts.dfy index 11ed8478a6b..c842b3bd8d8 100644 --- a/Test/comp/Ghosts.dfy +++ b/Test/comp/Ghosts.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method M1() returns (x: int) { print "hello, M1\n"; diff --git a/Test/comp/Ghosts.dfy.expect b/Test/comp/Ghosts.dfy.expect index e584e53feb0..38bd2cd6709 100644 --- a/Test/comp/Ghosts.dfy.expect +++ b/Test/comp/Ghosts.dfy.expect @@ -10,39 +10,3 @@ hello, M2 hello, M3 hello, N0 hello, N1 - -Dafny program verifier finished with 8 verified, 0 errors -hello, M2 -hello, M2 -hello, M3 -hello, M1 -hello, M1 -hello, M1 -hello, M2 -hello, M3 -hello, N0 -hello, N1 - -Dafny program verifier finished with 8 verified, 0 errors -hello, M2 -hello, M2 -hello, M3 -hello, M1 -hello, M1 -hello, M1 -hello, M2 -hello, M3 -hello, N0 -hello, N1 - -Dafny program verifier finished with 8 verified, 0 errors -hello, M2 -hello, M2 -hello, M3 -hello, M1 -hello, M1 -hello, M1 -hello, M2 -hello, M3 -hello, N0 -hello, N1 diff --git a/Test/comp/Hello.dfy.expect b/Test/comp/Hello.dfy.expect index 7346deaafd9..2350367ed58 100644 --- a/Test/comp/Hello.dfy.expect +++ b/Test/comp/Hello.dfy.expect @@ -3,18 +3,3 @@ Dafny program verifier finished with 0 verified, 0 errors Hello, JavaScript! Best, Dafny x is 14 y is false - -Dafny program verifier finished with 0 verified, 0 errors -Hello, JavaScript! Best, Dafny -x is 14 -y is false - -Dafny program verifier finished with 0 verified, 0 errors -Hello, JavaScript! Best, Dafny -x is 14 -y is false - -Dafny program verifier finished with 0 verified, 0 errors -Hello, JavaScript! Best, Dafny -x is 14 -y is false diff --git a/Test/comp/Iterators.dfy b/Test/comp/Iterators.dfy index e5a123e188e..03aa242c6b2 100644 --- a/Test/comp/Iterators.dfy +++ b/Test/comp/Iterators.dfy @@ -1,8 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - class C { var x: int // for variety, the following tests the use of an instance Main method diff --git a/Test/comp/Iterators.dfy.expect b/Test/comp/Iterators.dfy.expect index 22c894142fe..86d1b673939 100644 --- a/Test/comp/Iterators.dfy.expect +++ b/Test/comp/Iterators.dfy.expect @@ -4,15 +4,3 @@ hello, instance x is 0 0.0 1.0 2.0 3.0 4.0 5.0 0.0 1.0 2.0 - -Dafny program verifier finished with 5 verified, 0 errors -hello, instance -x is 0 -0.0 1.0 2.0 3.0 4.0 5.0 -0.0 1.0 2.0 - -Dafny program verifier finished with 5 verified, 0 errors -hello, instance -x is 0 -0.0 1.0 2.0 3.0 4.0 5.0 -0.0 1.0 2.0 diff --git a/Test/comp/Let.dfy b/Test/comp/Let.dfy index 521a5e06659..12685117927 100644 --- a/Test/comp/Let.dfy +++ b/Test/comp/Let.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method M() returns (x: int) { x := var y := 50; y; // non-top-level let } diff --git a/Test/comp/Let.dfy.expect b/Test/comp/Let.dfy.expect index 5e1d0433456..2c1728235f1 100644 --- a/Test/comp/Let.dfy.expect +++ b/Test/comp/Let.dfy.expect @@ -5,24 +5,3 @@ Tree.Node(Tree.Node(Tree.Leaf, 5, Tree.Node(Tree.Leaf, 7, Tree.Leaf)), 9, Tree.N 5 7 9 10 12 30 5 7 5 7 - -Dafny program verifier finished with 3 verified, 0 errors -50 58 -Tree.Node(Tree.Node(Tree.Leaf, 5, Tree.Node(Tree.Leaf, 7, Tree.Leaf)), 9, Tree.Node(Tree.Leaf, 10, Tree.Node(Tree.Leaf, 12, Tree.Node(Tree.Leaf, 30, Tree.Leaf)))) -5 7 9 10 12 30 -5 7 -5 7 - -Dafny program verifier finished with 3 verified, 0 errors -50 58 -Tree.Node(Tree.Node(Tree.Leaf, 5, Tree.Node(Tree.Leaf, 7, Tree.Leaf)), 9, Tree.Node(Tree.Leaf, 10, Tree.Node(Tree.Leaf, 12, Tree.Node(Tree.Leaf, 30, Tree.Leaf)))) -5 7 9 10 12 30 -5 7 -5 7 - -Dafny program verifier finished with 3 verified, 0 errors -50 58 -Tree.Node(Tree.Node(Tree.Leaf, 5, Tree.Node(Tree.Leaf, 7, Tree.Leaf)), 9, Tree.Node(Tree.Leaf, 10, Tree.Node(Tree.Leaf, 12, Tree.Node(Tree.Leaf, 30, Tree.Leaf)))) -5 7 9 10 12 30 -5 7 -5 7 diff --git a/Test/comp/Module.dfy b/Test/comp/Module.dfy index 909886f9ac5..4987cf6bd01 100644 --- a/Test/comp/Module.dfy +++ b/Test/comp/Module.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - // Simple sanity test of nested modules module Parent { module Child { diff --git a/Test/comp/Module.dfy.expect b/Test/comp/Module.dfy.expect index 5a18e86f45c..d2b3cafd1c9 100644 --- a/Test/comp/Module.dfy.expect +++ b/Test/comp/Module.dfy.expect @@ -1,12 +1,3 @@ Dafny program verifier finished with 1 verified, 0 errors hi from a nested module - -Dafny program verifier finished with 1 verified, 0 errors -hi from a nested module - -Dafny program verifier finished with 1 verified, 0 errors -hi from a nested module - -Dafny program verifier finished with 1 verified, 0 errors -hi from a nested module diff --git a/Test/comp/NativeNumbers.dfy.expect b/Test/comp/NativeNumbers.dfy.expect index bdadd002058..bcbb4d5ca89 100644 --- a/Test/comp/NativeNumbers.dfy.expect +++ b/Test/comp/NativeNumbers.dfy.expect @@ -82,171 +82,3 @@ uint64: 0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y 23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N - -Dafny program verifier finished with 25 verified, 0 errors -Casting: - -Small numbers: -0 0 0 0 0 0 0 0 0 -1 1 1 1 1 1 1 1 1 -2 2 2 2 2 2 2 2 2 -3 3 3 3 3 3 3 3 3 -4 4 4 4 4 4 4 4 4 -5 5 5 5 5 5 5 5 5 -6 6 6 6 6 6 6 6 6 -7 7 7 7 7 7 7 7 7 -8 8 8 8 8 8 8 8 8 - -Large unsigned numbers: -255 255 255 255 255 255 255 255 -65535 65535 65535 65535 65535 65535 -4294967295 4294967295 4294967295 4294967295 -18446744073709551615 18446744073709551615 - -Int to large unsigned: -255 65535 4294967295 18446744073709551615 - -Cast from cardinality operator: -0 0 0 0 0 0 0 0 0 -1 1 1 1 1 1 1 1 1 -2 2 2 2 2 2 2 2 2 -3 3 3 3 3 3 3 3 3 -4 4 4 4 4 4 4 4 4 - -Characters: -C 67 67 67 67 67 67 67 67 67 -0 127 32767 65535 65535 255 65535 65535 65535 - -Defaults: - -0 0 0 0 0 0 0 0 0 - -Byte arithmetic: - -20 30 -10 10 18 - -Short arithmetic: - -20 30 -10 10 18 - -Bitvectors: - -0 3 4 1 - -Comparison to zero: - -int8: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -int16: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -int32: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -int64: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -uint8: -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -uint16: -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -uint32: -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -uint64: -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N - - -Dafny program verifier finished with 25 verified, 0 errors -Casting: - -Small numbers: -0 0 0 0 0 0 0 0 0 -1 1 1 1 1 1 1 1 1 -2 2 2 2 2 2 2 2 2 -3 3 3 3 3 3 3 3 3 -4 4 4 4 4 4 4 4 4 -5 5 5 5 5 5 5 5 5 -6 6 6 6 6 6 6 6 6 -7 7 7 7 7 7 7 7 7 -8 8 8 8 8 8 8 8 8 - -Large unsigned numbers: -255 255 255 255 255 255 255 255 -65535 65535 65535 65535 65535 65535 -4294967295 4294967295 4294967295 4294967295 -18446744073709551615 18446744073709551615 - -Int to large unsigned: -255 65535 4294967295 18446744073709551615 - -Cast from cardinality operator: -0 0 0 0 0 0 0 0 0 -1 1 1 1 1 1 1 1 1 -2 2 2 2 2 2 2 2 2 -3 3 3 3 3 3 3 3 3 -4 4 4 4 4 4 4 4 4 - -Characters: -C 67 67 67 67 67 67 67 67 67 -0 127 32767 65535 65535 255 65535 65535 65535 - -Defaults: - -0 0 0 0 0 0 0 0 0 - -Byte arithmetic: - -20 30 -10 10 18 - -Short arithmetic: - -20 30 -10 10 18 - -Bitvectors: - -0 3 4 1 - -Comparison to zero: - -int8: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -int16: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -int32: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -int64: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -uint8: -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -uint16: -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -uint32: -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -uint64: -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N - diff --git a/Test/comp/NativeNumbers.dfy.js.expect b/Test/comp/NativeNumbers.dfy.js.expect new file mode 100644 index 00000000000..90a7c69a392 --- /dev/null +++ b/Test/comp/NativeNumbers.dfy.js.expect @@ -0,0 +1,9 @@ +NativeNumbers.dfy(17,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(18,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(19,28): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(20,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(22,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(23,31): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(24,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(25,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +8 resolution/type errors detected in NativeNumbers.dfy diff --git a/Test/comp/Numbers.dfy b/Test/comp/Numbers.dfy index 83643e5c22e..c3028251bf0 100644 --- a/Test/comp/Numbers.dfy +++ b/Test/comp/Numbers.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method Main() { Literals(); Arithmetic(); diff --git a/Test/comp/Numbers.dfy.expect b/Test/comp/Numbers.dfy.expect index caa0e3f1093..11d8d6ac08b 100644 --- a/Test/comp/Numbers.dfy.expect +++ b/Test/comp/Numbers.dfy.expect @@ -90,279 +90,3 @@ MyNumber: 120 120.0 120 120 120 x 120 120.0 120 120 120 x 120 120 - -Dafny program verifier finished with 29 verified, 0 errors -0 -0 -3 --5 -2147483647 (aka C# int.MaxValue) -2147483648 (aka 2^31) -2147483649 -4294967295 (aka C# uint.MaxValue) -4294967296 (aka 2^32) -590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) -590295810358705651712 (aka 2^53) --590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) --590295810358705651712 -9223372036854775807 (aka C# long.MaxValue) -9223372036854775808 (aka 2^63) -9223372036854775809 -18446744073709551615 (aka C# ulong.MaxValue) -18446744073709551616 (aka 2^64) -1267650600228229401496703205376 (aka 2^100) -170141183460469231731687303715884105727 (aka M_39) -0.0 -0.0 -3.0 --5.0 -3.14 --2.71 -1000000000.0 (aka a billion) -0.0000000000667408 (aka G) -0.0000000000667408 (aka G) -0.0000000000667408 (aka G) -17014118346046923173168730371588410572.7 (aka 1/10 of M_39) -0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 (aka 1/googol) -35 27 -27 -124 -124 --124 124 -g Q 2 -7 3 31 --8 1 -31 --7 3 31 -8 1 -31 -0 0 0 -0 0 0 -0.2 -35.2 27.2 -27.2 -124.8 -124.8 --124.8 124.8 -(312.0 / 40.0) (1248.0 / 40.0) -(-312.0 / 40.0) (-1248.0 / 40.0) -(-312.0 / 40.0) (1248.0 / 40.0) -(312.0 / 40.0) (-1248.0 / 40.0) -0.0 0.0 0.0 -120.0 120.0 (24.0 / 3.0) (-24.0 / 3.0) -123.4567 -123.4567 0.1234 -0.1234 (24.0 / 30.0) -0.2 0.02 0.00002 --0.2 -0.02 -0.00002 -(20.0 / 3.0) (-20.0 / 3.0) (-20.0 / 3.0) (20.0 / 3.0) -0.0 0.0 0.81 0.81 (8100.0 / 8100.0) -true false -0 2 2 2 -0 3 6 46 -0 1 36 2116 -0 0 0 0 -29 2 2 4294967283 2 -29 2 2 9007199254740979 2 -0 0 0 9 -0 0 0 9 -0 0 0 9 -0 4294967295 1267650600228229401496703205375 6 -8 4 -18 72 4 -0 -200 300 -100 100 18 -0: IsNat: true, Offset: 0, IsLimit: true, IsSucc: false -1: IsNat: true, Offset: 1, IsLimit: false, IsSucc: true -42: IsNat: true, Offset: 42, IsLimit: false, IsSucc: true -int: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -MyNumber: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -120 120.0 120 120 120 x -120 120.0 120 120 120 x -120 120 -120 120.0 120 120 120 x -120 120.0 120 120 120 x -120 120 - -Dafny program verifier finished with 29 verified, 0 errors -0 -0 -3 --5 -2147483647 (aka C# int.MaxValue) -2147483648 (aka 2^31) -2147483649 -4294967295 (aka C# uint.MaxValue) -4294967296 (aka 2^32) -590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) -590295810358705651712 (aka 2^53) --590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) --590295810358705651712 -9223372036854775807 (aka C# long.MaxValue) -9223372036854775808 (aka 2^63) -9223372036854775809 -18446744073709551615 (aka C# ulong.MaxValue) -18446744073709551616 (aka 2^64) -1267650600228229401496703205376 (aka 2^100) -170141183460469231731687303715884105727 (aka M_39) -0.0 -0.0 -3.0 --5.0 -3.14 --2.71 -1000000000.0 (aka a billion) -0.0000000000667408 (aka G) -0.0000000000667408 (aka G) -0.0000000000667408 (aka G) -17014118346046923173168730371588410572.7 (aka 1/10 of M_39) -0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 (aka 1/googol) -35 27 -27 -124 -124 --124 124 -g Q 2 -7 3 31 --8 1 -31 --7 3 31 -8 1 -31 -0 0 0 -0 0 0 -0.2 -35.2 27.2 -27.2 -124.8 -124.8 --124.8 124.8 -7.8 31.2 --7.8 -31.2 --7.8 31.2 -7.8 -31.2 -0.0 0.0 0.0 -120.0 120.0 8.0 -8.0 -123.4567 -123.4567 0.1234 -0.1234 0.8 -0.2 0.02 0.00002 --0.2 -0.02 -0.00002 -(20.0 / 3.0) (-20.0 / 3.0) (-20.0 / 3.0) (20.0 / 3.0) -0.0 0.0 0.81 0.81 1.0 -true false -0 2 2 2 -0 3 6 46 -0 1 36 2116 -0 0 0 0 -29 2 2 4294967283 2 -29 2 2 9007199254740979 2 -0 0 0 9 -0 0 0 9 -0 0 0 9 -0 4294967295 1267650600228229401496703205375 6 -8 4 -18 72 4 -0 -200 300 -100 100 18 -0: IsNat: true, Offset: 0, IsLimit: true, IsSucc: false -1: IsNat: true, Offset: 1, IsLimit: false, IsSucc: true -42: IsNat: true, Offset: 42, IsLimit: false, IsSucc: true -int: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -MyNumber: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -120 120.0 120 120 120 x -120 120.0 120 120 120 x -120 120 -120 120.0 120 120 120 x -120 120.0 120 120 120 x -120 120 - -Dafny program verifier finished with 29 verified, 0 errors -0 -0 -3 --5 -2147483647 (aka C# int.MaxValue) -2147483648 (aka 2^31) -2147483649 -4294967295 (aka C# uint.MaxValue) -4294967296 (aka 2^32) -590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) -590295810358705651712 (aka 2^53) --590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) --590295810358705651712 -9223372036854775807 (aka C# long.MaxValue) -9223372036854775808 (aka 2^63) -9223372036854775809 -18446744073709551615 (aka C# ulong.MaxValue) -18446744073709551616 (aka 2^64) -1267650600228229401496703205376 (aka 2^100) -170141183460469231731687303715884105727 (aka M_39) -0.0 -0.0 -3.0 --5.0 -3.14 --2.71 -1000000000.0 (aka a billion) -0.0000000000667408 (aka G) -0.0000000000667408 (aka G) -0.0000000000667408 (aka G) -17014118346046923173168730371588410572.7 (aka 1/10 of M_39) -0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 (aka 1/googol) -35 27 -27 -124 -124 --124 124 -g Q 2 -7 3 31 --8 1 -31 --7 3 31 -8 1 -31 -0 0 0 -0 0 0 -0.2 -35.2 27.2 -27.2 -124.8 -124.8 --124.8 124.8 -(312.0 / 40.0) (1248.0 / 40.0) -(-312.0 / 40.0) (-1248.0 / 40.0) -(-312.0 / 40.0) (1248.0 / 40.0) -(312.0 / 40.0) (-1248.0 / 40.0) -0.0 0.0 0.0 -120.0 120.0 (24.0 / 3.0) (-24.0 / 3.0) -123.4567 -123.4567 0.1234 -0.1234 (24.0 / 30.0) -0.2 0.02 0.00002 --0.2 -0.02 -0.00002 -(20.0 / 3.0) (-20.0 / 3.0) (-20.0 / 3.0) (20.0 / 3.0) -0.0 0.0 0.81 0.81 (8100.0 / 8100.0) -true false -0 2 2 2 -0 3 6 46 -0 1 36 2116 -0 0 0 0 -29 2 2 4294967283 2 -29 2 2 9007199254740979 2 -0 0 0 9 -0 0 0 9 -0 0 0 9 -0 4294967295 1267650600228229401496703205375 6 -8 4 -18 72 4 -0 -200 300 -100 100 18 -0: IsNat: true, Offset: 0, IsLimit: true, IsSucc: false -1: IsNat: true, Offset: 1, IsLimit: false, IsSucc: true -42: IsNat: true, Offset: 42, IsLimit: false, IsSucc: true -int: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -MyNumber: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -120 120.0 120 120 120 x -120 120.0 120 120 120 x -120 120 -120 120.0 120 120 120 x -120 120.0 120 120 120 x -120 120 diff --git a/Test/comp/Poly.dfy b/Test/comp/Poly.dfy index 9f7938c32b0..a34b7991c6b 100644 --- a/Test/comp/Poly.dfy +++ b/Test/comp/Poly.dfy @@ -1,7 +1,3 @@ -// Currently only enabled for Go due to seq etc. -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" > "%t" -// RUN: %diff "%s.expect" "%t" - trait Shape { function method Center(): (real, real) reads this method PrintCenter() { diff --git a/Test/comp/TailRecursion.dfy b/Test/comp/TailRecursion.dfy index 136c23a31fa..5ebb3f30083 100644 --- a/Test/comp/TailRecursion.dfy +++ b/Test/comp/TailRecursion.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method Main() { // In the following, 2_000_000 is too large an argument without tail-calls var x := M(2_000_000, 0); diff --git a/Test/comp/TailRecursion.dfy.expect b/Test/comp/TailRecursion.dfy.expect index 32ad9ffbd23..886da966a21 100644 --- a/Test/comp/TailRecursion.dfy.expect +++ b/Test/comp/TailRecursion.dfy.expect @@ -14,51 +14,3 @@ DownFrom(10) = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] Sum(List.Cons(100, List.Cons(40, List.Cons(60, List.Nil)))) = 200 TheBigSubtract(100) = -12 TailNat(10) = 50 - -Dafny program verifier finished with 20 verified, 0 errors -2000000 2000000 -2000000 2000000 -1000000 1000000 -10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 == 55 -TriangleNumber(10) = 55 -TriangleNumber_Real(10) = 55.0 -TriangleNumber_ORDINAL(10) = 55 -Factorial(5) = 120 -Union(8) = {0, 2, 4, 6, 8, 7, 5, 3, 1} -UpTo(10) = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -DownFrom(10) = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] -Sum(List.Cons(100, List.Cons(40, List.Cons(60, List.Nil)))) = 200 -TheBigSubtract(100) = -12 -TailNat(10) = 50 - -Dafny program verifier finished with 20 verified, 0 errors -2000000 2000000 -2000000 2000000 -1000000 1000000 -10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 == 55 -TriangleNumber(10) = 55 -TriangleNumber_Real(10) = 55.0 -TriangleNumber_ORDINAL(10) = 55 -Factorial(5) = 120 -Union(8) = {0, 2, 4, 6, 8, 7, 5, 3, 1} -UpTo(10) = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -DownFrom(10) = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] -Sum(List.Cons(100, List.Cons(40, List.Cons(60, List.Nil)))) = 200 -TheBigSubtract(100) = -12 -TailNat(10) = 50 - -Dafny program verifier finished with 20 verified, 0 errors -2000000 2000000 -2000000 2000000 -1000000 1000000 -10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 == 55 -TriangleNumber(10) = 55 -TriangleNumber_Real(10) = 55.0 -TriangleNumber_ORDINAL(10) = 55 -Factorial(5) = 120 -Union(8) = {0, 1, 2, 3, 4, 5, 6, 7, 8} -UpTo(10) = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -DownFrom(10) = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] -Sum(List.Cons(100, List.Cons(40, List.Cons(60, List.Nil)))) = 200 -TheBigSubtract(100) = -12 -TailNat(10) = 50 diff --git a/Test/comp/TypeParams.dfy b/Test/comp/TypeParams.dfy index 4b73ee67ba2..309a6e9e58a 100644 --- a/Test/comp/TypeParams.dfy +++ b/Test/comp/TypeParams.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - datatype Color = Orange | Pink | Teal type Six = x | x <= 6 newtype Even = x | x % 2 == 0 diff --git a/Test/comp/TypeParams.dfy.cs.expect b/Test/comp/TypeParams.dfy.cs.expect new file mode 100644 index 00000000000..ec5dac62f22 --- /dev/null +++ b/Test/comp/TypeParams.dfy.cs.expect @@ -0,0 +1,20 @@ + +Dafny program verifier finished with 20 verified, 0 errors +0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) +null null null null null +{} multiset{} [] map[] {} map[] +null null null null +0 [] Q {} 0 Dt.D0(0) +false Color.Orange 42 {} false Dt.D0(false) +NonemptyList.Atom(0) +6 7 8 9 +true true +IList.INil IList.ICons +IList.INil IList.INil Stream.Next IList.INil +NonemptyList.Atom(0) +NonemptyCoList.CoAtom +Color.Pink 19 -3 +null null null null +System.Func`2[Dafny.BigRational,System.Boolean] +System.Func`1[System.Numerics.BigInteger] +System.Func`3[_module.Color,Dafny.Set`1[System.UInt16],System.UInt32] diff --git a/Test/comp/TypeParams.dfy.expect b/Test/comp/TypeParams.dfy.expect deleted file mode 100644 index d6d8a685721..00000000000 --- a/Test/comp/TypeParams.dfy.expect +++ /dev/null @@ -1,80 +0,0 @@ - -Dafny program verifier finished with 20 verified, 0 errors -0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) -null null null null null -{} multiset{} [] map[] {} map[] -null null null null -0 [] Q {} 0 Dt.D0(0) -false Color.Orange 42 {} false Dt.D0(false) -NonemptyList.Atom(0) -6 7 8 9 -true true -IList.INil IList.ICons -IList.INil IList.INil Stream.Next IList.INil -NonemptyList.Atom(0) -NonemptyCoList.CoAtom -Color.Pink 19 -3 -null null null null -System.Func`2[Dafny.BigRational,System.Boolean] -System.Func`1[System.Numerics.BigInteger] -System.Func`3[_module.Color,Dafny.Set`1[System.UInt16],System.UInt32] - -Dafny program verifier finished with 20 verified, 0 errors -0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) -null null null null null -{} multiset{} [] map[] {} map[] -null null null null -0 [] Q {} 0 Dt.D0(0) -false Color.Orange 42 {} false Dt.D0(false) -NonemptyList.Atom(0) -6 7 8 9 -true true -IList.INil IList.ICons -IList.INil IList.INil Stream.Next IList.INil -NonemptyList.Atom(0) -NonemptyCoList.CoAtom -Color.Pink 19 -3 -null null null null -function () { return false; } -function () { return new BigNumber(0); } -function () { return new BigNumber(0); } - -Dafny program verifier finished with 20 verified, 0 errors -0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) -null null null null null -{} multiset{} [] map[] {} map[] -null null null null -0 [] Q {} 0 Dt.D0(0) -false Color.Orange 42 {} false Dt.D0(false) -NonemptyList.Atom(0) -6 7 8 9 -true true -IList.INil IList.ICons -IList.INil IList.INil Stream.Next IList.INil -NonemptyList.Atom(0) -NonemptyCoList.CoAtom -Color.Pink 19 -3 -null null null null -func(dafny.Real) bool -func() dafny.Int -func(main.Color, dafny.Set) uint32 - -Dafny program verifier finished with 20 verified, 0 errors -0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) -null null null null null -{} multiset{} [] map[] {} map[] -null null null null -0 [] Q {} 0 Dt.D0(0) -false Color.Orange 42 {} false Dt.D0(false) -NonemptyList.Atom(0) -6 7 8 9 -true true -IList.INil IList.ICons -IList.INil IList.INil Stream.Next IList.INil -NonemptyList.Atom(0) -NonemptyCoList.CoAtom -Color.Pink 19 -3 -null null null null -Function -Function -Function diff --git a/Test/comp/TypeParams.dfy.go.expect b/Test/comp/TypeParams.dfy.go.expect new file mode 100644 index 00000000000..d45da1d6a81 --- /dev/null +++ b/Test/comp/TypeParams.dfy.go.expect @@ -0,0 +1,20 @@ + +Dafny program verifier finished with 20 verified, 0 errors +0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) +null null null null null +{} multiset{} [] map[] {} map[] +null null null null +0 [] Q {} 0 Dt.D0(0) +false Color.Orange 42 {} false Dt.D0(false) +NonemptyList.Atom(0) +6 7 8 9 +true true +IList.INil IList.ICons +IList.INil IList.INil Stream.Next IList.INil +NonemptyList.Atom(0) +NonemptyCoList.CoAtom +Color.Pink 19 -3 +null null null null +func(dafny.Real) bool +func() dafny.Int +func(main.Color, dafny.Set) uint32 diff --git a/Test/comp/TypeParams.dfy.java.expect b/Test/comp/TypeParams.dfy.java.expect new file mode 100644 index 00000000000..87aa7de5390 --- /dev/null +++ b/Test/comp/TypeParams.dfy.java.expect @@ -0,0 +1,20 @@ + +Dafny program verifier finished with 20 verified, 0 errors +0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) +null null null null null +{} multiset{} [] map[] {} map[] +null null null null +0 [] Q {} 0 Dt.D0(0) +false Color.Orange 42 {} false Dt.D0(false) +NonemptyList.Atom(0) +6 7 8 9 +true true +IList.INil IList.ICons +IList.INil IList.INil Stream.Next IList.INil +NonemptyList.Atom(0) +NonemptyCoList.CoAtom +Color.Pink 19 -3 +null null null null +Function +Function +Function diff --git a/Test/comp/TypeParams.dfy.js.expect b/Test/comp/TypeParams.dfy.js.expect new file mode 100644 index 00000000000..08c5b3e415a --- /dev/null +++ b/Test/comp/TypeParams.dfy.js.expect @@ -0,0 +1,20 @@ + +Dafny program verifier finished with 20 verified, 0 errors +0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) +null null null null null +{} multiset{} [] map[] {} map[] +null null null null +0 [] Q {} 0 Dt.D0(0) +false Color.Orange 42 {} false Dt.D0(false) +NonemptyList.Atom(0) +6 7 8 9 +true true +IList.INil IList.ICons +IList.INil IList.INil Stream.Next IList.INil +NonemptyList.Atom(0) +NonemptyCoList.CoAtom +Color.Pink 19 -3 +null null null null +function () { return false; } +function () { return new BigNumber(0); } +function () { return new BigNumber(0); } From 396db408dd48b85be63da54de9788fb8a34c3c35 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 4 Jun 2020 16:33:11 -0700 Subject: [PATCH 006/192] Support skipping tests as well, more test fixes --- Test/DafnyTests/DafnyTests.cs | 11 +++++++---- .../CollectionPerTestCaseTheoryDiscoverer.cs | 6 +++--- Test/comp/CovariantCollections.dfy.java.expect | 4 ++-- Test/comp/NativeNumbers.dfy | 7 ------- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 50750b9c7aa..c8a5dc2bc05 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -55,21 +55,24 @@ public static string RunDafny(IEnumerable arguments) { dafnyProcess.StartInfo.RedirectStandardOutput = true; dafnyProcess.StartInfo.RedirectStandardError = true; dafnyProcess.StartInfo.CreateNoWindow = true; - dafnyProcess.StartInfo.WorkingDirectory = Path.Combine(DAFNY_ROOT, "Binaries"); + // Necessary for JS to find bignumber.js + dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; // Only preserve specific whitelisted environment variables dafnyProcess.StartInfo.EnvironmentVariables.Clear(); dafnyProcess.StartInfo.EnvironmentVariables.Add("PATH", System.Environment.GetEnvironmentVariable("PATH")); - + // Go requires this or GOCACHE + dafnyProcess.StartInfo.EnvironmentVariables.Add("HOME", System.Environment.GetEnvironmentVariable("HOME")); + dafnyProcess.Start(); dafnyProcess.WaitForExit(); string output = dafnyProcess.StandardOutput.ReadToEnd(); + string error = dafnyProcess.StandardError.ReadToEnd(); if (dafnyProcess.ExitCode != 0) { - string error = dafnyProcess.StandardError.ReadToEnd(); Assert.True(false, output + "\n" + error); } - return output; + return output + error; } } diff --git a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index 9868731e5d4..4a2c37b1654 100644 --- a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -6,18 +6,18 @@ namespace XUnitExtensions { public class CollectionPerTestCaseTheoryDiscoverer : IXunitTestCaseDiscoverer { - readonly TheoryDiscoverer theoryDiscoverer; + readonly IXunitTestCaseDiscoverer theoryDiscoverer; public CollectionPerTestCaseTheoryDiscoverer(IMessageSink diagnosticMessageSink) { - theoryDiscoverer = new TheoryDiscoverer(diagnosticMessageSink); + theoryDiscoverer = new SkippableTheoryDiscoverer(diagnosticMessageSink); } private TestCollection testCollectionForTestCase(IXunitTestCase testCase) { return new TestCollection(testCase.TestMethod.TestClass.TestCollection.TestAssembly, (ITypeInfo) null, "Test collection for " + testCase.DisplayName); } - + public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { return theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute) .Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); diff --git a/Test/comp/CovariantCollections.dfy.java.expect b/Test/comp/CovariantCollections.dfy.java.expect index f6e772177e6..270c270419b 100644 --- a/Test/comp/CovariantCollections.dfy.java.expect +++ b/Test/comp/CovariantCollections.dfy.java.expect @@ -1,5 +1,5 @@ Dafny program verifier finished with 2 verified, 0 errors -CovariantCollections.dfy(34,6): Error: compilation of seq is not supported; consider introducing a ghost -CovariantCollections.dfy(35,6): Error: compilation of seq is not supported; consider introducing a ghost +CovariantCollections.dfy(28,6): Error: compilation of seq is not supported; consider introducing a ghost +CovariantCollections.dfy(29,6): Error: compilation of seq is not supported; consider introducing a ghost File CovariantCollections/CovariantCollections.java contains the partially compiled program diff --git a/Test/comp/NativeNumbers.dfy b/Test/comp/NativeNumbers.dfy index bbf449cbdbd..e9c73eb0518 100644 --- a/Test/comp/NativeNumbers.dfy +++ b/Test/comp/NativeNumbers.dfy @@ -1,10 +1,3 @@ -// Skip JavaScript because JavaScript doesn't have the same native types - -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method Main() { CastTests(); DefaultTests(); From 42db43811a43539b25106e60f3d526d1f1250d6b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 5 Jun 2020 13:49:15 -0700 Subject: [PATCH 007/192] Generalized test cases to use pseudo-shebangs --- Binaries/testdafny | 2 + Test/DafnyTests/DafnyTests.cs | 71 ++++++++++++++++++------------ Test/comp/Arrays.dfy | 6 +-- Test/comp/Calls.dfy | 2 + Test/comp/Collections.dfy | 2 + Test/comp/Comprehensions.dfy | 2 + Test/comp/CovariantCollections.dfy | 2 + Test/comp/Dt.dfy | 2 + Test/comp/Forall.dfy | 2 + Test/comp/Ghosts.dfy | 2 + Test/comp/Hello.dfy | 6 +-- Test/comp/Iterators.dfy | 2 + Test/comp/Let.dfy | 2 + Test/comp/Module.dfy | 2 + Test/comp/Numbers.dfy | 2 + Test/comp/Poly.dfy | 2 + Test/comp/TailRecursion.dfy | 2 + Test/comp/TypeParams.dfy | 2 + Test/dafny0/AdvancedLHS.dfy | 4 +- 19 files changed, 78 insertions(+), 39 deletions(-) create mode 100755 Binaries/testdafny mode change 100644 => 100755 Test/comp/Hello.dfy diff --git a/Binaries/testdafny b/Binaries/testdafny new file mode 100755 index 00000000000..f78b2c9ec25 --- /dev/null +++ b/Binaries/testdafny @@ -0,0 +1,2 @@ +DAFNY_TEST_ARGV=$* +dotnet test ../../Test/DafnyTests/DafnyTests-NetCore.csproj diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index c8a5dc2bc05..6a274c9aa0f 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.CodeDom.Compiler; using System.Collections; using System.Collections.Generic; @@ -18,16 +19,12 @@ public class DafnyTests { private static string DAFNY_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()).Parent.Parent.Parent.Parent.Parent.FullName; private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); - private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test"); + private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; - private string RunDafnyProgram(string filePath, params string[] arguments) { + public static string RunDafny(IEnumerable arguments) { List dafnyArguments = new List { - - filePath, - "/compile:3", - // Expected output does not contain logo "-nologo", "-countVerificationErrors:0", @@ -40,14 +37,11 @@ private string RunDafnyProgram(string filePath, params string[] arguments) { "-compileVerbose:0" }; dafnyArguments.AddRange(arguments); - return RunDafny(dafnyArguments); - } - - public static string RunDafny(IEnumerable arguments) { + using (Process dafnyProcess = new Process()) { dafnyProcess.StartInfo.FileName = "mono"; dafnyProcess.StartInfo.ArgumentList.Add(DAFNY_EXE); - foreach (var argument in arguments) { + foreach (var argument in dafnyArguments) { dafnyProcess.StartInfo.ArgumentList.Add(argument); } @@ -76,14 +70,29 @@ public static string RunDafny(IEnumerable arguments) { } } - public static IEnumerable TestData() { - var filePaths = Directory.GetFiles(COMP_DIR, "*.dfy") - .Select(path => GetRelativePath(COMP_DIR, path)); -// var filePaths = new string[] {"NativeNumbers.dfy"}; - var languages = new string[] {"cs", "java", "go", "js"}; + public static IEnumerable AllTestFiles() { + var filePaths = Directory.GetFiles(TEST_ROOT, "*.dfy", SearchOption.AllDirectories) + .Select(path => GetRelativePath(TEST_ROOT, path)); foreach (var filePath in filePaths) { - foreach (var language in languages) { - yield return new object[] { filePath, language }; + // Parse the arguments to testdafny on the "shebang" line + // TODO: override with environment variable from testdafny itself when executed + string shebang = File.ReadLines(Path.Combine(TEST_ROOT, filePath)).FirstOrDefault(); + int semiIndex = shebang.IndexOf(";"); + if (semiIndex >= 0) { + string[] chunks = shebang.Substring(0, semiIndex).Split(); + List arguments = chunks.Skip(3).ToList(); + + if (!arguments.Any(arg => arg.StartsWith("/compile:"))) { + var languages = new string[] {"cs", "java", "go", "js"}; + foreach (var language in languages) { + yield return new object[] + {filePath, arguments.Concat(new[] {"/compile:3", "/compileTarget:" + language}).ToArray()}; + } + } else { + yield return new object[] {filePath, arguments}; + } + } else { + yield return new object[] {filePath, new[] { "/compile:0" }}; } } } @@ -111,19 +120,27 @@ private static string DiffText(List diffs) { } [ParallelTheory] - [MemberData(nameof(TestData))] - public void ValidProgramOutput(String program, String language) { - string fullInputPath = Path.Combine(COMP_DIR, program); + [MemberData(nameof(AllTestFiles))] + public void Test(string file, string[] args) { + string fullInputPath = Path.Combine(TEST_ROOT, file); string expectedOutputPath = fullInputPath + ".expect"; - string expectedLangOutputPath = fullInputPath + "." + language + ".expect"; - bool langSpecific = File.Exists(expectedLangOutputPath); - string expectedOutput = langSpecific ? File.ReadAllText(expectedLangOutputPath) : File.ReadAllText(expectedOutputPath); - - string output = RunDafnyProgram(fullInputPath, "/compileTarget:" + language); + bool specialCase = false; + string compileTarget = args.FirstOrDefault(arg => arg.StartsWith("/compileTarget:")); + if (compileTarget != null) { + string language = compileTarget.Substring("/compileTarget:".Length); + var specialCasePath = fullInputPath + "." + language + ".expect"; + if (File.Exists(specialCasePath)) { + specialCase = true; + expectedOutputPath = specialCasePath; + } + } + string expectedOutput = File.ReadAllText(expectedOutputPath); + + string output = RunDafny(new List{ fullInputPath }.Concat(args)); AssertEqualWithDiff(expectedOutput, output); - Skip.If(langSpecific, "Confirmed language-specific behavior"); + Skip.If(specialCase, "Confirmed known exception for arguments: " + args); } } } \ No newline at end of file diff --git a/Test/comp/Arrays.dfy b/Test/comp/Arrays.dfy index 754d68799e0..04bc672b6f8 100644 --- a/Test/comp/Arrays.dfy +++ b/Test/comp/Arrays.dfy @@ -1,8 +1,4 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" +//usr/bin/env testdafny "$0"; exit method LinearSearch(a: array, key: int) returns (n: nat) ensures 0 <= n <= a.Length diff --git a/Test/comp/Calls.dfy b/Test/comp/Calls.dfy index 4af874ff109..e06d54dc7ae 100644 --- a/Test/comp/Calls.dfy +++ b/Test/comp/Calls.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + function method F(x: int, y: bool): int { x + if y then 2 else 3 } diff --git a/Test/comp/Collections.dfy b/Test/comp/Collections.dfy index 8ed0b14e6fd..b9212dac4ba 100644 --- a/Test/comp/Collections.dfy +++ b/Test/comp/Collections.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + method Main() { Sets(); SubSets(); diff --git a/Test/comp/Comprehensions.dfy b/Test/comp/Comprehensions.dfy index 540643f3dad..cad3de9fd02 100644 --- a/Test/comp/Comprehensions.dfy +++ b/Test/comp/Comprehensions.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + method Main() { AssignSuchThat(); LetSuchThat(); diff --git a/Test/comp/CovariantCollections.dfy b/Test/comp/CovariantCollections.dfy index 47f25bc8992..ac0394c104e 100644 --- a/Test/comp/CovariantCollections.dfy +++ b/Test/comp/CovariantCollections.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + method Main() { // TODO-RS: Just testing the recent support for covariance in sequneces in C# for now. // These also work in Javascript, but hit runtime type errors in Go and still don't diff --git a/Test/comp/Dt.dfy b/Test/comp/Dt.dfy index d30e30ae8b8..e79342b97c6 100644 --- a/Test/comp/Dt.dfy +++ b/Test/comp/Dt.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + datatype List = Nil | Cons(head: int, tail: List) method Main() { diff --git a/Test/comp/Forall.dfy b/Test/comp/Forall.dfy index 6c858e3591a..8a6e0cefb5f 100644 --- a/Test/comp/Forall.dfy +++ b/Test/comp/Forall.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + method Main() { var arrayTests := new ArrayTests(); arrayTests.Run(); diff --git a/Test/comp/Ghosts.dfy b/Test/comp/Ghosts.dfy index c842b3bd8d8..2e581fb7701 100644 --- a/Test/comp/Ghosts.dfy +++ b/Test/comp/Ghosts.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + method M1() returns (x: int) { print "hello, M1\n"; diff --git a/Test/comp/Hello.dfy b/Test/comp/Hello.dfy old mode 100644 new mode 100755 index bcae164a158..0044627905b --- a/Test/comp/Hello.dfy +++ b/Test/comp/Hello.dfy @@ -1,8 +1,4 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" +//usr/bin/env testdafny "$0"; exit method Main() { print "Hello, JavaScript! Best, Dafny\n"; diff --git a/Test/comp/Iterators.dfy b/Test/comp/Iterators.dfy index 03aa242c6b2..26fc0a4ce88 100644 --- a/Test/comp/Iterators.dfy +++ b/Test/comp/Iterators.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + class C { var x: int // for variety, the following tests the use of an instance Main method diff --git a/Test/comp/Let.dfy b/Test/comp/Let.dfy index 12685117927..3b8d8e3295b 100644 --- a/Test/comp/Let.dfy +++ b/Test/comp/Let.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + method M() returns (x: int) { x := var y := 50; y; // non-top-level let } diff --git a/Test/comp/Module.dfy b/Test/comp/Module.dfy index 4987cf6bd01..61a2c42d390 100644 --- a/Test/comp/Module.dfy +++ b/Test/comp/Module.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + // Simple sanity test of nested modules module Parent { module Child { diff --git a/Test/comp/Numbers.dfy b/Test/comp/Numbers.dfy index c3028251bf0..547d37704b0 100644 --- a/Test/comp/Numbers.dfy +++ b/Test/comp/Numbers.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + method Main() { Literals(); Arithmetic(); diff --git a/Test/comp/Poly.dfy b/Test/comp/Poly.dfy index a34b7991c6b..80091adea9b 100644 --- a/Test/comp/Poly.dfy +++ b/Test/comp/Poly.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + trait Shape { function method Center(): (real, real) reads this method PrintCenter() { diff --git a/Test/comp/TailRecursion.dfy b/Test/comp/TailRecursion.dfy index 5ebb3f30083..fec4fdc812b 100644 --- a/Test/comp/TailRecursion.dfy +++ b/Test/comp/TailRecursion.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + method Main() { // In the following, 2_000_000 is too large an argument without tail-calls var x := M(2_000_000, 0); diff --git a/Test/comp/TypeParams.dfy b/Test/comp/TypeParams.dfy index 309a6e9e58a..5fc219da1f7 100644 --- a/Test/comp/TypeParams.dfy +++ b/Test/comp/TypeParams.dfy @@ -1,3 +1,5 @@ +//usr/bin/env testdafny "$0"; exit + datatype Color = Orange | Pink | Teal type Six = x | x <= 6 newtype Even = x | x % 2 == 0 diff --git a/Test/dafny0/AdvancedLHS.dfy b/Test/dafny0/AdvancedLHS.dfy index 71f7d7139e6..e37e318e912 100644 --- a/Test/dafny0/AdvancedLHS.dfy +++ b/Test/dafny0/AdvancedLHS.dfy @@ -1,5 +1,5 @@ -// RUN: %dafny /compile:0 /print:"%t.print" /dprint:"%t.dprint" "%s" > "%t" -// RUN: %diff "%s.expect" "%t" +//usr/bin/env testdafny "$0" /compile:0; exit + class C { var x: C? From 57f67083a9594ad61bdf156a62388f1c9056916b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 5 Jun 2020 15:40:38 -0700 Subject: [PATCH 008/192] Refactored to remove brackets from test names Avoids a bug in the Rider UI for running tests --- Test/DafnyTests/DafnyTests.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 6a274c9aa0f..0fdd664cabf 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -86,13 +86,13 @@ public static IEnumerable AllTestFiles() { var languages = new string[] {"cs", "java", "go", "js"}; foreach (var language in languages) { yield return new object[] - {filePath, arguments.Concat(new[] {"/compile:3", "/compileTarget:" + language}).ToArray()}; + {filePath, String.Join(" ",arguments.Concat(new[] {"/compile:3", "/compileTarget:" + language}))}; } } else { - yield return new object[] {filePath, arguments}; + yield return new object[] {filePath, String.Join(" ", arguments)}; } } else { - yield return new object[] {filePath, new[] { "/compile:0" }}; + yield return new object[] {filePath, "/compile:0"}; } } } @@ -121,12 +121,13 @@ private static string DiffText(List diffs) { [ParallelTheory] [MemberData(nameof(AllTestFiles))] - public void Test(string file, string[] args) { + public void Test(string file, string args) { string fullInputPath = Path.Combine(TEST_ROOT, file); + string[] arguments = args.Split(); string expectedOutputPath = fullInputPath + ".expect"; bool specialCase = false; - string compileTarget = args.FirstOrDefault(arg => arg.StartsWith("/compileTarget:")); + string compileTarget = arguments.FirstOrDefault(arg => arg.StartsWith("/compileTarget:")); if (compileTarget != null) { string language = compileTarget.Substring("/compileTarget:".Length); var specialCasePath = fullInputPath + "." + language + ".expect"; @@ -137,7 +138,7 @@ public void Test(string file, string[] args) { } string expectedOutput = File.ReadAllText(expectedOutputPath); - string output = RunDafny(new List{ fullInputPath }.Concat(args)); + string output = RunDafny(new List{ fullInputPath }.Concat(arguments)); AssertEqualWithDiff(expectedOutput, output); Skip.If(specialCase, "Confirmed known exception for arguments: " + args); From 358fb6c65f2bd6153c0d380cd115427474fb5878 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 8 Jun 2020 15:11:29 -0700 Subject: [PATCH 009/192] Checkpointing partial work on YAML-based config --- Test/DafnyTests/DafnyTests-NetCore.csproj | 1 + Test/DafnyTests/DafnyTests.cs | 103 +++++++++++++++++----- Test/VerifyThis2015/Problem1.dfy | 3 - Test/dafny4/git-issue250.dfy | 15 +--- 4 files changed, 85 insertions(+), 37 deletions(-) diff --git a/Test/DafnyTests/DafnyTests-NetCore.csproj b/Test/DafnyTests/DafnyTests-NetCore.csproj index 323172ccf3d..0a3e9c297ee 100644 --- a/Test/DafnyTests/DafnyTests-NetCore.csproj +++ b/Test/DafnyTests/DafnyTests-NetCore.csproj @@ -10,6 +10,7 @@ + diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 0fdd664cabf..a9b53a5aa0a 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -11,6 +11,10 @@ using Xunit; using Xunit.Abstractions; using Xunit.Sdk; +using YamlDotNet.Core; +using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; using Assert = Xunit.Assert; namespace DafnyTests { @@ -22,7 +26,6 @@ public class DafnyTests { private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; - public static string RunDafny(IEnumerable arguments) { List dafnyArguments = new List { // Expected output does not contain logo @@ -70,29 +73,80 @@ public static string RunDafny(IEnumerable arguments) { } } + private static string GetTestCaseConfigString(string filePath) { + // TODO-RS: Figure out how to do this cleanly on a TextReader instead, + // and to handle things like nested comments. + string fullText = File.ReadAllText(filePath); + var commentStart = fullText.IndexOf("/*"); + if (commentStart >= 0) { + var commentEnd = fullText.IndexOf("*/", commentStart + 2); + if (commentEnd >= 0) { + var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); + if (commentContent.StartsWith("---")) { + return commentContent; + } + } + } + + return null; + } + + private static IEnumerable Expand(YamlNode node) { + if (node is YamlSequenceNode seqNode) { + return seqNode.SelectMany(child => Expand(child)); + } else if (node is YamlMappingNode mappingNode) { + return CartesianProduct(mappingNode.Select(ExpandValue)).Select(FromPairs); + } else { + return new[]{ node }; + } + } + + private static IEnumerable> ExpandValue(KeyValuePair pair) { + return Expand(pair.Value).Select(v => KeyValuePair.Create(pair.Key, v)); + } + + private static YamlMappingNode FromPairs(IEnumerable> pairs) { + var result = new YamlMappingNode(); + foreach (var pair in pairs) { + result.Add(pair.Key, pair.Value); + } + + return result; + } + + /** + * Source: https://docs.microsoft.com/en-us/archive/blogs/ericlippert/computing-a-cartesian-product-with-linq + */ + private static IEnumerable> CartesianProduct(IEnumerable> sequences) + { + IEnumerable> emptyProduct = new[] { Enumerable.Empty() }; + return sequences.Aggregate( + emptyProduct, + (accumulator, sequence) => + from accseq in accumulator + from item in sequence + select accseq.Concat(new[] {item})); + } + public static IEnumerable AllTestFiles() { - var filePaths = Directory.GetFiles(TEST_ROOT, "*.dfy", SearchOption.AllDirectories) - .Select(path => GetRelativePath(TEST_ROOT, path)); +// var filePaths = Directory.GetFiles(TEST_ROOT, "*.dfy", SearchOption.AllDirectories) +// .Select(path => GetRelativePath(TEST_ROOT, path)); + var filePaths = new[] { "dafny4/git-issue250.dfy" }; foreach (var filePath in filePaths) { - // Parse the arguments to testdafny on the "shebang" line - // TODO: override with environment variable from testdafny itself when executed - string shebang = File.ReadLines(Path.Combine(TEST_ROOT, filePath)).FirstOrDefault(); - int semiIndex = shebang.IndexOf(";"); - if (semiIndex >= 0) { - string[] chunks = shebang.Substring(0, semiIndex).Split(); - List arguments = chunks.Skip(3).ToList(); - - if (!arguments.Any(arg => arg.StartsWith("/compile:"))) { - var languages = new string[] {"cs", "java", "go", "js"}; - foreach (var language in languages) { - yield return new object[] - {filePath, String.Join(" ",arguments.Concat(new[] {"/compile:3", "/compileTarget:" + language}))}; - } - } else { - yield return new object[] {filePath, String.Join(" ", arguments)}; + var fullFilePath = Path.Combine(TEST_ROOT, filePath); + string configString = GetTestCaseConfigString(fullFilePath); + if (configString != null) { + var yamlStream = new YamlStream(); + yamlStream.Load(new StringReader(configString)); + if (yamlStream.Documents[0].RootNode is YamlMappingNode mapping) { + Console.WriteLine(mapping["compile"]); } - } else { - yield return new object[] {filePath, "/compile:0"}; + } + + var languages = new string[] {"cs", "java", "go", "js"}; + foreach (var language in languages) { + yield return new object[] + {filePath, String.Join(" ", new[] {"/compile:3", "/compileTarget:" + language})}; } } } @@ -138,10 +192,15 @@ public void Test(string file, string args) { } string expectedOutput = File.ReadAllText(expectedOutputPath); - string output = RunDafny(new List{ fullInputPath }.Concat(arguments)); + string output = RunDafny(new List{ file }.Concat(arguments)); AssertEqualWithDiff(expectedOutput, output); Skip.If(specialCase, "Confirmed known exception for arguments: " + args); } + + [Fact] + public void ExpandTest() { + + } } } \ No newline at end of file diff --git a/Test/VerifyThis2015/Problem1.dfy b/Test/VerifyThis2015/Problem1.dfy index 1cc89c5405a..4cc62f7a590 100644 --- a/Test/VerifyThis2015/Problem1.dfy +++ b/Test/VerifyThis2015/Problem1.dfy @@ -1,6 +1,3 @@ -// RUN: %dafny /compile:3 /print:"%t.print" /dprint:"%t.dprint" "%s" > "%t" -// RUN: %diff "%s.expect" "%t" - // Rustan Leino // 12 April 2015 // VerifyThis 2015 diff --git a/Test/dafny4/git-issue250.dfy b/Test/dafny4/git-issue250.dfy index 3046034985f..f29220a2808 100644 --- a/Test/dafny4/git-issue250.dfy +++ b/Test/dafny4/git-issue250.dfy @@ -1,15 +1,6 @@ -// RUN: %dafny /arith:1 "%s" > "%t" -// RUN: %dafny /arith:2 "%s" >> "%t" -// RUN: %dafny /arith:3 "%s" >> "%t" -// RUN: %dafny /arith:4 "%s" >> "%t" -// RUN: %dafny /arith:5 "%s" >> "%t" -// RUN: %dafny /arith:6 "%s" >> "%t" -// RUN: %dafny /arith:7 "%s" >> "%t" -// RUN: %dafny /arith:8 "%s" >> "%t" -// RUN: %dafny /arith:9 "%s" >> "%t" -// RUN: %dafny /arith:10 "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - +/*--- +arith: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +*/ method Main() { } \ No newline at end of file From d852a10d536ff081d89e0b4dc174f8ec7d954088 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 9 Jun 2020 09:58:01 -0700 Subject: [PATCH 010/192] Got some combinatorial cases working --- Test/DafnyTests/DafnyTests.cs | 95 +++++++++++++------ Test/DafnyTests/YamlTests/multiple.expect.yml | 8 ++ Test/DafnyTests/YamlTests/multiple.yml | 2 + Test/dafny4/git-issue250.dfy | 3 +- Test/dafny4/git-issue250.dfy.expect | 18 ---- 5 files changed, 77 insertions(+), 49 deletions(-) create mode 100644 Test/DafnyTests/YamlTests/multiple.expect.yml create mode 100644 Test/DafnyTests/YamlTests/multiple.yml diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index a9b53a5aa0a..98bf5556e76 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -1,21 +1,13 @@ using System; -using System.Linq; -using System.CodeDom.Compiler; -using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using DiffMatchPatch; -using Microsoft.Extensions.DependencyModel; using Xunit; -using Xunit.Abstractions; using Xunit.Sdk; -using YamlDotNet.Core; using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; -using Assert = Xunit.Assert; namespace DafnyTests { @@ -57,9 +49,9 @@ public static string RunDafny(IEnumerable arguments) { // Only preserve specific whitelisted environment variables dafnyProcess.StartInfo.EnvironmentVariables.Clear(); - dafnyProcess.StartInfo.EnvironmentVariables.Add("PATH", System.Environment.GetEnvironmentVariable("PATH")); + dafnyProcess.StartInfo.EnvironmentVariables.Add("PATH", Environment.GetEnvironmentVariable("PATH")); // Go requires this or GOCACHE - dafnyProcess.StartInfo.EnvironmentVariables.Add("HOME", System.Environment.GetEnvironmentVariable("HOME")); + dafnyProcess.StartInfo.EnvironmentVariables.Add("HOME", Environment.GetEnvironmentVariable("HOME")); dafnyProcess.Start(); dafnyProcess.WaitForExit(); @@ -129,28 +121,55 @@ from item in sequence } public static IEnumerable AllTestFiles() { -// var filePaths = Directory.GetFiles(TEST_ROOT, "*.dfy", SearchOption.AllDirectories) -// .Select(path => GetRelativePath(TEST_ROOT, path)); - var filePaths = new[] { "dafny4/git-issue250.dfy" }; - foreach (var filePath in filePaths) { - var fullFilePath = Path.Combine(TEST_ROOT, filePath); - string configString = GetTestCaseConfigString(fullFilePath); - if (configString != null) { - var yamlStream = new YamlStream(); - yamlStream.Load(new StringReader(configString)); - if (yamlStream.Documents[0].RootNode is YamlMappingNode mapping) { - Console.WriteLine(mapping["compile"]); - } + var filePaths = Directory.GetFiles(TEST_ROOT, "*.dfy", SearchOption.AllDirectories) + .Select(path => GetRelativePath(TEST_ROOT, path)); + return filePaths.SelectMany(TestCasesForDafnyFile); + } + + private static IEnumerable TestCasesForDafnyFile(string filePath) { + var fullFilePath = Path.Combine(TEST_ROOT, filePath); + string configString = GetTestCaseConfigString(fullFilePath); + IEnumerable configs; + if (configString != null) { + var yamlStream = new YamlStream(); + yamlStream.Load(new StringReader(configString)); + var config = yamlStream.Documents[0].RootNode; + configs = Expand(config); + } else { + configs = new[] { new YamlMappingNode() }; + } + + IEnumerable mappings = configs.SelectMany(config => { + if (config is YamlMappingNode mapping) { + return ResolveCompile(filePath, mapping); + } else { + throw new ArgumentException("Bad test case configuration: " + config); } - + }); + + return mappings.Select(mapping => { + var flags = mapping.Select(pair => "/" + pair.Key + ":" + pair.Value); + return new[] {filePath, String.Join(" ", flags)}; + }); + } + + private static IEnumerable ResolveCompile(string filePath, YamlMappingNode mapping) { + if (!mapping.Children.ContainsKey(new YamlScalarNode("compile"))) { + mapping.Add("compile", "3"); + } + + if (mapping["compile"].Equals(new YamlScalarNode("3")) && !mapping.Children.ContainsKey("compileTarget")) { var languages = new string[] {"cs", "java", "go", "js"}; foreach (var language in languages) { - yield return new object[] - {filePath, String.Join(" ", new[] {"/compile:3", "/compileTarget:" + language})}; + var withLanguage = new YamlMappingNode(mapping.Children); + withLanguage.Add("compileTarget", language); + yield return withLanguage; } + } else { + yield return mapping; } - } - + } + // TODO-RS: Replace with Path.GetRelativePath() if we move to .NET Core 3.1 private static string GetRelativePath(string relativeTo, string path) { var fullRelativeTo = Path.GetFullPath(relativeTo); @@ -181,6 +200,7 @@ public void Test(string file, string args) { string expectedOutputPath = fullInputPath + ".expect"; bool specialCase = false; + // TODO-RS: Broken now that args are a single string. Needs a custom arguments class. string compileTarget = arguments.FirstOrDefault(arg => arg.StartsWith("/compileTarget:")); if (compileTarget != null) { string language = compileTarget.Substring("/compileTarget:".Length); @@ -198,9 +218,24 @@ public void Test(string file, string args) { Skip.If(specialCase, "Confirmed known exception for arguments: " + args); } - [Fact] - public void ExpandTest() { - + [Theory] + [InlineData("multiple.yml", "multiple.expect.yml")] + public void ExpandTest(string inputPath, string expectedOutputPath) { + string fullInputPath = Path.Combine(TEST_ROOT, "DafnyTests/YamlTests/" + inputPath); + string fullExpectedOutputPath = Path.Combine(TEST_ROOT, "DafnyTests/YamlTests/" + expectedOutputPath); + + using (var reader = File.OpenText(fullInputPath)) { + var yamlStream = new YamlStream(); + yamlStream.Load(reader); + var root = yamlStream.Documents[0].RootNode; + var expanded = Expand(root); + + var outputWriter = new StringWriter(); + var serializer = new SerializerBuilder().Build(); + serializer.Serialize(outputWriter, expanded); + string expectedOutput = File.ReadAllText(fullExpectedOutputPath); + Assert.Equal(expectedOutput, outputWriter.ToString()); + } } } } \ No newline at end of file diff --git a/Test/DafnyTests/YamlTests/multiple.expect.yml b/Test/DafnyTests/YamlTests/multiple.expect.yml new file mode 100644 index 00000000000..0596b9d116b --- /dev/null +++ b/Test/DafnyTests/YamlTests/multiple.expect.yml @@ -0,0 +1,8 @@ +- a: b + foo: bar +- a: b + foo: blarg +- a: c + foo: bar +- a: c + foo: blarg diff --git a/Test/DafnyTests/YamlTests/multiple.yml b/Test/DafnyTests/YamlTests/multiple.yml new file mode 100644 index 00000000000..e014496347c --- /dev/null +++ b/Test/DafnyTests/YamlTests/multiple.yml @@ -0,0 +1,2 @@ +a: [b, c] +foo: [bar, blarg] \ No newline at end of file diff --git a/Test/dafny4/git-issue250.dfy b/Test/dafny4/git-issue250.dfy index f29220a2808..ccc2be990d5 100644 --- a/Test/dafny4/git-issue250.dfy +++ b/Test/dafny4/git-issue250.dfy @@ -1,5 +1,6 @@ /*--- -arith: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +arith: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +compile: 1 */ method Main() { diff --git a/Test/dafny4/git-issue250.dfy.expect b/Test/dafny4/git-issue250.dfy.expect index 971a5ef441e..012f5b99379 100644 --- a/Test/dafny4/git-issue250.dfy.expect +++ b/Test/dafny4/git-issue250.dfy.expect @@ -1,20 +1,2 @@ Dafny program verifier finished with 0 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors From caf685d8d2a991e287392844a75195e801a85d53 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 9 Jun 2020 12:02:29 -0700 Subject: [PATCH 011/192] Remove incorrect comment --- Test/DafnyTests/DafnyTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 98bf5556e76..77835f58188 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -200,7 +200,6 @@ public void Test(string file, string args) { string expectedOutputPath = fullInputPath + ".expect"; bool specialCase = false; - // TODO-RS: Broken now that args are a single string. Needs a custom arguments class. string compileTarget = arguments.FirstOrDefault(arg => arg.StartsWith("/compileTarget:")); if (compileTarget != null) { string language = compileTarget.Substring("/compileTarget:".Length); From ac8a5c5307faa466dc3f5e4417c24a2ede5a112b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 9 Jun 2020 12:16:10 -0700 Subject: [PATCH 012/192] POC of coverting the allocated1 directory to the new format --- Test/allocated1/dafny0/AdvancedLHS.dfy | 3 --- Test/allocated1/dafny0/AdvancedLHS.dfy.expect | 6 ------ Test/dafny0/AdvancedLHS.dfy | 8 +++++--- Test/dafny0/AdvancedLHS.dfy.expect | 2 +- 4 files changed, 6 insertions(+), 13 deletions(-) delete mode 100644 Test/allocated1/dafny0/AdvancedLHS.dfy delete mode 100644 Test/allocated1/dafny0/AdvancedLHS.dfy.expect diff --git a/Test/allocated1/dafny0/AdvancedLHS.dfy b/Test/allocated1/dafny0/AdvancedLHS.dfy deleted file mode 100644 index b154bb9cd5f..00000000000 --- a/Test/allocated1/dafny0/AdvancedLHS.dfy +++ /dev/null @@ -1,3 +0,0 @@ -// RUN: %dafny /verifyAllModules /allocated:1 /compile:0 /print:"%t.print" /dprint:"%t.dprint" "%s" > "%t" -// RUN: %diff "%s.expect" "%t" -include "../../dafny0/AdvancedLHS.dfy" diff --git a/Test/allocated1/dafny0/AdvancedLHS.dfy.expect b/Test/allocated1/dafny0/AdvancedLHS.dfy.expect deleted file mode 100644 index 2bca6bb0f99..00000000000 --- a/Test/allocated1/dafny0/AdvancedLHS.dfy.expect +++ /dev/null @@ -1,6 +0,0 @@ -AdvancedLHS.dfy(34,22): Error: target object may be null -Execution trace: - (0,0): anon0 - (0,0): anon11_Else - -Dafny program verifier finished with 3 verified, 1 error diff --git a/Test/dafny0/AdvancedLHS.dfy b/Test/dafny0/AdvancedLHS.dfy index e37e318e912..775aea10e8b 100644 --- a/Test/dafny0/AdvancedLHS.dfy +++ b/Test/dafny0/AdvancedLHS.dfy @@ -1,6 +1,8 @@ -//usr/bin/env testdafny "$0" /compile:0; exit - - +/* +--- +compile: 0 +allocated: [1, 3] +*/ class C { var x: C? diff --git a/Test/dafny0/AdvancedLHS.dfy.expect b/Test/dafny0/AdvancedLHS.dfy.expect index 2bca6bb0f99..74ce5b2c0e9 100644 --- a/Test/dafny0/AdvancedLHS.dfy.expect +++ b/Test/dafny0/AdvancedLHS.dfy.expect @@ -1,4 +1,4 @@ -AdvancedLHS.dfy(34,22): Error: target object may be null +AdvancedLHS.dfy(36,22): Error: target object may be null Execution trace: (0,0): anon0 (0,0): anon11_Else From eb29654a6b3311a470a286c1e400d0e3baa930bb Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 9 Jun 2020 18:22:45 -0700 Subject: [PATCH 013/192] Converted BranchConverage.dfy --- Test/DafnyTests/DafnyTests.cs | 96 +++++--- Test/comp/BranchCoverage.dfy | 24 +- Test/comp/BranchCoverage.dfy.expect | 342 ++++------------------------ 3 files changed, 124 insertions(+), 338 deletions(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 77835f58188..0826b15ccba 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -3,7 +3,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using DiffMatchPatch; using Xunit; using Xunit.Sdk; using YamlDotNet.RepresentationModel; @@ -13,11 +12,13 @@ namespace DafnyTests { public class DafnyTests { - private static string DAFNY_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()).Parent.Parent.Parent.Parent.Parent.FullName; + private static string DAFNY_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()).Parent.Parent.Parent + .Parent.Parent.FullName; + private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; - + public static string RunDafny(IEnumerable arguments) { List dafnyArguments = new List { // Expected output does not contain logo @@ -32,7 +33,7 @@ public static string RunDafny(IEnumerable arguments) { "-compileVerbose:0" }; dafnyArguments.AddRange(arguments); - + using (Process dafnyProcess = new Process()) { dafnyProcess.StartInfo.FileName = "mono"; dafnyProcess.StartInfo.ArgumentList.Add(DAFNY_EXE); @@ -46,7 +47,7 @@ public static string RunDafny(IEnumerable arguments) { dafnyProcess.StartInfo.CreateNoWindow = true; // Necessary for JS to find bignumber.js dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; - + // Only preserve specific whitelisted environment variables dafnyProcess.StartInfo.EnvironmentVariables.Clear(); dafnyProcess.StartInfo.EnvironmentVariables.Add("PATH", Environment.GetEnvironmentVariable("PATH")); @@ -65,6 +66,28 @@ public static string RunDafny(IEnumerable arguments) { } } + private static string Exec(string file, params string[] arguments) { + using (Process dafnyProcess = new Process()) { + dafnyProcess.StartInfo.FileName = file; + foreach (var argument in arguments) { + dafnyProcess.StartInfo.ArgumentList.Add(argument); + } + + dafnyProcess.StartInfo.UseShellExecute = false; + dafnyProcess.StartInfo.RedirectStandardOutput = true; + dafnyProcess.StartInfo.RedirectStandardError = true; + dafnyProcess.StartInfo.CreateNoWindow = true; + + dafnyProcess.Start(); + dafnyProcess.WaitForExit(); +// if (dafnyProcess.ExitCode != 0) { +// string error = dafnyProcess.StandardError.ReadToEnd(); +// throw new Exception(error); +// } + return dafnyProcess.StandardOutput.ReadToEnd(); + } + } + private static string GetTestCaseConfigString(string filePath) { // TODO-RS: Figure out how to do this cleanly on a TextReader instead, // and to handle things like nested comments. @@ -89,11 +112,12 @@ private static IEnumerable Expand(YamlNode node) { } else if (node is YamlMappingNode mappingNode) { return CartesianProduct(mappingNode.Select(ExpandValue)).Select(FromPairs); } else { - return new[]{ node }; + return new[] {node}; } } - private static IEnumerable> ExpandValue(KeyValuePair pair) { + private static IEnumerable> + ExpandValue(KeyValuePair pair) { return Expand(pair.Value).Select(v => KeyValuePair.Create(pair.Key, v)); } @@ -105,13 +129,12 @@ private static YamlMappingNode FromPairs(IEnumerable> CartesianProduct(IEnumerable> sequences) - { - IEnumerable> emptyProduct = new[] { Enumerable.Empty() }; + private static IEnumerable> CartesianProduct(IEnumerable> sequences) { + IEnumerable> emptyProduct = new[] {Enumerable.Empty()}; return sequences.Aggregate( emptyProduct, (accumulator, sequence) => @@ -119,16 +142,17 @@ from accseq in accumulator from item in sequence select accseq.Concat(new[] {item})); } - + public static IEnumerable AllTestFiles() { - var filePaths = Directory.GetFiles(TEST_ROOT, "*.dfy", SearchOption.AllDirectories) - .Select(path => GetRelativePath(TEST_ROOT, path)); + var filePaths = Directory.GetFiles(COMP_DIR, "*.dfy", SearchOption.AllDirectories) + .Select(path => GetRelativePath(TEST_ROOT, path)); +// var filePaths = Assembly.GetExecutingAssembly().GetManifestResourceNames(); return filePaths.SelectMany(TestCasesForDafnyFile); } private static IEnumerable TestCasesForDafnyFile(string filePath) { var fullFilePath = Path.Combine(TEST_ROOT, filePath); - string configString = GetTestCaseConfigString(fullFilePath); + string configString = GetTestCaseConfigString(fullFilePath); IEnumerable configs; if (configString != null) { var yamlStream = new YamlStream(); @@ -136,7 +160,7 @@ private static IEnumerable TestCasesForDafnyFile(string filePath) { var config = yamlStream.Documents[0].RootNode; configs = Expand(config); } else { - configs = new[] { new YamlMappingNode() }; + configs = new[] {new YamlMappingNode()}; } IEnumerable mappings = configs.SelectMany(config => { @@ -148,8 +172,7 @@ private static IEnumerable TestCasesForDafnyFile(string filePath) { }); return mappings.Select(mapping => { - var flags = mapping.Select(pair => "/" + pair.Key + ":" + pair.Value); - return new[] {filePath, String.Join(" ", flags)}; + return new[] {filePath, String.Join(" ", mapping.Select(ConfigPairToArgument))}; }); } @@ -168,8 +191,16 @@ private static IEnumerable ResolveCompile(string filePath, Yaml } else { yield return mapping; } - } - + } + + private static string ConfigPairToArgument(KeyValuePair pair) { + if (pair.Key.Equals(new YamlScalarNode("otherFiles"))) { + return pair.Value.ToString(); + } else { + return String.Format("/{0}:{1}", pair.Key, pair.Value); + } + } + // TODO-RS: Replace with Path.GetRelativePath() if we move to .NET Core 3.1 private static string GetRelativePath(string relativeTo, string path) { var fullRelativeTo = Path.GetFullPath(relativeTo); @@ -180,24 +211,22 @@ private static string GetRelativePath(string relativeTo, string path) { private static void AssertEqualWithDiff(string expected, string actual) { if (expected != actual) { - DiffMatchPatch.DiffMatchPatch dmp = DiffMatchPatchModule.Default; - List diff = dmp.DiffMain(expected, actual); - dmp.DiffCleanupSemantic(diff); - string patch = DiffText(diff); - throw new AssertActualExpectedException(expected, actual, patch); + // TODO-RS: Do better than shelling out to a linux utility + string expectedPath = Path.GetTempFileName(); + File.WriteAllText(expectedPath, expected); + string actualPath = Path.GetTempFileName(); + File.WriteAllText(actualPath, actual); + string diff = Exec("diff", "--unified=3", expectedPath, actualPath); + throw new AssertActualExpectedException(expected, actual, diff); } } - private static string DiffText(List diffs) { - return ""; - } - [ParallelTheory] [MemberData(nameof(AllTestFiles))] public void Test(string file, string args) { string fullInputPath = Path.Combine(TEST_ROOT, file); string[] arguments = args.Split(); - + string expectedOutputPath = fullInputPath + ".expect"; bool specialCase = false; string compileTarget = arguments.FirstOrDefault(arg => arg.StartsWith("/compileTarget:")); @@ -209,10 +238,11 @@ public void Test(string file, string args) { expectedOutputPath = specialCasePath; } } + string expectedOutput = File.ReadAllText(expectedOutputPath); - string output = RunDafny(new List{ file }.Concat(arguments)); - + string output = RunDafny(new List {file}.Concat(arguments)); + AssertEqualWithDiff(expectedOutput, output); Skip.If(specialCase, "Confirmed known exception for arguments: " + args); } @@ -228,7 +258,7 @@ public void ExpandTest(string inputPath, string expectedOutputPath) { yamlStream.Load(reader); var root = yamlStream.Documents[0].RootNode; var expanded = Expand(root); - + var outputWriter = new StringWriter(); var serializer = new SerializerBuilder().Build(); serializer.Serialize(outputWriter, expanded); diff --git a/Test/comp/BranchCoverage.dfy b/Test/comp/BranchCoverage.dfy index 8b4206be84e..f4322acfd7f 100644 --- a/Test/comp/BranchCoverage.dfy +++ b/Test/comp/BranchCoverage.dfy @@ -1,8 +1,22 @@ -// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:cs BranchCoverage2.cs "%s" > "%t" -// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:js BranchCoverage3.js "%s" >> "%t" -// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:go BranchCoverage4.go "%s" >> "%t" -// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:java CodeCoverage.java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" +/* +--- +- compile: 3 + coverage: "-" + compileTarget: cs + otherFiles: comp/BranchCoverage2.cs +- compile: 3 + coverage: "-" + compileTarget: js + otherFiles: comp/BranchCoverage3.js +- compile: 3 + coverage: "-" + compileTarget: go + otherFiles: comp/BranchCoverage4.go +- compile: 3 + coverage: "-" + compileTarget: java + otherFiles: comp/CodeCoverage.java +*/ // The Main method is at the end of this file, because that makes it easier to maintain // this test file when adding more tests. diff --git a/Test/comp/BranchCoverage.dfy.expect b/Test/comp/BranchCoverage.dfy.expect index dfbc09cb43c..62b01571f0c 100644 --- a/Test/comp/BranchCoverage.dfy.expect +++ b/Test/comp/BranchCoverage.dfy.expect @@ -1,305 +1,47 @@ Dafny program verifier finished with 6 verified, 0 errors -0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor -1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled -2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled -3: BranchCoverage.dfy(30,35): entry to method _module._default.M -4: BranchCoverage.dfy(32,13): then branch -5: BranchCoverage.dfy(34,20): then branch -6: BranchCoverage.dfy(36,10): else branch -7: BranchCoverage.dfy(41,35): entry to method _module._default.N -8: BranchCoverage.dfy(44,13): then branch -9: BranchCoverage.dfy(46,20): then branch -10: BranchCoverage.dfy(46,10): implicit else branch -11: BranchCoverage.dfy(52,18): entry to method _module._default.P -12: BranchCoverage.dfy(55,8): then branch -13: BranchCoverage.dfy(55,3): implicit else branch -14: BranchCoverage.dfy(60,8): then branch -15: BranchCoverage.dfy(62,10): else branch -16: BranchCoverage.dfy(72,12): then branch -17: BranchCoverage.dfy(74,15): then branch -18: BranchCoverage.dfy(74,10): implicit else branch -19: BranchCoverage.dfy(82,35): entry to method _module._default.Q -20: BranchCoverage.dfy(94,3): if-case branch -21: BranchCoverage.dfy(96,3): if-case branch -22: BranchCoverage.dfy(99,3): if-case branch -23: BranchCoverage.dfy(105,35): entry to method _module._default.R -24: BranchCoverage.dfy(108,15): while body -25: BranchCoverage.dfy(113,11): while body -26: BranchCoverage.dfy(121,5): while-case branch -27: BranchCoverage.dfy(123,17): then branch -28: BranchCoverage.dfy(123,7): implicit else branch -29: BranchCoverage.dfy(128,5): while-case branch -30: BranchCoverage.dfy(131,5): while-case branch -31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib -32: BranchCoverage.dfy(144,5): then branch -33: BranchCoverage.dfy(146,5): then branch -34: BranchCoverage.dfy(146,5): else branch -35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial -36: BranchCoverage.dfy(156,13): then branch -37: BranchCoverage.dfy(156,13): else branch -38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial -39: BranchCoverage.dfy(165,3): then branch -40: BranchCoverage.dfy(167,3): else branch -41: BranchCoverage.dfy(173,15): entry to method _module._default.Main -0: 3 -1: 0 -2: 0 -3: 3 -4: 1 -5: 0 -6: 2 -7: 3 -8: 1 -9: 0 -10: 2 -11: 1 -12: 0 -13: 1 -14: 1 -15: 0 -16: 0 -17: 0 -18: 1 -19: 3 -20: 1 -21: 1 -22: 1 -23: 1 -24: 111 -25: 0 -26: 56 -27: 1 -28: 55 -29: 28 -30: 28 -31: 465 -32: 233 -33: 0 -34: 232 -35: 11 -36: 1 -37: 10 -38: 11 -39: 1 -40: 10 -41: 1 - -Dafny program verifier finished with 6 verified, 0 errors -0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor -1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled -2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled -3: BranchCoverage.dfy(30,35): entry to method _module._default.M -4: BranchCoverage.dfy(32,13): then branch -5: BranchCoverage.dfy(34,20): then branch -6: BranchCoverage.dfy(36,10): else branch -7: BranchCoverage.dfy(41,35): entry to method _module._default.N -8: BranchCoverage.dfy(44,13): then branch -9: BranchCoverage.dfy(46,20): then branch -10: BranchCoverage.dfy(46,10): implicit else branch -11: BranchCoverage.dfy(52,18): entry to method _module._default.P -12: BranchCoverage.dfy(55,8): then branch -13: BranchCoverage.dfy(55,3): implicit else branch -14: BranchCoverage.dfy(60,8): then branch -15: BranchCoverage.dfy(62,10): else branch -16: BranchCoverage.dfy(72,12): then branch -17: BranchCoverage.dfy(74,15): then branch -18: BranchCoverage.dfy(74,10): implicit else branch -19: BranchCoverage.dfy(82,35): entry to method _module._default.Q -20: BranchCoverage.dfy(94,3): if-case branch -21: BranchCoverage.dfy(96,3): if-case branch -22: BranchCoverage.dfy(99,3): if-case branch -23: BranchCoverage.dfy(105,35): entry to method _module._default.R -24: BranchCoverage.dfy(108,15): while body -25: BranchCoverage.dfy(113,11): while body -26: BranchCoverage.dfy(121,5): while-case branch -27: BranchCoverage.dfy(123,17): then branch -28: BranchCoverage.dfy(123,7): implicit else branch -29: BranchCoverage.dfy(128,5): while-case branch -30: BranchCoverage.dfy(131,5): while-case branch -31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib -32: BranchCoverage.dfy(144,5): then branch -33: BranchCoverage.dfy(146,5): then branch -34: BranchCoverage.dfy(146,5): else branch -35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial -36: BranchCoverage.dfy(156,13): then branch -37: BranchCoverage.dfy(156,13): else branch -38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial -39: BranchCoverage.dfy(165,3): then branch -40: BranchCoverage.dfy(167,3): else branch -41: BranchCoverage.dfy(173,15): entry to method _module._default.Main -0: 3 -1: 0 -2: 0 -3: 3 -4: 1 -5: 0 -6: 2 -7: 3 -8: 1 -9: 0 -10: 2 -11: 1 -12: 0 -13: 1 -14: 1 -15: 0 -16: 0 -17: 0 -18: 1 -19: 3 -20: 1 -21: 1 -22: 1 -23: 1 -24: 111 -25: 0 -26: 56 -27: 1 -28: 55 -29: 28 -30: 28 -31: 465 -32: 233 -33: 0 -34: 232 -35: 11 -36: 1 -37: 10 -38: 11 -39: 1 -40: 10 -41: 1 - -Dafny program verifier finished with 6 verified, 0 errors -0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor -1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled -2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled -3: BranchCoverage.dfy(30,35): entry to method _module._default.M -4: BranchCoverage.dfy(32,13): then branch -5: BranchCoverage.dfy(34,20): then branch -6: BranchCoverage.dfy(36,10): else branch -7: BranchCoverage.dfy(41,35): entry to method _module._default.N -8: BranchCoverage.dfy(44,13): then branch -9: BranchCoverage.dfy(46,20): then branch -10: BranchCoverage.dfy(46,10): implicit else branch -11: BranchCoverage.dfy(52,18): entry to method _module._default.P -12: BranchCoverage.dfy(55,8): then branch -13: BranchCoverage.dfy(55,3): implicit else branch -14: BranchCoverage.dfy(60,8): then branch -15: BranchCoverage.dfy(62,10): else branch -16: BranchCoverage.dfy(72,12): then branch -17: BranchCoverage.dfy(74,15): then branch -18: BranchCoverage.dfy(74,10): implicit else branch -19: BranchCoverage.dfy(82,35): entry to method _module._default.Q -20: BranchCoverage.dfy(94,3): if-case branch -21: BranchCoverage.dfy(96,3): if-case branch -22: BranchCoverage.dfy(99,3): if-case branch -23: BranchCoverage.dfy(105,35): entry to method _module._default.R -24: BranchCoverage.dfy(108,15): while body -25: BranchCoverage.dfy(113,11): while body -26: BranchCoverage.dfy(121,5): while-case branch -27: BranchCoverage.dfy(123,17): then branch -28: BranchCoverage.dfy(123,7): implicit else branch -29: BranchCoverage.dfy(128,5): while-case branch -30: BranchCoverage.dfy(131,5): while-case branch -31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib -32: BranchCoverage.dfy(144,5): then branch -33: BranchCoverage.dfy(146,5): then branch -34: BranchCoverage.dfy(146,5): else branch -35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial -36: BranchCoverage.dfy(156,13): then branch -37: BranchCoverage.dfy(156,13): else branch -38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial -39: BranchCoverage.dfy(165,3): then branch -40: BranchCoverage.dfy(167,3): else branch -41: BranchCoverage.dfy(173,15): entry to method _module._default.Main -0: 3 -1: 0 -2: 0 -3: 3 -4: 1 -5: 0 -6: 2 -7: 3 -8: 1 -9: 0 -10: 2 -11: 1 -12: 0 -13: 1 -14: 1 -15: 0 -16: 0 -17: 0 -18: 1 -19: 3 -20: 1 -21: 1 -22: 1 -23: 1 -24: 111 -25: 0 -26: 56 -27: 1 -28: 55 -29: 28 -30: 28 -31: 465 -32: 233 -33: 0 -34: 232 -35: 11 -36: 1 -37: 10 -38: 11 -39: 1 -40: 10 -41: 1 - -Dafny program verifier finished with 6 verified, 0 errors -0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor -1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled -2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled -3: BranchCoverage.dfy(30,35): entry to method _module._default.M -4: BranchCoverage.dfy(32,13): then branch -5: BranchCoverage.dfy(34,20): then branch -6: BranchCoverage.dfy(36,10): else branch -7: BranchCoverage.dfy(41,35): entry to method _module._default.N -8: BranchCoverage.dfy(44,13): then branch -9: BranchCoverage.dfy(46,20): then branch -10: BranchCoverage.dfy(46,10): implicit else branch -11: BranchCoverage.dfy(52,18): entry to method _module._default.P -12: BranchCoverage.dfy(55,8): then branch -13: BranchCoverage.dfy(55,3): implicit else branch -14: BranchCoverage.dfy(60,8): then branch -15: BranchCoverage.dfy(62,10): else branch -16: BranchCoverage.dfy(72,12): then branch -17: BranchCoverage.dfy(74,15): then branch -18: BranchCoverage.dfy(74,10): implicit else branch -19: BranchCoverage.dfy(82,35): entry to method _module._default.Q -20: BranchCoverage.dfy(94,3): if-case branch -21: BranchCoverage.dfy(96,3): if-case branch -22: BranchCoverage.dfy(99,3): if-case branch -23: BranchCoverage.dfy(105,35): entry to method _module._default.R -24: BranchCoverage.dfy(108,15): while body -25: BranchCoverage.dfy(113,11): while body -26: BranchCoverage.dfy(121,5): while-case branch -27: BranchCoverage.dfy(123,17): then branch -28: BranchCoverage.dfy(123,7): implicit else branch -29: BranchCoverage.dfy(128,5): while-case branch -30: BranchCoverage.dfy(131,5): while-case branch -31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib -32: BranchCoverage.dfy(144,5): then branch -33: BranchCoverage.dfy(146,5): then branch -34: BranchCoverage.dfy(146,5): else branch -35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial -36: BranchCoverage.dfy(156,13): then branch -37: BranchCoverage.dfy(156,13): else branch -38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial -39: BranchCoverage.dfy(165,3): then branch -40: BranchCoverage.dfy(167,3): else branch -41: BranchCoverage.dfy(173,15): entry to method _module._default.Main +0: BranchCoverage.dfy(27,18): entry to method _module.MyClass._ctor +1: BranchCoverage.dfy(33,22): entry to method _module._default.NeverCalled +2: BranchCoverage.dfy(39,3): entry to function _module._default.FunctionNeverCalled +3: BranchCoverage.dfy(44,35): entry to method _module._default.M +4: BranchCoverage.dfy(46,13): then branch +5: BranchCoverage.dfy(48,20): then branch +6: BranchCoverage.dfy(50,10): else branch +7: BranchCoverage.dfy(55,35): entry to method _module._default.N +8: BranchCoverage.dfy(58,13): then branch +9: BranchCoverage.dfy(60,20): then branch +10: BranchCoverage.dfy(60,10): implicit else branch +11: BranchCoverage.dfy(66,18): entry to method _module._default.P +12: BranchCoverage.dfy(69,8): then branch +13: BranchCoverage.dfy(69,3): implicit else branch +14: BranchCoverage.dfy(74,8): then branch +15: BranchCoverage.dfy(76,10): else branch +16: BranchCoverage.dfy(86,12): then branch +17: BranchCoverage.dfy(88,15): then branch +18: BranchCoverage.dfy(88,10): implicit else branch +19: BranchCoverage.dfy(96,35): entry to method _module._default.Q +20: BranchCoverage.dfy(108,3): if-case branch +21: BranchCoverage.dfy(110,3): if-case branch +22: BranchCoverage.dfy(113,3): if-case branch +23: BranchCoverage.dfy(119,35): entry to method _module._default.R +24: BranchCoverage.dfy(122,15): while body +25: BranchCoverage.dfy(127,11): while body +26: BranchCoverage.dfy(135,5): while-case branch +27: BranchCoverage.dfy(137,17): then branch +28: BranchCoverage.dfy(137,7): implicit else branch +29: BranchCoverage.dfy(142,5): while-case branch +30: BranchCoverage.dfy(145,5): while-case branch +31: BranchCoverage.dfy(157,3): entry to function _module._default.Fib +32: BranchCoverage.dfy(158,5): then branch +33: BranchCoverage.dfy(160,5): then branch +34: BranchCoverage.dfy(160,5): else branch +35: BranchCoverage.dfy(169,3): entry to function _module._default.Factorial +36: BranchCoverage.dfy(170,13): then branch +37: BranchCoverage.dfy(170,13): else branch +38: BranchCoverage.dfy(176,1): entry to method _module._default.ComputeFactorial +39: BranchCoverage.dfy(179,3): then branch +40: BranchCoverage.dfy(181,3): else branch +41: BranchCoverage.dfy(187,15): entry to method _module._default.Main 0: 3 1: 0 2: 0 From 262dd34a40b526be4198e224fa89193d1a6963d5 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 9 Jun 2020 18:36:10 -0700 Subject: [PATCH 014/192] =?UTF-8?q?Remove=20experimental=20=E2=80=9Csheban?= =?UTF-8?q?g=E2=80=9D=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Test/comp/Arrays.dfy | 2 -- Test/comp/Calls.dfy | 2 -- Test/comp/Collections.dfy | 2 -- Test/comp/Comprehensions.dfy | 2 -- Test/comp/CovariantCollections.dfy | 2 -- Test/comp/Dt.dfy | 2 -- Test/comp/Forall.dfy | 2 -- Test/comp/Ghosts.dfy | 2 -- Test/comp/Hello.dfy | 2 -- Test/comp/Iterators.dfy | 2 -- Test/comp/Let.dfy | 2 -- Test/comp/Module.dfy | 2 -- Test/comp/Numbers.dfy | 2 -- Test/comp/Poly.dfy | 2 -- Test/comp/TailRecursion.dfy | 2 -- Test/comp/TypeParams.dfy | 2 -- 16 files changed, 32 deletions(-) diff --git a/Test/comp/Arrays.dfy b/Test/comp/Arrays.dfy index 04bc672b6f8..eb80d4ba18d 100644 --- a/Test/comp/Arrays.dfy +++ b/Test/comp/Arrays.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method LinearSearch(a: array, key: int) returns (n: nat) ensures 0 <= n <= a.Length ensures n == a.Length || a[n] == key diff --git a/Test/comp/Calls.dfy b/Test/comp/Calls.dfy index e06d54dc7ae..4af874ff109 100644 --- a/Test/comp/Calls.dfy +++ b/Test/comp/Calls.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - function method F(x: int, y: bool): int { x + if y then 2 else 3 } diff --git a/Test/comp/Collections.dfy b/Test/comp/Collections.dfy index b9212dac4ba..8ed0b14e6fd 100644 --- a/Test/comp/Collections.dfy +++ b/Test/comp/Collections.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method Main() { Sets(); SubSets(); diff --git a/Test/comp/Comprehensions.dfy b/Test/comp/Comprehensions.dfy index cad3de9fd02..540643f3dad 100644 --- a/Test/comp/Comprehensions.dfy +++ b/Test/comp/Comprehensions.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method Main() { AssignSuchThat(); LetSuchThat(); diff --git a/Test/comp/CovariantCollections.dfy b/Test/comp/CovariantCollections.dfy index ac0394c104e..47f25bc8992 100644 --- a/Test/comp/CovariantCollections.dfy +++ b/Test/comp/CovariantCollections.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method Main() { // TODO-RS: Just testing the recent support for covariance in sequneces in C# for now. // These also work in Javascript, but hit runtime type errors in Go and still don't diff --git a/Test/comp/Dt.dfy b/Test/comp/Dt.dfy index e79342b97c6..d30e30ae8b8 100644 --- a/Test/comp/Dt.dfy +++ b/Test/comp/Dt.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - datatype List = Nil | Cons(head: int, tail: List) method Main() { diff --git a/Test/comp/Forall.dfy b/Test/comp/Forall.dfy index 8a6e0cefb5f..6c858e3591a 100644 --- a/Test/comp/Forall.dfy +++ b/Test/comp/Forall.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method Main() { var arrayTests := new ArrayTests(); arrayTests.Run(); diff --git a/Test/comp/Ghosts.dfy b/Test/comp/Ghosts.dfy index 2e581fb7701..c842b3bd8d8 100644 --- a/Test/comp/Ghosts.dfy +++ b/Test/comp/Ghosts.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method M1() returns (x: int) { print "hello, M1\n"; diff --git a/Test/comp/Hello.dfy b/Test/comp/Hello.dfy index 0044627905b..eb2ba3c7685 100755 --- a/Test/comp/Hello.dfy +++ b/Test/comp/Hello.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method Main() { print "Hello, JavaScript! Best, Dafny\n"; var x := 14; diff --git a/Test/comp/Iterators.dfy b/Test/comp/Iterators.dfy index 26fc0a4ce88..03aa242c6b2 100644 --- a/Test/comp/Iterators.dfy +++ b/Test/comp/Iterators.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - class C { var x: int // for variety, the following tests the use of an instance Main method diff --git a/Test/comp/Let.dfy b/Test/comp/Let.dfy index 3b8d8e3295b..12685117927 100644 --- a/Test/comp/Let.dfy +++ b/Test/comp/Let.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method M() returns (x: int) { x := var y := 50; y; // non-top-level let } diff --git a/Test/comp/Module.dfy b/Test/comp/Module.dfy index 61a2c42d390..4987cf6bd01 100644 --- a/Test/comp/Module.dfy +++ b/Test/comp/Module.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - // Simple sanity test of nested modules module Parent { module Child { diff --git a/Test/comp/Numbers.dfy b/Test/comp/Numbers.dfy index 547d37704b0..c3028251bf0 100644 --- a/Test/comp/Numbers.dfy +++ b/Test/comp/Numbers.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method Main() { Literals(); Arithmetic(); diff --git a/Test/comp/Poly.dfy b/Test/comp/Poly.dfy index 80091adea9b..a34b7991c6b 100644 --- a/Test/comp/Poly.dfy +++ b/Test/comp/Poly.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - trait Shape { function method Center(): (real, real) reads this method PrintCenter() { diff --git a/Test/comp/TailRecursion.dfy b/Test/comp/TailRecursion.dfy index fec4fdc812b..5ebb3f30083 100644 --- a/Test/comp/TailRecursion.dfy +++ b/Test/comp/TailRecursion.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - method Main() { // In the following, 2_000_000 is too large an argument without tail-calls var x := M(2_000_000, 0); diff --git a/Test/comp/TypeParams.dfy b/Test/comp/TypeParams.dfy index 5fc219da1f7..309a6e9e58a 100644 --- a/Test/comp/TypeParams.dfy +++ b/Test/comp/TypeParams.dfy @@ -1,5 +1,3 @@ -//usr/bin/env testdafny "$0"; exit - datatype Color = Orange | Pink | Teal type Six = x | x <= 6 newtype Even = x | x % 2 == 0 From 7b09ac6b753bd1b6cc37ab791cba01547babda66 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 9 Jun 2020 18:42:32 -0700 Subject: [PATCH 015/192] Special cases for comp/Poly.dfy --- Test/comp/Poly.dfy.cs.expect | 6 ++++++ Test/comp/Poly.dfy.java.expect | 6 ++++++ Test/comp/Poly.dfy.js.expect | 10 ++++++++++ 3 files changed, 22 insertions(+) create mode 100644 Test/comp/Poly.dfy.cs.expect create mode 100644 Test/comp/Poly.dfy.java.expect create mode 100644 Test/comp/Poly.dfy.js.expect diff --git a/Test/comp/Poly.dfy.cs.expect b/Test/comp/Poly.dfy.cs.expect new file mode 100644 index 00000000000..232c15bf822 --- /dev/null +++ b/Test/comp/Poly.dfy.cs.expect @@ -0,0 +1,6 @@ + +Dafny program verifier finished with 9 verified, 0 errors +Poly.dfy(59,16): Error: compilation of set is not supported; consider introducing a ghost +Poly.dfy(63,21): Error: compilation of multiset is not supported; consider introducing a ghost +Poly.dfy(80,13): Error: compilation of set is not supported; consider introducing a ghost +Poly.dfy(81,18): Error: compilation of multiset is not supported; consider introducing a ghost diff --git a/Test/comp/Poly.dfy.java.expect b/Test/comp/Poly.dfy.java.expect new file mode 100644 index 00000000000..957e41e14d7 --- /dev/null +++ b/Test/comp/Poly.dfy.java.expect @@ -0,0 +1,6 @@ + +Dafny program verifier finished with 9 verified, 0 errors +Poly.dfy(48,16): Error: compilation of seq is not supported; consider introducing a ghost +Poly.dfy(59,16): Error: compilation of set is not supported; consider introducing a ghost +Poly.dfy(63,21): Error: compilation of multiset is not supported; consider introducing a ghost +File Poly/Poly.java contains the partially compiled program diff --git a/Test/comp/Poly.dfy.js.expect b/Test/comp/Poly.dfy.js.expect new file mode 100644 index 00000000000..31bc5b17af1 --- /dev/null +++ b/Test/comp/Poly.dfy.js.expect @@ -0,0 +1,10 @@ + +Dafny program verifier finished with 9 verified, 0 errors +Center of square: ((1.0 / 2.0), (1.0 / 2.0)) +Center of circle: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Centers: {((1.0 / 2.0), (1.0 / 2.0)), (1.0, 1.0)} +Centers: {((1.0 / 2.0), (1.0 / 2.0)), (1.0, 1.0)} From b6a589a66e867ecc19611c5eeb4b4c75b3231c3c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 10 Jun 2020 11:50:03 -0700 Subject: [PATCH 016/192] Added support for .$lang.files directory --- Test/comp/BranchCoverage.dfy | 18 +--- .../BranchCoverage2.cs | 0 Test/comp/BranchCoverage.dfy.expect | 84 +++++++++---------- .../BranchCoverage4.go | 0 .../CodeCoverage.java | 0 .../BranchCoverage3.js | 0 Test/comp/JsModule.dfy | 8 +- 7 files changed, 49 insertions(+), 61 deletions(-) rename Test/comp/{ => BranchCoverage.dfy.cs.files}/BranchCoverage2.cs (100%) rename Test/comp/{ => BranchCoverage.dfy.go.files}/BranchCoverage4.go (100%) rename Test/comp/{ => BranchCoverage.dfy.java.files}/CodeCoverage.java (100%) rename Test/comp/{ => BranchCoverage.dfy.js.files}/BranchCoverage3.js (100%) diff --git a/Test/comp/BranchCoverage.dfy b/Test/comp/BranchCoverage.dfy index f4322acfd7f..b20432221a0 100644 --- a/Test/comp/BranchCoverage.dfy +++ b/Test/comp/BranchCoverage.dfy @@ -1,21 +1,7 @@ /* --- -- compile: 3 - coverage: "-" - compileTarget: cs - otherFiles: comp/BranchCoverage2.cs -- compile: 3 - coverage: "-" - compileTarget: js - otherFiles: comp/BranchCoverage3.js -- compile: 3 - coverage: "-" - compileTarget: go - otherFiles: comp/BranchCoverage4.go -- compile: 3 - coverage: "-" - compileTarget: java - otherFiles: comp/CodeCoverage.java +compile: 3 +coverage: "-" */ // The Main method is at the end of this file, because that makes it easier to maintain diff --git a/Test/comp/BranchCoverage2.cs b/Test/comp/BranchCoverage.dfy.cs.files/BranchCoverage2.cs similarity index 100% rename from Test/comp/BranchCoverage2.cs rename to Test/comp/BranchCoverage.dfy.cs.files/BranchCoverage2.cs diff --git a/Test/comp/BranchCoverage.dfy.expect b/Test/comp/BranchCoverage.dfy.expect index 62b01571f0c..ce18dfad615 100644 --- a/Test/comp/BranchCoverage.dfy.expect +++ b/Test/comp/BranchCoverage.dfy.expect @@ -1,47 +1,47 @@ Dafny program verifier finished with 6 verified, 0 errors -0: BranchCoverage.dfy(27,18): entry to method _module.MyClass._ctor -1: BranchCoverage.dfy(33,22): entry to method _module._default.NeverCalled -2: BranchCoverage.dfy(39,3): entry to function _module._default.FunctionNeverCalled -3: BranchCoverage.dfy(44,35): entry to method _module._default.M -4: BranchCoverage.dfy(46,13): then branch -5: BranchCoverage.dfy(48,20): then branch -6: BranchCoverage.dfy(50,10): else branch -7: BranchCoverage.dfy(55,35): entry to method _module._default.N -8: BranchCoverage.dfy(58,13): then branch -9: BranchCoverage.dfy(60,20): then branch -10: BranchCoverage.dfy(60,10): implicit else branch -11: BranchCoverage.dfy(66,18): entry to method _module._default.P -12: BranchCoverage.dfy(69,8): then branch -13: BranchCoverage.dfy(69,3): implicit else branch -14: BranchCoverage.dfy(74,8): then branch -15: BranchCoverage.dfy(76,10): else branch -16: BranchCoverage.dfy(86,12): then branch -17: BranchCoverage.dfy(88,15): then branch -18: BranchCoverage.dfy(88,10): implicit else branch -19: BranchCoverage.dfy(96,35): entry to method _module._default.Q -20: BranchCoverage.dfy(108,3): if-case branch -21: BranchCoverage.dfy(110,3): if-case branch -22: BranchCoverage.dfy(113,3): if-case branch -23: BranchCoverage.dfy(119,35): entry to method _module._default.R -24: BranchCoverage.dfy(122,15): while body -25: BranchCoverage.dfy(127,11): while body -26: BranchCoverage.dfy(135,5): while-case branch -27: BranchCoverage.dfy(137,17): then branch -28: BranchCoverage.dfy(137,7): implicit else branch -29: BranchCoverage.dfy(142,5): while-case branch -30: BranchCoverage.dfy(145,5): while-case branch -31: BranchCoverage.dfy(157,3): entry to function _module._default.Fib -32: BranchCoverage.dfy(158,5): then branch -33: BranchCoverage.dfy(160,5): then branch -34: BranchCoverage.dfy(160,5): else branch -35: BranchCoverage.dfy(169,3): entry to function _module._default.Factorial -36: BranchCoverage.dfy(170,13): then branch -37: BranchCoverage.dfy(170,13): else branch -38: BranchCoverage.dfy(176,1): entry to method _module._default.ComputeFactorial -39: BranchCoverage.dfy(179,3): then branch -40: BranchCoverage.dfy(181,3): else branch -41: BranchCoverage.dfy(187,15): entry to method _module._default.Main +0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor +1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled +2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled +3: BranchCoverage.dfy(30,35): entry to method _module._default.M +4: BranchCoverage.dfy(32,13): then branch +5: BranchCoverage.dfy(34,20): then branch +6: BranchCoverage.dfy(36,10): else branch +7: BranchCoverage.dfy(41,35): entry to method _module._default.N +8: BranchCoverage.dfy(44,13): then branch +9: BranchCoverage.dfy(46,20): then branch +10: BranchCoverage.dfy(46,10): implicit else branch +11: BranchCoverage.dfy(52,18): entry to method _module._default.P +12: BranchCoverage.dfy(55,8): then branch +13: BranchCoverage.dfy(55,3): implicit else branch +14: BranchCoverage.dfy(60,8): then branch +15: BranchCoverage.dfy(62,10): else branch +16: BranchCoverage.dfy(72,12): then branch +17: BranchCoverage.dfy(74,15): then branch +18: BranchCoverage.dfy(74,10): implicit else branch +19: BranchCoverage.dfy(82,35): entry to method _module._default.Q +20: BranchCoverage.dfy(94,3): if-case branch +21: BranchCoverage.dfy(96,3): if-case branch +22: BranchCoverage.dfy(99,3): if-case branch +23: BranchCoverage.dfy(105,35): entry to method _module._default.R +24: BranchCoverage.dfy(108,15): while body +25: BranchCoverage.dfy(113,11): while body +26: BranchCoverage.dfy(121,5): while-case branch +27: BranchCoverage.dfy(123,17): then branch +28: BranchCoverage.dfy(123,7): implicit else branch +29: BranchCoverage.dfy(128,5): while-case branch +30: BranchCoverage.dfy(131,5): while-case branch +31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib +32: BranchCoverage.dfy(144,5): then branch +33: BranchCoverage.dfy(146,5): then branch +34: BranchCoverage.dfy(146,5): else branch +35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial +36: BranchCoverage.dfy(156,13): then branch +37: BranchCoverage.dfy(156,13): else branch +38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial +39: BranchCoverage.dfy(165,3): then branch +40: BranchCoverage.dfy(167,3): else branch +41: BranchCoverage.dfy(173,15): entry to method _module._default.Main 0: 3 1: 0 2: 0 diff --git a/Test/comp/BranchCoverage4.go b/Test/comp/BranchCoverage.dfy.go.files/BranchCoverage4.go similarity index 100% rename from Test/comp/BranchCoverage4.go rename to Test/comp/BranchCoverage.dfy.go.files/BranchCoverage4.go diff --git a/Test/comp/CodeCoverage.java b/Test/comp/BranchCoverage.dfy.java.files/CodeCoverage.java similarity index 100% rename from Test/comp/CodeCoverage.java rename to Test/comp/BranchCoverage.dfy.java.files/CodeCoverage.java diff --git a/Test/comp/BranchCoverage3.js b/Test/comp/BranchCoverage.dfy.js.files/BranchCoverage3.js similarity index 100% rename from Test/comp/BranchCoverage3.js rename to Test/comp/BranchCoverage.dfy.js.files/BranchCoverage3.js diff --git a/Test/comp/JsModule.dfy b/Test/comp/JsModule.dfy index b4ddda74e73..2727a01947c 100644 --- a/Test/comp/JsModule.dfy +++ b/Test/comp/JsModule.dfy @@ -1,6 +1,8 @@ -// RUN: %dafny /compile:3 "%s" /compileTarget:js > "%t" -// note: putting /compileTarget:js after "%s" overrides user-provided option -// RUN: %diff "%s.expect" "%t" +/* +--- +compile: 3 +compileTarget: js +*/ // "url" is a built-in package in node, so it should be accessible to the // test suite without further requirements on the setup. From 866d2ec20ff45bb608a38c28b4e31e820c77992a Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 10 Jun 2020 11:50:18 -0700 Subject: [PATCH 017/192] Missing file --- Test/DafnyTests/DafnyTests.cs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 0826b15ccba..388c73f1814 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -22,15 +22,17 @@ public class DafnyTests { public static string RunDafny(IEnumerable arguments) { List dafnyArguments = new List { // Expected output does not contain logo - "-nologo", - "-countVerificationErrors:0", + "/nologo", + "/countVerificationErrors:0", // We do not want absolute or relative paths in error messages, just the basename of the file - "-useBaseNameForFileName", + "/useBaseNameForFileName", // We do not want output such as "Compiled program written to Foo.cs" // from the compilers, since that changes with the target language - "-compileVerbose:0" + "/compileVerbose:0", + + "/out:" + Path.GetTempPath(); }; dafnyArguments.AddRange(arguments); @@ -211,7 +213,9 @@ private static string GetRelativePath(string relativeTo, string path) { private static void AssertEqualWithDiff(string expected, string actual) { if (expected != actual) { - // TODO-RS: Do better than shelling out to a linux utility + // TODO-RS: Do better than shelling out to a linux utility. + // Disappointingly, I couldn't find any easy solutions for an in-memory + string expectedPath = Path.GetTempFileName(); File.WriteAllText(expectedPath, expected); string actualPath = Path.GetTempFileName(); @@ -237,8 +241,14 @@ public void Test(string file, string args) { specialCase = true; expectedOutputPath = specialCasePath; } + + // Include any additional files + var additionalFilesPath = fullInputPath + "." + language + ".files"; + if (Directory.Exists(additionalFilesPath)) { + arguments = arguments.Concat(Directory.GetFiles(additionalFilesPath)).ToArray(); + } } - + string expectedOutput = File.ReadAllText(expectedOutputPath); string output = RunDafny(new List {file}.Concat(arguments)); From 59269cd84e296be2ecd673809ef2281d1bf5335c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 11 Jun 2020 18:33:24 -0700 Subject: [PATCH 018/192] Relocated some more additional files --- Test/DafnyTests/DafnyTests.cs | 2 +- Test/comp/CSharpStyling.dfy | 7 +++++-- .../{ => CSharpStyling.dfy.cs.files}/CSharpStyling2.cs | 0 Test/comp/{ => Extern.dfy.cs.files}/Extern2.cs | 0 Test/comp/{ => Extern.dfy.go.files}/Extern4.go | 0 Test/comp/{ => Extern.dfy.java.files}/AllDafny.java | 0 Test/comp/{ => Extern.dfy.java.files}/AllExtern.java | 0 Test/comp/{ => Extern.dfy.java.files}/LibClass.java | 0 Test/comp/{ => Extern.dfy.java.files}/Mixed.java | 0 Test/comp/{ => Extern.dfy.java.files}/OtherClass.java | 0 Test/comp/{ => Extern.dfy.js.files}/Extern3.js | 0 Test/comp/ExternCtors.dfy | 7 +------ .../Library.cs | 0 Test/comp/ExternCtors.dfy.expect | 4 ---- .../Library.go | 0 .../Class.java | 0 .../Library.js | 0 Test/comp/GoModule.dfy | 7 ++++--- 18 files changed, 11 insertions(+), 16 deletions(-) rename Test/comp/{ => CSharpStyling.dfy.cs.files}/CSharpStyling2.cs (100%) rename Test/comp/{ => Extern.dfy.cs.files}/Extern2.cs (100%) rename Test/comp/{ => Extern.dfy.go.files}/Extern4.go (100%) rename Test/comp/{ => Extern.dfy.java.files}/AllDafny.java (100%) rename Test/comp/{ => Extern.dfy.java.files}/AllExtern.java (100%) rename Test/comp/{ => Extern.dfy.java.files}/LibClass.java (100%) rename Test/comp/{ => Extern.dfy.java.files}/Mixed.java (100%) rename Test/comp/{ => Extern.dfy.java.files}/OtherClass.java (100%) rename Test/comp/{ => Extern.dfy.js.files}/Extern3.js (100%) rename Test/comp/{ExternCtors-externs => ExternCtors.dfy.cs.files}/Library.cs (100%) rename Test/comp/{ExternCtors-externs => ExternCtors.dfy.go.files}/Library.go (100%) rename Test/comp/{ExternCtors-externs => ExternCtors.dfy.java.files}/Class.java (100%) rename Test/comp/{ExternCtors-externs => ExternCtors.dfy.js.files}/Library.js (100%) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 388c73f1814..645601195c9 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -32,7 +32,7 @@ public static string RunDafny(IEnumerable arguments) { // from the compilers, since that changes with the target language "/compileVerbose:0", - "/out:" + Path.GetTempPath(); + "/out:" + Path.Combine(COMP_DIR, "Output") }; dafnyArguments.AddRange(arguments); diff --git a/Test/comp/CSharpStyling.dfy b/Test/comp/CSharpStyling.dfy index 038eac22aef..226b9662377 100644 --- a/Test/comp/CSharpStyling.dfy +++ b/Test/comp/CSharpStyling.dfy @@ -1,5 +1,8 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" CSharpStyling2.cs > "%t" -// RUN: %diff "%s.expect" "%t" +/* +--- +compile: 3 +compileTarget: cs +*/ method Main() { var c := new MyClass(50); diff --git a/Test/comp/CSharpStyling2.cs b/Test/comp/CSharpStyling.dfy.cs.files/CSharpStyling2.cs similarity index 100% rename from Test/comp/CSharpStyling2.cs rename to Test/comp/CSharpStyling.dfy.cs.files/CSharpStyling2.cs diff --git a/Test/comp/Extern2.cs b/Test/comp/Extern.dfy.cs.files/Extern2.cs similarity index 100% rename from Test/comp/Extern2.cs rename to Test/comp/Extern.dfy.cs.files/Extern2.cs diff --git a/Test/comp/Extern4.go b/Test/comp/Extern.dfy.go.files/Extern4.go similarity index 100% rename from Test/comp/Extern4.go rename to Test/comp/Extern.dfy.go.files/Extern4.go diff --git a/Test/comp/AllDafny.java b/Test/comp/Extern.dfy.java.files/AllDafny.java similarity index 100% rename from Test/comp/AllDafny.java rename to Test/comp/Extern.dfy.java.files/AllDafny.java diff --git a/Test/comp/AllExtern.java b/Test/comp/Extern.dfy.java.files/AllExtern.java similarity index 100% rename from Test/comp/AllExtern.java rename to Test/comp/Extern.dfy.java.files/AllExtern.java diff --git a/Test/comp/LibClass.java b/Test/comp/Extern.dfy.java.files/LibClass.java similarity index 100% rename from Test/comp/LibClass.java rename to Test/comp/Extern.dfy.java.files/LibClass.java diff --git a/Test/comp/Mixed.java b/Test/comp/Extern.dfy.java.files/Mixed.java similarity index 100% rename from Test/comp/Mixed.java rename to Test/comp/Extern.dfy.java.files/Mixed.java diff --git a/Test/comp/OtherClass.java b/Test/comp/Extern.dfy.java.files/OtherClass.java similarity index 100% rename from Test/comp/OtherClass.java rename to Test/comp/Extern.dfy.java.files/OtherClass.java diff --git a/Test/comp/Extern3.js b/Test/comp/Extern.dfy.js.files/Extern3.js similarity index 100% rename from Test/comp/Extern3.js rename to Test/comp/Extern.dfy.js.files/Extern3.js diff --git a/Test/comp/ExternCtors.dfy b/Test/comp/ExternCtors.dfy index 35b6112bab6..524a7095429 100644 --- a/Test/comp/ExternCtors.dfy +++ b/Test/comp/ExternCtors.dfy @@ -1,9 +1,4 @@ -// RUN: %dafny /compile:3 /compileTarget:cs "%s" ExternCtors-externs/Library.cs > "%t" -// RUN: %dafny /compile:3 /compileTarget:java "%s" ExternCtors-externs/Class.java >> "%t" -// RUN: %diff "%s.expect" "%t" - -// FIXME: Extern constructors are currently broken in Go and JavaScript, -// so they are omitted +// FIXME: Extern constructors are currently broken in Go and JavaScript method Main() { Library.Class.SayHi(); diff --git a/Test/comp/ExternCtors-externs/Library.cs b/Test/comp/ExternCtors.dfy.cs.files/Library.cs similarity index 100% rename from Test/comp/ExternCtors-externs/Library.cs rename to Test/comp/ExternCtors.dfy.cs.files/Library.cs diff --git a/Test/comp/ExternCtors.dfy.expect b/Test/comp/ExternCtors.dfy.expect index 99ddf258037..1bcf5332a10 100644 --- a/Test/comp/ExternCtors.dfy.expect +++ b/Test/comp/ExternCtors.dfy.expect @@ -2,7 +2,3 @@ Dafny program verifier finished with 1 verified, 0 errors Hello! My value is 42 - -Dafny program verifier finished with 1 verified, 0 errors -Hello! -My value is 42 diff --git a/Test/comp/ExternCtors-externs/Library.go b/Test/comp/ExternCtors.dfy.go.files/Library.go similarity index 100% rename from Test/comp/ExternCtors-externs/Library.go rename to Test/comp/ExternCtors.dfy.go.files/Library.go diff --git a/Test/comp/ExternCtors-externs/Class.java b/Test/comp/ExternCtors.dfy.java.files/Class.java similarity index 100% rename from Test/comp/ExternCtors-externs/Class.java rename to Test/comp/ExternCtors.dfy.java.files/Class.java diff --git a/Test/comp/ExternCtors-externs/Library.js b/Test/comp/ExternCtors.dfy.js.files/Library.js similarity index 100% rename from Test/comp/ExternCtors-externs/Library.js rename to Test/comp/ExternCtors.dfy.js.files/Library.js diff --git a/Test/comp/GoModule.dfy b/Test/comp/GoModule.dfy index 6badfd10016..b578a8b7efa 100644 --- a/Test/comp/GoModule.dfy +++ b/Test/comp/GoModule.dfy @@ -1,6 +1,7 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 "%s" /compileTarget:go > "%t" -// note: putting /compileTarget:go after "%s" overrides user-provided option -// RUN: %diff "%s.expect" "%t" +/* +--- +compileTarget: go +*/ // "url" is a built-in package, so it should be accessible to the // test suite without further requirements on the setup. From bb01970b0092af0643a5b04e78a5fd36bb08eb9a Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 11 Jun 2020 18:36:15 -0700 Subject: [PATCH 019/192] Missed file --- Test/comp/Extern.dfy.expect | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/Test/comp/Extern.dfy.expect b/Test/comp/Extern.dfy.expect index a9bf891b291..f3b7991bdaf 100644 --- a/Test/comp/Extern.dfy.expect +++ b/Test/comp/Extern.dfy.expect @@ -8,33 +8,3 @@ Extern static method says: Mixed.P Extern instance method says: Mixed.IP 2002 AllExtern.P - -Dafny program verifier finished with 6 verified, 0 errors -Hello -31 62 45 OtherClass.CallMe -AllDafny.M -Extern static method says: Mixed.P -1001 -Extern instance method says: Mixed.IP -2002 -AllExtern.P - -Dafny program verifier finished with 6 verified, 0 errors -Hello -31 62 45 OtherClass.CallMe -AllDafny.M -Extern static method says: Mixed.P -1001 -Extern instance method says: Mixed.IP -2002 -AllExtern.P - -Dafny program verifier finished with 6 verified, 0 errors -Hello -31 62 45 OtherClass.CallMe -AllDafny.M -Extern static method says: Mixed.P -1001 -Extern instance method says: Mixed.IP -2002 -AllExtern.P From fd27e50f1eb6bf61ea9079b9c7a1829511743d14 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 12 Jun 2020 13:42:03 -0700 Subject: [PATCH 020/192] Lingering lit commands --- Test/DafnyTests/TemporaryDirectory.cs | 22 ++++++++++++++++++++++ Test/comp/Extern.dfy | 6 ------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 Test/DafnyTests/TemporaryDirectory.cs diff --git a/Test/DafnyTests/TemporaryDirectory.cs b/Test/DafnyTests/TemporaryDirectory.cs new file mode 100644 index 00000000000..fc0bb8fd96c --- /dev/null +++ b/Test/DafnyTests/TemporaryDirectory.cs @@ -0,0 +1,22 @@ +using System; +using System.IO; + +namespace DafnyTests { + public class TemporaryDirectory : IDisposable { + public readonly DirectoryInfo Dir; + + public TemporaryDirectory() { + string dirPath; + do { + dirPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + } while (File.Exists(dirPath) || Directory.Exists(dirPath)); + + Dir = Directory.CreateDirectory(dirPath); + } + + public void Dispose() { + // TODO-RS: Be way more careful with finalizers/threading/etc. + Dir.Delete(true); + } + } +} \ No newline at end of file diff --git a/Test/comp/Extern.dfy b/Test/comp/Extern.dfy index f375937fe0a..cc519998f0f 100644 --- a/Test/comp/Extern.dfy +++ b/Test/comp/Extern.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /compileTarget:cs "%s" Extern2.cs > "%t" -// RUN: %dafny /compile:3 /compileTarget:js "%s" Extern3.js >> "%t" -// RUN: %dafny /compile:3 /compileTarget:go "%s" Extern4.go >> "%t" -// RUN: %dafny /compile:3 /compileTarget:java "%s" LibClass.java OtherClass.java AllDafny.java Mixed.java AllExtern.java >> "%t" -// RUN: %diff "%s.expect" "%t" - method Main() { print "Hello\n"; var x, y := Library.LibClass.CallMeInt(30); From 56ae64124bfc1cd552c3cdafeb982cdd0ba69a73 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 16 Jun 2020 21:55:15 -0700 Subject: [PATCH 021/192] Possibly better /out value --- Test/DafnyTests/DafnyTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 645601195c9..b65f780c7e6 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -32,7 +32,7 @@ public static string RunDafny(IEnumerable arguments) { // from the compilers, since that changes with the target language "/compileVerbose:0", - "/out:" + Path.Combine(COMP_DIR, "Output") + "/out:Output/Files" }; dafnyArguments.AddRange(arguments); From a5874e6990297c902c0e1154f1cd32aedf5bbfc4 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 18 Jun 2020 13:16:11 -0700 Subject: [PATCH 022/192] Use temporary directory for build output --- Test/DafnyTests/DafnyTests.cs | 14 ++++++++++++-- Test/DafnyTests/TemporaryDirectory.cs | 27 +++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index b65f780c7e6..beb41e57347 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -18,6 +18,7 @@ public class DafnyTests { private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; + private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; public static string RunDafny(IEnumerable arguments) { List dafnyArguments = new List { @@ -249,9 +250,18 @@ public void Test(string file, string args) { } } - string expectedOutput = File.ReadAllText(expectedOutputPath); + var argumentsWithFile = new List {file}.Concat(arguments); + var expectedOutput = File.ReadAllText(expectedOutputPath); - string output = RunDafny(new List {file}.Concat(arguments)); + string output; + if (arguments.Any(arg => arg.StartsWith("/out"))) { + output = RunDafny(argumentsWithFile); + } else { + using (var tempDir = new TemporaryDirectory(OUTPUT_DIR)) { + argumentsWithFile = new List {"/out:" + tempDir.DirInfo.FullName + "/Program"}.Concat(argumentsWithFile); + output = RunDafny(argumentsWithFile); + } + } AssertEqualWithDiff(expectedOutput, output); Skip.If(specialCase, "Confirmed known exception for arguments: " + args); diff --git a/Test/DafnyTests/TemporaryDirectory.cs b/Test/DafnyTests/TemporaryDirectory.cs index fc0bb8fd96c..1dbe59773f9 100644 --- a/Test/DafnyTests/TemporaryDirectory.cs +++ b/Test/DafnyTests/TemporaryDirectory.cs @@ -3,20 +3,35 @@ namespace DafnyTests { public class TemporaryDirectory : IDisposable { - public readonly DirectoryInfo Dir; + public readonly DirectoryInfo DirInfo; - public TemporaryDirectory() { + public TemporaryDirectory(string parent, string prefix = "") { string dirPath; do { - dirPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + dirPath = Path.Combine(parent, prefix + Path.GetRandomFileName()); } while (File.Exists(dirPath) || Directory.Exists(dirPath)); - Dir = Directory.CreateDirectory(dirPath); + DirInfo = Directory.CreateDirectory(dirPath); } public void Dispose() { - // TODO-RS: Be way more careful with finalizers/threading/etc. - Dir.Delete(true); + Dispose(true); + } + + ~TemporaryDirectory() { + Dispose(false); + } + + protected virtual void Dispose(bool disposing) { + SafeDelete(); + } + + public void SafeDelete() { + try { + DirInfo.Delete(true); + } catch { + // Best effort only + } } } } \ No newline at end of file From 88ac6e4188b06c8c326710885fb1f2f590abae30 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 18 Jun 2020 14:04:22 -0700 Subject: [PATCH 023/192] All other deterministic known discrepencies Remaining cases require pattern matching --- .gitignore | 4 - Test/DafnyTests/DafnyTests.cs | 14 ++- Test/comp/Collections.dfy.cs.expect | 69 ++++++++++++++ Test/comp/Collections.dfy.expect | 16 ++-- Test/comp/Collections.dfy.java.expect | 69 ++++++++++++++ Test/comp/Comprehensions.dfy.java.expect | 22 +++++ .../comp/CovariantCollections.dfy.java.expect | 2 +- Test/comp/Dt.dfy.java.expect | 19 ++++ Test/comp/NativeNumbers.dfy.js.expect | 14 +-- Test/comp/Numbers.dfy.go.expect | 92 +++++++++++++++++++ Test/comp/Poly.dfy.java.expect | 2 +- Test/comp/TailRecursion.dfy.go.expect | 16 ++++ Test/comp/TailRecursion.dfy.js.expect | 16 ++++ 13 files changed, 331 insertions(+), 24 deletions(-) create mode 100644 Test/comp/Collections.dfy.cs.expect create mode 100644 Test/comp/Collections.dfy.java.expect create mode 100644 Test/comp/Comprehensions.dfy.java.expect create mode 100644 Test/comp/Dt.dfy.java.expect create mode 100644 Test/comp/Numbers.dfy.go.expect create mode 100644 Test/comp/TailRecursion.dfy.go.expect create mode 100644 Test/comp/TailRecursion.dfy.js.expect diff --git a/.gitignore b/.gitignore index ad441c8c294..f59232d71b2 100644 --- a/.gitignore +++ b/.gitignore @@ -60,10 +60,6 @@ Docs/OnlineTutorial/DocumentationTransducer.pdb Docs/OnlineTutorial/DocumentationTransducer/obj Docs/OnlineTutorial/manuscripts/*.htm Docs/OnlineTutorial/manuscripts/*.*.dfy -Test/comp/*.cs -Test/comp/*.js -Test/comp/*-go -Test/comp/**/*.java Test/expectations/*.cs Test/expectations/*.js Test/expectations/*-go diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index beb41e57347..74a7bec972d 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -62,7 +62,8 @@ public static string RunDafny(IEnumerable arguments) { string output = dafnyProcess.StandardOutput.ReadToEnd(); string error = dafnyProcess.StandardError.ReadToEnd(); if (dafnyProcess.ExitCode != 0) { - Assert.True(false, output + "\n" + error); + var message = String.Format("Non-zero Dafny exit code: {0}\n{1}\n{2}", dafnyProcess.ExitCode, output, error); + Assert.True(false, message); } return output + error; @@ -222,7 +223,8 @@ private static void AssertEqualWithDiff(string expected, string actual) { string actualPath = Path.GetTempFileName(); File.WriteAllText(actualPath, actual); string diff = Exec("diff", "--unified=3", expectedPath, actualPath); - throw new AssertActualExpectedException(expected, actual, diff); + string message = "AssertEqualWithDiff() Failure\n" + diff; + throw new AssertActualExpectedException(expected, actual, message); } } @@ -257,8 +259,14 @@ public void Test(string file, string args) { if (arguments.Any(arg => arg.StartsWith("/out"))) { output = RunDafny(argumentsWithFile); } else { + // Note that the temporary directory has to be an ancestor of Test + // or else Javascript won't be able to locate bignumber.js :( using (var tempDir = new TemporaryDirectory(OUTPUT_DIR)) { - argumentsWithFile = new List {"/out:" + tempDir.DirInfo.FullName + "/Program"}.Concat(argumentsWithFile); + // Add an extra component to the path to keep the files created inside the + // temporary directory, since some compilers will + // interpret the path as a single file basename rather than a directory. + var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; + argumentsWithFile = new List {outArgument}.Concat(argumentsWithFile); output = RunDafny(argumentsWithFile); } } diff --git a/Test/comp/Collections.dfy.cs.expect b/Test/comp/Collections.dfy.cs.expect new file mode 100644 index 00000000000..53a0cea6ebd --- /dev/null +++ b/Test/comp/Collections.dfy.cs.expect @@ -0,0 +1,69 @@ + +Dafny program verifier finished with 17 verified, 0 errors +Sets: {} {17, 82} {12, 17} + cardinality: 0 2 2 + union: {17, 82} {12, 17, 82} + intersection: {} {17} + difference: {} {82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true +false true +|s|=4 |S|=16 +{{a, c, d}, {a, b, c}, {}, {a}, {b}, {c}, {d}, {a, b, c, d}, {b, c, d}, {a, b, d}, {a, b}, {a, c}, {a, d}, {b, c}, {b, d}, {c, d}} +Multisets: multiset{} multiset{17, 17, 82, 82} multiset{12, 17} + cardinality: 0 4 2 + union: multiset{17, 17, 82, 82} multiset{12, 17, 17, 17, 82, 82} + intersection: multiset{} multiset{17} + difference: multiset{} multiset{17, 82, 82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true + update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{12, 17, 17} + multiplicity: 0 2 1 +Sequences: [] [17, 82, 17, 82] [12, 17] + cardinality: 0 4 2 + update: [42, 82, 17, 82] [42, 17] + index: 17 12 + subsequence ([lo..hi]): [82, 17] [17] + subsequence ([lo..]): [82, 17, 82] [17] + subsequence ([..hi]): [17, 82, 17][12] + subsequence ([..]): [] [17, 82, 17, 82] [12, 17] + concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] + prefix: true false true + proper prefix: true false false + membership: false true true +Bound Bound Bound Bound Bound Bound +ed ed ed ed ed ed +e e e e e e +hello +hEllo +[2, 4, 6, 8, 10] +[2, 0, 6, 8, 10] +Strings: uRuR gu + cardinality: 0 4 2 + concatenation: uRuR uRuRgu + prefix: true false true + proper prefix: true false false + membership: false true true + constructed as sequence: guru + mix: hello-d ddd-hello +Maps: map[] map[17 := 2, 82 := 2] map[12 := 26, 17 := 0] + cardinality: 0 2 2 + keys: {} {17, 82} {12, 17} + values: {} {2} {0, 26} + items: {} {(17, 2), (82, 2)} {(12, 26), (17, 0)} + update: map[17 := 6] map[17 := 6, 82 := 2] map[12 := 26, 17 := 6] + lookup: false 2 0 +m: map[Color.Blue := 30, Color.Yellow := 21] +keys: {Color.Blue, Color.Yellow} +values: {21, 30} +items: {(Color.Blue, 30), (Color.Yellow, 21)} +2: 0 1 1 +3: 0 1 2 +There are 0 occurrences of 58 in the multiset +There are 536870912 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of null in the multiset diff --git a/Test/comp/Collections.dfy.expect b/Test/comp/Collections.dfy.expect index 53a0cea6ebd..e46a75199ec 100644 --- a/Test/comp/Collections.dfy.expect +++ b/Test/comp/Collections.dfy.expect @@ -2,7 +2,7 @@ Dafny program verifier finished with 17 verified, 0 errors Sets: {} {17, 82} {12, 17} cardinality: 0 2 2 - union: {17, 82} {12, 17, 82} + union: {17, 82} {17, 82, 12} intersection: {} {17} difference: {} {82} disjoint: true false @@ -11,10 +11,10 @@ Sets: {} {17, 82} {12, 17} membership: false true true false true |s|=4 |S|=16 -{{a, c, d}, {a, b, c}, {}, {a}, {b}, {c}, {d}, {a, b, c, d}, {b, c, d}, {a, b, d}, {a, b}, {a, c}, {a, d}, {b, c}, {b, d}, {c, d}} +{{}, {a}, {b}, {b, a}, {c}, {c, a}, {c, b}, {c, b, a}, {d}, {d, a}, {d, b}, {d, b, a}, {d, c}, {d, c, a}, {d, c, b}, {d, c, b, a}} Multisets: multiset{} multiset{17, 17, 82, 82} multiset{12, 17} cardinality: 0 4 2 - union: multiset{17, 17, 82, 82} multiset{12, 17, 17, 17, 82, 82} + union: multiset{17, 17, 82, 82} multiset{17, 17, 17, 82, 82, 12} intersection: multiset{} multiset{17} difference: multiset{} multiset{17, 82, 82} disjoint: true false @@ -50,16 +50,16 @@ Strings: uRuR gu membership: false true true constructed as sequence: guru mix: hello-d ddd-hello -Maps: map[] map[17 := 2, 82 := 2] map[12 := 26, 17 := 0] +Maps: map[] map[17 := 2, 82 := 2] map[17 := 0, 12 := 26] cardinality: 0 2 2 - keys: {} {17, 82} {12, 17} + keys: {} {17, 82} {17, 12} values: {} {2} {0, 26} - items: {} {(17, 2), (82, 2)} {(12, 26), (17, 0)} - update: map[17 := 6] map[17 := 6, 82 := 2] map[12 := 26, 17 := 6] + items: {} {(17, 2), (82, 2)} {(17, 0), (12, 26)} + update: map[17 := 6] map[17 := 6, 82 := 2] map[17 := 6, 12 := 26] lookup: false 2 0 m: map[Color.Blue := 30, Color.Yellow := 21] keys: {Color.Blue, Color.Yellow} -values: {21, 30} +values: {30, 21} items: {(Color.Blue, 30), (Color.Yellow, 21)} 2: 0 1 1 3: 0 1 2 diff --git a/Test/comp/Collections.dfy.java.expect b/Test/comp/Collections.dfy.java.expect new file mode 100644 index 00000000000..9b20282899b --- /dev/null +++ b/Test/comp/Collections.dfy.java.expect @@ -0,0 +1,69 @@ + +Dafny program verifier finished with 17 verified, 0 errors +Sets: {} {17, 82} {17, 12} + cardinality: 0 2 2 + union: {17, 82} {17, 82, 12} + intersection: {} {17} + difference: {} {82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true +false true +|s|=4 |S|=16 +{{}, {a}, {b}, {a, b}, {c}, {a, c}, {d}, {b, c}, {a, d}, {a, b, c}, {b, d}, {a, b, d}, {c, d}, {a, c, d}, {b, c, d}, {a, b, c, d}} +Multisets: multiset{} multiset{17, 17, 82, 82} multiset{17, 12} + cardinality: 0 4 2 + union: multiset{17, 17, 82, 82} multiset{17, 17, 17, 82, 82, 12} + intersection: multiset{} multiset{17} + difference: multiset{} multiset{17, 82, 82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true + update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{17, 17, 12} + multiplicity: 0 2 1 +Sequences: [] [17, 82, 17, 82] [12, 17] + cardinality: 0 4 2 + update: [42, 82, 17, 82] [42, 17] + index: 17 12 + subsequence ([lo..hi]): [82, 17] [17] + subsequence ([lo..]): [82, 17, 82] [17] + subsequence ([..hi]): [17, 82, 17][12] + subsequence ([..]): [] [17, 82, 17, 82] [12, 17] + concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] + prefix: true false true + proper prefix: true false false + membership: false true true +[B, o, u, n, d] [B, o, u, n, d] [B, o, u, n, d] [B, o, u, n, d] [B, o, u, n, d] [B, o, u, n, d] +[e, d] [e, d] [e, d] [e, d] [e, d] [e, d] +e e e e e e +hello +hEllo +[2, 4, 6, 8, 10] +[2, 0, 6, 8, 10] +Strings: [] [u, R, u, R] [g, u] + cardinality: 0 4 2 + concatenation: [u, R, u, R] [u, R, u, R, g, u] + prefix: true false true + proper prefix: true false false + membership: false true true + constructed as sequence: [g, u, r, u] + mix: hello-d ddd-hello +Maps: map[] map[17 := 2, 82 := 2] map[17 := 0, 12 := 26] + cardinality: 0 2 2 + keys: {} {17, 82} {17, 12} + values: {} {2} {0, 26} + items: {} {(17, 2), (82, 2)} {(17, 0), (12, 26)} + update: map[17 := 6] map[17 := 6, 82 := 2] map[12 := 26, 17 := 6] + lookup: false 2 0 +m: map[Color.Yellow := 21, Color.Blue := 30] +keys: {Color.Yellow, Color.Blue} +values: {21, 30} +items: {(Color.Yellow, 21), (Color.Blue, 30)} +2: 0 1 1 +3: 0 1 2 +There are 0 occurrences of 58 in the multiset +There are 536870912 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of null in the multiset diff --git a/Test/comp/Comprehensions.dfy.java.expect b/Test/comp/Comprehensions.dfy.java.expect new file mode 100644 index 00000000000..ffe220bc356 --- /dev/null +++ b/Test/comp/Comprehensions.dfy.java.expect @@ -0,0 +1,22 @@ + +Dafny program verifier finished with 16 verified, 0 errors +x=13 y=14 +x=13 y=14 b=[y, e, s] +p=(13, 14) +q=(13, 14, [y, e, s]) +true false +true false +map[12 := 6, 13 := 6, 14 := 7] +map[16 := 12, 17 := 13, 18 := 14] +XP returned: 0 false +after: 0 before: 0 +after: 2 before: 2 +XM returned: 16 +0 12 3 8 +0 12 3 8 +0 12 3 8 +0 12 3 8 +[1, 1, 1, 1] +[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] +[0, 1, 4, 9, 16, 25, 36, 49] +[0, 1, 2, 3, 4, 5, 6, 7] diff --git a/Test/comp/CovariantCollections.dfy.java.expect b/Test/comp/CovariantCollections.dfy.java.expect index 270c270419b..5883880f24d 100644 --- a/Test/comp/CovariantCollections.dfy.java.expect +++ b/Test/comp/CovariantCollections.dfy.java.expect @@ -2,4 +2,4 @@ Dafny program verifier finished with 2 verified, 0 errors CovariantCollections.dfy(28,6): Error: compilation of seq is not supported; consider introducing a ghost CovariantCollections.dfy(29,6): Error: compilation of seq is not supported; consider introducing a ghost -File CovariantCollections/CovariantCollections.java contains the partially compiled program +File Program/Program.java contains the partially compiled program diff --git a/Test/comp/Dt.dfy.java.expect b/Test/comp/Dt.dfy.java.expect new file mode 100644 index 00000000000..a47e9c0e7d4 --- /dev/null +++ b/Test/comp/Dt.dfy.java.expect @@ -0,0 +1,19 @@ + +Dafny program verifier finished with 12 verified, 0 errors +List.Nil +List.Cons(5, List.Nil) +(List.Nil, List.Cons(5, List.Nil)) +(List.Cons(5, List.Nil), List.Nil) +() +5 List.Nil +List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Cons(4, List.Cons(5, List.Cons(6, List.Nil))))))) +0 + 1 + 2 + 3 + 4 + 5 + 6 == 21 (once more, that's 21) +List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Nil)))) +0 List.Cons(1, List.Cons(2, List.Cons(3, List.Nil))) +Stream.Next +Stream.Next +{Berry.Jordgubb, Berry.Smultron, Berry.Hallon} +CoBerry.Hjortron true true false +false +42 q hello +1701 diff --git a/Test/comp/NativeNumbers.dfy.js.expect b/Test/comp/NativeNumbers.dfy.js.expect index 90a7c69a392..13366e17630 100644 --- a/Test/comp/NativeNumbers.dfy.js.expect +++ b/Test/comp/NativeNumbers.dfy.js.expect @@ -1,9 +1,9 @@ -NativeNumbers.dfy(17,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(10,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(11,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(12,28): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(13,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(15,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(16,31): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(17,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. NativeNumbers.dfy(18,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(19,28): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(20,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(22,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(23,31): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(24,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(25,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. 8 resolution/type errors detected in NativeNumbers.dfy diff --git a/Test/comp/Numbers.dfy.go.expect b/Test/comp/Numbers.dfy.go.expect new file mode 100644 index 00000000000..8f76e618253 --- /dev/null +++ b/Test/comp/Numbers.dfy.go.expect @@ -0,0 +1,92 @@ + +Dafny program verifier finished with 29 verified, 0 errors +0 +0 +3 +-5 +2147483647 (aka C# int.MaxValue) +2147483648 (aka 2^31) +2147483649 +4294967295 (aka C# uint.MaxValue) +4294967296 (aka 2^32) +590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) +590295810358705651712 (aka 2^53) +-590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) +-590295810358705651712 +9223372036854775807 (aka C# long.MaxValue) +9223372036854775808 (aka 2^63) +9223372036854775809 +18446744073709551615 (aka C# ulong.MaxValue) +18446744073709551616 (aka 2^64) +1267650600228229401496703205376 (aka 2^100) +170141183460469231731687303715884105727 (aka M_39) +0.0 +0.0 +3.0 +-5.0 +3.14 +-2.71 +1000000000.0 (aka a billion) +0.0000000000667408 (aka G) +0.0000000000667408 (aka G) +0.0000000000667408 (aka G) +17014118346046923173168730371588410572.7 (aka 1/10 of M_39) +0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 (aka 1/googol) +35 27 -27 +124 -124 +-124 124 +g Q 2 +7 3 31 +-8 1 -31 +-7 3 31 +8 1 -31 +0 0 0 +0 0 0 +0.2 +35.2 27.2 -27.2 +124.8 -124.8 +-124.8 124.8 +7.8 31.2 +-7.8 -31.2 +-7.8 31.2 +7.8 -31.2 +0.0 0.0 0.0 +120.0 120.0 8.0 -8.0 +123.4567 -123.4567 0.1234 -0.1234 0.8 +0.2 0.02 0.00002 +-0.2 -0.02 -0.00002 +(20.0 / 3.0) (-20.0 / 3.0) (-20.0 / 3.0) (20.0 / 3.0) +0.0 0.0 0.81 0.81 1.0 +true false +0 2 2 2 +0 3 6 46 +0 1 36 2116 +0 0 0 0 +29 2 2 4294967283 2 +29 2 2 9007199254740979 2 +0 0 0 9 +0 0 0 9 +0 0 0 9 +0 4294967295 1267650600228229401496703205375 6 +8 4 +18 72 4 +0 +200 300 +100 100 18 +0: IsNat: true, Offset: 0, IsLimit: true, IsSucc: false +1: IsNat: true, Offset: 1, IsLimit: false, IsSucc: true +42: IsNat: true, Offset: 42, IsLimit: false, IsSucc: true +int: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +MyNumber: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +120 120.0 120 120 120 x +120 120.0 120 120 120 x +120 120 +120 120.0 120 120 120 x +120 120.0 120 120 120 x +120 120 diff --git a/Test/comp/Poly.dfy.java.expect b/Test/comp/Poly.dfy.java.expect index 957e41e14d7..cc1ae3008a1 100644 --- a/Test/comp/Poly.dfy.java.expect +++ b/Test/comp/Poly.dfy.java.expect @@ -3,4 +3,4 @@ Dafny program verifier finished with 9 verified, 0 errors Poly.dfy(48,16): Error: compilation of seq is not supported; consider introducing a ghost Poly.dfy(59,16): Error: compilation of set is not supported; consider introducing a ghost Poly.dfy(63,21): Error: compilation of multiset is not supported; consider introducing a ghost -File Poly/Poly.java contains the partially compiled program +File Program/Program.java contains the partially compiled program diff --git a/Test/comp/TailRecursion.dfy.go.expect b/Test/comp/TailRecursion.dfy.go.expect new file mode 100644 index 00000000000..e429e2b0a83 --- /dev/null +++ b/Test/comp/TailRecursion.dfy.go.expect @@ -0,0 +1,16 @@ + +Dafny program verifier finished with 20 verified, 0 errors +2000000 2000000 +2000000 2000000 +1000000 1000000 +10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 == 55 +TriangleNumber(10) = 55 +TriangleNumber_Real(10) = 55.0 +TriangleNumber_ORDINAL(10) = 55 +Factorial(5) = 120 +Union(8) = {0, 2, 4, 6, 8, 7, 5, 3, 1} +UpTo(10) = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +DownFrom(10) = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] +Sum(List.Cons(100, List.Cons(40, List.Cons(60, List.Nil)))) = 200 +TheBigSubtract(100) = -12 +TailNat(10) = 50 diff --git a/Test/comp/TailRecursion.dfy.js.expect b/Test/comp/TailRecursion.dfy.js.expect new file mode 100644 index 00000000000..e429e2b0a83 --- /dev/null +++ b/Test/comp/TailRecursion.dfy.js.expect @@ -0,0 +1,16 @@ + +Dafny program verifier finished with 20 verified, 0 errors +2000000 2000000 +2000000 2000000 +1000000 1000000 +10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 == 55 +TriangleNumber(10) = 55 +TriangleNumber_Real(10) = 55.0 +TriangleNumber_ORDINAL(10) = 55 +Factorial(5) = 120 +Union(8) = {0, 2, 4, 6, 8, 7, 5, 3, 1} +UpTo(10) = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +DownFrom(10) = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] +Sum(List.Cons(100, List.Cons(40, List.Cons(60, List.Nil)))) = 200 +TheBigSubtract(100) = -12 +TailNat(10) = 50 From 810e1b41975a901657366265589ff4a3ee4827a8 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 18 Jun 2020 16:18:06 -0700 Subject: [PATCH 024/192] Convert new test case --- Test/DafnyTests/DafnyTests.cs | 4 +- Test/comp/StaticMembersOfGenericTypes.dfy | 6 --- .../StaticMembersOfGenericTypes.dfy.expect | 54 ------------------- 3 files changed, 1 insertion(+), 63 deletions(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 74a7bec972d..c9aa27a8280 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -31,9 +31,7 @@ public static string RunDafny(IEnumerable arguments) { // We do not want output such as "Compiled program written to Foo.cs" // from the compilers, since that changes with the target language - "/compileVerbose:0", - - "/out:Output/Files" + "/compileVerbose:0" }; dafnyArguments.AddRange(arguments); diff --git a/Test/comp/StaticMembersOfGenericTypes.dfy b/Test/comp/StaticMembersOfGenericTypes.dfy index 9db643b01b4..9e5d733e9e9 100644 --- a/Test/comp/StaticMembersOfGenericTypes.dfy +++ b/Test/comp/StaticMembersOfGenericTypes.dfy @@ -1,9 +1,3 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - method Main() { GenericClass(); FunctionValues(); diff --git a/Test/comp/StaticMembersOfGenericTypes.dfy.expect b/Test/comp/StaticMembersOfGenericTypes.dfy.expect index 408d179dfe2..f07a820f11f 100644 --- a/Test/comp/StaticMembersOfGenericTypes.dfy.expect +++ b/Test/comp/StaticMembersOfGenericTypes.dfy.expect @@ -16,57 +16,3 @@ true false (5, 2.0, true) (5, 2.0, true) (6, 3.0, false) 50 3 4 149 - -Dafny program verifier finished with 9 verified, 0 errors -(0, 1) (true, 2, 3) -0 25 (20, 21) (20, 21) 23 22 -0 25 (20, 21) (20, 21) 23 22 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -0 25 (20, 21) (20, 21) 23 22 -0 25 (20, 21) (20, 21) 23 22 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -(2.0, true) (3.0, false) -(2.0, true) (3.0, false) -true false -(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) -(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) -50 3 4 -149 - -Dafny program verifier finished with 9 verified, 0 errors -(0, 1) (true, 2, 3) -0 25 (20, 21) (20, 21) 23 22 -0 25 (20, 21) (20, 21) 23 22 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -0 25 (20, 21) (20, 21) 23 22 -0 25 (20, 21) (20, 21) 23 22 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -(2.0, true) (3.0, false) -(2.0, true) (3.0, false) -true false -(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) -(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) -50 3 4 -149 - -Dafny program verifier finished with 9 verified, 0 errors -(0, 1) (true, 2, 3) -0 25 (20, 21) (20, 21) 23 22 -0 25 (20, 21) (20, 21) 23 22 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -0 25 (20, 21) (20, 21) 23 22 -0 25 (20, 21) (20, 21) 23 22 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -0 25 (true, 20, 21) (true, 20, 21) true 22 23 -(2.0, true) (3.0, false) -(2.0, true) (3.0, false) -true false -(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) -(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) -50 3 4 -149 From d2f7e3ceab0e3082bf939394a173e08b7096d760 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 18 Jun 2020 16:26:55 -0700 Subject: [PATCH 025/192] Remove binary from earlier experimentation --- Binaries/testdafny | 2 -- 1 file changed, 2 deletions(-) delete mode 100755 Binaries/testdafny diff --git a/Binaries/testdafny b/Binaries/testdafny deleted file mode 100755 index f78b2c9ec25..00000000000 --- a/Binaries/testdafny +++ /dev/null @@ -1,2 +0,0 @@ -DAFNY_TEST_ARGV=$* -dotnet test ../../Test/DafnyTests/DafnyTests-NetCore.csproj From 24dc118e64b5c10de5cb71ddeea9b05151b52da2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 20 Jun 2020 22:26:26 -0700 Subject: [PATCH 026/192] Progress on running from output directory Z3 dependency still a problem --- .gitignore | 2 ++ Source/Dafny-NetCore.sln => Dafny-NetCore.sln | 8 +++++- Source/Dafny.sln => Dafny.sln | 28 ++++++++++++++++--- Directory.Build.props | 1 + Test/DafnyTests-NetCore.sln | 16 ----------- Test/DafnyTests/DafnyTests-NetCore.csproj | 6 ++++ Test/DafnyTests/DafnyTests.cs | 23 ++++++--------- Test/DafnyTests/DafnyTests.csproj | 13 ++++++--- .../ParallelTheoryAttribute.cs | 2 +- .../{DafnyTests => testing}/TestAttribute.dfy | 0 .../TestAttribute.dfy.expect | 0 .../TestAttributeErrors.dfy | 0 .../TestAttributeErrors.dfy.expect | 0 13 files changed, 58 insertions(+), 41 deletions(-) rename Source/Dafny-NetCore.sln => Dafny-NetCore.sln (54%) rename Source/Dafny.sln => Dafny.sln (76%) delete mode 100644 Test/DafnyTests-NetCore.sln rename Test/{DafnyTests => testing}/TestAttribute.dfy (100%) rename Test/{DafnyTests => testing}/TestAttribute.dfy.expect (100%) rename Test/{exceptions => testing}/TestAttributeErrors.dfy (100%) rename Test/{exceptions => testing}/TestAttributeErrors.dfy.expect (100%) diff --git a/.gitignore b/.gitignore index 1e490747c20..6189df845d6 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ Package/ Source/*/bin/ Source/*/obj/ +Source/*/bin_core/ Source/*/obj_core/ Source/packages @@ -38,6 +39,7 @@ Source/DafnyExtension/z3.exe Test/*/bin/ Test/*/obj/ +Test/*/bin_core/ Test/*/obj_core/ Test/packages Test/traits/TraitCompile.cs diff --git a/Source/Dafny-NetCore.sln b/Dafny-NetCore.sln similarity index 54% rename from Source/Dafny-NetCore.sln rename to Dafny-NetCore.sln index b9e4a35f359..20953870148 100644 --- a/Source/Dafny-NetCore.sln +++ b/Dafny-NetCore.sln @@ -1,6 +1,8 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyRuntime-NetCore", "DafnyRuntime\DafnyRuntime-NetCore.csproj", "{F6DFCDF8-91AA-4222-A323-EBA22827793D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyRuntime-NetCore", "Source\DafnyRuntime\DafnyRuntime-NetCore.csproj", "{F6DFCDF8-91AA-4222-A323-EBA22827793D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests-NetCore", "Test\DafnyTests\DafnyTests-NetCore.csproj", "{9B0FD2BD-78C6-4D85-BCE3-557E85BC33C1}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -12,5 +14,9 @@ Global {F6DFCDF8-91AA-4222-A323-EBA22827793D}.Debug|Any CPU.Build.0 = Debug|Any CPU {F6DFCDF8-91AA-4222-A323-EBA22827793D}.Release|Any CPU.ActiveCfg = Release|Any CPU {F6DFCDF8-91AA-4222-A323-EBA22827793D}.Release|Any CPU.Build.0 = Release|Any CPU + {9B0FD2BD-78C6-4D85-BCE3-557E85BC33C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9B0FD2BD-78C6-4D85-BCE3-557E85BC33C1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9B0FD2BD-78C6-4D85-BCE3-557E85BC33C1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9B0FD2BD-78C6-4D85-BCE3-557E85BC33C1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/Source/Dafny.sln b/Dafny.sln similarity index 76% rename from Source/Dafny.sln rename to Dafny.sln index d4f6696edb8..5fdb74d1073 100644 --- a/Source/Dafny.sln +++ b/Dafny.sln @@ -1,16 +1,18 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 2012 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyDriver", "DafnyDriver\DafnyDriver.csproj", "{63400D1F-05B2-453E-9592-1EAB74B2C9CC}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyDriver", "Source\DafnyDriver\DafnyDriver.csproj", "{63400D1F-05B2-453E-9592-1EAB74B2C9CC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyPipeline", "Dafny\DafnyPipeline.csproj", "{FE44674A-1633-4917-99F4-57635E6FA740}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyPipeline", "Source\Dafny\DafnyPipeline.csproj", "{FE44674A-1633-4917-99F4-57635E6FA740}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyServer", "DafnyServer\DafnyServer.csproj", "{AC9B21AE-EBC1-4A27-AD11-ED031FC7B4A2}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyServer", "Source\DafnyServer\DafnyServer.csproj", "{AC9B21AE-EBC1-4A27-AD11-ED031FC7B4A2}" ProjectSection(ProjectDependencies) = postProject {FE44674A-1633-4917-99F4-57635E6FA740} = {FE44674A-1633-4917-99F4-57635E6FA740} EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyRuntime", "DafnyRuntime\DafnyRuntime.csproj", "{3E2944E9-3E1D-4E03-A881-FB04A7A6ED67}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyRuntime", "Source\DafnyRuntime\DafnyRuntime.csproj", "{3E2944E9-3E1D-4E03-A881-FB04A7A6ED67}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests", "Test\DafnyTests\DafnyTests.csproj", "{45AE3DDD-D0EA-4A27-943B-AC66A581BE03}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -89,6 +91,24 @@ Global {3E2944E9-3E1D-4E03-A881-FB04A7A6ED67}.Release|Any CPU.Build.0 = Release|Any CPU {3E2944E9-3E1D-4E03-A881-FB04A7A6ED67}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {3E2944E9-3E1D-4E03-A881-FB04A7A6ED67}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Checked|.NET.ActiveCfg = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Checked|.NET.Build.0 = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Checked|Any CPU.Build.0 = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Checked|Mixed Platforms.ActiveCfg = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Checked|Mixed Platforms.Build.0 = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|.NET.ActiveCfg = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|.NET.Build.0 = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|Any CPU.Build.0 = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|.NET.ActiveCfg = Release|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|.NET.Build.0 = Release|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Any CPU.ActiveCfg = Release|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Any CPU.Build.0 = Release|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Mixed Platforms.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Directory.Build.props b/Directory.Build.props index e18adfd972a..6d4f00371b1 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,5 +1,6 @@ obj_core\ + bin_core\ diff --git a/Test/DafnyTests-NetCore.sln b/Test/DafnyTests-NetCore.sln deleted file mode 100644 index 6e5e05bcbb0..00000000000 --- a/Test/DafnyTests-NetCore.sln +++ /dev/null @@ -1,16 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests", "DafnyTests\DafnyTests-NetCore.csproj", "{45AE3DDD-D0EA-4A27-943B-AC66A581BE03}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Debug|Any CPU.Build.0 = Debug|Any CPU - {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Any CPU.ActiveCfg = Release|Any CPU - {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection -EndGlobal diff --git a/Test/DafnyTests/DafnyTests-NetCore.csproj b/Test/DafnyTests/DafnyTests-NetCore.csproj index 0a3e9c297ee..76f7ab454b1 100644 --- a/Test/DafnyTests/DafnyTests-NetCore.csproj +++ b/Test/DafnyTests/DafnyTests-NetCore.csproj @@ -2,6 +2,7 @@ netcoreapp2.1 + DafnyTests @@ -11,6 +12,11 @@ + + + + + diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index c9aa27a8280..4436eaca771 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Reflection; using Xunit; using Xunit.Sdk; using YamlDotNet.RepresentationModel; @@ -12,11 +13,10 @@ namespace DafnyTests { public class DafnyTests { - private static string DAFNY_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()).Parent.Parent.Parent - .Parent.Parent.FullName; + private static string ROOT = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); - private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; + private static string DAFNY_EXE = Path.Combine(ROOT, "Dafny.exe"); + private static string TEST_ROOT = Path.Combine(ROOT, "Test") + Path.DirectorySeparatorChar; private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; @@ -37,9 +37,9 @@ public static string RunDafny(IEnumerable arguments) { using (Process dafnyProcess = new Process()) { dafnyProcess.StartInfo.FileName = "mono"; - dafnyProcess.StartInfo.ArgumentList.Add(DAFNY_EXE); + dafnyProcess.StartInfo.Arguments += DAFNY_EXE; foreach (var argument in dafnyArguments) { - dafnyProcess.StartInfo.ArgumentList.Add(argument); + dafnyProcess.StartInfo.Arguments += " " + argument; } dafnyProcess.StartInfo.UseShellExecute = false; @@ -71,10 +71,7 @@ public static string RunDafny(IEnumerable arguments) { private static string Exec(string file, params string[] arguments) { using (Process dafnyProcess = new Process()) { dafnyProcess.StartInfo.FileName = file; - foreach (var argument in arguments) { - dafnyProcess.StartInfo.ArgumentList.Add(argument); - } - + dafnyProcess.StartInfo.Arguments = String.Join(" ", arguments); dafnyProcess.StartInfo.UseShellExecute = false; dafnyProcess.StartInfo.RedirectStandardOutput = true; dafnyProcess.StartInfo.RedirectStandardError = true; @@ -82,10 +79,6 @@ private static string Exec(string file, params string[] arguments) { dafnyProcess.Start(); dafnyProcess.WaitForExit(); -// if (dafnyProcess.ExitCode != 0) { -// string error = dafnyProcess.StandardError.ReadToEnd(); -// throw new Exception(error); -// } return dafnyProcess.StandardOutput.ReadToEnd(); } } @@ -120,7 +113,7 @@ private static IEnumerable Expand(YamlNode node) { private static IEnumerable> ExpandValue(KeyValuePair pair) { - return Expand(pair.Value).Select(v => KeyValuePair.Create(pair.Key, v)); + return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); } private static YamlMappingNode FromPairs(IEnumerable> pairs) { diff --git a/Test/DafnyTests/DafnyTests.csproj b/Test/DafnyTests/DafnyTests.csproj index 85db0cb37ea..1ee231da0e7 100644 --- a/Test/DafnyTests/DafnyTests.csproj +++ b/Test/DafnyTests/DafnyTests.csproj @@ -35,23 +35,28 @@ 4 - - - + + + - + + + + Test\%(RecursiveDir)%(Filename)%(Extension) + + diff --git a/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs b/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs index 6ae3e119d2f..c7299727f86 100644 --- a/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs +++ b/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs @@ -3,5 +3,5 @@ using Xunit.Sdk; [AttributeUsage(AttributeTargets.Method)] -[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "DafnyTests-NetCore")] +[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "DafnyTests")] public class ParallelTheoryAttribute : TheoryAttribute { } \ No newline at end of file diff --git a/Test/DafnyTests/TestAttribute.dfy b/Test/testing/TestAttribute.dfy similarity index 100% rename from Test/DafnyTests/TestAttribute.dfy rename to Test/testing/TestAttribute.dfy diff --git a/Test/DafnyTests/TestAttribute.dfy.expect b/Test/testing/TestAttribute.dfy.expect similarity index 100% rename from Test/DafnyTests/TestAttribute.dfy.expect rename to Test/testing/TestAttribute.dfy.expect diff --git a/Test/exceptions/TestAttributeErrors.dfy b/Test/testing/TestAttributeErrors.dfy similarity index 100% rename from Test/exceptions/TestAttributeErrors.dfy rename to Test/testing/TestAttributeErrors.dfy diff --git a/Test/exceptions/TestAttributeErrors.dfy.expect b/Test/testing/TestAttributeErrors.dfy.expect similarity index 100% rename from Test/exceptions/TestAttributeErrors.dfy.expect rename to Test/testing/TestAttributeErrors.dfy.expect From 766ce711a38fbf9318e97689620670c59159f3f6 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 21 Jun 2020 12:11:54 -0700 Subject: [PATCH 027/192] Hack to locate Binaries directory --- Test/DafnyTests/DafnyTests-NetCore.csproj | 1 - Test/DafnyTests/DafnyTests.cs | 20 ++++++++++++++++---- Test/DafnyTests/DafnyTests.csproj | 1 - 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Test/DafnyTests/DafnyTests-NetCore.csproj b/Test/DafnyTests/DafnyTests-NetCore.csproj index 76f7ab454b1..65aa74f99e8 100644 --- a/Test/DafnyTests/DafnyTests-NetCore.csproj +++ b/Test/DafnyTests/DafnyTests-NetCore.csproj @@ -12,7 +12,6 @@ - diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 4436eaca771..f800dfea214 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -13,13 +13,25 @@ namespace DafnyTests { public class DafnyTests { - private static string ROOT = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - - private static string DAFNY_EXE = Path.Combine(ROOT, "Dafny.exe"); - private static string TEST_ROOT = Path.Combine(ROOT, "Test") + Path.DirectorySeparatorChar; + private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); + + private static string DAFNY_EXE = FindDafnyExe(); + private static string TEST_ROOT = Path.Combine(OUTPUT_ROOT.FullName, "Test") + Path.DirectorySeparatorChar; private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; + private static string FindDafnyExe() { + var path = OUTPUT_ROOT; + while (path.Parent != null) { + var exePath = Path.Combine(Path.Combine(path.FullName, "Binaries"), "Dafny.exe"); + if (File.Exists(exePath)) { + return exePath; + } + path = path.Parent; + } + throw new FileNotFoundException(); + } + public static string RunDafny(IEnumerable arguments) { List dafnyArguments = new List { // Expected output does not contain logo diff --git a/Test/DafnyTests/DafnyTests.csproj b/Test/DafnyTests/DafnyTests.csproj index 1ee231da0e7..6e21848f716 100644 --- a/Test/DafnyTests/DafnyTests.csproj +++ b/Test/DafnyTests/DafnyTests.csproj @@ -46,7 +46,6 @@ - From c9f3625470d0dda7d566efa0136f4a6894a04581 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 14:24:10 -0700 Subject: [PATCH 028/192] Test mostly running out of output directory --- .github/workflows/msbuild.yml | 9 ++------- Test/DafnyTests/DafnyTests-NetCore.csproj | 2 ++ Test/DafnyTests/DafnyTests.cs | 17 ++--------------- Test/DafnyTests/DafnyTests.csproj | 19 ++++++++++--------- 4 files changed, 16 insertions(+), 31 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index dd79cf7d02e..affae8c996e 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -49,12 +49,7 @@ jobs: run: nuget restore dafny/Source/Dafny.sln - name: Build Dafny run: msbuild dafny/Source/Dafny.sln - - uses: actions/setup-python@v1 - - name: Upgrade outdated pip - run: python -m pip install --upgrade pip - - name: Install lit - run: pip install lit OutputCheck pyyaml - uses: actions/setup-node@v1 - run: npm install bignumber.js - - name: Run lit tests - run: lit --time-tests -v --num-shards=5 --run-shard=${{ matrix.shard }} dafny/Test + - name: Run Dafny tests + run: msbuild =t:Test dafny/Test/DafnyTests/DafnyTests.csproj diff --git a/Test/DafnyTests/DafnyTests-NetCore.csproj b/Test/DafnyTests/DafnyTests-NetCore.csproj index 65aa74f99e8..0eafe984e91 100644 --- a/Test/DafnyTests/DafnyTests-NetCore.csproj +++ b/Test/DafnyTests/DafnyTests-NetCore.csproj @@ -12,10 +12,12 @@ + + diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index f800dfea214..980e5f35693 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -15,23 +15,11 @@ public class DafnyTests { private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); - private static string DAFNY_EXE = FindDafnyExe(); + private static string DAFNY_EXE = Path.Combine(OUTPUT_ROOT.FullName, "Dafny.exe"); private static string TEST_ROOT = Path.Combine(OUTPUT_ROOT.FullName, "Test") + Path.DirectorySeparatorChar; private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - private static string FindDafnyExe() { - var path = OUTPUT_ROOT; - while (path.Parent != null) { - var exePath = Path.Combine(Path.Combine(path.FullName, "Binaries"), "Dafny.exe"); - if (File.Exists(exePath)) { - return exePath; - } - path = path.Parent; - } - throw new FileNotFoundException(); - } - public static string RunDafny(IEnumerable arguments) { List dafnyArguments = new List { // Expected output does not contain logo @@ -123,8 +111,7 @@ private static IEnumerable Expand(YamlNode node) { } } - private static IEnumerable> - ExpandValue(KeyValuePair pair) { + private static IEnumerable> ExpandValue(KeyValuePair pair) { return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); } diff --git a/Test/DafnyTests/DafnyTests.csproj b/Test/DafnyTests/DafnyTests.csproj index 6e21848f716..edc169eb1f4 100644 --- a/Test/DafnyTests/DafnyTests.csproj +++ b/Test/DafnyTests/DafnyTests.csproj @@ -12,8 +12,6 @@ DafnyTests v4.8 512 - - ..\..\Binaries\ AnyCPU @@ -35,17 +33,11 @@ 4 - - - - - - - + @@ -55,6 +47,15 @@ Test\%(RecursiveDir)%(Filename)%(Extension) + + z3\%(RecursiveDir)%(Filename)%(Extension) + + + + + + + From 1f521e4eb44353b31f46598c72c2bc172923ff81 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 15:32:22 -0700 Subject: [PATCH 029/192] Just go with NetCore version for initial draft PR --- .github/workflows/msbuild.yml | 2 +- Test/DafnyTests/DafnyTests-NetCore.csproj | 12 ++++++------ Test/DafnyTests/DafnyTests.cs | 17 +++++++++-------- Test/DafnyTests/DafnyTests.csproj | 7 +++++-- Test/README.md | 13 +++++++++++++ 5 files changed, 34 insertions(+), 17 deletions(-) create mode 100644 Test/README.md diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index affae8c996e..29f029857b0 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -52,4 +52,4 @@ jobs: - uses: actions/setup-node@v1 - run: npm install bignumber.js - name: Run Dafny tests - run: msbuild =t:Test dafny/Test/DafnyTests/DafnyTests.csproj + run: dotnet test dafny/Test/DafnyTests/DafnyTests-NetCore.csproj --logger:"console;verbosity=normal" diff --git a/Test/DafnyTests/DafnyTests-NetCore.csproj b/Test/DafnyTests/DafnyTests-NetCore.csproj index 0eafe984e91..6878a6f1ac3 100644 --- a/Test/DafnyTests/DafnyTests-NetCore.csproj +++ b/Test/DafnyTests/DafnyTests-NetCore.csproj @@ -12,12 +12,12 @@ - - - - - - + + + + + + diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 980e5f35693..12f52488f54 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -13,13 +13,15 @@ namespace DafnyTests { public class DafnyTests { - private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); - - private static string DAFNY_EXE = Path.Combine(OUTPUT_ROOT.FullName, "Dafny.exe"); - private static string TEST_ROOT = Path.Combine(OUTPUT_ROOT.FullName, "Test") + Path.DirectorySeparatorChar; + private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); + private static string DAFNY_ROOT = OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; + + private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; + private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); + public static string RunDafny(IEnumerable arguments) { List dafnyArguments = new List { // Expected output does not contain logo @@ -139,8 +141,7 @@ from item in sequence public static IEnumerable AllTestFiles() { var filePaths = Directory.GetFiles(COMP_DIR, "*.dfy", SearchOption.AllDirectories) - .Select(path => GetRelativePath(TEST_ROOT, path)); -// var filePaths = Assembly.GetExecutingAssembly().GetManifestResourceNames(); + .Select(path => GetRelativePath(TEST_ROOT, path)); return filePaths.SelectMany(TestCasesForDafnyFile); } @@ -207,7 +208,7 @@ private static void AssertEqualWithDiff(string expected, string actual) { if (expected != actual) { // TODO-RS: Do better than shelling out to a linux utility. // Disappointingly, I couldn't find any easy solutions for an in-memory - + // unified diff calculation. string expectedPath = Path.GetTempFileName(); File.WriteAllText(expectedPath, expected); string actualPath = Path.GetTempFileName(); @@ -242,7 +243,7 @@ public void Test(string file, string args) { } } - var argumentsWithFile = new List {file}.Concat(arguments); + var argumentsWithFile = new List {fullInputPath}.Concat(arguments); var expectedOutput = File.ReadAllText(expectedOutputPath); string output; diff --git a/Test/DafnyTests/DafnyTests.csproj b/Test/DafnyTests/DafnyTests.csproj index edc169eb1f4..8076187d3f6 100644 --- a/Test/DafnyTests/DafnyTests.csproj +++ b/Test/DafnyTests/DafnyTests.csproj @@ -33,11 +33,14 @@ 4 + - + + true + @@ -71,6 +74,6 @@ --> - + \ No newline at end of file diff --git a/Test/README.md b/Test/README.md new file mode 100644 index 00000000000..a329ab9244f --- /dev/null +++ b/Test/README.md @@ -0,0 +1,13 @@ +# Tests + +Dafny test cases are based on single Dafny source files together with the expected output from the `dafny` CLI tool. + +The default behaviour is to assert that the source file verifies successfully and, for each currently-supported target language, can be compiled and run to produce the expected output. + +# TODO + +* More complete documentation about options (here or in the source code) +* Fix errors around missing types from System.dll when compiling to C#. +* Add `-NetCore.csproj` versions of projects as needed to support running tests using `dotnet`. +* Add support for regular expression matching against CLI output (needed to assert known limitations that cause errors with things like absolute paths names in them) +* Add support for sharding From 1e313c7e474406cf438e5acbc64c089f4079352b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 15:33:44 -0700 Subject: [PATCH 030/192] Remove reference not available outside of .NET Core --- Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs b/Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs index 8df40bc6a71..76decdf6562 100644 --- a/Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs +++ b/Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs @@ -21,7 +21,6 @@ public TestCaseWithCollection(IXunitTestCase testCase, ITestCollection collectio this.testMethod = new TestMethod(testClassWithCollection, testCase.TestMethod.Method); } - [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("Called by the de-serializer", error: true)] public TestCaseWithCollection() { } From e3e8cea32c2de0423305b928204f6ae471e2a1a2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 15:50:31 -0700 Subject: [PATCH 031/192] Fill out README --- Test/README.md | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Test/README.md b/Test/README.md index a329ab9244f..8e6a358bedc 100644 --- a/Test/README.md +++ b/Test/README.md @@ -1,13 +1,21 @@ # Tests -Dafny test cases are based on single Dafny source files together with the expected output from the `dafny` CLI tool. +Dafny test cases are based on single Dafny source files together with the expected output from the `dafny` CLI tool. If the source file is named `Foo.dfy`, the expected output will contained in the file named `Foo.dfy.expect`. -The default behaviour is to assert that the source file verifies successfully and, for each currently-supported target language, can be compiled and run to produce the expected output. +The default behaviour is to assert that the source file verifies successfully and, for each currently-supported target language, can be compiled and run to produce the expected output. When the behaviour is not 100% consistent between different target languages, expected discrepancies can be recorded in additional files with the name `Foo.dfy..expect`. For example: [Test/comp/NativeNumbers.dfy.js.expect](Test/comp/NativeNumbers.dfy.js.expect). Such exceptions are automatically flagged as "known issues" and classified as a "skipped/ignored" test. + +Test cases are configured and run using xUnit's support for parameterized tests, with extensions for running test cases in parallel. The sets of options passed to `dafny` can be configured using YAML embedded in the first multi-line comment in the source file. Lists of values are interpreted as multiple parameterizations. For example: [Test/dafny4/git-issue250.dfy](Test/dafny4/git-issue250.dfy). + +For details and more configuration options, see [the DafnyTests.cs source](Test/DafnyTests/DafnyTests.cs). # TODO -* More complete documentation about options (here or in the source code) -* Fix errors around missing types from System.dll when compiling to C#. -* Add `-NetCore.csproj` versions of projects as needed to support running tests using `dotnet`. +* More complete documentation about options (in this file or in the source code) +* Depend on only the project's output directory instead of the Binaries/Test directories + * This is mostly working except for errors around missing types from System.dll when compiling to C# +* Complete support for both the .NET Framework solution as well as the .NET Core one * Add support for regular expression matching against CLI output (needed to assert known limitations that cause errors with things like absolute paths names in them) -* Add support for sharding +* Add support for sharding ala lit's `--run-shard / --num-shards` options +* Finish converting the rest of the test cases + * Will write a small script to do this automatically for all recognized combinations of lit `// RUN` commands +* Extract most of the xUnit extensions as a separate package, since most of it is generically useful for any other data-driven xUnit tests. From e9838912005a7bb646ae9f32d01a44d134eda9bb Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 15:56:25 -0700 Subject: [PATCH 032/192] =?UTF-8?q?TODO=20to=20use=20David=E2=80=99s=20tri?= =?UTF-8?q?ck=20of=20verifying=20separately?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Test/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Test/README.md b/Test/README.md index 8e6a358bedc..4e2ea27b3c9 100644 --- a/Test/README.md +++ b/Test/README.md @@ -15,6 +15,7 @@ For details and more configuration options, see [the DafnyTests.cs source](Test/ * This is mostly working except for errors around missing types from System.dll when compiling to C# * Complete support for both the .NET Framework solution as well as the .NET Core one * Add support for regular expression matching against CLI output (needed to assert known limitations that cause errors with things like absolute paths names in them) +* By default, verify source files separately and skip verification when compiling to each target language * Add support for sharding ala lit's `--run-shard / --num-shards` options * Finish converting the rest of the test cases * Will write a small script to do this automatically for all recognized combinations of lit `// RUN` commands From 656775d3440a0a56cf23b98b1b0398e21be64b02 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 15:58:01 -0700 Subject: [PATCH 033/192] Temporarily remove sharding --- .github/workflows/msbuild.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index 29f029857b0..cb1c6931d13 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -15,7 +15,6 @@ jobs: matrix: # Windows build is not yet working due to issues with the checked-in z3.exe os: [ubuntu-latest] - shard: [1, 2, 3, 4, 5] fail-fast: false steps: - name: Setup Nuget From a70ff926e5434790f7c893a83b9d0f63aaa0856f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 16:02:08 -0700 Subject: [PATCH 034/192] Solution file moved --- .github/workflows/msbuild.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index cb1c6931d13..da785e36855 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -45,9 +45,9 @@ jobs: unzip z3*.zip && rm *.zip cp -r z3* dafny/Binaries/z3 - name: Nuget Restore Dafny - run: nuget restore dafny/Source/Dafny.sln + run: nuget restore dafny/Dafny.sln - name: Build Dafny - run: msbuild dafny/Source/Dafny.sln + run: msbuild dafny/Dafny.sln - uses: actions/setup-node@v1 - run: npm install bignumber.js - name: Run Dafny tests From 0157e2b90643dc9f7934492e32cd0c82c5a2cc72 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 16:55:27 -0700 Subject: [PATCH 035/192] Add TODO about traits --- Test/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Test/README.md b/Test/README.md index 4e2ea27b3c9..b2a994b37d2 100644 --- a/Test/README.md +++ b/Test/README.md @@ -17,6 +17,7 @@ For details and more configuration options, see [the DafnyTests.cs source](Test/ * Add support for regular expression matching against CLI output (needed to assert known limitations that cause errors with things like absolute paths names in them) * By default, verify source files separately and skip verification when compiling to each target language * Add support for sharding ala lit's `--run-shard / --num-shards` options +* Expose test case options as traits so that they can be filtered on (e.g. `dotnet test --filter compileTarget=java`) * Finish converting the rest of the test cases * Will write a small script to do this automatically for all recognized combinations of lit `// RUN` commands * Extract most of the xUnit extensions as a separate package, since most of it is generically useful for any other data-driven xUnit tests. From 0a7ab313ebccaddd782258e5771b8bd9413771da Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 17:03:13 -0700 Subject: [PATCH 036/192] Correct packages location --- Source/DafnyRuntime/DafnyRuntime.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/DafnyRuntime/DafnyRuntime.csproj b/Source/DafnyRuntime/DafnyRuntime.csproj index 9949dd4d636..bd726f1e113 100755 --- a/Source/DafnyRuntime/DafnyRuntime.csproj +++ b/Source/DafnyRuntime/DafnyRuntime.csproj @@ -126,11 +126,11 @@ - ..\packages\System.Collections.Immutable.1.7.0\lib\portable-net45+win8+wp8+wpa81\System.Collections.Immutable.dll + ..\..\packages\System.Collections.Immutable.1.7.0\lib\portable-net45+win8+wp8+wpa81\System.Collections.Immutable.dll True - ..\packages\System.Runtime.4.3.1\lib\net462\System.Runtime.dll + ..\..\packages\System.Runtime.4.3.1\lib\net462\System.Runtime.dll True From 6d9788603b55361544e95a687b951e5ee6d9ff50 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 17:45:15 -0700 Subject: [PATCH 037/192] Try msbuild version --- .github/workflows/msbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index da785e36855..635288010a8 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -51,4 +51,4 @@ jobs: - uses: actions/setup-node@v1 - run: npm install bignumber.js - name: Run Dafny tests - run: dotnet test dafny/Test/DafnyTests/DafnyTests-NetCore.csproj --logger:"console;verbosity=normal" + run: msbuild -t:Test dafny/Test/DafnyTests/DafnyTests.csproj From ac824df854224b4b090bc60a3605c42a22e8c331 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 22 Jun 2020 17:49:20 -0700 Subject: [PATCH 038/192] One fewer .Parent for msbuild --- Test/DafnyTests/DafnyTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index 12f52488f54..f85f68e49f6 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -14,7 +14,7 @@ namespace DafnyTests { public class DafnyTests { private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); - private static string DAFNY_ROOT = OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; + private static string DAFNY_ROOT = OUTPUT_ROOT.Parent.Parent.Parent.Parent.FullName; private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; From c462c007086071851a9c1c2164bb32c127a32bd7 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 25 Jun 2020 08:08:01 -0700 Subject: [PATCH 039/192] Another TODO --- Test/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Test/README.md b/Test/README.md index b2a994b37d2..4a171f2e8f1 100644 --- a/Test/README.md +++ b/Test/README.md @@ -14,6 +14,7 @@ For details and more configuration options, see [the DafnyTests.cs source](Test/ * Depend on only the project's output directory instead of the Binaries/Test directories * This is mostly working except for errors around missing types from System.dll when compiling to C# * Complete support for both the .NET Framework solution as well as the .NET Core one +* Figure out why test cases aren't executing in parallel when run using `msbuild` (they are when run in Rider) * Add support for regular expression matching against CLI output (needed to assert known limitations that cause errors with things like absolute paths names in them) * By default, verify source files separately and skip verification when compiling to each target language * Add support for sharding ala lit's `--run-shard / --num-shards` options From 926471cd21a28cefe768f4e709b9fa7b24acdbb6 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 25 Jun 2020 22:46:20 -0700 Subject: [PATCH 040/192] Fix parallel execution from msbuild, add sharding --- .github/workflows/msbuild.yml | 3 ++- .../CollectionPerTestCaseTheoryDiscoverer.cs | 20 +++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index 635288010a8..f5b9f4541a5 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -15,6 +15,7 @@ jobs: matrix: # Windows build is not yet working due to issues with the checked-in z3.exe os: [ubuntu-latest] + shard: [1, 2, 3, 4, 5] fail-fast: false steps: - name: Setup Nuget @@ -51,4 +52,4 @@ jobs: - uses: actions/setup-node@v1 - run: npm install bignumber.js - name: Run Dafny tests - run: msbuild -t:Test dafny/Test/DafnyTests/DafnyTests.csproj + run: XUNIT_SHARD=${{ matrix.shard }} XUNIT_NUM_SHARDS=5 msbuild -t:Test dafny/Test/DafnyTests/DafnyTests.csproj diff --git a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index 4a2c37b1654..f8e40300c59 100644 --- a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -1,5 +1,7 @@ +using System; using System.Collections.Generic; using System.Linq; +using Xunit; using Xunit.Abstractions; using Xunit.Sdk; @@ -19,8 +21,22 @@ private TestCollection testCollectionForTestCase(IXunitTestCase testCase) { } public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { - return theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute) - .Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); + // This discoverer requires pre-enumeration in order to assign a collection to each test case. + discoveryOptions.SetValue("xunit.discovery.PreEnumerateTheories", true); + + IEnumerable testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); + var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); + var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_NUM_SHARDS"); + if (shardEnvVar != null && numShardsEnvVar != null) { + var testCaseList = testCases.ToList(); + var shard = Int32.Parse(shardEnvVar); + var numShards = Int32.Parse(numShardsEnvVar); + var shardStart = (shard - 1) * testCaseList.Count / numShards; + var shardEnd = shard * testCaseList.Count / numShards; + testCases = testCaseList.GetRange(shardStart, shardEnd); + } + + return testCases.Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); } } } \ No newline at end of file From daf71e9512ba2174fdd6305b285ef99317258e30 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 27 Jun 2020 09:33:04 -0700 Subject: [PATCH 041/192] Fix sharding --- .github/workflows/msbuild.yml | 2 +- .../XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index f5b9f4541a5..a0ea7370d47 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -52,4 +52,4 @@ jobs: - uses: actions/setup-node@v1 - run: npm install bignumber.js - name: Run Dafny tests - run: XUNIT_SHARD=${{ matrix.shard }} XUNIT_NUM_SHARDS=5 msbuild -t:Test dafny/Test/DafnyTests/DafnyTests.csproj + run: XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 msbuild -t:Test dafny/Test/DafnyTests/DafnyTests.csproj diff --git a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index f8e40300c59..5b5b0d2daa3 100644 --- a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -26,14 +26,15 @@ public IEnumerable Discover(ITestFrameworkDiscoveryOptions disco IEnumerable testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); - var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_NUM_SHARDS"); + var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD_COUNT"); + // TODO-RS: More careful error checking if (shardEnvVar != null && numShardsEnvVar != null) { var testCaseList = testCases.ToList(); var shard = Int32.Parse(shardEnvVar); var numShards = Int32.Parse(numShardsEnvVar); var shardStart = (shard - 1) * testCaseList.Count / numShards; var shardEnd = shard * testCaseList.Count / numShards; - testCases = testCaseList.GetRange(shardStart, shardEnd); + testCases = testCaseList.GetRange(shardStart, shardEnd - shardStart); } return testCases.Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); From 9a5880e2ab44fd07a7e962eab4c943d44b994b8e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 2 Jul 2020 20:32:03 -0700 Subject: [PATCH 042/192] Use DiffPlex for unified diff --- Test/DafnyTests/DafnyTests.cs | 51 ++++++++++++++++--------------- Test/DafnyTests/DafnyTests.csproj | 1 + 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/Test/DafnyTests/DafnyTests.cs b/Test/DafnyTests/DafnyTests.cs index f85f68e49f6..6c58f9b9e6b 100644 --- a/Test/DafnyTests/DafnyTests.cs +++ b/Test/DafnyTests/DafnyTests.cs @@ -4,6 +4,10 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Text; +using DiffPlex; +using DiffPlex.DiffBuilder; +using DiffPlex.DiffBuilder.Model; using Xunit; using Xunit.Sdk; using YamlDotNet.RepresentationModel; @@ -70,21 +74,6 @@ public static string RunDafny(IEnumerable arguments) { } } - private static string Exec(string file, params string[] arguments) { - using (Process dafnyProcess = new Process()) { - dafnyProcess.StartInfo.FileName = file; - dafnyProcess.StartInfo.Arguments = String.Join(" ", arguments); - dafnyProcess.StartInfo.UseShellExecute = false; - dafnyProcess.StartInfo.RedirectStandardOutput = true; - dafnyProcess.StartInfo.RedirectStandardError = true; - dafnyProcess.StartInfo.CreateNoWindow = true; - - dafnyProcess.Start(); - dafnyProcess.WaitForExit(); - return dafnyProcess.StandardOutput.ReadToEnd(); - } - } - private static string GetTestCaseConfigString(string filePath) { // TODO-RS: Figure out how to do this cleanly on a TextReader instead, // and to handle things like nested comments. @@ -206,16 +195,28 @@ private static string GetRelativePath(string relativeTo, string path) { private static void AssertEqualWithDiff(string expected, string actual) { if (expected != actual) { - // TODO-RS: Do better than shelling out to a linux utility. - // Disappointingly, I couldn't find any easy solutions for an in-memory - // unified diff calculation. - string expectedPath = Path.GetTempFileName(); - File.WriteAllText(expectedPath, expected); - string actualPath = Path.GetTempFileName(); - File.WriteAllText(actualPath, actual); - string diff = Exec("diff", "--unified=3", expectedPath, actualPath); - string message = "AssertEqualWithDiff() Failure\n" + diff; - throw new AssertActualExpectedException(expected, actual, message); + var diffBuilder = new InlineDiffBuilder(new Differ()); + var diff = diffBuilder.BuildDiffModel(expected, actual); + + var message = new StringBuilder(); + message.AppendLine("AssertEqualWithDiff() Failure"); + message.AppendLine("Diff (changing expected into actual):"); + foreach (var line in diff.Lines) { + switch (line.Type) { + case ChangeType.Inserted: + message.Append("+"); + break; + case ChangeType.Deleted: + message.Append("-"); + break; + default: + message.Append(" "); + break; + } + message.AppendLine(line.Text); + } + + throw new AssertActualExpectedException(expected, actual, message.ToString()); } } diff --git a/Test/DafnyTests/DafnyTests.csproj b/Test/DafnyTests/DafnyTests.csproj index 8076187d3f6..77c953b9400 100644 --- a/Test/DafnyTests/DafnyTests.csproj +++ b/Test/DafnyTests/DafnyTests.csproj @@ -34,6 +34,7 @@ + From e1a39dd9e91b5f5ab62d95cdb45171b5bf8926b1 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 3 Jul 2020 14:18:05 -0700 Subject: [PATCH 043/192] Move solution files back under Source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Boogie does this as well, and it simplifies things if Test/ doesn’t contain any C# source code --- Dafny-NetCore.sln => Source/Dafny-NetCore.sln | 0 Dafny.sln => Source/Dafny.sln | 0 {Test => Source}/DafnyTests/DafnyTests-NetCore.csproj | 0 {Test => Source}/DafnyTests/DafnyTests.cs | 0 {Test => Source}/DafnyTests/DafnyTests.csproj | 0 {Test => Source}/DafnyTests/TemporaryDirectory.cs | 0 .../XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs | 0 .../DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs | 0 .../DafnyTests/XUnitExtensions/TestCaseWithCollection.cs | 0 {Test => Source}/DafnyTests/YamlTests/multiple.expect.yml | 0 {Test => Source}/DafnyTests/YamlTests/multiple.yml | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename Dafny-NetCore.sln => Source/Dafny-NetCore.sln (100%) rename Dafny.sln => Source/Dafny.sln (100%) rename {Test => Source}/DafnyTests/DafnyTests-NetCore.csproj (100%) rename {Test => Source}/DafnyTests/DafnyTests.cs (100%) rename {Test => Source}/DafnyTests/DafnyTests.csproj (100%) rename {Test => Source}/DafnyTests/TemporaryDirectory.cs (100%) rename {Test => Source}/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs (100%) rename {Test => Source}/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs (100%) rename {Test => Source}/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs (100%) rename {Test => Source}/DafnyTests/YamlTests/multiple.expect.yml (100%) rename {Test => Source}/DafnyTests/YamlTests/multiple.yml (100%) diff --git a/Dafny-NetCore.sln b/Source/Dafny-NetCore.sln similarity index 100% rename from Dafny-NetCore.sln rename to Source/Dafny-NetCore.sln diff --git a/Dafny.sln b/Source/Dafny.sln similarity index 100% rename from Dafny.sln rename to Source/Dafny.sln diff --git a/Test/DafnyTests/DafnyTests-NetCore.csproj b/Source/DafnyTests/DafnyTests-NetCore.csproj similarity index 100% rename from Test/DafnyTests/DafnyTests-NetCore.csproj rename to Source/DafnyTests/DafnyTests-NetCore.csproj diff --git a/Test/DafnyTests/DafnyTests.cs b/Source/DafnyTests/DafnyTests.cs similarity index 100% rename from Test/DafnyTests/DafnyTests.cs rename to Source/DafnyTests/DafnyTests.cs diff --git a/Test/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj similarity index 100% rename from Test/DafnyTests/DafnyTests.csproj rename to Source/DafnyTests/DafnyTests.csproj diff --git a/Test/DafnyTests/TemporaryDirectory.cs b/Source/DafnyTests/TemporaryDirectory.cs similarity index 100% rename from Test/DafnyTests/TemporaryDirectory.cs rename to Source/DafnyTests/TemporaryDirectory.cs diff --git a/Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs similarity index 100% rename from Test/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs rename to Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs diff --git a/Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs b/Source/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs similarity index 100% rename from Test/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs rename to Source/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs diff --git a/Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs b/Source/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs similarity index 100% rename from Test/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs rename to Source/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs diff --git a/Test/DafnyTests/YamlTests/multiple.expect.yml b/Source/DafnyTests/YamlTests/multiple.expect.yml similarity index 100% rename from Test/DafnyTests/YamlTests/multiple.expect.yml rename to Source/DafnyTests/YamlTests/multiple.expect.yml diff --git a/Test/DafnyTests/YamlTests/multiple.yml b/Source/DafnyTests/YamlTests/multiple.yml similarity index 100% rename from Test/DafnyTests/YamlTests/multiple.yml rename to Source/DafnyTests/YamlTests/multiple.yml From 0fa9b7a8a7f8524a368e1ea4f94dab9dcb62bb3e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 3 Jul 2020 14:18:41 -0700 Subject: [PATCH 044/192] Missing path changes --- Source/Dafny-NetCore.sln | 4 ++-- Source/Dafny.sln | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Source/Dafny-NetCore.sln b/Source/Dafny-NetCore.sln index 20953870148..3865dd0de98 100644 --- a/Source/Dafny-NetCore.sln +++ b/Source/Dafny-NetCore.sln @@ -1,8 +1,8 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyRuntime-NetCore", "Source\DafnyRuntime\DafnyRuntime-NetCore.csproj", "{F6DFCDF8-91AA-4222-A323-EBA22827793D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyRuntime-NetCore", "DafnyRuntime\DafnyRuntime-NetCore.csproj", "{F6DFCDF8-91AA-4222-A323-EBA22827793D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests-NetCore", "Test\DafnyTests\DafnyTests-NetCore.csproj", "{9B0FD2BD-78C6-4D85-BCE3-557E85BC33C1}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests-NetCore", "DafnyTests\DafnyTests-NetCore.csproj", "{9B0FD2BD-78C6-4D85-BCE3-557E85BC33C1}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/Source/Dafny.sln b/Source/Dafny.sln index 5fdb74d1073..94cbce153d5 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -1,18 +1,18 @@  Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 2012 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyDriver", "Source\DafnyDriver\DafnyDriver.csproj", "{63400D1F-05B2-453E-9592-1EAB74B2C9CC}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyDriver", "DafnyDriver\DafnyDriver.csproj", "{63400D1F-05B2-453E-9592-1EAB74B2C9CC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyPipeline", "Source\Dafny\DafnyPipeline.csproj", "{FE44674A-1633-4917-99F4-57635E6FA740}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyPipeline", "Dafny\DafnyPipeline.csproj", "{FE44674A-1633-4917-99F4-57635E6FA740}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyServer", "Source\DafnyServer\DafnyServer.csproj", "{AC9B21AE-EBC1-4A27-AD11-ED031FC7B4A2}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyServer", "DafnyServer\DafnyServer.csproj", "{AC9B21AE-EBC1-4A27-AD11-ED031FC7B4A2}" ProjectSection(ProjectDependencies) = postProject {FE44674A-1633-4917-99F4-57635E6FA740} = {FE44674A-1633-4917-99F4-57635E6FA740} EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyRuntime", "Source\DafnyRuntime\DafnyRuntime.csproj", "{3E2944E9-3E1D-4E03-A881-FB04A7A6ED67}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyRuntime", "DafnyRuntime\DafnyRuntime.csproj", "{3E2944E9-3E1D-4E03-A881-FB04A7A6ED67}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests", "Test\DafnyTests\DafnyTests.csproj", "{45AE3DDD-D0EA-4A27-943B-AC66A581BE03}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests", "DafnyTests\DafnyTests.csproj", "{45AE3DDD-D0EA-4A27-943B-AC66A581BE03}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From c6622b32ce6220d8b1e915fa80a8cf18aeabbe8f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 3 Jul 2020 18:04:50 -0700 Subject: [PATCH 045/192] More code reorganization, start on conversion script --- Source/Dafny.sln | 20 ++++ Source/DafnyTests/DafnyTests.cs | 91 +------------------ Source/DafnyTests/DafnyTests.csproj | 7 +- Source/DafnyTests/EnumerableUtils.cs | 19 ++++ .../XUnitExtensions/AssertWithDiff.cs | 35 +++++++ Source/DafnyTests/YamlUtils.cs | 54 +++++++++++ .../LitTestConverter/LitTestConverter.csproj | 52 +++++++++++ Source/LitTestConverter/LitTestConvertor.cs | 89 ++++++++++++++++++ .../Properties/AssemblyInfo.cs | 35 +++++++ 9 files changed, 313 insertions(+), 89 deletions(-) create mode 100644 Source/DafnyTests/EnumerableUtils.cs create mode 100644 Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs create mode 100644 Source/DafnyTests/YamlUtils.cs create mode 100644 Source/LitTestConverter/LitTestConverter.csproj create mode 100644 Source/LitTestConverter/LitTestConvertor.cs create mode 100644 Source/LitTestConverter/Properties/AssemblyInfo.cs diff --git a/Source/Dafny.sln b/Source/Dafny.sln index 94cbce153d5..2cf73f6e2c7 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -14,6 +14,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyRuntime", "DafnyRuntim EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests", "DafnyTests\DafnyTests.csproj", "{45AE3DDD-D0EA-4A27-943B-AC66A581BE03}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LitTestConverter", "LitTestConverter\LitTestConverter.csproj", "{2EA7D80A-B179-4352-BDF2-8A1599F8685A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Checked|.NET = Checked|.NET @@ -109,6 +111,24 @@ Global {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Any CPU.Build.0 = Release|Any CPU {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU {45AE3DDD-D0EA-4A27-943B-AC66A581BE03}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|.NET.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|.NET.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Any CPU.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Mixed Platforms.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Mixed Platforms.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|.NET.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|.NET.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|.NET.ActiveCfg = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|.NET.Build.0 = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Any CPU.Build.0 = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Mixed Platforms.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Source/DafnyTests/DafnyTests.cs b/Source/DafnyTests/DafnyTests.cs index 6c58f9b9e6b..665ba696a86 100644 --- a/Source/DafnyTests/DafnyTests.cs +++ b/Source/DafnyTests/DafnyTests.cs @@ -10,6 +10,7 @@ using DiffPlex.DiffBuilder.Model; using Xunit; using Xunit.Sdk; +using XUnitExtensions; using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; @@ -91,43 +92,6 @@ private static string GetTestCaseConfigString(string filePath) { return null; } - - private static IEnumerable Expand(YamlNode node) { - if (node is YamlSequenceNode seqNode) { - return seqNode.SelectMany(child => Expand(child)); - } else if (node is YamlMappingNode mappingNode) { - return CartesianProduct(mappingNode.Select(ExpandValue)).Select(FromPairs); - } else { - return new[] {node}; - } - } - - private static IEnumerable> ExpandValue(KeyValuePair pair) { - return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); - } - - private static YamlMappingNode FromPairs(IEnumerable> pairs) { - var result = new YamlMappingNode(); - foreach (var pair in pairs) { - result.Add(pair.Key, pair.Value); - } - - return result; - } - - /** - * Source: https://docs.microsoft.com/en-us/archive/blogs/ericlippert/computing-a-cartesian-product-with-linq - */ - private static IEnumerable> CartesianProduct(IEnumerable> sequences) { - IEnumerable> emptyProduct = new[] {Enumerable.Empty()}; - return sequences.Aggregate( - emptyProduct, - (accumulator, sequence) => - from accseq in accumulator - from item in sequence - select accseq.Concat(new[] {item})); - } - public static IEnumerable AllTestFiles() { var filePaths = Directory.GetFiles(COMP_DIR, "*.dfy", SearchOption.AllDirectories) .Select(path => GetRelativePath(TEST_ROOT, path)); @@ -142,7 +106,7 @@ private static IEnumerable TestCasesForDafnyFile(string filePath) { var yamlStream = new YamlStream(); yamlStream.Load(new StringReader(configString)); var config = yamlStream.Documents[0].RootNode; - configs = Expand(config); + configs = YamlUtils.Expand(config); } else { configs = new[] {new YamlMappingNode()}; } @@ -166,7 +130,7 @@ private static IEnumerable ResolveCompile(string filePath, Yaml } if (mapping["compile"].Equals(new YamlScalarNode("3")) && !mapping.Children.ContainsKey("compileTarget")) { - var languages = new string[] {"cs", "java", "go", "js"}; + var languages = new[] {"cs", "java", "go", "js"}; foreach (var language in languages) { var withLanguage = new YamlMappingNode(mapping.Children); withLanguage.Add("compileTarget", language); @@ -193,33 +157,6 @@ private static string GetRelativePath(string relativeTo, string path) { return fullPath.Substring(fullRelativeTo.Length); } - private static void AssertEqualWithDiff(string expected, string actual) { - if (expected != actual) { - var diffBuilder = new InlineDiffBuilder(new Differ()); - var diff = diffBuilder.BuildDiffModel(expected, actual); - - var message = new StringBuilder(); - message.AppendLine("AssertEqualWithDiff() Failure"); - message.AppendLine("Diff (changing expected into actual):"); - foreach (var line in diff.Lines) { - switch (line.Type) { - case ChangeType.Inserted: - message.Append("+"); - break; - case ChangeType.Deleted: - message.Append("-"); - break; - default: - message.Append(" "); - break; - } - message.AppendLine(line.Text); - } - - throw new AssertActualExpectedException(expected, actual, message.ToString()); - } - } - [ParallelTheory] [MemberData(nameof(AllTestFiles))] public void Test(string file, string args) { @@ -263,28 +200,8 @@ public void Test(string file, string args) { } } - AssertEqualWithDiff(expectedOutput, output); + AssertWithDiff.Equal(expectedOutput, output); Skip.If(specialCase, "Confirmed known exception for arguments: " + args); } - - [Theory] - [InlineData("multiple.yml", "multiple.expect.yml")] - public void ExpandTest(string inputPath, string expectedOutputPath) { - string fullInputPath = Path.Combine(TEST_ROOT, "DafnyTests/YamlTests/" + inputPath); - string fullExpectedOutputPath = Path.Combine(TEST_ROOT, "DafnyTests/YamlTests/" + expectedOutputPath); - - using (var reader = File.OpenText(fullInputPath)) { - var yamlStream = new YamlStream(); - yamlStream.Load(reader); - var root = yamlStream.Documents[0].RootNode; - var expanded = Expand(root); - - var outputWriter = new StringWriter(); - var serializer = new SerializerBuilder().Build(); - serializer.Serialize(outputWriter, expanded); - string expectedOutput = File.ReadAllText(fullExpectedOutputPath); - Assert.Equal(expectedOutput, outputWriter.ToString()); - } - } } } \ No newline at end of file diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index 77c953b9400..1993f79a8fd 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -39,7 +39,7 @@ - + true @@ -48,9 +48,12 @@ - + Test\%(RecursiveDir)%(Filename)%(Extension) + + YamlTests\%(RecursiveDir)%(Filename)%(Extension) + z3\%(RecursiveDir)%(Filename)%(Extension) diff --git a/Source/DafnyTests/EnumerableUtils.cs b/Source/DafnyTests/EnumerableUtils.cs new file mode 100644 index 00000000000..e0fe2d4909b --- /dev/null +++ b/Source/DafnyTests/EnumerableUtils.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; +using System.Linq; + +namespace DafnyTests { + public static class EnumerableUtils { + /** + * Source: https://docs.microsoft.com/en-us/archive/blogs/ericlippert/computing-a-cartesian-product-with-linq + */ + public static IEnumerable> CartesianProduct(this IEnumerable> sequences) { + IEnumerable> emptyProduct = new[] {Enumerable.Empty()}; + return sequences.Aggregate( + emptyProduct, + (accumulator, sequence) => + from accseq in accumulator + from item in sequence + select accseq.Concat(new[] {item})); + } + } +} \ No newline at end of file diff --git a/Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs b/Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs new file mode 100644 index 00000000000..9291263eae3 --- /dev/null +++ b/Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs @@ -0,0 +1,35 @@ +using System.Text; +using DiffPlex; +using DiffPlex.DiffBuilder; +using DiffPlex.DiffBuilder.Model; +using Xunit.Sdk; + +namespace XUnitExtensions { + public class AssertWithDiff { + public static void Equal(string expected, string actual) { + if (expected != actual) { + var diff = InlineDiffBuilder.Instance.BuildDiffModel(expected, actual); + + var message = new StringBuilder(); + message.AppendLine("AssertEqualWithDiff() Failure"); + message.AppendLine("Diff (changing expected into actual):"); + foreach (var line in diff.Lines) { + switch (line.Type) { + case ChangeType.Inserted: + message.Append("+"); + break; + case ChangeType.Deleted: + message.Append("-"); + break; + default: + message.Append(" "); + break; + } + message.AppendLine(line.Text); + } + + throw new AssertActualExpectedException(expected, actual, message.ToString()); + } + } + } +} \ No newline at end of file diff --git a/Source/DafnyTests/YamlUtils.cs b/Source/DafnyTests/YamlUtils.cs new file mode 100644 index 00000000000..5d54eb8e331 --- /dev/null +++ b/Source/DafnyTests/YamlUtils.cs @@ -0,0 +1,54 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Xunit; +using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; + +namespace DafnyTests { + public class YamlUtils { + + public static IEnumerable Expand(YamlNode node) { + if (node is YamlSequenceNode seqNode) { + return seqNode.SelectMany(child => Expand(child)); + } else if (node is YamlMappingNode mappingNode) { + return mappingNode.Select(ExpandValue).CartesianProduct().Select(FromPairs); + } else { + return new[] {node}; + } + } + + private static IEnumerable> ExpandValue(KeyValuePair pair) { + return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); + } + + private static YamlMappingNode FromPairs(IEnumerable> pairs) { + var result = new YamlMappingNode(); + foreach (var pair in pairs) { + result.Add(pair.Key, pair.Value); + } + + return result; + } + + [Theory] + [InlineData("multiple.yml", "multiple.expect.yml")] + public void ExpandTest(string inputPath, string expectedOutputPath) { + string fullInputPath = "./YamlTests/" + inputPath; + string fullExpectedOutputPath = "./YamlTests/" + expectedOutputPath; + + using (var reader = File.OpenText(fullInputPath)) { + var yamlStream = new YamlStream(); + yamlStream.Load(reader); + var root = yamlStream.Documents[0].RootNode; + var expanded = Expand(root); + + var outputWriter = new StringWriter(); + var serializer = new SerializerBuilder().Build(); + serializer.Serialize(outputWriter, expanded); + string expectedOutput = File.ReadAllText(fullExpectedOutputPath); + Assert.Equal(expectedOutput, outputWriter.ToString()); + } + } + } +} \ No newline at end of file diff --git a/Source/LitTestConverter/LitTestConverter.csproj b/Source/LitTestConverter/LitTestConverter.csproj new file mode 100644 index 00000000000..6d40400dfc1 --- /dev/null +++ b/Source/LitTestConverter/LitTestConverter.csproj @@ -0,0 +1,52 @@ + + + + + Debug + AnyCPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A} + Exe + Properties + LitTestConverter + LitTestConverter + v4.8 + 512 + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs new file mode 100644 index 00000000000..f23a2457057 --- /dev/null +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace DafnyTests { + public static class LitTestConvertor { + + private const string LIT_COMMAND_PREFIX = "// RUN: "; + private const string LIT_DIFF = "%diff"; + private const string LIT_DAFNY = "%dafny"; + private const string DAFNY_COMPILE = "/compile:"; + private const string DAFNY_COMPILE_TARGET = "/compileTarget:"; + private const string DAFNY_AUTO_TRIGGERS = "/autoTriggers:"; + private const string DAFNY_VERIFY_ALL_MODULES = "/verifyAllModules"; + private const string DAFNY_ALLOCATED = "/allocated:"; + + private static readonly string[] IGNORED_DAFNY_COMMAND_ARGUMENTS = { + "/print:\"%t.print\"", + "/dprint:\"%t.dprint\"", + "/rprint:\"%t.rprint\"", + "/rprint:\"%t.dprint\"", + "\"%s\"", ">", ">>", "\"%t\"" + }; + + public static void ConvertLitTest(string filePath) { + var compileLevel = 1; + var printToolTips = false; + var autoTriggers = 1; + var verifyAllModules = false; + var allocated = 3; + var reader = new StreamReader(filePath); + + string line; + while((line = reader.ReadLine()) != null) { + if (line.StartsWith(LIT_COMMAND_PREFIX)) { + var parts = line.Substring(LIT_COMMAND_PREFIX.Length) + .Split((char[])null, StringSplitOptions.RemoveEmptyEntries); + switch (parts[0]) { + case LIT_DIFF: + // Ignore: assume that if this is anything other than the standard + // %diff "%s.expect" "%t" line, then at least one other line will be non-standard + // as well (to produce the non-standard arguments) + break; + case LIT_DAFNY: + // Check the arguments for anything non-standard + foreach (var arg in parts.Skip(1)) { + if (IGNORED_DAFNY_COMMAND_ARGUMENTS.Contains(arg)) { + // Ignore + } else if (arg.Equals("/printTooltips")) { + printToolTips = true; + } else if (arg.Equals(DAFNY_VERIFY_ALL_MODULES)) { + verifyAllModules = true; + } else if (arg.StartsWith(DAFNY_COMPILE)) { + compileLevel = Int32.Parse(arg.Substring(DAFNY_COMPILE.Length)); + } else if (arg.StartsWith(DAFNY_ALLOCATED)) { + allocated = Int32.Parse(arg.Substring(DAFNY_ALLOCATED.Length)); + } else if (arg.StartsWith(DAFNY_COMPILE_TARGET)) { + // Ignore - assume it will work for all target language unless proven otherwise + } else { + throw new ArgumentException("Unrecognized dafny argument: " + arg); + } + } + break; + default: + throw new ArgumentException("Unrecognized lit command format: " + line); + } + } + } + } + + public static void Main(string[] args) { + var root = args[0]; + var count = 0; + var invalidCount = 0; + foreach (var file in Directory.GetFiles(root, "*.dfy", SearchOption.AllDirectories)) { + try { + count++; + ConvertLitTest(file); + } catch (ArgumentException e) { + invalidCount++; + Console.WriteLine(file + ": " + e.Message); + } + } + Console.WriteLine(invalidCount + "/" + count); + } + } +} \ No newline at end of file diff --git a/Source/LitTestConverter/Properties/AssemblyInfo.cs b/Source/LitTestConverter/Properties/AssemblyInfo.cs new file mode 100644 index 00000000000..fd9af8a2d05 --- /dev/null +++ b/Source/LitTestConverter/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("LitTestConverter")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("LitTestConverter")] +[assembly: AssemblyCopyright("Copyright © 2020")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("2EA7D80A-B179-4352-BDF2-8A1599F8685A")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] \ No newline at end of file From 6c607195e9dcdeed6601b0da8272560a17719fa7 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 8 Jul 2020 12:47:36 -0700 Subject: [PATCH 046/192] More progress on test conversion script --- Source/LitTestConverter/LitTestConvertor.cs | 47 +++++++++++++++------ 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index f23a2457057..0e91249cc41 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Text; namespace DafnyTests { @@ -12,21 +13,36 @@ public static class LitTestConvertor { private const string LIT_DAFNY = "%dafny"; private const string DAFNY_COMPILE = "/compile:"; private const string DAFNY_COMPILE_TARGET = "/compileTarget:"; - private const string DAFNY_AUTO_TRIGGERS = "/autoTriggers:"; - private const string DAFNY_VERIFY_ALL_MODULES = "/verifyAllModules"; - private const string DAFNY_ALLOCATED = "/allocated:"; private static readonly string[] IGNORED_DAFNY_COMMAND_ARGUMENTS = { "/print:\"%t.print\"", - "/dprint:\"%t.dprint\"", + "/dprint:\"%t.dprint\"", + "/dprint:\"%t.dfy\"", "/rprint:\"%t.rprint\"", "/rprint:\"%t.dprint\"", + "\"%s\"", ">", ">>", "\"%t\"" }; + + private static readonly string[] SUPPORTED_DAFNY_FLAGS = { + "/autoTriggers", + "/verifyAllModules", + "/allocated", + "/printTooltips", + "/env", + "/ironDafny", + "/definiteAssignment", + "/tracePOs", + "/optimizeResolution", + "/warnShadowing", + "/verifySnapshots", + "/traceCaching", + "/noNLarith", + "/errorTrace" + }; public static void ConvertLitTest(string filePath) { var compileLevel = 1; - var printToolTips = false; var autoTriggers = 1; var verifyAllModules = false; var allocated = 3; @@ -48,18 +64,12 @@ public static void ConvertLitTest(string filePath) { foreach (var arg in parts.Skip(1)) { if (IGNORED_DAFNY_COMMAND_ARGUMENTS.Contains(arg)) { // Ignore - } else if (arg.Equals("/printTooltips")) { - printToolTips = true; - } else if (arg.Equals(DAFNY_VERIFY_ALL_MODULES)) { - verifyAllModules = true; } else if (arg.StartsWith(DAFNY_COMPILE)) { compileLevel = Int32.Parse(arg.Substring(DAFNY_COMPILE.Length)); - } else if (arg.StartsWith(DAFNY_ALLOCATED)) { - allocated = Int32.Parse(arg.Substring(DAFNY_ALLOCATED.Length)); } else if (arg.StartsWith(DAFNY_COMPILE_TARGET)) { // Ignore - assume it will work for all target language unless proven otherwise } else { - throw new ArgumentException("Unrecognized dafny argument: " + arg); + ParseDafnyArgument(arg); } } break; @@ -70,6 +80,19 @@ public static void ConvertLitTest(string filePath) { } } + private static KeyValuePair ParseDafnyArgument(string argument) { + foreach (var supportedFlag in SUPPORTED_DAFNY_FLAGS) { + if (argument.StartsWith(supportedFlag)) { + if (argument.Equals(supportedFlag)) { + return new KeyValuePair(supportedFlag, "yes"); + } else if (argument[supportedFlag.Length] == ':') { + return new KeyValuePair(supportedFlag, argument.Substring(supportedFlag.Length + 1)); + } + } + } + throw new ArgumentException("Unrecognized dafny argument: " + argument); + } + public static void Main(string[] args) { var root = args[0]; var count = 0; From 5219755473b16958b0e38e0256e2ea6cb54187fb Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 9 Jul 2020 14:46:58 -0700 Subject: [PATCH 047/192] Be stricter about supported lit test pattern --- Source/LitTestConverter/LitTestConvertor.cs | 96 +++++++++++++-------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 0e91249cc41..401623f64d4 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -4,12 +4,12 @@ using System.Linq; using System.Runtime.InteropServices; using System.Text; +using Microsoft.SqlServer.Server; namespace DafnyTests { public static class LitTestConvertor { private const string LIT_COMMAND_PREFIX = "// RUN: "; - private const string LIT_DIFF = "%diff"; private const string LIT_DAFNY = "%dafny"; private const string DAFNY_COMPILE = "/compile:"; private const string DAFNY_COMPILE_TARGET = "/compileTarget:"; @@ -20,6 +20,7 @@ public static class LitTestConvertor { "/dprint:\"%t.dfy\"", "/rprint:\"%t.rprint\"", "/rprint:\"%t.dprint\"", + "/dprint:\"%t.dprint.dfy\"", "\"%s\"", ">", ">>", "\"%t\"" }; @@ -38,46 +39,64 @@ public static class LitTestConvertor { "/verifySnapshots", "/traceCaching", "/noNLarith", - "/errorTrace" + "/errorTrace", + "/arith", + "/noVerify", + "/dafnyVerify", + "/optimize", + // TODO-RS: Catch %t here? + "/dprint", + "/rprint" }; public static void ConvertLitTest(string filePath) { - var compileLevel = 1; - var autoTriggers = 1; - var verifyAllModules = false; - var allocated = 3; - var reader = new StreamReader(filePath); + string[] lines = File.ReadAllLines(filePath); + if (lines.Length < 2) { + throw new ArgumentException("Not enough lines to match expected lit test pattern"); + } - string line; - while((line = reader.ReadLine()) != null) { - if (line.StartsWith(LIT_COMMAND_PREFIX)) { - var parts = line.Substring(LIT_COMMAND_PREFIX.Length) - .Split((char[])null, StringSplitOptions.RemoveEmptyEntries); - switch (parts[0]) { - case LIT_DIFF: - // Ignore: assume that if this is anything other than the standard - // %diff "%s.expect" "%t" line, then at least one other line will be non-standard - // as well (to produce the non-standard arguments) - break; - case LIT_DAFNY: - // Check the arguments for anything non-standard - foreach (var arg in parts.Skip(1)) { - if (IGNORED_DAFNY_COMMAND_ARGUMENTS.Contains(arg)) { - // Ignore - } else if (arg.StartsWith(DAFNY_COMPILE)) { - compileLevel = Int32.Parse(arg.Substring(DAFNY_COMPILE.Length)); - } else if (arg.StartsWith(DAFNY_COMPILE_TARGET)) { - // Ignore - assume it will work for all target language unless proven otherwise - } else { - ParseDafnyArgument(arg); - } - } - break; - default: - throw new ArgumentException("Unrecognized lit command format: " + line); - } + string dafnyCommand = ExtractLitCommand(lines[0]); + if (dafnyCommand == null) { + // Doesn't look like a lit test + return; + } + if (!dafnyCommand.StartsWith(LIT_DAFNY)) { + throw new ArgumentException("First lit command is not expected %dafny: " + dafnyCommand); + } + var testConfig = ParseDafnyCommand(dafnyCommand.Substring(LIT_DAFNY.Length)); + + string diffCommand = ExtractLitCommand(lines[1]); + if (!diffCommand.Equals("%diff \"%s.expect\" \"%t\"")) { + throw new ArgumentException("Second lit command is not expected %diff: " + diffCommand); + } + } + + private static string ExtractLitCommand(string line) { + if (!line.StartsWith(LIT_COMMAND_PREFIX)) { + return null; + } + return line.Substring(LIT_COMMAND_PREFIX.Length); + } + + private static Dictionary ParseDafnyCommand(string line) { + var testConfig = new Dictionary(); + var parts = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries); + int compileLevel; + // Check the arguments for anything non-standard + foreach (var argument in parts) { + if (IGNORED_DAFNY_COMMAND_ARGUMENTS.Contains(argument)) { + // Ignore + } else if (argument.StartsWith(DAFNY_COMPILE)) { + compileLevel = Int32.Parse(argument.Substring(DAFNY_COMPILE.Length)); + } else if (argument.StartsWith(DAFNY_COMPILE_TARGET)) { + // Ignore - assume it will work for all target language unless proven otherwise + } else { + KeyValuePair pair = ParseDafnyArgument(argument); + testConfig.Add(pair.Key, pair.Value); } - } + } + + return testConfig; } private static KeyValuePair ParseDafnyArgument(string argument) { @@ -86,13 +105,14 @@ private static KeyValuePair ParseDafnyArgument(string argument) if (argument.Equals(supportedFlag)) { return new KeyValuePair(supportedFlag, "yes"); } else if (argument[supportedFlag.Length] == ':') { - return new KeyValuePair(supportedFlag, argument.Substring(supportedFlag.Length + 1)); + return new KeyValuePair(supportedFlag, + argument.Substring(supportedFlag.Length + 1)); } } } throw new ArgumentException("Unrecognized dafny argument: " + argument); } - + public static void Main(string[] args) { var root = args[0]; var count = 0; From 7988dbc6ef92d3d34999d89db694da1383a1a62b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 9 Jul 2020 16:57:29 -0700 Subject: [PATCH 048/192] Fix indendation --- Source/DafnyTests/DafnyTests.cs | 338 +++++++++--------- Source/DafnyTests/EnumerableUtils.cs | 26 +- Source/DafnyTests/TemporaryDirectory.cs | 50 +-- .../XUnitExtensions/AssertWithDiff.cs | 48 +-- .../CollectionPerTestCaseTheoryDiscoverer.cs | 54 +-- .../XUnitExtensions/TestCaseWithCollection.cs | 76 ++-- Source/DafnyTests/YamlUtils.cs | 74 ++-- Source/LitTestConverter/LitTestConvertor.cs | 224 ++++++------ 8 files changed, 449 insertions(+), 441 deletions(-) diff --git a/Source/DafnyTests/DafnyTests.cs b/Source/DafnyTests/DafnyTests.cs index 665ba696a86..92217546a69 100644 --- a/Source/DafnyTests/DafnyTests.cs +++ b/Source/DafnyTests/DafnyTests.cs @@ -16,192 +16,192 @@ namespace DafnyTests { - public class DafnyTests { + public class DafnyTests { - private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); - private static string DAFNY_ROOT = OUTPUT_ROOT.Parent.Parent.Parent.Parent.FullName; + private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); + private static string DAFNY_ROOT = OUTPUT_ROOT.Parent.Parent.Parent.Parent.FullName; - private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; - private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; - private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; + private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; + private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; + private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); + private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); - public static string RunDafny(IEnumerable arguments) { - List dafnyArguments = new List { - // Expected output does not contain logo - "/nologo", - "/countVerificationErrors:0", - - // We do not want absolute or relative paths in error messages, just the basename of the file - "/useBaseNameForFileName", - - // We do not want output such as "Compiled program written to Foo.cs" - // from the compilers, since that changes with the target language - "/compileVerbose:0" - }; - dafnyArguments.AddRange(arguments); - - using (Process dafnyProcess = new Process()) { - dafnyProcess.StartInfo.FileName = "mono"; - dafnyProcess.StartInfo.Arguments += DAFNY_EXE; - foreach (var argument in dafnyArguments) { - dafnyProcess.StartInfo.Arguments += " " + argument; - } - - dafnyProcess.StartInfo.UseShellExecute = false; - dafnyProcess.StartInfo.RedirectStandardOutput = true; - dafnyProcess.StartInfo.RedirectStandardError = true; - dafnyProcess.StartInfo.CreateNoWindow = true; - // Necessary for JS to find bignumber.js - dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; - - // Only preserve specific whitelisted environment variables - dafnyProcess.StartInfo.EnvironmentVariables.Clear(); - dafnyProcess.StartInfo.EnvironmentVariables.Add("PATH", Environment.GetEnvironmentVariable("PATH")); - // Go requires this or GOCACHE - dafnyProcess.StartInfo.EnvironmentVariables.Add("HOME", Environment.GetEnvironmentVariable("HOME")); - - dafnyProcess.Start(); - dafnyProcess.WaitForExit(); - string output = dafnyProcess.StandardOutput.ReadToEnd(); - string error = dafnyProcess.StandardError.ReadToEnd(); - if (dafnyProcess.ExitCode != 0) { - var message = String.Format("Non-zero Dafny exit code: {0}\n{1}\n{2}", dafnyProcess.ExitCode, output, error); - Assert.True(false, message); - } - - return output + error; - } + public static string RunDafny(IEnumerable arguments) { + List dafnyArguments = new List { + // Expected output does not contain logo + "/nologo", + "/countVerificationErrors:0", + + // We do not want absolute or relative paths in error messages, just the basename of the file + "/useBaseNameForFileName", + + // We do not want output such as "Compiled program written to Foo.cs" + // from the compilers, since that changes with the target language + "/compileVerbose:0" + }; + dafnyArguments.AddRange(arguments); + + using (Process dafnyProcess = new Process()) { + dafnyProcess.StartInfo.FileName = "mono"; + dafnyProcess.StartInfo.Arguments += DAFNY_EXE; + foreach (var argument in dafnyArguments) { + dafnyProcess.StartInfo.Arguments += " " + argument; } - private static string GetTestCaseConfigString(string filePath) { - // TODO-RS: Figure out how to do this cleanly on a TextReader instead, - // and to handle things like nested comments. - string fullText = File.ReadAllText(filePath); - var commentStart = fullText.IndexOf("/*"); - if (commentStart >= 0) { - var commentEnd = fullText.IndexOf("*/", commentStart + 2); - if (commentEnd >= 0) { - var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); - if (commentContent.StartsWith("---")) { - return commentContent; - } - } - } - - return null; - } - public static IEnumerable AllTestFiles() { - var filePaths = Directory.GetFiles(COMP_DIR, "*.dfy", SearchOption.AllDirectories) - .Select(path => GetRelativePath(TEST_ROOT, path)); - return filePaths.SelectMany(TestCasesForDafnyFile); + dafnyProcess.StartInfo.UseShellExecute = false; + dafnyProcess.StartInfo.RedirectStandardOutput = true; + dafnyProcess.StartInfo.RedirectStandardError = true; + dafnyProcess.StartInfo.CreateNoWindow = true; + // Necessary for JS to find bignumber.js + dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; + + // Only preserve specific whitelisted environment variables + dafnyProcess.StartInfo.EnvironmentVariables.Clear(); + dafnyProcess.StartInfo.EnvironmentVariables.Add("PATH", Environment.GetEnvironmentVariable("PATH")); + // Go requires this or GOCACHE + dafnyProcess.StartInfo.EnvironmentVariables.Add("HOME", Environment.GetEnvironmentVariable("HOME")); + + dafnyProcess.Start(); + dafnyProcess.WaitForExit(); + string output = dafnyProcess.StandardOutput.ReadToEnd(); + string error = dafnyProcess.StandardError.ReadToEnd(); + if (dafnyProcess.ExitCode != 0) { + var message = String.Format("Non-zero Dafny exit code: {0}\n{1}\n{2}", dafnyProcess.ExitCode, output, error); + Assert.True(false, message); } - private static IEnumerable TestCasesForDafnyFile(string filePath) { - var fullFilePath = Path.Combine(TEST_ROOT, filePath); - string configString = GetTestCaseConfigString(fullFilePath); - IEnumerable configs; - if (configString != null) { - var yamlStream = new YamlStream(); - yamlStream.Load(new StringReader(configString)); - var config = yamlStream.Documents[0].RootNode; - configs = YamlUtils.Expand(config); - } else { - configs = new[] {new YamlMappingNode()}; - } - - IEnumerable mappings = configs.SelectMany(config => { - if (config is YamlMappingNode mapping) { - return ResolveCompile(filePath, mapping); - } else { - throw new ArgumentException("Bad test case configuration: " + config); - } - }); - - return mappings.Select(mapping => { - return new[] {filePath, String.Join(" ", mapping.Select(ConfigPairToArgument))}; - }); - } + return output + error; + } + } - private static IEnumerable ResolveCompile(string filePath, YamlMappingNode mapping) { - if (!mapping.Children.ContainsKey(new YamlScalarNode("compile"))) { - mapping.Add("compile", "3"); - } - - if (mapping["compile"].Equals(new YamlScalarNode("3")) && !mapping.Children.ContainsKey("compileTarget")) { - var languages = new[] {"cs", "java", "go", "js"}; - foreach (var language in languages) { - var withLanguage = new YamlMappingNode(mapping.Children); - withLanguage.Add("compileTarget", language); - yield return withLanguage; - } - } else { - yield return mapping; - } + private static string GetTestCaseConfigString(string filePath) { + // TODO-RS: Figure out how to do this cleanly on a TextReader instead, + // and to handle things like nested comments. + string fullText = File.ReadAllText(filePath); + var commentStart = fullText.IndexOf("/*"); + if (commentStart >= 0) { + var commentEnd = fullText.IndexOf("*/", commentStart + 2); + if (commentEnd >= 0) { + var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); + if (commentContent.StartsWith("---")) { + return commentContent; + } } + } + + return null; + } + public static IEnumerable AllTestFiles() { + var filePaths = Directory.GetFiles(COMP_DIR, "*.dfy", SearchOption.AllDirectories) + .Select(path => GetRelativePath(TEST_ROOT, path)); + return filePaths.SelectMany(TestCasesForDafnyFile); + } - private static string ConfigPairToArgument(KeyValuePair pair) { - if (pair.Key.Equals(new YamlScalarNode("otherFiles"))) { - return pair.Value.ToString(); - } else { - return String.Format("/{0}:{1}", pair.Key, pair.Value); - } + private static IEnumerable TestCasesForDafnyFile(string filePath) { + var fullFilePath = Path.Combine(TEST_ROOT, filePath); + string configString = GetTestCaseConfigString(fullFilePath); + IEnumerable configs; + if (configString != null) { + var yamlStream = new YamlStream(); + yamlStream.Load(new StringReader(configString)); + var config = yamlStream.Documents[0].RootNode; + configs = YamlUtils.Expand(config); + } else { + configs = new[] {new YamlMappingNode()}; + } + + IEnumerable mappings = configs.SelectMany(config => { + if (config is YamlMappingNode mapping) { + return ResolveCompile(filePath, mapping); + } else { + throw new ArgumentException("Bad test case configuration: " + config); } + }); + + return mappings.Select(mapping => { + return new[] {filePath, String.Join(" ", mapping.Select(ConfigPairToArgument))}; + }); + } - // TODO-RS: Replace with Path.GetRelativePath() if we move to .NET Core 3.1 - private static string GetRelativePath(string relativeTo, string path) { - var fullRelativeTo = Path.GetFullPath(relativeTo); - var fullPath = Path.GetFullPath(path); - Assert.StartsWith(fullRelativeTo, fullPath); - return fullPath.Substring(fullRelativeTo.Length); + private static IEnumerable ResolveCompile(string filePath, YamlMappingNode mapping) { + if (!mapping.Children.ContainsKey(new YamlScalarNode("compile"))) { + mapping.Add("compile", "3"); + } + + if (mapping["compile"].Equals(new YamlScalarNode("3")) && !mapping.Children.ContainsKey("compileTarget")) { + var languages = new[] {"cs", "java", "go", "js"}; + foreach (var language in languages) { + var withLanguage = new YamlMappingNode(mapping.Children); + withLanguage.Add("compileTarget", language); + yield return withLanguage; } + } else { + yield return mapping; + } + } + + private static string ConfigPairToArgument(KeyValuePair pair) { + if (pair.Key.Equals(new YamlScalarNode("otherFiles"))) { + return pair.Value.ToString(); + } else { + return String.Format("/{0}:{1}", pair.Key, pair.Value); + } + } + + // TODO-RS: Replace with Path.GetRelativePath() if we move to .NET Core 3.1 + private static string GetRelativePath(string relativeTo, string path) { + var fullRelativeTo = Path.GetFullPath(relativeTo); + var fullPath = Path.GetFullPath(path); + Assert.StartsWith(fullRelativeTo, fullPath); + return fullPath.Substring(fullRelativeTo.Length); + } - [ParallelTheory] - [MemberData(nameof(AllTestFiles))] - public void Test(string file, string args) { - string fullInputPath = Path.Combine(TEST_ROOT, file); - string[] arguments = args.Split(); - - string expectedOutputPath = fullInputPath + ".expect"; - bool specialCase = false; - string compileTarget = arguments.FirstOrDefault(arg => arg.StartsWith("/compileTarget:")); - if (compileTarget != null) { - string language = compileTarget.Substring("/compileTarget:".Length); - var specialCasePath = fullInputPath + "." + language + ".expect"; - if (File.Exists(specialCasePath)) { - specialCase = true; - expectedOutputPath = specialCasePath; - } + [ParallelTheory] + [MemberData(nameof(AllTestFiles))] + public void Test(string file, string args) { + string fullInputPath = Path.Combine(TEST_ROOT, file); + string[] arguments = args.Split(); + + string expectedOutputPath = fullInputPath + ".expect"; + bool specialCase = false; + string compileTarget = arguments.FirstOrDefault(arg => arg.StartsWith("/compileTarget:")); + if (compileTarget != null) { + string language = compileTarget.Substring("/compileTarget:".Length); + var specialCasePath = fullInputPath + "." + language + ".expect"; + if (File.Exists(specialCasePath)) { + specialCase = true; + expectedOutputPath = specialCasePath; + } - // Include any additional files - var additionalFilesPath = fullInputPath + "." + language + ".files"; - if (Directory.Exists(additionalFilesPath)) { - arguments = arguments.Concat(Directory.GetFiles(additionalFilesPath)).ToArray(); - } - } + // Include any additional files + var additionalFilesPath = fullInputPath + "." + language + ".files"; + if (Directory.Exists(additionalFilesPath)) { + arguments = arguments.Concat(Directory.GetFiles(additionalFilesPath)).ToArray(); + } + } - var argumentsWithFile = new List {fullInputPath}.Concat(arguments); - var expectedOutput = File.ReadAllText(expectedOutputPath); - - string output; - if (arguments.Any(arg => arg.StartsWith("/out"))) { - output = RunDafny(argumentsWithFile); - } else { - // Note that the temporary directory has to be an ancestor of Test - // or else Javascript won't be able to locate bignumber.js :( - using (var tempDir = new TemporaryDirectory(OUTPUT_DIR)) { - // Add an extra component to the path to keep the files created inside the - // temporary directory, since some compilers will - // interpret the path as a single file basename rather than a directory. - var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; - argumentsWithFile = new List {outArgument}.Concat(argumentsWithFile); - output = RunDafny(argumentsWithFile); - } - } - - AssertWithDiff.Equal(expectedOutput, output); - Skip.If(specialCase, "Confirmed known exception for arguments: " + args); + var argumentsWithFile = new List {fullInputPath}.Concat(arguments); + var expectedOutput = File.ReadAllText(expectedOutputPath); + + string output; + if (arguments.Any(arg => arg.StartsWith("/out"))) { + output = RunDafny(argumentsWithFile); + } else { + // Note that the temporary directory has to be an ancestor of Test + // or else Javascript won't be able to locate bignumber.js :( + using (var tempDir = new TemporaryDirectory(OUTPUT_DIR)) { + // Add an extra component to the path to keep the files created inside the + // temporary directory, since some compilers will + // interpret the path as a single file basename rather than a directory. + var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; + argumentsWithFile = new List {outArgument}.Concat(argumentsWithFile); + output = RunDafny(argumentsWithFile); } + } + + AssertWithDiff.Equal(expectedOutput, output); + Skip.If(specialCase, "Confirmed known exception for arguments: " + args); } + } } \ No newline at end of file diff --git a/Source/DafnyTests/EnumerableUtils.cs b/Source/DafnyTests/EnumerableUtils.cs index e0fe2d4909b..cacd5efdf3c 100644 --- a/Source/DafnyTests/EnumerableUtils.cs +++ b/Source/DafnyTests/EnumerableUtils.cs @@ -2,18 +2,18 @@ using System.Linq; namespace DafnyTests { - public static class EnumerableUtils { - /** - * Source: https://docs.microsoft.com/en-us/archive/blogs/ericlippert/computing-a-cartesian-product-with-linq - */ - public static IEnumerable> CartesianProduct(this IEnumerable> sequences) { - IEnumerable> emptyProduct = new[] {Enumerable.Empty()}; - return sequences.Aggregate( - emptyProduct, - (accumulator, sequence) => - from accseq in accumulator - from item in sequence - select accseq.Concat(new[] {item})); - } + public static class EnumerableUtils { + /** + * Source: https://docs.microsoft.com/en-us/archive/blogs/ericlippert/computing-a-cartesian-product-with-linq + */ + public static IEnumerable> CartesianProduct(this IEnumerable> sequences) { + IEnumerable> emptyProduct = new[] {Enumerable.Empty()}; + return sequences.Aggregate( + emptyProduct, + (accumulator, sequence) => + from accseq in accumulator + from item in sequence + select accseq.Concat(new[] {item})); } + } } \ No newline at end of file diff --git a/Source/DafnyTests/TemporaryDirectory.cs b/Source/DafnyTests/TemporaryDirectory.cs index 1dbe59773f9..d1e1f5a090d 100644 --- a/Source/DafnyTests/TemporaryDirectory.cs +++ b/Source/DafnyTests/TemporaryDirectory.cs @@ -2,36 +2,36 @@ using System.IO; namespace DafnyTests { - public class TemporaryDirectory : IDisposable { - public readonly DirectoryInfo DirInfo; + public class TemporaryDirectory : IDisposable { + public readonly DirectoryInfo DirInfo; - public TemporaryDirectory(string parent, string prefix = "") { - string dirPath; - do { - dirPath = Path.Combine(parent, prefix + Path.GetRandomFileName()); - } while (File.Exists(dirPath) || Directory.Exists(dirPath)); + public TemporaryDirectory(string parent, string prefix = "") { + string dirPath; + do { + dirPath = Path.Combine(parent, prefix + Path.GetRandomFileName()); + } while (File.Exists(dirPath) || Directory.Exists(dirPath)); - DirInfo = Directory.CreateDirectory(dirPath); - } + DirInfo = Directory.CreateDirectory(dirPath); + } - public void Dispose() { - Dispose(true); - } + public void Dispose() { + Dispose(true); + } - ~TemporaryDirectory() { - Dispose(false); - } + ~TemporaryDirectory() { + Dispose(false); + } - protected virtual void Dispose(bool disposing) { - SafeDelete(); - } + protected virtual void Dispose(bool disposing) { + SafeDelete(); + } - public void SafeDelete() { - try { - DirInfo.Delete(true); - } catch { - // Best effort only - } - } + public void SafeDelete() { + try { + DirInfo.Delete(true); + } catch { + // Best effort only + } } + } } \ No newline at end of file diff --git a/Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs b/Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs index 9291263eae3..fabf42b95ba 100644 --- a/Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs +++ b/Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs @@ -5,31 +5,31 @@ using Xunit.Sdk; namespace XUnitExtensions { - public class AssertWithDiff { - public static void Equal(string expected, string actual) { - if (expected != actual) { - var diff = InlineDiffBuilder.Instance.BuildDiffModel(expected, actual); + public class AssertWithDiff { + public static void Equal(string expected, string actual) { + if (expected != actual) { + var diff = InlineDiffBuilder.Instance.BuildDiffModel(expected, actual); - var message = new StringBuilder(); - message.AppendLine("AssertEqualWithDiff() Failure"); - message.AppendLine("Diff (changing expected into actual):"); - foreach (var line in diff.Lines) { - switch (line.Type) { - case ChangeType.Inserted: - message.Append("+"); - break; - case ChangeType.Deleted: - message.Append("-"); - break; - default: - message.Append(" "); - break; - } - message.AppendLine(line.Text); - } - - throw new AssertActualExpectedException(expected, actual, message.ToString()); - } + var message = new StringBuilder(); + message.AppendLine("AssertEqualWithDiff() Failure"); + message.AppendLine("Diff (changing expected into actual):"); + foreach (var line in diff.Lines) { + switch (line.Type) { + case ChangeType.Inserted: + message.Append("+"); + break; + case ChangeType.Deleted: + message.Append("-"); + break; + default: + message.Append(" "); + break; + } + message.AppendLine(line.Text); } + + throw new AssertActualExpectedException(expected, actual, message.ToString()); + } } + } } \ No newline at end of file diff --git a/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index 5b5b0d2daa3..dc727405674 100644 --- a/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -6,38 +6,38 @@ using Xunit.Sdk; namespace XUnitExtensions { - public class CollectionPerTestCaseTheoryDiscoverer : IXunitTestCaseDiscoverer { + public class CollectionPerTestCaseTheoryDiscoverer : IXunitTestCaseDiscoverer { - readonly IXunitTestCaseDiscoverer theoryDiscoverer; + readonly IXunitTestCaseDiscoverer theoryDiscoverer; - public CollectionPerTestCaseTheoryDiscoverer(IMessageSink diagnosticMessageSink) - { - theoryDiscoverer = new SkippableTheoryDiscoverer(diagnosticMessageSink); - } + public CollectionPerTestCaseTheoryDiscoverer(IMessageSink diagnosticMessageSink) + { + theoryDiscoverer = new SkippableTheoryDiscoverer(diagnosticMessageSink); + } - private TestCollection testCollectionForTestCase(IXunitTestCase testCase) { - return new TestCollection(testCase.TestMethod.TestClass.TestCollection.TestAssembly, - (ITypeInfo) null, "Test collection for " + testCase.DisplayName); - } + private TestCollection testCollectionForTestCase(IXunitTestCase testCase) { + return new TestCollection(testCase.TestMethod.TestClass.TestCollection.TestAssembly, + (ITypeInfo) null, "Test collection for " + testCase.DisplayName); + } - public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { - // This discoverer requires pre-enumeration in order to assign a collection to each test case. - discoveryOptions.SetValue("xunit.discovery.PreEnumerateTheories", true); + public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { + // This discoverer requires pre-enumeration in order to assign a collection to each test case. + discoveryOptions.SetValue("xunit.discovery.PreEnumerateTheories", true); - IEnumerable testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); - var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); - var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD_COUNT"); - // TODO-RS: More careful error checking - if (shardEnvVar != null && numShardsEnvVar != null) { - var testCaseList = testCases.ToList(); - var shard = Int32.Parse(shardEnvVar); - var numShards = Int32.Parse(numShardsEnvVar); - var shardStart = (shard - 1) * testCaseList.Count / numShards; - var shardEnd = shard * testCaseList.Count / numShards; - testCases = testCaseList.GetRange(shardStart, shardEnd - shardStart); - } + IEnumerable testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); + var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); + var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD_COUNT"); + // TODO-RS: More careful error checking + if (shardEnvVar != null && numShardsEnvVar != null) { + var testCaseList = testCases.ToList(); + var shard = Int32.Parse(shardEnvVar); + var numShards = Int32.Parse(numShardsEnvVar); + var shardStart = (shard - 1) * testCaseList.Count / numShards; + var shardEnd = shard * testCaseList.Count / numShards; + testCases = testCaseList.GetRange(shardStart, shardEnd - shardStart); + } - return testCases.Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); - } + return testCases.Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); } + } } \ No newline at end of file diff --git a/Source/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs b/Source/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs index 76decdf6562..bf5e66e7e2b 100644 --- a/Source/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs +++ b/Source/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs @@ -8,51 +8,51 @@ using Xunit.Sdk; namespace XUnitExtensions { - public class TestCaseWithCollection : LongLivedMarshalByRefObject, IXunitTestCase { + public class TestCaseWithCollection : LongLivedMarshalByRefObject, IXunitTestCase { - private IXunitTestCase testCase; - private ITestMethod testMethod; + private IXunitTestCase testCase; + private ITestMethod testMethod; - public TestCaseWithCollection(IXunitTestCase testCase, ITestCollection collection) - { - this.testCase = testCase; + public TestCaseWithCollection(IXunitTestCase testCase, ITestCollection collection) + { + this.testCase = testCase; - var testClassWithCollection = new TestClass(collection, testCase.TestMethod.TestClass.Class); - this.testMethod = new TestMethod(testClassWithCollection, testCase.TestMethod.Method); - } + var testClassWithCollection = new TestClass(collection, testCase.TestMethod.TestClass.Class); + this.testMethod = new TestMethod(testClassWithCollection, testCase.TestMethod.Method); + } - [Obsolete("Called by the de-serializer", error: true)] - public TestCaseWithCollection() { } + [Obsolete("Called by the de-serializer", error: true)] + public TestCaseWithCollection() { } - public void Deserialize(IXunitSerializationInfo info) { - testCase = info.GetValue("InnerTestCase"); - testMethod = info.GetValue("TestMethod"); - } - - public void Serialize(IXunitSerializationInfo info) { - info.AddValue("InnerTestCase", testCase); - info.AddValue("TestMethod", testMethod); - } - public string DisplayName { get { return testCase.DisplayName; } } - public string SkipReason { get { return testCase.SkipReason; } } - public ISourceInformation SourceInformation - { - get { return testCase.SourceInformation; } - set { testCase.SourceInformation = value; } - } - public ITestMethod TestMethod { get { return testMethod; } } + public void Deserialize(IXunitSerializationInfo info) { + testCase = info.GetValue("InnerTestCase"); + testMethod = info.GetValue("TestMethod"); + } - public object[] TestMethodArguments { get { return testCase.TestMethodArguments; } } - public Dictionary> Traits { get { return testCase.Traits; } } - public string UniqueID { get { return testCase.UniqueID; } } + public void Serialize(IXunitSerializationInfo info) { + info.AddValue("InnerTestCase", testCase); + info.AddValue("TestMethod", testMethod); + } + public string DisplayName { get { return testCase.DisplayName; } } + public string SkipReason { get { return testCase.SkipReason; } } + public ISourceInformation SourceInformation + { + get { return testCase.SourceInformation; } + set { testCase.SourceInformation = value; } + } + public ITestMethod TestMethod { get { return testMethod; } } - public Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, - ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { - return testCase.RunAsync(diagnosticMessageSink, messageBus, constructorArguments, aggregator, cancellationTokenSource); - } + public object[] TestMethodArguments { get { return testCase.TestMethodArguments; } } + public Dictionary> Traits { get { return testCase.Traits; } } + public string UniqueID { get { return testCase.UniqueID; } } - public Exception InitializationException { get; } - public IMethodInfo Method { get; } - public int Timeout { get; } + public Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, + ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { + return testCase.RunAsync(diagnosticMessageSink, messageBus, constructorArguments, aggregator, cancellationTokenSource); } + + public Exception InitializationException { get; } + public IMethodInfo Method { get; } + public int Timeout { get; } + } } \ No newline at end of file diff --git a/Source/DafnyTests/YamlUtils.cs b/Source/DafnyTests/YamlUtils.cs index 5d54eb8e331..dfe5df2f4c6 100644 --- a/Source/DafnyTests/YamlUtils.cs +++ b/Source/DafnyTests/YamlUtils.cs @@ -6,49 +6,49 @@ using YamlDotNet.Serialization; namespace DafnyTests { - public class YamlUtils { + public class YamlUtils { - public static IEnumerable Expand(YamlNode node) { - if (node is YamlSequenceNode seqNode) { - return seqNode.SelectMany(child => Expand(child)); - } else if (node is YamlMappingNode mappingNode) { - return mappingNode.Select(ExpandValue).CartesianProduct().Select(FromPairs); - } else { - return new[] {node}; - } - } + public static IEnumerable Expand(YamlNode node) { + if (node is YamlSequenceNode seqNode) { + return seqNode.SelectMany(child => Expand(child)); + } else if (node is YamlMappingNode mappingNode) { + return mappingNode.Select(ExpandValue).CartesianProduct().Select(FromPairs); + } else { + return new[] {node}; + } + } - private static IEnumerable> ExpandValue(KeyValuePair pair) { - return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); - } + private static IEnumerable> ExpandValue(KeyValuePair pair) { + return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); + } - private static YamlMappingNode FromPairs(IEnumerable> pairs) { - var result = new YamlMappingNode(); - foreach (var pair in pairs) { - result.Add(pair.Key, pair.Value); - } + private static YamlMappingNode FromPairs(IEnumerable> pairs) { + var result = new YamlMappingNode(); + foreach (var pair in pairs) { + result.Add(pair.Key, pair.Value); + } - return result; - } + return result; + } - [Theory] - [InlineData("multiple.yml", "multiple.expect.yml")] - public void ExpandTest(string inputPath, string expectedOutputPath) { - string fullInputPath = "./YamlTests/" + inputPath; - string fullExpectedOutputPath = "./YamlTests/" + expectedOutputPath; + [Theory] + [InlineData("multiple.yml", "multiple.expect.yml")] + public void ExpandTest(string inputPath, string expectedOutputPath) { + string fullInputPath = "./YamlTests/" + inputPath; + string fullExpectedOutputPath = "./YamlTests/" + expectedOutputPath; - using (var reader = File.OpenText(fullInputPath)) { - var yamlStream = new YamlStream(); - yamlStream.Load(reader); - var root = yamlStream.Documents[0].RootNode; - var expanded = Expand(root); + using (var reader = File.OpenText(fullInputPath)) { + var yamlStream = new YamlStream(); + yamlStream.Load(reader); + var root = yamlStream.Documents[0].RootNode; + var expanded = Expand(root); - var outputWriter = new StringWriter(); - var serializer = new SerializerBuilder().Build(); - serializer.Serialize(outputWriter, expanded); - string expectedOutput = File.ReadAllText(fullExpectedOutputPath); - Assert.Equal(expectedOutput, outputWriter.ToString()); - } - } + var outputWriter = new StringWriter(); + var serializer = new SerializerBuilder().Build(); + serializer.Serialize(outputWriter, expanded); + string expectedOutput = File.ReadAllText(fullExpectedOutputPath); + Assert.Equal(expectedOutput, outputWriter.ToString()); + } } + } } \ No newline at end of file diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 401623f64d4..7d09bea4af9 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -6,127 +6,135 @@ using System.Text; using Microsoft.SqlServer.Server; -namespace DafnyTests { - public static class LitTestConvertor { +namespace DafnyTests { + public static class LitTestConvertor { - private const string LIT_COMMAND_PREFIX = "// RUN: "; - private const string LIT_DAFNY = "%dafny"; - private const string DAFNY_COMPILE = "/compile:"; - private const string DAFNY_COMPILE_TARGET = "/compileTarget:"; + private const string LIT_COMMAND_PREFIX = "// RUN: "; + private const string LIT_DAFNY = "%dafny"; + private const string DAFNY_COMPILE = "/compile:"; + private const string DAFNY_COMPILE_TARGET = "/compileTarget:"; - private static readonly string[] IGNORED_DAFNY_COMMAND_ARGUMENTS = { - "/print:\"%t.print\"", - "/dprint:\"%t.dprint\"", - "/dprint:\"%t.dfy\"", - "/rprint:\"%t.rprint\"", - "/rprint:\"%t.dprint\"", - "/dprint:\"%t.dprint.dfy\"", - - "\"%s\"", ">", ">>", "\"%t\"" - }; + private static readonly string[] IGNORED_DAFNY_COMMAND_ARGUMENTS = { + "/print:\"%t.print\"", + "/dprint:\"%t.dprint\"", + "/dprint:\"%t.dfy\"", + "/rprint:\"%t.rprint\"", + "/rprint:\"%t.dprint\"", + "/dprint:\"%t.dprint.dfy\"", + + "\"%s\"", ">", ">>", "\"%t\"" + }; + + private static readonly string[] SUPPORTED_DAFNY_FLAGS = { + "/allocated", + "/arith", + "/autoTriggers", + "/dafnyVerify", + "/definiteAssignment", + "/dprint", + "/env", + "/errorTrace", + "/ironDafny", + "/noNLarith", + "/noVerify", + "/optimize", + "/optimizeResolution", + "/print", + "/printTooltips", + "/rprint", + "/traceCaching", + "/tracePOs", + "/verifyAllModules", + "/verifySnapshots", + "/warnShadowing", + }; - private static readonly string[] SUPPORTED_DAFNY_FLAGS = { - "/autoTriggers", - "/verifyAllModules", - "/allocated", - "/printTooltips", - "/env", - "/ironDafny", - "/definiteAssignment", - "/tracePOs", - "/optimizeResolution", - "/warnShadowing", - "/verifySnapshots", - "/traceCaching", - "/noNLarith", - "/errorTrace", - "/arith", - "/noVerify", - "/dafnyVerify", - "/optimize", - // TODO-RS: Catch %t here? - "/dprint", - "/rprint" - }; + private static readonly string[] DAFNY_OUTPUT_FLAGS = { + "/print", + "/dprint", + "/rprint", + }; - public static void ConvertLitTest(string filePath) { - string[] lines = File.ReadAllLines(filePath); - if (lines.Length < 2) { - throw new ArgumentException("Not enough lines to match expected lit test pattern"); - } + public static void ConvertLitTest(string filePath) { + string[] lines = File.ReadAllLines(filePath); + if (lines.Length < 2) { + throw new ArgumentException("Not enough lines to match expected lit test pattern"); + } - string dafnyCommand = ExtractLitCommand(lines[0]); - if (dafnyCommand == null) { - // Doesn't look like a lit test - return; - } - if (!dafnyCommand.StartsWith(LIT_DAFNY)) { - throw new ArgumentException("First lit command is not expected %dafny: " + dafnyCommand); - } - var testConfig = ParseDafnyCommand(dafnyCommand.Substring(LIT_DAFNY.Length)); + string dafnyCommand = ExtractLitCommand(lines[0]); + if (dafnyCommand == null) { + // Doesn't look like a lit test + return; + } + if (!dafnyCommand.StartsWith(LIT_DAFNY)) { + throw new ArgumentException("First lit command is not expected %dafny: " + dafnyCommand); + } + var testConfig = ParseDafnyCommand(dafnyCommand.Substring(LIT_DAFNY.Length)); - string diffCommand = ExtractLitCommand(lines[1]); - if (!diffCommand.Equals("%diff \"%s.expect\" \"%t\"")) { - throw new ArgumentException("Second lit command is not expected %diff: " + diffCommand); - } + string diffCommand = ExtractLitCommand(lines[1]); + if (!diffCommand.Equals("%diff \"%s.expect\" \"%t\"")) { + throw new ArgumentException("Second lit command is not expected %diff: " + diffCommand); } + } - private static string ExtractLitCommand(string line) { - if (!line.StartsWith(LIT_COMMAND_PREFIX)) { - return null; - } - return line.Substring(LIT_COMMAND_PREFIX.Length); - } + private static string ExtractLitCommand(string line) { + if (!line.StartsWith(LIT_COMMAND_PREFIX)) { + return null; + } + return line.Substring(LIT_COMMAND_PREFIX.Length); + } - private static Dictionary ParseDafnyCommand(string line) { - var testConfig = new Dictionary(); - var parts = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries); - int compileLevel; - // Check the arguments for anything non-standard - foreach (var argument in parts) { - if (IGNORED_DAFNY_COMMAND_ARGUMENTS.Contains(argument)) { - // Ignore - } else if (argument.StartsWith(DAFNY_COMPILE)) { - compileLevel = Int32.Parse(argument.Substring(DAFNY_COMPILE.Length)); - } else if (argument.StartsWith(DAFNY_COMPILE_TARGET)) { - // Ignore - assume it will work for all target language unless proven otherwise - } else { - KeyValuePair pair = ParseDafnyArgument(argument); - testConfig.Add(pair.Key, pair.Value); - } - } - - return testConfig; + private static Dictionary ParseDafnyCommand(string line) { + var testConfig = new Dictionary(); + var parts = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries); + int compileLevel; + // Check the arguments for anything non-standard + foreach (var argument in parts) { + if (IGNORED_DAFNY_COMMAND_ARGUMENTS.Contains(argument)) { + // Ignore + } else if (argument.StartsWith(DAFNY_COMPILE)) { + compileLevel = Int32.Parse(argument.Substring(DAFNY_COMPILE.Length)); + } else if (argument.StartsWith(DAFNY_COMPILE_TARGET)) { + // Ignore - assume it will work for all target language unless proven otherwise + } else { + KeyValuePair pair = ParseDafnyArgument(argument); + if (!(DAFNY_OUTPUT_FLAGS.Contains(pair.Key) && pair.Value.Contains("%t"))) { + testConfig.Add(pair.Key, pair.Value); + } } + } - private static KeyValuePair ParseDafnyArgument(string argument) { - foreach (var supportedFlag in SUPPORTED_DAFNY_FLAGS) { - if (argument.StartsWith(supportedFlag)) { - if (argument.Equals(supportedFlag)) { - return new KeyValuePair(supportedFlag, "yes"); - } else if (argument[supportedFlag.Length] == ':') { - return new KeyValuePair(supportedFlag, - argument.Substring(supportedFlag.Length + 1)); - } - } - } - throw new ArgumentException("Unrecognized dafny argument: " + argument); + return testConfig; + } + + private static KeyValuePair ParseDafnyArgument(string argument) { + foreach (var supportedFlag in SUPPORTED_DAFNY_FLAGS) { + if (argument.StartsWith(supportedFlag)) { + if (argument.Equals(supportedFlag)) { + return new KeyValuePair(supportedFlag, "yes"); + } else if (argument[supportedFlag.Length] == ':') { + return new KeyValuePair(supportedFlag, + argument.Substring(supportedFlag.Length + 1)); + } } + } + throw new ArgumentException("Unrecognized dafny argument: " + argument); + } - public static void Main(string[] args) { - var root = args[0]; - var count = 0; - var invalidCount = 0; - foreach (var file in Directory.GetFiles(root, "*.dfy", SearchOption.AllDirectories)) { - try { - count++; - ConvertLitTest(file); - } catch (ArgumentException e) { - invalidCount++; - Console.WriteLine(file + ": " + e.Message); - } - } - Console.WriteLine(invalidCount + "/" + count); + public static void Main(string[] args) { + var root = args[0]; + var count = 0; + var invalidCount = 0; + foreach (var file in Directory.GetFiles(root, "*.dfy", SearchOption.AllDirectories)) { + try { + count++; + ConvertLitTest(file); + } catch (ArgumentException e) { + invalidCount++; + Console.WriteLine(file + ": " + e.Message); } + } + Console.WriteLine(invalidCount + "/" + count); } + } } \ No newline at end of file From 45b615252a34fc0a91bbd3024e0de3d5510c9280 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 9 Jul 2020 23:12:17 -0700 Subject: [PATCH 049/192] =?UTF-8?q?Applied=20David=E2=80=99s=20optimizatio?= =?UTF-8?q?n=20(verify=20once,=20compile=20many)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{DafnyTests.cs => DafnyTestCase.cs} | 221 ++++++++++++------ Source/DafnyTests/DafnyTests-NetCore.csproj | 37 +++ Source/LitTestConverter/LitTestConvertor.cs | 9 +- Test/comp/Arrays.dfy.expect | 2 +- Test/comp/BranchCoverage.dfy.expect | 2 +- Test/comp/CSharpStyling.dfy.expect | 2 +- Test/comp/Calls.dfy.expect | 2 +- Test/comp/Class.dfy.expect | 2 +- Test/comp/Collections.dfy.cs.expect | 2 +- Test/comp/Collections.dfy.expect | 2 +- Test/comp/Collections.dfy.java.expect | 2 +- Test/comp/Comprehensions.dfy.expect | 2 +- Test/comp/Comprehensions.dfy.java.expect | 2 +- Test/comp/CovariantCollections.dfy.expect | 2 +- .../comp/CovariantCollections.dfy.java.expect | 2 +- Test/comp/Dt.dfy.expect | 2 +- Test/comp/Dt.dfy.java.expect | 2 +- Test/comp/Extern.dfy.expect | 2 +- Test/comp/ExternCtors.dfy.expect | 2 +- Test/comp/Forall.dfy.expect | 2 +- Test/comp/Ghosts.dfy.expect | 2 +- Test/comp/GoModule.dfy.expect | 2 +- Test/comp/Iterators.dfy.expect | 2 +- Test/comp/Let.dfy.expect | 2 +- Test/comp/Module.dfy.expect | 2 +- Test/comp/NativeNumbers.dfy.expect | 2 +- Test/comp/Numbers.dfy.expect | 2 +- Test/comp/Numbers.dfy.go.expect | 2 +- Test/comp/Poly.dfy.cs.expect | 2 +- Test/comp/Poly.dfy.expect | 2 +- Test/comp/Poly.dfy.java.expect | 2 +- Test/comp/Poly.dfy.js.expect | 2 +- .../StaticMembersOfGenericTypes.dfy.expect | 2 +- Test/comp/TailRecursion.dfy.expect | 2 +- Test/comp/TailRecursion.dfy.go.expect | 2 +- Test/comp/TailRecursion.dfy.js.expect | 2 +- Test/comp/TypeParams.dfy.cs.expect | 2 +- Test/comp/TypeParams.dfy.go.expect | 2 +- Test/comp/TypeParams.dfy.java.expect | 2 +- Test/comp/TypeParams.dfy.js.expect | 2 +- 40 files changed, 220 insertions(+), 121 deletions(-) rename Source/DafnyTests/{DafnyTests.cs => DafnyTestCase.cs} (50%) diff --git a/Source/DafnyTests/DafnyTests.cs b/Source/DafnyTests/DafnyTestCase.cs similarity index 50% rename from Source/DafnyTests/DafnyTests.cs rename to Source/DafnyTests/DafnyTestCase.cs index 92217546a69..0b2220760e7 100644 --- a/Source/DafnyTests/DafnyTests.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -9,6 +9,7 @@ using DiffPlex.DiffBuilder; using DiffPlex.DiffBuilder.Model; using Xunit; +using Xunit.Abstractions; using Xunit.Sdk; using XUnitExtensions; using YamlDotNet.RepresentationModel; @@ -16,17 +17,75 @@ namespace DafnyTests { - public class DafnyTests { + public class DafnyTestCase : IXunitSerializable { private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); private static string DAFNY_ROOT = OUTPUT_ROOT.Parent.Parent.Parent.Parent.FullName; - private static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; - private static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; + public static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; + public static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); + public string DafnyFile; + public string[] Arguments; + + // May be null if the test is only doing verification + private readonly string CompileTarget; + + // May be null if the test doesn't need to check the output + public string ExpectedOutputFile; + private readonly string ExpectedExitCode; + + private readonly bool SpecialCase = false; + + public DafnyTestCase(string dafnyFile, Dictionary config, string compileTarget) { + DafnyFile = dafnyFile; + var fullDafnyFilePath = Path.Combine(TEST_ROOT, DafnyFile); + CompileTarget = compileTarget; + + config.TryGetValue("expect", out ExpectedOutputFile); + config.Remove("expect"); + if (ExpectedOutputFile == null) { + ExpectedOutputFile = dafnyFile + ".expect"; + if (compileTarget != null) { + var specialCasePath = fullDafnyFilePath + "." + compileTarget + ".expect"; + if (File.Exists(specialCasePath)) { + SpecialCase = true; + ExpectedOutputFile = specialCasePath; + } + } + } else if (ExpectedOutputFile.Equals("no")) { + ExpectedOutputFile = null; + } + + Arguments = config.Select(ConfigPairToArgument).ToArray(); + + if (compileTarget != null) { + Arguments = Arguments.Concat(new[] {"/compileTarget:" + compileTarget}).ToArray(); + // Include any additional files + var additionalFilesPath = fullDafnyFilePath + "." + compileTarget + ".files"; + if (Directory.Exists(additionalFilesPath)) { + Arguments = Arguments.Concat(Directory.GetFiles(additionalFilesPath)).ToArray(); + } + } + } + + public DafnyTestCase() { + + } + + public void Serialize(IXunitSerializationInfo info) { + info.AddValue(nameof(DafnyFile), DafnyFile); + info.AddValue(nameof(Arguments), Arguments); + } + + public void Deserialize(IXunitSerializationInfo info) { + DafnyFile = info.GetValue(nameof(DafnyFile)); + Arguments = info.GetValue(nameof(Arguments)); + } + public static string RunDafny(IEnumerable arguments) { List dafnyArguments = new List { // Expected output does not contain logo @@ -89,104 +148,81 @@ private static string GetTestCaseConfigString(string filePath) { } } } - + return null; } - public static IEnumerable AllTestFiles() { - var filePaths = Directory.GetFiles(COMP_DIR, "*.dfy", SearchOption.AllDirectories) - .Select(path => GetRelativePath(TEST_ROOT, path)); - return filePaths.SelectMany(TestCasesForDafnyFile); - } - - private static IEnumerable TestCasesForDafnyFile(string filePath) { + + public static IEnumerable TestCasesForDafnyFile(string filePath) { var fullFilePath = Path.Combine(TEST_ROOT, filePath); string configString = GetTestCaseConfigString(fullFilePath); - IEnumerable configs; + IEnumerable> configs; if (configString != null) { var yamlStream = new YamlStream(); yamlStream.Load(new StringReader(configString)); var config = yamlStream.Documents[0].RootNode; - configs = YamlUtils.Expand(config); + configs = YamlUtils.Expand(config).Select(ToDictionary); } else { - configs = new[] {new YamlMappingNode()}; + configs = new[] { new Dictionary() }; } - IEnumerable mappings = configs.SelectMany(config => { - if (config is YamlMappingNode mapping) { - return ResolveCompile(filePath, mapping); - } else { - throw new ArgumentException("Bad test case configuration: " + config); - } - }); - - return mappings.Select(mapping => { - return new[] {filePath, String.Join(" ", mapping.Select(ConfigPairToArgument))}; - }); + return configs.SelectMany(config => ResolveCompile(filePath, config)) + .Select(t => new object[] {t.DafnyFile, t.Arguments, t.ExpectedOutputFile}); } - private static IEnumerable ResolveCompile(string filePath, YamlMappingNode mapping) { - if (!mapping.Children.ContainsKey(new YamlScalarNode("compile"))) { - mapping.Add("compile", "3"); + private static Dictionary ToDictionary(YamlNode node) { + if (node is YamlMappingNode mapping) { + return mapping.Children.ToDictionary(pair => pair.Key.ToString(), pair => pair.Value.ToString()); + } else { + throw new ArgumentException("Bad test case configuration: " + node); } - - if (mapping["compile"].Equals(new YamlScalarNode("3")) && !mapping.Children.ContainsKey("compileTarget")) { - var languages = new[] {"cs", "java", "go", "js"}; - foreach (var language in languages) { - var withLanguage = new YamlMappingNode(mapping.Children); - withLanguage.Add("compileTarget", language); - yield return withLanguage; + } + + private static IEnumerable ResolveCompile(string filePath, Dictionary config) { + var compile = "3"; + if (config.ContainsKey("compile")) { + compile = config["compile"]; + config.Remove("compile"); + } + + if (compile.Equals("3") && !config.ContainsKey("compileTarget")) { + var compileTargets = new[] {"cs", "java", "go", "js"}; + + var justVerify = new Dictionary(config); + justVerify["compile"] = "0"; + justVerify["expect"] = "no"; + yield return new DafnyTestCase(filePath, justVerify, null); + + foreach (var compileTarget in compileTargets) { + var withLanguage = new Dictionary(config); + withLanguage["noVerify"] = "yes"; + withLanguage["compile"] = "4"; + yield return new DafnyTestCase(filePath, withLanguage, compileTarget); } } else { - yield return mapping; + var compileTarget = config["compileTarget"]?.ToString(); + config["compile"] = compile; + yield return new DafnyTestCase(filePath, config, compileTarget); } } - private static string ConfigPairToArgument(KeyValuePair pair) { - if (pair.Key.Equals(new YamlScalarNode("otherFiles"))) { + private static string ConfigPairToArgument(KeyValuePair pair) { + if (pair.Key.Equals("otherFiles")) { return pair.Value.ToString(); + } else if (pair.Value.Equals("yes")) { + return String.Format("/{0}", pair.Key); } else { return String.Format("/{0}:{1}", pair.Key, pair.Value); } } - // TODO-RS: Replace with Path.GetRelativePath() if we move to .NET Core 3.1 - private static string GetRelativePath(string relativeTo, string path) { - var fullRelativeTo = Path.GetFullPath(relativeTo); - var fullPath = Path.GetFullPath(path); - Assert.StartsWith(fullRelativeTo, fullPath); - return fullPath.Substring(fullRelativeTo.Length); - } - - [ParallelTheory] - [MemberData(nameof(AllTestFiles))] - public void Test(string file, string args) { - string fullInputPath = Path.Combine(TEST_ROOT, file); - string[] arguments = args.Split(); - - string expectedOutputPath = fullInputPath + ".expect"; - bool specialCase = false; - string compileTarget = arguments.FirstOrDefault(arg => arg.StartsWith("/compileTarget:")); - if (compileTarget != null) { - string language = compileTarget.Substring("/compileTarget:".Length); - var specialCasePath = fullInputPath + "." + language + ".expect"; - if (File.Exists(specialCasePath)) { - specialCase = true; - expectedOutputPath = specialCasePath; - } - - // Include any additional files - var additionalFilesPath = fullInputPath + "." + language + ".files"; - if (Directory.Exists(additionalFilesPath)) { - arguments = arguments.Concat(Directory.GetFiles(additionalFilesPath)).ToArray(); - } - } - - var argumentsWithFile = new List {fullInputPath}.Concat(arguments); - var expectedOutput = File.ReadAllText(expectedOutputPath); + public void Run() { + string fullInputPath = Path.Combine(TEST_ROOT, DafnyFile); + var arguments = new []{ fullInputPath }.Concat(Arguments); + string output; if (arguments.Any(arg => arg.StartsWith("/out"))) { - output = RunDafny(argumentsWithFile); + output = RunDafny(arguments); } else { // Note that the temporary directory has to be an ancestor of Test // or else Javascript won't be able to locate bignumber.js :( @@ -195,13 +231,44 @@ public void Test(string file, string args) { // temporary directory, since some compilers will // interpret the path as a single file basename rather than a directory. var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; - argumentsWithFile = new List {outArgument}.Concat(argumentsWithFile); - output = RunDafny(argumentsWithFile); + arguments = new []{ outArgument }.Concat(arguments); + output = RunDafny(arguments); } } - AssertWithDiff.Equal(expectedOutput, output); - Skip.If(specialCase, "Confirmed known exception for arguments: " + args); + if (ExpectedOutputFile != null) { + var expectedOutput = File.ReadAllText(Path.Combine(TEST_ROOT, ExpectedOutputFile)); + AssertWithDiff.Equal(expectedOutput, output); + } + + Skip.If(SpecialCase, "Confirmed known exception for arguments: " + arguments); + } + } + + public class DafnyTests { + + public static IEnumerable AllTestFiles() { + var filePaths = Directory.GetFiles(DafnyTestCase.COMP_DIR, "*.dfy", SearchOption.AllDirectories) + .Select(path => GetRelativePath(DafnyTestCase.TEST_ROOT, path)); + return filePaths.SelectMany(DafnyTestCase.TestCasesForDafnyFile); + } + + // TODO-RS: Replace with Path.GetRelativePath() if we move to .NET Core 3.1 + private static string GetRelativePath(string relativeTo, string path) { + var fullRelativeTo = Path.GetFullPath(relativeTo); + var fullPath = Path.GetFullPath(path); + Assert.StartsWith(fullRelativeTo, fullPath); + return fullPath.Substring(fullRelativeTo.Length); + } + + [ParallelTheory] + [MemberData(nameof(AllTestFiles))] + public static void Test(string dafnyFile, string[] arguments, string expectedOutputFile) { + var testCase = new DafnyTestCase(); + testCase.DafnyFile = dafnyFile; + testCase.Arguments = arguments; + testCase.ExpectedOutputFile = expectedOutputFile; + testCase.Run(); } } } \ No newline at end of file diff --git a/Source/DafnyTests/DafnyTests-NetCore.csproj b/Source/DafnyTests/DafnyTests-NetCore.csproj index 6878a6f1ac3..df2de859775 100644 --- a/Source/DafnyTests/DafnyTests-NetCore.csproj +++ b/Source/DafnyTests/DafnyTests-NetCore.csproj @@ -7,6 +7,7 @@ + @@ -15,6 +16,42 @@ + + <_UnmanagedRegistrationCache Remove="obj_core\DafnyTests-NetCore.csproj.UnmanagedRegistration.cache" /> + + + + <_ResolveComReferenceCache Remove="obj_core\Debug\netcoreapp2.1\DafnyTests-NetCore.csproj.ResolveComReference.cache" /> + + + + + + + + <_DebugSymbolsIntermediatePath Remove="obj_core\Debug\netcoreapp2.1\DafnyTests.pdb" /> + + + + <_DeploymentManifestEntryPoint Remove="obj_core\Debug\netcoreapp2.1\DafnyTests.dll" /> + + + + + + + + + + + + + + + + + + diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 7d09bea4af9..52f267e52ce 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -15,13 +15,6 @@ public static class LitTestConvertor { private const string DAFNY_COMPILE_TARGET = "/compileTarget:"; private static readonly string[] IGNORED_DAFNY_COMMAND_ARGUMENTS = { - "/print:\"%t.print\"", - "/dprint:\"%t.dprint\"", - "/dprint:\"%t.dfy\"", - "/rprint:\"%t.rprint\"", - "/rprint:\"%t.dprint\"", - "/dprint:\"%t.dprint.dfy\"", - "\"%s\"", ">", ">>", "\"%t\"" }; @@ -33,6 +26,7 @@ public static class LitTestConvertor { "/definiteAssignment", "/dprint", "/env", + "/errorLimit", "/errorTrace", "/ironDafny", "/noNLarith", @@ -41,6 +35,7 @@ public static class LitTestConvertor { "/optimizeResolution", "/print", "/printTooltips", + "/restartProver", "/rprint", "/traceCaching", "/tracePOs", diff --git a/Test/comp/Arrays.dfy.expect b/Test/comp/Arrays.dfy.expect index 3507348bea1..30a2d9d1911 100644 --- a/Test/comp/Arrays.dfy.expect +++ b/Test/comp/Arrays.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 9 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 17 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] diff --git a/Test/comp/BranchCoverage.dfy.expect b/Test/comp/BranchCoverage.dfy.expect index ce18dfad615..1eac6ec531a 100644 --- a/Test/comp/BranchCoverage.dfy.expect +++ b/Test/comp/BranchCoverage.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 6 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor 1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled 2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled diff --git a/Test/comp/CSharpStyling.dfy.expect b/Test/comp/CSharpStyling.dfy.expect index ad4acd57b94..90d9435e11b 100644 --- a/Test/comp/CSharpStyling.dfy.expect +++ b/Test/comp/CSharpStyling.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 1 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 50 2 3 diff --git a/Test/comp/Calls.dfy.expect b/Test/comp/Calls.dfy.expect index c9d9c3e9319..9aea927c251 100644 --- a/Test/comp/Calls.dfy.expect +++ b/Test/comp/Calls.dfy.expect @@ -1,3 +1,3 @@ -Dafny program verifier finished with 1 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 5 0 0 false 0 false 0 diff --git a/Test/comp/Class.dfy.expect b/Test/comp/Class.dfy.expect index d60d6ef7059..640a7391836 100644 --- a/Test/comp/Class.dfy.expect +++ b/Test/comp/Class.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 13 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors true true false 103 103 103 106 106 106 203 17 0 18 8 9 69 70 diff --git a/Test/comp/Collections.dfy.cs.expect b/Test/comp/Collections.dfy.cs.expect index 53a0cea6ebd..f9215367c73 100644 --- a/Test/comp/Collections.dfy.cs.expect +++ b/Test/comp/Collections.dfy.cs.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 17 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Sets: {} {17, 82} {12, 17} cardinality: 0 2 2 union: {17, 82} {12, 17, 82} diff --git a/Test/comp/Collections.dfy.expect b/Test/comp/Collections.dfy.expect index e46a75199ec..0779544d8d0 100644 --- a/Test/comp/Collections.dfy.expect +++ b/Test/comp/Collections.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 17 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Sets: {} {17, 82} {12, 17} cardinality: 0 2 2 union: {17, 82} {17, 82, 12} diff --git a/Test/comp/Collections.dfy.java.expect b/Test/comp/Collections.dfy.java.expect index bb98d940a70..e2181995389 100644 --- a/Test/comp/Collections.dfy.java.expect +++ b/Test/comp/Collections.dfy.java.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 17 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Sets: {} {17, 82} {17, 12} cardinality: 0 2 2 union: {17, 82} {17, 82, 12} diff --git a/Test/comp/Comprehensions.dfy.expect b/Test/comp/Comprehensions.dfy.expect index a6b302c2f34..c0ecb78560d 100644 --- a/Test/comp/Comprehensions.dfy.expect +++ b/Test/comp/Comprehensions.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 16 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors x=13 y=14 x=13 y=14 b=yes p=(13, 14) diff --git a/Test/comp/Comprehensions.dfy.java.expect b/Test/comp/Comprehensions.dfy.java.expect index ffe220bc356..0480a4aeae8 100644 --- a/Test/comp/Comprehensions.dfy.java.expect +++ b/Test/comp/Comprehensions.dfy.java.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 16 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors x=13 y=14 x=13 y=14 b=[y, e, s] p=(13, 14) diff --git a/Test/comp/CovariantCollections.dfy.expect b/Test/comp/CovariantCollections.dfy.expect index f504f38e3af..82ad857a44b 100644 --- a/Test/comp/CovariantCollections.dfy.expect +++ b/Test/comp/CovariantCollections.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 2 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Sequences: [] [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer] cardinality: 0 4 2 update: [_module.Integer, _module.Integer, _module.Integer, _module.Integer] [_module.Integer, _module.Integer] diff --git a/Test/comp/CovariantCollections.dfy.java.expect b/Test/comp/CovariantCollections.dfy.java.expect index 5883880f24d..15e182eac73 100644 --- a/Test/comp/CovariantCollections.dfy.java.expect +++ b/Test/comp/CovariantCollections.dfy.java.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 2 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors CovariantCollections.dfy(28,6): Error: compilation of seq is not supported; consider introducing a ghost CovariantCollections.dfy(29,6): Error: compilation of seq is not supported; consider introducing a ghost File Program/Program.java contains the partially compiled program diff --git a/Test/comp/Dt.dfy.expect b/Test/comp/Dt.dfy.expect index 01f48244ae3..bf3cd8f6e31 100644 --- a/Test/comp/Dt.dfy.expect +++ b/Test/comp/Dt.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 12 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors List.Nil List.Cons(5, List.Nil) (List.Nil, List.Cons(5, List.Nil)) diff --git a/Test/comp/Dt.dfy.java.expect b/Test/comp/Dt.dfy.java.expect index a47e9c0e7d4..0b97de481fc 100644 --- a/Test/comp/Dt.dfy.java.expect +++ b/Test/comp/Dt.dfy.java.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 12 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors List.Nil List.Cons(5, List.Nil) (List.Nil, List.Cons(5, List.Nil)) diff --git a/Test/comp/Extern.dfy.expect b/Test/comp/Extern.dfy.expect index f3b7991bdaf..e58765e42de 100644 --- a/Test/comp/Extern.dfy.expect +++ b/Test/comp/Extern.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 6 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Hello 31 62 45 OtherClass.CallMe AllDafny.M diff --git a/Test/comp/ExternCtors.dfy.expect b/Test/comp/ExternCtors.dfy.expect index 1bcf5332a10..475f85604f1 100644 --- a/Test/comp/ExternCtors.dfy.expect +++ b/Test/comp/ExternCtors.dfy.expect @@ -1,4 +1,4 @@ -Dafny program verifier finished with 1 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Hello! My value is 42 diff --git a/Test/comp/Forall.dfy.expect b/Test/comp/Forall.dfy.expect index c595c63df8e..0aab1261432 100644 --- a/Test/comp/Forall.dfy.expect +++ b/Test/comp/Forall.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 25 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Arrays: Basic cases [0, 1, 2, 3, 4] [0, 1, 2, 3, 4] diff --git a/Test/comp/Ghosts.dfy.expect b/Test/comp/Ghosts.dfy.expect index 38bd2cd6709..ce70626b69f 100644 --- a/Test/comp/Ghosts.dfy.expect +++ b/Test/comp/Ghosts.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 8 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors hello, M2 hello, M2 hello, M3 diff --git a/Test/comp/GoModule.dfy.expect b/Test/comp/GoModule.dfy.expect index cb4ac018965..3f5c3c9d0cc 100644 --- a/Test/comp/GoModule.dfy.expect +++ b/Test/comp/GoModule.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 2 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors The address http://localhost:8080/default.htm?year=1915&month=august&day=29 has the following parts: host: localhost:8080 diff --git a/Test/comp/Iterators.dfy.expect b/Test/comp/Iterators.dfy.expect index 86d1b673939..d276f6118fa 100644 --- a/Test/comp/Iterators.dfy.expect +++ b/Test/comp/Iterators.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 5 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors hello, instance x is 0 0.0 1.0 2.0 3.0 4.0 5.0 diff --git a/Test/comp/Let.dfy.expect b/Test/comp/Let.dfy.expect index 2c1728235f1..ac2c0c88017 100644 --- a/Test/comp/Let.dfy.expect +++ b/Test/comp/Let.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 3 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 50 58 Tree.Node(Tree.Node(Tree.Leaf, 5, Tree.Node(Tree.Leaf, 7, Tree.Leaf)), 9, Tree.Node(Tree.Leaf, 10, Tree.Node(Tree.Leaf, 12, Tree.Node(Tree.Leaf, 30, Tree.Leaf)))) 5 7 9 10 12 30 diff --git a/Test/comp/Module.dfy.expect b/Test/comp/Module.dfy.expect index d2b3cafd1c9..649b27bc62e 100644 --- a/Test/comp/Module.dfy.expect +++ b/Test/comp/Module.dfy.expect @@ -1,3 +1,3 @@ -Dafny program verifier finished with 1 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors hi from a nested module diff --git a/Test/comp/NativeNumbers.dfy.expect b/Test/comp/NativeNumbers.dfy.expect index bcbb4d5ca89..35a343942cd 100644 --- a/Test/comp/NativeNumbers.dfy.expect +++ b/Test/comp/NativeNumbers.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 25 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Casting: Small numbers: diff --git a/Test/comp/Numbers.dfy.expect b/Test/comp/Numbers.dfy.expect index 11d8d6ac08b..4a364f1430a 100644 --- a/Test/comp/Numbers.dfy.expect +++ b/Test/comp/Numbers.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 29 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 0 0 3 diff --git a/Test/comp/Numbers.dfy.go.expect b/Test/comp/Numbers.dfy.go.expect index 8f76e618253..351c9c97688 100644 --- a/Test/comp/Numbers.dfy.go.expect +++ b/Test/comp/Numbers.dfy.go.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 29 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 0 0 3 diff --git a/Test/comp/Poly.dfy.cs.expect b/Test/comp/Poly.dfy.cs.expect index 232c15bf822..0b1b05fc2cb 100644 --- a/Test/comp/Poly.dfy.cs.expect +++ b/Test/comp/Poly.dfy.cs.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 9 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Poly.dfy(59,16): Error: compilation of set is not supported; consider introducing a ghost Poly.dfy(63,21): Error: compilation of multiset is not supported; consider introducing a ghost Poly.dfy(80,13): Error: compilation of set is not supported; consider introducing a ghost diff --git a/Test/comp/Poly.dfy.expect b/Test/comp/Poly.dfy.expect index 9ee53ecbe63..c9e163a80e1 100644 --- a/Test/comp/Poly.dfy.expect +++ b/Test/comp/Poly.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 9 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Center of square: (0.5, 0.5) Center of circle: (1.0, 1.0) Center: (0.5, 0.5) diff --git a/Test/comp/Poly.dfy.java.expect b/Test/comp/Poly.dfy.java.expect index cc1ae3008a1..a96b1e5eed5 100644 --- a/Test/comp/Poly.dfy.java.expect +++ b/Test/comp/Poly.dfy.java.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 9 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Poly.dfy(48,16): Error: compilation of seq is not supported; consider introducing a ghost Poly.dfy(59,16): Error: compilation of set is not supported; consider introducing a ghost Poly.dfy(63,21): Error: compilation of multiset is not supported; consider introducing a ghost diff --git a/Test/comp/Poly.dfy.js.expect b/Test/comp/Poly.dfy.js.expect index 31bc5b17af1..2af9bb1e22d 100644 --- a/Test/comp/Poly.dfy.js.expect +++ b/Test/comp/Poly.dfy.js.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 9 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors Center of square: ((1.0 / 2.0), (1.0 / 2.0)) Center of circle: (1.0, 1.0) Center: ((1.0 / 2.0), (1.0 / 2.0)) diff --git a/Test/comp/StaticMembersOfGenericTypes.dfy.expect b/Test/comp/StaticMembersOfGenericTypes.dfy.expect index f07a820f11f..d4e2ae816a0 100644 --- a/Test/comp/StaticMembersOfGenericTypes.dfy.expect +++ b/Test/comp/StaticMembersOfGenericTypes.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 9 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors (0, 1) (true, 2, 3) 0 25 (20, 21) (20, 21) 23 22 0 25 (20, 21) (20, 21) 23 22 diff --git a/Test/comp/TailRecursion.dfy.expect b/Test/comp/TailRecursion.dfy.expect index 886da966a21..14631b26260 100644 --- a/Test/comp/TailRecursion.dfy.expect +++ b/Test/comp/TailRecursion.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 20 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 2000000 2000000 2000000 2000000 1000000 1000000 diff --git a/Test/comp/TailRecursion.dfy.go.expect b/Test/comp/TailRecursion.dfy.go.expect index e429e2b0a83..ff45438c853 100644 --- a/Test/comp/TailRecursion.dfy.go.expect +++ b/Test/comp/TailRecursion.dfy.go.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 20 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 2000000 2000000 2000000 2000000 1000000 1000000 diff --git a/Test/comp/TailRecursion.dfy.js.expect b/Test/comp/TailRecursion.dfy.js.expect index e429e2b0a83..ff45438c853 100644 --- a/Test/comp/TailRecursion.dfy.js.expect +++ b/Test/comp/TailRecursion.dfy.js.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 20 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 2000000 2000000 2000000 2000000 1000000 1000000 diff --git a/Test/comp/TypeParams.dfy.cs.expect b/Test/comp/TypeParams.dfy.cs.expect index c9e307ce060..a1906ead70c 100644 --- a/Test/comp/TypeParams.dfy.cs.expect +++ b/Test/comp/TypeParams.dfy.cs.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 21 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) null null null null null {} multiset{} [] map[] {} map[] diff --git a/Test/comp/TypeParams.dfy.go.expect b/Test/comp/TypeParams.dfy.go.expect index a6895aa8361..c6a6c186a0d 100644 --- a/Test/comp/TypeParams.dfy.go.expect +++ b/Test/comp/TypeParams.dfy.go.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 21 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) null null null null null {} multiset{} [] map[] {} map[] diff --git a/Test/comp/TypeParams.dfy.java.expect b/Test/comp/TypeParams.dfy.java.expect index c28d92c1738..42ffdaa9640 100644 --- a/Test/comp/TypeParams.dfy.java.expect +++ b/Test/comp/TypeParams.dfy.java.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 21 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) null null null null null {} multiset{} [] map[] {} map[] diff --git a/Test/comp/TypeParams.dfy.js.expect b/Test/comp/TypeParams.dfy.js.expect index 0545d716adb..a2049a5c46e 100644 --- a/Test/comp/TypeParams.dfy.js.expect +++ b/Test/comp/TypeParams.dfy.js.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 21 verified, 0 errors +Dafny program verifier finished with 0 verified, 0 errors 0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) null null null null null {} multiset{} [] map[] {} map[] From 55b7d7636828ced6bb70062db2fd57167016cac3 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 10 Jul 2020 07:30:01 -0700 Subject: [PATCH 050/192] Fix solution reference in build --- .github/workflows/msbuild.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index a0ea7370d47..5d230add877 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -46,10 +46,10 @@ jobs: unzip z3*.zip && rm *.zip cp -r z3* dafny/Binaries/z3 - name: Nuget Restore Dafny - run: nuget restore dafny/Dafny.sln + run: nuget restore dafny/Source/Dafny.sln - name: Build Dafny - run: msbuild dafny/Dafny.sln + run: msbuild dafny/Source/Dafny.sln - uses: actions/setup-node@v1 - run: npm install bignumber.js - name: Run Dafny tests - run: XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 msbuild -t:Test dafny/Test/DafnyTests/DafnyTests.csproj + run: XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 msbuild -t:Test dafny/Source/DafnyTests/DafnyTests.csproj From b6bdb52d42e6961a3e03e893cb3d73af644f2e8d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 10 Jul 2020 09:44:02 -0700 Subject: [PATCH 051/192] Fix CSharpStyling.dfy.expect --- Source/DafnyTests/DafnyTests-NetCore.csproj | 37 --------------------- Test/comp/CSharpStyling.dfy.expect | 2 +- 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/Source/DafnyTests/DafnyTests-NetCore.csproj b/Source/DafnyTests/DafnyTests-NetCore.csproj index df2de859775..9c7afa74073 100644 --- a/Source/DafnyTests/DafnyTests-NetCore.csproj +++ b/Source/DafnyTests/DafnyTests-NetCore.csproj @@ -6,7 +6,6 @@ - @@ -16,42 +15,6 @@ - - <_UnmanagedRegistrationCache Remove="obj_core\DafnyTests-NetCore.csproj.UnmanagedRegistration.cache" /> - - - - <_ResolveComReferenceCache Remove="obj_core\Debug\netcoreapp2.1\DafnyTests-NetCore.csproj.ResolveComReference.cache" /> - - - - - - - - <_DebugSymbolsIntermediatePath Remove="obj_core\Debug\netcoreapp2.1\DafnyTests.pdb" /> - - - - <_DeploymentManifestEntryPoint Remove="obj_core\Debug\netcoreapp2.1\DafnyTests.dll" /> - - - - - - - - - - - - - - - - - - diff --git a/Test/comp/CSharpStyling.dfy.expect b/Test/comp/CSharpStyling.dfy.expect index 90d9435e11b..ad4acd57b94 100644 --- a/Test/comp/CSharpStyling.dfy.expect +++ b/Test/comp/CSharpStyling.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier finished with 0 verified, 0 errors +Dafny program verifier finished with 1 verified, 0 errors 50 2 3 From 1d9f6de3c701158a67a729144d88aff067ddd5b0 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 10 Jul 2020 12:54:37 -0700 Subject: [PATCH 052/192] Improved path handling between frameworks --- Source/DafnyTests/DafnyTestCase.cs | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 0b2220760e7..92f440bba7a 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Runtime.InteropServices; using System.Text; using DiffPlex; using DiffPlex.DiffBuilder; @@ -20,20 +21,24 @@ namespace DafnyTests { public class DafnyTestCase : IXunitSerializable { private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); - private static string DAFNY_ROOT = OUTPUT_ROOT.Parent.Parent.Parent.Parent.FullName; + private static bool IS_NET_CORE => RuntimeInformation.FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase); + // TODO-RS: This is an ugly method of locating the project root - .NET Core happens to + // have the current directly one level deeper. The proper fix is to run entirely out of + // the output directory, and the projects are at least partially configured to make that possible, + // but it's not quite working yet. + private static string DAFNY_ROOT = IS_NET_CORE ? + OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName : + OUTPUT_ROOT.Parent.Parent.Parent.Parent.FullName; - public static string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; + public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; public static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - + private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); public string DafnyFile; public string[] Arguments; - // May be null if the test is only doing verification - private readonly string CompileTarget; - // May be null if the test doesn't need to check the output public string ExpectedOutputFile; private readonly string ExpectedExitCode; @@ -43,15 +48,14 @@ public class DafnyTestCase : IXunitSerializable { public DafnyTestCase(string dafnyFile, Dictionary config, string compileTarget) { DafnyFile = dafnyFile; var fullDafnyFilePath = Path.Combine(TEST_ROOT, DafnyFile); - CompileTarget = compileTarget; config.TryGetValue("expect", out ExpectedOutputFile); config.Remove("expect"); if (ExpectedOutputFile == null) { ExpectedOutputFile = dafnyFile + ".expect"; if (compileTarget != null) { - var specialCasePath = fullDafnyFilePath + "." + compileTarget + ".expect"; - if (File.Exists(specialCasePath)) { + var specialCasePath = dafnyFile + "." + compileTarget + ".expect"; + if (File.Exists(Path.Combine(TEST_ROOT, specialCasePath))) { SpecialCase = true; ExpectedOutputFile = specialCasePath; } @@ -67,7 +71,9 @@ public DafnyTestCase(string dafnyFile, Dictionary config, string // Include any additional files var additionalFilesPath = fullDafnyFilePath + "." + compileTarget + ".files"; if (Directory.Exists(additionalFilesPath)) { - Arguments = Arguments.Concat(Directory.GetFiles(additionalFilesPath)).ToArray(); + var relativePaths = Directory.GetFiles(additionalFilesPath) + .Select(path => DafnyTests.GetRelativePath(TEST_ROOT, path)); + Arguments = Arguments.Concat(relativePaths).ToArray(); } } } @@ -199,7 +205,7 @@ private static IEnumerable ResolveCompile(string filePath, Dictio yield return new DafnyTestCase(filePath, withLanguage, compileTarget); } } else { - var compileTarget = config["compileTarget"]?.ToString(); + config.TryGetValue("compileTarget", out var compileTarget); config["compile"] = compile; yield return new DafnyTestCase(filePath, config, compileTarget); } @@ -254,7 +260,7 @@ public static IEnumerable AllTestFiles() { } // TODO-RS: Replace with Path.GetRelativePath() if we move to .NET Core 3.1 - private static string GetRelativePath(string relativeTo, string path) { + public static string GetRelativePath(string relativeTo, string path) { var fullRelativeTo = Path.GetFullPath(relativeTo); var fullPath = Path.GetFullPath(path); Assert.StartsWith(fullRelativeTo, fullPath); From 350ac23e5ccf2c6d9b915fff9c5722cb164b0d8e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 12 Jul 2020 10:29:59 -0700 Subject: [PATCH 053/192] Switch back to single test argument (working now?) --- Source/Dafny/DafnyOptions.cs | 1 + Source/DafnyTests/DafnyTestCase.cs | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Source/Dafny/DafnyOptions.cs b/Source/Dafny/DafnyOptions.cs index 8082411fde0..5a5c2b82c75 100644 --- a/Source/Dafny/DafnyOptions.cs +++ b/Source/Dafny/DafnyOptions.cs @@ -60,6 +60,7 @@ public static void Install(DafnyOptions options) { public enum PrintModes { Everything, DllEmbed, NoIncludes, NoGhost }; public PrintModes PrintMode = PrintModes.Everything; // Default to printing everything public bool DafnyVerify = true; + public bool VerifyVerbose = true; public string DafnyPrintResolvedFile = null; public List DafnyPrintExportedViews = new List(); public bool Compile = true; diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 92f440bba7a..ad90e281afa 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -172,7 +172,8 @@ public static IEnumerable TestCasesForDafnyFile(string filePath) { } return configs.SelectMany(config => ResolveCompile(filePath, config)) - .Select(t => new object[] {t.DafnyFile, t.Arguments, t.ExpectedOutputFile}); +// .Select(t => new object[] {t.DafnyFile, t.Arguments, t.ExpectedOutputFile}); + .Select(t => new object[] {t}); } private static Dictionary ToDictionary(YamlNode node) { @@ -269,11 +270,12 @@ public static string GetRelativePath(string relativeTo, string path) { [ParallelTheory] [MemberData(nameof(AllTestFiles))] - public static void Test(string dafnyFile, string[] arguments, string expectedOutputFile) { - var testCase = new DafnyTestCase(); - testCase.DafnyFile = dafnyFile; - testCase.Arguments = arguments; - testCase.ExpectedOutputFile = expectedOutputFile; + public static void Test(DafnyTestCase testCase) { +// public static void Test(string dafnyFile, string[] arguments, string expectedOutputFile) { +// var testCase = new DafnyTestCase(); +// testCase.DafnyFile = dafnyFile; +// testCase.Arguments = arguments; +// testCase.ExpectedOutputFile = expectedOutputFile; testCase.Run(); } } From e098b39b98d04569800a055feca20b239950d2c7 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Sep 2020 16:06:47 -0700 Subject: [PATCH 054/192] Add DafnyTests project back --- Source/Dafny.sln | 26 +++++ Source/DafnyTests/DafnyTests-NetCore.csproj | 23 ---- Source/DafnyTests/DafnyTests.csproj | 110 ++++---------------- 3 files changed, 47 insertions(+), 112 deletions(-) delete mode 100644 Source/DafnyTests/DafnyTests-NetCore.csproj diff --git a/Source/Dafny.sln b/Source/Dafny.sln index 9d9003d4247..7f0583b77e2 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -48,6 +48,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution version.cs = version.cs EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests", "DafnyTests\DafnyTests.csproj", "{08A3665B-9854-4CF6-BF5F-C532DA39A043}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Checked|.NET = Checked|.NET @@ -520,6 +522,30 @@ Global {71208DB9-31D6-4071-838B-8D44A37CF0F1}.Release|Mixed Platforms.Build.0 = Release|Any CPU {71208DB9-31D6-4071-838B-8D44A37CF0F1}.Release|x86.ActiveCfg = Release|Any CPU {71208DB9-31D6-4071-838B-8D44A37CF0F1}.Release|x86.Build.0 = Release|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Checked|.NET.ActiveCfg = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Checked|.NET.Build.0 = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Checked|Any CPU.Build.0 = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Checked|Mixed Platforms.ActiveCfg = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Checked|Mixed Platforms.Build.0 = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Checked|x86.ActiveCfg = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Checked|x86.Build.0 = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Debug|.NET.ActiveCfg = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Debug|.NET.Build.0 = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Debug|Any CPU.Build.0 = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Debug|x86.ActiveCfg = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Debug|x86.Build.0 = Debug|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|.NET.ActiveCfg = Release|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|.NET.Build.0 = Release|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|Any CPU.ActiveCfg = Release|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|Any CPU.Build.0 = Release|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|x86.ActiveCfg = Release|Any CPU + {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Source/DafnyTests/DafnyTests-NetCore.csproj b/Source/DafnyTests/DafnyTests-NetCore.csproj deleted file mode 100644 index 9c7afa74073..00000000000 --- a/Source/DafnyTests/DafnyTests-NetCore.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - netcoreapp2.1 - DafnyTests - - - - - - - - - - - - - - - - - - diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index 169a9a8e67e..e1041a092e0 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -1,91 +1,23 @@ - - - - - Debug - AnyCPU - {45AE3DDD-D0EA-4A27-943B-AC66A581BE03} - {FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} - Library - Properties - DafnyTests - DafnyTests - v4.8 - 512 - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - CS0162 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - - - - - - - - - - - true - - - - - - - - - Test\%(RecursiveDir)%(Filename)%(Extension) - - - YamlTests\%(RecursiveDir)%(Filename)%(Extension) - - - z3\%(RecursiveDir)%(Filename)%(Extension) - - - - - - - - - - - - - + + - This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105.The missing file is {0}. + netcoreapp3.1 + DafnyTests - - - - - - \ No newline at end of file + + + + + + + + + + + + + + + + + From e132a82f392a0ffb3213b5fb89fa1dec1aa7abab Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Sep 2020 16:39:46 -0700 Subject: [PATCH 055/192] Fix reference to Dafny binary --- Source/DafnyTests/DafnyTestCase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index ad90e281afa..9cc9c70ba32 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -34,7 +34,7 @@ public class DafnyTestCase : IXunitSerializable { public static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.exe"); + private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.dll"); public string DafnyFile; public string[] Arguments; From 16b17b9d56a45910ab10666ef2eccecd114fcc94 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 28 Sep 2020 17:31:19 -0700 Subject: [PATCH 056/192] Better reference to Dafny binary May need something better to work on all platforms --- Source/DafnyTests/DafnyTestCase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 9cc9c70ba32..4011684d603 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -34,7 +34,7 @@ public class DafnyTestCase : IXunitSerializable { public static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/Dafny.dll"); + private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/dafny"); public string DafnyFile; public string[] Arguments; @@ -108,8 +108,8 @@ public static string RunDafny(IEnumerable arguments) { dafnyArguments.AddRange(arguments); using (Process dafnyProcess = new Process()) { - dafnyProcess.StartInfo.FileName = "mono"; - dafnyProcess.StartInfo.Arguments += DAFNY_EXE; +// dafnyProcess.StartInfo.FileName = "mono"; + dafnyProcess.StartInfo.FileName = DAFNY_EXE; foreach (var argument in dafnyArguments) { dafnyProcess.StartInfo.Arguments += " " + argument; } From bcdff33580cc882b87604b5110af8902c75c629c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 29 Sep 2020 14:11:43 -0700 Subject: [PATCH 057/192] Lots of fixes to the framework, updating expectations --- Source/Dafny.sln | 26 ++++ Source/DafnyTests/DafnyTestCase.cs | 131 ++++++++++-------- .../LitTestConverter/LitTestConverter.csproj | 51 +------ Test/comp/AutoInit.dfy | 7 - Test/comp/AutoInit.dfy.expect | 38 ----- Test/comp/Comprehensions.dfy.expect | 23 +++ Test/comp/Comprehensions.dfy.java.expect | 25 +++- .../comp/CovariantCollections.dfy.java.expect | 5 - Test/comp/Numbers.dfy.go.expect | 10 ++ Test/comp/Poly.dfy.cs.expect | 6 - Test/comp/Poly.dfy.expect | 12 +- Test/comp/Poly.dfy.go.expect | 14 ++ Test/comp/Poly.dfy.java.expect | 6 - Test/comp/Poly.dfy.js.expect | 10 -- Test/comp/TypeParams.dfy.cs.expect | 2 +- 15 files changed, 180 insertions(+), 186 deletions(-) delete mode 100644 Test/comp/CovariantCollections.dfy.java.expect delete mode 100644 Test/comp/Poly.dfy.cs.expect create mode 100644 Test/comp/Poly.dfy.go.expect delete mode 100644 Test/comp/Poly.dfy.java.expect delete mode 100644 Test/comp/Poly.dfy.js.expect diff --git a/Source/Dafny.sln b/Source/Dafny.sln index 7f0583b77e2..03140937735 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -50,6 +50,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests", "DafnyTests\DafnyTests.csproj", "{08A3665B-9854-4CF6-BF5F-C532DA39A043}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LitTestConverter", "LitTestConverter\LitTestConverter.csproj", "{2EA7D80A-B179-4352-BDF2-8A1599F8685A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Checked|.NET = Checked|.NET @@ -546,6 +548,30 @@ Global {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|Mixed Platforms.Build.0 = Release|Any CPU {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|x86.ActiveCfg = Release|Any CPU {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|x86.Build.0 = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|.NET.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|.NET.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Any CPU.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Mixed Platforms.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Mixed Platforms.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|x86.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|x86.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|.NET.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|.NET.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|x86.ActiveCfg = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|x86.Build.0 = Debug|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|.NET.ActiveCfg = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|.NET.Build.0 = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Any CPU.Build.0 = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|x86.ActiveCfg = Release|Any CPU + {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 4011684d603..7fa9fe49fc5 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -18,17 +18,16 @@ namespace DafnyTests { - public class DafnyTestCase : IXunitSerializable { + public class DafnyTestCase { private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); - private static bool IS_NET_CORE => RuntimeInformation.FrameworkDescription.StartsWith(".NET Core", StringComparison.OrdinalIgnoreCase); - // TODO-RS: This is an ugly method of locating the project root - .NET Core happens to - // have the current directly one level deeper. The proper fix is to run entirely out of - // the output directory, and the projects are at least partially configured to make that possible, + + // TODO-RS: This is an ugly method of locating the project root. + // The proper fix is to run entirely out of the output directory, + // and the projects are at least partially configured to make that possible, // but it's not quite working yet. - private static string DAFNY_ROOT = IS_NET_CORE ? - OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName : - OUTPUT_ROOT.Parent.Parent.Parent.Parent.FullName; + private static string DAFNY_ROOT = + OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; public static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; @@ -38,41 +37,70 @@ public class DafnyTestCase : IXunitSerializable { public string DafnyFile; public string[] Arguments; - - // May be null if the test doesn't need to check the output - public string ExpectedOutputFile; - private readonly string ExpectedExitCode; - private readonly bool SpecialCase = false; + public class Expectation : IXunitSerializable { + // May be null if the test doesn't need to check the output + public string OutputFile; + public int ExitCode = 0; - public DafnyTestCase(string dafnyFile, Dictionary config, string compileTarget) { - DafnyFile = dafnyFile; - var fullDafnyFilePath = Path.Combine(TEST_ROOT, DafnyFile); + public bool SpecialCase = false; - config.TryGetValue("expect", out ExpectedOutputFile); - config.Remove("expect"); - if (ExpectedOutputFile == null) { - ExpectedOutputFile = dafnyFile + ".expect"; - if (compileTarget != null) { - var specialCasePath = dafnyFile + "." + compileTarget + ".expect"; - if (File.Exists(Path.Combine(TEST_ROOT, specialCasePath))) { - SpecialCase = true; - ExpectedOutputFile = specialCasePath; + public Expectation(string dafnyFile, Dictionary config, string compileTarget) { + config.TryGetValue("expect", out OutputFile); + config.Remove("expect"); + if (OutputFile == null) { + OutputFile = dafnyFile + ".expect"; + if (compileTarget != null) { + var specialCasePath = dafnyFile + "." + compileTarget + ".expect"; + if (File.Exists(Path.Combine(TEST_ROOT, specialCasePath))) { + SpecialCase = true; + OutputFile = specialCasePath; + } } + } else if (OutputFile.Equals("no")) { + OutputFile = null; + } + } + + public Expectation() { + + } + + public void Deserialize(IXunitSerializationInfo info) { + OutputFile = info.GetValue(nameof(OutputFile)); + ExitCode = info.GetValue(nameof(ExitCode)); + SpecialCase = info.GetValue(nameof(SpecialCase)); + } + + public void Serialize(IXunitSerializationInfo info) { + if (OutputFile != null) { + info.AddValue(nameof(OutputFile), OutputFile); + } + if (ExitCode != 0) { + info.AddValue(nameof(ExitCode), ExitCode); + } + if (SpecialCase) { + info.AddValue(nameof(SpecialCase), true); } - } else if (ExpectedOutputFile.Equals("no")) { - ExpectedOutputFile = null; } + } + public Expectation Expected; + + public DafnyTestCase(string dafnyFile, Dictionary config, string compileTarget) { + Expected = new Expectation(dafnyFile, config, compileTarget); + + DafnyFile = dafnyFile; Arguments = config.Select(ConfigPairToArgument).ToArray(); + var fullDafnyFilePath = Path.Combine(TEST_ROOT, DafnyFile); if (compileTarget != null) { Arguments = Arguments.Concat(new[] {"/compileTarget:" + compileTarget}).ToArray(); // Include any additional files var additionalFilesPath = fullDafnyFilePath + "." + compileTarget + ".files"; if (Directory.Exists(additionalFilesPath)) { var relativePaths = Directory.GetFiles(additionalFilesPath) - .Select(path => DafnyTests.GetRelativePath(TEST_ROOT, path)); + .Select(path => Path.GetRelativePath(TEST_ROOT, path)); Arguments = Arguments.Concat(relativePaths).ToArray(); } } @@ -81,15 +109,10 @@ public DafnyTestCase(string dafnyFile, Dictionary config, string public DafnyTestCase() { } - - public void Serialize(IXunitSerializationInfo info) { - info.AddValue(nameof(DafnyFile), DafnyFile); - info.AddValue(nameof(Arguments), Arguments); - } - - public void Deserialize(IXunitSerializationInfo info) { - DafnyFile = info.GetValue(nameof(DafnyFile)); - Arguments = info.GetValue(nameof(Arguments)); + + public object[] ToMemberData() { + string[] args = new[] {DafnyFile}.Concat(Arguments).ToArray(); + return new object[] {args, Expected}; } public static string RunDafny(IEnumerable arguments) { @@ -172,8 +195,7 @@ public static IEnumerable TestCasesForDafnyFile(string filePath) { } return configs.SelectMany(config => ResolveCompile(filePath, config)) -// .Select(t => new object[] {t.DafnyFile, t.Arguments, t.ExpectedOutputFile}); - .Select(t => new object[] {t}); + .Select(t => t.ToMemberData()); } private static Dictionary ToDictionary(YamlNode node) { @@ -223,9 +245,7 @@ private static string ConfigPairToArgument(KeyValuePair pair) { } public void Run() { - string fullInputPath = Path.Combine(TEST_ROOT, DafnyFile); - - var arguments = new []{ fullInputPath }.Concat(Arguments); + var arguments = new []{ DafnyFile }.Concat(Arguments); string output; if (arguments.Any(arg => arg.StartsWith("/out"))) { @@ -243,12 +263,12 @@ public void Run() { } } - if (ExpectedOutputFile != null) { - var expectedOutput = File.ReadAllText(Path.Combine(TEST_ROOT, ExpectedOutputFile)); + if (Expected.OutputFile != null) { + var expectedOutput = File.ReadAllText(Path.Combine(TEST_ROOT, Expected.OutputFile)); AssertWithDiff.Equal(expectedOutput, output); } - Skip.If(SpecialCase, "Confirmed known exception for arguments: " + arguments); + Skip.If(Expected.SpecialCase, "Confirmed known exception for arguments: " + arguments); } } @@ -256,26 +276,17 @@ public class DafnyTests { public static IEnumerable AllTestFiles() { var filePaths = Directory.GetFiles(DafnyTestCase.COMP_DIR, "*.dfy", SearchOption.AllDirectories) - .Select(path => GetRelativePath(DafnyTestCase.TEST_ROOT, path)); + .Select(path => Path.GetRelativePath(DafnyTestCase.TEST_ROOT, path)); return filePaths.SelectMany(DafnyTestCase.TestCasesForDafnyFile); } - // TODO-RS: Replace with Path.GetRelativePath() if we move to .NET Core 3.1 - public static string GetRelativePath(string relativeTo, string path) { - var fullRelativeTo = Path.GetFullPath(relativeTo); - var fullPath = Path.GetFullPath(path); - Assert.StartsWith(fullRelativeTo, fullPath); - return fullPath.Substring(fullRelativeTo.Length); - } - [ParallelTheory] [MemberData(nameof(AllTestFiles))] - public static void Test(DafnyTestCase testCase) { -// public static void Test(string dafnyFile, string[] arguments, string expectedOutputFile) { -// var testCase = new DafnyTestCase(); -// testCase.DafnyFile = dafnyFile; -// testCase.Arguments = arguments; -// testCase.ExpectedOutputFile = expectedOutputFile; + public static void Test(string[] args, DafnyTestCase.Expectation expected) { + var testCase = new DafnyTestCase(); + testCase.DafnyFile = args[0]; + testCase.Arguments = args.Skip(1).ToArray(); + testCase.Expected = expected; testCase.Run(); } } diff --git a/Source/LitTestConverter/LitTestConverter.csproj b/Source/LitTestConverter/LitTestConverter.csproj index 6d40400dfc1..aa00977a2ba 100644 --- a/Source/LitTestConverter/LitTestConverter.csproj +++ b/Source/LitTestConverter/LitTestConverter.csproj @@ -1,52 +1,11 @@  - - + + - Debug - AnyCPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A} Exe - Properties - LitTestConverter + netcoreapp3.1 LitTestConverter - v4.8 - 512 + false - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - - - - - - - - - - + \ No newline at end of file diff --git a/Test/comp/AutoInit.dfy b/Test/comp/AutoInit.dfy index f0982bdd63a..849a1805cce 100644 --- a/Test/comp/AutoInit.dfy +++ b/Test/comp/AutoInit.dfy @@ -1,10 +1,3 @@ -// RUN: %dafny /compile:0 "%s" > "%t" -// RUN: %dafny /noVerify /compile:4 /compileTarget:cs "%s" >> "%t" -// RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" -// RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - // This file tests the compilation of some default values. It also includes some // regression tests for equality, printing, and tuples. diff --git a/Test/comp/AutoInit.dfy.expect b/Test/comp/AutoInit.dfy.expect index 88a5a1c17dd..3565f96f65c 100644 --- a/Test/comp/AutoInit.dfy.expect +++ b/Test/comp/AutoInit.dfy.expect @@ -1,42 +1,4 @@ -Dafny program verifier finished with 9 verified, 0 errors - -Dafny program verifier finished with 0 verified, 0 errors -3 false 0 -Cell.Cell(false) false true -() (0, false, false, []) -(null, Cell.Cell(0)) (null, Cell.Cell(0)) true -(null, null, null, Cell.Cell(0)) (null, null, null, Cell.Cell(0)) true -true false false true -true false false true -17 -1 3 9 0 0 ThisOrThat.Or ThisOrThat.Or -1 - -Dafny program verifier finished with 0 verified, 0 errors -3 false 0 -Cell.Cell(false) false true -() (0, false, false, []) -(null, Cell.Cell(0)) (null, Cell.Cell(0)) true -(null, null, null, Cell.Cell(0)) (null, null, null, Cell.Cell(0)) true -true false false true -true false false true -17 -1 3 9 0 0 ThisOrThat.Or ThisOrThat.Or -1 - -Dafny program verifier finished with 0 verified, 0 errors -3 false 0 -Cell.Cell(false) false true -() (0, false, false, []) -(null, Cell.Cell(0)) (null, Cell.Cell(0)) true -(null, null, null, Cell.Cell(0)) (null, null, null, Cell.Cell(0)) true -true false false true -true false false true -17 -1 3 9 0 0 ThisOrThat.Or ThisOrThat.Or -1 - Dafny program verifier finished with 0 verified, 0 errors 3 false 0 Cell.Cell(false) false true diff --git a/Test/comp/Comprehensions.dfy.expect b/Test/comp/Comprehensions.dfy.expect index c0ecb78560d..9d09f0a864d 100644 --- a/Test/comp/Comprehensions.dfy.expect +++ b/Test/comp/Comprehensions.dfy.expect @@ -20,3 +20,26 @@ XM returned: 16 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] [0, 1, 4, 9, 16, 25, 36, 49] [0, 1, 2, 3, 4, 5, 6, 7] +4 4 2 2 0 4 2 true +5 4 2 2 0 4 2 true +5 5 3 3 1 5 3 true +2 2 1 1 0 2 false true +3 3 +451 +671 +221 +281 +1 +true false 1 +451 +671 +221 +281 +2 +true true false 2 +the intersection is {null} +there are 3 elements in the union +true true true +false false false +true true true +false false false diff --git a/Test/comp/Comprehensions.dfy.java.expect b/Test/comp/Comprehensions.dfy.java.expect index 0480a4aeae8..92af75e4523 100644 --- a/Test/comp/Comprehensions.dfy.java.expect +++ b/Test/comp/Comprehensions.dfy.java.expect @@ -1,7 +1,7 @@ Dafny program verifier finished with 0 verified, 0 errors x=13 y=14 -x=13 y=14 b=[y, e, s] +x=13 y=14 b=yes p=(13, 14) q=(13, 14, [y, e, s]) true false @@ -20,3 +20,26 @@ XM returned: 16 [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] [0, 1, 4, 9, 16, 25, 36, 49] [0, 1, 2, 3, 4, 5, 6, 7] +4 4 2 2 0 4 2 true +5 4 2 2 0 4 2 true +5 5 3 3 1 5 3 true +2 2 1 1 0 2 false true +3 3 +451 +671 +221 +281 +1 +true false 1 +451 +671 +221 +281 +2 +true true false 2 +the intersection is {null} +there are 3 elements in the union +true true true +false false false +true true true +false false false diff --git a/Test/comp/CovariantCollections.dfy.java.expect b/Test/comp/CovariantCollections.dfy.java.expect deleted file mode 100644 index 15e182eac73..00000000000 --- a/Test/comp/CovariantCollections.dfy.java.expect +++ /dev/null @@ -1,5 +0,0 @@ - -Dafny program verifier finished with 0 verified, 0 errors -CovariantCollections.dfy(28,6): Error: compilation of seq is not supported; consider introducing a ghost -CovariantCollections.dfy(29,6): Error: compilation of seq is not supported; consider introducing a ghost -File Program/Program.java contains the partially compiled program diff --git a/Test/comp/Numbers.dfy.go.expect b/Test/comp/Numbers.dfy.go.expect index 351c9c97688..a3aa8c2334f 100644 --- a/Test/comp/Numbers.dfy.go.expect +++ b/Test/comp/Numbers.dfy.go.expect @@ -42,6 +42,16 @@ g Q 2 8 1 -31 0 0 0 0 0 0 +uint8: 10:1 0:231 +uint16: 2848:7 0:65511 +uint32: 186737707:10 0:4294967271 +uint64: 802032351030850069:4 0:18446744073709551591 +via real: 7:12 -8:1 -7:12 8:1 +int: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int8: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int16: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int32: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int64: 7:12 -8:1 -7:12 8:1 -12:0 12:0 0.2 35.2 27.2 -27.2 124.8 -124.8 diff --git a/Test/comp/Poly.dfy.cs.expect b/Test/comp/Poly.dfy.cs.expect deleted file mode 100644 index 0b1b05fc2cb..00000000000 --- a/Test/comp/Poly.dfy.cs.expect +++ /dev/null @@ -1,6 +0,0 @@ - -Dafny program verifier finished with 0 verified, 0 errors -Poly.dfy(59,16): Error: compilation of set is not supported; consider introducing a ghost -Poly.dfy(63,21): Error: compilation of multiset is not supported; consider introducing a ghost -Poly.dfy(80,13): Error: compilation of set is not supported; consider introducing a ghost -Poly.dfy(81,18): Error: compilation of multiset is not supported; consider introducing a ghost diff --git a/Test/comp/Poly.dfy.expect b/Test/comp/Poly.dfy.expect index 2c8cb9e2f31..be80efca6d7 100644 --- a/Test/comp/Poly.dfy.expect +++ b/Test/comp/Poly.dfy.expect @@ -1,14 +1,14 @@ Dafny program verifier finished with 0 verified, 0 errors -Center of square: (0.5, 0.5) +Center of square: ((1.0 / 2.0), (1.0 / 2.0)) Center of circle: (1.0, 1.0) -Center: (0.5, 0.5) +Center: ((1.0 / 2.0), (1.0 / 2.0)) Center: (1.0, 1.0) -Center: (0.5, 0.5) +Center: ((1.0 / 2.0), (1.0 / 2.0)) Center: (1.0, 1.0) -Center: (0.5, 0.5) +Center: ((1.0 / 2.0), (1.0 / 2.0)) Center: (1.0, 1.0) -Center: (0.5, 0.5) +Center: ((1.0 / 2.0), (1.0 / 2.0)) Center: (1.0, 1.0) -Center: (0.5, 0.5) +Center: ((1.0 / 2.0), (1.0 / 2.0)) Center: (1.0, 1.0) diff --git a/Test/comp/Poly.dfy.go.expect b/Test/comp/Poly.dfy.go.expect new file mode 100644 index 00000000000..2c8cb9e2f31 --- /dev/null +++ b/Test/comp/Poly.dfy.go.expect @@ -0,0 +1,14 @@ + +Dafny program verifier finished with 0 verified, 0 errors +Center of square: (0.5, 0.5) +Center of circle: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) diff --git a/Test/comp/Poly.dfy.java.expect b/Test/comp/Poly.dfy.java.expect deleted file mode 100644 index a96b1e5eed5..00000000000 --- a/Test/comp/Poly.dfy.java.expect +++ /dev/null @@ -1,6 +0,0 @@ - -Dafny program verifier finished with 0 verified, 0 errors -Poly.dfy(48,16): Error: compilation of seq is not supported; consider introducing a ghost -Poly.dfy(59,16): Error: compilation of set is not supported; consider introducing a ghost -Poly.dfy(63,21): Error: compilation of multiset is not supported; consider introducing a ghost -File Program/Program.java contains the partially compiled program diff --git a/Test/comp/Poly.dfy.js.expect b/Test/comp/Poly.dfy.js.expect deleted file mode 100644 index 2af9bb1e22d..00000000000 --- a/Test/comp/Poly.dfy.js.expect +++ /dev/null @@ -1,10 +0,0 @@ - -Dafny program verifier finished with 0 verified, 0 errors -Center of square: ((1.0 / 2.0), (1.0 / 2.0)) -Center of circle: (1.0, 1.0) -Center: ((1.0 / 2.0), (1.0 / 2.0)) -Center: (1.0, 1.0) -Center: ((1.0 / 2.0), (1.0 / 2.0)) -Center: (1.0, 1.0) -Centers: {((1.0 / 2.0), (1.0 / 2.0)), (1.0, 1.0)} -Centers: {((1.0 / 2.0), (1.0 / 2.0)), (1.0, 1.0)} diff --git a/Test/comp/TypeParams.dfy.cs.expect b/Test/comp/TypeParams.dfy.cs.expect index a1906ead70c..288c714fb1e 100644 --- a/Test/comp/TypeParams.dfy.cs.expect +++ b/Test/comp/TypeParams.dfy.cs.expect @@ -17,5 +17,5 @@ Color.Pink 19 -3 null null null null System.Func`2[Dafny.BigRational,System.Boolean] System.Func`1[System.Numerics.BigInteger] -System.Func`3[_module.Color,Dafny.Set`1[System.UInt16],System.UInt32] +System.Func`3[_module.Color,Dafny.ISet`1[System.UInt16],System.UInt32] 0 0 From 96477c3e4b372901d77cadc321c4cdc931a7ddaa Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 29 Sep 2020 15:14:50 -0700 Subject: [PATCH 058/192] A few more fixes --- Source/DafnyTests/DafnyTestCase.cs | 22 ++++++++++------------ Test/comp/Collections.dfy.cs.expect | 5 +++++ Test/comp/Collections.dfy.java.expect | 5 +++++ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 7fa9fe49fc5..8ad3b811a40 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -73,15 +73,13 @@ public void Deserialize(IXunitSerializationInfo info) { } public void Serialize(IXunitSerializationInfo info) { - if (OutputFile != null) { - info.AddValue(nameof(OutputFile), OutputFile); - } - if (ExitCode != 0) { - info.AddValue(nameof(ExitCode), ExitCode); - } - if (SpecialCase) { - info.AddValue(nameof(SpecialCase), true); - } + info.AddValue(nameof(OutputFile), OutputFile); + info.AddValue(nameof(ExitCode), ExitCode); + info.AddValue(nameof(SpecialCase), SpecialCase); + } + + public override string ToString() { + return OutputFile ?? "-"; } } @@ -258,8 +256,8 @@ public void Run() { // temporary directory, since some compilers will // interpret the path as a single file basename rather than a directory. var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; - arguments = new []{ outArgument }.Concat(arguments); - output = RunDafny(arguments); + var dafnyArguments = new []{ outArgument }.Concat(arguments); + output = RunDafny(dafnyArguments); } } @@ -268,7 +266,7 @@ public void Run() { AssertWithDiff.Equal(expectedOutput, output); } - Skip.If(Expected.SpecialCase, "Confirmed known exception for arguments: " + arguments); + Skip.If(Expected.SpecialCase, "Confirmed known exception for arguments: " + String.Join(" ", arguments)); } } diff --git a/Test/comp/Collections.dfy.cs.expect b/Test/comp/Collections.dfy.cs.expect index f9215367c73..91f1a2f44a2 100644 --- a/Test/comp/Collections.dfy.cs.expect +++ b/Test/comp/Collections.dfy.cs.expect @@ -9,6 +9,7 @@ Sets: {} {17, 82} {12, 17} subset: true false true proper subset: true false false membership: false true true + eq covariance: 1 true true false true |s|=4 |S|=16 {{a, c, d}, {a, b, c}, {}, {a}, {b}, {c}, {d}, {a, b, c, d}, {b, c, d}, {a, b, d}, {a, b}, {a, c}, {a, d}, {b, c}, {b, d}, {c, d}} @@ -42,6 +43,10 @@ hello hEllo [2, 4, 6, 8, 10] [2, 0, 6, 8, 10] +true false false +true true false true +false false false true + eq covariance: true true Strings: uRuR gu cardinality: 0 4 2 concatenation: uRuR uRuRgu diff --git a/Test/comp/Collections.dfy.java.expect b/Test/comp/Collections.dfy.java.expect index e2181995389..4d9798d25c1 100644 --- a/Test/comp/Collections.dfy.java.expect +++ b/Test/comp/Collections.dfy.java.expect @@ -9,6 +9,7 @@ Sets: {} {17, 82} {17, 12} subset: true false true proper subset: true false false membership: false true true + eq covariance: 1 true true false true |s|=4 |S|=16 {{}, {a}, {b}, {a, b}, {c}, {a, c}, {d}, {b, c}, {a, d}, {a, b, c}, {b, d}, {a, b, d}, {c, d}, {a, c, d}, {b, c, d}, {a, b, c, d}} @@ -42,6 +43,10 @@ hello hEllo [2, 4, 6, 8, 10] [2, 0, 6, 8, 10] +true false false +true true false true +false false false true + eq covariance: true true Strings: uRuR gu cardinality: 0 4 2 concatenation: uRuR uRuRgu From 01653f784049fc8ed5df2d1d9d0760211e44a697 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 30 Sep 2020 16:26:28 -0700 Subject: [PATCH 059/192] Progress on lit test convertor --- Source/DafnyTests/DafnyTestCase.cs | 2 +- Source/LitTestConverter/LitTestConvertor.cs | 218 ++++++++++++-------- 2 files changed, 135 insertions(+), 85 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 8ad3b811a40..7e57220312b 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -98,7 +98,7 @@ public DafnyTestCase(string dafnyFile, Dictionary config, string var additionalFilesPath = fullDafnyFilePath + "." + compileTarget + ".files"; if (Directory.Exists(additionalFilesPath)) { var relativePaths = Directory.GetFiles(additionalFilesPath) - .Select(path => Path.GetRelativePath(TEST_ROOT, path)); + .Select(path => Path.GetRelativePath(TEST_ROOT, path)); Arguments = Arguments.Concat(relativePaths).ToArray(); } } diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 52f267e52ce..93ed16e0a76 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -6,120 +6,164 @@ using System.Text; using Microsoft.SqlServer.Server; -namespace DafnyTests { - public static class LitTestConvertor { - - private const string LIT_COMMAND_PREFIX = "// RUN: "; - private const string LIT_DAFNY = "%dafny"; - private const string DAFNY_COMPILE = "/compile:"; - private const string DAFNY_COMPILE_TARGET = "/compileTarget:"; +namespace DafnyTests { + + public class LitTestConvertor { - private static readonly string[] IGNORED_DAFNY_COMMAND_ARGUMENTS = { - "\"%s\"", ">", ">>", "\"%t\"" - }; + private const string DAFNY_COMMENT_PREFIX = "//"; + private const string LIT_COMMAND_PREFIX = "RUN:"; + private const string LIT_DAFNY = "%dafny"; + private const string DAFNY_COMPILE = "compile"; + private const string DAFNY_COMPILE_TARGET = "compileTarget"; private static readonly string[] SUPPORTED_DAFNY_FLAGS = { - "/allocated", - "/arith", - "/autoTriggers", - "/dafnyVerify", - "/definiteAssignment", - "/dprint", - "/env", - "/errorLimit", - "/errorTrace", - "/ironDafny", - "/noNLarith", - "/noVerify", - "/optimize", - "/optimizeResolution", - "/print", - "/printTooltips", - "/restartProver", - "/rprint", - "/traceCaching", - "/tracePOs", - "/verifyAllModules", - "/verifySnapshots", - "/warnShadowing", + "allocated", + "arith", + "autoTriggers", + DAFNY_COMPILE, + DAFNY_COMPILE_TARGET, + "dafnyVerify", + "definiteAssignment", + "dprint", + "env", + "errorLimit", + "errorTrace", + "ironDafny", + "noNLarith", + "noVerify", + "optimize", + "optimizeResolution", + "print", + "printTooltips", + "restartProver", + "rprint", + "spillTargetCode", + "traceCaching", + "tracePOs", + "verifyAllModules", + "verifySnapshots", + "warnShadowing", }; private static readonly string[] DAFNY_OUTPUT_FLAGS = { - "/print", - "/dprint", - "/rprint", + "print", + "dprint", + "rprint", }; + + private int count = 0; + private int verifyOnlyCount = 0; + private int defaultCount = 0; + private int invalidCount = 0; - public static void ConvertLitTest(string filePath) { + public void ConvertLitTest(string filePath) { string[] lines = File.ReadAllLines(filePath); - if (lines.Length < 2) { - throw new ArgumentException("Not enough lines to match expected lit test pattern"); - } - - string dafnyCommand = ExtractLitCommand(lines[0]); - if (dafnyCommand == null) { - // Doesn't look like a lit test - return; - } - if (!dafnyCommand.StartsWith(LIT_DAFNY)) { - throw new ArgumentException("First lit command is not expected %dafny: " + dafnyCommand); - } - var testConfig = ParseDafnyCommand(dafnyCommand.Substring(LIT_DAFNY.Length)); - - string diffCommand = ExtractLitCommand(lines[1]); - if (!diffCommand.Equals("%diff \"%s.expect\" \"%t\"")) { - throw new ArgumentException("Second lit command is not expected %diff: " + diffCommand); - } + + var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); + if (!litCommands.Any()) { + // TODO-RS: Check if in an Inputs directory + return; + } + + + // Make sure the commands are consecutive + if (lines.Skip(litCommands.Count).Any(line => ExtractLitCommand(line) != null)) { + throw new ArgumentException("Lit commands are not consecutive"); + } + + if (!litCommands[^1].Equals("%diff \"%s.expect\" \"%t\"")) { + throw new ArgumentException("Last lit command is not expected %diff: " + litCommands[^1]); + } + litCommands.RemoveAt(litCommands.Count - 1); + + var testConfigs = litCommands.Select(ParseDafnyCommandArguments).ToList(); + + if (testConfigs.Count == 1 && + testConfigs[0].Count == 1 && + DictionaryContainsEntry(testConfigs[0], DAFNY_COMPILE, "0")) { + verifyOnlyCount++; + } else if (testConfigs.Count(c => c.ContainsKey(DAFNY_COMPILE_TARGET)) > 1) { + defaultCount++; + } } + private static bool DictionaryContainsEntry(Dictionary dictionary, K key, V value) { + if (dictionary.TryGetValue(key, out var dictionaryValue)) { + return value.Equals(dictionaryValue); + } else { + return false; + } + } + private static string ExtractLitCommand(string line) { + if (!line.StartsWith(DAFNY_COMMENT_PREFIX)) { + return null; + } + line = line.Substring(DAFNY_COMMENT_PREFIX.Length).Trim(); + if (!line.StartsWith(LIT_COMMAND_PREFIX)) { return null; } - return line.Substring(LIT_COMMAND_PREFIX.Length); + return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); } - private static Dictionary ParseDafnyCommand(string line) { - var testConfig = new Dictionary(); - var parts = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries); - int compileLevel; + private static Dictionary ParseDafnyCommandArguments(string dafnyCommand) { + if (!dafnyCommand.StartsWith(LIT_DAFNY)) { + throw new ArgumentException("Lit command is not expected %dafny: " + dafnyCommand); + } + IEnumerable arguments = RemoveInputFileAndOutput(dafnyCommand.Substring(LIT_DAFNY.Length)); + // Check the arguments for anything non-standard - foreach (var argument in parts) { - if (IGNORED_DAFNY_COMMAND_ARGUMENTS.Contains(argument)) { - // Ignore - } else if (argument.StartsWith(DAFNY_COMPILE)) { - compileLevel = Int32.Parse(argument.Substring(DAFNY_COMPILE.Length)); - } else if (argument.StartsWith(DAFNY_COMPILE_TARGET)) { - // Ignore - assume it will work for all target language unless proven otherwise - } else { - KeyValuePair pair = ParseDafnyArgument(argument); - if (!(DAFNY_OUTPUT_FLAGS.Contains(pair.Key) && pair.Value.Contains("%t"))) { - testConfig.Add(pair.Key, pair.Value); - } + var testConfig = new Dictionary(); + foreach (var argument in arguments) { + KeyValuePair pair = ParseDafnyArgument(argument); + if (!(DAFNY_OUTPUT_FLAGS.Contains(pair.Key) && pair.Value.Contains("%t"))) { + testConfig.Add(pair.Key, pair.Value); } } return testConfig; } + private static IEnumerable RemoveInputFileAndOutput(string line) { + var arguments = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).ToList(); + + // Ensure the last two parts are > "%t" or >> "%t" + if (arguments.Count < 3) { + throw new ArgumentException("Not enough arguments to %dafny command: " + line); + } + if (!arguments[^1].Equals("\"%t\"") + || !(arguments[^2].Equals(">") || arguments[^2].Equals(">>"))) + { + throw new ArgumentException("Non-standard output in %dafny command: " + line); + } + arguments.RemoveRange(arguments.Count - 2, 2); + + if (!arguments.Remove("\"%s\"")) { + throw new ArgumentException("Test file (\"%s\") does not appear as input in %dafny command: " + line); + } + + return arguments; + } + private static KeyValuePair ParseDafnyArgument(string argument) { - foreach (var supportedFlag in SUPPORTED_DAFNY_FLAGS) { - if (argument.StartsWith(supportedFlag)) { - if (argument.Equals(supportedFlag)) { - return new KeyValuePair(supportedFlag, "yes"); - } else if (argument[supportedFlag.Length] == ':') { - return new KeyValuePair(supportedFlag, - argument.Substring(supportedFlag.Length + 1)); + if (argument.StartsWith("-") || argument.StartsWith("/")) { + argument = argument.Substring(1); + foreach (var supportedFlag in SUPPORTED_DAFNY_FLAGS) { + if (argument.StartsWith(supportedFlag)) { + if (argument.Equals(supportedFlag)) { + return new KeyValuePair(supportedFlag, "yes"); + } else if (argument[supportedFlag.Length] == ':') { + return new KeyValuePair(supportedFlag, + argument.Substring(supportedFlag.Length + 1)); + } } } } throw new ArgumentException("Unrecognized dafny argument: " + argument); } - public static void Main(string[] args) { - var root = args[0]; - var count = 0; - var invalidCount = 0; + public void Run(string root) { foreach (var file in Directory.GetFiles(root, "*.dfy", SearchOption.AllDirectories)) { try { count++; @@ -129,7 +173,13 @@ public static void Main(string[] args) { Console.WriteLine(file + ": " + e.Message); } } - Console.WriteLine(invalidCount + "/" + count); + Console.WriteLine("Default: " + defaultCount + "/" + count); + Console.WriteLine("Verify only: " + verifyOnlyCount + "/" + count); + Console.WriteLine("Invalid: " + invalidCount + "/" + count); + } + + public static void Main(string[] args) { + new LitTestConvertor().Run(args[0]); } } } \ No newline at end of file From e6deb41d5911688ce7d9f510f6f5b4f63843e2a8 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 1 Oct 2020 12:06:31 -0700 Subject: [PATCH 060/192] Improve lit test conversion approach --- Source/DafnyTests/DafnyTestCase.cs | 8 -- Source/LitTestConverter/LitTestConvertor.cs | 120 +++++++++----------- Test/README.md | 4 - 3 files changed, 54 insertions(+), 78 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 7e57220312b..3da201c1bbc 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -3,18 +3,10 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Reflection; -using System.Runtime.InteropServices; -using System.Text; -using DiffPlex; -using DiffPlex.DiffBuilder; -using DiffPlex.DiffBuilder.Model; using Xunit; using Xunit.Abstractions; -using Xunit.Sdk; using XUnitExtensions; using YamlDotNet.RepresentationModel; -using YamlDotNet.Serialization; namespace DafnyTests { diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 93ed16e0a76..48c86015067 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -15,40 +15,17 @@ public class LitTestConvertor { private const string LIT_DAFNY = "%dafny"; private const string DAFNY_COMPILE = "compile"; private const string DAFNY_COMPILE_TARGET = "compileTarget"; + // Fake options to which files are passed to the CLI + private const string TEST_CONFIG_OTHER_FILES = "otherFiles"; + private const string TEST_CONFIG_INCLUDE_THIS_FILE = "includeThisFile"; - private static readonly string[] SUPPORTED_DAFNY_FLAGS = { - "allocated", - "arith", - "autoTriggers", - DAFNY_COMPILE, - DAFNY_COMPILE_TARGET, - "dafnyVerify", - "definiteAssignment", - "dprint", - "env", - "errorLimit", - "errorTrace", - "ironDafny", - "noNLarith", - "noVerify", - "optimize", - "optimizeResolution", - "print", - "printTooltips", - "restartProver", - "rprint", - "spillTargetCode", - "traceCaching", - "tracePOs", - "verifyAllModules", - "verifySnapshots", - "warnShadowing", - }; - - private static readonly string[] DAFNY_OUTPUT_FLAGS = { + // Arguments that are taken care of automatically. If a test is actually using the output of + // one of these as input in another command, that will be flagged as an unsupported + // use of lit substitution. + private static readonly string[] DAFNY_IGNORED_OPTIONS = { "print", "dprint", - "rprint", + "rprint" }; private int count = 0; @@ -57,19 +34,22 @@ public class LitTestConvertor { private int invalidCount = 0; public void ConvertLitTest(string filePath) { - string[] lines = File.ReadAllLines(filePath); - - var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); - if (!litCommands.Any()) { - // TODO-RS: Check if in an Inputs directory + if (filePath.Contains("/Inputs/") || filePath.Contains("/comp/")) { + // TODO-RS: Need to add .common.yml file to disable Inputs/*.dfy return; } - + string[] lines = File.ReadAllLines(filePath); + + var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); // Make sure the commands are consecutive if (lines.Skip(litCommands.Count).Any(line => ExtractLitCommand(line) != null)) { throw new ArgumentException("Lit commands are not consecutive"); } + if (!litCommands.Any()) { + // TODO-RS: Check if in an Inputs directory + throw new ArgumentException("No lit commands found"); + } if (!litCommands[^1].Equals("%diff \"%s.expect\" \"%t\"")) { throw new ArgumentException("Last lit command is not expected %diff: " + litCommands[^1]); @@ -108,59 +88,67 @@ private static string ExtractLitCommand(string line) { } private static Dictionary ParseDafnyCommandArguments(string dafnyCommand) { + var testConfig = new Dictionary(); + var otherFiles = new List(); + if (!dafnyCommand.StartsWith(LIT_DAFNY)) { throw new ArgumentException("Lit command is not expected %dafny: " + dafnyCommand); } - IEnumerable arguments = RemoveInputFileAndOutput(dafnyCommand.Substring(LIT_DAFNY.Length)); - - // Check the arguments for anything non-standard - var testConfig = new Dictionary(); - foreach (var argument in arguments) { - KeyValuePair pair = ParseDafnyArgument(argument); - if (!(DAFNY_OUTPUT_FLAGS.Contains(pair.Key) && pair.Value.Contains("%t"))) { - testConfig.Add(pair.Key, pair.Value); - } - } - return testConfig; - } - - private static IEnumerable RemoveInputFileAndOutput(string line) { - var arguments = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).ToList(); + string argumentsString = dafnyCommand.Substring(LIT_DAFNY.Length); + var arguments = argumentsString.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).ToList(); // Ensure the last two parts are > "%t" or >> "%t" if (arguments.Count < 3) { - throw new ArgumentException("Not enough arguments to %dafny command: " + line); + throw new ArgumentException("Not enough arguments to %dafny command: " + dafnyCommand); } if (!arguments[^1].Equals("\"%t\"") || !(arguments[^2].Equals(">") || arguments[^2].Equals(">>"))) { - throw new ArgumentException("Non-standard output in %dafny command: " + line); + throw new ArgumentException("Non-standard output in %dafny command: " + dafnyCommand); } arguments.RemoveRange(arguments.Count - 2, 2); if (!arguments.Remove("\"%s\"")) { - throw new ArgumentException("Test file (\"%s\") does not appear as input in %dafny command: " + line); + testConfig[TEST_CONFIG_INCLUDE_THIS_FILE] = "no"; + } + + // Check the arguments for anything non-standard + foreach (var argument in arguments) { + KeyValuePair pair = ParseDafnyArgument(argument); + if (DAFNY_IGNORED_OPTIONS.Contains(pair.Key)) { + continue; + } + if (pair.Value.Contains("%")) { + throw new ArgumentException("Use of lit substitution (% variable) requires manual conversion: " + argument); + } + if (pair.Key.Equals(TEST_CONFIG_OTHER_FILES)) { + otherFiles.Add(pair.Value); + } else { + testConfig.Add(pair.Key, pair.Value); + } } - return arguments; + if (otherFiles.Any()) { + testConfig[TEST_CONFIG_OTHER_FILES] = String.Join(" ", otherFiles); + } + return testConfig; } private static KeyValuePair ParseDafnyArgument(string argument) { if (argument.StartsWith("-") || argument.StartsWith("/")) { argument = argument.Substring(1); - foreach (var supportedFlag in SUPPORTED_DAFNY_FLAGS) { - if (argument.StartsWith(supportedFlag)) { - if (argument.Equals(supportedFlag)) { - return new KeyValuePair(supportedFlag, "yes"); - } else if (argument[supportedFlag.Length] == ':') { - return new KeyValuePair(supportedFlag, - argument.Substring(supportedFlag.Length + 1)); - } - } + int colonIndex = argument.IndexOf(":"); + if (colonIndex >= 0) { + return new KeyValuePair( + argument.Substring(0, colonIndex), + argument.Substring(colonIndex + 1)); + } else { + return new KeyValuePair(argument, "yes"); } + } else { + return new KeyValuePair(TEST_CONFIG_OTHER_FILES, argument); } - throw new ArgumentException("Unrecognized dafny argument: " + argument); } public void Run(string root) { diff --git a/Test/README.md b/Test/README.md index 4a171f2e8f1..c567c6ed269 100644 --- a/Test/README.md +++ b/Test/README.md @@ -13,11 +13,7 @@ For details and more configuration options, see [the DafnyTests.cs source](Test/ * More complete documentation about options (in this file or in the source code) * Depend on only the project's output directory instead of the Binaries/Test directories * This is mostly working except for errors around missing types from System.dll when compiling to C# -* Complete support for both the .NET Framework solution as well as the .NET Core one -* Figure out why test cases aren't executing in parallel when run using `msbuild` (they are when run in Rider) * Add support for regular expression matching against CLI output (needed to assert known limitations that cause errors with things like absolute paths names in them) -* By default, verify source files separately and skip verification when compiling to each target language -* Add support for sharding ala lit's `--run-shard / --num-shards` options * Expose test case options as traits so that they can be filtered on (e.g. `dotnet test --filter compileTarget=java`) * Finish converting the rest of the test cases * Will write a small script to do this automatically for all recognized combinations of lit `// RUN` commands From 56a1e4b801a47c51602a8c21d697e7b53eeaac7f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 1 Oct 2020 14:29:07 -0700 Subject: [PATCH 061/192] First cut of generic YamlFileDataAttribute --- Source/DafnyTests/DafnyTests.csproj | 6 +- .../XUnitExtensions/YamlFileDataAttribute.cs | 80 +++++++++++++++++++ Source/DafnyTests/YamlTests/calculator.yml | 6 ++ Source/DafnyTests/YamlUtils.cs | 7 ++ 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs create mode 100644 Source/DafnyTests/YamlTests/calculator.yml diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index e1041a092e0..d727a72f524 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -15,9 +15,11 @@ - + - + + + diff --git a/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs b/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs new file mode 100644 index 00000000000..3e202b2944a --- /dev/null +++ b/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using DafnyTests; +using Xunit.Sdk; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; + +namespace XUnitExtensions { + public class YamlFileDataAttribute : DataAttribute { + + private string RootPath; + private IDeserializer Deserializer = new DeserializerBuilder().Build(); + + public YamlFileDataAttribute(string rootPath) { + RootPath = rootPath; + } + + private class ParsingEventBuffer : IEmitter { + public List ParsingEvents = new List(); + + public void Emit(ParsingEvent parsingEvent) { + ParsingEvents.Add(parsingEvent); + } + } + + private class ParsingEventListParser : IParser { + private readonly IEnumerator Enumerator; + + public ParsingEventListParser(IEnumerator enumerator) { + Enumerator = enumerator; + } + + public bool MoveNext() { + return Enumerator.MoveNext(); + } + + public ParsingEvent Current => Enumerator.Current; + } + + private IParser NodeParser(YamlNode node) { + ParsingEventBuffer buffer = new ParsingEventBuffer(); + YamlDocument doc = new YamlDocument(node); + YamlStream stream = new YamlStream(); + stream.Add(doc); + stream.Save(buffer, false); + return new ParsingEventListParser(buffer.ParsingEvents.GetEnumerator()); + } + + private object DeserializeParameter(ParameterInfo parameterInfo, YamlNode mappingNode) { + return Deserializer.Deserialize(NodeParser(mappingNode), parameterInfo.ParameterType); + } + + private object[] MethodArguments(MethodInfo testMethod, YamlMappingNode mappingNode) { + return testMethod.GetParameters().Select(param => DeserializeParameter(param, mappingNode[param.Name])).ToArray(); + } + + private IEnumerable FileData(MethodInfo testMethod, string path) { + var yamlStream = new YamlStream(); + yamlStream.Load(new StreamReader(path)); + // TODO: error checking and expansion + YamlSequenceNode node = (YamlSequenceNode)yamlStream.Documents[0].RootNode; + return node.Select(childNode => MethodArguments(testMethod, (YamlMappingNode)childNode)); + } + + public override IEnumerable GetData(MethodInfo testMethod) { + if (RootPath.EndsWith(".yml")) { + return FileData(testMethod, RootPath); + } else { + var filePaths = Directory.GetFiles(RootPath, "*.yml", SearchOption.AllDirectories) + .Select(path => Path.GetRelativePath(RootPath, path)); + return filePaths.SelectMany(path => FileData(testMethod, path)); + } + } + } +} \ No newline at end of file diff --git a/Source/DafnyTests/YamlTests/calculator.yml b/Source/DafnyTests/YamlTests/calculator.yml new file mode 100644 index 00000000000..1f1f8294389 --- /dev/null +++ b/Source/DafnyTests/YamlTests/calculator.yml @@ -0,0 +1,6 @@ +- lhs: 2 + rhs: 2 + expected: 4 +- lhs: 3 + rhs: 4 + expected: 7 diff --git a/Source/DafnyTests/YamlUtils.cs b/Source/DafnyTests/YamlUtils.cs index dfe5df2f4c6..251290526ee 100644 --- a/Source/DafnyTests/YamlUtils.cs +++ b/Source/DafnyTests/YamlUtils.cs @@ -2,6 +2,7 @@ using System.IO; using System.Linq; using Xunit; +using XUnitExtensions; using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; @@ -50,5 +51,11 @@ public void ExpandTest(string inputPath, string expectedOutputPath) { Assert.Equal(expectedOutput, outputWriter.ToString()); } } + + [Theory] + [YamlFileData("YamlTests/calculator.yml")] + public void CalculatorTest(int lhs, int rhs, int expected) { + Assert.Equal(expected, lhs + rhs); + } } } \ No newline at end of file From 77cc5bac5e4bfcfdf72fce242bb2dca792febb3f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 1 Oct 2020 16:08:12 -0700 Subject: [PATCH 062/192] Add switches to YamlFileDataAttribute --- .../XUnitExtensions/YamlFileDataAttribute.cs | 25 ++++++++--- Source/DafnyTests/YamlTests/configs.yml | 6 +++ Source/DafnyTests/YamlUtils.cs | 42 +++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 Source/DafnyTests/YamlTests/configs.yml diff --git a/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs b/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs index 3e202b2944a..a358d03e8fd 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs @@ -14,10 +14,14 @@ namespace XUnitExtensions { public class YamlFileDataAttribute : DataAttribute { private string RootPath; + private bool WithParameterNames; + private bool WithSourceFile; private IDeserializer Deserializer = new DeserializerBuilder().Build(); - public YamlFileDataAttribute(string rootPath) { + public YamlFileDataAttribute(string rootPath, bool withParameterNames = true, bool withSourceFile = false) { RootPath = rootPath; + WithParameterNames = withParameterNames; + WithSourceFile = withSourceFile; } private class ParsingEventBuffer : IEmitter { @@ -55,8 +59,19 @@ private object DeserializeParameter(ParameterInfo parameterInfo, YamlNode mappin return Deserializer.Deserialize(NodeParser(mappingNode), parameterInfo.ParameterType); } - private object[] MethodArguments(MethodInfo testMethod, YamlMappingNode mappingNode) { - return testMethod.GetParameters().Select(param => DeserializeParameter(param, mappingNode[param.Name])).ToArray(); + private IEnumerable MethodArguments(MethodInfo testMethod, string path, YamlMappingNode mappingNode) { + IEnumerable result; + IEnumerable parameters = testMethod.GetParameters(); + if (WithSourceFile) { + parameters = parameters.Skip(1); + } + if (WithParameterNames) { + result = parameters.Select(param => DeserializeParameter(param, mappingNode[param.Name])).ToArray(); + } else { + result = new[] {DeserializeParameter(parameters.Single(), mappingNode)}; + } + + return WithSourceFile ? new[] {path}.Concat(result) : result; } private IEnumerable FileData(MethodInfo testMethod, string path) { @@ -64,7 +79,7 @@ private IEnumerable FileData(MethodInfo testMethod, string path) { yamlStream.Load(new StreamReader(path)); // TODO: error checking and expansion YamlSequenceNode node = (YamlSequenceNode)yamlStream.Documents[0].RootNode; - return node.Select(childNode => MethodArguments(testMethod, (YamlMappingNode)childNode)); + return node.Select(childNode => MethodArguments(testMethod, path, (YamlMappingNode)childNode).ToArray()); } public override IEnumerable GetData(MethodInfo testMethod) { @@ -72,7 +87,7 @@ public override IEnumerable GetData(MethodInfo testMethod) { return FileData(testMethod, RootPath); } else { var filePaths = Directory.GetFiles(RootPath, "*.yml", SearchOption.AllDirectories) - .Select(path => Path.GetRelativePath(RootPath, path)); + .Select(path => Path.GetRelativePath(RootPath, path)); return filePaths.SelectMany(path => FileData(testMethod, path)); } } diff --git a/Source/DafnyTests/YamlTests/configs.yml b/Source/DafnyTests/YamlTests/configs.yml new file mode 100644 index 00000000000..ec67b14a12c --- /dev/null +++ b/Source/DafnyTests/YamlTests/configs.yml @@ -0,0 +1,6 @@ +- one: 1 + two: 2 + three: 3 +- four: 4 + five: 5 + six: 6 diff --git a/Source/DafnyTests/YamlUtils.cs b/Source/DafnyTests/YamlUtils.cs index 251290526ee..87432ac5ac3 100644 --- a/Source/DafnyTests/YamlUtils.cs +++ b/Source/DafnyTests/YamlUtils.cs @@ -1,8 +1,10 @@ +using System; using System.Collections.Generic; using System.IO; using System.Linq; using Xunit; using XUnitExtensions; +using YamlDotNet.Core; using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; @@ -57,5 +59,45 @@ public void ExpandTest(string inputPath, string expectedOutputPath) { public void CalculatorTest(int lhs, int rhs, int expected) { Assert.Equal(expected, lhs + rhs); } + + [Theory] + [YamlFileData("YamlTests/configs.yml", withParameterNames: false, withSourceFile: true)] + public void DictionaryTest(string path, Dictionary config) { + Assert.Equal(3, config.Count); + } + + private class MyConfigTypeConvertor : IYamlTypeConverter { + + public bool Accepts(Type type) { + return type == typeof (MyConfig); + } + + public object ReadYaml(IParser parser, Type type) { + Dictionary config = new Deserializer().Deserialize>(parser); + return new MyConfig(config); + } + + public void WriteYaml(IEmitter emitter, object value, Type type) { + throw new NotImplementedException(); + } + } + + private static readonly IDeserializer DESERIALIZER = new DeserializerBuilder() + .WithTypeConverter(new MyConfigTypeConvertor()) + .Build(); + + public class MyConfig { + + public MyConfig(Dictionary config) { + Config = config; + } + + private Dictionary Config; + + public override string ToString() { + return ""; + } + } + } } \ No newline at end of file From 3995823d93fb1857a1b42c5b3f4c7adc241aec48 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 2 Oct 2020 12:53:51 -0700 Subject: [PATCH 063/192] Extract YamlFileDataDiscoverer (for customization) --- .../XUnitExtensions/YamlFileDataAttribute.cs | 79 +----------- .../XUnitExtensions/YamlFileDataDiscoverer.cs | 122 ++++++++++++++++++ Source/DafnyTests/YamlUtils.cs | 47 ++----- 3 files changed, 141 insertions(+), 107 deletions(-) create mode 100644 Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs diff --git a/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs b/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs index a358d03e8fd..218fd177be6 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reflection; using DafnyTests; +using Xunit.Abstractions; using Xunit.Sdk; using YamlDotNet.Core; using YamlDotNet.Core.Events; @@ -11,85 +12,15 @@ using YamlDotNet.Serialization; namespace XUnitExtensions { + + [DataDiscoverer("XUnitExtensions.YamlFileDataDiscoverer", "DafnyTests")] public class YamlFileDataAttribute : DataAttribute { - private string RootPath; - private bool WithParameterNames; - private bool WithSourceFile; - private IDeserializer Deserializer = new DeserializerBuilder().Build(); - - public YamlFileDataAttribute(string rootPath, bool withParameterNames = true, bool withSourceFile = false) { - RootPath = rootPath; - WithParameterNames = withParameterNames; - WithSourceFile = withSourceFile; - } - - private class ParsingEventBuffer : IEmitter { - public List ParsingEvents = new List(); - - public void Emit(ParsingEvent parsingEvent) { - ParsingEvents.Add(parsingEvent); - } - } - - private class ParsingEventListParser : IParser { - private readonly IEnumerator Enumerator; - - public ParsingEventListParser(IEnumerator enumerator) { - Enumerator = enumerator; - } - - public bool MoveNext() { - return Enumerator.MoveNext(); - } - - public ParsingEvent Current => Enumerator.Current; - } - - private IParser NodeParser(YamlNode node) { - ParsingEventBuffer buffer = new ParsingEventBuffer(); - YamlDocument doc = new YamlDocument(node); - YamlStream stream = new YamlStream(); - stream.Add(doc); - stream.Save(buffer, false); - return new ParsingEventListParser(buffer.ParsingEvents.GetEnumerator()); - } - - private object DeserializeParameter(ParameterInfo parameterInfo, YamlNode mappingNode) { - return Deserializer.Deserialize(NodeParser(mappingNode), parameterInfo.ParameterType); - } - - private IEnumerable MethodArguments(MethodInfo testMethod, string path, YamlMappingNode mappingNode) { - IEnumerable result; - IEnumerable parameters = testMethod.GetParameters(); - if (WithSourceFile) { - parameters = parameters.Skip(1); - } - if (WithParameterNames) { - result = parameters.Select(param => DeserializeParameter(param, mappingNode[param.Name])).ToArray(); - } else { - result = new[] {DeserializeParameter(parameters.Single(), mappingNode)}; - } - - return WithSourceFile ? new[] {path}.Concat(result) : result; - } - - private IEnumerable FileData(MethodInfo testMethod, string path) { - var yamlStream = new YamlStream(); - yamlStream.Load(new StreamReader(path)); - // TODO: error checking and expansion - YamlSequenceNode node = (YamlSequenceNode)yamlStream.Documents[0].RootNode; - return node.Select(childNode => MethodArguments(testMethod, path, (YamlMappingNode)childNode).ToArray()); + public YamlFileDataAttribute(string rootPath, bool withParameterNames = true) { } public override IEnumerable GetData(MethodInfo testMethod) { - if (RootPath.EndsWith(".yml")) { - return FileData(testMethod, RootPath); - } else { - var filePaths = Directory.GetFiles(RootPath, "*.yml", SearchOption.AllDirectories) - .Select(path => Path.GetRelativePath(RootPath, path)); - return filePaths.SelectMany(path => FileData(testMethod, path)); - } + throw new NotImplementedException(); } } } \ No newline at end of file diff --git a/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs new file mode 100644 index 00000000000..15316d38e46 --- /dev/null +++ b/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs @@ -0,0 +1,122 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; + +namespace XUnitExtensions { + public class YamlFileDataDiscoverer : IDataDiscoverer { + public IParser GetYamlParser(string filePath) { + return new Parser(new StreamReader(filePath)); + } + + public IDeserializer GetDeserializer() { + return new Deserializer(); + } + + public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { + return true; + } + + private class ParsingEventBuffer : IEmitter { + public List ParsingEvents = new List(); + + public void Emit(ParsingEvent parsingEvent) { + ParsingEvents.Add(parsingEvent); + } + } + + private class ParsingEventListParser : IParser { + private readonly IEnumerator Enumerator; + + public ParsingEventListParser(IEnumerator enumerator) { + Enumerator = enumerator; + } + + public bool MoveNext() { + return Enumerator.MoveNext(); + } + + public ParsingEvent Current => Enumerator.Current; + } + + private IParser NodeParser(YamlNode node) { + ParsingEventBuffer buffer = new ParsingEventBuffer(); + YamlDocument doc = new YamlDocument(node); + YamlStream stream = new YamlStream(); + stream.Add(doc); + stream.Save(buffer, false); + return new ParsingEventListParser(buffer.ParsingEvents.GetEnumerator()); + } + + private object DeserializeParameter(IDeserializer deserializer, ParameterInfo parameterInfo, YamlNode mappingNode) { + return deserializer.Deserialize(NodeParser(mappingNode), parameterInfo.ParameterType); + } + + private IEnumerable MethodArguments(IDeserializer deserializer, MethodInfo testMethod, string path, bool withParameterNames, YamlMappingNode mappingNode) { + IEnumerable result; + IEnumerable parameters = testMethod.GetParameters(); + bool withSourceFile = parameters.First().ParameterType.Equals(typeof(ISourceInformation)); + if (withSourceFile) { + parameters = parameters.Skip(1); + } + if (withParameterNames) { + result = parameters.Select(param => DeserializeParameter(deserializer, param, mappingNode[param.Name])).ToArray(); + } else { + result = new[] {DeserializeParameter(deserializer, parameters.Single(), mappingNode)}; + } + + return withSourceFile ? new[] {new SourceFile(path)}.Concat(result) : result; + } + + private IEnumerable FileData(IDeserializer deserializer, MethodInfo testMethod, string path, bool withParameterNames) { + var yamlStream = new YamlStream(); + yamlStream.Load(new StreamReader(path)); + // TODO: error checking and expansion + YamlSequenceNode node = (YamlSequenceNode)yamlStream.Documents[0].RootNode; + return node.Select(childNode => MethodArguments(deserializer, testMethod, path, withParameterNames, (YamlMappingNode)childNode).ToArray()); + } + + public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo testMethod) { + // YamlDotNet's deserialization framework requires runtime type information + MethodInfo methodInfo = testMethod.ToRuntimeMethod(); + if (methodInfo == null) { + return null; + } + + IDeserializer deserializer = GetDeserializer(); + + List attributeArgs = attributeInfo.GetConstructorArguments().ToList(); + string rootPath = (string)attributeArgs[0]; + bool withParameterNames = (bool)attributeArgs[1]; + + if (rootPath.EndsWith(".yml")) { + return FileData(deserializer, methodInfo, rootPath, withParameterNames); + } else { + var filePaths = Directory.GetFiles(rootPath, "*.yml", SearchOption.AllDirectories) + .Select(path => Path.GetRelativePath(rootPath, path)); + return filePaths.SelectMany(path => FileData(deserializer, methodInfo, path, withParameterNames)); + } + } + } + + public class SourceFile : SourceInformation { + + public SourceFile() { + + } + + public SourceFile(string fileName) { + FileName = fileName; + } + + public override string ToString() { + return FileName; + } + } +} diff --git a/Source/DafnyTests/YamlUtils.cs b/Source/DafnyTests/YamlUtils.cs index 87432ac5ac3..9bbaa1a0617 100644 --- a/Source/DafnyTests/YamlUtils.cs +++ b/Source/DafnyTests/YamlUtils.cs @@ -1,10 +1,9 @@ -using System; using System.Collections.Generic; using System.IO; using System.Linq; using Xunit; +using Xunit.Abstractions; using XUnitExtensions; -using YamlDotNet.Core; using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; @@ -61,43 +60,25 @@ public void CalculatorTest(int lhs, int rhs, int expected) { } [Theory] - [YamlFileData("YamlTests/configs.yml", withParameterNames: false, withSourceFile: true)] - public void DictionaryTest(string path, Dictionary config) { + [YamlFileData("YamlTests/configs.yml", withParameterNames: false)] + public void DictionaryTest(ISourceInformation source, Dictionary config) { Assert.Equal(3, config.Count); } - private class MyConfigTypeConvertor : IYamlTypeConverter { + public class ExpandList : List { - public bool Accepts(Type type) { - return type == typeof (MyConfig); - } - - public object ReadYaml(IParser parser, Type type) { - Dictionary config = new Deserializer().Deserialize>(parser); - return new MyConfig(config); - } - - public void WriteYaml(IEmitter emitter, object value, Type type) { - throw new NotImplementedException(); - } } - - private static readonly IDeserializer DESERIALIZER = new DeserializerBuilder() - .WithTypeConverter(new MyConfigTypeConvertor()) - .Build(); - - public class MyConfig { - - public MyConfig(Dictionary config) { - Config = config; - } - - private Dictionary Config; - public override string ToString() { - return ""; - } + [Fact] + public void TagTest() { + var yaml = @" +!foreach [a, b, c] +"; + var deserializer = new DeserializerBuilder() + .WithTagMapping("!foreach", typeof(ExpandList)) + .Build(); + var value = deserializer.Deserialize(new StringReader(yaml)); + Assert.IsType>(value); } - } } \ No newline at end of file From 0bdfddae7d07dc9e2d30a2aefb8be81c06c10422 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 2 Oct 2020 15:32:13 -0700 Subject: [PATCH 064/192] Hook up GetYamlParser (whew) --- .../XUnitExtensions/YamlFileDataDiscoverer.cs | 136 +++++++++++------- 1 file changed, 81 insertions(+), 55 deletions(-) diff --git a/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs index 15316d38e46..9fa2eb89ed0 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -8,6 +9,9 @@ using YamlDotNet.Core.Events; using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NodeDeserializers; +using YamlDotNet.Serialization.ObjectFactories; +using YamlDotNet.Serialization.Utilities; namespace XUnitExtensions { public class YamlFileDataDiscoverer : IDataDiscoverer { @@ -19,67 +23,47 @@ public IDeserializer GetDeserializer() { return new Deserializer(); } - public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { - return true; - } - - private class ParsingEventBuffer : IEmitter { - public List ParsingEvents = new List(); - - public void Emit(ParsingEvent parsingEvent) { - ParsingEvents.Add(parsingEvent); - } - } - - private class ParsingEventListParser : IParser { - private readonly IEnumerator Enumerator; - - public ParsingEventListParser(IEnumerator enumerator) { - Enumerator = enumerator; - } - - public bool MoveNext() { - return Enumerator.MoveNext(); - } - - public ParsingEvent Current => Enumerator.Current; + private Func ForMethodInfoDeserializeFn(IDeserializer deserializer, MethodInfo testMethod, string path) { + return (parser, type) => { + if (type == typeof(MethodArguments)) { + MethodArguments arguments = new MethodArguments(testMethod, path); + arguments.Read(parser, type, t => deserializer.Deserialize(parser, t)); + return arguments; + } else { + return deserializer.Deserialize(parser, type); + } + }; } - private IParser NodeParser(YamlNode node) { - ParsingEventBuffer buffer = new ParsingEventBuffer(); - YamlDocument doc = new YamlDocument(node); - YamlStream stream = new YamlStream(); - stream.Add(doc); - stream.Save(buffer, false); - return new ParsingEventListParser(buffer.ParsingEvents.GetEnumerator()); - } - - private object DeserializeParameter(IDeserializer deserializer, ParameterInfo parameterInfo, YamlNode mappingNode) { - return deserializer.Deserialize(NodeParser(mappingNode), parameterInfo.ParameterType); + public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { + return true; } - - private IEnumerable MethodArguments(IDeserializer deserializer, MethodInfo testMethod, string path, bool withParameterNames, YamlMappingNode mappingNode) { - IEnumerable result; - IEnumerable parameters = testMethod.GetParameters(); - bool withSourceFile = parameters.First().ParameterType.Equals(typeof(ISourceInformation)); - if (withSourceFile) { - parameters = parameters.Skip(1); - } + private IEnumerable FileData(IDeserializer deserializer, MethodInfo testMethod, string path, bool withParameterNames) { + IParser parser = GetYamlParser(path); + parser.Consume(); + parser.Consume(); + if (withParameterNames) { - result = parameters.Select(param => DeserializeParameter(deserializer, param, mappingNode[param.Name])).ToArray(); + IObjectFactory argumentsFactory = new DefaultObjectFactory(); + INodeDeserializer collectionDeserializer = new CollectionNodeDeserializer(argumentsFactory); + if (collectionDeserializer.Deserialize(parser, typeof(List), ForMethodInfoDeserializeFn(deserializer, testMethod, path), out var value)) { + List argumentses = (List) value; + return argumentses.Select(a => a.Arguments); + } else { + throw new ArgumentException(); + } } else { - result = new[] {DeserializeParameter(deserializer, parameters.Single(), mappingNode)}; + IEnumerable parameters = testMethod.GetParameters(); + bool withSourceFile = parameters.First().ParameterType.Equals(typeof(ISourceInformation)); + if (withSourceFile) { + parameters = parameters.Skip(1); + } + IEnumerable results = (IEnumerable)deserializer.Deserialize(parser, typeof(List<>).MakeGenericType(parameters.Single().ParameterType)); + + return withSourceFile ? + results.Select(value => new[]{ new SourceFile(path), value }) : + results.Select(value => new[]{ value }); } - - return withSourceFile ? new[] {new SourceFile(path)}.Concat(result) : result; - } - - private IEnumerable FileData(IDeserializer deserializer, MethodInfo testMethod, string path, bool withParameterNames) { - var yamlStream = new YamlStream(); - yamlStream.Load(new StreamReader(path)); - // TODO: error checking and expansion - YamlSequenceNode node = (YamlSequenceNode)yamlStream.Documents[0].RootNode; - return node.Select(childNode => MethodArguments(deserializer, testMethod, path, withParameterNames, (YamlMappingNode)childNode).ToArray()); } public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo testMethod) { @@ -104,6 +88,48 @@ public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo t } } } + + public class MethodArguments : IYamlConvertible { + + private MethodInfo Method; + private Dictionary ParameterNameIndexes; + public object[] Arguments { get; } + + public MethodArguments(MethodInfo methodInfo, string path) { + Method = methodInfo; + Arguments = new object[Method.GetParameters().Length]; + ParameterNameIndexes = Method.GetParameters() + .Select((value, index) => new KeyValuePair(value.Name, index)) + .ToDictionary(p => p.Key, p => p.Value); + var sourceFileParameter = Method.GetParameters().FirstOrDefault(p => p.ParameterType.Equals(typeof(ISourceInformation))); + if (sourceFileParameter != null) { + Arguments[ParameterNameIndexes[sourceFileParameter.Name]] = new SourceFile(path); + } + } + + public void Read(IParser parser, Type expectedType, ObjectDeserializer nestedObjectDeserializer) { + if (!parser.TryConsume(out MappingStart _)) { + throw new ArgumentException(); + } + while (!parser.TryConsume(out MappingEnd _)) { + Scalar scalar = parser.Consume(); + int parameterIndex = ParameterNameIndexes[scalar.Value]; + Type parameterType = Method.GetParameters()[parameterIndex].ParameterType; + + object argument = nestedObjectDeserializer(parameterType); + IValuePromise valuePromise = argument as IValuePromise; + if (valuePromise != null) { + valuePromise.ValueAvailable += (Action) (v => Arguments[parameterIndex] = TypeConverter.ChangeType(v, parameterType)); + } else { + Arguments[parameterIndex] = TypeConverter.ChangeType(argument, parameterType); + } + } + } + + public void Write(IEmitter emitter, ObjectSerializer nestedObjectSerializer) { + throw new NotImplementedException(); + } + } public class SourceFile : SourceInformation { From 55096e9beaee44acafe0f458d5a5c531c2c711f7 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 3 Oct 2020 09:11:46 -0700 Subject: [PATCH 065/192] Add ForEachAttribute --- .../XUnitExtensions/ForEachAttribute.cs | 17 ++++++++++ .../XUnitExtensions/YamlFileDataAttribute.cs | 8 ----- .../XUnitExtensions/YamlFileDataDiscoverer.cs | 31 ++++++++++++++----- .../YamlTests/calculator-combinatorial.yml | 2 ++ Source/DafnyTests/YamlUtils.cs | 6 ++++ 5 files changed, 49 insertions(+), 15 deletions(-) create mode 100644 Source/DafnyTests/XUnitExtensions/ForEachAttribute.cs create mode 100644 Source/DafnyTests/YamlTests/calculator-combinatorial.yml diff --git a/Source/DafnyTests/XUnitExtensions/ForEachAttribute.cs b/Source/DafnyTests/XUnitExtensions/ForEachAttribute.cs new file mode 100644 index 00000000000..4a9a9487e4c --- /dev/null +++ b/Source/DafnyTests/XUnitExtensions/ForEachAttribute.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; + +namespace XUnitExtensions { + public class ForEachAttribute : Attribute { + + private readonly Type EnumeratorClass; + + public ForEachAttribute(Type enumeratorClass = null) { + EnumeratorClass = enumeratorClass ?? typeof(IEnumerable<>); + } + + public Type EnumerableTypeOf(Type elementType) { + return EnumeratorClass.MakeGenericType(elementType); + } + } +} \ No newline at end of file diff --git a/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs b/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs index 218fd177be6..1962b0672c7 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs @@ -1,15 +1,7 @@ using System; using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Reflection; -using DafnyTests; -using Xunit.Abstractions; using Xunit.Sdk; -using YamlDotNet.Core; -using YamlDotNet.Core.Events; -using YamlDotNet.RepresentationModel; -using YamlDotNet.Serialization; namespace XUnitExtensions { diff --git a/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs index 9fa2eb89ed0..b740dbaaf6a 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs @@ -1,13 +1,14 @@ using System; +using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; using System.Reflection; +using DafnyTests; using Xunit.Abstractions; using Xunit.Sdk; using YamlDotNet.Core; using YamlDotNet.Core.Events; -using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NodeDeserializers; using YamlDotNet.Serialization.ObjectFactories; @@ -15,11 +16,11 @@ namespace XUnitExtensions { public class YamlFileDataDiscoverer : IDataDiscoverer { - public IParser GetYamlParser(string filePath) { + public virtual IParser GetYamlParser(string filePath) { return new Parser(new StreamReader(filePath)); } - public IDeserializer GetDeserializer() { + public virtual IDeserializer GetDeserializer() { return new Deserializer(); } @@ -46,9 +47,10 @@ private IEnumerable FileData(IDeserializer deserializer, MethodInfo te if (withParameterNames) { IObjectFactory argumentsFactory = new DefaultObjectFactory(); INodeDeserializer collectionDeserializer = new CollectionNodeDeserializer(argumentsFactory); - if (collectionDeserializer.Deserialize(parser, typeof(List), ForMethodInfoDeserializeFn(deserializer, testMethod, path), out var value)) { + var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, testMethod, path); + if (collectionDeserializer.Deserialize(parser, typeof(List), nestedObjectDeserializer, out var value)) { List argumentses = (List) value; - return argumentses.Select(a => a.Arguments); + return argumentses.SelectMany(a => a.Combinations()); } else { throw new ArgumentException(); } @@ -93,7 +95,7 @@ public class MethodArguments : IYamlConvertible { private MethodInfo Method; private Dictionary ParameterNameIndexes; - public object[] Arguments { get; } + private object[] Arguments; public MethodArguments(MethodInfo methodInfo, string path) { Method = methodInfo; @@ -114,7 +116,12 @@ public void Read(IParser parser, Type expectedType, ObjectDeserializer nestedObj while (!parser.TryConsume(out MappingEnd _)) { Scalar scalar = parser.Consume(); int parameterIndex = ParameterNameIndexes[scalar.Value]; - Type parameterType = Method.GetParameters()[parameterIndex].ParameterType; + ParameterInfo parameter = Method.GetParameters()[parameterIndex]; + Type parameterType = parameter.ParameterType; + ForEachAttribute forEach = parameter.GetCustomAttribute(); + if (forEach != null) { + parameterType = forEach.EnumerableTypeOf(parameterType); + } object argument = nestedObjectDeserializer(parameterType); IValuePromise valuePromise = argument as IValuePromise; @@ -129,6 +136,16 @@ public void Read(IParser parser, Type expectedType, ObjectDeserializer nestedObj public void Write(IEmitter emitter, ObjectSerializer nestedObjectSerializer) { throw new NotImplementedException(); } + + public IEnumerable Combinations() { + // TODO: Optimize this away when there are no ForEach attributes + IEnumerable> lifted = Arguments.Select((arg, index) => { + ParameterInfo parameter = Method.GetParameters()[index]; + ForEachAttribute forEach = parameter.GetCustomAttribute(); + return forEach == null ? new[] {arg} : ((IEnumerable)arg).Cast(); + }); + return lifted.CartesianProduct().Select(e => e.ToArray()); + } } public class SourceFile : SourceInformation { diff --git a/Source/DafnyTests/YamlTests/calculator-combinatorial.yml b/Source/DafnyTests/YamlTests/calculator-combinatorial.yml new file mode 100644 index 00000000000..d363f891c5a --- /dev/null +++ b/Source/DafnyTests/YamlTests/calculator-combinatorial.yml @@ -0,0 +1,2 @@ +- lhs: [0, 1, 2, 3, 4] + rhs: [0, 1, 2, 3] diff --git a/Source/DafnyTests/YamlUtils.cs b/Source/DafnyTests/YamlUtils.cs index 9bbaa1a0617..8931137d50a 100644 --- a/Source/DafnyTests/YamlUtils.cs +++ b/Source/DafnyTests/YamlUtils.cs @@ -59,6 +59,12 @@ public void CalculatorTest(int lhs, int rhs, int expected) { Assert.Equal(expected, lhs + rhs); } + [Theory] + [YamlFileData("YamlTests/calculator-combinatorial.yml")] + public void CalculatorCombinatorialTest([ForEach()] int lhs, [ForEach()] int rhs) { + Assert.Equal(rhs + lhs, lhs + rhs); + } + [Theory] [YamlFileData("YamlTests/configs.yml", withParameterNames: false)] public void DictionaryTest(ISourceInformation source, Dictionary config) { From b6424aaedde72491dee9a1f14d4e88c1626349d4 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 3 Oct 2020 09:16:44 -0700 Subject: [PATCH 066/192] Move EnumerableUtils inside XunitExtensions --- Source/DafnyTests/{ => XUnitExtensions}/EnumerableUtils.cs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Source/DafnyTests/{ => XUnitExtensions}/EnumerableUtils.cs (100%) diff --git a/Source/DafnyTests/EnumerableUtils.cs b/Source/DafnyTests/XUnitExtensions/EnumerableUtils.cs similarity index 100% rename from Source/DafnyTests/EnumerableUtils.cs rename to Source/DafnyTests/XUnitExtensions/EnumerableUtils.cs From abcfc8e7e9c8499d5e7efda58ac113b33648a081 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 3 Oct 2020 11:51:32 -0700 Subject: [PATCH 067/192] Add custom data discoverer test --- .../YamlTests/calculator-tagged.yml | 6 +++ Source/DafnyTests/YamlUtils.cs | 48 +++++++++++++++---- 2 files changed, 44 insertions(+), 10 deletions(-) create mode 100644 Source/DafnyTests/YamlTests/calculator-tagged.yml diff --git a/Source/DafnyTests/YamlTests/calculator-tagged.yml b/Source/DafnyTests/YamlTests/calculator-tagged.yml new file mode 100644 index 00000000000..25bb6deb6e3 --- /dev/null +++ b/Source/DafnyTests/YamlTests/calculator-tagged.yml @@ -0,0 +1,6 @@ +- lhs: !range + start: 0 + end: 5 + rhs: !range + start: 0 + end: 4 diff --git a/Source/DafnyTests/YamlUtils.cs b/Source/DafnyTests/YamlUtils.cs index 8931137d50a..6235d3e2373 100644 --- a/Source/DafnyTests/YamlUtils.cs +++ b/Source/DafnyTests/YamlUtils.cs @@ -1,11 +1,17 @@ +using System; +using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using Xunit; using Xunit.Abstractions; +using Xunit.Sdk; using XUnitExtensions; +using YamlDotNet.Core; using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; namespace DafnyTests { public class YamlUtils { @@ -75,16 +81,38 @@ public class ExpandList : List { } - [Fact] - public void TagTest() { - var yaml = @" -!foreach [a, b, c] -"; - var deserializer = new DeserializerBuilder() - .WithTagMapping("!foreach", typeof(ExpandList)) - .Build(); - var value = deserializer.Deserialize(new StringReader(yaml)); - Assert.IsType>(value); + [DataDiscoverer("DafnyTests.CustomDiscoverer", "DafnyTests")] + public class CustomYamlFileDataAttribute : YamlFileDataAttribute { + public CustomYamlFileDataAttribute(string rootPath, bool withParameterNames = true) : base(rootPath, withParameterNames) { + } + } + + [Theory] + [CustomYamlFileData("YamlTests/calculator-tagged.yml")] + public void CustomDataDiscovererTest([ForEach()] int lhs, [ForEach()] int rhs) { + Assert.Equal(rhs + lhs, lhs + rhs); + } + } + + public class Range : IEnumerable { + + public int Start; + public int End; + + + public IEnumerator GetEnumerator() { + return Enumerable.Range(Start, End).GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() { + return GetEnumerator(); + } + } + + public class CustomDiscoverer : YamlFileDataDiscoverer { + public override IDeserializer GetDeserializer() { + return new DeserializerBuilder().WithTagMapping("!range", typeof(Range)) + .WithNamingConvention(CamelCaseNamingConvention.Instance).Build(); } } } \ No newline at end of file From a6627c9dc4db34efbe91ab71b9ef0cfc69eab833 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 3 Oct 2020 17:17:17 -0700 Subject: [PATCH 068/192] Use resources instead of files --- Source/DafnyTests/DafnyTestCase.cs | 32 +++++- Source/DafnyTests/DafnyTests.csproj | 5 +- ...eDataAttribute.cs => YamlDataAttribute.cs} | 6 +- ...ataDiscoverer.cs => YamlDataDiscoverer.cs} | 33 +++---- .../XUnitExtensions/YamlDataTests.cs | 97 +++++++++++++++++++ .../CalculatorCombinatorialTest.yml} | 0 .../YamlDataTests/CalculatorTest.yml} | 0 .../CustomDataDiscovererTest.yml} | 0 .../YamlDataTests/DictionaryTest.yml} | 0 .../DafnyTests/YamlTests/multiple.expect.yml | 8 -- Source/DafnyTests/YamlTests/multiple.yml | 2 - Source/DafnyTests/YamlUtils.cs | 76 --------------- 12 files changed, 150 insertions(+), 109 deletions(-) rename Source/DafnyTests/XUnitExtensions/{YamlFileDataAttribute.cs => YamlDataAttribute.cs} (55%) rename Source/DafnyTests/XUnitExtensions/{YamlFileDataDiscoverer.cs => YamlDataDiscoverer.cs} (84%) create mode 100644 Source/DafnyTests/XUnitExtensions/YamlDataTests.cs rename Source/DafnyTests/{YamlTests/calculator-combinatorial.yml => XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml} (100%) rename Source/DafnyTests/{YamlTests/calculator.yml => XUnitExtensions/YamlDataTests/CalculatorTest.yml} (100%) rename Source/DafnyTests/{YamlTests/calculator-tagged.yml => XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml} (100%) rename Source/DafnyTests/{YamlTests/configs.yml => XUnitExtensions/YamlDataTests/DictionaryTest.yml} (100%) delete mode 100644 Source/DafnyTests/YamlTests/multiple.expect.yml delete mode 100644 Source/DafnyTests/YamlTests/multiple.yml diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 3da201c1bbc..fc8389c3403 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -5,7 +5,9 @@ using System.Linq; using Xunit; using Xunit.Abstractions; +using Xunit.Sdk; using XUnitExtensions; +using YamlDotNet.Core; using YamlDotNet.RepresentationModel; namespace DafnyTests { @@ -262,6 +264,32 @@ public void Run() { } } + public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { + public override IParser GetYamlParser(Stream stream) { + // TODO-RS: Figure out how to do this cleanly on a TextReader instead, + // and to handle things like nested comments. + string fullText = new StreamReader(stream).ReadToEnd(); + var commentStart = fullText.IndexOf("/*"); + if (commentStart >= 0) { + var commentEnd = fullText.IndexOf("*/", commentStart + 2); + if (commentEnd >= 0) { + var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); + if (commentContent.StartsWith("---")) { + return new Parser(new StringReader(commentContent)); + } + } + } + + return null; + } + } + + [DataDiscoverer("DafnyTests.DafnyTestYamlFileDataDiscoverer", "DafnyTests")] + public class DafnyTestDataAttribute : YamlDataAttribute { + public DafnyTestDataAttribute(bool withParameterNames = true) : base(withParameterNames) { + } + } + public class DafnyTests { public static IEnumerable AllTestFiles() { @@ -271,8 +299,8 @@ public static IEnumerable AllTestFiles() { } [ParallelTheory] - [MemberData(nameof(AllTestFiles))] - public static void Test(string[] args, DafnyTestCase.Expectation expected) { + [DafnyTestData()] + public static void Test(string[] args, [ForEach()] DafnyTestCase.Expectation expected) { var testCase = new DafnyTestCase(); testCase.DafnyFile = args[0]; testCase.Arguments = args.Skip(1).ToArray(); diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index d727a72f524..b16adae698b 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -18,7 +18,10 @@ - + + + + diff --git a/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs b/Source/DafnyTests/XUnitExtensions/YamlDataAttribute.cs similarity index 55% rename from Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs rename to Source/DafnyTests/XUnitExtensions/YamlDataAttribute.cs index 1962b0672c7..ea57d54d10b 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlFileDataAttribute.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlDataAttribute.cs @@ -5,10 +5,10 @@ namespace XUnitExtensions { - [DataDiscoverer("XUnitExtensions.YamlFileDataDiscoverer", "DafnyTests")] - public class YamlFileDataAttribute : DataAttribute { + [DataDiscoverer("XUnitExtensions.YamlDataDiscoverer", "DafnyTests")] + public class YamlDataAttribute : DataAttribute { - public YamlFileDataAttribute(string rootPath, bool withParameterNames = true) { + public YamlDataAttribute(bool withParameterNames = true) { } public override IEnumerable GetData(MethodInfo testMethod) { diff --git a/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs similarity index 84% rename from Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs rename to Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs index b740dbaaf6a..5524ecadeae 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlFileDataDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs @@ -15,9 +15,9 @@ using YamlDotNet.Serialization.Utilities; namespace XUnitExtensions { - public class YamlFileDataDiscoverer : IDataDiscoverer { - public virtual IParser GetYamlParser(string filePath) { - return new Parser(new StreamReader(filePath)); + public class YamlDataDiscoverer : IDataDiscoverer { + public virtual IParser GetYamlParser(Stream stream) { + return new Parser(new StreamReader(stream)); } public virtual IDeserializer GetDeserializer() { @@ -39,15 +39,17 @@ private Func ForMethodInfoDeserializeFn(IDeserializer des public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { return true; } - private IEnumerable FileData(IDeserializer deserializer, MethodInfo testMethod, string path, bool withParameterNames) { - IParser parser = GetYamlParser(path); + + private IEnumerable ResourceData(IDeserializer deserializer, MethodInfo testMethod, string resourceName, bool withParameterNames) { + Stream stream = testMethod.DeclaringType.Assembly.GetManifestResourceStream(resourceName); + IParser parser = GetYamlParser(stream); parser.Consume(); parser.Consume(); if (withParameterNames) { IObjectFactory argumentsFactory = new DefaultObjectFactory(); INodeDeserializer collectionDeserializer = new CollectionNodeDeserializer(argumentsFactory); - var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, testMethod, path); + var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, testMethod, resourceName); if (collectionDeserializer.Deserialize(parser, typeof(List), nestedObjectDeserializer, out var value)) { List argumentses = (List) value; return argumentses.SelectMany(a => a.Combinations()); @@ -63,7 +65,7 @@ private IEnumerable FileData(IDeserializer deserializer, MethodInfo te IEnumerable results = (IEnumerable)deserializer.Deserialize(parser, typeof(List<>).MakeGenericType(parameters.Single().ParameterType)); return withSourceFile ? - results.Select(value => new[]{ new SourceFile(path), value }) : + results.Select(value => new[]{ new SourceFile(resourceName), value }) : results.Select(value => new[]{ value }); } } @@ -78,16 +80,13 @@ public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo t IDeserializer deserializer = GetDeserializer(); List attributeArgs = attributeInfo.GetConstructorArguments().ToList(); - string rootPath = (string)attributeArgs[0]; - bool withParameterNames = (bool)attributeArgs[1]; - - if (rootPath.EndsWith(".yml")) { - return FileData(deserializer, methodInfo, rootPath, withParameterNames); - } else { - var filePaths = Directory.GetFiles(rootPath, "*.yml", SearchOption.AllDirectories) - .Select(path => Path.GetRelativePath(rootPath, path)); - return filePaths.SelectMany(path => FileData(deserializer, methodInfo, path, withParameterNames)); - } + bool withParameterNames = (bool)attributeArgs[0]; + + string resourceNamePrefix = methodInfo.DeclaringType.FullName + "." + methodInfo.Name; +// throw new ArgumentException(resourceNamePrefix); + return methodInfo.DeclaringType.Assembly.GetManifestResourceNames() + .Where(n => n.StartsWith(resourceNamePrefix)) + .SelectMany(path => ResourceData(deserializer, methodInfo, path, withParameterNames)); } } diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs b/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs new file mode 100644 index 00000000000..1637bb2dd3a --- /dev/null +++ b/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs @@ -0,0 +1,97 @@ +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; +using XUnitExtensions; +using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace DafnyTests.XUnitExtensions { + public class YamlDataTests { + + [Fact] + public void ExpandTest() { + string multipleYaml = @"a: [b, c] +foo: [bar, blarg] +"; + string multipleYamlExpanded = @"- a: b + foo: bar +- a: b + foo: blarg +- a: c + foo: bar +- a: c + foo: blarg +"; + var yamlStream = new YamlStream(); + yamlStream.Load(new StringReader(multipleYaml)); + var root = yamlStream.Documents[0].RootNode; + var expanded = YamlUtils.Expand(root); + + var outputWriter = new StringWriter(); + var serializer = new SerializerBuilder().Build(); + serializer.Serialize(outputWriter, expanded); + Assert.Equal(multipleYamlExpanded, outputWriter.ToString()); + } + + [Theory] + [YamlData()] + public void CalculatorTest(int lhs, int rhs, int expected) { + Assert.Equal(expected, lhs + rhs); + } + + [Theory] + [YamlData()] + public void CalculatorCombinatorialTest([ForEach()] int lhs, [ForEach()] int rhs) { + Assert.Equal(rhs + lhs, lhs + rhs); + } + + [Theory] + [YamlData(false)] + public void DictionaryTest(ISourceInformation source, Dictionary config) { + Assert.Equal(3, config.Count); + } + + public class ExpandList : List { + + } + + [DataDiscoverer("DafnyTests.XUnitExtensions.CustomDiscoverer", "DafnyTests")] + public class CustomYamlDataAttribute : YamlDataAttribute { + public CustomYamlDataAttribute(bool withParameterNames = true) : base(withParameterNames) { + } + } + + [Theory] + [CustomYamlData()] + public void CustomDataDiscovererTest([ForEach()] int lhs, [ForEach()] int rhs) { + Assert.Equal(rhs + lhs, lhs + rhs); + } + } + + public class Range : IEnumerable { + + public int Start; + public int End; + + + public IEnumerator GetEnumerator() { + return Enumerable.Range(Start, End).GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() { + return GetEnumerator(); + } + } + + public class CustomDiscoverer : YamlDataDiscoverer { + public override IDeserializer GetDeserializer() { + return new DeserializerBuilder().WithTagMapping("!range", typeof(Range)) + .WithNamingConvention(CamelCaseNamingConvention.Instance).Build(); + } + } +} \ No newline at end of file diff --git a/Source/DafnyTests/YamlTests/calculator-combinatorial.yml b/Source/DafnyTests/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml similarity index 100% rename from Source/DafnyTests/YamlTests/calculator-combinatorial.yml rename to Source/DafnyTests/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml diff --git a/Source/DafnyTests/YamlTests/calculator.yml b/Source/DafnyTests/XUnitExtensions/YamlDataTests/CalculatorTest.yml similarity index 100% rename from Source/DafnyTests/YamlTests/calculator.yml rename to Source/DafnyTests/XUnitExtensions/YamlDataTests/CalculatorTest.yml diff --git a/Source/DafnyTests/YamlTests/calculator-tagged.yml b/Source/DafnyTests/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml similarity index 100% rename from Source/DafnyTests/YamlTests/calculator-tagged.yml rename to Source/DafnyTests/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml diff --git a/Source/DafnyTests/YamlTests/configs.yml b/Source/DafnyTests/XUnitExtensions/YamlDataTests/DictionaryTest.yml similarity index 100% rename from Source/DafnyTests/YamlTests/configs.yml rename to Source/DafnyTests/XUnitExtensions/YamlDataTests/DictionaryTest.yml diff --git a/Source/DafnyTests/YamlTests/multiple.expect.yml b/Source/DafnyTests/YamlTests/multiple.expect.yml deleted file mode 100644 index 0596b9d116b..00000000000 --- a/Source/DafnyTests/YamlTests/multiple.expect.yml +++ /dev/null @@ -1,8 +0,0 @@ -- a: b - foo: bar -- a: b - foo: blarg -- a: c - foo: bar -- a: c - foo: blarg diff --git a/Source/DafnyTests/YamlTests/multiple.yml b/Source/DafnyTests/YamlTests/multiple.yml deleted file mode 100644 index e014496347c..00000000000 --- a/Source/DafnyTests/YamlTests/multiple.yml +++ /dev/null @@ -1,2 +0,0 @@ -a: [b, c] -foo: [bar, blarg] \ No newline at end of file diff --git a/Source/DafnyTests/YamlUtils.cs b/Source/DafnyTests/YamlUtils.cs index 6235d3e2373..39f0ddeddc2 100644 --- a/Source/DafnyTests/YamlUtils.cs +++ b/Source/DafnyTests/YamlUtils.cs @@ -38,81 +38,5 @@ private static YamlMappingNode FromPairs(IEnumerable config) { - Assert.Equal(3, config.Count); - } - - public class ExpandList : List { - - } - - [DataDiscoverer("DafnyTests.CustomDiscoverer", "DafnyTests")] - public class CustomYamlFileDataAttribute : YamlFileDataAttribute { - public CustomYamlFileDataAttribute(string rootPath, bool withParameterNames = true) : base(rootPath, withParameterNames) { - } - } - - [Theory] - [CustomYamlFileData("YamlTests/calculator-tagged.yml")] - public void CustomDataDiscovererTest([ForEach()] int lhs, [ForEach()] int rhs) { - Assert.Equal(rhs + lhs, lhs + rhs); - } - } - - public class Range : IEnumerable { - - public int Start; - public int End; - - - public IEnumerator GetEnumerator() { - return Enumerable.Range(Start, End).GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() { - return GetEnumerator(); - } - } - - public class CustomDiscoverer : YamlFileDataDiscoverer { - public override IDeserializer GetDeserializer() { - return new DeserializerBuilder().WithTagMapping("!range", typeof(Range)) - .WithNamingConvention(CamelCaseNamingConvention.Instance).Build(); - } } } \ No newline at end of file From fe15a29a028a7f2fd39fd5702eca0dfab2b1df48 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 5 Oct 2020 09:56:45 -0700 Subject: [PATCH 069/192] Progress on actual dafny test cases --- Source/DafnyTests/DafnyTestCase.cs | 229 +++++++++--------- Source/DafnyTests/DafnyTests.csproj | 1 + .../XUnitExtensions/YamlDataDiscoverer.cs | 12 +- .../XUnitExtensions/YamlDataTests.cs | 29 --- Source/DafnyTests/YamlUtils.cs | 42 ---- 5 files changed, 128 insertions(+), 185 deletions(-) delete mode 100644 Source/DafnyTests/YamlUtils.cs diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index fc8389c3403..61aa2c4d5bf 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -1,17 +1,87 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; +using System.Reflection; using Xunit; using Xunit.Abstractions; using Xunit.Sdk; using XUnitExtensions; using YamlDotNet.Core; using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; namespace DafnyTests { + + public class DafnyTestSpec : IEnumerable { + + public string SourcePath; + public Dictionary DafnyArguments = new Dictionary(); + + public DafnyTestCase.Expectation Expect = new DafnyTestCase.Expectation(); + + public IEnumerator GetEnumerator() { + return ExpandArguments(DafnyArguments) + .SelectMany(args => ResolveCompile(SourcePath, args, Expect)) + .GetEnumerator(); + } + + + private static IEnumerable ResolveCompile(string sourcePath, Dictionary config, DafnyTestCase.Expectation expect) { + var compile = "3"; + if (config.ContainsKey("compile")) { + compile = config["compile"]; + config.Remove("compile"); + } + + if (compile.Equals("3") && !config.ContainsKey("compileTarget")) { + var compileTargets = new[] {"cs", "java", "go", "js"}; + + var justVerify = new Dictionary(config); + justVerify["compile"] = "0"; + justVerify["expect"] = "no"; + yield return new DafnyTestCase(sourcePath, justVerify, null, expect); + + foreach (var compileTarget in compileTargets) { + var withLanguage = new Dictionary(config); + withLanguage["noVerify"] = "yes"; + withLanguage["compile"] = "4"; + yield return new DafnyTestCase(sourcePath, withLanguage, compileTarget, expect); + } + } else { + config.TryGetValue("compileTarget", out var compileTarget); + config["compile"] = compile; + yield return new DafnyTestCase(sourcePath, config, compileTarget, expect); + } + } + + IEnumerator IEnumerable.GetEnumerator() { + return GetEnumerator(); + } + + private static IEnumerable> ExpandArguments(Dictionary arguments) { + return arguments.Select(ExpandValue) + .CartesianProduct() + .Select(e => e.ToDictionary(p => p.Key, p => p.Value)); + } + + public static IEnumerable Expand(object obj) { + if (obj is IEnumerable e) { + return e; + } else { + return new[] {(string)obj}; + } + } + + private static IEnumerable> ExpandValue(KeyValuePair pair) { + return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); + } + } + public class DafnyTestCase { private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); @@ -39,21 +109,22 @@ public class Expectation : IXunitSerializable { public bool SpecialCase = false; - public Expectation(string dafnyFile, Dictionary config, string compileTarget) { - config.TryGetValue("expect", out OutputFile); - config.Remove("expect"); - if (OutputFile == null) { - OutputFile = dafnyFile + ".expect"; + public Expectation Resolve(string sourceFile, string compileTarget) { + Expectation result = (Expectation)MemberwiseClone(); + if (result.OutputFile == null) { + result.OutputFile = sourceFile + ".expect"; if (compileTarget != null) { - var specialCasePath = dafnyFile + "." + compileTarget + ".expect"; + var specialCasePath = sourceFile + "." + compileTarget + ".expect"; if (File.Exists(Path.Combine(TEST_ROOT, specialCasePath))) { - SpecialCase = true; - OutputFile = specialCasePath; + result.SpecialCase = true; + result.OutputFile = specialCasePath; } } - } else if (OutputFile.Equals("no")) { - OutputFile = null; + } else if (result.OutputFile.Equals("no")) { + result.OutputFile = null; } + + return result; } public Expectation() { @@ -79,22 +150,19 @@ public override string ToString() { public Expectation Expected; - public DafnyTestCase(string dafnyFile, Dictionary config, string compileTarget) { - Expected = new Expectation(dafnyFile, config, compileTarget); + public DafnyTestCase(string dafnyFile, Dictionary config, string compileTarget, Expectation expected) { + Expected = expected.Resolve(dafnyFile, compileTarget); DafnyFile = dafnyFile; Arguments = config.Select(ConfigPairToArgument).ToArray(); - var fullDafnyFilePath = Path.Combine(TEST_ROOT, DafnyFile); if (compileTarget != null) { Arguments = Arguments.Concat(new[] {"/compileTarget:" + compileTarget}).ToArray(); // Include any additional files - var additionalFilesPath = fullDafnyFilePath + "." + compileTarget + ".files"; - if (Directory.Exists(additionalFilesPath)) { - var relativePaths = Directory.GetFiles(additionalFilesPath) - .Select(path => Path.GetRelativePath(TEST_ROOT, path)); - Arguments = Arguments.Concat(relativePaths).ToArray(); - } + var additionalFilesPath = dafnyFile + "." + compileTarget + ".files"; + IEnumerable additionalFiles = GetType().Assembly.GetManifestResourceNames() + .Where(name => name.StartsWith(additionalFilesPath)); + Arguments = Arguments.Concat(additionalFiles).ToArray(); } } @@ -102,12 +170,8 @@ public DafnyTestCase() { } - public object[] ToMemberData() { - string[] args = new[] {DafnyFile}.Concat(Arguments).ToArray(); - return new object[] {args, Expected}; - } - public static string RunDafny(IEnumerable arguments) { + // TODO-RS: Let these be overridden List dafnyArguments = new List { // Expected output does not contain logo "/nologo", @@ -155,77 +219,6 @@ public static string RunDafny(IEnumerable arguments) { } } - private static string GetTestCaseConfigString(string filePath) { - // TODO-RS: Figure out how to do this cleanly on a TextReader instead, - // and to handle things like nested comments. - string fullText = File.ReadAllText(filePath); - var commentStart = fullText.IndexOf("/*"); - if (commentStart >= 0) { - var commentEnd = fullText.IndexOf("*/", commentStart + 2); - if (commentEnd >= 0) { - var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); - if (commentContent.StartsWith("---")) { - return commentContent; - } - } - } - - return null; - } - - public static IEnumerable TestCasesForDafnyFile(string filePath) { - var fullFilePath = Path.Combine(TEST_ROOT, filePath); - string configString = GetTestCaseConfigString(fullFilePath); - IEnumerable> configs; - if (configString != null) { - var yamlStream = new YamlStream(); - yamlStream.Load(new StringReader(configString)); - var config = yamlStream.Documents[0].RootNode; - configs = YamlUtils.Expand(config).Select(ToDictionary); - } else { - configs = new[] { new Dictionary() }; - } - - return configs.SelectMany(config => ResolveCompile(filePath, config)) - .Select(t => t.ToMemberData()); - } - - private static Dictionary ToDictionary(YamlNode node) { - if (node is YamlMappingNode mapping) { - return mapping.Children.ToDictionary(pair => pair.Key.ToString(), pair => pair.Value.ToString()); - } else { - throw new ArgumentException("Bad test case configuration: " + node); - } - } - - private static IEnumerable ResolveCompile(string filePath, Dictionary config) { - var compile = "3"; - if (config.ContainsKey("compile")) { - compile = config["compile"]; - config.Remove("compile"); - } - - if (compile.Equals("3") && !config.ContainsKey("compileTarget")) { - var compileTargets = new[] {"cs", "java", "go", "js"}; - - var justVerify = new Dictionary(config); - justVerify["compile"] = "0"; - justVerify["expect"] = "no"; - yield return new DafnyTestCase(filePath, justVerify, null); - - foreach (var compileTarget in compileTargets) { - var withLanguage = new Dictionary(config); - withLanguage["noVerify"] = "yes"; - withLanguage["compile"] = "4"; - yield return new DafnyTestCase(filePath, withLanguage, compileTarget); - } - } else { - config.TryGetValue("compileTarget", out var compileTarget); - config["compile"] = compile; - yield return new DafnyTestCase(filePath, config, compileTarget); - } - } - private static string ConfigPairToArgument(KeyValuePair pair) { if (pair.Key.Equals("otherFiles")) { return pair.Value.ToString(); @@ -264,8 +257,18 @@ public void Run() { } } + public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { + + private const string DEFAULT_CONFIG = @" +!dafnyTestSpec +dafnyArguments: + compile: 3 +"; + public override IParser GetYamlParser(Stream stream) { + string content = DEFAULT_CONFIG; + // TODO-RS: Figure out how to do this cleanly on a TextReader instead, // and to handle things like nested comments. string fullText = new StreamReader(stream).ReadToEnd(); @@ -275,36 +278,42 @@ public override IParser GetYamlParser(Stream stream) { if (commentEnd >= 0) { var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); if (commentContent.StartsWith("---")) { - return new Parser(new StringReader(commentContent)); + content = commentContent; } } } - return null; + return new Parser(new StringReader(content)); + } + + public override IDeserializer GetDeserializer() { + return new DeserializerBuilder() + .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); } } - [DataDiscoverer("DafnyTests.DafnyTestYamlFileDataDiscoverer", "DafnyTests")] + [DataDiscoverer("DafnyTests.DafnyTestYamlDataDiscoverer", "DafnyTests")] public class DafnyTestDataAttribute : YamlDataAttribute { public DafnyTestDataAttribute(bool withParameterNames = true) : base(withParameterNames) { } } public class DafnyTests { - - public static IEnumerable AllTestFiles() { - var filePaths = Directory.GetFiles(DafnyTestCase.COMP_DIR, "*.dfy", SearchOption.AllDirectories) - .Select(path => Path.GetRelativePath(DafnyTestCase.TEST_ROOT, path)); - return filePaths.SelectMany(DafnyTestCase.TestCasesForDafnyFile); - } + [Fact] + public static void MetaTest() { + var discoverer = new DafnyTestYamlDataDiscoverer(); + var testMethod = typeof(DafnyTests).GetMethod(nameof(Test)); + var dataAttribute = testMethod.GetCustomAttributesData().ToList()[1]; + IEnumerable data = discoverer.GetData(Reflector.Wrap(dataAttribute), Reflector.Wrap(testMethod)); + List list = data.ToList(); + } + [ParallelTheory] - [DafnyTestData()] - public static void Test(string[] args, [ForEach()] DafnyTestCase.Expectation expected) { - var testCase = new DafnyTestCase(); - testCase.DafnyFile = args[0]; - testCase.Arguments = args.Skip(1).ToArray(); - testCase.Expected = expected; + [DafnyTestData(false)] + public static void Test(DafnyTestCase testCase) { testCase.Run(); } } diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index b16adae698b..5547aa6ef16 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -23,6 +23,7 @@ + diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs index 5524ecadeae..9b25b566906 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs @@ -24,7 +24,7 @@ public virtual IDeserializer GetDeserializer() { return new Deserializer(); } - private Func ForMethodInfoDeserializeFn(IDeserializer deserializer, MethodInfo testMethod, string path) { + private static Func ForMethodInfoDeserializeFn(IDeserializer deserializer, MethodInfo testMethod, string path) { return (parser, type) => { if (type == typeof(MethodArguments)) { MethodArguments arguments = new MethodArguments(testMethod, path); @@ -62,7 +62,7 @@ private IEnumerable ResourceData(IDeserializer deserializer, MethodInf if (withSourceFile) { parameters = parameters.Skip(1); } - IEnumerable results = (IEnumerable)deserializer.Deserialize(parser, typeof(List<>).MakeGenericType(parameters.Single().ParameterType)); + IEnumerable results = (IEnumerable)deserializer.Deserialize(parser); return withSourceFile ? results.Select(value => new[]{ new SourceFile(resourceName), value }) : @@ -83,10 +83,14 @@ public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo t bool withParameterNames = (bool)attributeArgs[0]; string resourceNamePrefix = methodInfo.DeclaringType.FullName + "." + methodInfo.Name; -// throw new ArgumentException(resourceNamePrefix); - return methodInfo.DeclaringType.Assembly.GetManifestResourceNames() + string[] resourceNames = methodInfo.DeclaringType.Assembly.GetManifestResourceNames(); + IEnumerable result = methodInfo.DeclaringType.Assembly.GetManifestResourceNames() .Where(n => n.StartsWith(resourceNamePrefix)) .SelectMany(path => ResourceData(deserializer, methodInfo, path, withParameterNames)); + if (!result.Any()) { + throw new ArgumentException("No data found for resource prefix: " + resourceNamePrefix); + } + return result; } } diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs b/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs index 1637bb2dd3a..c81a2fc6907 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs @@ -13,31 +13,6 @@ namespace DafnyTests.XUnitExtensions { public class YamlDataTests { - [Fact] - public void ExpandTest() { - string multipleYaml = @"a: [b, c] -foo: [bar, blarg] -"; - string multipleYamlExpanded = @"- a: b - foo: bar -- a: b - foo: blarg -- a: c - foo: bar -- a: c - foo: blarg -"; - var yamlStream = new YamlStream(); - yamlStream.Load(new StringReader(multipleYaml)); - var root = yamlStream.Documents[0].RootNode; - var expanded = YamlUtils.Expand(root); - - var outputWriter = new StringWriter(); - var serializer = new SerializerBuilder().Build(); - serializer.Serialize(outputWriter, expanded); - Assert.Equal(multipleYamlExpanded, outputWriter.ToString()); - } - [Theory] [YamlData()] public void CalculatorTest(int lhs, int rhs, int expected) { @@ -56,10 +31,6 @@ public void DictionaryTest(ISourceInformation source, Dictionary Assert.Equal(3, config.Count); } - public class ExpandList : List { - - } - [DataDiscoverer("DafnyTests.XUnitExtensions.CustomDiscoverer", "DafnyTests")] public class CustomYamlDataAttribute : YamlDataAttribute { public CustomYamlDataAttribute(bool withParameterNames = true) : base(withParameterNames) { diff --git a/Source/DafnyTests/YamlUtils.cs b/Source/DafnyTests/YamlUtils.cs deleted file mode 100644 index 39f0ddeddc2..00000000000 --- a/Source/DafnyTests/YamlUtils.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; -using XUnitExtensions; -using YamlDotNet.Core; -using YamlDotNet.RepresentationModel; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; - -namespace DafnyTests { - public class YamlUtils { - - public static IEnumerable Expand(YamlNode node) { - if (node is YamlSequenceNode seqNode) { - return seqNode.SelectMany(child => Expand(child)); - } else if (node is YamlMappingNode mappingNode) { - return mappingNode.Select(ExpandValue).CartesianProduct().Select(FromPairs); - } else { - return new[] {node}; - } - } - - private static IEnumerable> ExpandValue(KeyValuePair pair) { - return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); - } - - private static YamlMappingNode FromPairs(IEnumerable> pairs) { - var result = new YamlMappingNode(); - foreach (var pair in pairs) { - result.Add(pair.Key, pair.Value); - } - - return result; - } - } -} \ No newline at end of file From fa68a9751f95c84adf0e0d1c124732e81a084acf Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 5 Oct 2020 11:31:40 -0700 Subject: [PATCH 070/192] Better solution for capturing the resource name --- Source/DafnyTests/DafnyTestCase.cs | 23 ++++++-- .../XUnitExtensions/YamlDataDiscoverer.cs | 53 +++++-------------- .../XUnitExtensions/YamlDataTests.cs | 5 +- 3 files changed, 34 insertions(+), 47 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 61aa2c4d5bf..fb9c8fde160 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -13,6 +13,7 @@ using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; +using YamlDotNet.Serialization.ObjectFactories; namespace DafnyTests { @@ -24,6 +25,10 @@ public class DafnyTestSpec : IEnumerable { public DafnyTestCase.Expectation Expect = new DafnyTestCase.Expectation(); + public DafnyTestSpec(string manifestResourceName) { + SourcePath = manifestResourceName; + } + public IEnumerator GetEnumerator() { return ExpandArguments(DafnyArguments) .SelectMany(args => ResolveCompile(SourcePath, args, Expect)) @@ -221,7 +226,7 @@ public static string RunDafny(IEnumerable arguments) { private static string ConfigPairToArgument(KeyValuePair pair) { if (pair.Key.Equals("otherFiles")) { - return pair.Value.ToString(); + return pair.Value; } else if (pair.Value.Equals("yes")) { return String.Format("/{0}", pair.Key); } else { @@ -266,7 +271,7 @@ public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { compile: 3 "; - public override IParser GetYamlParser(Stream stream) { + public override IParser GetYamlParser(string manifestResourceName, Stream stream) { string content = DEFAULT_CONFIG; // TODO-RS: Figure out how to do this cleanly on a TextReader instead, @@ -286,10 +291,20 @@ public override IParser GetYamlParser(Stream stream) { return new Parser(new StringReader(content)); } - public override IDeserializer GetDeserializer() { + public override IDeserializer GetDeserializer(string manifestResourceName) { + var defaultObjectFactory = new DefaultObjectFactory(); + var customObjectFactory = new LambdaObjectFactory(type => { + if (type == typeof(DafnyTestSpec)) { + return new DafnyTestSpec(manifestResourceName); + } else { + return defaultObjectFactory.Create(type); + } + }); + return new DeserializerBuilder() - .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) .WithNamingConvention(CamelCaseNamingConvention.Instance) + .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) + .WithObjectFactory(customObjectFactory) .Build(); } } diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs index 9b25b566906..63b54888c8b 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs @@ -16,18 +16,18 @@ namespace XUnitExtensions { public class YamlDataDiscoverer : IDataDiscoverer { - public virtual IParser GetYamlParser(Stream stream) { + public virtual IParser GetYamlParser(string manifestResourceName, Stream stream) { return new Parser(new StreamReader(stream)); } - public virtual IDeserializer GetDeserializer() { + public virtual IDeserializer GetDeserializer(string manifestResourceName) { return new Deserializer(); } - private static Func ForMethodInfoDeserializeFn(IDeserializer deserializer, MethodInfo testMethod, string path) { + private static Func ForMethodInfoDeserializeFn(IDeserializer deserializer, MethodInfo testMethod) { return (parser, type) => { if (type == typeof(MethodArguments)) { - MethodArguments arguments = new MethodArguments(testMethod, path); + MethodArguments arguments = new MethodArguments(testMethod); arguments.Read(parser, type, t => deserializer.Deserialize(parser, t)); return arguments; } else { @@ -40,16 +40,17 @@ public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodIn return true; } - private IEnumerable ResourceData(IDeserializer deserializer, MethodInfo testMethod, string resourceName, bool withParameterNames) { + private IEnumerable ResourceData(MethodInfo testMethod, string resourceName, bool withParameterNames) { Stream stream = testMethod.DeclaringType.Assembly.GetManifestResourceStream(resourceName); - IParser parser = GetYamlParser(stream); + IParser parser = GetYamlParser(resourceName, stream); + IDeserializer deserializer = GetDeserializer(resourceName); parser.Consume(); parser.Consume(); if (withParameterNames) { IObjectFactory argumentsFactory = new DefaultObjectFactory(); INodeDeserializer collectionDeserializer = new CollectionNodeDeserializer(argumentsFactory); - var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, testMethod, resourceName); + var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, testMethod); if (collectionDeserializer.Deserialize(parser, typeof(List), nestedObjectDeserializer, out var value)) { List argumentses = (List) value; return argumentses.SelectMany(a => a.Combinations()); @@ -58,15 +59,9 @@ private IEnumerable ResourceData(IDeserializer deserializer, MethodInf } } else { IEnumerable parameters = testMethod.GetParameters(); - bool withSourceFile = parameters.First().ParameterType.Equals(typeof(ISourceInformation)); - if (withSourceFile) { - parameters = parameters.Skip(1); - } - IEnumerable results = (IEnumerable)deserializer.Deserialize(parser); - - return withSourceFile ? - results.Select(value => new[]{ new SourceFile(resourceName), value }) : - results.Select(value => new[]{ value }); + Type targetType = typeof(IEnumerable<>).MakeGenericType(parameters.Single().ParameterType); + IEnumerable results = (IEnumerable) deserializer.Deserialize(parser, targetType); + return results.Select(value => new[] {value}); } } @@ -77,16 +72,13 @@ public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo t return null; } - IDeserializer deserializer = GetDeserializer(); - List attributeArgs = attributeInfo.GetConstructorArguments().ToList(); bool withParameterNames = (bool)attributeArgs[0]; string resourceNamePrefix = methodInfo.DeclaringType.FullName + "." + methodInfo.Name; - string[] resourceNames = methodInfo.DeclaringType.Assembly.GetManifestResourceNames(); IEnumerable result = methodInfo.DeclaringType.Assembly.GetManifestResourceNames() .Where(n => n.StartsWith(resourceNamePrefix)) - .SelectMany(path => ResourceData(deserializer, methodInfo, path, withParameterNames)); + .SelectMany(path => ResourceData(methodInfo, path, withParameterNames)); if (!result.Any()) { throw new ArgumentException("No data found for resource prefix: " + resourceNamePrefix); } @@ -100,16 +92,12 @@ public class MethodArguments : IYamlConvertible { private Dictionary ParameterNameIndexes; private object[] Arguments; - public MethodArguments(MethodInfo methodInfo, string path) { + public MethodArguments(MethodInfo methodInfo) { Method = methodInfo; Arguments = new object[Method.GetParameters().Length]; ParameterNameIndexes = Method.GetParameters() .Select((value, index) => new KeyValuePair(value.Name, index)) .ToDictionary(p => p.Key, p => p.Value); - var sourceFileParameter = Method.GetParameters().FirstOrDefault(p => p.ParameterType.Equals(typeof(ISourceInformation))); - if (sourceFileParameter != null) { - Arguments[ParameterNameIndexes[sourceFileParameter.Name]] = new SourceFile(path); - } } public void Read(IParser parser, Type expectedType, ObjectDeserializer nestedObjectDeserializer) { @@ -150,19 +138,4 @@ public IEnumerable Combinations() { return lifted.CartesianProduct().Select(e => e.ToArray()); } } - - public class SourceFile : SourceInformation { - - public SourceFile() { - - } - - public SourceFile(string fileName) { - FileName = fileName; - } - - public override string ToString() { - return FileName; - } - } } diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs b/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs index c81a2fc6907..14a9ee02dcb 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs @@ -27,7 +27,7 @@ public void CalculatorCombinatorialTest([ForEach()] int lhs, [ForEach()] int rhs [Theory] [YamlData(false)] - public void DictionaryTest(ISourceInformation source, Dictionary config) { + public void DictionaryTest(Dictionary config) { Assert.Equal(3, config.Count); } @@ -49,7 +49,6 @@ public class Range : IEnumerable { public int Start; public int End; - public IEnumerator GetEnumerator() { return Enumerable.Range(Start, End).GetEnumerator(); } @@ -60,7 +59,7 @@ IEnumerator IEnumerable.GetEnumerator() { } public class CustomDiscoverer : YamlDataDiscoverer { - public override IDeserializer GetDeserializer() { + public override IDeserializer GetDeserializer(string manifestResourceName) { return new DeserializerBuilder().WithTagMapping("!range", typeof(Range)) .WithNamingConvention(CamelCaseNamingConvention.Instance).Build(); } From 6dd4597032bba3ac9e9e3668b4367f1231eff770 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 5 Oct 2020 11:57:51 -0700 Subject: [PATCH 071/192] Got an actual Dafny test working again! --- Source/DafnyTests/DafnyTestCase.cs | 35 +++++++++++++------ Source/DafnyTests/DafnyTests.csproj | 1 + .../XUnitExtensions/YamlDataDiscoverer.cs | 3 ++ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index fb9c8fde160..222621e070e 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -48,19 +48,18 @@ private static IEnumerable ResolveCompile(string sourcePath, Dict var justVerify = new Dictionary(config); justVerify["compile"] = "0"; - justVerify["expect"] = "no"; - yield return new DafnyTestCase(sourcePath, justVerify, null, expect); + yield return new DafnyTestCase(sourcePath, justVerify, null, DafnyTestCase.Expectation.NO_OUTPUT); foreach (var compileTarget in compileTargets) { var withLanguage = new Dictionary(config); withLanguage["noVerify"] = "yes"; withLanguage["compile"] = "4"; - yield return new DafnyTestCase(sourcePath, withLanguage, compileTarget, expect); + yield return new DafnyTestCase(sourcePath, withLanguage, compileTarget, expect.Resolve(sourcePath, compileTarget)); } } else { config.TryGetValue("compileTarget", out var compileTarget); config["compile"] = compile; - yield return new DafnyTestCase(sourcePath, config, compileTarget, expect); + yield return new DafnyTestCase(sourcePath, config, compileTarget, expect.Resolve(sourcePath, compileTarget)); } } @@ -114,6 +113,8 @@ public class Expectation : IXunitSerializable { public bool SpecialCase = false; + public static Expectation NO_OUTPUT = new Expectation(0, null); + public Expectation Resolve(string sourceFile, string compileTarget) { Expectation result = (Expectation)MemberwiseClone(); if (result.OutputFile == null) { @@ -132,6 +133,11 @@ public Expectation Resolve(string sourceFile, string compileTarget) { return result; } + public Expectation(int exitCode, string outputFile) { + ExitCode = exitCode; + OutputFile = outputFile; + } + public Expectation() { } @@ -156,7 +162,7 @@ public override string ToString() { public Expectation Expected; public DafnyTestCase(string dafnyFile, Dictionary config, string compileTarget, Expectation expected) { - Expected = expected.Resolve(dafnyFile, compileTarget); + Expected = expected; DafnyFile = dafnyFile; Arguments = config.Select(ConfigPairToArgument).ToArray(); @@ -175,7 +181,7 @@ public DafnyTestCase() { } - public static string RunDafny(IEnumerable arguments) { + public static string RunDafny(string workingDirectory, IEnumerable arguments) { // TODO-RS: Let these be overridden List dafnyArguments = new List { // Expected output does not contain logo @@ -203,7 +209,8 @@ public static string RunDafny(IEnumerable arguments) { dafnyProcess.StartInfo.RedirectStandardError = true; dafnyProcess.StartInfo.CreateNoWindow = true; // Necessary for JS to find bignumber.js - dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; +// dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; + dafnyProcess.StartInfo.WorkingDirectory = workingDirectory; // Only preserve specific whitelisted environment variables dafnyProcess.StartInfo.EnvironmentVariables.Clear(); @@ -239,22 +246,26 @@ public void Run() { string output; if (arguments.Any(arg => arg.StartsWith("/out"))) { - output = RunDafny(arguments); + output = RunDafny(TEST_ROOT, arguments); } else { // Note that the temporary directory has to be an ancestor of Test // or else Javascript won't be able to locate bignumber.js :( using (var tempDir = new TemporaryDirectory(OUTPUT_DIR)) { + using (var fileStream = File.Create(tempDir.DirInfo.FullName + "/" + DafnyFile)) { + GetType().Assembly.GetManifestResourceStream(DafnyFile).CopyTo(fileStream); + } + // Add an extra component to the path to keep the files created inside the // temporary directory, since some compilers will // interpret the path as a single file basename rather than a directory. var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; var dafnyArguments = new []{ outArgument }.Concat(arguments); - output = RunDafny(dafnyArguments); + output = RunDafny(tempDir.DirInfo.FullName, dafnyArguments); } } if (Expected.OutputFile != null) { - var expectedOutput = File.ReadAllText(Path.Combine(TEST_ROOT, Expected.OutputFile)); + var expectedOutput = new StreamReader(GetType().Assembly.GetManifestResourceStream(Expected.OutputFile)).ReadToEnd(); AssertWithDiff.Equal(expectedOutput, output); } @@ -272,6 +283,10 @@ public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { "; public override IParser GetYamlParser(string manifestResourceName, Stream stream) { + if (!manifestResourceName.EndsWith(".dfy")) { + return null; + } + string content = DEFAULT_CONFIG; // TODO-RS: Figure out how to do this cleanly on a TextReader instead, diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index 5547aa6ef16..7d2d65cac5c 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -24,6 +24,7 @@ + diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs index 63b54888c8b..0ea8c865def 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs @@ -43,6 +43,9 @@ public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodIn private IEnumerable ResourceData(MethodInfo testMethod, string resourceName, bool withParameterNames) { Stream stream = testMethod.DeclaringType.Assembly.GetManifestResourceStream(resourceName); IParser parser = GetYamlParser(resourceName, stream); + if (parser == null) { + return Enumerable.Empty(); + } IDeserializer deserializer = GetDeserializer(resourceName); parser.Consume(); parser.Consume(); From b98cc632b5ece0ead7905c96efbeec990947a7fe Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 6 Oct 2020 11:37:54 -0700 Subject: [PATCH 072/192] More test spec fixes --- Test/DafnyTests/out.raw | Bin 2174 -> 0 bytes Test/comp/BranchCoverage.dfy | 6 ++++-- Test/comp/CSharpStyling.dfy | 6 ++++-- Test/comp/GoModule.dfy | 4 +++- Test/comp/JsModule.dfy | 5 +++-- 5 files changed, 14 insertions(+), 7 deletions(-) delete mode 100644 Test/DafnyTests/out.raw diff --git a/Test/DafnyTests/out.raw b/Test/DafnyTests/out.raw deleted file mode 100644 index 84efc0444556367abc90621c98fca5d6b8087db1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2174 zcmeH|Pfx-?5XIlw#P7gSF9iHcxDnJ~0(ww`i8T$TgydG4Uc1qDix9 zJM(6Cc6Z*+^zF5&x&qxOlhcfMpeyCbP6=|RRCDk?dcfCd3SwT7dK%#KqCK>0EE)3x zohTbXc18+*vbkS z5z>GthUH*qKH|hjjhDL%)zlErz8>+ghzmbEqx-~eiIOlgXT3YHhR)U2z1KFAt?zR9 zk(;T{#F*V|Ym3wTMeb^#YQR^#n}+FD;n{I6`i12$oo AD*ylh diff --git a/Test/comp/BranchCoverage.dfy b/Test/comp/BranchCoverage.dfy index b20432221a0..5ec1fb9b86c 100644 --- a/Test/comp/BranchCoverage.dfy +++ b/Test/comp/BranchCoverage.dfy @@ -1,7 +1,9 @@ /* --- -compile: 3 -coverage: "-" +!dafnyTestSpec +dafnyArguments: + compile: 3 + coverage: "-" */ // The Main method is at the end of this file, because that makes it easier to maintain diff --git a/Test/comp/CSharpStyling.dfy b/Test/comp/CSharpStyling.dfy index 226b9662377..2cd92a70e7d 100644 --- a/Test/comp/CSharpStyling.dfy +++ b/Test/comp/CSharpStyling.dfy @@ -1,7 +1,9 @@ /* --- -compile: 3 -compileTarget: cs +!dafnyTestSpec +dafnyArguments: + compile: 3 + compileTarget: cs */ method Main() { diff --git a/Test/comp/GoModule.dfy b/Test/comp/GoModule.dfy index b578a8b7efa..dde4b7f764e 100644 --- a/Test/comp/GoModule.dfy +++ b/Test/comp/GoModule.dfy @@ -1,6 +1,8 @@ /* --- -compileTarget: go +!dafnyTestSpec +dafnyArguments: + compileTarget: go */ // "url" is a built-in package, so it should be accessible to the diff --git a/Test/comp/JsModule.dfy b/Test/comp/JsModule.dfy index 2727a01947c..769d0d57284 100644 --- a/Test/comp/JsModule.dfy +++ b/Test/comp/JsModule.dfy @@ -1,7 +1,8 @@ /* --- -compile: 3 -compileTarget: js +!dafnyTestSpec +dafnyArguments: + compileTarget: js */ // "url" is a built-in package in node, so it should be accessible to the From 970379afe4841d029d84b1b9707f811e57bdce8c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 6 Oct 2020 18:54:00 -0700 Subject: [PATCH 073/192] Got most of the comp tests working again --- Source/DafnyTests/DafnyTestCase.cs | 143 ++++++++++-------- Source/DafnyTests/DafnyTests.csproj | 15 +- .../XUnitExtensions/YamlDataDiscoverer.cs | 59 +++++--- Source/LitTestConverter/LitTestConvertor.cs | 5 +- 4 files changed, 121 insertions(+), 101 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 222621e070e..dee9139f6dd 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -5,28 +5,59 @@ using System.IO; using System.Linq; using System.Reflection; +using Microsoft.Extensions.FileProviders; using Xunit; using Xunit.Abstractions; using Xunit.Sdk; using XUnitExtensions; using YamlDotNet.Core; -using YamlDotNet.RepresentationModel; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; using YamlDotNet.Serialization.ObjectFactories; namespace DafnyTests { + + public class DafnyTestSpec : IEnumerable { + private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); + + // TODO-RS: This is an ugly method of locating the project root. + // The proper fix is to run entirely out of the output directory, + // and the projects are at least partially configured to make that possible, + // but it's not quite working yet. + private static string DAFNY_ROOT = + OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; - public class DafnyTestSpec : IEnumerable { + public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; + public static readonly string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; + + public static readonly string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/dafny"); + private static readonly IFileProvider manifestFileProvider = new ManifestEmbeddedFileProvider( + Assembly.GetExecutingAssembly()); + private static readonly Dictionary PathsForResourceNames = GetPathsForResourceNames( + "DafnyTests", manifestFileProvider, "DafnyTests"); + public string SourcePath; public Dictionary DafnyArguments = new Dictionary(); public DafnyTestCase.Expectation Expect = new DafnyTestCase.Expectation(); public DafnyTestSpec(string manifestResourceName) { - SourcePath = manifestResourceName; + SourcePath = "comp/" + PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); + } + + private static Dictionary GetPathsForResourceNames(string assemblyName, IFileProvider fileProvider, string path = null) { + return fileProvider.GetDirectoryContents(path).SelectMany(file => { + var childName = path == null ? file.Name : path + "/" + file.Name; + if (file.IsDirectory) { + return GetPathsForResourceNames(assemblyName, fileProvider, childName); + } else { + var result = new Dictionary(); + result[assemblyName + "." + childName.Replace("/", ".")] = childName; + return result; + } + }).ToDictionary(pair => pair.Key, pair => pair.Value); } public IEnumerator GetEnumerator() { @@ -35,7 +66,6 @@ public IEnumerator GetEnumerator() { .GetEnumerator(); } - private static IEnumerable ResolveCompile(string sourcePath, Dictionary config, DafnyTestCase.Expectation expect) { var compile = "3"; if (config.ContainsKey("compile")) { @@ -86,24 +116,8 @@ private static IEnumerable> ExpandValue(KeyValuePai } } - public class DafnyTestCase { + public class DafnyTestCase: IXunitSerializable { - private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); - - // TODO-RS: This is an ugly method of locating the project root. - // The proper fix is to run entirely out of the output directory, - // and the projects are at least partially configured to make that possible, - // but it's not quite working yet. - private static string DAFNY_ROOT = - OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; - - public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; - public static string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; - private static string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - - private static string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/dafny"); - - public string DafnyFile; public string[] Arguments; public class Expectation : IXunitSerializable { @@ -121,7 +135,7 @@ public Expectation Resolve(string sourceFile, string compileTarget) { result.OutputFile = sourceFile + ".expect"; if (compileTarget != null) { var specialCasePath = sourceFile + "." + compileTarget + ".expect"; - if (File.Exists(Path.Combine(TEST_ROOT, specialCasePath))) { + if (File.Exists(DafnyTestSpec.TEST_ROOT + specialCasePath)) { result.SpecialCase = true; result.OutputFile = specialCasePath; } @@ -164,24 +178,37 @@ public override string ToString() { public DafnyTestCase(string dafnyFile, Dictionary config, string compileTarget, Expectation expected) { Expected = expected; - DafnyFile = dafnyFile; - Arguments = config.Select(ConfigPairToArgument).ToArray(); + var arguments = new []{ dafnyFile }.Concat(config.Select(ConfigPairToArgument)).ToList(); if (compileTarget != null) { - Arguments = Arguments.Concat(new[] {"/compileTarget:" + compileTarget}).ToArray(); + arguments.Add("/compileTarget:" + compileTarget); // Include any additional files - var additionalFilesPath = dafnyFile + "." + compileTarget + ".files"; - IEnumerable additionalFiles = GetType().Assembly.GetManifestResourceNames() - .Where(name => name.StartsWith(additionalFilesPath)); - Arguments = Arguments.Concat(additionalFiles).ToArray(); + var additionalFilesPath = DafnyTestSpec.TEST_ROOT + dafnyFile + "." + compileTarget + ".files"; + if (Directory.Exists(additionalFilesPath)) { + var relativePaths = Directory.GetFiles(additionalFilesPath) + .Select(path => Path.GetRelativePath(DafnyTestSpec.TEST_ROOT, path)); + arguments.AddRange(relativePaths); + } } + + Arguments = arguments.ToArray(); } public DafnyTestCase() { } + + public void Serialize(IXunitSerializationInfo info) { + info.AddValue(nameof(Arguments), Arguments); + info.AddValue(nameof(Expected), Expected); + } + + public void Deserialize(IXunitSerializationInfo info) { + Arguments = info.GetValue(nameof(Arguments)); + Expected = info.GetValue(nameof(Expected)); + } - public static string RunDafny(string workingDirectory, IEnumerable arguments) { + public static string RunDafny(IEnumerable arguments) { // TODO-RS: Let these be overridden List dafnyArguments = new List { // Expected output does not contain logo @@ -198,8 +225,7 @@ public static string RunDafny(string workingDirectory, IEnumerable argum dafnyArguments.AddRange(arguments); using (Process dafnyProcess = new Process()) { -// dafnyProcess.StartInfo.FileName = "mono"; - dafnyProcess.StartInfo.FileName = DAFNY_EXE; + dafnyProcess.StartInfo.FileName = DafnyTestSpec.DAFNY_EXE; foreach (var argument in dafnyArguments) { dafnyProcess.StartInfo.Arguments += " " + argument; } @@ -209,8 +235,7 @@ public static string RunDafny(string workingDirectory, IEnumerable argum dafnyProcess.StartInfo.RedirectStandardError = true; dafnyProcess.StartInfo.CreateNoWindow = true; // Necessary for JS to find bignumber.js -// dafnyProcess.StartInfo.WorkingDirectory = TEST_ROOT; - dafnyProcess.StartInfo.WorkingDirectory = workingDirectory; + dafnyProcess.StartInfo.WorkingDirectory = DafnyTestSpec.TEST_ROOT; // Only preserve specific whitelisted environment variables dafnyProcess.StartInfo.EnvironmentVariables.Clear(); @@ -232,9 +257,7 @@ public static string RunDafny(string workingDirectory, IEnumerable argum } private static string ConfigPairToArgument(KeyValuePair pair) { - if (pair.Key.Equals("otherFiles")) { - return pair.Value; - } else if (pair.Value.Equals("yes")) { + if (pair.Value.Equals("yes")) { return String.Format("/{0}", pair.Key); } else { return String.Format("/{0}:{1}", pair.Key, pair.Value); @@ -242,44 +265,41 @@ private static string ConfigPairToArgument(KeyValuePair pair) { } public void Run() { - var arguments = new []{ DafnyFile }.Concat(Arguments); - string output; - if (arguments.Any(arg => arg.StartsWith("/out"))) { - output = RunDafny(TEST_ROOT, arguments); + if (Arguments.Any(arg => arg.StartsWith("/out"))) { + output = RunDafny(Arguments); } else { // Note that the temporary directory has to be an ancestor of Test // or else Javascript won't be able to locate bignumber.js :( - using (var tempDir = new TemporaryDirectory(OUTPUT_DIR)) { - using (var fileStream = File.Create(tempDir.DirInfo.FullName + "/" + DafnyFile)) { - GetType().Assembly.GetManifestResourceStream(DafnyFile).CopyTo(fileStream); - } - + using (var tempDir = new TemporaryDirectory(DafnyTestSpec.OUTPUT_DIR)) { // Add an extra component to the path to keep the files created inside the // temporary directory, since some compilers will // interpret the path as a single file basename rather than a directory. var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; - var dafnyArguments = new []{ outArgument }.Concat(arguments); - output = RunDafny(tempDir.DirInfo.FullName, dafnyArguments); + var dafnyArguments = new []{ outArgument }.Concat(Arguments); + output = RunDafny(dafnyArguments); } } if (Expected.OutputFile != null) { - var expectedOutput = new StreamReader(GetType().Assembly.GetManifestResourceStream(Expected.OutputFile)).ReadToEnd(); + var expectedOutput = File.ReadAllText(Path.Combine(DafnyTestSpec.TEST_ROOT, Expected.OutputFile)); AssertWithDiff.Equal(expectedOutput, output); } - Skip.If(Expected.SpecialCase, "Confirmed known exception for arguments: " + String.Join(" ", arguments)); + Skip.If(Expected.SpecialCase, "Confirmed known exception for arguments: " + String.Join(" ", Arguments)); + } + + public override string ToString() { + return String.Join(" ", Arguments) + " => " + Expected; } } public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { - private const string DEFAULT_CONFIG = @" -!dafnyTestSpec -dafnyArguments: - compile: 3 + private const string DEFAULT_CONFIG = +@"!dafnyTestSpec +dafnyArguments: {} "; public override IParser GetYamlParser(string manifestResourceName, Stream stream) { @@ -308,13 +328,8 @@ public override IParser GetYamlParser(string manifestResourceName, Stream stream public override IDeserializer GetDeserializer(string manifestResourceName) { var defaultObjectFactory = new DefaultObjectFactory(); - var customObjectFactory = new LambdaObjectFactory(type => { - if (type == typeof(DafnyTestSpec)) { - return new DafnyTestSpec(manifestResourceName); - } else { - return defaultObjectFactory.Create(type); - } - }); + var customObjectFactory = new LambdaObjectFactory(type => + type == typeof(DafnyTestSpec) ? new DafnyTestSpec(manifestResourceName) : defaultObjectFactory.Create(type)); return new DeserializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) @@ -336,11 +351,9 @@ public class DafnyTests { public static void MetaTest() { var discoverer = new DafnyTestYamlDataDiscoverer(); var testMethod = typeof(DafnyTests).GetMethod(nameof(Test)); - var dataAttribute = testMethod.GetCustomAttributesData().ToList()[1]; - IEnumerable data = discoverer.GetData(Reflector.Wrap(dataAttribute), Reflector.Wrap(testMethod)); - List list = data.ToList(); + discoverer.GetData(testMethod, false).ToList(); } - + [ParallelTheory] [DafnyTestData(false)] public static void Test(DafnyTestCase testCase) { diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index 7d2d65cac5c..c8db9d26eb6 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -3,28 +3,21 @@ netcoreapp3.1 DafnyTests + true + - - - - - - - - - - + + - diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs index 0ea8c865def..ccc23b449ba 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs @@ -41,30 +41,37 @@ public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodIn } private IEnumerable ResourceData(MethodInfo testMethod, string resourceName, bool withParameterNames) { - Stream stream = testMethod.DeclaringType.Assembly.GetManifestResourceStream(resourceName); - IParser parser = GetYamlParser(resourceName, stream); - if (parser == null) { - return Enumerable.Empty(); - } - IDeserializer deserializer = GetDeserializer(resourceName); - parser.Consume(); - parser.Consume(); - - if (withParameterNames) { - IObjectFactory argumentsFactory = new DefaultObjectFactory(); - INodeDeserializer collectionDeserializer = new CollectionNodeDeserializer(argumentsFactory); - var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, testMethod); - if (collectionDeserializer.Deserialize(parser, typeof(List), nestedObjectDeserializer, out var value)) { - List argumentses = (List) value; - return argumentses.SelectMany(a => a.Combinations()); + try { + Stream stream = testMethod.DeclaringType.Assembly.GetManifestResourceStream(resourceName); + IParser parser = GetYamlParser(resourceName, stream); + if (parser == null) { + return Enumerable.Empty(); + } + + IDeserializer deserializer = GetDeserializer(resourceName); + parser.Consume(); + parser.Consume(); + + if (withParameterNames) { + IObjectFactory argumentsFactory = new DefaultObjectFactory(); + INodeDeserializer collectionDeserializer = new CollectionNodeDeserializer(argumentsFactory); + var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, testMethod); + if (collectionDeserializer.Deserialize(parser, typeof(List), nestedObjectDeserializer, + out var value)) { + List argumentses = (List) value; + return argumentses.SelectMany(a => a.Combinations()); + } else { + throw new ArgumentException(); + } } else { - throw new ArgumentException(); + IEnumerable parameters = testMethod.GetParameters(); + Type targetType = typeof(IEnumerable<>).MakeGenericType(parameters.Single().ParameterType); + IEnumerable results = (IEnumerable) deserializer.Deserialize(parser, targetType); + return results.Select(value => new[] {value}); } - } else { - IEnumerable parameters = testMethod.GetParameters(); - Type targetType = typeof(IEnumerable<>).MakeGenericType(parameters.Single().ParameterType); - IEnumerable results = (IEnumerable) deserializer.Deserialize(parser, targetType); - return results.Select(value => new[] {value}); + } catch (Exception e) { + throw new ArgumentException( + "Exception thrown while trying to deserialize test data from manifest resource: " + resourceName, e); } } @@ -74,10 +81,14 @@ public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo t if (methodInfo == null) { return null; } - + List attributeArgs = attributeInfo.GetConstructorArguments().ToList(); - bool withParameterNames = (bool)attributeArgs[0]; + bool withParameterNames = (bool) attributeArgs[0]; + + return GetData(methodInfo, withParameterNames); + } + public IEnumerable GetData(MethodInfo methodInfo, bool withParameterNames) { string resourceNamePrefix = methodInfo.DeclaringType.FullName + "." + methodInfo.Name; IEnumerable result = methodInfo.DeclaringType.Assembly.GetManifestResourceNames() .Where(n => n.StartsWith(resourceNamePrefix)) diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 48c86015067..8c5f39fa0c7 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -31,6 +31,7 @@ public class LitTestConvertor { private int count = 0; private int verifyOnlyCount = 0; private int defaultCount = 0; + private int otherFilesCount = 0; private int invalidCount = 0; public void ConvertLitTest(string filePath) { @@ -87,7 +88,7 @@ private static string ExtractLitCommand(string line) { return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); } - private static Dictionary ParseDafnyCommandArguments(string dafnyCommand) { + private Dictionary ParseDafnyCommandArguments(string dafnyCommand) { var testConfig = new Dictionary(); var otherFiles = new List(); @@ -131,6 +132,7 @@ private static Dictionary ParseDafnyCommandArguments(string dafn if (otherFiles.Any()) { testConfig[TEST_CONFIG_OTHER_FILES] = String.Join(" ", otherFiles); + otherFilesCount++; } return testConfig; } @@ -163,6 +165,7 @@ public void Run(string root) { } Console.WriteLine("Default: " + defaultCount + "/" + count); Console.WriteLine("Verify only: " + verifyOnlyCount + "/" + count); + Console.WriteLine("With other files: " + otherFilesCount + "/" + count); Console.WriteLine("Invalid: " + invalidCount + "/" + count); } From 0141c504342f40c78fa4fdb9d0abf22d76ad24ea Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 7 Oct 2020 10:41:38 -0700 Subject: [PATCH 074/192] Fix BranchCoverage.dfy.expect line numbers --- Test/comp/BranchCoverage.dfy.expect | 84 ++++++++++++++--------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Test/comp/BranchCoverage.dfy.expect b/Test/comp/BranchCoverage.dfy.expect index 1eac6ec531a..185c342a12a 100644 --- a/Test/comp/BranchCoverage.dfy.expect +++ b/Test/comp/BranchCoverage.dfy.expect @@ -1,47 +1,47 @@ Dafny program verifier finished with 0 verified, 0 errors -0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor -1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled -2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled -3: BranchCoverage.dfy(30,35): entry to method _module._default.M -4: BranchCoverage.dfy(32,13): then branch -5: BranchCoverage.dfy(34,20): then branch -6: BranchCoverage.dfy(36,10): else branch -7: BranchCoverage.dfy(41,35): entry to method _module._default.N -8: BranchCoverage.dfy(44,13): then branch -9: BranchCoverage.dfy(46,20): then branch -10: BranchCoverage.dfy(46,10): implicit else branch -11: BranchCoverage.dfy(52,18): entry to method _module._default.P -12: BranchCoverage.dfy(55,8): then branch -13: BranchCoverage.dfy(55,3): implicit else branch -14: BranchCoverage.dfy(60,8): then branch -15: BranchCoverage.dfy(62,10): else branch -16: BranchCoverage.dfy(72,12): then branch -17: BranchCoverage.dfy(74,15): then branch -18: BranchCoverage.dfy(74,10): implicit else branch -19: BranchCoverage.dfy(82,35): entry to method _module._default.Q -20: BranchCoverage.dfy(94,3): if-case branch -21: BranchCoverage.dfy(96,3): if-case branch -22: BranchCoverage.dfy(99,3): if-case branch -23: BranchCoverage.dfy(105,35): entry to method _module._default.R -24: BranchCoverage.dfy(108,15): while body -25: BranchCoverage.dfy(113,11): while body -26: BranchCoverage.dfy(121,5): while-case branch -27: BranchCoverage.dfy(123,17): then branch -28: BranchCoverage.dfy(123,7): implicit else branch -29: BranchCoverage.dfy(128,5): while-case branch -30: BranchCoverage.dfy(131,5): while-case branch -31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib -32: BranchCoverage.dfy(144,5): then branch -33: BranchCoverage.dfy(146,5): then branch -34: BranchCoverage.dfy(146,5): else branch -35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial -36: BranchCoverage.dfy(156,13): then branch -37: BranchCoverage.dfy(156,13): else branch -38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial -39: BranchCoverage.dfy(165,3): then branch -40: BranchCoverage.dfy(167,3): else branch -41: BranchCoverage.dfy(173,15): entry to method _module._default.Main +0: BranchCoverage.dfy(15,18): entry to method _module.MyClass._ctor +1: BranchCoverage.dfy(21,22): entry to method _module._default.NeverCalled +2: BranchCoverage.dfy(27,3): entry to function _module._default.FunctionNeverCalled +3: BranchCoverage.dfy(32,35): entry to method _module._default.M +4: BranchCoverage.dfy(34,13): then branch +5: BranchCoverage.dfy(36,20): then branch +6: BranchCoverage.dfy(38,10): else branch +7: BranchCoverage.dfy(43,35): entry to method _module._default.N +8: BranchCoverage.dfy(46,13): then branch +9: BranchCoverage.dfy(48,20): then branch +10: BranchCoverage.dfy(48,10): implicit else branch +11: BranchCoverage.dfy(54,18): entry to method _module._default.P +12: BranchCoverage.dfy(57,8): then branch +13: BranchCoverage.dfy(57,3): implicit else branch +14: BranchCoverage.dfy(62,8): then branch +15: BranchCoverage.dfy(64,10): else branch +16: BranchCoverage.dfy(74,12): then branch +17: BranchCoverage.dfy(76,15): then branch +18: BranchCoverage.dfy(76,10): implicit else branch +19: BranchCoverage.dfy(84,35): entry to method _module._default.Q +20: BranchCoverage.dfy(96,3): if-case branch +21: BranchCoverage.dfy(98,3): if-case branch +22: BranchCoverage.dfy(101,3): if-case branch +23: BranchCoverage.dfy(107,35): entry to method _module._default.R +24: BranchCoverage.dfy(110,15): while body +25: BranchCoverage.dfy(115,11): while body +26: BranchCoverage.dfy(123,5): while-case branch +27: BranchCoverage.dfy(125,17): then branch +28: BranchCoverage.dfy(125,7): implicit else branch +29: BranchCoverage.dfy(130,5): while-case branch +30: BranchCoverage.dfy(133,5): while-case branch +31: BranchCoverage.dfy(145,3): entry to function _module._default.Fib +32: BranchCoverage.dfy(146,5): then branch +33: BranchCoverage.dfy(148,5): then branch +34: BranchCoverage.dfy(148,5): else branch +35: BranchCoverage.dfy(157,3): entry to function _module._default.Factorial +36: BranchCoverage.dfy(158,13): then branch +37: BranchCoverage.dfy(158,13): else branch +38: BranchCoverage.dfy(164,1): entry to method _module._default.ComputeFactorial +39: BranchCoverage.dfy(167,3): then branch +40: BranchCoverage.dfy(169,3): else branch +41: BranchCoverage.dfy(175,15): entry to method _module._default.Main 0: 3 1: 0 2: 0 From 55530f4fb4b95e4a37944b334455d886e5742022 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 8 Oct 2020 10:17:23 -0700 Subject: [PATCH 075/192] Hook up expected exit code --- Source/DafnyTests/DafnyTestCase.cs | 19 ++++++++----------- Source/DafnyTests/DafnyTests.csproj | 1 + Source/DafnyTests/ProcessResult.cs | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 Source/DafnyTests/ProcessResult.cs diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index dee9139f6dd..097654f4531 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Reflection; +using FluentAssertions; using Microsoft.Extensions.FileProviders; using Xunit; using Xunit.Abstractions; @@ -208,7 +209,7 @@ public void Deserialize(IXunitSerializationInfo info) { Expected = info.GetValue(nameof(Expected)); } - public static string RunDafny(IEnumerable arguments) { + public static ProcessResult RunDafny(IEnumerable arguments) { // TODO-RS: Let these be overridden List dafnyArguments = new List { // Expected output does not contain logo @@ -247,12 +248,7 @@ public static string RunDafny(IEnumerable arguments) { dafnyProcess.WaitForExit(); string output = dafnyProcess.StandardOutput.ReadToEnd(); string error = dafnyProcess.StandardError.ReadToEnd(); - if (dafnyProcess.ExitCode != 0) { - var message = String.Format("Non-zero Dafny exit code: {0}\n{1}\n{2}", dafnyProcess.ExitCode, output, error); - Assert.True(false, message); - } - - return output + error; + return new ProcessResult(dafnyProcess.ExitCode, output, error); } } @@ -265,9 +261,9 @@ private static string ConfigPairToArgument(KeyValuePair pair) { } public void Run() { - string output; + ProcessResult dafnyResult; if (Arguments.Any(arg => arg.StartsWith("/out"))) { - output = RunDafny(Arguments); + dafnyResult = RunDafny(Arguments); } else { // Note that the temporary directory has to be an ancestor of Test // or else Javascript won't be able to locate bignumber.js :( @@ -277,13 +273,14 @@ public void Run() { // interpret the path as a single file basename rather than a directory. var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; var dafnyArguments = new []{ outArgument }.Concat(Arguments); - output = RunDafny(dafnyArguments); + dafnyResult = RunDafny(dafnyArguments); } } + dafnyResult.ExitCode.Should().Be(Expected.ExitCode); if (Expected.OutputFile != null) { var expectedOutput = File.ReadAllText(Path.Combine(DafnyTestSpec.TEST_ROOT, Expected.OutputFile)); - AssertWithDiff.Equal(expectedOutput, output); + AssertWithDiff.Equal(expectedOutput, dafnyResult.StandardOutput); } Skip.If(Expected.SpecialCase, "Confirmed known exception for arguments: " + String.Join(" ", Arguments)); diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index c8db9d26eb6..b1afc42ac82 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -8,6 +8,7 @@ + diff --git a/Source/DafnyTests/ProcessResult.cs b/Source/DafnyTests/ProcessResult.cs new file mode 100644 index 00000000000..52e9d3b0ad3 --- /dev/null +++ b/Source/DafnyTests/ProcessResult.cs @@ -0,0 +1,14 @@ +namespace DafnyTests { + public class ProcessResult { + + public int ExitCode; + public string StandardOutput; + public string StandardError; + + public ProcessResult(int exitCode, string standardOutput, string standardError) { + ExitCode = exitCode; + StandardOutput = standardOutput; + StandardError = standardError; + } + } +} \ No newline at end of file From f786ec363d387ab6ec7799a181d0c621607e5a53 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 8 Oct 2020 17:43:40 -0700 Subject: [PATCH 076/192] Better approach to compile target overrides --- Source/DafnyTests/DafnyTestCase.cs | 199 +++++++++++------- Test/comp/BranchCoverage.dfy | 13 ++ .../BranchCoverage2.cs | 20 -- Test/comp/BranchCoverage.dfy.expect | 84 ++++---- .../BranchCoverage3.js | 25 --- .../BranchCoverage4.go | 0 Test/comp/CSharpStyling.dfy | 2 + .../CSharpStyling2.cs | 37 ---- .../CodeCoverage.java | 0 Test/comp/Collections.dfy | 11 + Test/comp/TypeParams.dfy | 17 ++ 11 files changed, 203 insertions(+), 205 deletions(-) delete mode 100644 Test/comp/BranchCoverage.dfy.cs.files/BranchCoverage2.cs delete mode 100644 Test/comp/BranchCoverage.dfy.js.files/BranchCoverage3.js rename Test/comp/{BranchCoverage.dfy.go.files => }/BranchCoverage4.go (100%) delete mode 100644 Test/comp/CSharpStyling.dfy.cs.files/CSharpStyling2.cs rename Test/comp/{BranchCoverage.dfy.java.files => }/CodeCoverage.java (100%) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 097654f4531..646fea65794 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.ComponentModel.Design; using System.Diagnostics; using System.IO; using System.Linq; @@ -30,6 +31,7 @@ public class DafnyTestSpec : IEnumerable { OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; + public static readonly string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; public static readonly string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; public static readonly string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/dafny"); @@ -39,13 +41,25 @@ public class DafnyTestSpec : IEnumerable { private static readonly Dictionary PathsForResourceNames = GetPathsForResourceNames( "DafnyTests", manifestFileProvider, "DafnyTests"); + // Absolute file system path to the main Dafny file public string SourcePath; + public Dictionary DafnyArguments = new Dictionary(); - - public DafnyTestCase.Expectation Expect = new DafnyTestCase.Expectation(); + public IEnumerable OtherFiles = Enumerable.Empty(); + + public Dictionary CompileTargetOverrides = new Dictionary(); + + public DafnyTestCase.Expectation Expected; public DafnyTestSpec(string manifestResourceName) { - SourcePath = "comp/" + PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); + SourcePath = COMP_DIR + PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); + } + + private DafnyTestSpec(string sourcePath, Dictionary dafnyArguments, IEnumerable otherFiles, DafnyTestCase.Expectation expected) { + SourcePath = sourcePath; + DafnyArguments = dafnyArguments; + OtherFiles = otherFiles; + Expected = expected; } private static Dictionary GetPathsForResourceNames(string assemblyName, IFileProvider fileProvider, string path = null) { @@ -62,35 +76,85 @@ private static Dictionary GetPathsForResourceNames(string assemb } public IEnumerator GetEnumerator() { - return ExpandArguments(DafnyArguments) - .SelectMany(args => ResolveCompile(SourcePath, args, Expect)) + return ResolveCompile() + .SelectMany(spec => spec.ExpandArguments()) + .Select(spec => spec.ResolveExpected().ToTestCase()) .GetEnumerator(); } - - private static IEnumerable ResolveCompile(string sourcePath, Dictionary config, DafnyTestCase.Expectation expect) { - var compile = "3"; - if (config.ContainsKey("compile")) { - compile = config["compile"]; - config.Remove("compile"); + + public string Compile { + get { + if (DafnyArguments.TryGetValue("compile", out var compile)) { + return (string) compile; + } + return null; } - - if (compile.Equals("3") && !config.ContainsKey("compileTarget")) { + } + public string CompileTarget { + get { + if (DafnyArguments.TryGetValue("compileTarget", out var compileTarget)) { + return (string) compileTarget; + } + return null; + } + } + + private IEnumerable ResolveCompile() { + if ("3".Equals(Compile ?? "3") && CompileTarget == null) { var compileTargets = new[] {"cs", "java", "go", "js"}; - - var justVerify = new Dictionary(config); - justVerify["compile"] = "0"; - yield return new DafnyTestCase(sourcePath, justVerify, null, DafnyTestCase.Expectation.NO_OUTPUT); + + var justVerify = new Dictionary(DafnyArguments) { + ["compile"] = "0" + }; + yield return new DafnyTestSpec(SourcePath, justVerify, OtherFiles, DafnyTestCase.Expectation.NO_OUTPUT); foreach (var compileTarget in compileTargets) { - var withLanguage = new Dictionary(config); - withLanguage["noVerify"] = "yes"; - withLanguage["compile"] = "4"; - yield return new DafnyTestCase(sourcePath, withLanguage, compileTarget, expect.Resolve(sourcePath, compileTarget)); + var withLanguage = new Dictionary(DafnyArguments) { + ["noVerify"] = "yes", + ["compile"] = "4", + ["compileTarget"] = compileTarget + }; + var specForLanguage = new DafnyTestSpec(SourcePath, withLanguage, OtherFiles, Expected); + if (CompileTargetOverrides.TryGetValue(compileTarget, out var compileTargetOverride)) { + yield return specForLanguage.ApplyOverride(compileTargetOverride); + } else { + yield return specForLanguage; + } } } else { - config.TryGetValue("compileTarget", out var compileTarget); - config["compile"] = compile; - yield return new DafnyTestCase(sourcePath, config, compileTarget, expect.Resolve(sourcePath, compileTarget)); + yield return this; + } + } + + private DafnyTestSpec ApplyOverride(DafnyTestSpec otherSpec) { + var mergedArguments = new Dictionary(DafnyArguments); + foreach (KeyValuePair pair in otherSpec.DafnyArguments) { + mergedArguments[pair.Key] = pair.Value; + } + return new DafnyTestSpec(otherSpec.SourcePath, mergedArguments, OtherFiles.Concat(otherSpec.OtherFiles), otherSpec.Expected); + } + + private DafnyTestSpec ResolveExpected() { + if (Expected == null) { + return new DafnyTestSpec(SourcePath, DafnyArguments, OtherFiles, + new DafnyTestCase.Expectation(0, Path.GetFileName(SourcePath) + ".expect", null)); + } else { + return this; + } + } + + private DafnyTestCase ToTestCase() { + var arguments = new []{ Path.GetRelativePath(TEST_ROOT, SourcePath) } + .Concat(DafnyArguments.Select(ConfigPairToArgument)) + .Concat(OtherFiles.Select(otherFile => "comp/" + otherFile)); + return new DafnyTestCase(arguments, Expected.Adjust()); + } + + private static string ConfigPairToArgument(KeyValuePair pair) { + if (pair.Value.Equals("yes")) { + return String.Format("/{0}", pair.Key); + } else { + return String.Format("/{0}:{1}", pair.Key, pair.Value); } } @@ -98,24 +162,27 @@ IEnumerator IEnumerable.GetEnumerator() { return GetEnumerator(); } - private static IEnumerable> ExpandArguments(Dictionary arguments) { - return arguments.Select(ExpandValue) - .CartesianProduct() - .Select(e => e.ToDictionary(p => p.Key, p => p.Value)); + private IEnumerable ExpandArguments() { + return DafnyArguments.Select(ExpandValue) + .CartesianProduct() + .Select(e => e.ToDictionary(p => p.Key, p => p.Value)) + .Select(args => new DafnyTestSpec(SourcePath, args, OtherFiles, Expected)); } - + public static IEnumerable Expand(object obj) { - if (obj is IEnumerable e) { - return e; + if (obj is ForEachArgumentList forEach) { + return forEach; } else { return new[] {(string)obj}; } } - private static IEnumerable> ExpandValue(KeyValuePair pair) { - return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); + private static IEnumerable> ExpandValue(KeyValuePair pair) { + return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); } } + + public class ForEachArgumentList : List { } public class DafnyTestCase: IXunitSerializable { @@ -126,31 +193,14 @@ public class Expectation : IXunitSerializable { public string OutputFile; public int ExitCode = 0; - public bool SpecialCase = false; + public string SpecialCaseReason; - public static Expectation NO_OUTPUT = new Expectation(0, null); + public static Expectation NO_OUTPUT = new Expectation(0, null, null); - public Expectation Resolve(string sourceFile, string compileTarget) { - Expectation result = (Expectation)MemberwiseClone(); - if (result.OutputFile == null) { - result.OutputFile = sourceFile + ".expect"; - if (compileTarget != null) { - var specialCasePath = sourceFile + "." + compileTarget + ".expect"; - if (File.Exists(DafnyTestSpec.TEST_ROOT + specialCasePath)) { - result.SpecialCase = true; - result.OutputFile = specialCasePath; - } - } - } else if (result.OutputFile.Equals("no")) { - result.OutputFile = null; - } - - return result; - } - - public Expectation(int exitCode, string outputFile) { + public Expectation(int exitCode, string outputFile, string specialCaseReason) { ExitCode = exitCode; OutputFile = outputFile; + SpecialCaseReason = specialCaseReason; } public Expectation() { @@ -160,15 +210,19 @@ public Expectation() { public void Deserialize(IXunitSerializationInfo info) { OutputFile = info.GetValue(nameof(OutputFile)); ExitCode = info.GetValue(nameof(ExitCode)); - SpecialCase = info.GetValue(nameof(SpecialCase)); + SpecialCaseReason = info.GetValue(nameof(SpecialCaseReason)); } public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(OutputFile), OutputFile); info.AddValue(nameof(ExitCode), ExitCode); - info.AddValue(nameof(SpecialCase), SpecialCase); + info.AddValue(nameof(SpecialCaseReason), SpecialCaseReason); } + public Expectation Adjust() { + return new Expectation(ExitCode, OutputFile == null ? null : "comp/" + OutputFile, SpecialCaseReason); + } + public override string ToString() { return OutputFile ?? "-"; } @@ -176,22 +230,8 @@ public override string ToString() { public Expectation Expected; - public DafnyTestCase(string dafnyFile, Dictionary config, string compileTarget, Expectation expected) { + public DafnyTestCase(IEnumerable arguments, Expectation expected) { Expected = expected; - - var arguments = new []{ dafnyFile }.Concat(config.Select(ConfigPairToArgument)).ToList(); - - if (compileTarget != null) { - arguments.Add("/compileTarget:" + compileTarget); - // Include any additional files - var additionalFilesPath = DafnyTestSpec.TEST_ROOT + dafnyFile + "." + compileTarget + ".files"; - if (Directory.Exists(additionalFilesPath)) { - var relativePaths = Directory.GetFiles(additionalFilesPath) - .Select(path => Path.GetRelativePath(DafnyTestSpec.TEST_ROOT, path)); - arguments.AddRange(relativePaths); - } - } - Arguments = arguments.ToArray(); } @@ -252,14 +292,6 @@ public static ProcessResult RunDafny(IEnumerable arguments) { } } - private static string ConfigPairToArgument(KeyValuePair pair) { - if (pair.Value.Equals("yes")) { - return String.Format("/{0}", pair.Key); - } else { - return String.Format("/{0}:{1}", pair.Key, pair.Value); - } - } - public void Run() { ProcessResult dafnyResult; if (Arguments.Any(arg => arg.StartsWith("/out"))) { @@ -277,13 +309,17 @@ public void Run() { } } - dafnyResult.ExitCode.Should().Be(Expected.ExitCode); + if (dafnyResult.ExitCode != Expected.ExitCode) { + throw new AssertActualExpectedException(Expected.ExitCode, dafnyResult.ExitCode, + String.Format("Expected exit code to be {0} but was {1}. Standard error output:\n{2}", + Expected.ExitCode, dafnyResult.ExitCode, dafnyResult.StandardError)); + } if (Expected.OutputFile != null) { var expectedOutput = File.ReadAllText(Path.Combine(DafnyTestSpec.TEST_ROOT, Expected.OutputFile)); AssertWithDiff.Equal(expectedOutput, dafnyResult.StandardOutput); } - Skip.If(Expected.SpecialCase, "Confirmed known exception for arguments: " + String.Join(" ", Arguments)); + Skip.If(Expected.SpecialCaseReason != null, Expected.SpecialCaseReason); } public override string ToString() { @@ -331,6 +367,7 @@ public override IDeserializer GetDeserializer(string manifestResourceName) { return new DeserializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) + .WithTagMapping("!foreach", typeof(ForEachArgumentList)) .WithObjectFactory(customObjectFactory) .Build(); } diff --git a/Test/comp/BranchCoverage.dfy b/Test/comp/BranchCoverage.dfy index 5ec1fb9b86c..ec82463342d 100644 --- a/Test/comp/BranchCoverage.dfy +++ b/Test/comp/BranchCoverage.dfy @@ -4,6 +4,19 @@ dafnyArguments: compile: 3 coverage: "-" +compileTargetOverrides: + java: + otherFiles: + - CodeCoverage.java + cs: + otherFiles: + - BranchCoverage2.cs + js: + otherFiles: + - BranchCoverage3.js + go: + otherFiles: + - BranchCoverage4.go */ // The Main method is at the end of this file, because that makes it easier to maintain diff --git a/Test/comp/BranchCoverage.dfy.cs.files/BranchCoverage2.cs b/Test/comp/BranchCoverage.dfy.cs.files/BranchCoverage2.cs deleted file mode 100644 index 174d297ee7e..00000000000 --- a/Test/comp/BranchCoverage.dfy.cs.files/BranchCoverage2.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; - -namespace DafnyProfiling { - public class CodeCoverage { - static uint[] tallies; - public static void Setup(int size) { - tallies = new uint[size]; - } - public static void TearDown() { - for (var i = 0; i < tallies.Length; i++) { - Console.WriteLine("{0}: {1}", i, tallies[i]); - } - tallies = null; - } - public static bool Record(int id) { - tallies[id]++; - return true; - } - } -} diff --git a/Test/comp/BranchCoverage.dfy.expect b/Test/comp/BranchCoverage.dfy.expect index 185c342a12a..16d271fa49f 100644 --- a/Test/comp/BranchCoverage.dfy.expect +++ b/Test/comp/BranchCoverage.dfy.expect @@ -1,47 +1,47 @@ Dafny program verifier finished with 0 verified, 0 errors -0: BranchCoverage.dfy(15,18): entry to method _module.MyClass._ctor -1: BranchCoverage.dfy(21,22): entry to method _module._default.NeverCalled -2: BranchCoverage.dfy(27,3): entry to function _module._default.FunctionNeverCalled -3: BranchCoverage.dfy(32,35): entry to method _module._default.M -4: BranchCoverage.dfy(34,13): then branch -5: BranchCoverage.dfy(36,20): then branch -6: BranchCoverage.dfy(38,10): else branch -7: BranchCoverage.dfy(43,35): entry to method _module._default.N -8: BranchCoverage.dfy(46,13): then branch -9: BranchCoverage.dfy(48,20): then branch -10: BranchCoverage.dfy(48,10): implicit else branch -11: BranchCoverage.dfy(54,18): entry to method _module._default.P -12: BranchCoverage.dfy(57,8): then branch -13: BranchCoverage.dfy(57,3): implicit else branch -14: BranchCoverage.dfy(62,8): then branch -15: BranchCoverage.dfy(64,10): else branch -16: BranchCoverage.dfy(74,12): then branch -17: BranchCoverage.dfy(76,15): then branch -18: BranchCoverage.dfy(76,10): implicit else branch -19: BranchCoverage.dfy(84,35): entry to method _module._default.Q -20: BranchCoverage.dfy(96,3): if-case branch -21: BranchCoverage.dfy(98,3): if-case branch -22: BranchCoverage.dfy(101,3): if-case branch -23: BranchCoverage.dfy(107,35): entry to method _module._default.R -24: BranchCoverage.dfy(110,15): while body -25: BranchCoverage.dfy(115,11): while body -26: BranchCoverage.dfy(123,5): while-case branch -27: BranchCoverage.dfy(125,17): then branch -28: BranchCoverage.dfy(125,7): implicit else branch -29: BranchCoverage.dfy(130,5): while-case branch -30: BranchCoverage.dfy(133,5): while-case branch -31: BranchCoverage.dfy(145,3): entry to function _module._default.Fib -32: BranchCoverage.dfy(146,5): then branch -33: BranchCoverage.dfy(148,5): then branch -34: BranchCoverage.dfy(148,5): else branch -35: BranchCoverage.dfy(157,3): entry to function _module._default.Factorial -36: BranchCoverage.dfy(158,13): then branch -37: BranchCoverage.dfy(158,13): else branch -38: BranchCoverage.dfy(164,1): entry to method _module._default.ComputeFactorial -39: BranchCoverage.dfy(167,3): then branch -40: BranchCoverage.dfy(169,3): else branch -41: BranchCoverage.dfy(175,15): entry to method _module._default.Main +0: BranchCoverage.dfy(28,18): entry to method _module.MyClass._ctor +1: BranchCoverage.dfy(34,22): entry to method _module._default.NeverCalled +2: BranchCoverage.dfy(40,3): entry to function _module._default.FunctionNeverCalled +3: BranchCoverage.dfy(45,35): entry to method _module._default.M +4: BranchCoverage.dfy(47,13): then branch +5: BranchCoverage.dfy(49,20): then branch +6: BranchCoverage.dfy(51,10): else branch +7: BranchCoverage.dfy(56,35): entry to method _module._default.N +8: BranchCoverage.dfy(59,13): then branch +9: BranchCoverage.dfy(61,20): then branch +10: BranchCoverage.dfy(61,10): implicit else branch +11: BranchCoverage.dfy(67,18): entry to method _module._default.P +12: BranchCoverage.dfy(70,8): then branch +13: BranchCoverage.dfy(70,3): implicit else branch +14: BranchCoverage.dfy(75,8): then branch +15: BranchCoverage.dfy(77,10): else branch +16: BranchCoverage.dfy(87,12): then branch +17: BranchCoverage.dfy(89,15): then branch +18: BranchCoverage.dfy(89,10): implicit else branch +19: BranchCoverage.dfy(97,35): entry to method _module._default.Q +20: BranchCoverage.dfy(109,3): if-case branch +21: BranchCoverage.dfy(111,3): if-case branch +22: BranchCoverage.dfy(114,3): if-case branch +23: BranchCoverage.dfy(120,35): entry to method _module._default.R +24: BranchCoverage.dfy(123,15): while body +25: BranchCoverage.dfy(128,11): while body +26: BranchCoverage.dfy(136,5): while-case branch +27: BranchCoverage.dfy(138,17): then branch +28: BranchCoverage.dfy(138,7): implicit else branch +29: BranchCoverage.dfy(143,5): while-case branch +30: BranchCoverage.dfy(146,5): while-case branch +31: BranchCoverage.dfy(158,3): entry to function _module._default.Fib +32: BranchCoverage.dfy(159,5): then branch +33: BranchCoverage.dfy(161,5): then branch +34: BranchCoverage.dfy(161,5): else branch +35: BranchCoverage.dfy(170,3): entry to function _module._default.Factorial +36: BranchCoverage.dfy(171,13): then branch +37: BranchCoverage.dfy(171,13): else branch +38: BranchCoverage.dfy(177,1): entry to method _module._default.ComputeFactorial +39: BranchCoverage.dfy(180,3): then branch +40: BranchCoverage.dfy(182,3): else branch +41: BranchCoverage.dfy(188,15): entry to method _module._default.Main 0: 3 1: 0 2: 0 diff --git a/Test/comp/BranchCoverage.dfy.js.files/BranchCoverage3.js b/Test/comp/BranchCoverage.dfy.js.files/BranchCoverage3.js deleted file mode 100644 index 7923dd9ac8e..00000000000 --- a/Test/comp/BranchCoverage.dfy.js.files/BranchCoverage3.js +++ /dev/null @@ -1,25 +0,0 @@ -let DafnyProfiling = (function() { - let $module = {}; - - $module.tallies = [] - - $module.CodeCoverage = class CodeCoverage { - static Setup(size) { - for (let i = 0; i < size; i++) { - $module.tallies[i] = 0 - } - } - static TearDown() { - for (let i = 0; i < $module.tallies.length; i++) { - process.stdout.write("" + i + ": " + $module.tallies[i] + "\n"); - } - $module.tallies = null; - } - static Record(id) { - $module.tallies[id]++ - return true; - } - }; - - return $module; -})(); diff --git a/Test/comp/BranchCoverage.dfy.go.files/BranchCoverage4.go b/Test/comp/BranchCoverage4.go similarity index 100% rename from Test/comp/BranchCoverage.dfy.go.files/BranchCoverage4.go rename to Test/comp/BranchCoverage4.go diff --git a/Test/comp/CSharpStyling.dfy b/Test/comp/CSharpStyling.dfy index 2cd92a70e7d..b99078cd843 100644 --- a/Test/comp/CSharpStyling.dfy +++ b/Test/comp/CSharpStyling.dfy @@ -4,6 +4,8 @@ dafnyArguments: compile: 3 compileTarget: cs +otherFiles: + - CSharpStyling2.cs */ method Main() { diff --git a/Test/comp/CSharpStyling.dfy.cs.files/CSharpStyling2.cs b/Test/comp/CSharpStyling.dfy.cs.files/CSharpStyling2.cs deleted file mode 100644 index 7a7b6e9eeb2..00000000000 --- a/Test/comp/CSharpStyling.dfy.cs.files/CSharpStyling2.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.Numerics; - -namespace _module { - public partial class MyClass { - public MyClass() { } // needed by Dafny in order to call constructor "Init" - public MyClass(BigInteger x) { - this.u = x; - } - public MyClass(bool y) { - this.u = y ? 3 : -3; - } - - public BigInteger OneResultExtern(BigInteger z) { - BigInteger r = 12; - if (z == 0) { - r = 8; - return r; - } else if (z == 1) { - r = 10; - return r; - } - return r; - } - - public void TwoResultsExtern(BigInteger z, out BigInteger r, out BigInteger s) { - if (z == 0) { - r = 5; - s = 7; - return; - } else { - r = 9; - s = 11; - } - return; - } - } -} diff --git a/Test/comp/BranchCoverage.dfy.java.files/CodeCoverage.java b/Test/comp/CodeCoverage.java similarity index 100% rename from Test/comp/BranchCoverage.dfy.java.files/CodeCoverage.java rename to Test/comp/CodeCoverage.java diff --git a/Test/comp/Collections.dfy b/Test/comp/Collections.dfy index 74498d216a9..6f9012b326f 100644 --- a/Test/comp/Collections.dfy +++ b/Test/comp/Collections.dfy @@ -1,3 +1,14 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + cs: + expected: + outputFile: Collections.dfy.cs.expect + java: + expected: + outputFile: Collections.dfy.java.expect +*/ method Main() { Sets(); SubSets(); diff --git a/Test/comp/TypeParams.dfy b/Test/comp/TypeParams.dfy index 6d19b84d61e..42a49873dbb 100644 --- a/Test/comp/TypeParams.dfy +++ b/Test/comp/TypeParams.dfy @@ -1,3 +1,20 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + cs: + expected: + outputFile: TypeParams.dfy.cs.expect + java: + expected: + outputFile: TypeParams.dfy.java.expect + js: + expected: + outputFile: TypeParams.dfy.js.expect + go: + expected: + outputFile: TypeParams.dfy.go.expect +*/ datatype Color = Orange | Pink | Teal type Six = x | x <= 6 newtype Even = x | x % 2 == 0 From 6b01bfd105c0923b377fde835124e3490a95a9da Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 8 Oct 2020 23:09:12 -0700 Subject: [PATCH 077/192] All comp tests passing (with lots marked as known issues) --- Source/DafnyTests/DafnyTestCase.cs | 9 ++- .../{Extern.dfy.java.files => }/AllDafny.java | 0 .../AllExtern.java | 0 .../Class.java | 0 Test/comp/Collections.dfy | 2 + Test/comp/Comprehensions.dfy | 9 +++ Test/comp/Dt.dfy | 9 +++ Test/comp/Extern.dfy | 21 +++++++ Test/comp/Extern.dfy.cs.files/Extern2.cs | 46 ---------------- Test/comp/Extern.dfy.js.files/Extern3.js | 55 ------------------- .../comp/{Extern.dfy.go.files => }/Extern4.go | 0 Test/comp/ExternCtors.dfy | 25 ++++++++- Test/comp/ExternCtors.dfy.cs.files/Library.cs | 19 ------- Test/comp/ExternCtors.dfy.js.files/Library.js | 21 ------- Test/comp/Iterators.dfy | 10 ++++ .../{Extern.dfy.java.files => }/LibClass.java | 0 .../{ExternCtors.dfy.go.files => }/Library.go | 0 .../{Extern.dfy.java.files => }/Mixed.java | 0 Test/comp/NativeNumbers.dfy | 9 +++ Test/comp/NativeNumbers.dfy.js.expect | 16 +++--- Test/comp/Numbers.dfy | 9 +++ .../OtherClass.java | 0 Test/comp/Poly.dfy | 9 +++ Test/comp/TailRecursion.dfy | 13 +++++ Test/comp/TypeParams.dfy | 4 ++ 25 files changed, 134 insertions(+), 152 deletions(-) rename Test/comp/{Extern.dfy.java.files => }/AllDafny.java (100%) rename Test/comp/{Extern.dfy.java.files => }/AllExtern.java (100%) rename Test/comp/{ExternCtors.dfy.java.files => }/Class.java (100%) delete mode 100644 Test/comp/Extern.dfy.cs.files/Extern2.cs delete mode 100644 Test/comp/Extern.dfy.js.files/Extern3.js rename Test/comp/{Extern.dfy.go.files => }/Extern4.go (100%) delete mode 100644 Test/comp/ExternCtors.dfy.cs.files/Library.cs delete mode 100644 Test/comp/ExternCtors.dfy.js.files/Library.js rename Test/comp/{Extern.dfy.java.files => }/LibClass.java (100%) rename Test/comp/{ExternCtors.dfy.go.files => }/Library.go (100%) rename Test/comp/{Extern.dfy.java.files => }/Mixed.java (100%) rename Test/comp/{Extern.dfy.java.files => }/OtherClass.java (100%) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 646fea65794..94e986f7817 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -89,6 +89,9 @@ public string Compile { } return null; } + set { + DafnyArguments["compile"] = value; + } } public string CompileTarget { get { @@ -100,7 +103,11 @@ public string CompileTarget { } private IEnumerable ResolveCompile() { - if ("3".Equals(Compile ?? "3") && CompileTarget == null) { + if (Compile == null) { + Compile = "3"; + } + + if ("3".Equals(Compile) && CompileTarget == null) { var compileTargets = new[] {"cs", "java", "go", "js"}; var justVerify = new Dictionary(DafnyArguments) { diff --git a/Test/comp/Extern.dfy.java.files/AllDafny.java b/Test/comp/AllDafny.java similarity index 100% rename from Test/comp/Extern.dfy.java.files/AllDafny.java rename to Test/comp/AllDafny.java diff --git a/Test/comp/Extern.dfy.java.files/AllExtern.java b/Test/comp/AllExtern.java similarity index 100% rename from Test/comp/Extern.dfy.java.files/AllExtern.java rename to Test/comp/AllExtern.java diff --git a/Test/comp/ExternCtors.dfy.java.files/Class.java b/Test/comp/Class.java similarity index 100% rename from Test/comp/ExternCtors.dfy.java.files/Class.java rename to Test/comp/Class.java diff --git a/Test/comp/Collections.dfy b/Test/comp/Collections.dfy index 6f9012b326f..3269a32cd02 100644 --- a/Test/comp/Collections.dfy +++ b/Test/comp/Collections.dfy @@ -5,9 +5,11 @@ compileTargetOverrides: cs: expected: outputFile: Collections.dfy.cs.expect + specialCaseReason: Set output is inconsistently ordered java: expected: outputFile: Collections.dfy.java.expect + specialCaseReason: Set output is inconsistently ordered */ method Main() { Sets(); diff --git a/Test/comp/Comprehensions.dfy b/Test/comp/Comprehensions.dfy index 2ed46d8fe62..1883c8838d2 100644 --- a/Test/comp/Comprehensions.dfy +++ b/Test/comp/Comprehensions.dfy @@ -1,3 +1,12 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + java: + expected: + outputFile: Comprehensions.dfy.java.expect + specialCaseReason: Java doesn't always print strings correctly +*/ method Main() { AssignSuchThat(); LetSuchThat(); diff --git a/Test/comp/Dt.dfy b/Test/comp/Dt.dfy index d30e30ae8b8..b639e4ef3e5 100644 --- a/Test/comp/Dt.dfy +++ b/Test/comp/Dt.dfy @@ -1,3 +1,12 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + java: + expected: + outputFile: Dt.dfy.java.expect + specialCaseReason: Set output is inconsistently ordered +*/ datatype List = Nil | Cons(head: int, tail: List) method Main() { diff --git a/Test/comp/Extern.dfy b/Test/comp/Extern.dfy index cc519998f0f..634e42ee00c 100644 --- a/Test/comp/Extern.dfy +++ b/Test/comp/Extern.dfy @@ -1,3 +1,24 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + java: + otherFiles: + - AllDafny.java + - AllExtern.java + - LibClass.java + - Mixed.java + - OtherClass.java + cs: + otherFiles: + - Extern2.cs + js: + otherFiles: + - Extern3.js + go: + otherFiles: + - Extern4.go +*/ method Main() { print "Hello\n"; var x, y := Library.LibClass.CallMeInt(30); diff --git a/Test/comp/Extern.dfy.cs.files/Extern2.cs b/Test/comp/Extern.dfy.cs.files/Extern2.cs deleted file mode 100644 index 8ea42b9a2b4..00000000000 --- a/Test/comp/Extern.dfy.cs.files/Extern2.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System.Numerics; - -namespace Library { - public partial class LibClass { - // static method CallMeInt(x: int) returns (y: int, z: int) - public static void CallMeInt(BigInteger x, out BigInteger y, out BigInteger z) { - y = x + BigInteger.One; - z = y + y; - } - // static method CallMeNative(x: MyInt, b: bool) returns (y: MyInt) - public static int CallMeNative(int x, bool b) { - var y = b ? x + 1 : x - 1; - return y; - } - } - - class OtherClass { - public static object CallMe() { - return "OtherClass.CallMe"; - } - } - - // must be partial, since Dafny will also generate some methods into this class - public partial class Mixed { - public static void P() { - System.Console.WriteLine("Mixed.P"); - } - public void IP() { - System.Console.WriteLine("Mixed.IP"); - } - public static BigInteger G() { - return 1; - } - public BigInteger IG() { - return 2; - } - } - // It's okay for the following class to not be partial, since Dafny won't be adding - // any members to it. In fact, this test is to make sure that Dafny does not - // generate this class at all. - public class AllExtern { - public static void P() { - System.Console.WriteLine("AllExtern.P"); - } - } -} diff --git a/Test/comp/Extern.dfy.js.files/Extern3.js b/Test/comp/Extern.dfy.js.files/Extern3.js deleted file mode 100644 index 0992fc65dcb..00000000000 --- a/Test/comp/Extern.dfy.js.files/Extern3.js +++ /dev/null @@ -1,55 +0,0 @@ -let Library = (function() { - let $module = {}; - - $module.LibClass = class LibClass { - // static method CallMeInt(x: int) returns (y: int, z: int) - static CallMeInt(x) { - let y = x.plus(new BigNumber(1)); - let z = y.plus(y); - return [y, z]; - } - // static method CallMeNative(x: MyInt, b: bool) returns (y: MyInt) - static CallMeNative(x, b) { - let y = b ? x + 1 : x - 1; - return y; - } - }; - - $module.OtherClass = class OtherClass { - static CallMe() { - return "OtherClass.CallMe"; - } - } - - $module.AllDafny = class AllDafny { - // Just here so the generated class can extend it - } - - $module.Mixed = class Mixed { - constructor() { } - - // static method P() - static P() { - process.stdout.write("Mixed.P\n"); - } - - IP() { - process.stdout.write("Mixed.IP\n"); - } - - static G() { - return new BigNumber(1); - } - - IG() { - return new BigNumber(2); - } - } - $module.AllExtern = class AllExtern { - static P() { - process.stdout.write("AllExtern.P\n"); - } - }; - - return $module; -})(); diff --git a/Test/comp/Extern.dfy.go.files/Extern4.go b/Test/comp/Extern4.go similarity index 100% rename from Test/comp/Extern.dfy.go.files/Extern4.go rename to Test/comp/Extern4.go diff --git a/Test/comp/ExternCtors.dfy b/Test/comp/ExternCtors.dfy index 524a7095429..4c75b0b8e30 100644 --- a/Test/comp/ExternCtors.dfy +++ b/Test/comp/ExternCtors.dfy @@ -1,5 +1,26 @@ -// FIXME: Extern constructors are currently broken in Go and JavaScript - +/* +--- +!dafnyTestSpec +compileTargetOverrides: + java: + otherFiles: + - Class.java + cs: + otherFiles: + - Library.cs + js: + otherFiles: + - Library.js + expected: + outputFile: ~ + specialCaseReason: Extern constructors are currently broken in JavaScript + go: + otherFiles: + - Library.go + expected: + outputFile: ~ + specialCaseReason: Extern constructors are currently broken in Go +*/ method Main() { Library.Class.SayHi(); var obj := new Library.Class(42); diff --git a/Test/comp/ExternCtors.dfy.cs.files/Library.cs b/Test/comp/ExternCtors.dfy.cs.files/Library.cs deleted file mode 100644 index cb6e918524b..00000000000 --- a/Test/comp/ExternCtors.dfy.cs.files/Library.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System.Numerics; - -namespace Library { - public partial class Class { - private readonly BigInteger n; - - public Class(BigInteger n) { - this.n = n; - } - - public static void SayHi() { - System.Console.WriteLine("Hello!"); - } - - public BigInteger Get() { - return n; - } - } -} diff --git a/Test/comp/ExternCtors.dfy.js.files/Library.js b/Test/comp/ExternCtors.dfy.js.files/Library.js deleted file mode 100644 index 99f81dff92f..00000000000 --- a/Test/comp/ExternCtors.dfy.js.files/Library.js +++ /dev/null @@ -1,21 +0,0 @@ -// FIXME Test currently fails on JavaScript - -let Library = (function() { - let $module = {}; - - $module.Class = class Class { - constructor(n) { - this.n = n; - } - - static SayHi() { - process.stdout.write("Hello!\n"); - } - - Get() { - return this.n; - } - } - - return $module; -})(); \ No newline at end of file diff --git a/Test/comp/Iterators.dfy b/Test/comp/Iterators.dfy index 03aa242c6b2..40afd70a32e 100644 --- a/Test/comp/Iterators.dfy +++ b/Test/comp/Iterators.dfy @@ -1,3 +1,13 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + java: + expected: + exitCode: 134 + outputFile: ~ + specialCaseReason: Iterators are not implemented for Java +*/ class C { var x: int // for variety, the following tests the use of an instance Main method diff --git a/Test/comp/Extern.dfy.java.files/LibClass.java b/Test/comp/LibClass.java similarity index 100% rename from Test/comp/Extern.dfy.java.files/LibClass.java rename to Test/comp/LibClass.java diff --git a/Test/comp/ExternCtors.dfy.go.files/Library.go b/Test/comp/Library.go similarity index 100% rename from Test/comp/ExternCtors.dfy.go.files/Library.go rename to Test/comp/Library.go diff --git a/Test/comp/Extern.dfy.java.files/Mixed.java b/Test/comp/Mixed.java similarity index 100% rename from Test/comp/Extern.dfy.java.files/Mixed.java rename to Test/comp/Mixed.java diff --git a/Test/comp/NativeNumbers.dfy b/Test/comp/NativeNumbers.dfy index e9c73eb0518..9fe0dcbc508 100644 --- a/Test/comp/NativeNumbers.dfy +++ b/Test/comp/NativeNumbers.dfy @@ -1,3 +1,12 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + js: + expected: + outputFile: NativeNumbers.dfy.js.expect + specialCaseReason: :nativeType doesn't worry correctly for JavaScript +*/ method Main() { CastTests(); DefaultTests(); diff --git a/Test/comp/NativeNumbers.dfy.js.expect b/Test/comp/NativeNumbers.dfy.js.expect index 13366e17630..0645aac4dee 100644 --- a/Test/comp/NativeNumbers.dfy.js.expect +++ b/Test/comp/NativeNumbers.dfy.js.expect @@ -1,9 +1,9 @@ -NativeNumbers.dfy(10,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(11,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(12,28): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(13,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(15,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(16,31): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(17,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(18,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(19,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(20,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(21,28): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(22,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(24,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(25,31): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(26,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. +NativeNumbers.dfy(27,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. 8 resolution/type errors detected in NativeNumbers.dfy diff --git a/Test/comp/Numbers.dfy b/Test/comp/Numbers.dfy index 6cd34a868f7..b616094e60d 100644 --- a/Test/comp/Numbers.dfy +++ b/Test/comp/Numbers.dfy @@ -1,3 +1,12 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + go: + expected: + outputFile: Numbers.dfy.go.expect + specialCaseReason: Inconsistent printing of rationals +*/ method Main() { Literals(); Arithmetic(); diff --git a/Test/comp/Extern.dfy.java.files/OtherClass.java b/Test/comp/OtherClass.java similarity index 100% rename from Test/comp/Extern.dfy.java.files/OtherClass.java rename to Test/comp/OtherClass.java diff --git a/Test/comp/Poly.dfy b/Test/comp/Poly.dfy index 04544953907..66508645025 100644 --- a/Test/comp/Poly.dfy +++ b/Test/comp/Poly.dfy @@ -1,3 +1,12 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + go: + expected: + outputFile: Poly.dfy.go.expect + specialCaseReason: Inconsistent printing of rationals +*/ trait Shape { function method Center(): (real, real) reads this method PrintCenter() { diff --git a/Test/comp/TailRecursion.dfy b/Test/comp/TailRecursion.dfy index 5ebb3f30083..6a2d1eb9be5 100644 --- a/Test/comp/TailRecursion.dfy +++ b/Test/comp/TailRecursion.dfy @@ -1,3 +1,16 @@ +/* +--- +!dafnyTestSpec +compileTargetOverrides: + js: + expected: + outputFile: TailRecursion.dfy.js.expect + specialCaseReason: Set output is inconsistently ordered + go: + expected: + outputFile: TailRecursion.dfy.go.expect + specialCaseReason: Set output is inconsistently ordered +*/ method Main() { // In the following, 2_000_000 is too large an argument without tail-calls var x := M(2_000_000, 0); diff --git a/Test/comp/TypeParams.dfy b/Test/comp/TypeParams.dfy index 42a49873dbb..29b9d5b45fd 100644 --- a/Test/comp/TypeParams.dfy +++ b/Test/comp/TypeParams.dfy @@ -5,15 +5,19 @@ compileTargetOverrides: cs: expected: outputFile: TypeParams.dfy.cs.expect + specialCaseReason: Function output is inconsistent java: expected: outputFile: TypeParams.dfy.java.expect + specialCaseReason: Function output is inconsistent js: expected: outputFile: TypeParams.dfy.js.expect + specialCaseReason: Function output is inconsistent go: expected: outputFile: TypeParams.dfy.go.expect + specialCaseReason: Function output is inconsistent */ datatype Color = Orange | Pink | Teal type Six = x | x <= 6 From 65aec14d48fe0a41f22dd1afc69d02111ec51208 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 9 Oct 2020 07:17:44 -0700 Subject: [PATCH 078/192] Fix GHA build to use dotnet --- .github/workflows/msbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index 63ee1a4d030..da98ee13bbe 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -48,4 +48,4 @@ jobs: - uses: actions/setup-node@v1 - run: npm install bignumber.js - name: Run Dafny tests - run: XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 msbuild -t:Test dafny/Source/DafnyTests/DafnyTests.csproj + run: XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 dotnet test dafny/Source/Dafny.sln From 2f5e65012609da5246316608b910830cc3191a2d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 9 Oct 2020 07:38:58 -0700 Subject: [PATCH 079/192] Force add missing ignored files --- Test/comp/BranchCoverage2.cs | 20 +++++++++++++ Test/comp/BranchCoverage3.js | 25 ++++++++++++++++ Test/comp/CSharpStyling2.cs | 37 ++++++++++++++++++++++++ Test/comp/Extern2.cs | 46 ++++++++++++++++++++++++++++++ Test/comp/Extern3.js | 55 ++++++++++++++++++++++++++++++++++++ Test/comp/Library.cs | 19 +++++++++++++ Test/comp/Library.js | 21 ++++++++++++++ 7 files changed, 223 insertions(+) create mode 100644 Test/comp/BranchCoverage2.cs create mode 100644 Test/comp/BranchCoverage3.js create mode 100644 Test/comp/CSharpStyling2.cs create mode 100644 Test/comp/Extern2.cs create mode 100644 Test/comp/Extern3.js create mode 100644 Test/comp/Library.cs create mode 100644 Test/comp/Library.js diff --git a/Test/comp/BranchCoverage2.cs b/Test/comp/BranchCoverage2.cs new file mode 100644 index 00000000000..174d297ee7e --- /dev/null +++ b/Test/comp/BranchCoverage2.cs @@ -0,0 +1,20 @@ +using System; + +namespace DafnyProfiling { + public class CodeCoverage { + static uint[] tallies; + public static void Setup(int size) { + tallies = new uint[size]; + } + public static void TearDown() { + for (var i = 0; i < tallies.Length; i++) { + Console.WriteLine("{0}: {1}", i, tallies[i]); + } + tallies = null; + } + public static bool Record(int id) { + tallies[id]++; + return true; + } + } +} diff --git a/Test/comp/BranchCoverage3.js b/Test/comp/BranchCoverage3.js new file mode 100644 index 00000000000..7923dd9ac8e --- /dev/null +++ b/Test/comp/BranchCoverage3.js @@ -0,0 +1,25 @@ +let DafnyProfiling = (function() { + let $module = {}; + + $module.tallies = [] + + $module.CodeCoverage = class CodeCoverage { + static Setup(size) { + for (let i = 0; i < size; i++) { + $module.tallies[i] = 0 + } + } + static TearDown() { + for (let i = 0; i < $module.tallies.length; i++) { + process.stdout.write("" + i + ": " + $module.tallies[i] + "\n"); + } + $module.tallies = null; + } + static Record(id) { + $module.tallies[id]++ + return true; + } + }; + + return $module; +})(); diff --git a/Test/comp/CSharpStyling2.cs b/Test/comp/CSharpStyling2.cs new file mode 100644 index 00000000000..7a7b6e9eeb2 --- /dev/null +++ b/Test/comp/CSharpStyling2.cs @@ -0,0 +1,37 @@ +using System.Numerics; + +namespace _module { + public partial class MyClass { + public MyClass() { } // needed by Dafny in order to call constructor "Init" + public MyClass(BigInteger x) { + this.u = x; + } + public MyClass(bool y) { + this.u = y ? 3 : -3; + } + + public BigInteger OneResultExtern(BigInteger z) { + BigInteger r = 12; + if (z == 0) { + r = 8; + return r; + } else if (z == 1) { + r = 10; + return r; + } + return r; + } + + public void TwoResultsExtern(BigInteger z, out BigInteger r, out BigInteger s) { + if (z == 0) { + r = 5; + s = 7; + return; + } else { + r = 9; + s = 11; + } + return; + } + } +} diff --git a/Test/comp/Extern2.cs b/Test/comp/Extern2.cs new file mode 100644 index 00000000000..8ea42b9a2b4 --- /dev/null +++ b/Test/comp/Extern2.cs @@ -0,0 +1,46 @@ +using System.Numerics; + +namespace Library { + public partial class LibClass { + // static method CallMeInt(x: int) returns (y: int, z: int) + public static void CallMeInt(BigInteger x, out BigInteger y, out BigInteger z) { + y = x + BigInteger.One; + z = y + y; + } + // static method CallMeNative(x: MyInt, b: bool) returns (y: MyInt) + public static int CallMeNative(int x, bool b) { + var y = b ? x + 1 : x - 1; + return y; + } + } + + class OtherClass { + public static object CallMe() { + return "OtherClass.CallMe"; + } + } + + // must be partial, since Dafny will also generate some methods into this class + public partial class Mixed { + public static void P() { + System.Console.WriteLine("Mixed.P"); + } + public void IP() { + System.Console.WriteLine("Mixed.IP"); + } + public static BigInteger G() { + return 1; + } + public BigInteger IG() { + return 2; + } + } + // It's okay for the following class to not be partial, since Dafny won't be adding + // any members to it. In fact, this test is to make sure that Dafny does not + // generate this class at all. + public class AllExtern { + public static void P() { + System.Console.WriteLine("AllExtern.P"); + } + } +} diff --git a/Test/comp/Extern3.js b/Test/comp/Extern3.js new file mode 100644 index 00000000000..0992fc65dcb --- /dev/null +++ b/Test/comp/Extern3.js @@ -0,0 +1,55 @@ +let Library = (function() { + let $module = {}; + + $module.LibClass = class LibClass { + // static method CallMeInt(x: int) returns (y: int, z: int) + static CallMeInt(x) { + let y = x.plus(new BigNumber(1)); + let z = y.plus(y); + return [y, z]; + } + // static method CallMeNative(x: MyInt, b: bool) returns (y: MyInt) + static CallMeNative(x, b) { + let y = b ? x + 1 : x - 1; + return y; + } + }; + + $module.OtherClass = class OtherClass { + static CallMe() { + return "OtherClass.CallMe"; + } + } + + $module.AllDafny = class AllDafny { + // Just here so the generated class can extend it + } + + $module.Mixed = class Mixed { + constructor() { } + + // static method P() + static P() { + process.stdout.write("Mixed.P\n"); + } + + IP() { + process.stdout.write("Mixed.IP\n"); + } + + static G() { + return new BigNumber(1); + } + + IG() { + return new BigNumber(2); + } + } + $module.AllExtern = class AllExtern { + static P() { + process.stdout.write("AllExtern.P\n"); + } + }; + + return $module; +})(); diff --git a/Test/comp/Library.cs b/Test/comp/Library.cs new file mode 100644 index 00000000000..cb6e918524b --- /dev/null +++ b/Test/comp/Library.cs @@ -0,0 +1,19 @@ +using System.Numerics; + +namespace Library { + public partial class Class { + private readonly BigInteger n; + + public Class(BigInteger n) { + this.n = n; + } + + public static void SayHi() { + System.Console.WriteLine("Hello!"); + } + + public BigInteger Get() { + return n; + } + } +} diff --git a/Test/comp/Library.js b/Test/comp/Library.js new file mode 100644 index 00000000000..99f81dff92f --- /dev/null +++ b/Test/comp/Library.js @@ -0,0 +1,21 @@ +// FIXME Test currently fails on JavaScript + +let Library = (function() { + let $module = {}; + + $module.Class = class Class { + constructor(n) { + this.n = n; + } + + static SayHi() { + process.stdout.write("Hello!\n"); + } + + Get() { + return this.n; + } + } + + return $module; +})(); \ No newline at end of file From 12e1e9cf883645062d7b0f5297650c5422b01f1e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 9 Oct 2020 09:09:47 -0700 Subject: [PATCH 080/192] Reset Hello.dfy execute bits (from experimentation with shebangs) --- Test/comp/Hello.dfy | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 Test/comp/Hello.dfy diff --git a/Test/comp/Hello.dfy b/Test/comp/Hello.dfy old mode 100755 new mode 100644 From 8e85a2f36b9a246e7996a21a2a789ad478799e40 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 9 Oct 2020 09:15:17 -0700 Subject: [PATCH 081/192] Move Library.* files for ExternCtors.dfy back --- Test/comp/{ => ExternCtors-externs}/Library.cs | 0 Test/comp/{ => ExternCtors-externs}/Library.go | 0 Test/comp/{ => ExternCtors-externs}/Library.js | 0 Test/comp/ExternCtors.dfy | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename Test/comp/{ => ExternCtors-externs}/Library.cs (100%) rename Test/comp/{ => ExternCtors-externs}/Library.go (100%) rename Test/comp/{ => ExternCtors-externs}/Library.js (100%) diff --git a/Test/comp/Library.cs b/Test/comp/ExternCtors-externs/Library.cs similarity index 100% rename from Test/comp/Library.cs rename to Test/comp/ExternCtors-externs/Library.cs diff --git a/Test/comp/Library.go b/Test/comp/ExternCtors-externs/Library.go similarity index 100% rename from Test/comp/Library.go rename to Test/comp/ExternCtors-externs/Library.go diff --git a/Test/comp/Library.js b/Test/comp/ExternCtors-externs/Library.js similarity index 100% rename from Test/comp/Library.js rename to Test/comp/ExternCtors-externs/Library.js diff --git a/Test/comp/ExternCtors.dfy b/Test/comp/ExternCtors.dfy index 4c75b0b8e30..fcd3f575ec4 100644 --- a/Test/comp/ExternCtors.dfy +++ b/Test/comp/ExternCtors.dfy @@ -7,16 +7,16 @@ compileTargetOverrides: - Class.java cs: otherFiles: - - Library.cs + - ExternCtors-externs/Library.cs js: otherFiles: - - Library.js + - ExternCtors-externs/Library.js expected: outputFile: ~ specialCaseReason: Extern constructors are currently broken in JavaScript go: otherFiles: - - Library.go + - ExternCtors-externs/Library.go expected: outputFile: ~ specialCaseReason: Extern constructors are currently broken in Go From 596196a0b874843879b103a5e264e073adc63de0 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 11 Oct 2020 08:41:17 -0700 Subject: [PATCH 082/192] Refactor some constants --- Source/DafnyTests/DafnyTestCase.cs | 46 +++++++++++-------- .../XUnitExtensions/YamlDataDiscoverer.cs | 2 +- .../LitTestConverter/LitTestConverter.csproj | 4 ++ Source/LitTestConverter/LitTestConvertor.cs | 24 +++++----- 4 files changed, 43 insertions(+), 33 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 94e986f7817..cfeba230e1f 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -36,6 +36,11 @@ public class DafnyTestSpec : IEnumerable { public static readonly string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/dafny"); + // Dafny options with special handling + public const string DAFNY_COMPILE_OPTION = "compile"; + public const string DAFNY_COMPILE_TARGET_OPTION = "compileTarget"; + public const string DAFNY_NO_VERIFY_OPTION = "noVerify"; + private static readonly IFileProvider manifestFileProvider = new ManifestEmbeddedFileProvider( Assembly.GetExecutingAssembly()); private static readonly Dictionary PathsForResourceNames = GetPathsForResourceNames( @@ -84,18 +89,18 @@ public IEnumerator GetEnumerator() { public string Compile { get { - if (DafnyArguments.TryGetValue("compile", out var compile)) { + if (DafnyArguments.TryGetValue(DAFNY_COMPILE_OPTION, out var compile)) { return (string) compile; } return null; } set { - DafnyArguments["compile"] = value; + DafnyArguments[DAFNY_COMPILE_OPTION] = value; } } public string CompileTarget { get { - if (DafnyArguments.TryGetValue("compileTarget", out var compileTarget)) { + if (DafnyArguments.TryGetValue(DAFNY_COMPILE_TARGET_OPTION, out var compileTarget)) { return (string) compileTarget; } return null; @@ -111,15 +116,15 @@ private IEnumerable ResolveCompile() { var compileTargets = new[] {"cs", "java", "go", "js"}; var justVerify = new Dictionary(DafnyArguments) { - ["compile"] = "0" + [DAFNY_COMPILE_OPTION] = "0" }; yield return new DafnyTestSpec(SourcePath, justVerify, OtherFiles, DafnyTestCase.Expectation.NO_OUTPUT); foreach (var compileTarget in compileTargets) { var withLanguage = new Dictionary(DafnyArguments) { - ["noVerify"] = "yes", - ["compile"] = "4", - ["compileTarget"] = compileTarget + [DAFNY_NO_VERIFY_OPTION] = "yes", + [DAFNY_COMPILE_OPTION] = "4", + [DAFNY_COMPILE_TARGET_OPTION] = compileTarget }; var specForLanguage = new DafnyTestSpec(SourcePath, withLanguage, OtherFiles, Expected); if (CompileTargetOverrides.TryGetValue(compileTarget, out var compileTargetOverride)) { @@ -191,6 +196,8 @@ private static IEnumerable> ExpandValue(KeyValuePai public class ForEachArgumentList : List { } + // TODO-RS: This is really close to a generic CLI test case - + // can we move all the dafny-specific stuff to DafnyTestSpec instead? public class DafnyTestCase: IXunitSerializable { public string[] Arguments; @@ -341,27 +348,28 @@ public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { @"!dafnyTestSpec dafnyArguments: {} "; - - public override IParser GetYamlParser(string manifestResourceName, Stream stream) { - if (!manifestResourceName.EndsWith(".dfy")) { - return null; - } - - string content = DEFAULT_CONFIG; - - // TODO-RS: Figure out how to do this cleanly on a TextReader instead, - // and to handle things like nested comments. - string fullText = new StreamReader(stream).ReadToEnd(); + + public static string GetTestCaseConfigYaml(string fullText) { var commentStart = fullText.IndexOf("/*"); if (commentStart >= 0) { var commentEnd = fullText.IndexOf("*/", commentStart + 2); if (commentEnd >= 0) { var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); if (commentContent.StartsWith("---")) { - content = commentContent; + return commentContent; } } } + + return null; + } + + public override IParser GetYamlParser(string manifestResourceName, Stream stream) { + if (!manifestResourceName.EndsWith(".dfy")) { + return null; + } + + string content = GetTestCaseConfigYaml(new StreamReader(stream).ReadToEnd()) ?? DEFAULT_CONFIG; return new Parser(new StringReader(content)); } diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs index ccc23b449ba..e48aa8bb246 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs @@ -42,7 +42,7 @@ public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodIn private IEnumerable ResourceData(MethodInfo testMethod, string resourceName, bool withParameterNames) { try { - Stream stream = testMethod.DeclaringType.Assembly.GetManifestResourceStream(resourceName); + using Stream stream = testMethod.DeclaringType.Assembly.GetManifestResourceStream(resourceName); IParser parser = GetYamlParser(resourceName, stream); if (parser == null) { return Enumerable.Empty(); diff --git a/Source/LitTestConverter/LitTestConverter.csproj b/Source/LitTestConverter/LitTestConverter.csproj index aa00977a2ba..66280b7c884 100644 --- a/Source/LitTestConverter/LitTestConverter.csproj +++ b/Source/LitTestConverter/LitTestConverter.csproj @@ -8,4 +8,8 @@ false + + + + \ No newline at end of file diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 8c5f39fa0c7..79951bcc714 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -2,9 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using Microsoft.SqlServer.Server; namespace DafnyTests { @@ -13,12 +10,12 @@ public class LitTestConvertor { private const string DAFNY_COMMENT_PREFIX = "//"; private const string LIT_COMMAND_PREFIX = "RUN:"; private const string LIT_DAFNY = "%dafny"; - private const string DAFNY_COMPILE = "compile"; - private const string DAFNY_COMPILE_TARGET = "compileTarget"; + private const string LIT_SERVER = "%server"; + // Fake options to which files are passed to the CLI - private const string TEST_CONFIG_OTHER_FILES = "otherFiles"; - private const string TEST_CONFIG_INCLUDE_THIS_FILE = "includeThisFile"; - + public const string TEST_CONFIG_OTHER_FILES = "otherFiles"; + public const string TEST_CONFIG_INCLUDE_THIS_FILE = "includeThisFile"; + // Arguments that are taken care of automatically. If a test is actually using the output of // one of these as input in another command, that will be flagged as an unsupported // use of lit substitution. @@ -35,7 +32,7 @@ public class LitTestConvertor { private int invalidCount = 0; public void ConvertLitTest(string filePath) { - if (filePath.Contains("/Inputs/") || filePath.Contains("/comp/")) { + if (filePath.Contains("/Inputs/")) { // TODO-RS: Need to add .common.yml file to disable Inputs/*.dfy return; } @@ -48,8 +45,7 @@ public void ConvertLitTest(string filePath) { throw new ArgumentException("Lit commands are not consecutive"); } if (!litCommands.Any()) { - // TODO-RS: Check if in an Inputs directory - throw new ArgumentException("No lit commands found"); + return; } if (!litCommands[^1].Equals("%diff \"%s.expect\" \"%t\"")) { @@ -61,9 +57,10 @@ public void ConvertLitTest(string filePath) { if (testConfigs.Count == 1 && testConfigs[0].Count == 1 && - DictionaryContainsEntry(testConfigs[0], DAFNY_COMPILE, "0")) { + DictionaryContainsEntry(testConfigs[0], DafnyTestSpec.DAFNY_COMPILE_OPTION, "0")) { verifyOnlyCount++; - } else if (testConfigs.Count(c => c.ContainsKey(DAFNY_COMPILE_TARGET)) > 1) { + + } else if (testConfigs.Count(c => c.ContainsKey(DafnyTestSpec.DAFNY_COMPILE_TARGET_OPTION)) > 1) { defaultCount++; } } @@ -154,6 +151,7 @@ private static KeyValuePair ParseDafnyArgument(string argument) } public void Run(string root) { + // TODO-RS: Search for "*.transcript" too foreach (var file in Directory.GetFiles(root, "*.dfy", SearchOption.AllDirectories)) { try { count++; From 03658d7630478d2cebc4de042bc5298b242fdefd Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 13 Oct 2020 11:02:21 -0700 Subject: [PATCH 083/192] Weird issue around Process.ExitCode being wrong? --- Source/DafnyTests/DafnyTestCase.cs | 4 ++-- Source/LitTestConverter/LitTestConvertor.cs | 6 +++--- Test/comp/DontRunMe.dfy | 1 + Test/dafny4/git-issue250.dfy | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 Test/comp/DontRunMe.dfy diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index cfeba230e1f..16d6ab1ab0c 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -281,7 +281,7 @@ public static ProcessResult RunDafny(IEnumerable arguments) { using (Process dafnyProcess = new Process()) { dafnyProcess.StartInfo.FileName = DafnyTestSpec.DAFNY_EXE; - foreach (var argument in dafnyArguments) { + foreach(var argument in dafnyArguments) { dafnyProcess.StartInfo.Arguments += " " + argument; } @@ -299,9 +299,9 @@ public static ProcessResult RunDafny(IEnumerable arguments) { dafnyProcess.StartInfo.EnvironmentVariables.Add("HOME", Environment.GetEnvironmentVariable("HOME")); dafnyProcess.Start(); - dafnyProcess.WaitForExit(); string output = dafnyProcess.StandardOutput.ReadToEnd(); string error = dafnyProcess.StandardError.ReadToEnd(); + dafnyProcess.WaitForExit(); return new ProcessResult(dafnyProcess.ExitCode, output, error); } } diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 79951bcc714..8c78077e406 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -85,8 +85,8 @@ private static string ExtractLitCommand(string line) { return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); } - private Dictionary ParseDafnyCommandArguments(string dafnyCommand) { - var testConfig = new Dictionary(); + private Dictionary ParseDafnyCommandArguments(string dafnyCommand) { + var testConfig = new Dictionary(); var otherFiles = new List(); if (!dafnyCommand.StartsWith(LIT_DAFNY)) { @@ -128,7 +128,7 @@ private Dictionary ParseDafnyCommandArguments(string dafnyComman } if (otherFiles.Any()) { - testConfig[TEST_CONFIG_OTHER_FILES] = String.Join(" ", otherFiles); + testConfig[TEST_CONFIG_OTHER_FILES] = otherFiles; otherFilesCount++; } return testConfig; diff --git a/Test/comp/DontRunMe.dfy b/Test/comp/DontRunMe.dfy new file mode 100644 index 00000000000..f100088c38f --- /dev/null +++ b/Test/comp/DontRunMe.dfy @@ -0,0 +1 @@ +Not a valid Dafny file \ No newline at end of file diff --git a/Test/dafny4/git-issue250.dfy b/Test/dafny4/git-issue250.dfy index ccc2be990d5..49562c6e4e1 100644 --- a/Test/dafny4/git-issue250.dfy +++ b/Test/dafny4/git-issue250.dfy @@ -1,5 +1,5 @@ /*--- -arith: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +arith: !foreach [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] compile: 1 */ method Main() { From a70ebb83d0145aad2b07d51b36678ffd2215d48a Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 13 Oct 2020 14:28:44 -0700 Subject: [PATCH 084/192] Remove DontRunMe.dfy, refactor covertor code I was tripped up by /countVerificationErrors:0 --- Source/DafnyTests/DafnyTestCase.cs | 59 +++++++++++---------- Source/LitTestConverter/LitTestConvertor.cs | 29 +++++----- Test/comp/DontRunMe.dfy | 1 - 3 files changed, 46 insertions(+), 43 deletions(-) delete mode 100644 Test/comp/DontRunMe.dfy diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 16d6ab1ab0c..aafa85a6634 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -1,12 +1,10 @@ using System; using System.Collections; using System.Collections.Generic; -using System.ComponentModel.Design; using System.Diagnostics; using System.IO; using System.Linq; using System.Reflection; -using FluentAssertions; using Microsoft.Extensions.FileProviders; using Xunit; using Xunit.Abstractions; @@ -41,45 +39,28 @@ public class DafnyTestSpec : IEnumerable { public const string DAFNY_COMPILE_TARGET_OPTION = "compileTarget"; public const string DAFNY_NO_VERIFY_OPTION = "noVerify"; - private static readonly IFileProvider manifestFileProvider = new ManifestEmbeddedFileProvider( - Assembly.GetExecutingAssembly()); - private static readonly Dictionary PathsForResourceNames = GetPathsForResourceNames( - "DafnyTests", manifestFileProvider, "DafnyTests"); - // Absolute file system path to the main Dafny file - public string SourcePath; + [YamlIgnore] + public readonly string SourcePath; public Dictionary DafnyArguments = new Dictionary(); - public IEnumerable OtherFiles = Enumerable.Empty(); + public List OtherFiles = new List(); public Dictionary CompileTargetOverrides = new Dictionary(); public DafnyTestCase.Expectation Expected; - public DafnyTestSpec(string manifestResourceName) { - SourcePath = COMP_DIR + PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); + public DafnyTestSpec(string sourcePath) { + SourcePath = sourcePath; } - private DafnyTestSpec(string sourcePath, Dictionary dafnyArguments, IEnumerable otherFiles, DafnyTestCase.Expectation expected) { + private DafnyTestSpec(string sourcePath, Dictionary dafnyArguments, List otherFiles, DafnyTestCase.Expectation expected) { SourcePath = sourcePath; DafnyArguments = dafnyArguments; OtherFiles = otherFiles; Expected = expected; } - private static Dictionary GetPathsForResourceNames(string assemblyName, IFileProvider fileProvider, string path = null) { - return fileProvider.GetDirectoryContents(path).SelectMany(file => { - var childName = path == null ? file.Name : path + "/" + file.Name; - if (file.IsDirectory) { - return GetPathsForResourceNames(assemblyName, fileProvider, childName); - } else { - var result = new Dictionary(); - result[assemblyName + "." + childName.Replace("/", ".")] = childName; - return result; - } - }).ToDictionary(pair => pair.Key, pair => pair.Value); - } - public IEnumerator GetEnumerator() { return ResolveCompile() .SelectMany(spec => spec.ExpandArguments()) @@ -143,7 +124,7 @@ private DafnyTestSpec ApplyOverride(DafnyTestSpec otherSpec) { foreach (KeyValuePair pair in otherSpec.DafnyArguments) { mergedArguments[pair.Key] = pair.Value; } - return new DafnyTestSpec(otherSpec.SourcePath, mergedArguments, OtherFiles.Concat(otherSpec.OtherFiles), otherSpec.Expected); + return new DafnyTestSpec(otherSpec.SourcePath, mergedArguments, OtherFiles.Concat(otherSpec.OtherFiles).ToList(), otherSpec.Expected); } private DafnyTestSpec ResolveExpected() { @@ -349,6 +330,30 @@ public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { dafnyArguments: {} "; + private static readonly IFileProvider ManifestFileProvider = new ManifestEmbeddedFileProvider( + Assembly.GetExecutingAssembly()); + private static readonly Dictionary PathsForResourceNames = GetPathsForResourceNames( + "DafnyTests", ManifestFileProvider, "DafnyTests"); + + private static Dictionary GetPathsForResourceNames(string assemblyName, IFileProvider fileProvider, string path = null) { + return fileProvider.GetDirectoryContents(path).SelectMany(file => { + var childName = path == null ? file.Name : path + "/" + file.Name; + if (file.IsDirectory) { + return GetPathsForResourceNames(assemblyName, fileProvider, childName); + } else { + var result = new Dictionary(); + result[assemblyName + "." + childName.Replace("/", ".")] = childName; + return result; + } + }).ToDictionary(pair => pair.Key, pair => pair.Value); + } + + private static DafnyTestSpec SpecForResourceName(string manifestResourceName) { + string filePath = DafnyTestSpec.COMP_DIR + + PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); + return new DafnyTestSpec(filePath); + } + public static string GetTestCaseConfigYaml(string fullText) { var commentStart = fullText.IndexOf("/*"); if (commentStart >= 0) { @@ -377,7 +382,7 @@ public override IParser GetYamlParser(string manifestResourceName, Stream stream public override IDeserializer GetDeserializer(string manifestResourceName) { var defaultObjectFactory = new DefaultObjectFactory(); var customObjectFactory = new LambdaObjectFactory(type => - type == typeof(DafnyTestSpec) ? new DafnyTestSpec(manifestResourceName) : defaultObjectFactory.Create(type)); + type == typeof(DafnyTestSpec) ? SpecForResourceName(manifestResourceName) : defaultObjectFactory.Create(type)); return new DeserializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 8c78077e406..df335649ca6 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -53,16 +53,20 @@ public void ConvertLitTest(string filePath) { } litCommands.RemoveAt(litCommands.Count - 1); - var testConfigs = litCommands.Select(ParseDafnyCommandArguments).ToList(); + List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(filePath, c)).ToList(); if (testConfigs.Count == 1 && - testConfigs[0].Count == 1 && - DictionaryContainsEntry(testConfigs[0], DafnyTestSpec.DAFNY_COMPILE_OPTION, "0")) { + testConfigs[0].DafnyArguments.Count == 1 && + DictionaryContainsEntry(testConfigs[0].DafnyArguments, DafnyTestSpec.DAFNY_COMPILE_OPTION, "0")) { verifyOnlyCount++; - } else if (testConfigs.Count(c => c.ContainsKey(DafnyTestSpec.DAFNY_COMPILE_TARGET_OPTION)) > 1) { + } else if (testConfigs.Count(c => c.CompileTarget != null) > 1) { defaultCount++; } + + IEnumerable otherLines = lines.Skip(litCommands.Count); + + } private static bool DictionaryContainsEntry(Dictionary dictionary, K key, V value) { @@ -85,9 +89,8 @@ private static string ExtractLitCommand(string line) { return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); } - private Dictionary ParseDafnyCommandArguments(string dafnyCommand) { - var testConfig = new Dictionary(); - var otherFiles = new List(); + private DafnyTestSpec ParseDafnyCommandArguments(string filePath, string dafnyCommand) { + var spec = new DafnyTestSpec(filePath); if (!dafnyCommand.StartsWith(LIT_DAFNY)) { throw new ArgumentException("Lit command is not expected %dafny: " + dafnyCommand); @@ -108,7 +111,7 @@ private Dictionary ParseDafnyCommandArguments(string dafnyComman arguments.RemoveRange(arguments.Count - 2, 2); if (!arguments.Remove("\"%s\"")) { - testConfig[TEST_CONFIG_INCLUDE_THIS_FILE] = "no"; + spec.DafnyArguments[TEST_CONFIG_INCLUDE_THIS_FILE] = "no"; } // Check the arguments for anything non-standard @@ -121,17 +124,13 @@ private Dictionary ParseDafnyCommandArguments(string dafnyComman throw new ArgumentException("Use of lit substitution (% variable) requires manual conversion: " + argument); } if (pair.Key.Equals(TEST_CONFIG_OTHER_FILES)) { - otherFiles.Add(pair.Value); + spec.OtherFiles.Add(pair.Value); } else { - testConfig.Add(pair.Key, pair.Value); + spec.DafnyArguments.Add(pair.Key, pair.Value); } } - if (otherFiles.Any()) { - testConfig[TEST_CONFIG_OTHER_FILES] = otherFiles; - otherFilesCount++; - } - return testConfig; + return spec; } private static KeyValuePair ParseDafnyArgument(string argument) { diff --git a/Test/comp/DontRunMe.dfy b/Test/comp/DontRunMe.dfy deleted file mode 100644 index f100088c38f..00000000000 --- a/Test/comp/DontRunMe.dfy +++ /dev/null @@ -1 +0,0 @@ -Not a valid Dafny file \ No newline at end of file From ce0a8abc03988229515b40deaa31d28f0654ce36 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 13 Oct 2020 15:00:04 -0700 Subject: [PATCH 085/192] Tweak test configuration to make xunit conversion easier --- Test/c++/datatypes.dfy | 4 +--- Test/c++/datatypes.dfy.expect | 6 +++--- Test/dafny0/Extern.dfy | 2 +- Test/dafny0/ExternCopyFromTrait.dfy | 2 +- Test/dafny0/snapshots/Snapshots0.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots1.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots2.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots3.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots4.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots5.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots6.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots7.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots8.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots9.run.dfy | 2 +- Test/dafny4/git-issue158.dfy | 3 +-- Test/exceptions/GenericOutcomeDt.dfy | 2 +- Test/exceptions/NatOutcome.dfy | 2 +- Test/exceptions/NatOutcomeDt.dfy | 2 +- Test/exceptions/VoidOutcome.dfy | 2 +- Test/exceptions/VoidOutcomeDt.dfy | 2 +- Test/expectations/ExpectAndExceptions.dfy | 2 +- Test/expectations/ExpectWithNonStringMessage.dfy | 2 +- Test/git-issues/git-issue-633A.dfy | 2 +- Test/irondafny0/optimize0.dfy | 3 +-- 24 files changed, 26 insertions(+), 30 deletions(-) diff --git a/Test/c++/datatypes.dfy b/Test/c++/datatypes.dfy index dbdc430f62a..d8e47e627a6 100644 --- a/Test/c++/datatypes.dfy +++ b/Test/c++/datatypes.dfy @@ -1,6 +1,4 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cpp "%s" > "%t".raw -// Remove the absolute file path before the expected error -// RUN: sed 's/[^:]*://' "%t".raw > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cpp "%s" > "%t" // RUN: %diff "%s.expect" "%t" newtype uint32 = i:int | 0 <= i < 0x100000000 diff --git a/Test/c++/datatypes.dfy.expect b/Test/c++/datatypes.dfy.expect index 0b1210403f3..ec681caa598 100644 --- a/Test/c++/datatypes.dfy.expect +++ b/Test/c++/datatypes.dfy.expect @@ -1,11 +1,11 @@ Dafny program verifier finished with 10 verified, 0 errors - 42 - 1 +Case A: 42 +Case B: 1 This is expected 42 1 -32 +Got some:32 Example1s are equal Example4s are equal Example1s are not equal diff --git a/Test/dafny0/Extern.dfy b/Test/dafny0/Extern.dfy index 21644b7a3c1..f5e66f416e3 100644 --- a/Test/dafny0/Extern.dfy +++ b/Test/dafny0/Extern.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:1 /compileTarget:cs /print:"%t.print" /dprint:"%t.dprint" "%s" "%S%{dirsep}Extern2.cs" "%S%{dirsep}ExternHelloLibrary.dll" > "%t" +// RUN: %dafny /compile:1 /compileTarget:cs /print:"%t.print" /dprint:"%t.dprint" "%s" Extern2.cs ExternHelloLibrary.dll > "%t" // RUN: %diff "%s.expect" "%t" method Main() { diff --git a/Test/dafny0/ExternCopyFromTrait.dfy b/Test/dafny0/ExternCopyFromTrait.dfy index 1132aa1e473..8ead0b18c8c 100644 --- a/Test/dafny0/ExternCopyFromTrait.dfy +++ b/Test/dafny0/ExternCopyFromTrait.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:1 /compileTarget:cs /print:"%t.print" /dprint:"%t.dprint" "%s" "%S%{dirsep}ExternCopyFromTraitLibrary.cs" > "%t" +// RUN: %dafny /compile:1 /compileTarget:cs /print:"%t.print" /dprint:"%t.dprint" "%s" ExternCopyFromTraitLibrary.cs > "%t" // RUN: %diff "%s.expect" "%t" module {:extern "M"} M { trait {:extern} T1 { diff --git a/Test/dafny0/snapshots/Snapshots0.run.dfy b/Test/dafny0/snapshots/Snapshots0.run.dfy index d20379e5a5d..4aeef2aae64 100644 --- a/Test/dafny0/snapshots/Snapshots0.run.dfy +++ b/Test/dafny0/snapshots/Snapshots0.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 "%S/Inputs/Snapshots0.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots0.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots1.run.dfy b/Test/dafny0/snapshots/Snapshots1.run.dfy index d4d9085c7b5..5e8a323f8ad 100644 --- a/Test/dafny0/snapshots/Snapshots1.run.dfy +++ b/Test/dafny0/snapshots/Snapshots1.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 "%S/Inputs/Snapshots1.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots1.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots2.run.dfy b/Test/dafny0/snapshots/Snapshots2.run.dfy index ddeb24f9cbf..1658838a09e 100644 --- a/Test/dafny0/snapshots/Snapshots2.run.dfy +++ b/Test/dafny0/snapshots/Snapshots2.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 "%S/Inputs/Snapshots2.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots2.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots3.run.dfy b/Test/dafny0/snapshots/Snapshots3.run.dfy index 681e4d12552..593a13ed18a 100644 --- a/Test/dafny0/snapshots/Snapshots3.run.dfy +++ b/Test/dafny0/snapshots/Snapshots3.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 "%S/Inputs/Snapshots3.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots3.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots4.run.dfy b/Test/dafny0/snapshots/Snapshots4.run.dfy index bdfa722a299..83443bca05f 100644 --- a/Test/dafny0/snapshots/Snapshots4.run.dfy +++ b/Test/dafny0/snapshots/Snapshots4.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 "%S/Inputs/Snapshots4.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots4.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots5.run.dfy b/Test/dafny0/snapshots/Snapshots5.run.dfy index 5393eb52a7a..2ca08b01a09 100644 --- a/Test/dafny0/snapshots/Snapshots5.run.dfy +++ b/Test/dafny0/snapshots/Snapshots5.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 "%S/Inputs/Snapshots5.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots5.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots6.run.dfy b/Test/dafny0/snapshots/Snapshots6.run.dfy index e201988f4c3..fcdeb3692c7 100644 --- a/Test/dafny0/snapshots/Snapshots6.run.dfy +++ b/Test/dafny0/snapshots/Snapshots6.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 "%S/Inputs/Snapshots6.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots6.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots7.run.dfy b/Test/dafny0/snapshots/Snapshots7.run.dfy index c25043306be..b225f46126b 100644 --- a/Test/dafny0/snapshots/Snapshots7.run.dfy +++ b/Test/dafny0/snapshots/Snapshots7.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 "%S/Inputs/Snapshots7.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots7.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots8.run.dfy b/Test/dafny0/snapshots/Snapshots8.run.dfy index f3e9d79f5e3..c277023df73 100644 --- a/Test/dafny0/snapshots/Snapshots8.run.dfy +++ b/Test/dafny0/snapshots/Snapshots8.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:3 /traceCaching:1 /errorTrace:0 "%S/Inputs/Snapshots8.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:3 /traceCaching:1 /errorTrace:0 Inputs/Snapshots8.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots9.run.dfy b/Test/dafny0/snapshots/Snapshots9.run.dfy index 0c1e265b19b..de77de560cc 100644 --- a/Test/dafny0/snapshots/Snapshots9.run.dfy +++ b/Test/dafny0/snapshots/Snapshots9.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:3 /traceCaching:1 "%S/Inputs/Snapshots9.dfy" > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:3 /traceCaching:1 Inputs/Snapshots9.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny4/git-issue158.dfy b/Test/dafny4/git-issue158.dfy index 38c4bfbb3fb..e646fd2e6c6 100644 --- a/Test/dafny4/git-issue158.dfy +++ b/Test/dafny4/git-issue158.dfy @@ -1,8 +1,7 @@ -// See https://github.com/dafny-lang/dafny/issues/508 -// UNSUPPORTED: mono // RUN: %dafny /compile:3 "%s" > "%t" // RUN: %dafny /compile:3 /optimize "%s" >> "%t" // RUN: %diff "%s.expect" "%t" +// See https://github.com/dafny-lang/dafny/issues/508 class Class { } diff --git a/Test/exceptions/GenericOutcomeDt.dfy b/Test/exceptions/GenericOutcomeDt.dfy index e6d8589a9fe..56355c36663 100644 --- a/Test/exceptions/GenericOutcomeDt.dfy +++ b/Test/exceptions/GenericOutcomeDt.dfy @@ -1,6 +1,6 @@ -// Does not test anything Exceptions-related, but is included by other tests // RUN: %dafny "%s" > "%t" // RUN: %diff "%s.expect" "%t" +// Does not test anything Exceptions-related, but is included by other tests datatype Outcome = | Success(value: T) diff --git a/Test/exceptions/NatOutcome.dfy b/Test/exceptions/NatOutcome.dfy index 4394ff5aa2e..1cd20d79ffd 100644 --- a/Test/exceptions/NatOutcome.dfy +++ b/Test/exceptions/NatOutcome.dfy @@ -1,6 +1,6 @@ -// Does not test anything Exceptions-related, but is included by other tests // RUN: %dafny "%s" > "%t" // RUN: %diff "%s.expect" "%t" +// Does not test anything Exceptions-related, but is included by other tests datatype Option = None | Some(get: T) diff --git a/Test/exceptions/NatOutcomeDt.dfy b/Test/exceptions/NatOutcomeDt.dfy index a4480be8c49..1c48070ba12 100644 --- a/Test/exceptions/NatOutcomeDt.dfy +++ b/Test/exceptions/NatOutcomeDt.dfy @@ -1,6 +1,6 @@ -// Does not test anything Exceptions-related, but is included by other tests // RUN: %dafny "%s" > "%t" // RUN: %diff "%s.expect" "%t" +// Does not test anything Exceptions-related, but is included by other tests datatype NatOutcome = | NatSuccess(value: nat) diff --git a/Test/exceptions/VoidOutcome.dfy b/Test/exceptions/VoidOutcome.dfy index db41a1c4b62..b354076ca73 100644 --- a/Test/exceptions/VoidOutcome.dfy +++ b/Test/exceptions/VoidOutcome.dfy @@ -1,6 +1,6 @@ -// Does not test anything Exceptions-related, but is included by other tests // RUN: %dafny "%s" > "%t" // RUN: %diff "%s.expect" "%t" +// Does not test anything Exceptions-related, but is included by other tests trait VoidOutcome { predicate method IsFailure() diff --git a/Test/exceptions/VoidOutcomeDt.dfy b/Test/exceptions/VoidOutcomeDt.dfy index c3b0fc29977..65380b75a1f 100644 --- a/Test/exceptions/VoidOutcomeDt.dfy +++ b/Test/exceptions/VoidOutcomeDt.dfy @@ -1,6 +1,6 @@ -// Does not test anything Exceptions-related, but is included by other tests // RUN: %dafny "%s" > "%t" // RUN: %diff "%s.expect" "%t" +// Does not test anything Exceptions-related, but is included by other tests datatype VoidOutcome = | VoidSuccess diff --git a/Test/expectations/ExpectAndExceptions.dfy b/Test/expectations/ExpectAndExceptions.dfy index 735510069e6..80d3a40eb12 100644 --- a/Test/expectations/ExpectAndExceptions.dfy +++ b/Test/expectations/ExpectAndExceptions.dfy @@ -1,9 +1,9 @@ // RUN: %dafny /compile:3 /compileTarget:cs "%s" > "%t" // RUN: %dafny /compile:3 /compileTarget:go "%s" >> "%t" -// TODO-RS: Need to fix the inconsistent handling of verbatimString() in Java // RUN: %dafny /compile:3 /compileTarget:java "%s" >> "%t" // RUN: %dafny /compile:3 /compileTarget:js "%s" >> "%t" // RUN: %diff "%s.expect" "%t" +// TODO-RS: Need to fix the inconsistent handling of verbatimString() in Java include "../exceptions/NatOutcomeDt.dfy" include "../exceptions/GenericOutcomeDt.dfy" diff --git a/Test/expectations/ExpectWithNonStringMessage.dfy b/Test/expectations/ExpectWithNonStringMessage.dfy index c16a8853ade..c218a8e58ca 100644 --- a/Test/expectations/ExpectWithNonStringMessage.dfy +++ b/Test/expectations/ExpectWithNonStringMessage.dfy @@ -1,9 +1,9 @@ // RUN: %dafny /compile:3 /compileTarget:cs "%s" > "%t" // RUN: %dafny /compile:3 /compileTarget:go "%s" >> "%t" -// TODO-RS: Need to fix the inconsistent handling of verbatimString() in Java // RUN: %dafny /compile:3 /compileTarget:java "%s" >> "%t" // RUN: %dafny /compile:3 /compileTarget:js "%s" >> "%t" // RUN: %diff "%s.expect" "%t" +// TODO-RS: Need to fix the inconsistent handling of verbatimString() in Java datatype Option = None | Some(get: T) diff --git a/Test/git-issues/git-issue-633A.dfy b/Test/git-issues/git-issue-633A.dfy index 3a91bb32bc8..dab31320331 100644 --- a/Test/git-issues/git-issue-633A.dfy +++ b/Test/git-issues/git-issue-633A.dfy @@ -1,5 +1,5 @@ +// RUN: echo "" // This file is not a test - just a companion file for git-issue-633.dfy -// RUN: %diff "%s" "%s" method mm() { m(); diff --git a/Test/irondafny0/optimize0.dfy b/Test/irondafny0/optimize0.dfy index 53e84ce7270..c4b8b2122ca 100644 --- a/Test/irondafny0/optimize0.dfy +++ b/Test/irondafny0/optimize0.dfy @@ -1,7 +1,6 @@ -// See https://github.com/dafny-lang/dafny/issues/508 -// UNSUPPORTED: mono // RUN: %dafny /compile:3 /optimize /print:"%t.print" /dprint:"%t.dprint" "%s" > "%t" // RUN: %diff "%s.expect" "%t" +// See https://github.com/dafny-lang/dafny/issues/508 method Main() { print "o hai!"; From 2da02ccf8e607f46808d9b3ec2baab374d7a5283 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 15 Oct 2020 13:59:32 -0700 Subject: [PATCH 086/192] More progress on test conversion script --- Source/DafnyTests/DafnyTestCase.cs | 41 ++++++-- Source/LitTestConverter/LitTestConvertor.cs | 101 ++++++++++++++------ Test/comp/ExternCtors.dfy | 4 + Test/comp/TypeParams.dfy | 8 +- 4 files changed, 112 insertions(+), 42 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index aafa85a6634..3392e488c67 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -5,7 +5,9 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Text; using Microsoft.Extensions.FileProviders; +using Microsoft.VisualBasic.CompilerServices; using Xunit; using Xunit.Abstractions; using Xunit.Sdk; @@ -68,15 +70,19 @@ public IEnumerator GetEnumerator() { .GetEnumerator(); } - public string Compile { + public int? Compile { get { if (DafnyArguments.TryGetValue(DAFNY_COMPILE_OPTION, out var compile)) { - return (string) compile; + return Int32.Parse((string) compile); } return null; } set { - DafnyArguments[DAFNY_COMPILE_OPTION] = value; + if (value != null) { + DafnyArguments[DAFNY_COMPILE_OPTION] = value.ToString(); + } else { + DafnyArguments.Remove(DAFNY_COMPILE_OPTION); + } } } public string CompileTarget { @@ -88,12 +94,21 @@ public string CompileTarget { } } + public bool NoVerify { + get { + if (DafnyArguments.TryGetValue(DAFNY_NO_VERIFY_OPTION, out var noVerify)) { + return (string)noVerify == "yes"; + } + return false; + } + } + private IEnumerable ResolveCompile() { if (Compile == null) { - Compile = "3"; + Compile = 3; } - if ("3".Equals(Compile) && CompileTarget == null) { + if (Compile > 0) { var compileTargets = new[] {"cs", "java", "go", "js"}; var justVerify = new Dictionary(DafnyArguments) { @@ -104,7 +119,7 @@ private IEnumerable ResolveCompile() { foreach (var compileTarget in compileTargets) { var withLanguage = new Dictionary(DafnyArguments) { [DAFNY_NO_VERIFY_OPTION] = "yes", - [DAFNY_COMPILE_OPTION] = "4", + [DAFNY_COMPILE_OPTION] = Compile > 2 ? "4" : "2", [DAFNY_COMPILE_TARGET_OPTION] = compileTarget }; var specForLanguage = new DafnyTestSpec(SourcePath, withLanguage, OtherFiles, Expected); @@ -219,7 +234,19 @@ public Expectation Adjust() { } public override string ToString() { - return OutputFile ?? "-"; + string result; + if (ExitCode != 0) { + result = String.Format("", ExitCode); + } else if (OutputFile != null) { + result = OutputFile; + } else { + result = ""; + } + + if (SpecialCaseReason != null) { + result += " (special case)"; + } + return result; } } diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index df335649ca6..ce8afa3740d 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using YamlDotNet.Serialization; namespace DafnyTests { @@ -22,30 +23,59 @@ public class LitTestConvertor { private static readonly string[] DAFNY_IGNORED_OPTIONS = { "print", "dprint", - "rprint" + "rprint", + "spillTargetCode" }; private int count = 0; private int verifyOnlyCount = 0; private int defaultCount = 0; - private int otherFilesCount = 0; + private int alreadyConverted = 0; private int invalidCount = 0; - + public void ConvertLitTest(string filePath) { + IEnumerable testSpec; + IEnumerable testContent; + if (filePath.Contains("/Inputs/")) { - // TODO-RS: Need to add .common.yml file to disable Inputs/*.dfy - return; + testSpec = Enumerable.Empty(); + testContent = File.ReadAllLines(filePath); + } else { + + string[] lines = File.ReadAllLines(filePath); + + var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); + if (!litCommands.Any()) { + alreadyConverted++; + return; + } + testContent = lines.Skip(litCommands.Count); + + // Make sure the commands are consecutive + if (testContent.Any(line => ExtractLitCommand(line) != null)) { + throw new ArgumentException("Lit commands are not consecutive"); + } + + testSpec = ConvertLitCommands(filePath, litCommands); } - - string[] lines = File.ReadAllLines(filePath); - var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); - // Make sure the commands are consecutive - if (lines.Skip(litCommands.Count).Any(line => ExtractLitCommand(line) != null)) { - throw new ArgumentException("Lit commands are not consecutive"); + using (StreamWriter file = new StreamWriter(filePath)) { + if (testSpec != null) { + file.WriteLine("/*"); + file.WriteLine("---"); + new Serializer().Serialize(file, testSpec); + file.WriteLine("*/"); + foreach(var line in testContent) { + file.WriteLine(line); + } + } } - if (!litCommands.Any()) { - return; + } + + private IEnumerable ConvertLitCommands(string filePath, List litCommands) { + if (litCommands.Count == 1 && litCommands.Single().StartsWith("echo")) { + // This is an idiom for Dafny files used elsewhere + return Enumerable.Empty(); } if (!litCommands[^1].Equals("%diff \"%s.expect\" \"%t\"")) { @@ -55,26 +85,33 @@ public void ConvertLitTest(string filePath) { List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(filePath, c)).ToList(); - if (testConfigs.Count == 1 && - testConfigs[0].DafnyArguments.Count == 1 && - DictionaryContainsEntry(testConfigs[0].DafnyArguments, DafnyTestSpec.DAFNY_COMPILE_OPTION, "0")) { - verifyOnlyCount++; - - } else if (testConfigs.Count(c => c.CompileTarget != null) > 1) { + if (testConfigs.Count == 1) { + var single = testConfigs.Single(); + if (IsStandardVerifyOnly(single)) { + verifyOnlyCount++; + } + return single; + } + + if (IsStandardVerifyOnly(testConfigs[0]) && testConfigs.Skip(1).All(IsStandardCompileAndRun) + || testConfigs.Skip(1).All(IsStandardCompileAndRun)) { defaultCount++; + return null; } - - IEnumerable otherLines = lines.Skip(litCommands.Count); - + throw new ArgumentException("Multi-command lit tests require manual conversion"); } - - private static bool DictionaryContainsEntry(Dictionary dictionary, K key, V value) { - if (dictionary.TryGetValue(key, out var dictionaryValue)) { - return value.Equals(dictionaryValue); - } else { - return false; - } + + private static bool IsStandardCompileAndRun(DafnyTestSpec spec) { + + return spec.CompileTarget != null && + ((spec.Compile == 3 && spec.DafnyArguments.Count == 2) || + // /compile:4 might be used with /noVerify + (spec.Compile == 4 && spec.DafnyArguments.Count <= 3)); + } + + private static bool IsStandardVerifyOnly(DafnyTestSpec spec) { + return spec.Compile == 0 && spec.DafnyArguments.Count == 1; } private static string ExtractLitCommand(string line) { @@ -91,7 +128,7 @@ private static string ExtractLitCommand(string line) { private DafnyTestSpec ParseDafnyCommandArguments(string filePath, string dafnyCommand) { var spec = new DafnyTestSpec(filePath); - + if (!dafnyCommand.StartsWith(LIT_DAFNY)) { throw new ArgumentException("Lit command is not expected %dafny: " + dafnyCommand); } @@ -160,9 +197,11 @@ public void Run(string root) { Console.WriteLine(file + ": " + e.Message); } } + + Console.WriteLine(""); + Console.WriteLine("Already converted: " + alreadyConverted + "/" + count); Console.WriteLine("Default: " + defaultCount + "/" + count); Console.WriteLine("Verify only: " + verifyOnlyCount + "/" + count); - Console.WriteLine("With other files: " + otherFilesCount + "/" + count); Console.WriteLine("Invalid: " + invalidCount + "/" + count); } diff --git a/Test/comp/ExternCtors.dfy b/Test/comp/ExternCtors.dfy index fcd3f575ec4..0ca0a6eff83 100644 --- a/Test/comp/ExternCtors.dfy +++ b/Test/comp/ExternCtors.dfy @@ -12,12 +12,16 @@ compileTargetOverrides: otherFiles: - ExternCtors-externs/Library.js expected: + # The expected exit code should be 3, but /countVerificationErrors:0 is overriding that + # https://github.com/dafny-lang/dafny/issues/887 outputFile: ~ specialCaseReason: Extern constructors are currently broken in JavaScript go: otherFiles: - ExternCtors-externs/Library.go expected: + # The expected exit code should be 3, but /countVerificationErrors:0 is overriding that + # https://github.com/dafny-lang/dafny/issues/887 outputFile: ~ specialCaseReason: Extern constructors are currently broken in Go */ diff --git a/Test/comp/TypeParams.dfy b/Test/comp/TypeParams.dfy index 29b9d5b45fd..c478e5fcbe6 100644 --- a/Test/comp/TypeParams.dfy +++ b/Test/comp/TypeParams.dfy @@ -5,19 +5,19 @@ compileTargetOverrides: cs: expected: outputFile: TypeParams.dfy.cs.expect - specialCaseReason: Function output is inconsistent + specialCaseReason: Function printing is inconsistent java: expected: outputFile: TypeParams.dfy.java.expect - specialCaseReason: Function output is inconsistent + specialCaseReason: Function printing is inconsistent js: expected: outputFile: TypeParams.dfy.js.expect - specialCaseReason: Function output is inconsistent + specialCaseReason: Function printing is inconsistent go: expected: outputFile: TypeParams.dfy.go.expect - specialCaseReason: Function output is inconsistent + specialCaseReason: Function printing is inconsistent */ datatype Color = Orange | Pink | Teal type Six = x | x <= 6 From 703e3b0d3f9b98732038061726d3da24e2c83c29 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 16 May 2021 09:14:15 -0700 Subject: [PATCH 087/192] Partial work on expanding out of comp dir .expect file path is not right yet --- Source/DafnyTests/DafnyTestCase.cs | 50 ++++++++++----------- Source/DafnyTests/DafnyTests.csproj | 3 +- Source/LitTestConverter/LitTestConvertor.cs | 47 +++++++++++++++---- 3 files changed, 65 insertions(+), 35 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 3392e488c67..92e37a2730f 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -15,11 +15,12 @@ using YamlDotNet.Core; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; +using YamlDotNet.Serialization.NodeDeserializers; using YamlDotNet.Serialization.ObjectFactories; namespace DafnyTests { - public class DafnyTestSpec : IEnumerable { + public class DafnyTestSpec: IEnumerable { private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); @@ -31,7 +32,6 @@ public class DafnyTestSpec : IEnumerable { OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; - public static readonly string COMP_DIR = Path.Combine(TEST_ROOT, "comp") + Path.DirectorySeparatorChar; public static readonly string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; public static readonly string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/dafny"); @@ -70,6 +70,11 @@ public IEnumerator GetEnumerator() { .GetEnumerator(); } + IEnumerator IEnumerable.GetEnumerator() { + return GetEnumerator(); + } + + [YamlIgnore] public int? Compile { get { if (DafnyArguments.TryGetValue(DAFNY_COMPILE_OPTION, out var compile)) { @@ -85,6 +90,8 @@ public int? Compile { } } } + + [YamlIgnore] public string CompileTarget { get { if (DafnyArguments.TryGetValue(DAFNY_COMPILE_TARGET_OPTION, out var compileTarget)) { @@ -94,21 +101,13 @@ public string CompileTarget { } } - public bool NoVerify { - get { - if (DafnyArguments.TryGetValue(DAFNY_NO_VERIFY_OPTION, out var noVerify)) { - return (string)noVerify == "yes"; - } - return false; - } - } - private IEnumerable ResolveCompile() { if (Compile == null) { Compile = 3; } - if (Compile > 0) { + if (Compile > 0 && CompileTarget == null) { + // TODO: Include c++ but flag as special case by default? var compileTargets = new[] {"cs", "java", "go", "js"}; var justVerify = new Dictionary(DafnyArguments) { @@ -152,10 +151,9 @@ private DafnyTestSpec ResolveExpected() { } private DafnyTestCase ToTestCase() { - var arguments = new []{ Path.GetRelativePath(TEST_ROOT, SourcePath) } - .Concat(DafnyArguments.Select(ConfigPairToArgument)) - .Concat(OtherFiles.Select(otherFile => "comp/" + otherFile)); - return new DafnyTestCase(arguments, Expected.Adjust()); + var arguments = new []{ SourcePath } + .Concat(DafnyArguments.Select(ConfigPairToArgument)); + return new DafnyTestCase(arguments, Expected); } private static string ConfigPairToArgument(KeyValuePair pair) { @@ -166,10 +164,6 @@ private static string ConfigPairToArgument(KeyValuePair pair) { } } - IEnumerator IEnumerable.GetEnumerator() { - return GetEnumerator(); - } - private IEnumerable ExpandArguments() { return DafnyArguments.Select(ExpandValue) .CartesianProduct() @@ -369,15 +363,18 @@ private static Dictionary GetPathsForResourceNames(string assemb return GetPathsForResourceNames(assemblyName, fileProvider, childName); } else { var result = new Dictionary(); - result[assemblyName + "." + childName.Replace("/", ".")] = childName; + result[ResourceNameForFilePath(assemblyName, childName)] = childName; return result; } }).ToDictionary(pair => pair.Key, pair => pair.Value); } + private static string ResourceNameForFilePath(string assemblyName, string filePath) { + return assemblyName + "." + filePath.Replace("/", ".").Replace("+", "_"); + } + private static DafnyTestSpec SpecForResourceName(string manifestResourceName) { - string filePath = DafnyTestSpec.COMP_DIR + - PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); + string filePath = PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); return new DafnyTestSpec(filePath); } @@ -419,10 +416,10 @@ public override IDeserializer GetDeserializer(string manifestResourceName) { .Build(); } } - + [DataDiscoverer("DafnyTests.DafnyTestYamlDataDiscoverer", "DafnyTests")] public class DafnyTestDataAttribute : YamlDataAttribute { - public DafnyTestDataAttribute(bool withParameterNames = true) : base(withParameterNames) { + public DafnyTestDataAttribute(bool withParameterNames) : base(withParameterNames) { } } @@ -432,7 +429,8 @@ public class DafnyTests { public static void MetaTest() { var discoverer = new DafnyTestYamlDataDiscoverer(); var testMethod = typeof(DafnyTests).GetMethod(nameof(Test)); - discoverer.GetData(testMethod, false).ToList(); + var testData = discoverer.GetData(testMethod, false).ToList(); + Assert.True(testData.Any()); } [ParallelTheory] diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index b1afc42ac82..9473fbc7cfc 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -19,6 +19,7 @@ - + + diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index ce8afa3740d..8d78c84a5a2 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -3,6 +3,8 @@ using System.IO; using System.Linq; using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; +using YamlDotNet.Serialization.ObjectGraphTraversalStrategies; namespace DafnyTests { @@ -23,8 +25,7 @@ public class LitTestConvertor { private static readonly string[] DAFNY_IGNORED_OPTIONS = { "print", "dprint", - "rprint", - "spillTargetCode" + "rprint" }; private int count = 0; @@ -34,7 +35,7 @@ public class LitTestConvertor { private int invalidCount = 0; public void ConvertLitTest(string filePath) { - IEnumerable testSpec; + object testSpec; IEnumerable testContent; if (filePath.Contains("/Inputs/")) { @@ -59,20 +60,29 @@ public void ConvertLitTest(string filePath) { testSpec = ConvertLitCommands(filePath, litCommands); } + ISerializer serializer = new SerializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) + .WithObjectGraphTraversalStrategyFactory((inspector, resolver, converters, recursion) => + new Foo(inspector, resolver, converters, recursion)) + .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) + .WithTagMapping("!foreach", typeof(ForEachArgumentList)) + .Build(); + using (StreamWriter file = new StreamWriter(filePath)) { if (testSpec != null) { file.WriteLine("/*"); file.WriteLine("---"); - new Serializer().Serialize(file, testSpec); + serializer.Serialize(file, testSpec); file.WriteLine("*/"); - foreach(var line in testContent) { - file.WriteLine(line); - } + } + foreach (var line in testContent) { + file.WriteLine(line); } } } - private IEnumerable ConvertLitCommands(string filePath, List litCommands) { + private object ConvertLitCommands(string filePath, List litCommands) { if (litCommands.Count == 1 && litCommands.Single().StartsWith("echo")) { // This is an idiom for Dafny files used elsewhere return Enumerable.Empty(); @@ -188,6 +198,10 @@ private static KeyValuePair ParseDafnyArgument(string argument) public void Run(string root) { // TODO-RS: Search for "*.transcript" too + if (!Directory.Exists(root)) { + ConvertLitTest(root); + return; + } foreach (var file in Directory.GetFiles(root, "*.dfy", SearchOption.AllDirectories)) { try { count++; @@ -204,6 +218,23 @@ public void Run(string root) { Console.WriteLine("Verify only: " + verifyOnlyCount + "/" + count); Console.WriteLine("Invalid: " + invalidCount + "/" + count); } + + private class Foo : FullObjectGraphTraversalStrategy { + public Foo( + ITypeInspector typeDescriptor, + ITypeResolver typeResolver, + int maxRecursion, + INamingConvention namingConvention): base(typeDescriptor, typeResolver, maxRecursion, namingConvention) { + } + + protected override void TraverseObject(IObjectDescriptor value, IObjectGraphVisitor visitor, TContext context, Stack path) { + if (value.Value is DafnyTestSpec) { + TraverseProperties(value, visitor, context, path); + } else { + base.TraverseObject(value, visitor, context, path); + } + } + } public static void Main(string[] args) { new LitTestConvertor().Run(args[0]); From 4402f5338bedff29e111adcc6bdc05fd9100b2a7 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 29 May 2021 11:43:08 -0700 Subject: [PATCH 088/192] Switch to invoking dotnet instead of dafny executable --- Source/DafnyTests/DafnyTestCase.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 92e37a2730f..dbce3fad547 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -34,7 +34,7 @@ public class DafnyTestSpec: IEnumerable { public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; public static readonly string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - public static readonly string DAFNY_EXE = Path.Combine(DAFNY_ROOT, "Binaries/dafny"); + public static readonly string DAFNY_PROJ = Path.Combine(DAFNY_ROOT, "Source/DafnyDriver/DafnyDriver.csproj"); // Dafny options with special handling public const string DAFNY_COMPILE_OPTION = "compile"; @@ -282,11 +282,13 @@ public static ProcessResult RunDafny(IEnumerable arguments) { dafnyArguments.AddRange(arguments); using (Process dafnyProcess = new Process()) { - dafnyProcess.StartInfo.FileName = DafnyTestSpec.DAFNY_EXE; + dafnyProcess.StartInfo.FileName = "dotnet"; + dafnyProcess.StartInfo.Arguments = "run --no-build --project "; + dafnyProcess.StartInfo.Arguments += DafnyTestSpec.DAFNY_PROJ; foreach(var argument in dafnyArguments) { dafnyProcess.StartInfo.Arguments += " " + argument; } - + dafnyProcess.StartInfo.UseShellExecute = false; dafnyProcess.StartInfo.RedirectStandardOutput = true; dafnyProcess.StartInfo.RedirectStandardError = true; From a0a24bdf11417a047f757af1890a454ba5d9d776 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 29 May 2021 12:45:51 -0700 Subject: [PATCH 089/192] Fix expectation path, fix arguments --- Source/DafnyTests/DafnyTestCase.cs | 8 +++----- Source/LitTestConverter/LitTestConvertor.cs | 19 ------------------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index dbce3fad547..4d66f3ee2f7 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -144,10 +144,9 @@ private DafnyTestSpec ApplyOverride(DafnyTestSpec otherSpec) { private DafnyTestSpec ResolveExpected() { if (Expected == null) { return new DafnyTestSpec(SourcePath, DafnyArguments, OtherFiles, - new DafnyTestCase.Expectation(0, Path.GetFileName(SourcePath) + ".expect", null)); - } else { - return this; + new DafnyTestCase.Expectation(0, SourcePath + ".expect", null)); } + return this; } private DafnyTestCase ToTestCase() { @@ -268,8 +267,6 @@ public void Deserialize(IXunitSerializationInfo info) { public static ProcessResult RunDafny(IEnumerable arguments) { // TODO-RS: Let these be overridden List dafnyArguments = new List { - // Expected output does not contain logo - "/nologo", "/countVerificationErrors:0", // We do not want absolute or relative paths in error messages, just the basename of the file @@ -285,6 +282,7 @@ public static ProcessResult RunDafny(IEnumerable arguments) { dafnyProcess.StartInfo.FileName = "dotnet"; dafnyProcess.StartInfo.Arguments = "run --no-build --project "; dafnyProcess.StartInfo.Arguments += DafnyTestSpec.DAFNY_PROJ; + dafnyProcess.StartInfo.Arguments += " --"; foreach(var argument in dafnyArguments) { dafnyProcess.StartInfo.Arguments += " " + argument; } diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 8d78c84a5a2..dd494a6f14f 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -63,8 +63,6 @@ public void ConvertLitTest(string filePath) { ISerializer serializer = new SerializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) - .WithObjectGraphTraversalStrategyFactory((inspector, resolver, converters, recursion) => - new Foo(inspector, resolver, converters, recursion)) .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) .WithTagMapping("!foreach", typeof(ForEachArgumentList)) .Build(); @@ -219,23 +217,6 @@ public void Run(string root) { Console.WriteLine("Invalid: " + invalidCount + "/" + count); } - private class Foo : FullObjectGraphTraversalStrategy { - public Foo( - ITypeInspector typeDescriptor, - ITypeResolver typeResolver, - int maxRecursion, - INamingConvention namingConvention): base(typeDescriptor, typeResolver, maxRecursion, namingConvention) { - } - - protected override void TraverseObject(IObjectDescriptor value, IObjectGraphVisitor visitor, TContext context, Stack path) { - if (value.Value is DafnyTestSpec) { - TraverseProperties(value, visitor, context, path); - } else { - base.TraverseObject(value, visitor, context, path); - } - } - } - public static void Main(string[] args) { new LitTestConvertor().Run(args[0]); } From 89b160f60984a4f59837c0d98306d4dc1721138d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 29 May 2021 17:07:06 -0700 Subject: [PATCH 090/192] Poke CI --- .github/workflows/msbuild.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index 8551d353120..d2271659a0a 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -25,7 +25,6 @@ jobs: ] ## Size of the following array must match --num-shards in lit command shard: [1, 2, 3, 4, 5] - ##shard: [1, 2, 3] include: - os: 'ubuntu-latest' os_for_build: 'ubuntu' From 081f3679c7e0a650e20e32d318879a5051b80451 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 30 May 2021 09:12:17 -0700 Subject: [PATCH 091/192] fix: Temporarily locking down previous dotnet SDK version --- .github/workflows/msbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index d2271659a0a..e414b1af373 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -44,7 +44,7 @@ jobs: - name: Setup dotnet 5.0 uses: actions/setup-dotnet@v1.7.2 with: - dotnet-version: '5.0.x' # SDK Version for building Dafny; x will use the latest version of the 5.0 channel + dotnet-version: '5.0.203' # See https://github.com/dafny-lang/dafny/issues/1238 - name: C++ for ubuntu 16.04 if: matrix.os == 'ubuntu-16.04' run: | From 2d1a00b7da680b16e3b90f0710fa7909455a8996 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 30 May 2021 11:22:35 -0700 Subject: [PATCH 092/192] Fix target framework on new projects --- Source/DafnyTests/DafnyTests.csproj | 2 +- Source/LitTestConverter/LitTestConverter.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index 9473fbc7cfc..9aed5d99a0c 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 + net5.0 DafnyTests true diff --git a/Source/LitTestConverter/LitTestConverter.csproj b/Source/LitTestConverter/LitTestConverter.csproj index 66280b7c884..89176ef67eb 100644 --- a/Source/LitTestConverter/LitTestConverter.csproj +++ b/Source/LitTestConverter/LitTestConverter.csproj @@ -3,7 +3,7 @@ Exe - netcoreapp3.1 + net5.0 LitTestConverter false From b935ecb6098eeddd50a7a9769be22a6265fd4aa2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 6 Aug 2021 11:14:26 -0700 Subject: [PATCH 093/192] Update Source/DafnyTests/DafnyTestCase.cs Co-authored-by: Alex Cioc <4691979+acioc@users.noreply.github.com> --- Source/DafnyTests/DafnyTestCase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 4d66f3ee2f7..9601bb81018 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -171,8 +171,8 @@ private IEnumerable ExpandArguments() { } public static IEnumerable Expand(object obj) { - if (obj is ForEachArgumentList forEach) { - return forEach; + if (obj is ForEachArgumentList forEachArgumentList) { + return forEachArgumentList; } else { return new[] {(string)obj}; } @@ -439,4 +439,4 @@ public static void Test(DafnyTestCase testCase) { testCase.Run(); } } -} \ No newline at end of file +} From 7af490aed03d89f00dd3ff0383f2e9848066e2a9 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 6 Aug 2021 11:14:39 -0700 Subject: [PATCH 094/192] Update Source/DafnyTests/DafnyTestCase.cs Co-authored-by: Alex Cioc <4691979+acioc@users.noreply.github.com> --- Source/DafnyTests/DafnyTestCase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyTests/DafnyTestCase.cs index 9601bb81018..8842e8e3092 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyTests/DafnyTestCase.cs @@ -283,7 +283,7 @@ public static ProcessResult RunDafny(IEnumerable arguments) { dafnyProcess.StartInfo.Arguments = "run --no-build --project "; dafnyProcess.StartInfo.Arguments += DafnyTestSpec.DAFNY_PROJ; dafnyProcess.StartInfo.Arguments += " --"; - foreach(var argument in dafnyArguments) { + foreach (var argument in dafnyArguments) { dafnyProcess.StartInfo.Arguments += " " + argument; } From 3d37243721c374af8a7f5fb30fcba4dc9fbcca1d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 6 Aug 2021 11:32:25 -0700 Subject: [PATCH 095/192] Misc. PR feedback --- Source/DafnyTests/DafnyTests.csproj | 2 +- Source/DafnyTests/TemporaryDirectory.cs | 1 + .../CollectionPerTestCaseTheoryDiscoverer.cs | 23 ++++++++++++++++--- Test/comp/NativeNumbers.dfy | 2 +- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyTests/DafnyTests.csproj index 9aed5d99a0c..6c2a7f559da 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyTests/DafnyTests.csproj @@ -9,7 +9,7 @@ - + diff --git a/Source/DafnyTests/TemporaryDirectory.cs b/Source/DafnyTests/TemporaryDirectory.cs index d1e1f5a090d..6fcb4dfd85f 100644 --- a/Source/DafnyTests/TemporaryDirectory.cs +++ b/Source/DafnyTests/TemporaryDirectory.cs @@ -7,6 +7,7 @@ public class TemporaryDirectory : IDisposable { public TemporaryDirectory(string parent, string prefix = "") { string dirPath; + // Loop until we pick a random name that isn't already taken. do { dirPath = Path.Combine(parent, prefix + Path.GetRandomFileName()); } while (File.Exists(dirPath) || Directory.Exists(dirPath)); diff --git a/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index dc727405674..08eca11e130 100644 --- a/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -25,13 +25,30 @@ public IEnumerable Discover(ITestFrameworkDiscoveryOptions disco discoveryOptions.SetValue("xunit.discovery.PreEnumerateTheories", true); IEnumerable testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); + + // Select the requested fraction of the test cases if using the XUNIT_SHARD[_COUNT} environment variables. + // Ideally this would be handled at a higher level so that cases from different test methods could be + // balanced as a whole. var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD_COUNT"); - // TODO-RS: More careful error checking - if (shardEnvVar != null && numShardsEnvVar != null) { - var testCaseList = testCases.ToList(); + if (shardEnvVar != null || numShardsEnvVar != null) { + if (shardEnvVar == null || numShardsEnvVar == null) { + throw new ArgumentNullException( + "The XUNIT_SHARD and XUNIT_SHARD_COUNT environment variables must both be provided."); + } + var shard = Int32.Parse(shardEnvVar); var numShards = Int32.Parse(numShardsEnvVar); + if (numShards <= 0) { + throw new ArgumentNullException( + "XUNIT_SHARD_COUNT must be greater than 0."); + } + if (shard <= 0 || shard > numShards) { + throw new ArgumentNullException( + "XUNIT_SHARD must be at least 1 and at most XUNIT_SHARD_COUNT."); + } + + var testCaseList = testCases.ToList(); var shardStart = (shard - 1) * testCaseList.Count / numShards; var shardEnd = shard * testCaseList.Count / numShards; testCases = testCaseList.GetRange(shardStart, shardEnd - shardStart); diff --git a/Test/comp/NativeNumbers.dfy b/Test/comp/NativeNumbers.dfy index 9fe0dcbc508..cf9b142bf25 100644 --- a/Test/comp/NativeNumbers.dfy +++ b/Test/comp/NativeNumbers.dfy @@ -5,7 +5,7 @@ compileTargetOverrides: js: expected: outputFile: NativeNumbers.dfy.js.expect - specialCaseReason: :nativeType doesn't worry correctly for JavaScript + specialCaseReason: :nativeType doesn't work correctly for JavaScript */ method Main() { CastTests(); From ec84d254b101fe3459b77b4a32b78729e615adab Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 6 Aug 2021 13:06:11 -0700 Subject: [PATCH 096/192] Rename DafnyTests -> DafnyDriver.Test --- Source/Dafny.sln | 3 ++- .../DafnyDriver.Test.csproj} | 2 +- .../{DafnyTests => DafnyDriver.Test}/DafnyTestCase.cs | 10 ++++------ .../{DafnyTests => DafnyDriver.Test}/ProcessResult.cs | 2 +- .../TemporaryDirectory.cs | 2 +- .../XUnitExtensions/AssertWithDiff.cs | 0 .../CollectionPerTestCaseTheoryDiscoverer.cs | 0 .../XUnitExtensions/EnumerableUtils.cs | 2 +- .../XUnitExtensions/ForEachAttribute.cs | 2 +- .../XUnitExtensions/ParallelTheoryAttribute.cs | 2 +- .../XUnitExtensions/TestCaseWithCollection.cs | 0 .../XUnitExtensions/YamlDataAttribute.cs | 2 +- .../XUnitExtensions/YamlDataDiscoverer.cs | 4 ++-- .../XUnitExtensions/YamlDataTests.cs | 4 ++-- .../YamlDataTests/CalculatorCombinatorialTest.yml | 0 .../XUnitExtensions/YamlDataTests/CalculatorTest.yml | 0 .../YamlDataTests/CustomDataDiscovererTest.yml | 0 .../XUnitExtensions/YamlDataTests/DictionaryTest.yml | 0 Source/LitTestConverter/LitTestConverter.csproj | 2 +- Source/LitTestConverter/LitTestConvertor.cs | 6 +++--- 20 files changed, 21 insertions(+), 22 deletions(-) rename Source/{DafnyTests/DafnyTests.csproj => DafnyDriver.Test/DafnyDriver.Test.csproj} (95%) rename Source/{DafnyTests => DafnyDriver.Test}/DafnyTestCase.cs (98%) rename Source/{DafnyTests => DafnyDriver.Test}/ProcessResult.cs (91%) rename Source/{DafnyTests => DafnyDriver.Test}/TemporaryDirectory.cs (96%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/AssertWithDiff.cs (100%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs (100%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/EnumerableUtils.cs (95%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/ForEachAttribute.cs (89%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/ParallelTheoryAttribute.cs (84%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/TestCaseWithCollection.cs (100%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/YamlDataAttribute.cs (83%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/YamlDataDiscoverer.cs (98%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/YamlDataTests.cs (96%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml (100%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/YamlDataTests/CalculatorTest.yml (100%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml (100%) rename Source/{DafnyTests => DafnyDriver.Test}/XUnitExtensions/YamlDataTests/DictionaryTest.yml (100%) diff --git a/Source/Dafny.sln b/Source/Dafny.sln index eae8373cca7..40932c390f9 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -16,9 +16,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution version.cs = version.cs EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTests", "DafnyTests\DafnyTests.csproj", "{08A3665B-9854-4CF6-BF5F-C532DA39A043}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyDriver.Test", "DafnyDriver.Test\DafnyDriver.Test.csproj", "{08A3665B-9854-4CF6-BF5F-C532DA39A043}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LitTestConverter", "LitTestConverter\LitTestConverter.csproj", "{2EA7D80A-B179-4352-BDF2-8A1599F8685A}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DafnyLanguageServer", "DafnyLanguageServer\DafnyLanguageServer.csproj", "{CD05D26C-C672-4F43-835E-7A3E1741E4D8}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DafnyLanguageServer.Test", "DafnyLanguageServer.Test\DafnyLanguageServer.Test.csproj", "{320C02A3-D3E6-4AC7-A7E2-170AF86A6EEC}" diff --git a/Source/DafnyTests/DafnyTests.csproj b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj similarity index 95% rename from Source/DafnyTests/DafnyTests.csproj rename to Source/DafnyDriver.Test/DafnyDriver.Test.csproj index 6c2a7f559da..082e0f9c9a9 100644 --- a/Source/DafnyTests/DafnyTests.csproj +++ b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj @@ -2,8 +2,8 @@ net5.0 - DafnyTests true + DafnyDriver.Test diff --git a/Source/DafnyTests/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs similarity index 98% rename from Source/DafnyTests/DafnyTestCase.cs rename to Source/DafnyDriver.Test/DafnyTestCase.cs index 8842e8e3092..f05d2232c3f 100644 --- a/Source/DafnyTests/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -5,9 +5,8 @@ using System.IO; using System.Linq; using System.Reflection; -using System.Text; +using DafnyDriver.Test.XUnitExtensions; using Microsoft.Extensions.FileProviders; -using Microsoft.VisualBasic.CompilerServices; using Xunit; using Xunit.Abstractions; using Xunit.Sdk; @@ -15,10 +14,9 @@ using YamlDotNet.Core; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; -using YamlDotNet.Serialization.NodeDeserializers; using YamlDotNet.Serialization.ObjectFactories; -namespace DafnyTests { +namespace DafnyDriver.Test { public class DafnyTestSpec: IEnumerable { @@ -354,7 +352,7 @@ public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { private static readonly IFileProvider ManifestFileProvider = new ManifestEmbeddedFileProvider( Assembly.GetExecutingAssembly()); private static readonly Dictionary PathsForResourceNames = GetPathsForResourceNames( - "DafnyTests", ManifestFileProvider, "DafnyTests"); + "DafnyDriver.Test", ManifestFileProvider, "DafnyTests"); private static Dictionary GetPathsForResourceNames(string assemblyName, IFileProvider fileProvider, string path = null) { return fileProvider.GetDirectoryContents(path).SelectMany(file => { @@ -417,7 +415,7 @@ public override IDeserializer GetDeserializer(string manifestResourceName) { } } - [DataDiscoverer("DafnyTests.DafnyTestYamlDataDiscoverer", "DafnyTests")] + [DataDiscoverer("DafnyDriver.Test.DafnyTestYamlDataDiscoverer", "DafnyDriver.Test")] public class DafnyTestDataAttribute : YamlDataAttribute { public DafnyTestDataAttribute(bool withParameterNames) : base(withParameterNames) { } diff --git a/Source/DafnyTests/ProcessResult.cs b/Source/DafnyDriver.Test/ProcessResult.cs similarity index 91% rename from Source/DafnyTests/ProcessResult.cs rename to Source/DafnyDriver.Test/ProcessResult.cs index 52e9d3b0ad3..e9966ba048f 100644 --- a/Source/DafnyTests/ProcessResult.cs +++ b/Source/DafnyDriver.Test/ProcessResult.cs @@ -1,4 +1,4 @@ -namespace DafnyTests { +namespace DafnyDriver.Test { public class ProcessResult { public int ExitCode; diff --git a/Source/DafnyTests/TemporaryDirectory.cs b/Source/DafnyDriver.Test/TemporaryDirectory.cs similarity index 96% rename from Source/DafnyTests/TemporaryDirectory.cs rename to Source/DafnyDriver.Test/TemporaryDirectory.cs index 6fcb4dfd85f..a73993b0d78 100644 --- a/Source/DafnyTests/TemporaryDirectory.cs +++ b/Source/DafnyDriver.Test/TemporaryDirectory.cs @@ -1,7 +1,7 @@ using System; using System.IO; -namespace DafnyTests { +namespace DafnyDriver.Test { public class TemporaryDirectory : IDisposable { public readonly DirectoryInfo DirInfo; diff --git a/Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs b/Source/DafnyDriver.Test/XUnitExtensions/AssertWithDiff.cs similarity index 100% rename from Source/DafnyTests/XUnitExtensions/AssertWithDiff.cs rename to Source/DafnyDriver.Test/XUnitExtensions/AssertWithDiff.cs diff --git a/Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs similarity index 100% rename from Source/DafnyTests/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs rename to Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs diff --git a/Source/DafnyTests/XUnitExtensions/EnumerableUtils.cs b/Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs similarity index 95% rename from Source/DafnyTests/XUnitExtensions/EnumerableUtils.cs rename to Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs index cacd5efdf3c..8ac8ea01da9 100644 --- a/Source/DafnyTests/XUnitExtensions/EnumerableUtils.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Linq; -namespace DafnyTests { +namespace DafnyDriver.Test { public static class EnumerableUtils { /** * Source: https://docs.microsoft.com/en-us/archive/blogs/ericlippert/computing-a-cartesian-product-with-linq diff --git a/Source/DafnyTests/XUnitExtensions/ForEachAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs similarity index 89% rename from Source/DafnyTests/XUnitExtensions/ForEachAttribute.cs rename to Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs index 4a9a9487e4c..0e68ea90b52 100644 --- a/Source/DafnyTests/XUnitExtensions/ForEachAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace XUnitExtensions { +namespace DafnyDriver.Test.XUnitExtensions { public class ForEachAttribute : Attribute { private readonly Type EnumeratorClass; diff --git a/Source/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/ParallelTheoryAttribute.cs similarity index 84% rename from Source/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs rename to Source/DafnyDriver.Test/XUnitExtensions/ParallelTheoryAttribute.cs index c7299727f86..4bd67701e75 100644 --- a/Source/DafnyTests/XUnitExtensions/ParallelTheoryAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/ParallelTheoryAttribute.cs @@ -3,5 +3,5 @@ using Xunit.Sdk; [AttributeUsage(AttributeTargets.Method)] -[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "DafnyTests")] +[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "DafnyDriver.Test")] public class ParallelTheoryAttribute : TheoryAttribute { } \ No newline at end of file diff --git a/Source/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs b/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs similarity index 100% rename from Source/DafnyTests/XUnitExtensions/TestCaseWithCollection.cs rename to Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs similarity index 83% rename from Source/DafnyTests/XUnitExtensions/YamlDataAttribute.cs rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs index ea57d54d10b..464f337e999 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs @@ -5,7 +5,7 @@ namespace XUnitExtensions { - [DataDiscoverer("XUnitExtensions.YamlDataDiscoverer", "DafnyTests")] + [DataDiscoverer("XUnitExtensions.YamlDataDiscoverer", "DafnyDriver.Test")] public class YamlDataAttribute : DataAttribute { public YamlDataAttribute(bool withParameterNames = true) { diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs similarity index 98% rename from Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs index e48aa8bb246..6f0b92f54fe 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs @@ -4,7 +4,7 @@ using System.IO; using System.Linq; using System.Reflection; -using DafnyTests; +using DafnyDriver.Test; using Xunit.Abstractions; using Xunit.Sdk; using YamlDotNet.Core; @@ -14,7 +14,7 @@ using YamlDotNet.Serialization.ObjectFactories; using YamlDotNet.Serialization.Utilities; -namespace XUnitExtensions { +namespace DafnyDriver.Test.XUnitExtensions { public class YamlDataDiscoverer : IDataDiscoverer { public virtual IParser GetYamlParser(string manifestResourceName, Stream stream) { return new Parser(new StreamReader(stream)); diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs similarity index 96% rename from Source/DafnyTests/XUnitExtensions/YamlDataTests.cs rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs index 14a9ee02dcb..08494d03242 100644 --- a/Source/DafnyTests/XUnitExtensions/YamlDataTests.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs @@ -10,7 +10,7 @@ using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; -namespace DafnyTests.XUnitExtensions { +namespace DafnyDriver.Test.XUnitExtensions { public class YamlDataTests { [Theory] @@ -31,7 +31,7 @@ public void DictionaryTest(Dictionary config) { Assert.Equal(3, config.Count); } - [DataDiscoverer("DafnyTests.XUnitExtensions.CustomDiscoverer", "DafnyTests")] + [DataDiscoverer("DafnyTests.XUnitExtensions.CustomDiscoverer", "DafnyDriver.Test")] public class CustomYamlDataAttribute : YamlDataAttribute { public CustomYamlDataAttribute(bool withParameterNames = true) : base(withParameterNames) { } diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml similarity index 100% rename from Source/DafnyTests/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataTests/CalculatorTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorTest.yml similarity index 100% rename from Source/DafnyTests/XUnitExtensions/YamlDataTests/CalculatorTest.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorTest.yml diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml similarity index 100% rename from Source/DafnyTests/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml diff --git a/Source/DafnyTests/XUnitExtensions/YamlDataTests/DictionaryTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/DictionaryTest.yml similarity index 100% rename from Source/DafnyTests/XUnitExtensions/YamlDataTests/DictionaryTest.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/DictionaryTest.yml diff --git a/Source/LitTestConverter/LitTestConverter.csproj b/Source/LitTestConverter/LitTestConverter.csproj index 89176ef67eb..1548e223922 100644 --- a/Source/LitTestConverter/LitTestConverter.csproj +++ b/Source/LitTestConverter/LitTestConverter.csproj @@ -9,7 +9,7 @@ - + \ No newline at end of file diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index dd494a6f14f..0b3393c878e 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -2,11 +2,11 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using DafnyDriver.Test; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; -using YamlDotNet.Serialization.ObjectGraphTraversalStrategies; -namespace DafnyTests { +namespace LitTestConvertor { public class LitTestConvertor { @@ -15,7 +15,7 @@ public class LitTestConvertor { private const string LIT_DAFNY = "%dafny"; private const string LIT_SERVER = "%server"; - // Fake options to which files are passed to the CLI + // Fake options for which files are passed to the CLI public const string TEST_CONFIG_OTHER_FILES = "otherFiles"; public const string TEST_CONFIG_INCLUDE_THIS_FILE = "includeThisFile"; From 153d830a74d969f7bb93f48d284609abfe18b0c7 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 6 Aug 2021 13:12:38 -0700 Subject: [PATCH 097/192] Missing package reference --- Source/DafnyDriver.Test/DafnyDriver.Test.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj index 082e0f9c9a9..697c0823cf5 100644 --- a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj +++ b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj @@ -15,6 +15,7 @@ + From b1191351f9b8f4c1c38ce581960744e352908868 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 7 Aug 2021 07:38:27 -0700 Subject: [PATCH 098/192] Extract some classes into their own files --- Source/DafnyDriver.Test/DafnyTestCase.cs | 262 ------------------ Source/DafnyDriver.Test/DafnyTestSpec.cs | 176 ++++++++++++ .../DafnyTestYamlDataDiscoverer.cs | 93 +++++++ Source/DafnyDriver.Test/IntegrationTest.cs | 24 ++ .../XUnitExtensions/YamlDataAttribute.cs | 2 + .../XUnitExtensions/YamlDataDiscoverer.cs | 1 - 6 files changed, 295 insertions(+), 263 deletions(-) create mode 100644 Source/DafnyDriver.Test/DafnyTestSpec.cs create mode 100644 Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs create mode 100644 Source/DafnyDriver.Test/IntegrationTest.cs diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index f05d2232c3f..01a6f813cbb 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -18,171 +18,6 @@ namespace DafnyDriver.Test { - public class DafnyTestSpec: IEnumerable { - - private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); - - // TODO-RS: This is an ugly method of locating the project root. - // The proper fix is to run entirely out of the output directory, - // and the projects are at least partially configured to make that possible, - // but it's not quite working yet. - private static string DAFNY_ROOT = - OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; - - public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; - public static readonly string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - - public static readonly string DAFNY_PROJ = Path.Combine(DAFNY_ROOT, "Source/DafnyDriver/DafnyDriver.csproj"); - - // Dafny options with special handling - public const string DAFNY_COMPILE_OPTION = "compile"; - public const string DAFNY_COMPILE_TARGET_OPTION = "compileTarget"; - public const string DAFNY_NO_VERIFY_OPTION = "noVerify"; - - // Absolute file system path to the main Dafny file - [YamlIgnore] - public readonly string SourcePath; - - public Dictionary DafnyArguments = new Dictionary(); - public List OtherFiles = new List(); - - public Dictionary CompileTargetOverrides = new Dictionary(); - - public DafnyTestCase.Expectation Expected; - - public DafnyTestSpec(string sourcePath) { - SourcePath = sourcePath; - } - - private DafnyTestSpec(string sourcePath, Dictionary dafnyArguments, List otherFiles, DafnyTestCase.Expectation expected) { - SourcePath = sourcePath; - DafnyArguments = dafnyArguments; - OtherFiles = otherFiles; - Expected = expected; - } - - public IEnumerator GetEnumerator() { - return ResolveCompile() - .SelectMany(spec => spec.ExpandArguments()) - .Select(spec => spec.ResolveExpected().ToTestCase()) - .GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() { - return GetEnumerator(); - } - - [YamlIgnore] - public int? Compile { - get { - if (DafnyArguments.TryGetValue(DAFNY_COMPILE_OPTION, out var compile)) { - return Int32.Parse((string) compile); - } - return null; - } - set { - if (value != null) { - DafnyArguments[DAFNY_COMPILE_OPTION] = value.ToString(); - } else { - DafnyArguments.Remove(DAFNY_COMPILE_OPTION); - } - } - } - - [YamlIgnore] - public string CompileTarget { - get { - if (DafnyArguments.TryGetValue(DAFNY_COMPILE_TARGET_OPTION, out var compileTarget)) { - return (string) compileTarget; - } - return null; - } - } - - private IEnumerable ResolveCompile() { - if (Compile == null) { - Compile = 3; - } - - if (Compile > 0 && CompileTarget == null) { - // TODO: Include c++ but flag as special case by default? - var compileTargets = new[] {"cs", "java", "go", "js"}; - - var justVerify = new Dictionary(DafnyArguments) { - [DAFNY_COMPILE_OPTION] = "0" - }; - yield return new DafnyTestSpec(SourcePath, justVerify, OtherFiles, DafnyTestCase.Expectation.NO_OUTPUT); - - foreach (var compileTarget in compileTargets) { - var withLanguage = new Dictionary(DafnyArguments) { - [DAFNY_NO_VERIFY_OPTION] = "yes", - [DAFNY_COMPILE_OPTION] = Compile > 2 ? "4" : "2", - [DAFNY_COMPILE_TARGET_OPTION] = compileTarget - }; - var specForLanguage = new DafnyTestSpec(SourcePath, withLanguage, OtherFiles, Expected); - if (CompileTargetOverrides.TryGetValue(compileTarget, out var compileTargetOverride)) { - yield return specForLanguage.ApplyOverride(compileTargetOverride); - } else { - yield return specForLanguage; - } - } - } else { - yield return this; - } - } - - private DafnyTestSpec ApplyOverride(DafnyTestSpec otherSpec) { - var mergedArguments = new Dictionary(DafnyArguments); - foreach (KeyValuePair pair in otherSpec.DafnyArguments) { - mergedArguments[pair.Key] = pair.Value; - } - return new DafnyTestSpec(otherSpec.SourcePath, mergedArguments, OtherFiles.Concat(otherSpec.OtherFiles).ToList(), otherSpec.Expected); - } - - private DafnyTestSpec ResolveExpected() { - if (Expected == null) { - return new DafnyTestSpec(SourcePath, DafnyArguments, OtherFiles, - new DafnyTestCase.Expectation(0, SourcePath + ".expect", null)); - } - return this; - } - - private DafnyTestCase ToTestCase() { - var arguments = new []{ SourcePath } - .Concat(DafnyArguments.Select(ConfigPairToArgument)); - return new DafnyTestCase(arguments, Expected); - } - - private static string ConfigPairToArgument(KeyValuePair pair) { - if (pair.Value.Equals("yes")) { - return String.Format("/{0}", pair.Key); - } else { - return String.Format("/{0}:{1}", pair.Key, pair.Value); - } - } - - private IEnumerable ExpandArguments() { - return DafnyArguments.Select(ExpandValue) - .CartesianProduct() - .Select(e => e.ToDictionary(p => p.Key, p => p.Value)) - .Select(args => new DafnyTestSpec(SourcePath, args, OtherFiles, Expected)); - } - - public static IEnumerable Expand(object obj) { - if (obj is ForEachArgumentList forEachArgumentList) { - return forEachArgumentList; - } else { - return new[] {(string)obj}; - } - } - - private static IEnumerable> ExpandValue(KeyValuePair pair) { - return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); - } - } - - public class ForEachArgumentList : List { } - // TODO-RS: This is really close to a generic CLI test case - // can we move all the dafny-specific stuff to DafnyTestSpec instead? public class DafnyTestCase: IXunitSerializable { @@ -340,101 +175,4 @@ public override string ToString() { return String.Join(" ", Arguments) + " => " + Expected; } } - - - public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { - - private const string DEFAULT_CONFIG = -@"!dafnyTestSpec -dafnyArguments: {} -"; - - private static readonly IFileProvider ManifestFileProvider = new ManifestEmbeddedFileProvider( - Assembly.GetExecutingAssembly()); - private static readonly Dictionary PathsForResourceNames = GetPathsForResourceNames( - "DafnyDriver.Test", ManifestFileProvider, "DafnyTests"); - - private static Dictionary GetPathsForResourceNames(string assemblyName, IFileProvider fileProvider, string path = null) { - return fileProvider.GetDirectoryContents(path).SelectMany(file => { - var childName = path == null ? file.Name : path + "/" + file.Name; - if (file.IsDirectory) { - return GetPathsForResourceNames(assemblyName, fileProvider, childName); - } else { - var result = new Dictionary(); - result[ResourceNameForFilePath(assemblyName, childName)] = childName; - return result; - } - }).ToDictionary(pair => pair.Key, pair => pair.Value); - } - - private static string ResourceNameForFilePath(string assemblyName, string filePath) { - return assemblyName + "." + filePath.Replace("/", ".").Replace("+", "_"); - } - - private static DafnyTestSpec SpecForResourceName(string manifestResourceName) { - string filePath = PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); - return new DafnyTestSpec(filePath); - } - - public static string GetTestCaseConfigYaml(string fullText) { - var commentStart = fullText.IndexOf("/*"); - if (commentStart >= 0) { - var commentEnd = fullText.IndexOf("*/", commentStart + 2); - if (commentEnd >= 0) { - var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); - if (commentContent.StartsWith("---")) { - return commentContent; - } - } - } - - return null; - } - - public override IParser GetYamlParser(string manifestResourceName, Stream stream) { - if (!manifestResourceName.EndsWith(".dfy")) { - return null; - } - - string content = GetTestCaseConfigYaml(new StreamReader(stream).ReadToEnd()) ?? DEFAULT_CONFIG; - - return new Parser(new StringReader(content)); - } - - public override IDeserializer GetDeserializer(string manifestResourceName) { - var defaultObjectFactory = new DefaultObjectFactory(); - var customObjectFactory = new LambdaObjectFactory(type => - type == typeof(DafnyTestSpec) ? SpecForResourceName(manifestResourceName) : defaultObjectFactory.Create(type)); - - return new DeserializerBuilder() - .WithNamingConvention(CamelCaseNamingConvention.Instance) - .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) - .WithTagMapping("!foreach", typeof(ForEachArgumentList)) - .WithObjectFactory(customObjectFactory) - .Build(); - } - } - - [DataDiscoverer("DafnyDriver.Test.DafnyTestYamlDataDiscoverer", "DafnyDriver.Test")] - public class DafnyTestDataAttribute : YamlDataAttribute { - public DafnyTestDataAttribute(bool withParameterNames) : base(withParameterNames) { - } - } - - public class DafnyTests { - - [Fact] - public static void MetaTest() { - var discoverer = new DafnyTestYamlDataDiscoverer(); - var testMethod = typeof(DafnyTests).GetMethod(nameof(Test)); - var testData = discoverer.GetData(testMethod, false).ToList(); - Assert.True(testData.Any()); - } - - [ParallelTheory] - [DafnyTestData(false)] - public static void Test(DafnyTestCase testCase) { - testCase.Run(); - } - } } diff --git a/Source/DafnyDriver.Test/DafnyTestSpec.cs b/Source/DafnyDriver.Test/DafnyTestSpec.cs new file mode 100644 index 00000000000..a7505601541 --- /dev/null +++ b/Source/DafnyDriver.Test/DafnyTestSpec.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; +using YamlDotNet.Serialization.ObjectFactories; + +namespace DafnyDriver.Test +{ + public class ForEachArgumentList : List { } + + public class DafnyTestSpec: IEnumerable { + + private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); + + // TODO-RS: This is an ugly method of locating the project root. + // The proper fix is to run entirely out of the output directory, + // and the projects are at least partially configured to make that possible, + // but it's not quite working yet. + private static string DAFNY_ROOT = + OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; + + public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; + public static readonly string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; + + public static readonly string DAFNY_PROJ = Path.Combine(DAFNY_ROOT, "Source/DafnyDriver/DafnyDriver.csproj"); + + // Dafny options with special handling + public const string DAFNY_COMPILE_OPTION = "compile"; + public const string DAFNY_COMPILE_TARGET_OPTION = "compileTarget"; + public const string DAFNY_NO_VERIFY_OPTION = "noVerify"; + + // Absolute file system path to the main Dafny file + [YamlIgnore] + public readonly string SourcePath; + + public Dictionary DafnyArguments = new Dictionary(); + public List OtherFiles = new List(); + + public Dictionary CompileTargetOverrides = new Dictionary(); + + public DafnyTestCase.Expectation Expected; + + public DafnyTestSpec(string sourcePath) { + SourcePath = sourcePath; + } + + private DafnyTestSpec(string sourcePath, Dictionary dafnyArguments, List otherFiles, DafnyTestCase.Expectation expected) { + SourcePath = sourcePath; + DafnyArguments = dafnyArguments; + OtherFiles = otherFiles; + Expected = expected; + } + + public IEnumerator GetEnumerator() { + return ResolveCompile() + .SelectMany(spec => spec.ExpandArguments()) + .Select(spec => spec.ResolveExpected().ToTestCase()) + .GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() { + return GetEnumerator(); + } + + [YamlIgnore] + public int? Compile { + get { + if (DafnyArguments.TryGetValue(DAFNY_COMPILE_OPTION, out var compile)) { + return Int32.Parse((string) compile); + } + return null; + } + set { + if (value != null) { + DafnyArguments[DAFNY_COMPILE_OPTION] = value.ToString(); + } else { + DafnyArguments.Remove(DAFNY_COMPILE_OPTION); + } + } + } + + [YamlIgnore] + public string CompileTarget { + get { + if (DafnyArguments.TryGetValue(DAFNY_COMPILE_TARGET_OPTION, out var compileTarget)) { + return (string) compileTarget; + } + return null; + } + } + + private IEnumerable ResolveCompile() { + if (Compile == null) { + Compile = 3; + } + + if (Compile > 0 && CompileTarget == null) { + // TODO: Include c++ but flag as special case by default? + var compileTargets = new[] {"cs", "java", "go", "js"}; + + var justVerify = new Dictionary(DafnyArguments) { + [DAFNY_COMPILE_OPTION] = "0" + }; + yield return new DafnyTestSpec(SourcePath, justVerify, OtherFiles, DafnyTestCase.Expectation.NO_OUTPUT); + + foreach (var compileTarget in compileTargets) { + var withLanguage = new Dictionary(DafnyArguments) { + [DAFNY_NO_VERIFY_OPTION] = "yes", + [DAFNY_COMPILE_OPTION] = Compile > 2 ? "4" : "2", + [DAFNY_COMPILE_TARGET_OPTION] = compileTarget + }; + var specForLanguage = new DafnyTestSpec(SourcePath, withLanguage, OtherFiles, Expected); + if (CompileTargetOverrides.TryGetValue(compileTarget, out var compileTargetOverride)) { + yield return specForLanguage.ApplyOverride(compileTargetOverride); + } else { + yield return specForLanguage; + } + } + } else { + yield return this; + } + } + + private DafnyTestSpec ApplyOverride(DafnyTestSpec otherSpec) { + var mergedArguments = new Dictionary(DafnyArguments); + foreach (KeyValuePair pair in otherSpec.DafnyArguments) { + mergedArguments[pair.Key] = pair.Value; + } + return new DafnyTestSpec(otherSpec.SourcePath, mergedArguments, OtherFiles.Concat(otherSpec.OtherFiles).ToList(), otherSpec.Expected); + } + + private DafnyTestSpec ResolveExpected() { + if (Expected == null) { + return new DafnyTestSpec(SourcePath, DafnyArguments, OtherFiles, + new DafnyTestCase.Expectation(0, SourcePath + ".expect", null)); + } + return this; + } + + private DafnyTestCase ToTestCase() { + var arguments = new []{ SourcePath } + .Concat(DafnyArguments.Select(ConfigPairToArgument)); + return new DafnyTestCase(arguments, Expected); + } + + private static string ConfigPairToArgument(KeyValuePair pair) { + if (pair.Value.Equals("yes")) { + return String.Format("/{0}", pair.Key); + } else { + return String.Format("/{0}:{1}", pair.Key, pair.Value); + } + } + + private IEnumerable ExpandArguments() { + return DafnyArguments.Select(ExpandValue) + .CartesianProduct() + .Select(e => e.ToDictionary(p => p.Key, p => p.Value)) + .Select(args => new DafnyTestSpec(SourcePath, args, OtherFiles, Expected)); + } + + public static IEnumerable Expand(object obj) { + if (obj is ForEachArgumentList forEachArgumentList) { + return forEachArgumentList; + } else { + return new[] {(string)obj}; + } + } + + private static IEnumerable> ExpandValue(KeyValuePair pair) { + return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); + } + } +} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs b/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs new file mode 100644 index 00000000000..61f914e7fca --- /dev/null +++ b/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs @@ -0,0 +1,93 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using DafnyDriver.Test.XUnitExtensions; +using Microsoft.Extensions.FileProviders; +using Xunit.Sdk; +using XUnitExtensions; +using YamlDotNet.Core; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; +using YamlDotNet.Serialization.ObjectFactories; + +namespace DafnyDriver.Test { + public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { + + private const string DEFAULT_CONFIG = +@"!dafnyTestSpec +dafnyArguments: {} +"; + + private static readonly IFileProvider ManifestFileProvider = new ManifestEmbeddedFileProvider( + Assembly.GetExecutingAssembly()); + private static readonly Dictionary PathsForResourceNames = GetPathsForResourceNames( + "DafnyDriver.Test", ManifestFileProvider, "DafnyTests"); + + private static Dictionary GetPathsForResourceNames(string assemblyName, IFileProvider fileProvider, string path = null) { + return fileProvider.GetDirectoryContents(path).SelectMany(file => { + var childName = path == null ? file.Name : path + "/" + file.Name; + if (file.IsDirectory) { + return GetPathsForResourceNames(assemblyName, fileProvider, childName); + } else { + var result = new Dictionary(); + result[ResourceNameForFilePath(assemblyName, childName)] = childName; + return result; + } + }).ToDictionary(pair => pair.Key, pair => pair.Value); + } + + private static string ResourceNameForFilePath(string assemblyName, string filePath) { + return assemblyName + "." + filePath.Replace("/", ".").Replace("+", "_"); + } + + private static DafnyTestSpec SpecForResourceName(string manifestResourceName) { + string filePath = PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); + return new DafnyTestSpec(filePath); + } + + public static string GetTestCaseConfigYaml(string fullText) { + var commentStart = fullText.IndexOf("/*"); + if (commentStart >= 0) { + var commentEnd = fullText.IndexOf("*/", commentStart + 2); + if (commentEnd >= 0) { + var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); + if (commentContent.StartsWith("---")) { + return commentContent; + } + } + } + + return null; + } + + public override IParser GetYamlParser(string manifestResourceName, Stream stream) { + if (!manifestResourceName.EndsWith(".dfy")) { + return null; + } + + string content = GetTestCaseConfigYaml(new StreamReader(stream).ReadToEnd()) ?? DEFAULT_CONFIG; + + return new Parser(new StringReader(content)); + } + + public override IDeserializer GetDeserializer(string manifestResourceName) { + var defaultObjectFactory = new DefaultObjectFactory(); + var customObjectFactory = new LambdaObjectFactory(type => + type == typeof(DafnyTestSpec) ? SpecForResourceName(manifestResourceName) : defaultObjectFactory.Create(type)); + + return new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) + .WithTagMapping("!foreach", typeof(ForEachArgumentList)) + .WithObjectFactory(customObjectFactory) + .Build(); + } + } +} + +[DataDiscoverer("DafnyDriver.Test.DafnyTestYamlDataDiscoverer", "DafnyDriver.Test")] +public class DafnyTestDataAttribute : YamlDataAttribute { + public DafnyTestDataAttribute(bool withParameterNames) : base(withParameterNames) { + } +} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/IntegrationTest.cs b/Source/DafnyDriver.Test/IntegrationTest.cs new file mode 100644 index 00000000000..f67937a12b3 --- /dev/null +++ b/Source/DafnyDriver.Test/IntegrationTest.cs @@ -0,0 +1,24 @@ +using System.Linq; +using Xunit; +using Xunit.Sdk; +using XUnitExtensions; + +namespace DafnyDriver.Test { + + public class DafnyTests { + + [Fact] + public static void MetaTest() { + var discoverer = new DafnyTestYamlDataDiscoverer(); + var testMethod = typeof(DafnyTests).GetMethod(nameof(Test)); + var testData = discoverer.GetData(testMethod, false).ToList(); + Assert.True(testData.Any()); + } + + [ParallelTheory] + [DafnyTestData(false)] + public static void Test(DafnyTestCase testCase) { + testCase.Run(); + } + } +} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs index 464f337e999..60be24aa21e 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs @@ -12,6 +12,8 @@ public YamlDataAttribute(bool withParameterNames = true) { } public override IEnumerable GetData(MethodInfo testMethod) { + // This method is not used - the YamlDataDiscoverer has all of the actual logic instead + // because it exposes some methods for subclasses to customize such as GetYamlParser. throw new NotImplementedException(); } } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs index 6f0b92f54fe..cbd36ba3738 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs @@ -4,7 +4,6 @@ using System.IO; using System.Linq; using System.Reflection; -using DafnyDriver.Test; using Xunit.Abstractions; using Xunit.Sdk; using YamlDotNet.Core; From 147b481931626004d1cb538352d8a86a73a19759 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 8 Aug 2021 09:49:16 -0700 Subject: [PATCH 099/192] Factor out (mostly) generic CLITestCase --- Source/DafnyDriver.Test/DafnyTestCase.cs | 209 +++++------------- Source/DafnyDriver.Test/DafnyTestSpec.cs | 33 +-- Source/DafnyDriver.Test/IntegrationTest.cs | 9 +- Source/DafnyDriver.Test/ProcessResult.cs | 14 -- Source/DafnyDriver.Test/TemporaryDirectory.cs | 3 +- .../XUnitExtensions/CLITestCase.cs | 167 ++++++++++++++ Source/LitTestConverter/LitTestConvertor.cs | 5 +- 7 files changed, 242 insertions(+), 198 deletions(-) delete mode 100644 Source/DafnyDriver.Test/ProcessResult.cs create mode 100644 Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index 01a6f813cbb..6ff9b32eb74 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -1,178 +1,75 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Diagnostics; -using System.IO; using System.Linq; -using System.Reflection; using DafnyDriver.Test.XUnitExtensions; -using Microsoft.Extensions.FileProviders; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; -using XUnitExtensions; -using YamlDotNet.Core; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; -using YamlDotNet.Serialization.ObjectFactories; namespace DafnyDriver.Test { - // TODO-RS: This is really close to a generic CLI test case - - // can we move all the dafny-specific stuff to DafnyTestSpec instead? - public class DafnyTestCase: IXunitSerializable { - - public string[] Arguments; - - public class Expectation : IXunitSerializable { - // May be null if the test doesn't need to check the output - public string OutputFile; - public int ExitCode = 0; - - public string SpecialCaseReason; - - public static Expectation NO_OUTPUT = new Expectation(0, null, null); - - public Expectation(int exitCode, string outputFile, string specialCaseReason) { - ExitCode = exitCode; - OutputFile = outputFile; - SpecialCaseReason = specialCaseReason; - } - - public Expectation() { - - } - - public void Deserialize(IXunitSerializationInfo info) { - OutputFile = info.GetValue(nameof(OutputFile)); - ExitCode = info.GetValue(nameof(ExitCode)); - SpecialCaseReason = info.GetValue(nameof(SpecialCaseReason)); - } - - public void Serialize(IXunitSerializationInfo info) { - info.AddValue(nameof(OutputFile), OutputFile); - info.AddValue(nameof(ExitCode), ExitCode); - info.AddValue(nameof(SpecialCaseReason), SpecialCaseReason); + /** + * Specialization of CLITestCase that mainly exists to support a much more + * consice definition of ToString(). + */ + public class DafnyTestCase : CLITestCase { + + private static readonly Dictionary defaultDafnyOptions = new() { + ["countVerificationErrors"] = "0", + + // We do not want absolute or relative paths in error messages, just the basename of the file + ["useBaseNameForFileName"] = "yes", + + // We do not want output such as "Compiled program written to Foo.cs" + // from the compilers, since that changes with the target language + ["compileVerbose"] = "0" + }; + + private static IEnumerable OptionsToFullArguments(string sourcePath, + Dictionary dafnyOptions, List otherFiles) + { + Dictionary optionsWithDefaults = new(defaultDafnyOptions); + foreach (var (key, value) in dafnyOptions) { + optionsWithDefaults.Add(key, value); } - public Expectation Adjust() { - return new Expectation(ExitCode, OutputFile == null ? null : "comp/" + OutputFile, SpecialCaseReason); - } - - public override string ToString() { - string result; - if (ExitCode != 0) { - result = String.Format("", ExitCode); - } else if (OutputFile != null) { - result = OutputFile; - } else { - result = ""; - } + return new[] { + "run --no-build --project ", + DafnyTestSpec.DAFNY_PROJ, + " --" + }.Concat(OptionsToArguments(sourcePath, optionsWithDefaults, otherFiles)); + } - if (SpecialCaseReason != null) { - result += " (special case)"; - } - return result; + private static IEnumerable OptionsToArguments(string sourcePath, Dictionary dafnyOptions, List otherFiles) { + return new []{ sourcePath } + .Concat(otherFiles) + .Concat(dafnyOptions.Select(ConfigPairToArgument)); + } + + private static string ConfigPairToArgument(KeyValuePair pair) { + if (pair.Value.Equals("yes")) { + return $"/{pair.Key}"; } + return $"/{pair.Key}:{pair.Value}"; } - public Expectation Expected; - - public DafnyTestCase(IEnumerable arguments, Expectation expected) { + public readonly string SourcePath; + + public Dictionary DafnyOptions = new(); + public List OtherFiles = new(); + + public DafnyTestCase(string sourcePath, Dictionary dafnyOptions, List otherFiles, Expectation expected) + : base("dotnet", OptionsToFullArguments(sourcePath, dafnyOptions, otherFiles), expected) + { + SourcePath = sourcePath; + DafnyOptions = dafnyOptions; + OtherFiles = otherFiles; Expected = expected; - Arguments = arguments.ToArray(); } - + public DafnyTestCase() { } - - public void Serialize(IXunitSerializationInfo info) { - info.AddValue(nameof(Arguments), Arguments); - info.AddValue(nameof(Expected), Expected); - } - public void Deserialize(IXunitSerializationInfo info) { - Arguments = info.GetValue(nameof(Arguments)); - Expected = info.GetValue(nameof(Expected)); - } - - public static ProcessResult RunDafny(IEnumerable arguments) { - // TODO-RS: Let these be overridden - List dafnyArguments = new List { - "/countVerificationErrors:0", - - // We do not want absolute or relative paths in error messages, just the basename of the file - "/useBaseNameForFileName", - - // We do not want output such as "Compiled program written to Foo.cs" - // from the compilers, since that changes with the target language - "/compileVerbose:0" - }; - dafnyArguments.AddRange(arguments); - - using (Process dafnyProcess = new Process()) { - dafnyProcess.StartInfo.FileName = "dotnet"; - dafnyProcess.StartInfo.Arguments = "run --no-build --project "; - dafnyProcess.StartInfo.Arguments += DafnyTestSpec.DAFNY_PROJ; - dafnyProcess.StartInfo.Arguments += " --"; - foreach (var argument in dafnyArguments) { - dafnyProcess.StartInfo.Arguments += " " + argument; - } - - dafnyProcess.StartInfo.UseShellExecute = false; - dafnyProcess.StartInfo.RedirectStandardOutput = true; - dafnyProcess.StartInfo.RedirectStandardError = true; - dafnyProcess.StartInfo.CreateNoWindow = true; - // Necessary for JS to find bignumber.js - dafnyProcess.StartInfo.WorkingDirectory = DafnyTestSpec.TEST_ROOT; - - // Only preserve specific whitelisted environment variables - dafnyProcess.StartInfo.EnvironmentVariables.Clear(); - dafnyProcess.StartInfo.EnvironmentVariables.Add("PATH", Environment.GetEnvironmentVariable("PATH")); - // Go requires this or GOCACHE - dafnyProcess.StartInfo.EnvironmentVariables.Add("HOME", Environment.GetEnvironmentVariable("HOME")); - - dafnyProcess.Start(); - string output = dafnyProcess.StandardOutput.ReadToEnd(); - string error = dafnyProcess.StandardError.ReadToEnd(); - dafnyProcess.WaitForExit(); - return new ProcessResult(dafnyProcess.ExitCode, output, error); - } - } - - public void Run() { - ProcessResult dafnyResult; - if (Arguments.Any(arg => arg.StartsWith("/out"))) { - dafnyResult = RunDafny(Arguments); - } else { - // Note that the temporary directory has to be an ancestor of Test - // or else Javascript won't be able to locate bignumber.js :( - using (var tempDir = new TemporaryDirectory(DafnyTestSpec.OUTPUT_DIR)) { - // Add an extra component to the path to keep the files created inside the - // temporary directory, since some compilers will - // interpret the path as a single file basename rather than a directory. - var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; - var dafnyArguments = new []{ outArgument }.Concat(Arguments); - dafnyResult = RunDafny(dafnyArguments); - } - } - - if (dafnyResult.ExitCode != Expected.ExitCode) { - throw new AssertActualExpectedException(Expected.ExitCode, dafnyResult.ExitCode, - String.Format("Expected exit code to be {0} but was {1}. Standard error output:\n{2}", - Expected.ExitCode, dafnyResult.ExitCode, dafnyResult.StandardError)); - } - if (Expected.OutputFile != null) { - var expectedOutput = File.ReadAllText(Path.Combine(DafnyTestSpec.TEST_ROOT, Expected.OutputFile)); - AssertWithDiff.Equal(expectedOutput, dafnyResult.StandardOutput); - } - - Skip.If(Expected.SpecialCaseReason != null, Expected.SpecialCaseReason); - } - public override string ToString() { - return String.Join(" ", Arguments) + " => " + Expected; + return String.Join(" ", OptionsToArguments(SourcePath, DafnyOptions, OtherFiles)) + " => " + Expected; } } -} +} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/DafnyTestSpec.cs b/Source/DafnyDriver.Test/DafnyTestSpec.cs index a7505601541..18cd5a86d3d 100644 --- a/Source/DafnyDriver.Test/DafnyTestSpec.cs +++ b/Source/DafnyDriver.Test/DafnyTestSpec.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using DafnyDriver.Test.XUnitExtensions; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; using YamlDotNet.Serialization.ObjectFactories; @@ -13,7 +14,7 @@ public class ForEachArgumentList : List { } public class DafnyTestSpec: IEnumerable { - private static DirectoryInfo OUTPUT_ROOT = new DirectoryInfo(Directory.GetCurrentDirectory()); + private static readonly DirectoryInfo OUTPUT_ROOT = new(Directory.GetCurrentDirectory()); // TODO-RS: This is an ugly method of locating the project root. // The proper fix is to run entirely out of the output directory, @@ -36,18 +37,18 @@ public class DafnyTestSpec: IEnumerable { [YamlIgnore] public readonly string SourcePath; - public Dictionary DafnyArguments = new Dictionary(); - public List OtherFiles = new List(); + public Dictionary DafnyArguments = new(); + public List OtherFiles = new(); - public Dictionary CompileTargetOverrides = new Dictionary(); + public Dictionary CompileTargetOverrides = new(); - public DafnyTestCase.Expectation Expected; + public CLITestCase.Expectation Expected; public DafnyTestSpec(string sourcePath) { SourcePath = sourcePath; } - private DafnyTestSpec(string sourcePath, Dictionary dafnyArguments, List otherFiles, DafnyTestCase.Expectation expected) { + private DafnyTestSpec(string sourcePath, Dictionary dafnyArguments, List otherFiles, CLITestCase.Expectation expected) { SourcePath = sourcePath; DafnyArguments = dafnyArguments; OtherFiles = otherFiles; @@ -104,7 +105,7 @@ private IEnumerable ResolveCompile() { var justVerify = new Dictionary(DafnyArguments) { [DAFNY_COMPILE_OPTION] = "0" }; - yield return new DafnyTestSpec(SourcePath, justVerify, OtherFiles, DafnyTestCase.Expectation.NO_OUTPUT); + yield return new DafnyTestSpec(SourcePath, justVerify, OtherFiles, CLITestCase.Expectation.NO_OUTPUT); foreach (var compileTarget in compileTargets) { var withLanguage = new Dictionary(DafnyArguments) { @@ -126,8 +127,8 @@ private IEnumerable ResolveCompile() { private DafnyTestSpec ApplyOverride(DafnyTestSpec otherSpec) { var mergedArguments = new Dictionary(DafnyArguments); - foreach (KeyValuePair pair in otherSpec.DafnyArguments) { - mergedArguments[pair.Key] = pair.Value; + foreach (var (key, value) in otherSpec.DafnyArguments) { + mergedArguments[key] = value; } return new DafnyTestSpec(otherSpec.SourcePath, mergedArguments, OtherFiles.Concat(otherSpec.OtherFiles).ToList(), otherSpec.Expected); } @@ -135,25 +136,15 @@ private DafnyTestSpec ApplyOverride(DafnyTestSpec otherSpec) { private DafnyTestSpec ResolveExpected() { if (Expected == null) { return new DafnyTestSpec(SourcePath, DafnyArguments, OtherFiles, - new DafnyTestCase.Expectation(0, SourcePath + ".expect", null)); + new CLITestCase.Expectation(0, SourcePath + ".expect", null)); } return this; } private DafnyTestCase ToTestCase() { - var arguments = new []{ SourcePath } - .Concat(DafnyArguments.Select(ConfigPairToArgument)); - return new DafnyTestCase(arguments, Expected); + return new DafnyTestCase(SourcePath, DafnyArguments, OtherFiles, Expected); } - private static string ConfigPairToArgument(KeyValuePair pair) { - if (pair.Value.Equals("yes")) { - return String.Format("/{0}", pair.Key); - } else { - return String.Format("/{0}:{1}", pair.Key, pair.Value); - } - } - private IEnumerable ExpandArguments() { return DafnyArguments.Select(ExpandValue) .CartesianProduct() diff --git a/Source/DafnyDriver.Test/IntegrationTest.cs b/Source/DafnyDriver.Test/IntegrationTest.cs index f67937a12b3..6400d192513 100644 --- a/Source/DafnyDriver.Test/IntegrationTest.cs +++ b/Source/DafnyDriver.Test/IntegrationTest.cs @@ -1,14 +1,15 @@ using System.Linq; +using DafnyDriver.Test.XUnitExtensions; using Xunit; -using Xunit.Sdk; -using XUnitExtensions; namespace DafnyDriver.Test { public class DafnyTests { [Fact] - public static void MetaTest() { + public static void DafnyTestDataDiscovererDiscoversAtLeastOneTest() { + // This test is much easier to debug than the main parameterized test + // if the discoverer is not working correctly var discoverer = new DafnyTestYamlDataDiscoverer(); var testMethod = typeof(DafnyTests).GetMethod(nameof(Test)); var testData = discoverer.GetData(testMethod, false).ToList(); @@ -17,7 +18,7 @@ public static void MetaTest() { [ParallelTheory] [DafnyTestData(false)] - public static void Test(DafnyTestCase testCase) { + public static void Test(CLITestCase testCase) { testCase.Run(); } } diff --git a/Source/DafnyDriver.Test/ProcessResult.cs b/Source/DafnyDriver.Test/ProcessResult.cs deleted file mode 100644 index e9966ba048f..00000000000 --- a/Source/DafnyDriver.Test/ProcessResult.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace DafnyDriver.Test { - public class ProcessResult { - - public int ExitCode; - public string StandardOutput; - public string StandardError; - - public ProcessResult(int exitCode, string standardOutput, string standardError) { - ExitCode = exitCode; - StandardOutput = standardOutput; - StandardError = standardError; - } - } -} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/TemporaryDirectory.cs b/Source/DafnyDriver.Test/TemporaryDirectory.cs index a73993b0d78..943817c81ae 100644 --- a/Source/DafnyDriver.Test/TemporaryDirectory.cs +++ b/Source/DafnyDriver.Test/TemporaryDirectory.cs @@ -17,6 +17,7 @@ public TemporaryDirectory(string parent, string prefix = "") { public void Dispose() { Dispose(true); + GC.SuppressFinalize(this); } ~TemporaryDirectory() { @@ -27,7 +28,7 @@ protected virtual void Dispose(bool disposing) { SafeDelete(); } - public void SafeDelete() { + private void SafeDelete() { try { DirInfo.Delete(true); } catch { diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs new file mode 100644 index 00000000000..9ada976b844 --- /dev/null +++ b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs @@ -0,0 +1,167 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; +using XUnitExtensions; + +namespace DafnyDriver.Test.XUnitExtensions { + + public class CLITestCase: IXunitSerializable + { + public string CLIPath; + public string[] Arguments; + + public class Expectation : IXunitSerializable { + // May be null if the test doesn't need to check the output + public string OutputFile; + public int ExitCode = 0; + + public string SpecialCaseReason; + + public static Expectation NO_OUTPUT = new Expectation(0, null, null); + + public Expectation(int exitCode, string outputFile, string specialCaseReason) { + ExitCode = exitCode; + OutputFile = outputFile; + SpecialCaseReason = specialCaseReason; + } + + public Expectation() { + + } + + public void Deserialize(IXunitSerializationInfo info) { + OutputFile = info.GetValue(nameof(OutputFile)); + ExitCode = info.GetValue(nameof(ExitCode)); + SpecialCaseReason = info.GetValue(nameof(SpecialCaseReason)); + } + + public void Serialize(IXunitSerializationInfo info) { + info.AddValue(nameof(OutputFile), OutputFile); + info.AddValue(nameof(ExitCode), ExitCode); + info.AddValue(nameof(SpecialCaseReason), SpecialCaseReason); + } + + public Expectation Adjust() { + return new Expectation(ExitCode, OutputFile == null ? null : "comp/" + OutputFile, SpecialCaseReason); + } + + public override string ToString() { + string result; + if (ExitCode != 0) { + result = String.Format("", ExitCode); + } else if (OutputFile != null) { + result = OutputFile; + } else { + result = ""; + } + + if (SpecialCaseReason != null) { + result += " (special case)"; + } + return result; + } + } + + public class CLIResult { + + public int ExitCode; + public string StandardOutput; + public string StandardError; + + public CLIResult(int exitCode, string standardOutput, string standardError) { + ExitCode = exitCode; + StandardOutput = standardOutput; + StandardError = standardError; + } + } + + public Expectation Expected; + + public CLITestCase(string cliPath, IEnumerable arguments, Expectation expected) { + CLIPath = cliPath; + Arguments = arguments.ToArray(); + Expected = expected; + } + + public CLITestCase() { + + } + + public void Serialize(IXunitSerializationInfo info) { + info.AddValue(nameof(Arguments), Arguments); + info.AddValue(nameof(Expected), Expected); + } + + public void Deserialize(IXunitSerializationInfo info) { + Arguments = info.GetValue(nameof(Arguments)); + Expected = info.GetValue(nameof(Expected)); + } + + public CLIResult RunDafny(IEnumerable arguments) { + using (Process process = new Process()) { + process.StartInfo.FileName = CLIPath; + foreach (var argument in arguments) { + process.StartInfo.Arguments += " " + argument; + } + + process.StartInfo.UseShellExecute = false; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.CreateNoWindow = true; + // Necessary for JS to find bignumber.js + process.StartInfo.WorkingDirectory = DafnyTestSpec.TEST_ROOT; + + // Only preserve specific whitelisted environment variables + process.StartInfo.EnvironmentVariables.Clear(); + process.StartInfo.EnvironmentVariables.Add("PATH", Environment.GetEnvironmentVariable("PATH")); + // Go requires this or GOCACHE + process.StartInfo.EnvironmentVariables.Add("HOME", Environment.GetEnvironmentVariable("HOME")); + + process.Start(); + string output = process.StandardOutput.ReadToEnd(); + string error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + return new CLIResult(process.ExitCode, output, error); + } + } + + public void Run() { + CLIResult result; + if (Arguments.Any(arg => arg.StartsWith("/out"))) { + result = RunDafny(Arguments); + } else { + // Note that the temporary directory has to be an ancestor of Test + // or else Javascript won't be able to locate bignumber.js :( + using var tempDir = new TemporaryDirectory(DafnyTestSpec.OUTPUT_DIR); + + // Add an extra component to the path to keep the files created inside the + // temporary directory, since some compilers will + // interpret the path as a single file basename rather than a directory. + var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; + var dafnyArguments = Arguments.Concat(new []{ outArgument }); + result = RunDafny(dafnyArguments); + } + + if (result.ExitCode != Expected.ExitCode) { + throw new AssertActualExpectedException(Expected.ExitCode, result.ExitCode, + String.Format("Expected exit code to be {0} but was {1}. Standard error output:\n{2}", + Expected.ExitCode, result.ExitCode, result.StandardError)); + } + if (Expected.OutputFile != null) { + var expectedOutput = File.ReadAllText(Path.Combine(DafnyTestSpec.TEST_ROOT, Expected.OutputFile)); + AssertWithDiff.Equal(expectedOutput, result.StandardOutput); + } + + Skip.If(Expected.SpecialCaseReason != null, Expected.SpecialCaseReason); + } + + public override string ToString() { + return String.Join(" ", Arguments) + " => " + Expected; + } + } +} diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 0b3393c878e..9e77ae3da18 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using DafnyDriver.Test; +using DafnyDriver.Test.XUnitExtensions; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; @@ -39,7 +40,7 @@ public void ConvertLitTest(string filePath) { IEnumerable testContent; if (filePath.Contains("/Inputs/")) { - testSpec = Enumerable.Empty(); + testSpec = Enumerable.Empty(); testContent = File.ReadAllLines(filePath); } else { @@ -83,7 +84,7 @@ public void ConvertLitTest(string filePath) { private object ConvertLitCommands(string filePath, List litCommands) { if (litCommands.Count == 1 && litCommands.Single().StartsWith("echo")) { // This is an idiom for Dafny files used elsewhere - return Enumerable.Empty(); + return Enumerable.Empty(); } if (!litCommands[^1].Equals("%diff \"%s.expect\" \"%t\"")) { From d0f3502c5070e077707b9c960bcf65847cf5b070 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 9 Aug 2021 12:40:53 -0700 Subject: [PATCH 100/192] Add LitTestConvertor.Test package --- Source/Dafny.sln | 26 +++++++++++++++++++ .../ConvertingFromLitToXunit.cs | 14 ++++++++++ .../LitTestConvertor.Test.csproj | 26 +++++++++++++++++++ .../TestFiles/HelloWorldLitTest.dfy | 13 ++++++++++ 4 files changed, 79 insertions(+) create mode 100644 Source/LitTestConvertor.Test/ConvertingFromLitToXunit.cs create mode 100644 Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj create mode 100644 Source/LitTestConvertor.Test/TestFiles/HelloWorldLitTest.dfy diff --git a/Source/Dafny.sln b/Source/Dafny.sln index 40932c390f9..fd7250e6e31 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -24,6 +24,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DafnyLanguageServer", "Dafn EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DafnyLanguageServer.Test", "DafnyLanguageServer.Test\DafnyLanguageServer.Test.csproj", "{320C02A3-D3E6-4AC7-A7E2-170AF86A6EEC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LitTestConvertor.Test", "LitTestConvertor.Test\LitTestConvertor.Test.csproj", "{A4BC2C77-3A48-4917-AA22-41978DFFE24E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Checked|.NET = Checked|.NET @@ -232,6 +234,30 @@ Global {320C02A3-D3E6-4AC7-A7E2-170AF86A6EEC}.Release|Mixed Platforms.Build.0 = Release|Any CPU {320C02A3-D3E6-4AC7-A7E2-170AF86A6EEC}.Release|x86.ActiveCfg = Release|Any CPU {320C02A3-D3E6-4AC7-A7E2-170AF86A6EEC}.Release|x86.Build.0 = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|.NET.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|.NET.Build.0 = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|Any CPU.Build.0 = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|Mixed Platforms.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|Mixed Platforms.Build.0 = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|x86.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|x86.Build.0 = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|.NET.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|.NET.Build.0 = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|x86.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|x86.Build.0 = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|.NET.ActiveCfg = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|.NET.Build.0 = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|Any CPU.Build.0 = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|x86.ActiveCfg = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Source/LitTestConvertor.Test/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/ConvertingFromLitToXunit.cs new file mode 100644 index 00000000000..68f9eace538 --- /dev/null +++ b/Source/LitTestConvertor.Test/ConvertingFromLitToXunit.cs @@ -0,0 +1,14 @@ +using System; +using Xunit; + +namespace LitTestConvertor.Test +{ + public class ConvertingFromLitToXunit + { + [Fact] + public void HelloWorld() { + var convertor = new LitTestConvertor(); + // TODO + } + } +} \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj new file mode 100644 index 00000000000..ab9de4fcd23 --- /dev/null +++ b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj @@ -0,0 +1,26 @@ + + + + net5.0 + + false + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + diff --git a/Source/LitTestConvertor.Test/TestFiles/HelloWorldLitTest.dfy b/Source/LitTestConvertor.Test/TestFiles/HelloWorldLitTest.dfy new file mode 100644 index 00000000000..3fca71de079 --- /dev/null +++ b/Source/LitTestConvertor.Test/TestFiles/HelloWorldLitTest.dfy @@ -0,0 +1,13 @@ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + +method Main() { + print "Hello, World! Best, Dafny\n"; + var x := 14; + print "x is ", x, "\n"; + var y := false; + print "y is ", y, "\n"; +} From 42c6781814f292fc013c4bbed940c003303ec738 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 9 Aug 2021 12:51:36 -0700 Subject: [PATCH 101/192] Revert experimental Test refactorings --- .../{testing => DafnyTests}/TestAttribute.dfy | 0 .../TestAttribute.dfy.expect | 0 Test/README.md | 20 - Test/VerifyThis2015/Problem1.dfy | 3 + Test/allocated1/dafny0/AdvancedLHS.dfy | 3 + Test/comp/Arrays.dfy | 7 + Test/comp/Arrays.dfy.expect | 155 ++++++++ Test/comp/AutoInit.dfy | 7 + Test/comp/AutoInit.dfy.expect | 119 ++++++ Test/comp/BranchCoverage.dfy | 25 +- Test/comp/BranchCoverage.dfy.expect | 344 ++++++++++++++--- Test/comp/CSharpStyling.dfy | 11 +- Test/comp/Calls.dfy | 7 + Test/comp/Calls.dfy.expect | 29 ++ Test/comp/Class.dfy | 6 + Test/comp/Class.dfy.expect | 41 +- Test/comp/Collections.dfy | 20 +- Test/comp/Collections.dfy.cs.expect | 74 ---- Test/comp/Collections.dfy.expect | 364 ++++++++++++++++++ Test/comp/Collections.dfy.java.expect | 74 ---- Test/comp/Comprehensions.dfy | 15 +- Test/comp/Comprehensions.dfy.expect | 191 ++++++++- Test/comp/Comprehensions.dfy.java.expect | 45 --- Test/comp/CovariantCollections.dfy | 7 + Test/comp/CovariantCollections.dfy.expect | 164 ++++++++ Test/comp/Dt.dfy | 15 +- Test/comp/Dt.dfy.expect | 59 ++- Test/comp/Dt.dfy.java.expect | 19 - Test/comp/Extern.dfy | 27 +- Test/comp/Extern.dfy.expect | 32 +- .../comp/{ => ExternCtors-externs}/Class.java | 0 Test/comp/ExternCtors.dfy | 34 +- Test/comp/ExternCtors.dfy.expect | 6 +- Test/comp/Forall.dfy | 6 + Test/comp/Forall.dfy.expect | 134 ++++++- Test/comp/Ghosts.dfy | 6 + Test/comp/Ghosts.dfy.expect | 38 +- Test/comp/GoModule.dfy | 9 +- Test/comp/Hello.dfy | 6 + Test/comp/Hello.dfy.expect | 17 +- Test/comp/Iterators.dfy | 15 +- Test/comp/Iterators.dfy.expect | 14 +- Test/comp/JsModule.dfy | 9 +- Test/comp/Let.dfy | 6 + Test/comp/Let.dfy.expect | 23 +- Test/comp/Module.dfy | 6 + Test/comp/Module.dfy.expect | 11 +- Test/comp/NativeNumbers.dfy | 16 +- Test/comp/NativeNumbers.dfy.expect | 170 +++++++- Test/comp/NativeNumbers.dfy.js.expect | 9 - Test/comp/Numbers.dfy | 16 +- Test/comp/Numbers.dfy.expect | 326 ++++++++++++++++ Test/comp/Numbers.dfy.go.expect | 102 ----- Test/comp/Poly.dfy | 16 +- Test/comp/Poly.dfy.expect | 44 +++ Test/comp/Poly.dfy.go.expect | 14 - Test/comp/StaticMembersOfGenericTypes.dfy | 6 + .../StaticMembersOfGenericTypes.dfy.expect | 56 ++- Test/comp/TailRecursion.dfy | 7 + Test/comp/TailRecursion.dfy.expect | 53 +++ Test/comp/TypeParams.dfy | 27 +- Test/comp/TypeParams.dfy.cs.expect | 21 - Test/comp/TypeParams.dfy.expect | 84 ++++ Test/comp/TypeParams.dfy.go.expect | 21 - Test/comp/TypeParams.dfy.java.expect | 21 - Test/comp/TypeParams.dfy.js.expect | 21 - Test/dafny0/AdvancedLHS.dfy | 8 +- Test/dafny0/AdvancedLHS.dfy.expect | 1 - Test/dafny4/git-issue250.dfy | 16 +- Test/dafny4/git-issue250.dfy.expect | 18 + .../TestAttributeErrors.dfy | 0 .../TestAttributeErrors.dfy.expect | 0 72 files changed, 2611 insertions(+), 685 deletions(-) rename Test/{testing => DafnyTests}/TestAttribute.dfy (100%) rename Test/{testing => DafnyTests}/TestAttribute.dfy.expect (100%) delete mode 100644 Test/README.md create mode 100644 Test/allocated1/dafny0/AdvancedLHS.dfy delete mode 100644 Test/comp/Collections.dfy.cs.expect delete mode 100644 Test/comp/Collections.dfy.java.expect delete mode 100644 Test/comp/Comprehensions.dfy.java.expect delete mode 100644 Test/comp/Dt.dfy.java.expect rename Test/comp/{ => ExternCtors-externs}/Class.java (100%) delete mode 100644 Test/comp/NativeNumbers.dfy.js.expect delete mode 100644 Test/comp/Numbers.dfy.go.expect delete mode 100644 Test/comp/Poly.dfy.go.expect delete mode 100644 Test/comp/TypeParams.dfy.cs.expect create mode 100644 Test/comp/TypeParams.dfy.expect delete mode 100644 Test/comp/TypeParams.dfy.go.expect delete mode 100644 Test/comp/TypeParams.dfy.java.expect delete mode 100644 Test/comp/TypeParams.dfy.js.expect rename Test/{testing => exceptions}/TestAttributeErrors.dfy (100%) rename Test/{testing => exceptions}/TestAttributeErrors.dfy.expect (100%) diff --git a/Test/testing/TestAttribute.dfy b/Test/DafnyTests/TestAttribute.dfy similarity index 100% rename from Test/testing/TestAttribute.dfy rename to Test/DafnyTests/TestAttribute.dfy diff --git a/Test/testing/TestAttribute.dfy.expect b/Test/DafnyTests/TestAttribute.dfy.expect similarity index 100% rename from Test/testing/TestAttribute.dfy.expect rename to Test/DafnyTests/TestAttribute.dfy.expect diff --git a/Test/README.md b/Test/README.md deleted file mode 100644 index c567c6ed269..00000000000 --- a/Test/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Tests - -Dafny test cases are based on single Dafny source files together with the expected output from the `dafny` CLI tool. If the source file is named `Foo.dfy`, the expected output will contained in the file named `Foo.dfy.expect`. - -The default behaviour is to assert that the source file verifies successfully and, for each currently-supported target language, can be compiled and run to produce the expected output. When the behaviour is not 100% consistent between different target languages, expected discrepancies can be recorded in additional files with the name `Foo.dfy..expect`. For example: [Test/comp/NativeNumbers.dfy.js.expect](Test/comp/NativeNumbers.dfy.js.expect). Such exceptions are automatically flagged as "known issues" and classified as a "skipped/ignored" test. - -Test cases are configured and run using xUnit's support for parameterized tests, with extensions for running test cases in parallel. The sets of options passed to `dafny` can be configured using YAML embedded in the first multi-line comment in the source file. Lists of values are interpreted as multiple parameterizations. For example: [Test/dafny4/git-issue250.dfy](Test/dafny4/git-issue250.dfy). - -For details and more configuration options, see [the DafnyTests.cs source](Test/DafnyTests/DafnyTests.cs). - -# TODO - -* More complete documentation about options (in this file or in the source code) -* Depend on only the project's output directory instead of the Binaries/Test directories - * This is mostly working except for errors around missing types from System.dll when compiling to C# -* Add support for regular expression matching against CLI output (needed to assert known limitations that cause errors with things like absolute paths names in them) -* Expose test case options as traits so that they can be filtered on (e.g. `dotnet test --filter compileTarget=java`) -* Finish converting the rest of the test cases - * Will write a small script to do this automatically for all recognized combinations of lit `// RUN` commands -* Extract most of the xUnit extensions as a separate package, since most of it is generically useful for any other data-driven xUnit tests. diff --git a/Test/VerifyThis2015/Problem1.dfy b/Test/VerifyThis2015/Problem1.dfy index 269079e63eb..12b28f242c0 100644 --- a/Test/VerifyThis2015/Problem1.dfy +++ b/Test/VerifyThis2015/Problem1.dfy @@ -1,3 +1,6 @@ +// RUN: %dafny /compile:3 /print:"%t.print" /dprint:"%t.dprint" "%s" > "%t" +// RUN: %diff "%s.expect" "%t" + // Rustan Leino // 12 April 2015 // VerifyThis 2015 diff --git a/Test/allocated1/dafny0/AdvancedLHS.dfy b/Test/allocated1/dafny0/AdvancedLHS.dfy new file mode 100644 index 00000000000..b154bb9cd5f --- /dev/null +++ b/Test/allocated1/dafny0/AdvancedLHS.dfy @@ -0,0 +1,3 @@ +// RUN: %dafny /verifyAllModules /allocated:1 /compile:0 /print:"%t.print" /dprint:"%t.dprint" "%s" > "%t" +// RUN: %diff "%s.expect" "%t" +include "../../dafny0/AdvancedLHS.dfy" diff --git a/Test/comp/Arrays.dfy b/Test/comp/Arrays.dfy index d54bcca25ce..d4651343d99 100644 --- a/Test/comp/Arrays.dfy +++ b/Test/comp/Arrays.dfy @@ -1,3 +1,10 @@ +// RUN: %dafny /compile:0 "%s" > "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:cs "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method LinearSearch(a: array, key: int) returns (n: nat) ensures 0 <= n <= a.Length ensures n == a.Length || a[n] == key diff --git a/Test/comp/Arrays.dfy.expect b/Test/comp/Arrays.dfy.expect index 7cfc6d7bcdf..6af6e3e356e 100644 --- a/Test/comp/Arrays.dfy.expect +++ b/Test/comp/Arrays.dfy.expect @@ -1,4 +1,159 @@ +Dafny program verifier finished with 42 verified, 0 errors + +Dafny program verifier did not attempt verification +0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 +17 +[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] +[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] +[20, 21, 22] +[0, 1, 2, 3, 4, 5, 6, 7] +[0, 1, 2, 3, 4, 5, 6, 7] +d d d +h e l l o +0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 +1 0 0 0 0 +0 1 0 0 0 +0 0 1 0 0 +cube dims: 3 0 4 +[] [0] [0, 0] [0, 0, 0] [0, 0, 0, 0] +It's null +hi hello tjena hej hola +hi hello tjena hej hola +hi hello tjena hej hola +d d d +h e l l o +8 8 8 8 +true true true +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +(d, 8, true, 1, 2, 3, 4) +D D D +0 0 0 0 +false false false +0 0 0 0 0 +0 0 0 0 0 +0 0 0 0 0 +0 0 0 0 0 +(D, 0, false, 0, 0, 0, 0) +8 0 +false 0 null null +8 8 +8 8 +8 8 +8 8 +D D D +D D D +D D D +D D r (D, D, r) +D D r +[19, 18, 9, 8] + +Dafny program verifier did not attempt verification +0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 +17 +[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] +[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] +[20, 21, 22] +[0, 1, 2, 3, 4, 5, 6, 7] +[0, 1, 2, 3, 4, 5, 6, 7] +d d d +h e l l o +0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 +1 0 0 0 0 +0 1 0 0 0 +0 0 1 0 0 +cube dims: 3 0 4 +[] [0] [0, 0] [0, 0, 0] [0, 0, 0, 0] +It's null +hi hello tjena hej hola +hi hello tjena hej hola +hi hello tjena hej hola +d d d +h e l l o +8 8 8 8 +true true true +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +(d, 8, true, 1, 2, 3, 4) +D D D +0 0 0 0 +false false false +0 0 0 0 0 +0 0 0 0 0 +0 0 0 0 0 +0 0 0 0 0 +(D, 0, false, 0, 0, 0, 0) +8 0 +false 0 null null +8 8 +8 8 +8 8 +8 8 +D D D +D D D +D D D +D D r (D, D, r) +D D r +[19, 18, 9, 8] + +Dafny program verifier did not attempt verification +0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 +17 +[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] +[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] +[20, 21, 22] +[0, 1, 2, 3, 4, 5, 6, 7] +[0, 1, 2, 3, 4, 5, 6, 7] +d d d +h e l l o +0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 +1 0 0 0 0 +0 1 0 0 0 +0 0 1 0 0 +cube dims: 3 0 4 +[] [0] [0, 0] [0, 0, 0] [0, 0, 0, 0] +It's null +hi hello tjena hej hola +hi hello tjena hej hola +hi hello tjena hej hola +d d d +h e l l o +8 8 8 8 +true true true +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 +4 4 4 4 4 +(d, 8, true, 1, 2, 3, 4) +D D D +0 0 0 0 +false false false +0 0 0 0 0 +0 0 0 0 0 +0 0 0 0 0 +0 0 0 0 0 +(D, 0, false, 0, 0, 0, 0) +8 0 +false 0 null null +8 8 +8 8 +8 8 +8 8 +D D D +D D D +D D D +D D r (D, D, r) +D D r +[19, 18, 9, 8] + Dafny program verifier did not attempt verification 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 17 diff --git a/Test/comp/AutoInit.dfy b/Test/comp/AutoInit.dfy index 25413f24f79..61c2a4438fb 100644 --- a/Test/comp/AutoInit.dfy +++ b/Test/comp/AutoInit.dfy @@ -1,3 +1,10 @@ +// RUN: %dafny /compile:0 "%s" > "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:cs "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + // This file tests the compilation of some default values. It also includes some // regression tests for equality, printing, and tuples. diff --git a/Test/comp/AutoInit.dfy.expect b/Test/comp/AutoInit.dfy.expect index 160b278648c..2bf2fb537f2 100644 --- a/Test/comp/AutoInit.dfy.expect +++ b/Test/comp/AutoInit.dfy.expect @@ -1,4 +1,123 @@ +Dafny program verifier finished with 22 verified, 0 errors + +Dafny program verifier did not attempt verification +3 false 0 +Cell.Cell(false) false true +() (0, false, false, []) +(null, Cell.Cell(0)) (null, Cell.Cell(0)) true +(null, null, null, Cell.Cell(0)) (null, null, null, Cell.Cell(0)) true +true false false true +true false false true +17 +1 3 9 0 0 ThisOrThat.Or ThisOrThat.Or +(1, 3, 9, 0, 0, ThisOrThat.Or, ThisOrThat.Or) +1 +ww == null -- yes, as expected +true true true true +true true true +true true true +null null +null null +null null null +null null null +null null null +null null null +null null +null null null +null null null +DatatypeDefaultValues_Compile.EnumDatatype.MakeZero + DatatypeDefaultValues_Compile.IntList.Nil + DatatypeDefaultValues_Compile.Cell.MakeCell(0) +DatatypeDefaultValues_Compile.GenericTree.Leaf + DatatypeDefaultValues_Compile.Complicated.ComplA(0, false) + DatatypeDefaultValues_Compile.CellCell.MakeCellCell(DatatypeDefaultValues_Compile.Cell.MakeCell(0)) +DatatypeDefaultValues_Compile.Difficult.MakeDifficult(null) + DatatypeDefaultValues_Compile.GenDifficult.MakeGenDifficult(null) +ImportedTypes_mLibrary_Compile.Color.Red(0) and ImportedTypes_mLibrary_Compile.Color.Red(0) +ImportedTypes_mLibrary_Compile.CoColor.Yellow and ImportedTypes_mLibrary_Compile.CoColor.Yellow +ImportedTypes_mLibrary_Compile.MoColor.MoYellow and ImportedTypes_mLibrary_Compile.MoColor.MoYellow +0.0 and 0.0 +13 + +Dafny program verifier did not attempt verification +3 false 0 +Cell.Cell(false) false true +() (0, false, false, []) +(null, Cell.Cell(0)) (null, Cell.Cell(0)) true +(null, null, null, Cell.Cell(0)) (null, null, null, Cell.Cell(0)) true +true false false true +true false false true +17 +1 3 9 0 0 ThisOrThat.Or ThisOrThat.Or +(1, 3, 9, 0, 0, ThisOrThat.Or, ThisOrThat.Or) +1 +ww == null -- yes, as expected +true true true true +true true true +true true true +null null +null null +null null null +null null null +null null null +null null null +null null +null null null +null null null +DatatypeDefaultValues_Compile.EnumDatatype.MakeZero + DatatypeDefaultValues_Compile.IntList.Nil + DatatypeDefaultValues_Compile.Cell.MakeCell(0) +DatatypeDefaultValues_Compile.GenericTree.Leaf + DatatypeDefaultValues_Compile.Complicated.ComplA(0, false) + DatatypeDefaultValues_Compile.CellCell.MakeCellCell(DatatypeDefaultValues_Compile.Cell.MakeCell(0)) +DatatypeDefaultValues_Compile.Difficult.MakeDifficult(null) + DatatypeDefaultValues_Compile.GenDifficult.MakeGenDifficult(null) +ImportedTypes_mLibrary_Compile.Color.Red(0) and ImportedTypes_mLibrary_Compile.Color.Red(0) +ImportedTypes_mLibrary_Compile.CoColor.Yellow and ImportedTypes_mLibrary_Compile.CoColor.Yellow +ImportedTypes_mLibrary_Compile.MoColor.MoYellow and ImportedTypes_mLibrary_Compile.MoColor.MoYellow +0.0 and 0.0 +13 + +Dafny program verifier did not attempt verification +3 false 0 +Cell.Cell(false) false true +() (0, false, false, []) +(null, Cell.Cell(0)) (null, Cell.Cell(0)) true +(null, null, null, Cell.Cell(0)) (null, null, null, Cell.Cell(0)) true +true false false true +true false false true +17 +1 3 9 0 0 ThisOrThat.Or ThisOrThat.Or +(1, 3, 9, 0, 0, ThisOrThat.Or, ThisOrThat.Or) +1 +ww == null -- yes, as expected +true true true true +true true true +true true true +null null +null null +null null null +null null null +null null null +null null null +null null +null null null +null null null +DatatypeDefaultValues_Compile.EnumDatatype.MakeZero + DatatypeDefaultValues_Compile.IntList.Nil + DatatypeDefaultValues_Compile.Cell.MakeCell(0) +DatatypeDefaultValues_Compile.GenericTree.Leaf + DatatypeDefaultValues_Compile.Complicated.ComplA(0, false) + DatatypeDefaultValues_Compile.CellCell.MakeCellCell(DatatypeDefaultValues_Compile.Cell.MakeCell(0)) +DatatypeDefaultValues_Compile.Difficult.MakeDifficult(null) + DatatypeDefaultValues_Compile.GenDifficult.MakeGenDifficult(null) +ImportedTypes_mLibrary_Compile.Color.Red(0) and ImportedTypes_mLibrary_Compile.Color.Red(0) +ImportedTypes_mLibrary_Compile.CoColor.Yellow and ImportedTypes_mLibrary_Compile.CoColor.Yellow +ImportedTypes_mLibrary_Compile.MoColor.MoYellow and ImportedTypes_mLibrary_Compile.MoColor.MoYellow +0.0 and 0.0 +13 + Dafny program verifier did not attempt verification 3 false 0 Cell.Cell(false) false true diff --git a/Test/comp/BranchCoverage.dfy b/Test/comp/BranchCoverage.dfy index ec82463342d..8b4206be84e 100644 --- a/Test/comp/BranchCoverage.dfy +++ b/Test/comp/BranchCoverage.dfy @@ -1,23 +1,8 @@ -/* ---- -!dafnyTestSpec -dafnyArguments: - compile: 3 - coverage: "-" -compileTargetOverrides: - java: - otherFiles: - - CodeCoverage.java - cs: - otherFiles: - - BranchCoverage2.cs - js: - otherFiles: - - BranchCoverage3.js - go: - otherFiles: - - BranchCoverage4.go -*/ +// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:cs BranchCoverage2.cs "%s" > "%t" +// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:js BranchCoverage3.js "%s" >> "%t" +// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:go BranchCoverage4.go "%s" >> "%t" +// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:java CodeCoverage.java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" // The Main method is at the end of this file, because that makes it easier to maintain // this test file when adding more tests. diff --git a/Test/comp/BranchCoverage.dfy.expect b/Test/comp/BranchCoverage.dfy.expect index 7f7e0922c49..dfbc09cb43c 100644 --- a/Test/comp/BranchCoverage.dfy.expect +++ b/Test/comp/BranchCoverage.dfy.expect @@ -1,47 +1,305 @@ -Dafny program verifier did not attempt verification -0: BranchCoverage.dfy(28,18): entry to method _module.MyClass._ctor -1: BranchCoverage.dfy(34,22): entry to method _module._default.NeverCalled -2: BranchCoverage.dfy(40,3): entry to function _module._default.FunctionNeverCalled -3: BranchCoverage.dfy(45,35): entry to method _module._default.M -4: BranchCoverage.dfy(47,13): then branch -5: BranchCoverage.dfy(49,20): then branch -6: BranchCoverage.dfy(51,10): else branch -7: BranchCoverage.dfy(56,35): entry to method _module._default.N -8: BranchCoverage.dfy(59,13): then branch -9: BranchCoverage.dfy(61,20): then branch -10: BranchCoverage.dfy(61,10): implicit else branch -11: BranchCoverage.dfy(67,18): entry to method _module._default.P -12: BranchCoverage.dfy(70,8): then branch -13: BranchCoverage.dfy(70,3): implicit else branch -14: BranchCoverage.dfy(75,8): then branch -15: BranchCoverage.dfy(77,10): else branch -16: BranchCoverage.dfy(87,12): then branch -17: BranchCoverage.dfy(89,15): then branch -18: BranchCoverage.dfy(89,10): implicit else branch -19: BranchCoverage.dfy(97,35): entry to method _module._default.Q -20: BranchCoverage.dfy(109,3): if-case branch -21: BranchCoverage.dfy(111,3): if-case branch -22: BranchCoverage.dfy(114,3): if-case branch -23: BranchCoverage.dfy(120,35): entry to method _module._default.R -24: BranchCoverage.dfy(123,15): while body -25: BranchCoverage.dfy(128,11): while body -26: BranchCoverage.dfy(136,5): while-case branch -27: BranchCoverage.dfy(138,17): then branch -28: BranchCoverage.dfy(138,7): implicit else branch -29: BranchCoverage.dfy(143,5): while-case branch -30: BranchCoverage.dfy(146,5): while-case branch -31: BranchCoverage.dfy(158,3): entry to function _module._default.Fib -32: BranchCoverage.dfy(159,5): then branch -33: BranchCoverage.dfy(161,5): then branch -34: BranchCoverage.dfy(161,5): else branch -35: BranchCoverage.dfy(170,3): entry to function _module._default.Factorial -36: BranchCoverage.dfy(171,13): then branch -37: BranchCoverage.dfy(171,13): else branch -38: BranchCoverage.dfy(177,1): entry to method _module._default.ComputeFactorial -39: BranchCoverage.dfy(180,3): then branch -40: BranchCoverage.dfy(182,3): else branch -41: BranchCoverage.dfy(188,15): entry to method _module._default.Main +Dafny program verifier finished with 6 verified, 0 errors +0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor +1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled +2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled +3: BranchCoverage.dfy(30,35): entry to method _module._default.M +4: BranchCoverage.dfy(32,13): then branch +5: BranchCoverage.dfy(34,20): then branch +6: BranchCoverage.dfy(36,10): else branch +7: BranchCoverage.dfy(41,35): entry to method _module._default.N +8: BranchCoverage.dfy(44,13): then branch +9: BranchCoverage.dfy(46,20): then branch +10: BranchCoverage.dfy(46,10): implicit else branch +11: BranchCoverage.dfy(52,18): entry to method _module._default.P +12: BranchCoverage.dfy(55,8): then branch +13: BranchCoverage.dfy(55,3): implicit else branch +14: BranchCoverage.dfy(60,8): then branch +15: BranchCoverage.dfy(62,10): else branch +16: BranchCoverage.dfy(72,12): then branch +17: BranchCoverage.dfy(74,15): then branch +18: BranchCoverage.dfy(74,10): implicit else branch +19: BranchCoverage.dfy(82,35): entry to method _module._default.Q +20: BranchCoverage.dfy(94,3): if-case branch +21: BranchCoverage.dfy(96,3): if-case branch +22: BranchCoverage.dfy(99,3): if-case branch +23: BranchCoverage.dfy(105,35): entry to method _module._default.R +24: BranchCoverage.dfy(108,15): while body +25: BranchCoverage.dfy(113,11): while body +26: BranchCoverage.dfy(121,5): while-case branch +27: BranchCoverage.dfy(123,17): then branch +28: BranchCoverage.dfy(123,7): implicit else branch +29: BranchCoverage.dfy(128,5): while-case branch +30: BranchCoverage.dfy(131,5): while-case branch +31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib +32: BranchCoverage.dfy(144,5): then branch +33: BranchCoverage.dfy(146,5): then branch +34: BranchCoverage.dfy(146,5): else branch +35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial +36: BranchCoverage.dfy(156,13): then branch +37: BranchCoverage.dfy(156,13): else branch +38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial +39: BranchCoverage.dfy(165,3): then branch +40: BranchCoverage.dfy(167,3): else branch +41: BranchCoverage.dfy(173,15): entry to method _module._default.Main +0: 3 +1: 0 +2: 0 +3: 3 +4: 1 +5: 0 +6: 2 +7: 3 +8: 1 +9: 0 +10: 2 +11: 1 +12: 0 +13: 1 +14: 1 +15: 0 +16: 0 +17: 0 +18: 1 +19: 3 +20: 1 +21: 1 +22: 1 +23: 1 +24: 111 +25: 0 +26: 56 +27: 1 +28: 55 +29: 28 +30: 28 +31: 465 +32: 233 +33: 0 +34: 232 +35: 11 +36: 1 +37: 10 +38: 11 +39: 1 +40: 10 +41: 1 + +Dafny program verifier finished with 6 verified, 0 errors +0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor +1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled +2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled +3: BranchCoverage.dfy(30,35): entry to method _module._default.M +4: BranchCoverage.dfy(32,13): then branch +5: BranchCoverage.dfy(34,20): then branch +6: BranchCoverage.dfy(36,10): else branch +7: BranchCoverage.dfy(41,35): entry to method _module._default.N +8: BranchCoverage.dfy(44,13): then branch +9: BranchCoverage.dfy(46,20): then branch +10: BranchCoverage.dfy(46,10): implicit else branch +11: BranchCoverage.dfy(52,18): entry to method _module._default.P +12: BranchCoverage.dfy(55,8): then branch +13: BranchCoverage.dfy(55,3): implicit else branch +14: BranchCoverage.dfy(60,8): then branch +15: BranchCoverage.dfy(62,10): else branch +16: BranchCoverage.dfy(72,12): then branch +17: BranchCoverage.dfy(74,15): then branch +18: BranchCoverage.dfy(74,10): implicit else branch +19: BranchCoverage.dfy(82,35): entry to method _module._default.Q +20: BranchCoverage.dfy(94,3): if-case branch +21: BranchCoverage.dfy(96,3): if-case branch +22: BranchCoverage.dfy(99,3): if-case branch +23: BranchCoverage.dfy(105,35): entry to method _module._default.R +24: BranchCoverage.dfy(108,15): while body +25: BranchCoverage.dfy(113,11): while body +26: BranchCoverage.dfy(121,5): while-case branch +27: BranchCoverage.dfy(123,17): then branch +28: BranchCoverage.dfy(123,7): implicit else branch +29: BranchCoverage.dfy(128,5): while-case branch +30: BranchCoverage.dfy(131,5): while-case branch +31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib +32: BranchCoverage.dfy(144,5): then branch +33: BranchCoverage.dfy(146,5): then branch +34: BranchCoverage.dfy(146,5): else branch +35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial +36: BranchCoverage.dfy(156,13): then branch +37: BranchCoverage.dfy(156,13): else branch +38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial +39: BranchCoverage.dfy(165,3): then branch +40: BranchCoverage.dfy(167,3): else branch +41: BranchCoverage.dfy(173,15): entry to method _module._default.Main +0: 3 +1: 0 +2: 0 +3: 3 +4: 1 +5: 0 +6: 2 +7: 3 +8: 1 +9: 0 +10: 2 +11: 1 +12: 0 +13: 1 +14: 1 +15: 0 +16: 0 +17: 0 +18: 1 +19: 3 +20: 1 +21: 1 +22: 1 +23: 1 +24: 111 +25: 0 +26: 56 +27: 1 +28: 55 +29: 28 +30: 28 +31: 465 +32: 233 +33: 0 +34: 232 +35: 11 +36: 1 +37: 10 +38: 11 +39: 1 +40: 10 +41: 1 + +Dafny program verifier finished with 6 verified, 0 errors +0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor +1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled +2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled +3: BranchCoverage.dfy(30,35): entry to method _module._default.M +4: BranchCoverage.dfy(32,13): then branch +5: BranchCoverage.dfy(34,20): then branch +6: BranchCoverage.dfy(36,10): else branch +7: BranchCoverage.dfy(41,35): entry to method _module._default.N +8: BranchCoverage.dfy(44,13): then branch +9: BranchCoverage.dfy(46,20): then branch +10: BranchCoverage.dfy(46,10): implicit else branch +11: BranchCoverage.dfy(52,18): entry to method _module._default.P +12: BranchCoverage.dfy(55,8): then branch +13: BranchCoverage.dfy(55,3): implicit else branch +14: BranchCoverage.dfy(60,8): then branch +15: BranchCoverage.dfy(62,10): else branch +16: BranchCoverage.dfy(72,12): then branch +17: BranchCoverage.dfy(74,15): then branch +18: BranchCoverage.dfy(74,10): implicit else branch +19: BranchCoverage.dfy(82,35): entry to method _module._default.Q +20: BranchCoverage.dfy(94,3): if-case branch +21: BranchCoverage.dfy(96,3): if-case branch +22: BranchCoverage.dfy(99,3): if-case branch +23: BranchCoverage.dfy(105,35): entry to method _module._default.R +24: BranchCoverage.dfy(108,15): while body +25: BranchCoverage.dfy(113,11): while body +26: BranchCoverage.dfy(121,5): while-case branch +27: BranchCoverage.dfy(123,17): then branch +28: BranchCoverage.dfy(123,7): implicit else branch +29: BranchCoverage.dfy(128,5): while-case branch +30: BranchCoverage.dfy(131,5): while-case branch +31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib +32: BranchCoverage.dfy(144,5): then branch +33: BranchCoverage.dfy(146,5): then branch +34: BranchCoverage.dfy(146,5): else branch +35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial +36: BranchCoverage.dfy(156,13): then branch +37: BranchCoverage.dfy(156,13): else branch +38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial +39: BranchCoverage.dfy(165,3): then branch +40: BranchCoverage.dfy(167,3): else branch +41: BranchCoverage.dfy(173,15): entry to method _module._default.Main +0: 3 +1: 0 +2: 0 +3: 3 +4: 1 +5: 0 +6: 2 +7: 3 +8: 1 +9: 0 +10: 2 +11: 1 +12: 0 +13: 1 +14: 1 +15: 0 +16: 0 +17: 0 +18: 1 +19: 3 +20: 1 +21: 1 +22: 1 +23: 1 +24: 111 +25: 0 +26: 56 +27: 1 +28: 55 +29: 28 +30: 28 +31: 465 +32: 233 +33: 0 +34: 232 +35: 11 +36: 1 +37: 10 +38: 11 +39: 1 +40: 10 +41: 1 + +Dafny program verifier finished with 6 verified, 0 errors +0: BranchCoverage.dfy(13,18): entry to method _module.MyClass._ctor +1: BranchCoverage.dfy(19,22): entry to method _module._default.NeverCalled +2: BranchCoverage.dfy(25,3): entry to function _module._default.FunctionNeverCalled +3: BranchCoverage.dfy(30,35): entry to method _module._default.M +4: BranchCoverage.dfy(32,13): then branch +5: BranchCoverage.dfy(34,20): then branch +6: BranchCoverage.dfy(36,10): else branch +7: BranchCoverage.dfy(41,35): entry to method _module._default.N +8: BranchCoverage.dfy(44,13): then branch +9: BranchCoverage.dfy(46,20): then branch +10: BranchCoverage.dfy(46,10): implicit else branch +11: BranchCoverage.dfy(52,18): entry to method _module._default.P +12: BranchCoverage.dfy(55,8): then branch +13: BranchCoverage.dfy(55,3): implicit else branch +14: BranchCoverage.dfy(60,8): then branch +15: BranchCoverage.dfy(62,10): else branch +16: BranchCoverage.dfy(72,12): then branch +17: BranchCoverage.dfy(74,15): then branch +18: BranchCoverage.dfy(74,10): implicit else branch +19: BranchCoverage.dfy(82,35): entry to method _module._default.Q +20: BranchCoverage.dfy(94,3): if-case branch +21: BranchCoverage.dfy(96,3): if-case branch +22: BranchCoverage.dfy(99,3): if-case branch +23: BranchCoverage.dfy(105,35): entry to method _module._default.R +24: BranchCoverage.dfy(108,15): while body +25: BranchCoverage.dfy(113,11): while body +26: BranchCoverage.dfy(121,5): while-case branch +27: BranchCoverage.dfy(123,17): then branch +28: BranchCoverage.dfy(123,7): implicit else branch +29: BranchCoverage.dfy(128,5): while-case branch +30: BranchCoverage.dfy(131,5): while-case branch +31: BranchCoverage.dfy(143,3): entry to function _module._default.Fib +32: BranchCoverage.dfy(144,5): then branch +33: BranchCoverage.dfy(146,5): then branch +34: BranchCoverage.dfy(146,5): else branch +35: BranchCoverage.dfy(155,3): entry to function _module._default.Factorial +36: BranchCoverage.dfy(156,13): then branch +37: BranchCoverage.dfy(156,13): else branch +38: BranchCoverage.dfy(162,1): entry to method _module._default.ComputeFactorial +39: BranchCoverage.dfy(165,3): then branch +40: BranchCoverage.dfy(167,3): else branch +41: BranchCoverage.dfy(173,15): entry to method _module._default.Main 0: 3 1: 0 2: 0 diff --git a/Test/comp/CSharpStyling.dfy b/Test/comp/CSharpStyling.dfy index b99078cd843..038eac22aef 100644 --- a/Test/comp/CSharpStyling.dfy +++ b/Test/comp/CSharpStyling.dfy @@ -1,12 +1,5 @@ -/* ---- -!dafnyTestSpec -dafnyArguments: - compile: 3 - compileTarget: cs -otherFiles: - - CSharpStyling2.cs -*/ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" CSharpStyling2.cs > "%t" +// RUN: %diff "%s.expect" "%t" method Main() { var c := new MyClass(50); diff --git a/Test/comp/Calls.dfy b/Test/comp/Calls.dfy index 8ffc707aefe..a30ce9560ff 100644 --- a/Test/comp/Calls.dfy +++ b/Test/comp/Calls.dfy @@ -1,3 +1,10 @@ +// RUN: %dafny /compile:0 "%s" > "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:cs "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + function method F(x: int, y: bool): int { x + if y then 2 else 3 } diff --git a/Test/comp/Calls.dfy.expect b/Test/comp/Calls.dfy.expect index 2f7ac985a44..c55c552c172 100644 --- a/Test/comp/Calls.dfy.expect +++ b/Test/comp/Calls.dfy.expect @@ -1,4 +1,33 @@ +Dafny program verifier finished with 6 verified, 0 errors + +Dafny program verifier did not attempt verification +5 0 0 false 0 false 0 +5 3 5 3 5 3 +5 3 5 3 5 3 +15 +[0, 1, 2, 4] +[0, 2, 4] +--> 5 <-- + +Dafny program verifier did not attempt verification +5 0 0 false 0 false 0 +5 3 5 3 5 3 +5 3 5 3 5 3 +15 +[0, 1, 2, 4] +[0, 2, 4] +--> 5 <-- + +Dafny program verifier did not attempt verification +5 0 0 false 0 false 0 +5 3 5 3 5 3 +5 3 5 3 5 3 +15 +[0, 1, 2, 4] +[0, 2, 4] +--> 5 <-- + Dafny program verifier did not attempt verification 5 0 0 false 0 false 0 5 3 5 3 5 3 diff --git a/Test/comp/Class.dfy b/Test/comp/Class.dfy index 7f8d90cac95..644de2bb66b 100644 --- a/Test/comp/Class.dfy +++ b/Test/comp/Class.dfy @@ -1,3 +1,9 @@ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + class MyClass { var a: int const b: int diff --git a/Test/comp/Class.dfy.expect b/Test/comp/Class.dfy.expect index 37ee1b8b3d0..a75cb43552a 100644 --- a/Test/comp/Class.dfy.expect +++ b/Test/comp/Class.dfy.expect @@ -1,5 +1,44 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 13 verified, 0 errors +true true false +103 103 103 106 106 106 +203 17 0 18 8 9 69 70 +203 17 0 18 8 9 69 70 +203 17 0 18 8 9 69 70 +0 18 9 70 +0 18 9 70 +0 18 9 70 +70 +hi later +21 4 42 5 1 + +Dafny program verifier finished with 13 verified, 0 errors +true true false +103 103 103 106 106 106 +203 17 0 18 8 9 69 70 +203 17 0 18 8 9 69 70 +203 17 0 18 8 9 69 70 +0 18 9 70 +0 18 9 70 +0 18 9 70 +70 +hi later +21 4 42 5 1 + +Dafny program verifier finished with 13 verified, 0 errors +true true false +103 103 103 106 106 106 +203 17 0 18 8 9 69 70 +203 17 0 18 8 9 69 70 +203 17 0 18 8 9 69 70 +0 18 9 70 +0 18 9 70 +0 18 9 70 +70 +hi later +21 4 42 5 1 + +Dafny program verifier finished with 13 verified, 0 errors true true false 103 103 103 106 106 106 203 17 0 18 8 9 69 70 diff --git a/Test/comp/Collections.dfy b/Test/comp/Collections.dfy index 2d243aa5d7b..4a361a2cc1a 100644 --- a/Test/comp/Collections.dfy +++ b/Test/comp/Collections.dfy @@ -1,16 +1,10 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - cs: - expected: - outputFile: Collections.dfy.cs.expect - specialCaseReason: Set output is inconsistently ordered - java: - expected: - outputFile: Collections.dfy.java.expect - specialCaseReason: Set output is inconsistently ordered -*/ +// RUN: %dafny /compile:0 "%s" > "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:cs "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { Sets(); SubSets(); diff --git a/Test/comp/Collections.dfy.cs.expect b/Test/comp/Collections.dfy.cs.expect deleted file mode 100644 index 88c7cf974f0..00000000000 --- a/Test/comp/Collections.dfy.cs.expect +++ /dev/null @@ -1,74 +0,0 @@ - -Dafny program verifier did not attempt verification -Sets: {} {17, 82} {12, 17} - cardinality: 0 2 2 - union: {17, 82} {12, 17, 82} - intersection: {} {17} - difference: {} {82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true - eq covariance: 1 true true -false true -|s|=4 |S|=16 -{{a, c, d}, {a, b, c}, {}, {a}, {b}, {c}, {d}, {a, b, c, d}, {b, c, d}, {a, b, d}, {a, b}, {a, c}, {a, d}, {b, c}, {b, d}, {c, d}} -Multisets: multiset{} multiset{17, 17, 82, 82} multiset{12, 17} - cardinality: 0 4 2 - union: multiset{17, 17, 82, 82} multiset{12, 17, 17, 17, 82, 82} - intersection: multiset{} multiset{17} - difference: multiset{} multiset{17, 82, 82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true - update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{12, 17, 17} - multiplicity: 0 2 1 -Sequences: [] [17, 82, 17, 82] [12, 17] - cardinality: 0 4 2 - update: [42, 82, 17, 82] [42, 17] - index: 17 12 - subsequence ([lo..hi]): [82, 17] [17] - subsequence ([lo..]): [82, 17, 82] [17] - subsequence ([..hi]): [17, 82, 17][12] - subsequence ([..]): [] [17, 82, 17, 82] [12, 17] - concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] - prefix: true false true - proper prefix: true false false - membership: false true true -Bound Bound Bound Bound Bound Bound -ed ed ed ed ed ed -e e e e e e -hello -hEllo -[2, 4, 6, 8, 10] -[2, 0, 6, 8, 10] -true false false -true true false true -false false false true - eq covariance: true true -Strings: uRuR gu - cardinality: 0 4 2 - concatenation: uRuR uRuRgu - prefix: true false true - proper prefix: true false false - membership: false true true - constructed as sequence: guru - mix: hello-d ddd-hello -Maps: map[] map[17 := 2, 82 := 2] map[12 := 26, 17 := 0] - cardinality: 0 2 2 - keys: {} {17, 82} {12, 17} - values: {} {2} {0, 26} - items: {} {(17, 2), (82, 2)} {(12, 26), (17, 0)} - update: map[17 := 6] map[17 := 6, 82 := 2] map[12 := 26, 17 := 6] - lookup: false 2 0 -m: map[Color.Blue := 30, Color.Yellow := 21] -keys: {Color.Blue, Color.Yellow} -values: {21, 30} -items: {(Color.Blue, 30), (Color.Yellow, 21)} -2: 0 1 1 -3: 0 1 2 -There are 0 occurrences of 58 in the multiset -There are 536870912 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of null in the multiset diff --git a/Test/comp/Collections.dfy.expect b/Test/comp/Collections.dfy.expect index b33d84e7cf9..48bd9d464f5 100644 --- a/Test/comp/Collections.dfy.expect +++ b/Test/comp/Collections.dfy.expect @@ -1,4 +1,236 @@ +Dafny program verifier finished with 27 verified, 0 errors + +Dafny program verifier did not attempt verification +Sets: {} {17, 82} {12, 17} + cardinality: 0 2 2 + union: {17, 82} {12, 17, 82} + intersection: {} {17} + difference: {} {82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true + eq covariance: 1 true true +false true +|s|=4 |S|=16 +{{a, c, d}, {a, b, c}, {}, {a}, {b}, {c}, {d}, {a, b, c, d}, {b, c, d}, {a, b, d}, {a, b}, {a, c}, {a, d}, {b, c}, {b, d}, {c, d}} +Multisets: multiset{} multiset{17, 17, 82, 82} multiset{12, 17} + cardinality: 0 4 2 + union: multiset{17, 17, 82, 82} multiset{12, 17, 17, 17, 82, 82} + intersection: multiset{} multiset{17} + difference: multiset{} multiset{17, 82, 82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true + update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{12, 17, 17} + multiplicity: 0 2 1 +Sequences: [] [17, 82, 17, 82] [12, 17] + cardinality: 0 4 2 + update: [42, 82, 17, 82] [42, 17] + index: 17 12 + subsequence ([lo..hi]): [82, 17] [17] + subsequence ([lo..]): [82, 17, 82] [17] + subsequence ([..hi]): [17, 82, 17][12] + subsequence ([..]): [] [17, 82, 17, 82] [12, 17] + concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] + prefix: true false true + proper prefix: true false false + membership: false true true +Bound Bound Bound Bound Bound Bound +ed ed ed ed ed ed +e e e e e e +hello +hEllo +[2, 4, 6, 8, 10] +[2, 0, 6, 8, 10] +true false false +true true false true +false false false true + eq covariance: true true +Strings: uRuR gu + cardinality: 0 4 2 + concatenation: uRuR uRuRgu + prefix: true false true + proper prefix: true false false + membership: false true true + constructed as sequence: guru + mix: hello-d ddd-hello +Maps: map[] map[17 := 2, 82 := 2] map[12 := 26, 17 := 0] + cardinality: 0 2 2 + keys: {} {17, 82} {12, 17} + values: {} {2} {0, 26} + items: {} {(17, 2), (82, 2)} {(12, 26), (17, 0)} + update: 6 (1 0) 6 (2 2 2) 6 (2 2 0) + lookup: false 2 0 +m: map[Color.Blue := 30, Color.Yellow := 21] +keys: {Color.Blue, Color.Yellow} +values: {21, 30} +items: {(Color.Blue, 30), (Color.Yellow, 21)} +3 2 4 +70 52 66 +8 67 67 +4 4 3 +true false true false +true false false false +false false true +false false false true +0 20 4 +198 20 4 +0 2 false +true true true false +0 20 4 +198 20 4 +0 2 false +false false true +false true true true +jack wendy null +jack null null +ronald ronald false +true true true false +jack wendy null +jack null null +ronald ronald false +2: 0 1 1 +3: 0 1 2 +There are 0 occurrences of 58 in the multiset +There are 536870912 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of null in the multiset +Map=== +|s|=1 |u|=1 +s[1]=123 u[1]=40 +|S|=1 |U|=1 +S[1]=123 U[1]=41 +Seq=== +|s|=2 |u|=2 +123 42, 123 42, 123 42 +|s|=2 |u|=2 +123 43, 123 43, 123 43 +|s|=2 |u|=2 +123 44, 123 44, 123 44 +Multiset=== +|s|=2 |u|=4 +1 3 +|s|=2 |u|=4 +1 3 + +Dafny program verifier did not attempt verification +Sets: {} {17, 82} {12, 17} + cardinality: 0 2 2 + union: {17, 82} {17, 82, 12} + intersection: {} {17} + difference: {} {82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true + eq covariance: 1 true true +false true +|s|=4 |S|=16 +{{}, {a}, {b}, {b, a}, {c}, {c, a}, {c, b}, {c, b, a}, {d}, {d, a}, {d, b}, {d, b, a}, {d, c}, {d, c, a}, {d, c, b}, {d, c, b, a}} +Multisets: multiset{} multiset{17, 17, 82, 82} multiset{12, 17} + cardinality: 0 4 2 + union: multiset{17, 17, 82, 82} multiset{17, 17, 17, 82, 82, 12} + intersection: multiset{} multiset{17} + difference: multiset{} multiset{17, 82, 82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true + update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{12, 17, 17} + multiplicity: 0 2 1 +Sequences: [] [17, 82, 17, 82] [12, 17] + cardinality: 0 4 2 + update: [42, 82, 17, 82] [42, 17] + index: 17 12 + subsequence ([lo..hi]): [82, 17] [17] + subsequence ([lo..]): [82, 17, 82] [17] + subsequence ([..hi]): [17, 82, 17][12] + subsequence ([..]): [] [17, 82, 17, 82] [12, 17] + concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] + prefix: true false true + proper prefix: true false false + membership: false true true +Bound Bound Bound Bound Bound Bound +ed ed ed ed ed ed +e e e e e e +hello +hEllo +[2, 4, 6, 8, 10] +[2, 0, 6, 8, 10] +true false false +true true false true +false false false true + eq covariance: true true +Strings: uRuR gu + cardinality: 0 4 2 + concatenation: uRuR uRuRgu + prefix: true false true + proper prefix: true false false + membership: false true true + constructed as sequence: guru + mix: hello-d ddd-hello +Maps: map[] map[17 := 2, 82 := 2] map[17 := 0, 12 := 26] + cardinality: 0 2 2 + keys: {} {17, 82} {17, 12} + values: {} {2} {0, 26} + items: {} {(17, 2), (82, 2)} {(17, 0), (12, 26)} + update: 6 (1 0) 6 (2 2 2) 6 (2 2 0) + lookup: false 2 0 +m: map[Color.Blue := 30, Color.Yellow := 21] +keys: {Color.Blue, Color.Yellow} +values: {30, 21} +items: {(Color.Blue, 30), (Color.Yellow, 21)} +3 2 4 +70 52 66 +8 67 67 +4 4 3 +true false true false +true false false false +false false true +false false false true +0 20 4 +198 20 4 +0 2 false +true true true false +0 20 4 +198 20 4 +0 2 false +false false true +false true true true +jack wendy null +jack null null +ronald ronald false +true true true false +jack wendy null +jack null null +ronald ronald false +2: 0 1 1 +3: 0 1 2 +There are 0 occurrences of 58 in the multiset +There are 536870912 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of null in the multiset +Map=== +|s|=1 |u|=1 +s[1]=123 u[1]=40 +|S|=1 |U|=1 +S[1]=123 U[1]=41 +Seq=== +|s|=2 |u|=2 +123 42, 123 42, 123 42 +|s|=2 |u|=2 +123 43, 123 43, 123 43 +|s|=2 |u|=2 +123 44, 123 44, 123 44 +Multiset=== +|s|=2 |u|=4 +1 3 +|s|=2 |u|=4 +1 3 + Dafny program verifier did not attempt verification Sets: {} {17, 82} {12, 17} cardinality: 0 2 2 @@ -96,3 +328,135 @@ There are 0 occurrences of 58 in the multiset There are 536870912 occurrences of 58 in the multiset There are 633825300114114700748351602688 occurrences of 58 in the multiset There are 633825300114114700748351602688 occurrences of null in the multiset +Map=== +|s|=1 |u|=1 +s[1]=123 u[1]=40 +|S|=1 |U|=1 +S[1]=123 U[1]=41 +Seq=== +|s|=2 |u|=2 +123 42, 123 42, 123 42 +|s|=2 |u|=2 +123 43, 123 43, 123 43 +|s|=2 |u|=2 +123 44, 123 44, 123 44 +Multiset=== +|s|=2 |u|=4 +1 3 +|s|=2 |u|=4 +1 3 + +Dafny program verifier did not attempt verification +Sets: {} {17, 82} {17, 12} + cardinality: 0 2 2 + union: {17, 82} {17, 82, 12} + intersection: {} {17} + difference: {} {82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true + eq covariance: 1 true true +false true +|s|=4 |S|=16 +{{}, {a}, {b}, {a, b}, {c}, {a, c}, {d}, {b, c}, {a, d}, {a, b, c}, {b, d}, {a, b, d}, {c, d}, {a, c, d}, {b, c, d}, {a, b, c, d}} +Multisets: multiset{} multiset{17, 17, 82, 82} multiset{17, 12} + cardinality: 0 4 2 + union: multiset{17, 17, 82, 82} multiset{17, 17, 17, 82, 82, 12} + intersection: multiset{} multiset{17} + difference: multiset{} multiset{17, 82, 82} + disjoint: true false + subset: true false true + proper subset: true false false + membership: false true true + update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{17, 17, 12} + multiplicity: 0 2 1 +Sequences: [] [17, 82, 17, 82] [12, 17] + cardinality: 0 4 2 + update: [42, 82, 17, 82] [42, 17] + index: 17 12 + subsequence ([lo..hi]): [82, 17] [17] + subsequence ([lo..]): [82, 17, 82] [17] + subsequence ([..hi]): [17, 82, 17][12] + subsequence ([..]): [] [17, 82, 17, 82] [12, 17] + concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] + prefix: true false true + proper prefix: true false false + membership: false true true +Bound Bound Bound Bound Bound Bound +ed ed ed ed ed ed +e e e e e e +hello +hEllo +[2, 4, 6, 8, 10] +[2, 0, 6, 8, 10] +true false false +true true false true +false false false true + eq covariance: true true +Strings: uRuR gu + cardinality: 0 4 2 + concatenation: uRuR uRuRgu + prefix: true false true + proper prefix: true false false + membership: false true true + constructed as sequence: guru + mix: hello-d ddd-hello +Maps: map[] map[17 := 2, 82 := 2] map[17 := 0, 12 := 26] + cardinality: 0 2 2 + keys: {} {17, 82} {17, 12} + values: {} {2} {0, 26} + items: {} {(17, 2), (82, 2)} {(17, 0), (12, 26)} + update: 6 (1 0) 6 (2 2 2) 6 (2 2 0) + lookup: false 2 0 +m: map[Color.Yellow := 21, Color.Blue := 30] +keys: {Color.Yellow, Color.Blue} +values: {21, 30} +items: {(Color.Yellow, 21), (Color.Blue, 30)} +3 2 4 +70 52 66 +8 67 67 +4 4 3 +true false true false +true false false false +false false true +false false false true +0 20 4 +198 20 4 +0 2 false +true true true false +0 20 4 +198 20 4 +0 2 false +false false true +false true true true +jack wendy null +jack null null +ronald ronald false +true true true false +jack wendy null +jack null null +ronald ronald false +2: 0 1 1 +3: 0 1 2 +There are 0 occurrences of 58 in the multiset +There are 536870912 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of 58 in the multiset +There are 633825300114114700748351602688 occurrences of null in the multiset +Map=== +|s|=1 |u|=1 +s[1]=123 u[1]=40 +|S|=1 |U|=1 +S[1]=123 U[1]=41 +Seq=== +|s|=2 |u|=2 +123 42, 123 42, 123 42 +|s|=2 |u|=2 +123 43, 123 43, 123 43 +|s|=2 |u|=2 +123 44, 123 44, 123 44 +Multiset=== +|s|=2 |u|=4 +1 3 +|s|=2 |u|=4 +1 3 diff --git a/Test/comp/Collections.dfy.java.expect b/Test/comp/Collections.dfy.java.expect deleted file mode 100644 index a9b9533204c..00000000000 --- a/Test/comp/Collections.dfy.java.expect +++ /dev/null @@ -1,74 +0,0 @@ - -Dafny program verifier did not attempt verification -Sets: {} {17, 82} {17, 12} - cardinality: 0 2 2 - union: {17, 82} {17, 82, 12} - intersection: {} {17} - difference: {} {82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true - eq covariance: 1 true true -false true -|s|=4 |S|=16 -{{}, {a}, {b}, {a, b}, {c}, {a, c}, {d}, {b, c}, {a, d}, {a, b, c}, {b, d}, {a, b, d}, {c, d}, {a, c, d}, {b, c, d}, {a, b, c, d}} -Multisets: multiset{} multiset{17, 17, 82, 82} multiset{17, 12} - cardinality: 0 4 2 - union: multiset{17, 17, 82, 82} multiset{17, 17, 17, 82, 82, 12} - intersection: multiset{} multiset{17} - difference: multiset{} multiset{17, 82, 82} - disjoint: true false - subset: true false true - proper subset: true false false - membership: false true true - update: multiset{17, 17} multiset{17, 17, 82, 82} multiset{17, 17, 12} - multiplicity: 0 2 1 -Sequences: [] [17, 82, 17, 82] [12, 17] - cardinality: 0 4 2 - update: [42, 82, 17, 82] [42, 17] - index: 17 12 - subsequence ([lo..hi]): [82, 17] [17] - subsequence ([lo..]): [82, 17, 82] [17] - subsequence ([..hi]): [17, 82, 17][12] - subsequence ([..]): [] [17, 82, 17, 82] [12, 17] - concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] - prefix: true false true - proper prefix: true false false - membership: false true true -Bound Bound Bound Bound Bound Bound -ed ed ed ed ed ed -e e e e e e -hello -hEllo -[2, 4, 6, 8, 10] -[2, 0, 6, 8, 10] -true false false -true true false true -false false false true - eq covariance: true true -Strings: uRuR gu - cardinality: 0 4 2 - concatenation: uRuR uRuRgu - prefix: true false true - proper prefix: true false false - membership: false true true - constructed as sequence: guru - mix: hello-d ddd-hello -Maps: map[] map[17 := 2, 82 := 2] map[17 := 0, 12 := 26] - cardinality: 0 2 2 - keys: {} {17, 82} {17, 12} - values: {} {2} {0, 26} - items: {} {(17, 2), (82, 2)} {(17, 0), (12, 26)} - update: map[17 := 6] map[17 := 6, 82 := 2] map[12 := 26, 17 := 6] - lookup: false 2 0 -m: map[Color.Yellow := 21, Color.Blue := 30] -keys: {Color.Yellow, Color.Blue} -values: {21, 30} -items: {(Color.Yellow, 21), (Color.Blue, 30)} -2: 0 1 1 -3: 0 1 2 -There are 0 occurrences of 58 in the multiset -There are 536870912 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of 58 in the multiset -There are 633825300114114700748351602688 occurrences of null in the multiset diff --git a/Test/comp/Comprehensions.dfy b/Test/comp/Comprehensions.dfy index 8efebb1d246..764024b55a0 100644 --- a/Test/comp/Comprehensions.dfy +++ b/Test/comp/Comprehensions.dfy @@ -1,12 +1,9 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - java: - expected: - outputFile: Comprehensions.dfy.java.expect - specialCaseReason: Java doesn't always print strings correctly -*/ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { AssignSuchThat(); LetSuchThat(); diff --git a/Test/comp/Comprehensions.dfy.expect b/Test/comp/Comprehensions.dfy.expect index 6fdbdd01982..ba391ac0121 100644 --- a/Test/comp/Comprehensions.dfy.expect +++ b/Test/comp/Comprehensions.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 21 verified, 0 errors x=13 y=14 x=13 y=14 b=yes true false @@ -41,3 +41,192 @@ true true true false false false true true true false false false +4 3 2 1 +true true true true +4 3 2 1 +true true true true +false false false false +false false false false +true true true true +false false false false +true true true true +false true +false true +null +false true +true true +3 + +Dafny program verifier finished with 21 verified, 0 errors +x=13 y=14 +x=13 y=14 b=yes +true false +true false +map[12 := 6, 13 := 6, 14 := 7] +map[16 := 12, 17 := 13, 18 := 14] +XP returned: 0 false +after: 0 before: 0 +after: 2 before: 2 +XM returned: 16 +0 12 3 8 +0 12 3 8 +0 12 3 8 +0 12 3 8 +[1, 1, 1, 1] +[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] +[0, 1, 4, 9, 16, 25, 36, 49] +[0, 1, 2, 3, 4, 5, 6, 7] +4 4 2 2 0 4 2 true +5 4 2 2 0 4 2 true +5 5 3 3 1 5 3 true +2 2 1 1 0 2 false true +3 3 +451 +671 +221 +281 +1 +true false 1 +451 +671 +221 +281 +2 +true true false 2 +the intersection is {null} +there are 3 elements in the union +true true true +false false false +true true true +false false false +4 3 2 1 +true true true true +4 3 2 1 +true true true true +false false false false +false false false false +true true true true +false false false false +true true true true +false true +false true +null +false true +true true +3 + +Dafny program verifier finished with 21 verified, 0 errors +x=13 y=14 +x=13 y=14 b=yes +true false +true false +map[12 := 6, 13 := 6, 14 := 7] +map[16 := 12, 17 := 13, 18 := 14] +XP returned: 0 false +after: 0 before: 0 +after: 2 before: 2 +XM returned: 16 +0 12 3 8 +0 12 3 8 +0 12 3 8 +0 12 3 8 +[1, 1, 1, 1] +[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] +[0, 1, 4, 9, 16, 25, 36, 49] +[0, 1, 2, 3, 4, 5, 6, 7] +4 4 2 2 0 4 2 true +5 4 2 2 0 4 2 true +5 5 3 3 1 5 3 true +2 2 1 1 0 2 false true +3 3 +451 +671 +221 +281 +1 +true false 1 +451 +671 +221 +281 +2 +true true false 2 +the intersection is {null} +there are 3 elements in the union +true true true +false false false +true true true +false false false +4 3 2 1 +true true true true +4 3 2 1 +true true true true +false false false false +false false false false +true true true true +false false false false +true true true true +false true +false true +null +false true +true true +3 + +Dafny program verifier finished with 21 verified, 0 errors +x=13 y=14 +x=13 y=14 b=yes +true false +true false +map[12 := 6, 13 := 6, 14 := 7] +map[16 := 12, 17 := 13, 18 := 14] +XP returned: 0 false +after: 0 before: 0 +after: 2 before: 2 +XM returned: 16 +0 12 3 8 +0 12 3 8 +0 12 3 8 +0 12 3 8 +[1, 1, 1, 1] +[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] +[0, 1, 4, 9, 16, 25, 36, 49] +[0, 1, 2, 3, 4, 5, 6, 7] +4 4 2 2 0 4 2 true +5 4 2 2 0 4 2 true +5 5 3 3 1 5 3 true +2 2 1 1 0 2 false true +3 3 +451 +671 +221 +281 +1 +true false 1 +451 +671 +221 +281 +2 +true true false 2 +the intersection is {null} +there are 3 elements in the union +true true true +false false false +true true true +false false false +4 3 2 1 +true true true true +4 3 2 1 +true true true true +false false false false +false false false false +true true true true +false false false false +true true true true +false true +false true +null +false true +true true +3 diff --git a/Test/comp/Comprehensions.dfy.java.expect b/Test/comp/Comprehensions.dfy.java.expect deleted file mode 100644 index 402bdc9456d..00000000000 --- a/Test/comp/Comprehensions.dfy.java.expect +++ /dev/null @@ -1,45 +0,0 @@ - -Dafny program verifier did not attempt verification -x=13 y=14 -x=13 y=14 b=yes -p=(13, 14) -q=(13, 14, [y, e, s]) -true false -true false -map[12 := 6, 13 := 6, 14 := 7] -map[16 := 12, 17 := 13, 18 := 14] -XP returned: 0 false -after: 0 before: 0 -after: 2 before: 2 -XM returned: 16 -0 12 3 8 -0 12 3 8 -0 12 3 8 -0 12 3 8 -[1, 1, 1, 1] -[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] -[0, 1, 4, 9, 16, 25, 36, 49] -[0, 1, 2, 3, 4, 5, 6, 7] -4 4 2 2 0 4 2 true -5 4 2 2 0 4 2 true -5 5 3 3 1 5 3 true -2 2 1 1 0 2 false true -3 3 -451 -671 -221 -281 -1 -true false 1 -451 -671 -221 -281 -2 -true true false 2 -the intersection is {null} -there are 3 elements in the union -true true true -false false false -true true true -false false false diff --git a/Test/comp/CovariantCollections.dfy b/Test/comp/CovariantCollections.dfy index 4d95797acfd..22ee046dbe3 100644 --- a/Test/comp/CovariantCollections.dfy +++ b/Test/comp/CovariantCollections.dfy @@ -1,3 +1,10 @@ +// RUN: %dafny /compile:0 "%s" > "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:cs "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { Sequences(); Sets(); diff --git a/Test/comp/CovariantCollections.dfy.expect b/Test/comp/CovariantCollections.dfy.expect index cdc87b9e162..afd1879d9a0 100644 --- a/Test/comp/CovariantCollections.dfy.expect +++ b/Test/comp/CovariantCollections.dfy.expect @@ -1,4 +1,168 @@ +Dafny program verifier finished with 25 verified, 0 errors + +Dafny program verifier did not attempt verification +Sequences: [] [17, 82, 17, 82] [12, 17] + cardinality: 0 4 2 + update: [42, 82, 17, 82] [42, 17] + index: 17 12 + subsequence ([lo..hi]): [82, 17] [17] + subsequence ([lo..]): [82, 17, 82] [17] + subsequence ([..hi]): [17, 82, 17] [12, 17] + subsequence ([..]): [] [17, 82, 17, 82] [12, 17] + concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] + prefix: true false true + proper prefix: true false false + membership: false true true +Sets: {} {17, 82} {12, 17} + cardinality: 0 2 2 + comprehension: {82} + union: {17, 82} {12, 17, 82} + intersection: {} {17} + difference: {} {82} + subset: true false true + proper subset: true false false + membership: false true true +Multisets: {} {17, 17, 82, 82} {12, 17} + cardinality: 0 4 2 + update: {17, 17, 42, 42, 42} {12, 17, 42} + multiplicity: 2 0 20 + union: {17, 17, 82, 82} {12, 17, 17, 17, 82, 82} + intersection: {} {17, 82, 82} + difference: {} {17, 82, 82} + subset: true false true + proper subset: true false false + membership: false true true +Maps: {} {12 := 17, 17 := 82, 82 := 17} {12 := 17, 17 := 17} + cardinality: 0 3 2 + update: {12 := 17, 17 := 82, 42 := 17, 82 := 17} {12 := 42, 17 := 17} + comprehension: {17 := 12, 82 := 12} + Keys: {12, 17, 82} + Values: {17, 82} + eq: false true true + eq: true true true true true true + eq: true true true true true true + eq: true false true false true false + eq: true false true false true false + eq: true true true true true true + eq: true true true true true true +set: {20, 30} +multiset: {20, 30} +seq: [20, 30] +map: {20 := 30, 30 := 20} +true +true +1 1 +1 1 + +Dafny program verifier did not attempt verification +Sequences: [] [17, 82, 17, 82] [12, 17] + cardinality: 0 4 2 + update: [42, 82, 17, 82] [42, 17] + index: 17 12 + subsequence ([lo..hi]): [82, 17] [17] + subsequence ([lo..]): [82, 17, 82] [17] + subsequence ([..hi]): [17, 82, 17] [12, 17] + subsequence ([..]): [] [17, 82, 17, 82] [12, 17] + concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] + prefix: true false true + proper prefix: true false false + membership: false true true +Sets: {} {17, 82} {12, 17} + cardinality: 0 2 2 + comprehension: {82} + union: {17, 82} {12, 17, 82} + intersection: {} {17} + difference: {} {82} + subset: true false true + proper subset: true false false + membership: false true true +Multisets: {} {17, 17, 82, 82} {12, 17} + cardinality: 0 4 2 + update: {17, 17, 42, 42, 42} {12, 17, 42} + multiplicity: 2 0 20 + union: {17, 17, 82, 82} {12, 17, 17, 17, 82, 82} + intersection: {} {17, 82, 82} + difference: {} {17, 82, 82} + subset: true false true + proper subset: true false false + membership: false true true +Maps: {} {12 := 17, 17 := 82, 82 := 17} {12 := 17, 17 := 17} + cardinality: 0 3 2 + update: {12 := 17, 17 := 82, 42 := 17, 82 := 17} {12 := 42, 17 := 17} + comprehension: {17 := 12, 82 := 12} + Keys: {12, 17, 82} + Values: {17, 82} + eq: false true true + eq: true true true true true true + eq: true true true true true true + eq: true false true false true false + eq: true false true false true false + eq: true true true true true true + eq: true true true true true true +set: {20, 30} +multiset: {20, 30} +seq: [20, 30] +map: {20 := 30, 30 := 20} +true +true +1 1 +1 1 + +Dafny program verifier did not attempt verification +Sequences: [] [17, 82, 17, 82] [12, 17] + cardinality: 0 4 2 + update: [42, 82, 17, 82] [42, 17] + index: 17 12 + subsequence ([lo..hi]): [82, 17] [17] + subsequence ([lo..]): [82, 17, 82] [17] + subsequence ([..hi]): [17, 82, 17] [12, 17] + subsequence ([..]): [] [17, 82, 17, 82] [12, 17] + concatenation: [17, 82, 17, 82] [17, 82, 17, 82, 12, 17] + prefix: true false true + proper prefix: true false false + membership: false true true +Sets: {} {17, 82} {12, 17} + cardinality: 0 2 2 + comprehension: {82} + union: {17, 82} {12, 17, 82} + intersection: {} {17} + difference: {} {82} + subset: true false true + proper subset: true false false + membership: false true true +Multisets: {} {17, 17, 82, 82} {12, 17} + cardinality: 0 4 2 + update: {17, 17, 42, 42, 42} {12, 17, 42} + multiplicity: 2 0 20 + union: {17, 17, 82, 82} {12, 17, 17, 17, 82, 82} + intersection: {} {17, 82, 82} + difference: {} {17, 82, 82} + subset: true false true + proper subset: true false false + membership: false true true +Maps: {} {12 := 17, 17 := 82, 82 := 17} {12 := 17, 17 := 17} + cardinality: 0 3 2 + update: {12 := 17, 17 := 82, 42 := 17, 82 := 17} {12 := 42, 17 := 17} + comprehension: {17 := 12, 82 := 12} + Keys: {12, 17, 82} + Values: {17, 82} + eq: false true true + eq: true true true true true true + eq: true true true true true true + eq: true false true false true false + eq: true false true false true false + eq: true true true true true true + eq: true true true true true true +set: {20, 30} +multiset: {20, 30} +seq: [20, 30] +map: {20 := 30, 30 := 20} +true +true +1 1 +1 1 + Dafny program verifier did not attempt verification Sequences: [] [17, 82, 17, 82] [12, 17] cardinality: 0 4 2 diff --git a/Test/comp/Dt.dfy b/Test/comp/Dt.dfy index b639e4ef3e5..aced23c719f 100644 --- a/Test/comp/Dt.dfy +++ b/Test/comp/Dt.dfy @@ -1,12 +1,9 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - java: - expected: - outputFile: Dt.dfy.java.expect - specialCaseReason: Set output is inconsistently ordered -*/ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + datatype List = Nil | Cons(head: int, tail: List) method Main() { diff --git a/Test/comp/Dt.dfy.expect b/Test/comp/Dt.dfy.expect index 4fff4226b5d..2a037c60b5d 100644 --- a/Test/comp/Dt.dfy.expect +++ b/Test/comp/Dt.dfy.expect @@ -1,5 +1,5 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 12 verified, 0 errors List.Nil List.Cons(5, List.Nil) (List.Nil, List.Cons(5, List.Nil)) @@ -17,3 +17,60 @@ CoBerry.Hjortron true true false false 42 q hello 1701 + +Dafny program verifier finished with 12 verified, 0 errors +List.Nil +List.Cons(5, List.Nil) +(List.Nil, List.Cons(5, List.Nil)) +(List.Cons(5, List.Nil), List.Nil) +() +5 List.Nil +List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Cons(4, List.Cons(5, List.Cons(6, List.Nil))))))) +0 + 1 + 2 + 3 + 4 + 5 + 6 == 21 (once more, that's 21) +List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Nil)))) +0 List.Cons(1, List.Cons(2, List.Cons(3, List.Nil))) +Stream.Next +Stream.Next +{Berry.Smultron, Berry.Jordgubb, Berry.Hallon} +CoBerry.Hjortron true true false +false +42 q hello +1701 + +Dafny program verifier finished with 12 verified, 0 errors +List.Nil +List.Cons(5, List.Nil) +(List.Nil, List.Cons(5, List.Nil)) +(List.Cons(5, List.Nil), List.Nil) +() +5 List.Nil +List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Cons(4, List.Cons(5, List.Cons(6, List.Nil))))))) +0 + 1 + 2 + 3 + 4 + 5 + 6 == 21 (once more, that's 21) +List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Nil)))) +0 List.Cons(1, List.Cons(2, List.Cons(3, List.Nil))) +Stream.Next +Stream.Next +{Berry.Smultron, Berry.Jordgubb, Berry.Hallon} +CoBerry.Hjortron true true false +false +42 q hello +1701 + +Dafny program verifier finished with 12 verified, 0 errors +List.Nil +List.Cons(5, List.Nil) +(List.Nil, List.Cons(5, List.Nil)) +(List.Cons(5, List.Nil), List.Nil) +() +5 List.Nil +List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Cons(4, List.Cons(5, List.Cons(6, List.Nil))))))) +0 + 1 + 2 + 3 + 4 + 5 + 6 == 21 (once more, that's 21) +List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Nil)))) +0 List.Cons(1, List.Cons(2, List.Cons(3, List.Nil))) +Stream.Next +Stream.Next +{Berry.Jordgubb, Berry.Smultron, Berry.Hallon} +CoBerry.Hjortron true true false +false +42 q hello +1701 diff --git a/Test/comp/Dt.dfy.java.expect b/Test/comp/Dt.dfy.java.expect deleted file mode 100644 index 34275817243..00000000000 --- a/Test/comp/Dt.dfy.java.expect +++ /dev/null @@ -1,19 +0,0 @@ - -Dafny program verifier did not attempt verification -List.Nil -List.Cons(5, List.Nil) -(List.Nil, List.Cons(5, List.Nil)) -(List.Cons(5, List.Nil), List.Nil) -() -5 List.Nil -List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Cons(4, List.Cons(5, List.Cons(6, List.Nil))))))) -0 + 1 + 2 + 3 + 4 + 5 + 6 == 21 (once more, that's 21) -List.Cons(0, List.Cons(1, List.Cons(2, List.Cons(3, List.Nil)))) -0 List.Cons(1, List.Cons(2, List.Cons(3, List.Nil))) -Stream.Next -Stream.Next -{Berry.Jordgubb, Berry.Smultron, Berry.Hallon} -CoBerry.Hjortron true true false -false -42 q hello -1701 diff --git a/Test/comp/Extern.dfy b/Test/comp/Extern.dfy index 634e42ee00c..f375937fe0a 100644 --- a/Test/comp/Extern.dfy +++ b/Test/comp/Extern.dfy @@ -1,24 +1,9 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - java: - otherFiles: - - AllDafny.java - - AllExtern.java - - LibClass.java - - Mixed.java - - OtherClass.java - cs: - otherFiles: - - Extern2.cs - js: - otherFiles: - - Extern3.js - go: - otherFiles: - - Extern4.go -*/ +// RUN: %dafny /compile:3 /compileTarget:cs "%s" Extern2.cs > "%t" +// RUN: %dafny /compile:3 /compileTarget:js "%s" Extern3.js >> "%t" +// RUN: %dafny /compile:3 /compileTarget:go "%s" Extern4.go >> "%t" +// RUN: %dafny /compile:3 /compileTarget:java "%s" LibClass.java OtherClass.java AllDafny.java Mixed.java AllExtern.java >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { print "Hello\n"; var x, y := Library.LibClass.CallMeInt(30); diff --git a/Test/comp/Extern.dfy.expect b/Test/comp/Extern.dfy.expect index 199c0c95eb7..a9bf891b291 100644 --- a/Test/comp/Extern.dfy.expect +++ b/Test/comp/Extern.dfy.expect @@ -1,5 +1,35 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 6 verified, 0 errors +Hello +31 62 45 OtherClass.CallMe +AllDafny.M +Extern static method says: Mixed.P +1001 +Extern instance method says: Mixed.IP +2002 +AllExtern.P + +Dafny program verifier finished with 6 verified, 0 errors +Hello +31 62 45 OtherClass.CallMe +AllDafny.M +Extern static method says: Mixed.P +1001 +Extern instance method says: Mixed.IP +2002 +AllExtern.P + +Dafny program verifier finished with 6 verified, 0 errors +Hello +31 62 45 OtherClass.CallMe +AllDafny.M +Extern static method says: Mixed.P +1001 +Extern instance method says: Mixed.IP +2002 +AllExtern.P + +Dafny program verifier finished with 6 verified, 0 errors Hello 31 62 45 OtherClass.CallMe AllDafny.M diff --git a/Test/comp/Class.java b/Test/comp/ExternCtors-externs/Class.java similarity index 100% rename from Test/comp/Class.java rename to Test/comp/ExternCtors-externs/Class.java diff --git a/Test/comp/ExternCtors.dfy b/Test/comp/ExternCtors.dfy index 0ca0a6eff83..35b6112bab6 100644 --- a/Test/comp/ExternCtors.dfy +++ b/Test/comp/ExternCtors.dfy @@ -1,30 +1,10 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - java: - otherFiles: - - Class.java - cs: - otherFiles: - - ExternCtors-externs/Library.cs - js: - otherFiles: - - ExternCtors-externs/Library.js - expected: - # The expected exit code should be 3, but /countVerificationErrors:0 is overriding that - # https://github.com/dafny-lang/dafny/issues/887 - outputFile: ~ - specialCaseReason: Extern constructors are currently broken in JavaScript - go: - otherFiles: - - ExternCtors-externs/Library.go - expected: - # The expected exit code should be 3, but /countVerificationErrors:0 is overriding that - # https://github.com/dafny-lang/dafny/issues/887 - outputFile: ~ - specialCaseReason: Extern constructors are currently broken in Go -*/ +// RUN: %dafny /compile:3 /compileTarget:cs "%s" ExternCtors-externs/Library.cs > "%t" +// RUN: %dafny /compile:3 /compileTarget:java "%s" ExternCtors-externs/Class.java >> "%t" +// RUN: %diff "%s.expect" "%t" + +// FIXME: Extern constructors are currently broken in Go and JavaScript, +// so they are omitted + method Main() { Library.Class.SayHi(); var obj := new Library.Class(42); diff --git a/Test/comp/ExternCtors.dfy.expect b/Test/comp/ExternCtors.dfy.expect index b661ce9e0a3..99ddf258037 100644 --- a/Test/comp/ExternCtors.dfy.expect +++ b/Test/comp/ExternCtors.dfy.expect @@ -1,4 +1,8 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 1 verified, 0 errors +Hello! +My value is 42 + +Dafny program verifier finished with 1 verified, 0 errors Hello! My value is 42 diff --git a/Test/comp/Forall.dfy b/Test/comp/Forall.dfy index 6c858e3591a..4c54ba101ad 100644 --- a/Test/comp/Forall.dfy +++ b/Test/comp/Forall.dfy @@ -1,3 +1,9 @@ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { var arrayTests := new ArrayTests(); arrayTests.Run(); diff --git a/Test/comp/Forall.dfy.expect b/Test/comp/Forall.dfy.expect index defe440e58d..b79a345bbc6 100644 --- a/Test/comp/Forall.dfy.expect +++ b/Test/comp/Forall.dfy.expect @@ -1,5 +1,137 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 25 verified, 0 errors +Arrays: Basic cases +[0, 1, 2, 3, 4] +[0, 1, 2, 3, 4] +[0, 1, 4, 3, 8] + +Arrays: Wrong index +[0, 0, 1, 2, 3] +[0, 0, 1, 2, 3] +[1, 2, 3, 4, 5] + +Arrays: Sequence conversion +[2, 1, 4, 5, 6] +[0, 3, 3, 3, 4] + +Arrays: Index collection +[1, 1, 2, 3, 4] +[1, 1, 2, 3, 4] + +Arrays: Functions +[1, 1, 3, 4, 5] +[1, 1, 3, 4, 5] +[1, 2, 3, 3, 5] +[1, 2, 3, 4, 5] + +Multi-dimensional arrays +[[0, 2, 4], [1, 3, 5], [2, 4, 6]] +[[0, 1, 2], [2, 3, 4], [4, 5, 6]] + +Objects: Basic cases +[(0, 1.0), (0, 2.0), (0, 3.0)] +[(2, 1.0), (2, 2.0), (4, 3.0)] +[(2, 1.0), (3, 2.0), (3, 3.0)] + +Objects: Bad field accesses +[(2, 1.0), (3, 2.0), (4, 3.0)] +[(2, 1.0), (2, 2.0), (2, 3.0)] +[(2, 1.0), (3, 2.0), (1, 3.0)] + +Objects: Functions +[(2, 1.0), (4, 2.0), (6, 3.0)] +[(2, 1.0), (3, 2.0), (3, 3.0)] +[(3, 1.0), (4, 2.0), (5, 3.0)] + +Dafny program verifier finished with 25 verified, 0 errors +Arrays: Basic cases +[0, 1, 2, 3, 4] +[0, 1, 2, 3, 4] +[0, 1, 4, 3, 8] + +Arrays: Wrong index +[0, 0, 1, 2, 3] +[0, 0, 1, 2, 3] +[1, 2, 3, 4, 5] + +Arrays: Sequence conversion +[2, 1, 4, 5, 6] +[0, 3, 3, 3, 4] + +Arrays: Index collection +[1, 1, 2, 3, 4] +[1, 1, 2, 3, 4] + +Arrays: Functions +[1, 1, 3, 4, 5] +[1, 1, 3, 4, 5] +[1, 2, 3, 3, 5] +[1, 2, 3, 4, 5] + +Multi-dimensional arrays +[[0, 2, 4], [1, 3, 5], [2, 4, 6]] +[[0, 1, 2], [2, 3, 4], [4, 5, 6]] + +Objects: Basic cases +[(0, 1.0), (0, 2.0), (0, 3.0)] +[(2, 1.0), (2, 2.0), (4, 3.0)] +[(2, 1.0), (3, 2.0), (3, 3.0)] + +Objects: Bad field accesses +[(2, 1.0), (3, 2.0), (4, 3.0)] +[(2, 1.0), (2, 2.0), (2, 3.0)] +[(2, 1.0), (3, 2.0), (1, 3.0)] + +Objects: Functions +[(2, 1.0), (4, 2.0), (6, 3.0)] +[(2, 1.0), (3, 2.0), (3, 3.0)] +[(3, 1.0), (4, 2.0), (5, 3.0)] + +Dafny program verifier finished with 25 verified, 0 errors +Arrays: Basic cases +[0, 1, 2, 3, 4] +[0, 1, 2, 3, 4] +[0, 1, 4, 3, 8] + +Arrays: Wrong index +[0, 0, 1, 2, 3] +[0, 0, 1, 2, 3] +[1, 2, 3, 4, 5] + +Arrays: Sequence conversion +[2, 1, 4, 5, 6] +[0, 3, 3, 3, 4] + +Arrays: Index collection +[1, 1, 2, 3, 4] +[1, 1, 2, 3, 4] + +Arrays: Functions +[1, 1, 3, 4, 5] +[1, 1, 3, 4, 5] +[1, 2, 3, 3, 5] +[1, 2, 3, 4, 5] + +Multi-dimensional arrays +[[0, 2, 4], [1, 3, 5], [2, 4, 6]] +[[0, 1, 2], [2, 3, 4], [4, 5, 6]] + +Objects: Basic cases +[(0, 1.0), (0, 2.0), (0, 3.0)] +[(2, 1.0), (2, 2.0), (4, 3.0)] +[(2, 1.0), (3, 2.0), (3, 3.0)] + +Objects: Bad field accesses +[(2, 1.0), (3, 2.0), (4, 3.0)] +[(2, 1.0), (2, 2.0), (2, 3.0)] +[(2, 1.0), (3, 2.0), (1, 3.0)] + +Objects: Functions +[(2, 1.0), (4, 2.0), (6, 3.0)] +[(2, 1.0), (3, 2.0), (3, 3.0)] +[(3, 1.0), (4, 2.0), (5, 3.0)] + +Dafny program verifier finished with 25 verified, 0 errors Arrays: Basic cases [0, 1, 2, 3, 4] [0, 1, 2, 3, 4] diff --git a/Test/comp/Ghosts.dfy b/Test/comp/Ghosts.dfy index c842b3bd8d8..11ed8478a6b 100644 --- a/Test/comp/Ghosts.dfy +++ b/Test/comp/Ghosts.dfy @@ -1,3 +1,9 @@ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method M1() returns (x: int) { print "hello, M1\n"; diff --git a/Test/comp/Ghosts.dfy.expect b/Test/comp/Ghosts.dfy.expect index 807ef099639..e584e53feb0 100644 --- a/Test/comp/Ghosts.dfy.expect +++ b/Test/comp/Ghosts.dfy.expect @@ -1,5 +1,41 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 8 verified, 0 errors +hello, M2 +hello, M2 +hello, M3 +hello, M1 +hello, M1 +hello, M1 +hello, M2 +hello, M3 +hello, N0 +hello, N1 + +Dafny program verifier finished with 8 verified, 0 errors +hello, M2 +hello, M2 +hello, M3 +hello, M1 +hello, M1 +hello, M1 +hello, M2 +hello, M3 +hello, N0 +hello, N1 + +Dafny program verifier finished with 8 verified, 0 errors +hello, M2 +hello, M2 +hello, M3 +hello, M1 +hello, M1 +hello, M1 +hello, M2 +hello, M3 +hello, N0 +hello, N1 + +Dafny program verifier finished with 8 verified, 0 errors hello, M2 hello, M2 hello, M3 diff --git a/Test/comp/GoModule.dfy b/Test/comp/GoModule.dfy index dde4b7f764e..6badfd10016 100644 --- a/Test/comp/GoModule.dfy +++ b/Test/comp/GoModule.dfy @@ -1,9 +1,6 @@ -/* ---- -!dafnyTestSpec -dafnyArguments: - compileTarget: go -*/ +// RUN: %dafny /compile:3 /spillTargetCode:2 "%s" /compileTarget:go > "%t" +// note: putting /compileTarget:go after "%s" overrides user-provided option +// RUN: %diff "%s.expect" "%t" // "url" is a built-in package, so it should be accessible to the // test suite without further requirements on the setup. diff --git a/Test/comp/Hello.dfy b/Test/comp/Hello.dfy index eb2ba3c7685..bcae164a158 100644 --- a/Test/comp/Hello.dfy +++ b/Test/comp/Hello.dfy @@ -1,3 +1,9 @@ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { print "Hello, JavaScript! Best, Dafny\n"; var x := 14; diff --git a/Test/comp/Hello.dfy.expect b/Test/comp/Hello.dfy.expect index 5fa80698081..7346deaafd9 100644 --- a/Test/comp/Hello.dfy.expect +++ b/Test/comp/Hello.dfy.expect @@ -1,5 +1,20 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 0 verified, 0 errors +Hello, JavaScript! Best, Dafny +x is 14 +y is false + +Dafny program verifier finished with 0 verified, 0 errors +Hello, JavaScript! Best, Dafny +x is 14 +y is false + +Dafny program verifier finished with 0 verified, 0 errors +Hello, JavaScript! Best, Dafny +x is 14 +y is false + +Dafny program verifier finished with 0 verified, 0 errors Hello, JavaScript! Best, Dafny x is 14 y is false diff --git a/Test/comp/Iterators.dfy b/Test/comp/Iterators.dfy index 055426f7eb7..48ed18f8355 100644 --- a/Test/comp/Iterators.dfy +++ b/Test/comp/Iterators.dfy @@ -1,13 +1,8 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - java: - expected: - exitCode: 134 - outputFile: ~ - specialCaseReason: Iterators are not implemented for Java -*/ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + class C { var x: int // for variety, the following tests the use of an instance Main method diff --git a/Test/comp/Iterators.dfy.expect b/Test/comp/Iterators.dfy.expect index 18a161c359c..22c894142fe 100644 --- a/Test/comp/Iterators.dfy.expect +++ b/Test/comp/Iterators.dfy.expect @@ -1,5 +1,17 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 5 verified, 0 errors +hello, instance +x is 0 +0.0 1.0 2.0 3.0 4.0 5.0 +0.0 1.0 2.0 + +Dafny program verifier finished with 5 verified, 0 errors +hello, instance +x is 0 +0.0 1.0 2.0 3.0 4.0 5.0 +0.0 1.0 2.0 + +Dafny program verifier finished with 5 verified, 0 errors hello, instance x is 0 0.0 1.0 2.0 3.0 4.0 5.0 diff --git a/Test/comp/JsModule.dfy b/Test/comp/JsModule.dfy index 769d0d57284..b4ddda74e73 100644 --- a/Test/comp/JsModule.dfy +++ b/Test/comp/JsModule.dfy @@ -1,9 +1,6 @@ -/* ---- -!dafnyTestSpec -dafnyArguments: - compileTarget: js -*/ +// RUN: %dafny /compile:3 "%s" /compileTarget:js > "%t" +// note: putting /compileTarget:js after "%s" overrides user-provided option +// RUN: %diff "%s.expect" "%t" // "url" is a built-in package in node, so it should be accessible to the // test suite without further requirements on the setup. diff --git a/Test/comp/Let.dfy b/Test/comp/Let.dfy index 12685117927..521a5e06659 100644 --- a/Test/comp/Let.dfy +++ b/Test/comp/Let.dfy @@ -1,3 +1,9 @@ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method M() returns (x: int) { x := var y := 50; y; // non-top-level let } diff --git a/Test/comp/Let.dfy.expect b/Test/comp/Let.dfy.expect index d16cc4276a4..5e1d0433456 100644 --- a/Test/comp/Let.dfy.expect +++ b/Test/comp/Let.dfy.expect @@ -1,5 +1,26 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 3 verified, 0 errors +50 58 +Tree.Node(Tree.Node(Tree.Leaf, 5, Tree.Node(Tree.Leaf, 7, Tree.Leaf)), 9, Tree.Node(Tree.Leaf, 10, Tree.Node(Tree.Leaf, 12, Tree.Node(Tree.Leaf, 30, Tree.Leaf)))) +5 7 9 10 12 30 +5 7 +5 7 + +Dafny program verifier finished with 3 verified, 0 errors +50 58 +Tree.Node(Tree.Node(Tree.Leaf, 5, Tree.Node(Tree.Leaf, 7, Tree.Leaf)), 9, Tree.Node(Tree.Leaf, 10, Tree.Node(Tree.Leaf, 12, Tree.Node(Tree.Leaf, 30, Tree.Leaf)))) +5 7 9 10 12 30 +5 7 +5 7 + +Dafny program verifier finished with 3 verified, 0 errors +50 58 +Tree.Node(Tree.Node(Tree.Leaf, 5, Tree.Node(Tree.Leaf, 7, Tree.Leaf)), 9, Tree.Node(Tree.Leaf, 10, Tree.Node(Tree.Leaf, 12, Tree.Node(Tree.Leaf, 30, Tree.Leaf)))) +5 7 9 10 12 30 +5 7 +5 7 + +Dafny program verifier finished with 3 verified, 0 errors 50 58 Tree.Node(Tree.Node(Tree.Leaf, 5, Tree.Node(Tree.Leaf, 7, Tree.Leaf)), 9, Tree.Node(Tree.Leaf, 10, Tree.Node(Tree.Leaf, 12, Tree.Node(Tree.Leaf, 30, Tree.Leaf)))) 5 7 9 10 12 30 diff --git a/Test/comp/Module.dfy b/Test/comp/Module.dfy index 4987cf6bd01..909886f9ac5 100644 --- a/Test/comp/Module.dfy +++ b/Test/comp/Module.dfy @@ -1,3 +1,9 @@ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + // Simple sanity test of nested modules module Parent { module Child { diff --git a/Test/comp/Module.dfy.expect b/Test/comp/Module.dfy.expect index 300e4a642a0..5a18e86f45c 100644 --- a/Test/comp/Module.dfy.expect +++ b/Test/comp/Module.dfy.expect @@ -1,3 +1,12 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 1 verified, 0 errors +hi from a nested module + +Dafny program verifier finished with 1 verified, 0 errors +hi from a nested module + +Dafny program verifier finished with 1 verified, 0 errors +hi from a nested module + +Dafny program verifier finished with 1 verified, 0 errors hi from a nested module diff --git a/Test/comp/NativeNumbers.dfy b/Test/comp/NativeNumbers.dfy index cf9b142bf25..bbf449cbdbd 100644 --- a/Test/comp/NativeNumbers.dfy +++ b/Test/comp/NativeNumbers.dfy @@ -1,12 +1,10 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - js: - expected: - outputFile: NativeNumbers.dfy.js.expect - specialCaseReason: :nativeType doesn't work correctly for JavaScript -*/ +// Skip JavaScript because JavaScript doesn't have the same native types + +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { CastTests(); DefaultTests(); diff --git a/Test/comp/NativeNumbers.dfy.expect b/Test/comp/NativeNumbers.dfy.expect index 2132aae876a..bdadd002058 100644 --- a/Test/comp/NativeNumbers.dfy.expect +++ b/Test/comp/NativeNumbers.dfy.expect @@ -1,5 +1,173 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 25 verified, 0 errors +Casting: + +Small numbers: +0 0 0 0 0 0 0 0 0 +1 1 1 1 1 1 1 1 1 +2 2 2 2 2 2 2 2 2 +3 3 3 3 3 3 3 3 3 +4 4 4 4 4 4 4 4 4 +5 5 5 5 5 5 5 5 5 +6 6 6 6 6 6 6 6 6 +7 7 7 7 7 7 7 7 7 +8 8 8 8 8 8 8 8 8 + +Large unsigned numbers: +255 255 255 255 255 255 255 255 +65535 65535 65535 65535 65535 65535 +4294967295 4294967295 4294967295 4294967295 +18446744073709551615 18446744073709551615 + +Int to large unsigned: +255 65535 4294967295 18446744073709551615 + +Cast from cardinality operator: +0 0 0 0 0 0 0 0 0 +1 1 1 1 1 1 1 1 1 +2 2 2 2 2 2 2 2 2 +3 3 3 3 3 3 3 3 3 +4 4 4 4 4 4 4 4 4 + +Characters: +C 67 67 67 67 67 67 67 67 67 +0 127 32767 65535 65535 255 65535 65535 65535 + +Defaults: + +0 0 0 0 0 0 0 0 0 + +Byte arithmetic: + +20 30 +10 10 18 + +Short arithmetic: + +20 30 +10 10 18 + +Bitvectors: + +0 3 4 1 + +Comparison to zero: + +int8: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +int16: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +int32: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +int64: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +uint8: +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +uint16: +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +uint32: +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +uint64: +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N + + +Dafny program verifier finished with 25 verified, 0 errors +Casting: + +Small numbers: +0 0 0 0 0 0 0 0 0 +1 1 1 1 1 1 1 1 1 +2 2 2 2 2 2 2 2 2 +3 3 3 3 3 3 3 3 3 +4 4 4 4 4 4 4 4 4 +5 5 5 5 5 5 5 5 5 +6 6 6 6 6 6 6 6 6 +7 7 7 7 7 7 7 7 7 +8 8 8 8 8 8 8 8 8 + +Large unsigned numbers: +255 255 255 255 255 255 255 255 +65535 65535 65535 65535 65535 65535 +4294967295 4294967295 4294967295 4294967295 +18446744073709551615 18446744073709551615 + +Int to large unsigned: +255 65535 4294967295 18446744073709551615 + +Cast from cardinality operator: +0 0 0 0 0 0 0 0 0 +1 1 1 1 1 1 1 1 1 +2 2 2 2 2 2 2 2 2 +3 3 3 3 3 3 3 3 3 +4 4 4 4 4 4 4 4 4 + +Characters: +C 67 67 67 67 67 67 67 67 67 +0 127 32767 65535 65535 255 65535 65535 65535 + +Defaults: + +0 0 0 0 0 0 0 0 0 + +Byte arithmetic: + +20 30 +10 10 18 + +Short arithmetic: + +20 30 +10 10 18 + +Bitvectors: + +0 3 4 1 + +Comparison to zero: + +int8: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +int16: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +int32: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +int64: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +uint8: +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +uint16: +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +uint32: +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +uint64: +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N + + +Dafny program verifier finished with 25 verified, 0 errors Casting: Small numbers: diff --git a/Test/comp/NativeNumbers.dfy.js.expect b/Test/comp/NativeNumbers.dfy.js.expect deleted file mode 100644 index 0645aac4dee..00000000000 --- a/Test/comp/NativeNumbers.dfy.js.expect +++ /dev/null @@ -1,9 +0,0 @@ -NativeNumbers.dfy(19,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(20,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(21,28): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(22,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(24,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(25,31): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(26,29): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -NativeNumbers.dfy(27,30): Error: None of the types given in :nativeType arguments is supported by the current compilation target. Try supplying others. -8 resolution/type errors detected in NativeNumbers.dfy diff --git a/Test/comp/Numbers.dfy b/Test/comp/Numbers.dfy index 944b7e2e462..1b7888a40e1 100644 --- a/Test/comp/Numbers.dfy +++ b/Test/comp/Numbers.dfy @@ -1,12 +1,10 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - go: - expected: - outputFile: Numbers.dfy.go.expect - specialCaseReason: Inconsistent printing of rationals -*/ +// RUN: %dafny /compile:0 "%s" > "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:cs "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { Literals(); Arithmetic(); diff --git a/Test/comp/Numbers.dfy.expect b/Test/comp/Numbers.dfy.expect index 24dff607160..43f392e399b 100644 --- a/Test/comp/Numbers.dfy.expect +++ b/Test/comp/Numbers.dfy.expect @@ -1,4 +1,330 @@ +Dafny program verifier finished with 56 verified, 0 errors + +Dafny program verifier did not attempt verification +0 +0 +3 +-5 +2147483647 (aka C# int.MaxValue) +2147483648 (aka 2^31) +2147483649 +4294967295 (aka C# uint.MaxValue) +4294967296 (aka 2^32) +590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) +590295810358705651712 (aka 2^53) +-590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) +-590295810358705651712 +9223372036854775807 (aka C# long.MaxValue) +9223372036854775808 (aka 2^63) +9223372036854775809 +18446744073709551615 (aka C# ulong.MaxValue) +18446744073709551616 (aka 2^64) +1267650600228229401496703205376 (aka 2^100) +170141183460469231731687303715884105727 (aka M_39) +0.0 +0.0 +3.0 +-5.0 +3.14 +-2.71 +1000000000.0 (aka a billion) +0.0000000000667408 (aka G) +0.0000000000667408 (aka G) +0.0000000000667408 (aka G) +17014118346046923173168730371588410572.7 (aka 1/10 of M_39) +0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 (aka 1/googol) +35 27 -27 +124 -124 +-124 124 +g Q 2 +7 3 31 +-8 1 -31 +-7 3 31 +8 1 -31 +0 0 0 +0 0 0 +uint8: 10:1 0:231 +uint16: 2848:7 0:65511 +uint32: 186737707:10 0:4294967271 +uint64: 802032351030850069:4 0:18446744073709551591 +via real: 7:12 -8:1 -7:12 8:1 +int: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int8: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int16: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int32: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int64: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +0.2 +35.2 27.2 -27.2 +124.8 -124.8 +-124.8 124.8 +(312.0 / 40.0) (1248.0 / 40.0) +(-312.0 / 40.0) (-1248.0 / 40.0) +(-312.0 / 40.0) (1248.0 / 40.0) +(312.0 / 40.0) (-1248.0 / 40.0) +0.0 0.0 0.0 +120.0 120.0 (24.0 / 3.0) (-24.0 / 3.0) +123.4567 -123.4567 0.1234 -0.1234 (24.0 / 30.0) +0.2 0.02 0.00002 +-0.2 -0.02 -0.00002 +(20.0 / 3.0) (-20.0 / 3.0) (-20.0 / 3.0) (20.0 / 3.0) +0.0 0.0 0.81 0.81 (8100.0 / 8100.0) +true false +0 2 2 2 +0 3 6 46 +0 1 36 2116 +0 0 0 0 +29 2 2 4294967283 2 +29 2 2 9007199254740979 2 +0 0 0 9 +0 0 0 9 +0 0 0 9 +0 4294967295 1267650600228229401496703205375 6 +8 4 +18 72 4 +0 +200 300 +100 100 18 +0: IsNat: true, Offset: 0, IsLimit: true, IsSucc: false +1: IsNat: true, Offset: 1, IsLimit: false, IsSucc: true +42: IsNat: true, Offset: 42, IsLimit: false, IsSucc: true +int: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +MyNumber: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +120 120.0 120 120 120 x +120 120.0 120 120 120 x +120 120 +120 120.0 120 120 120 x +120 120.0 120 120 120 x +120 120 +false true false true +true false true false +false true false true +true false true false + +Dafny program verifier did not attempt verification +0 +0 +3 +-5 +2147483647 (aka C# int.MaxValue) +2147483648 (aka 2^31) +2147483649 +4294967295 (aka C# uint.MaxValue) +4294967296 (aka 2^32) +590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) +590295810358705651712 (aka 2^53) +-590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) +-590295810358705651712 +9223372036854775807 (aka C# long.MaxValue) +9223372036854775808 (aka 2^63) +9223372036854775809 +18446744073709551615 (aka C# ulong.MaxValue) +18446744073709551616 (aka 2^64) +1267650600228229401496703205376 (aka 2^100) +170141183460469231731687303715884105727 (aka M_39) +0.0 +0.0 +3.0 +-5.0 +3.14 +-2.71 +1000000000.0 (aka a billion) +0.0000000000667408 (aka G) +0.0000000000667408 (aka G) +0.0000000000667408 (aka G) +17014118346046923173168730371588410572.7 (aka 1/10 of M_39) +0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 (aka 1/googol) +35 27 -27 +124 -124 +-124 124 +g Q 2 +7 3 31 +-8 1 -31 +-7 3 31 +8 1 -31 +0 0 0 +0 0 0 +uint8: 10:1 0:231 +uint16: 2848:7 0:65511 +uint32: 186737707:10 0:4294967271 +uint64: 802032351030850069:4 0:18446744073709551591 +via real: 7:12 -8:1 -7:12 8:1 +int: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int8: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int16: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int32: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int64: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +0.2 +35.2 27.2 -27.2 +124.8 -124.8 +-124.8 124.8 +(312.0 / 40.0) (1248.0 / 40.0) +(-312.0 / 40.0) (-1248.0 / 40.0) +(-312.0 / 40.0) (1248.0 / 40.0) +(312.0 / 40.0) (-1248.0 / 40.0) +0.0 0.0 0.0 +120.0 120.0 (24.0 / 3.0) (-24.0 / 3.0) +123.4567 -123.4567 0.1234 -0.1234 (24.0 / 30.0) +0.2 0.02 0.00002 +-0.2 -0.02 -0.00002 +(20.0 / 3.0) (-20.0 / 3.0) (-20.0 / 3.0) (20.0 / 3.0) +0.0 0.0 0.81 0.81 (8100.0 / 8100.0) +true false +0 2 2 2 +0 3 6 46 +0 1 36 2116 +0 0 0 0 +29 2 2 4294967283 2 +29 2 2 9007199254740979 2 +0 0 0 9 +0 0 0 9 +0 0 0 9 +0 4294967295 1267650600228229401496703205375 6 +8 4 +18 72 4 +0 +200 300 +100 100 18 +0: IsNat: true, Offset: 0, IsLimit: true, IsSucc: false +1: IsNat: true, Offset: 1, IsLimit: false, IsSucc: true +42: IsNat: true, Offset: 42, IsLimit: false, IsSucc: true +int: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +MyNumber: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +120 120.0 120 120 120 x +120 120.0 120 120 120 x +120 120 +120 120.0 120 120 120 x +120 120.0 120 120 120 x +120 120 +false true false true +true false true false +false true false true +true false true false + +Dafny program verifier did not attempt verification +0 +0 +3 +-5 +2147483647 (aka C# int.MaxValue) +2147483648 (aka 2^31) +2147483649 +4294967295 (aka C# uint.MaxValue) +4294967296 (aka 2^32) +590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) +590295810358705651712 (aka 2^53) +-590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) +-590295810358705651712 +9223372036854775807 (aka C# long.MaxValue) +9223372036854775808 (aka 2^63) +9223372036854775809 +18446744073709551615 (aka C# ulong.MaxValue) +18446744073709551616 (aka 2^64) +1267650600228229401496703205376 (aka 2^100) +170141183460469231731687303715884105727 (aka M_39) +0.0 +0.0 +3.0 +-5.0 +3.14 +-2.71 +1000000000.0 (aka a billion) +0.0000000000667408 (aka G) +0.0000000000667408 (aka G) +0.0000000000667408 (aka G) +17014118346046923173168730371588410572.7 (aka 1/10 of M_39) +0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 (aka 1/googol) +35 27 -27 +124 -124 +-124 124 +g Q 2 +7 3 31 +-8 1 -31 +-7 3 31 +8 1 -31 +0 0 0 +0 0 0 +uint8: 10:1 0:231 +uint16: 2848:7 0:65511 +uint32: 186737707:10 0:4294967271 +uint64: 802032351030850069:4 0:18446744073709551591 +via real: 7:12 -8:1 -7:12 8:1 +int: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int8: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int16: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int32: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +int64: 7:12 -8:1 -7:12 8:1 -12:0 12:0 +0.2 +35.2 27.2 -27.2 +124.8 -124.8 +-124.8 124.8 +7.8 31.2 +-7.8 -31.2 +-7.8 31.2 +7.8 -31.2 +0.0 0.0 0.0 +120.0 120.0 8.0 -8.0 +123.4567 -123.4567 0.1234 -0.1234 0.8 +0.2 0.02 0.00002 +-0.2 -0.02 -0.00002 +(20.0 / 3.0) (-20.0 / 3.0) (-20.0 / 3.0) (20.0 / 3.0) +0.0 0.0 0.81 0.81 1.0 +true false +0 2 2 2 +0 3 6 46 +0 1 36 2116 +0 0 0 0 +29 2 2 4294967283 2 +29 2 2 9007199254740979 2 +0 0 0 9 +0 0 0 9 +0 0 0 9 +0 4294967295 1267650600228229401496703205375 6 +8 4 +18 72 4 +0 +200 300 +100 100 18 +0: IsNat: true, Offset: 0, IsLimit: true, IsSucc: false +1: IsNat: true, Offset: 1, IsLimit: false, IsSucc: true +42: IsNat: true, Offset: 42, IsLimit: false, IsSucc: true +int: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +MyNumber: +-42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y +23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N +120 120.0 120 120 120 x +120 120.0 120 120 120 x +120 120 +120 120.0 120 120 120 x +120 120.0 120 120 120 x +120 120 +false true false true +true false true false +false true false true +true false true false + Dafny program verifier did not attempt verification 0 0 diff --git a/Test/comp/Numbers.dfy.go.expect b/Test/comp/Numbers.dfy.go.expect deleted file mode 100644 index d6fc8c8882d..00000000000 --- a/Test/comp/Numbers.dfy.go.expect +++ /dev/null @@ -1,102 +0,0 @@ - -Dafny program verifier did not attempt verification -0 -0 -3 --5 -2147483647 (aka C# int.MaxValue) -2147483648 (aka 2^31) -2147483649 -4294967295 (aka C# uint.MaxValue) -4294967296 (aka 2^32) -590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) -590295810358705651712 (aka 2^53) --590295810358705651711 (aka JavaScript Number.MAX_SAFE_INTEGER) --590295810358705651712 -9223372036854775807 (aka C# long.MaxValue) -9223372036854775808 (aka 2^63) -9223372036854775809 -18446744073709551615 (aka C# ulong.MaxValue) -18446744073709551616 (aka 2^64) -1267650600228229401496703205376 (aka 2^100) -170141183460469231731687303715884105727 (aka M_39) -0.0 -0.0 -3.0 --5.0 -3.14 --2.71 -1000000000.0 (aka a billion) -0.0000000000667408 (aka G) -0.0000000000667408 (aka G) -0.0000000000667408 (aka G) -17014118346046923173168730371588410572.7 (aka 1/10 of M_39) -0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001 (aka 1/googol) -35 27 -27 -124 -124 --124 124 -g Q 2 -7 3 31 --8 1 -31 --7 3 31 -8 1 -31 -0 0 0 -0 0 0 -uint8: 10:1 0:231 -uint16: 2848:7 0:65511 -uint32: 186737707:10 0:4294967271 -uint64: 802032351030850069:4 0:18446744073709551591 -via real: 7:12 -8:1 -7:12 8:1 -int: 7:12 -8:1 -7:12 8:1 -12:0 12:0 -int8: 7:12 -8:1 -7:12 8:1 -12:0 12:0 -int16: 7:12 -8:1 -7:12 8:1 -12:0 12:0 -int32: 7:12 -8:1 -7:12 8:1 -12:0 12:0 -int64: 7:12 -8:1 -7:12 8:1 -12:0 12:0 -0.2 -35.2 27.2 -27.2 -124.8 -124.8 --124.8 124.8 -7.8 31.2 --7.8 -31.2 --7.8 31.2 -7.8 -31.2 -0.0 0.0 0.0 -120.0 120.0 8.0 -8.0 -123.4567 -123.4567 0.1234 -0.1234 0.8 -0.2 0.02 0.00002 --0.2 -0.02 -0.00002 -(20.0 / 3.0) (-20.0 / 3.0) (-20.0 / 3.0) (20.0 / 3.0) -0.0 0.0 0.81 0.81 1.0 -true false -0 2 2 2 -0 3 6 46 -0 1 36 2116 -0 0 0 0 -29 2 2 4294967283 2 -29 2 2 9007199254740979 2 -0 0 0 9 -0 0 0 9 -0 0 0 9 -0 4294967295 1267650600228229401496703205375 6 -8 4 -18 72 4 -0 -200 300 -100 100 18 -0: IsNat: true, Offset: 0, IsLimit: true, IsSucc: false -1: IsNat: true, Offset: 1, IsLimit: false, IsSucc: true -42: IsNat: true, Offset: 42, IsLimit: false, IsSucc: true -int: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -MyNumber: --42 <0 Y <=0 Y ==0 N !=0 Y >0 N >=0 N 0< N 0<= N 0== N 0!= Y 0> Y 0>= Y -0 <0 N <=0 Y ==0 Y !=0 N >0 N >=0 Y 0< N 0<= Y 0== Y 0!= N 0> N 0>= Y -23 <0 N <=0 N ==0 N !=0 Y >0 Y >=0 Y 0< Y 0<= Y 0== N 0!= Y 0> N 0>= N -120 120.0 120 120 120 x -120 120.0 120 120 120 x -120 120 -120 120.0 120 120 120 x -120 120.0 120 120 120 x -120 120 diff --git a/Test/comp/Poly.dfy b/Test/comp/Poly.dfy index 66508645025..17ac3ca0193 100644 --- a/Test/comp/Poly.dfy +++ b/Test/comp/Poly.dfy @@ -1,12 +1,10 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - go: - expected: - outputFile: Poly.dfy.go.expect - specialCaseReason: Inconsistent printing of rationals -*/ +// RUN: %dafny /compile:0 "%s" > "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:cs "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + trait Shape { function method Center(): (real, real) reads this method PrintCenter() { diff --git a/Test/comp/Poly.dfy.expect b/Test/comp/Poly.dfy.expect index 4c328cf0c4a..498c7152718 100644 --- a/Test/comp/Poly.dfy.expect +++ b/Test/comp/Poly.dfy.expect @@ -1,4 +1,6 @@ +Dafny program verifier finished with 15 verified, 0 errors + Dafny program verifier did not attempt verification Center of square: ((1.0 / 2.0), (1.0 / 2.0)) Center of circle: (1.0, 1.0) @@ -12,3 +14,45 @@ Center: ((1.0 / 2.0), (1.0 / 2.0)) Center: (1.0, 1.0) Center: ((1.0 / 2.0), (1.0 / 2.0)) Center: (1.0, 1.0) + +Dafny program verifier did not attempt verification +Center of square: ((1.0 / 2.0), (1.0 / 2.0)) +Center of circle: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) + +Dafny program verifier did not attempt verification +Center of square: ((1.0 / 2.0), (1.0 / 2.0)) +Center of circle: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) +Center: ((1.0 / 2.0), (1.0 / 2.0)) +Center: (1.0, 1.0) + +Dafny program verifier did not attempt verification +Center of square: (0.5, 0.5) +Center of circle: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) +Center: (0.5, 0.5) +Center: (1.0, 1.0) diff --git a/Test/comp/Poly.dfy.go.expect b/Test/comp/Poly.dfy.go.expect deleted file mode 100644 index 054f2188473..00000000000 --- a/Test/comp/Poly.dfy.go.expect +++ /dev/null @@ -1,14 +0,0 @@ - -Dafny program verifier did not attempt verification -Center of square: (0.5, 0.5) -Center of circle: (1.0, 1.0) -Center: (0.5, 0.5) -Center: (1.0, 1.0) -Center: (0.5, 0.5) -Center: (1.0, 1.0) -Center: (0.5, 0.5) -Center: (1.0, 1.0) -Center: (0.5, 0.5) -Center: (1.0, 1.0) -Center: (0.5, 0.5) -Center: (1.0, 1.0) diff --git a/Test/comp/StaticMembersOfGenericTypes.dfy b/Test/comp/StaticMembersOfGenericTypes.dfy index 58b1e68a0c5..c91977b787b 100644 --- a/Test/comp/StaticMembersOfGenericTypes.dfy +++ b/Test/comp/StaticMembersOfGenericTypes.dfy @@ -1,3 +1,9 @@ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { GenericClass(); FunctionValues(); diff --git a/Test/comp/StaticMembersOfGenericTypes.dfy.expect b/Test/comp/StaticMembersOfGenericTypes.dfy.expect index 44ea117e23a..408d179dfe2 100644 --- a/Test/comp/StaticMembersOfGenericTypes.dfy.expect +++ b/Test/comp/StaticMembersOfGenericTypes.dfy.expect @@ -1,5 +1,59 @@ -Dafny program verifier did not attempt verification +Dafny program verifier finished with 9 verified, 0 errors +(0, 1) (true, 2, 3) +0 25 (20, 21) (20, 21) 23 22 +0 25 (20, 21) (20, 21) 23 22 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +0 25 (20, 21) (20, 21) 23 22 +0 25 (20, 21) (20, 21) 23 22 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +(2.0, true) (3.0, false) +(2.0, true) (3.0, false) +true false +(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) +(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) +50 3 4 +149 + +Dafny program verifier finished with 9 verified, 0 errors +(0, 1) (true, 2, 3) +0 25 (20, 21) (20, 21) 23 22 +0 25 (20, 21) (20, 21) 23 22 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +0 25 (20, 21) (20, 21) 23 22 +0 25 (20, 21) (20, 21) 23 22 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +(2.0, true) (3.0, false) +(2.0, true) (3.0, false) +true false +(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) +(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) +50 3 4 +149 + +Dafny program verifier finished with 9 verified, 0 errors +(0, 1) (true, 2, 3) +0 25 (20, 21) (20, 21) 23 22 +0 25 (20, 21) (20, 21) 23 22 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +0 25 (20, 21) (20, 21) 23 22 +0 25 (20, 21) (20, 21) 23 22 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +0 25 (true, 20, 21) (true, 20, 21) true 22 23 +(2.0, true) (3.0, false) +(2.0, true) (3.0, false) +true false +(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) +(5, 2.0, true) (5, 2.0, true) (6, 3.0, false) +50 3 4 +149 + +Dafny program verifier finished with 9 verified, 0 errors (0, 1) (true, 2, 3) 0 25 (20, 21) (20, 21) 23 22 0 25 (20, 21) (20, 21) 23 22 diff --git a/Test/comp/TailRecursion.dfy b/Test/comp/TailRecursion.dfy index 8da90c04fba..541884ca6d1 100644 --- a/Test/comp/TailRecursion.dfy +++ b/Test/comp/TailRecursion.dfy @@ -1,3 +1,10 @@ +// RUN: %dafny /compile:0 "%s" > "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:cs "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { // In the following, 2_000_000 is too large an argument without tail-calls var x := M(2_000_000, 0); diff --git a/Test/comp/TailRecursion.dfy.expect b/Test/comp/TailRecursion.dfy.expect index c53f69551a9..053ffdca6c2 100644 --- a/Test/comp/TailRecursion.dfy.expect +++ b/Test/comp/TailRecursion.dfy.expect @@ -1,4 +1,57 @@ +Dafny program verifier finished with 27 verified, 0 errors + +Dafny program verifier did not attempt verification +2000000 2000000 +2000000 2000000 +1000000 1000000 +10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 == 55 +TriangleNumber(10) = 55 +TriangleNumber_Real(10) = 55.0 +TriangleNumber_ORDINAL(10) = 55 +Factorial(5) = 120 +Union(8) = [0, 1, 2, 3, 4, 5, 6, 7, 8] +UpTo(10) = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +DownFrom(10) = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] +Sum(List.Cons(100, List.Cons(40, List.Cons(60, List.Nil)))) = 200 +TheBigSubtract(100) = -12 +TailNat(10) = 50 +17 17 17 + +Dafny program verifier did not attempt verification +2000000 2000000 +2000000 2000000 +1000000 1000000 +10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 == 55 +TriangleNumber(10) = 55 +TriangleNumber_Real(10) = 55.0 +TriangleNumber_ORDINAL(10) = 55 +Factorial(5) = 120 +Union(8) = [0, 1, 2, 3, 4, 5, 6, 7, 8] +UpTo(10) = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +DownFrom(10) = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] +Sum(List.Cons(100, List.Cons(40, List.Cons(60, List.Nil)))) = 200 +TheBigSubtract(100) = -12 +TailNat(10) = 50 +17 17 17 + +Dafny program verifier did not attempt verification +2000000 2000000 +2000000 2000000 +1000000 1000000 +10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 == 55 +TriangleNumber(10) = 55 +TriangleNumber_Real(10) = 55.0 +TriangleNumber_ORDINAL(10) = 55 +Factorial(5) = 120 +Union(8) = [0, 1, 2, 3, 4, 5, 6, 7, 8] +UpTo(10) = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +DownFrom(10) = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] +Sum(List.Cons(100, List.Cons(40, List.Cons(60, List.Nil)))) = 200 +TheBigSubtract(100) = -12 +TailNat(10) = 50 +17 17 17 + Dafny program verifier did not attempt verification 2000000 2000000 2000000 2000000 diff --git a/Test/comp/TypeParams.dfy b/Test/comp/TypeParams.dfy index c478e5fcbe6..48a23b12045 100644 --- a/Test/comp/TypeParams.dfy +++ b/Test/comp/TypeParams.dfy @@ -1,24 +1,9 @@ -/* ---- -!dafnyTestSpec -compileTargetOverrides: - cs: - expected: - outputFile: TypeParams.dfy.cs.expect - specialCaseReason: Function printing is inconsistent - java: - expected: - outputFile: TypeParams.dfy.java.expect - specialCaseReason: Function printing is inconsistent - js: - expected: - outputFile: TypeParams.dfy.js.expect - specialCaseReason: Function printing is inconsistent - go: - expected: - outputFile: TypeParams.dfy.go.expect - specialCaseReason: Function printing is inconsistent -*/ +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + datatype Color = Orange | Pink | Teal type Six = x | x <= 6 newtype Even = x | x % 2 == 0 diff --git a/Test/comp/TypeParams.dfy.cs.expect b/Test/comp/TypeParams.dfy.cs.expect deleted file mode 100644 index 85f574c09d1..00000000000 --- a/Test/comp/TypeParams.dfy.cs.expect +++ /dev/null @@ -1,21 +0,0 @@ - -Dafny program verifier did not attempt verification -0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) -null null null null null -{} multiset{} [] map[] {} map[] -null null null null -0 [] Q {} 0 Dt.D0(0) -false Color.Orange 42 {} false Dt.D0(false) -NonemptyList.Atom(0) -6 7 8 9 -true true -IList.INil IList.ICons -IList.INil IList.INil Stream.Next IList.INil -NonemptyList.Atom(0) -NonemptyCoList.CoAtom -Color.Pink 19 -3 -null null null null -System.Func`2[Dafny.BigRational,System.Boolean] -System.Func`1[System.Numerics.BigInteger] -System.Func`3[_module.Color,Dafny.ISet`1[System.UInt16],System.UInt32] -0 0 diff --git a/Test/comp/TypeParams.dfy.expect b/Test/comp/TypeParams.dfy.expect new file mode 100644 index 00000000000..aceea6b184a --- /dev/null +++ b/Test/comp/TypeParams.dfy.expect @@ -0,0 +1,84 @@ + +Dafny program verifier finished with 20 verified, 0 errors +0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) +null null null null null +{} multiset{} [] map[] {} map[] +null null null null +0 [] Q {} 0 Dt.D0(0) +false Color.Orange 42 {} false Dt.D0(false) +NonemptyList.Atom(0) +6 7 8 9 +true true +IList.INil IList.ICons +IList.INil IList.INil Stream.Next IList.INil +NonemptyList.Atom(0) +NonemptyCoList.CoAtom +Color.Pink 19 -3 +null null null null +System.Func`2[Dafny.BigRational,System.Boolean] +System.Func`1[System.Numerics.BigInteger] +System.Func`3[_module.Color,Dafny.ISet`1[System.UInt16],System.UInt32] +0 0 + +Dafny program verifier finished with 20 verified, 0 errors +0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) +null null null null null +{} multiset{} [] map[] {} map[] +null null null null +0 [] Q {} 0 Dt.D0(0) +false Color.Orange 42 {} false Dt.D0(false) +NonemptyList.Atom(0) +6 7 8 9 +true true +IList.INil IList.ICons +IList.INil IList.INil Stream.Next IList.INil +NonemptyList.Atom(0) +NonemptyCoList.CoAtom +Color.Pink 19 -3 +null null null null +function () { return false; } +function () { return _dafny.ZERO; } +function () { return _dafny.ZERO; } +0 0 + +Dafny program verifier finished with 20 verified, 0 errors +0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) +null null null null null +{} multiset{} [] map[] {} map[] +null null null null +0 [] Q {} 0 Dt.D0(0) +false Color.Orange 42 {} false Dt.D0(false) +NonemptyList.Atom(0) +6 7 8 9 +true true +IList.INil IList.ICons +IList.INil IList.INil Stream.Next IList.INil +NonemptyList.Atom(0) +NonemptyCoList.CoAtom +Color.Pink 19 -3 +null null null null +func(dafny.Real) bool +func() dafny.Int +func(main.Color, dafny.Set) uint32 +0 0 + +Dafny program verifier finished with 20 verified, 0 errors +0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) +null null null null null +{} multiset{} [] map[] {} map[] +null null null null +0 [] Q {} 0 Dt.D0(0) +false Color.Orange 42 {} false Dt.D0(false) +NonemptyList.Atom(0) +6 7 8 9 +true true +IList.INil IList.ICons +IList.INil IList.INil Stream.Next IList.INil +NonemptyList.Atom(0) +NonemptyCoList.CoAtom +Color.Pink 19 -3 +null null null null +Function +Function +Function +0 0 diff --git a/Test/comp/TypeParams.dfy.go.expect b/Test/comp/TypeParams.dfy.go.expect deleted file mode 100644 index e61dd555ad3..00000000000 --- a/Test/comp/TypeParams.dfy.go.expect +++ /dev/null @@ -1,21 +0,0 @@ - -Dafny program verifier did not attempt verification -0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) -null null null null null -{} multiset{} [] map[] {} map[] -null null null null -0 [] Q {} 0 Dt.D0(0) -false Color.Orange 42 {} false Dt.D0(false) -NonemptyList.Atom(0) -6 7 8 9 -true true -IList.INil IList.ICons -IList.INil IList.INil Stream.Next IList.INil -NonemptyList.Atom(0) -NonemptyCoList.CoAtom -Color.Pink 19 -3 -null null null null -func(dafny.Real) bool -func() dafny.Int -func(main.Color, dafny.Set) uint32 -0 0 diff --git a/Test/comp/TypeParams.dfy.java.expect b/Test/comp/TypeParams.dfy.java.expect deleted file mode 100644 index 0bcf45b24b7..00000000000 --- a/Test/comp/TypeParams.dfy.java.expect +++ /dev/null @@ -1,21 +0,0 @@ - -Dafny program verifier did not attempt verification -0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) -null null null null null -{} multiset{} [] map[] {} map[] -null null null null -0 [] Q {} 0 Dt.D0(0) -false Color.Orange 42 {} false Dt.D0(false) -NonemptyList.Atom(0) -6 7 8 9 -true true -IList.INil IList.ICons -IList.INil IList.INil Stream.Next IList.INil -NonemptyList.Atom(0) -NonemptyCoList.CoAtom -Color.Pink 19 -3 -null null null null -Function -Function -Function -0 0 diff --git a/Test/comp/TypeParams.dfy.js.expect b/Test/comp/TypeParams.dfy.js.expect deleted file mode 100644 index aadfb98b352..00000000000 --- a/Test/comp/TypeParams.dfy.js.expect +++ /dev/null @@ -1,21 +0,0 @@ - -Dafny program verifier did not attempt verification -0 0 0 false Color.Orange 0.0 {} DtZ.DZ0(Color.Orange) -null null null null null -{} multiset{} [] map[] {} map[] -null null null null -0 [] Q {} 0 Dt.D0(0) -false Color.Orange 42 {} false Dt.D0(false) -NonemptyList.Atom(0) -6 7 8 9 -true true -IList.INil IList.ICons -IList.INil IList.INil Stream.Next IList.INil -NonemptyList.Atom(0) -NonemptyCoList.CoAtom -Color.Pink 19 -3 -null null null null -function () { return false; } -function () { return new BigNumber(0); } -function () { return new BigNumber(0); } -0 0 diff --git a/Test/dafny0/AdvancedLHS.dfy b/Test/dafny0/AdvancedLHS.dfy index 775aea10e8b..71f7d7139e6 100644 --- a/Test/dafny0/AdvancedLHS.dfy +++ b/Test/dafny0/AdvancedLHS.dfy @@ -1,8 +1,6 @@ -/* ---- -compile: 0 -allocated: [1, 3] -*/ +// RUN: %dafny /compile:0 /print:"%t.print" /dprint:"%t.dprint" "%s" > "%t" +// RUN: %diff "%s.expect" "%t" + class C { var x: C? diff --git a/Test/dafny0/AdvancedLHS.dfy.expect b/Test/dafny0/AdvancedLHS.dfy.expect index 843e5f827ac..96fad02e8a9 100644 --- a/Test/dafny0/AdvancedLHS.dfy.expect +++ b/Test/dafny0/AdvancedLHS.dfy.expect @@ -1,4 +1,3 @@ - AdvancedLHS.dfy(34,22): Error: target object may be null Dafny program verifier finished with 3 verified, 1 error diff --git a/Test/dafny4/git-issue250.dfy b/Test/dafny4/git-issue250.dfy index 49562c6e4e1..3046034985f 100644 --- a/Test/dafny4/git-issue250.dfy +++ b/Test/dafny4/git-issue250.dfy @@ -1,7 +1,15 @@ -/*--- -arith: !foreach [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -compile: 1 -*/ +// RUN: %dafny /arith:1 "%s" > "%t" +// RUN: %dafny /arith:2 "%s" >> "%t" +// RUN: %dafny /arith:3 "%s" >> "%t" +// RUN: %dafny /arith:4 "%s" >> "%t" +// RUN: %dafny /arith:5 "%s" >> "%t" +// RUN: %dafny /arith:6 "%s" >> "%t" +// RUN: %dafny /arith:7 "%s" >> "%t" +// RUN: %dafny /arith:8 "%s" >> "%t" +// RUN: %dafny /arith:9 "%s" >> "%t" +// RUN: %dafny /arith:10 "%s" >> "%t" +// RUN: %diff "%s.expect" "%t" + method Main() { } \ No newline at end of file diff --git a/Test/dafny4/git-issue250.dfy.expect b/Test/dafny4/git-issue250.dfy.expect index 012f5b99379..971a5ef441e 100644 --- a/Test/dafny4/git-issue250.dfy.expect +++ b/Test/dafny4/git-issue250.dfy.expect @@ -1,2 +1,20 @@ Dafny program verifier finished with 0 verified, 0 errors + +Dafny program verifier finished with 0 verified, 0 errors + +Dafny program verifier finished with 0 verified, 0 errors + +Dafny program verifier finished with 0 verified, 0 errors + +Dafny program verifier finished with 0 verified, 0 errors + +Dafny program verifier finished with 0 verified, 0 errors + +Dafny program verifier finished with 0 verified, 0 errors + +Dafny program verifier finished with 0 verified, 0 errors + +Dafny program verifier finished with 0 verified, 0 errors + +Dafny program verifier finished with 0 verified, 0 errors diff --git a/Test/testing/TestAttributeErrors.dfy b/Test/exceptions/TestAttributeErrors.dfy similarity index 100% rename from Test/testing/TestAttributeErrors.dfy rename to Test/exceptions/TestAttributeErrors.dfy diff --git a/Test/testing/TestAttributeErrors.dfy.expect b/Test/exceptions/TestAttributeErrors.dfy.expect similarity index 100% rename from Test/testing/TestAttributeErrors.dfy.expect rename to Test/exceptions/TestAttributeErrors.dfy.expect From 4afdd358b6c6568a8b4c8d8dafee03ab160e036b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 9 Aug 2021 13:27:44 -0700 Subject: [PATCH 102/192] Nice merge job, dude :) --- Source/Dafny.sln | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/Dafny.sln b/Source/Dafny.sln index a1d16c6c183..cc101458fe7 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -236,7 +236,6 @@ Global {320C02A3-D3E6-4AC7-A7E2-170AF86A6EEC}.Release|Mixed Platforms.Build.0 = Release|Any CPU {320C02A3-D3E6-4AC7-A7E2-170AF86A6EEC}.Release|x86.ActiveCfg = Release|Any CPU {320C02A3-D3E6-4AC7-A7E2-170AF86A6EEC}.Release|x86.Build.0 = Release|Any CPU -<<<<<<< HEAD {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|.NET.ActiveCfg = Debug|Any CPU {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|.NET.Build.0 = Debug|Any CPU {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Checked|Any CPU.ActiveCfg = Debug|Any CPU @@ -261,7 +260,6 @@ Global {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|Mixed Platforms.Build.0 = Release|Any CPU {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|x86.ActiveCfg = Release|Any CPU {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Release|x86.Build.0 = Release|Any CPU -======= {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Checked|.NET.ActiveCfg = Debug|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Checked|.NET.Build.0 = Debug|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Checked|Any CPU.ActiveCfg = Debug|Any CPU @@ -286,7 +284,6 @@ Global {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Mixed Platforms.Build.0 = Release|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|x86.ActiveCfg = Release|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|x86.Build.0 = Release|Any CPU ->>>>>>> 4c5da6ff884054a920559dbf6fe9d41bf67a98da EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 94ce61371047ea8f028d73aa96c52e5069c4a832 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 9 Aug 2021 13:50:11 -0700 Subject: [PATCH 103/192] Revert changes to lit job and temporarily disable (makes the CI a lot cheaper while iterating) --- .github/workflows/msbuild.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index d38d05931e5..9e453e1991e 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -3,8 +3,8 @@ name: Build and Test on: push: branches: [ master ] - pull_request: - branches: [ master ] +# pull_request: +# branches: [ master ] jobs: build: @@ -60,6 +60,8 @@ jobs: java-version: 1.8 - name: Upgrade outdated pip run: python -m pip install --upgrade pip + - name: Install lit + run: pip install lit OutputCheck pyyaml - uses: actions/setup-node@v1 - run: npm install bignumber.js - name: Checkout Dafny @@ -77,6 +79,7 @@ jobs: - if: runner.os != 'Windows' run: | unzip dafny/Package/CI.zip -d unzippedRelease - - name: Run tests + - name: Run lit tests run: | - XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 dotnet test dafny/Source/Dafny.sln + ## lit in this context needs the executables specified + lit --time-tests -v --num-shards=5 --run-shard=${{ matrix.shard }} --param executable=$PWD/unzippedRelease/dafny/dafny --param serverExecutable=$PWD/unzippedRelease/dafny/DafnyServer dafny/Test From 62056b9fcb27cb1e96ecf0782f47e848428d4275 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 9 Aug 2021 21:59:35 -0700 Subject: [PATCH 104/192] Starting to add proper meta unit tests --- .../DafnyDriver.Test/DafnyDriver.Test.csproj | 6 +- .../XUnitExtensions/YamlDataAttribute.cs | 9 +- .../XUnitExtensions/YamlDataTests.cs | 84 ++++++++++++++++++- .../CalculatorCombinatorialTest.yml | 4 +- .../CustomDataDiscovererTest.yml | 2 +- .../YamlDataTests/MultiFileTest/Data1.yml | 4 + .../YamlDataTests/MultiFileTest/Data2.yml | 4 + Source/LitTestConverter/LitTestConvertor.cs | 41 ++++----- .../Conversion/ConvertingFromLitToXunit.cs | 45 ++++++++++ .../ConvertingFromLitToXunit.cs | 14 ---- .../LitTestConvertor.Test.csproj | 8 +- .../TestFiles/VerifyOnlyLitTest.dfy | 6 ++ 12 files changed, 179 insertions(+), 48 deletions(-) create mode 100644 Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data1.yml create mode 100644 Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data2.yml create mode 100644 Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs delete mode 100644 Source/LitTestConvertor.Test/ConvertingFromLitToXunit.cs create mode 100644 Source/LitTestConvertor.Test/TestFiles/VerifyOnlyLitTest.dfy diff --git a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj index 697c0823cf5..acf654b9c9f 100644 --- a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj +++ b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj @@ -19,8 +19,12 @@ - + + + + + diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs index 60be24aa21e..6d5a11caaaa 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs @@ -5,10 +5,13 @@ namespace XUnitExtensions { - [DataDiscoverer("XUnitExtensions.YamlDataDiscoverer", "DafnyDriver.Test")] - public class YamlDataAttribute : DataAttribute { - + [DataDiscoverer("DafnyDriver.Test.XUnitExtensions.YamlDataDiscoverer", "DafnyDriver.Test")] + public class YamlDataAttribute : DataAttribute + { + public readonly bool WithParameterNames; + public YamlDataAttribute(bool withParameterNames = true) { + WithParameterNames = withParameterNames; } public override IEnumerable GetData(MethodInfo testMethod) { diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs index 08494d03242..435e2d70872 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs @@ -1,3 +1,4 @@ +using System; using System.Collections; using System.Collections.Generic; using System.IO; @@ -12,12 +13,27 @@ namespace DafnyDriver.Test.XUnitExtensions { public class YamlDataTests { - + + private void AssertTheoryData(string methodName, IEnumerable expectedData) { + var method = typeof(YamlDataTests).GetMethod(methodName); + var attribute = (YamlDataAttribute)Attribute.GetCustomAttribute(method!, typeof(YamlDataAttribute)); + var discoverer = new YamlDataDiscoverer(); + Assert.Equal(expectedData, discoverer.GetData(method, attribute!.WithParameterNames)); + } + [Theory] [YamlData()] public void CalculatorTest(int lhs, int rhs, int expected) { Assert.Equal(expected, lhs + rhs); } + + [Fact] + public void CalculatorTestData() { + AssertTheoryData(nameof(CalculatorTest), new [] { + new object[]{ 2, 2, 4 }, + new object[]{ 3, 4, 7 } + }); + } [Theory] [YamlData()] @@ -25,13 +41,61 @@ public void CalculatorCombinatorialTest([ForEach()] int lhs, [ForEach()] int rhs Assert.Equal(rhs + lhs, lhs + rhs); } + [Fact] + public void CalculatorCombinatorialTestData() { + AssertTheoryData(nameof(CalculatorCombinatorialTest), new [] { + new object[]{ 1, 5 }, + new object[]{ 1, 10 }, + new object[]{ 2, 5 }, + new object[]{ 2, 10 }, + new object[]{ 3, 5 }, + new object[]{ 3, 10 }, + new object[]{ 4, 5 }, + new object[]{ 4, 10 }, + new object[]{ 5, 5 }, + new object[]{ 5, 10 }, + }); + } + + [Theory] + [YamlData()] + public void MultiFileTest(int a, int b) { + Assert.Equal(a + 1, b); + } + + [Fact] + public void MultiFileTestData() { + AssertTheoryData(nameof(MultiFileTest), new [] { + new object[]{ 1, 2 }, + new object[]{ 3, 4 }, + new object[]{ 5, 6 }, + new object[]{ 7, 8 }, + }); + } + [Theory] [YamlData(false)] public void DictionaryTest(Dictionary config) { Assert.Equal(3, config.Count); } + + [Fact] + public void DictionaryTestData() { + AssertTheoryData(nameof(DictionaryTest), new [] { + new object[]{ new Dictionary { + ["one"] = "1", + ["two"] = "2", + ["three"] = "3" + }}, + new object[]{ new Dictionary { + ["four"] = "4", + ["five"] = "5", + ["six"] = "6" + }} + }); + } - [DataDiscoverer("DafnyTests.XUnitExtensions.CustomDiscoverer", "DafnyDriver.Test")] + [DataDiscoverer("DafnyDriver.Test.XUnitExtensions.CustomDiscoverer", "DafnyDriver.Test")] public class CustomYamlDataAttribute : YamlDataAttribute { public CustomYamlDataAttribute(bool withParameterNames = true) : base(withParameterNames) { } @@ -42,6 +106,22 @@ public CustomYamlDataAttribute(bool withParameterNames = true) : base(withParame public void CustomDataDiscovererTest([ForEach()] int lhs, [ForEach()] int rhs) { Assert.Equal(rhs + lhs, lhs + rhs); } + + [Fact] + public void CustomDataDiscovererTestData() { + AssertTheoryData(nameof(CustomDataDiscovererTest), new [] { + new object[]{ 0, 0 }, + new object[]{ 0, 1 }, + new object[]{ 1, 0 }, + new object[]{ 1, 1 }, + new object[]{ 2, 0 }, + new object[]{ 2, 1 }, + new object[]{ 3, 0 }, + new object[]{ 3, 1 }, + new object[]{ 4, 0 }, + new object[]{ 4, 1 }, + }); + } } public class Range : IEnumerable { diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml index d363f891c5a..05fe6690d86 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml @@ -1,2 +1,2 @@ -- lhs: [0, 1, 2, 3, 4] - rhs: [0, 1, 2, 3] +- lhs: [1, 2, 3, 4, 5] + rhs: [5, 10] diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml index 25bb6deb6e3..9305876ab94 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml @@ -3,4 +3,4 @@ end: 5 rhs: !range start: 0 - end: 4 + end: 2 diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data1.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data1.yml new file mode 100644 index 00000000000..344e8af8d98 --- /dev/null +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data1.yml @@ -0,0 +1,4 @@ +- a: 1 + b: 2 +- a: 3 + b: 4 \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data2.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data2.yml new file mode 100644 index 00000000000..15a08ae994d --- /dev/null +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data2.yml @@ -0,0 +1,4 @@ +- a: 5 + b: 6 +- a: 7 + b: 8 \ No newline at end of file diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 9e77ae3da18..e6ca5f290e9 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -36,31 +36,15 @@ public class LitTestConvertor { private int invalidCount = 0; public void ConvertLitTest(string filePath) { - object testSpec; + IEnumerable testSpec; IEnumerable testContent; - if (filePath.Contains("/Inputs/")) { testSpec = Enumerable.Empty(); testContent = File.ReadAllLines(filePath); } else { - - string[] lines = File.ReadAllLines(filePath); - - var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); - if (!litCommands.Any()) { - alreadyConverted++; - return; - } - testContent = lines.Skip(litCommands.Count); - - // Make sure the commands are consecutive - if (testContent.Any(line => ExtractLitCommand(line) != null)) { - throw new ArgumentException("Lit commands are not consecutive"); - } - - testSpec = ConvertLitCommands(filePath, litCommands); + (testSpec, testContent) = ConvertLitCommands(filePath, File.ReadAllLines(filePath)); } - + ISerializer serializer = new SerializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) @@ -81,10 +65,21 @@ public void ConvertLitTest(string filePath) { } } - private object ConvertLitCommands(string filePath, List litCommands) { + public (IEnumerable spec, IEnumerable content) ConvertLitCommands(string filePath, IEnumerable lines) { + var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); + if (!litCommands.Any()) { + throw new ArgumentException("No lit commands found"); + } + + // Make sure the commands are consecutive + var testContent = lines.Skip(litCommands.Count); + if (testContent.Any(line => ExtractLitCommand(line) != null)) { + throw new ArgumentException("Lit commands are not consecutive"); + } + if (litCommands.Count == 1 && litCommands.Single().StartsWith("echo")) { // This is an idiom for Dafny files used elsewhere - return Enumerable.Empty(); + return (Enumerable.Empty(), testContent); } if (!litCommands[^1].Equals("%diff \"%s.expect\" \"%t\"")) { @@ -99,13 +94,13 @@ private object ConvertLitCommands(string filePath, List litCommands) { if (IsStandardVerifyOnly(single)) { verifyOnlyCount++; } - return single; + return (single, testContent); } if (IsStandardVerifyOnly(testConfigs[0]) && testConfigs.Skip(1).All(IsStandardCompileAndRun) || testConfigs.Skip(1).All(IsStandardCompileAndRun)) { defaultCount++; - return null; + return (null, testContent); } throw new ArgumentException("Multi-command lit tests require manual conversion"); diff --git a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs new file mode 100644 index 00000000000..39b37edf0cf --- /dev/null +++ b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using DafnyDriver.Test; +using Xunit; + +namespace LitTestConvertor.Test +{ + public class ConvertingFromLitToXunit + { + + + private static IEnumerable ReadLines(StreamReader reader) { + string line; + while ((line = reader.ReadLine()) != null) { + yield return line; + } + } + + [Fact] + public void HelloWorld() { + var convertor = new LitTestConvertor(); + using var stream = + Assembly.GetExecutingAssembly().GetManifestResourceStream("LitTestConvertor.Test.TestFiles.HelloWorldLitTest.dfy"); + using var reader = new StreamReader(stream); + var lines = ReadLines(reader); + var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles/HelloWorldLitTest.dfy", lines); + DafnyTestSpec spec = (DafnyTestSpec)testCases; + Assert.Equal(3, spec.Compile); + } + + [Fact] + public void VerifyOnly() { + var convertor = new LitTestConvertor(); + using var stream = + Assembly.GetExecutingAssembly().GetManifestResourceStream("LitTestConvertor.Test.TestFiles.VerifyOnlyLitTest.dfy"); + using var reader = new StreamReader(stream); + var lines = ReadLines(reader); + var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles/VerifyOnly.dfy", lines); + DafnyTestSpec spec = (DafnyTestSpec)testCases; + Assert.Equal(0, spec.Compile); + } + } +} \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/ConvertingFromLitToXunit.cs deleted file mode 100644 index 68f9eace538..00000000000 --- a/Source/LitTestConvertor.Test/ConvertingFromLitToXunit.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; -using Xunit; - -namespace LitTestConvertor.Test -{ - public class ConvertingFromLitToXunit - { - [Fact] - public void HelloWorld() { - var convertor = new LitTestConvertor(); - // TODO - } - } -} \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj index ab9de4fcd23..6a01d1d0515 100644 --- a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj +++ b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj @@ -2,7 +2,7 @@ net5.0 - + true false @@ -20,7 +20,11 @@ - + + + + + diff --git a/Source/LitTestConvertor.Test/TestFiles/VerifyOnlyLitTest.dfy b/Source/LitTestConvertor.Test/TestFiles/VerifyOnlyLitTest.dfy new file mode 100644 index 00000000000..8b2364aa0f2 --- /dev/null +++ b/Source/LitTestConvertor.Test/TestFiles/VerifyOnlyLitTest.dfy @@ -0,0 +1,6 @@ +// RUN: %dafny /compile:0 "%s" > "%t" +// RUN: %diff "%s.expect" "%t" + +method Main() { + print "Hello, World! Best, Dafny\n"; +} From 7bb72c29c003698cf33d492c6993d5de729ae149 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 11 Aug 2021 09:39:18 -0700 Subject: [PATCH 105/192] Partial work dropping the need for EmbeddedResource --- .../DafnyDriver.Test/DafnyDriver.Test.csproj | 16 ++++--- Source/DafnyDriver.Test/README.md | 43 +++++++++++++++++++ .../XUnitExtensions/YamlDataTests.cs | 8 ++-- .../CalculatorCombinatorialTest.yml | 0 .../{ => TestFiles}/CalculatorTest.yml | 0 .../CustomDataDiscovererTest.yml | 0 .../{ => TestFiles}/DictionaryTest.yml | 0 .../{ => TestFiles}/MultiFileTest/Data1.yml | 0 .../{ => TestFiles}/MultiFileTest/Data2.yml | 0 .../DafnyLanguageServer.Test.csproj | 14 +----- 10 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 Source/DafnyDriver.Test/README.md rename Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/{ => TestFiles}/CalculatorCombinatorialTest.yml (100%) rename Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/{ => TestFiles}/CalculatorTest.yml (100%) rename Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/{ => TestFiles}/CustomDataDiscovererTest.yml (100%) rename Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/{ => TestFiles}/DictionaryTest.yml (100%) rename Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/{ => TestFiles}/MultiFileTest/Data1.yml (100%) rename Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/{ => TestFiles}/MultiFileTest/Data2.yml (100%) diff --git a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj index acf654b9c9f..eb733c18994 100644 --- a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj +++ b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj @@ -19,12 +19,14 @@ - - - - - - - + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + diff --git a/Source/DafnyDriver.Test/README.md b/Source/DafnyDriver.Test/README.md new file mode 100644 index 00000000000..410e8791240 --- /dev/null +++ b/Source/DafnyDriver.Test/README.md @@ -0,0 +1,43 @@ +# Dafny CLI Testing + +Dafny test cases are based on single Dafny source files together with the expected output from the `dafny` CLI tool. If the source file is named `Foo.dfy`, the expected output will contained in the file named `Foo.dfy.expect`. + +The default behaviour is to assert that the source file verifies successfully and, for each currently-supported target language, can be compiled and run to produce the expected output. When the behaviour is not 100% consistent between different target languages, expected discrepancies can be recorded in additional files with the name `Foo.dfy..expect`. For example: [Test/comp/NativeNumbers.dfy.js.expect](Test/comp/NativeNumbers.dfy.js.expect). Such exceptions are automatically flagged as "known issues" and classified as a "skipped/ignored" test. + +## Test configuration syntax + +Test cases are configured and run using xUnit's support for parameterized tests, with extensions for running test cases in parallel. The sets of options passed to `dafny` can be configured using YAML embedded in the first multi-line comment in the source file. Lists of values are interpreted as multiple parameterizations. For example: [Test/dafny4/git-issue250.dfy](Test/dafny4/git-issue250.dfy). + +For details and more configuration options, see [the DafnyTests.cs source](Test/DafnyTests/DafnyTests.cs). + + +```yaml +!dafnyTestSpec +options: + compile: 3 + coverage: "-" +compileTargetOverrides: + java: + otherFiles: + - CodeCoverage.java + cs: + otherFiles: + - BranchCoverage2.cs + js: + otherFiles: + - BranchCoverage3.js + go: + otherFiles: + - BranchCoverage4.go +``` + +## TODO + +* More complete documentation about options (in this file or in the source code) +* Depend on only the project's output directory instead of the Binaries/Test directories + * This is mostly working except for errors around missing types from System.dll when compiling to C# +* Add support for regular expression matching against CLI output (needed to assert known limitations that cause errors with things like absolute paths names in them) +* Expose test case options as traits so that they can be filtered on (e.g. `dotnet test --filter compileTarget=java`) +* Convert all the existing lit-based test cases under Test/ to this format + * The `LitTestConvertor` package is able to understand common lit command patterns and convert them into the new YAML format automatically. +* Extract most of the xUnit extensions as a separate package, since most of it is generically useful for any other data-driven xUnit tests, especially CLIs. diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs index 435e2d70872..e521e69af2a 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs @@ -22,7 +22,7 @@ private void AssertTheoryData(string methodName, IEnumerable expectedDat } [Theory] - [YamlData()] + [YamlData] public void CalculatorTest(int lhs, int rhs, int expected) { Assert.Equal(expected, lhs + rhs); } @@ -36,8 +36,8 @@ public void CalculatorTestData() { } [Theory] - [YamlData()] - public void CalculatorCombinatorialTest([ForEach()] int lhs, [ForEach()] int rhs) { + [YamlData] + public void CalculatorCombinatorialTest([ForEach] int lhs, [ForEach] int rhs) { Assert.Equal(rhs + lhs, lhs + rhs); } @@ -58,7 +58,7 @@ public void CalculatorCombinatorialTestData() { } [Theory] - [YamlData()] + [YamlData] public void MultiFileTest(int a, int b) { Assert.Equal(a + 1, b); } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CalculatorCombinatorialTest.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorCombinatorialTest.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CalculatorCombinatorialTest.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CalculatorTest.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CalculatorTest.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CalculatorTest.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CustomDataDiscovererTest.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/CustomDataDiscovererTest.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CustomDataDiscovererTest.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/DictionaryTest.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/DictionaryTest.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/DictionaryTest.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/DictionaryTest.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data1.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/MultiFileTest/Data1.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data1.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/MultiFileTest/Data1.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data2.yml b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/MultiFileTest/Data2.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/MultiFileTest/Data2.yml rename to Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/MultiFileTest/Data2.yml diff --git a/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj b/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj index 6ffecdd3b5d..fb151794b7b 100644 --- a/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj +++ b/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj @@ -20,19 +20,7 @@ - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - + PreserveNewest From ebf71dadb8344a36fd5789c13947913158926610 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 11 Aug 2021 14:33:49 -0700 Subject: [PATCH 106/192] Removing EmbeddedResource support, just use files! --- .../DafnyDriver.Test/DafnyDriver.Test.csproj | 8 ++- .../DafnyTestYamlDataDiscoverer.cs | 42 +++------------ Source/DafnyDriver.Test/IntegrationTest.cs | 22 ++++---- .../CalculatorCombinatorialTest.yml | 0 .../YamlDataTests}/CalculatorTest.yml | 0 .../CustomDataDiscovererTest.yml | 0 .../YamlDataTests}/DictionaryTest.yml | 0 .../YamlDataTests}/MultiFileTest/Data1.yml | 0 .../YamlDataTests}/MultiFileTest/Data2.yml | 0 .../XUnitExtensions/YamlDataAttribute.cs | 8 ++- .../XUnitExtensions/YamlDataDiscoverer.cs | 51 ++++++++++++------- .../XUnitExtensions/YamlDataTests.cs | 2 +- 12 files changed, 59 insertions(+), 74 deletions(-) rename Source/DafnyDriver.Test/{XUnitExtensions/YamlDataTests/TestFiles => TestFiles/YamlDataTests}/CalculatorCombinatorialTest.yml (100%) rename Source/DafnyDriver.Test/{XUnitExtensions/YamlDataTests/TestFiles => TestFiles/YamlDataTests}/CalculatorTest.yml (100%) rename Source/DafnyDriver.Test/{XUnitExtensions/YamlDataTests/TestFiles => TestFiles/YamlDataTests}/CustomDataDiscovererTest.yml (100%) rename Source/DafnyDriver.Test/{XUnitExtensions/YamlDataTests/TestFiles => TestFiles/YamlDataTests}/DictionaryTest.yml (100%) rename Source/DafnyDriver.Test/{XUnitExtensions/YamlDataTests/TestFiles => TestFiles/YamlDataTests}/MultiFileTest/Data1.yml (100%) rename Source/DafnyDriver.Test/{XUnitExtensions/YamlDataTests/TestFiles => TestFiles/YamlDataTests}/MultiFileTest/Data2.yml (100%) diff --git a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj index eb733c18994..3bcb28c7960 100644 --- a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj +++ b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj @@ -2,14 +2,12 @@ net5.0 - true DafnyDriver.Test - @@ -19,13 +17,13 @@ - + PreserveNewest - + PreserveNewest - + PreserveNewest diff --git a/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs b/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs index 61f914e7fca..b6e35d4e5d9 100644 --- a/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs +++ b/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs @@ -1,9 +1,5 @@ -using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Reflection; using DafnyDriver.Test.XUnitExtensions; -using Microsoft.Extensions.FileProviders; using Xunit.Sdk; using XUnitExtensions; using YamlDotNet.Core; @@ -19,30 +15,8 @@ public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { dafnyArguments: {} "; - private static readonly IFileProvider ManifestFileProvider = new ManifestEmbeddedFileProvider( - Assembly.GetExecutingAssembly()); - private static readonly Dictionary PathsForResourceNames = GetPathsForResourceNames( - "DafnyDriver.Test", ManifestFileProvider, "DafnyTests"); - - private static Dictionary GetPathsForResourceNames(string assemblyName, IFileProvider fileProvider, string path = null) { - return fileProvider.GetDirectoryContents(path).SelectMany(file => { - var childName = path == null ? file.Name : path + "/" + file.Name; - if (file.IsDirectory) { - return GetPathsForResourceNames(assemblyName, fileProvider, childName); - } else { - var result = new Dictionary(); - result[ResourceNameForFilePath(assemblyName, childName)] = childName; - return result; - } - }).ToDictionary(pair => pair.Key, pair => pair.Value); - } - - private static string ResourceNameForFilePath(string assemblyName, string filePath) { - return assemblyName + "." + filePath.Replace("/", ".").Replace("+", "_"); - } - - private static DafnyTestSpec SpecForResourceName(string manifestResourceName) { - string filePath = PathsForResourceNames[manifestResourceName].Substring("DafnyTests/Test".Length + 1); + private static DafnyTestSpec SpecForFileName(string fileName) { + string filePath = fileName.Substring("TestFiles/DafnyTests/Test".Length + 1); return new DafnyTestSpec(filePath); } @@ -61,20 +35,16 @@ public static string GetTestCaseConfigYaml(string fullText) { return null; } - public override IParser GetYamlParser(string manifestResourceName, Stream stream) { - if (!manifestResourceName.EndsWith(".dfy")) { - return null; - } - + public override IParser GetYamlParser(string fileName, Stream stream) { string content = GetTestCaseConfigYaml(new StreamReader(stream).ReadToEnd()) ?? DEFAULT_CONFIG; return new Parser(new StringReader(content)); } - public override IDeserializer GetDeserializer(string manifestResourceName) { + public override IDeserializer GetDeserializer(string fileName) { var defaultObjectFactory = new DefaultObjectFactory(); var customObjectFactory = new LambdaObjectFactory(type => - type == typeof(DafnyTestSpec) ? SpecForResourceName(manifestResourceName) : defaultObjectFactory.Create(type)); + type == typeof(DafnyTestSpec) ? SpecForFileName(fileName) : defaultObjectFactory.Create(type)); return new DeserializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) @@ -88,6 +58,6 @@ public override IDeserializer GetDeserializer(string manifestResourceName) { [DataDiscoverer("DafnyDriver.Test.DafnyTestYamlDataDiscoverer", "DafnyDriver.Test")] public class DafnyTestDataAttribute : YamlDataAttribute { - public DafnyTestDataAttribute(bool withParameterNames) : base(withParameterNames) { + public DafnyTestDataAttribute() : base(false, null, ".dfy") { } } \ No newline at end of file diff --git a/Source/DafnyDriver.Test/IntegrationTest.cs b/Source/DafnyDriver.Test/IntegrationTest.cs index 6400d192513..96d5544e50c 100644 --- a/Source/DafnyDriver.Test/IntegrationTest.cs +++ b/Source/DafnyDriver.Test/IntegrationTest.cs @@ -4,20 +4,20 @@ namespace DafnyDriver.Test { - public class DafnyTests { + public static class DafnyTests { - [Fact] - public static void DafnyTestDataDiscovererDiscoversAtLeastOneTest() { - // This test is much easier to debug than the main parameterized test - // if the discoverer is not working correctly - var discoverer = new DafnyTestYamlDataDiscoverer(); - var testMethod = typeof(DafnyTests).GetMethod(nameof(Test)); - var testData = discoverer.GetData(testMethod, false).ToList(); - Assert.True(testData.Any()); - } + // [Fact] + // public static void DafnyTestDataDiscovererDiscoversAtLeastOneTest() { + // // This test is much easier to debug than the main parameterized test + // // if the discoverer is not working correctly + // var discoverer = new DafnyTestYamlDataDiscoverer(); + // var testMethod = typeof(DafnyTests).GetMethod(nameof(Test)); + // var testData = discoverer.GetData(testMethod, false, ).ToList(); + // Assert.True(testData.Any()); + // } [ParallelTheory] - [DafnyTestData(false)] + [DafnyTestData] public static void Test(CLITestCase testCase) { testCase.Run(); } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CalculatorCombinatorialTest.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorCombinatorialTest.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CalculatorCombinatorialTest.yml rename to Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorCombinatorialTest.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CalculatorTest.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorTest.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CalculatorTest.yml rename to Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorTest.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CustomDataDiscovererTest.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CustomDataDiscovererTest.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/CustomDataDiscovererTest.yml rename to Source/DafnyDriver.Test/TestFiles/YamlDataTests/CustomDataDiscovererTest.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/DictionaryTest.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/DictionaryTest.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/DictionaryTest.yml rename to Source/DafnyDriver.Test/TestFiles/YamlDataTests/DictionaryTest.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/MultiFileTest/Data1.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data1.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/MultiFileTest/Data1.yml rename to Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data1.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/MultiFileTest/Data2.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data2.yml similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests/TestFiles/MultiFileTest/Data2.yml rename to Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data2.yml diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs index 6d5a11caaaa..58c676d2d7d 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs @@ -9,9 +9,13 @@ namespace XUnitExtensions { public class YamlDataAttribute : DataAttribute { public readonly bool WithParameterNames; - - public YamlDataAttribute(bool withParameterNames = true) { + public readonly string Path; + public readonly string Extension; + + public YamlDataAttribute(bool withParameterNames = true, string path = null, string extension = ".yml") { WithParameterNames = withParameterNames; + Path = path; + Extension = extension; } public override IEnumerable GetData(MethodInfo testMethod) { diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs index cbd36ba3738..b2330574108 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Reflection; +using JetBrains.Annotations; using Xunit.Abstractions; using Xunit.Sdk; using YamlDotNet.Core; @@ -15,11 +16,11 @@ namespace DafnyDriver.Test.XUnitExtensions { public class YamlDataDiscoverer : IDataDiscoverer { - public virtual IParser GetYamlParser(string manifestResourceName, Stream stream) { + public virtual IParser GetYamlParser(string fileName, Stream stream) { return new Parser(new StreamReader(stream)); } - public virtual IDeserializer GetDeserializer(string manifestResourceName) { + public virtual IDeserializer GetDeserializer(string fileName) { return new Deserializer(); } @@ -39,15 +40,15 @@ public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodIn return true; } - private IEnumerable ResourceData(MethodInfo testMethod, string resourceName, bool withParameterNames) { + private IEnumerable FileData(MethodInfo testMethod, string fileName, bool withParameterNames) { try { - using Stream stream = testMethod.DeclaringType.Assembly.GetManifestResourceStream(resourceName); - IParser parser = GetYamlParser(resourceName, stream); - if (parser == null) { + using Stream stream = File.OpenRead(fileName); + IParser parser = GetYamlParser(fileName, stream); + if (parser == null) { return Enumerable.Empty(); } - IDeserializer deserializer = GetDeserializer(resourceName); + IDeserializer deserializer = GetDeserializer(fileName); parser.Consume(); parser.Consume(); @@ -70,7 +71,7 @@ private IEnumerable ResourceData(MethodInfo testMethod, string resourc } } catch (Exception e) { throw new ArgumentException( - "Exception thrown while trying to deserialize test data from manifest resource: " + resourceName, e); + "Exception thrown while trying to deserialize test data from file: " + fileName, e); } } @@ -80,22 +81,34 @@ public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo t if (methodInfo == null) { return null; } - + List attributeArgs = attributeInfo.GetConstructorArguments().ToList(); - bool withParameterNames = (bool) attributeArgs[0]; + var withParameterNames = attributeInfo.GetNamedArgument("WithParameterNames"); + var path = attributeInfo.GetNamedArgument("Path"); + var extension = attributeInfo.GetNamedArgument("Extension"); - return GetData(methodInfo, withParameterNames); + return GetData(methodInfo, withParameterNames, path, extension); } - public IEnumerable GetData(MethodInfo methodInfo, bool withParameterNames) { - string resourceNamePrefix = methodInfo.DeclaringType.FullName + "." + methodInfo.Name; - IEnumerable result = methodInfo.DeclaringType.Assembly.GetManifestResourceNames() - .Where(n => n.StartsWith(resourceNamePrefix)) - .SelectMany(path => ResourceData(methodInfo, path, withParameterNames)); - if (!result.Any()) { - throw new ArgumentException("No data found for resource prefix: " + resourceNamePrefix); + public IEnumerable GetData(MethodInfo methodInfo, bool withParameterNames, string path, string extension) { + if (path == null) { + path = Path.Combine("TestFiles", methodInfo.DeclaringType.Name, methodInfo.Name); + } + if (extension == null) { + extension = ".yml"; + } + + if (Directory.Exists(path)) { + return Directory.EnumerateFiles(path, "*" + extension, SearchOption.AllDirectories) + .SelectMany(childPath => FileData(methodInfo, childPath, withParameterNames)); + } + + var yamlFileName = path + ".yml"; + if (File.Exists(yamlFileName)) { + return FileData(methodInfo, yamlFileName, withParameterNames); } - return result; + + throw new ArgumentException("No data found for path: " + path); } } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs index e521e69af2a..dc6a8e423f4 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs @@ -18,7 +18,7 @@ private void AssertTheoryData(string methodName, IEnumerable expectedDat var method = typeof(YamlDataTests).GetMethod(methodName); var attribute = (YamlDataAttribute)Attribute.GetCustomAttribute(method!, typeof(YamlDataAttribute)); var discoverer = new YamlDataDiscoverer(); - Assert.Equal(expectedData, discoverer.GetData(method, attribute!.WithParameterNames)); + Assert.Equal(expectedData, discoverer.GetData(method, attribute!.WithParameterNames, attribute.Path, attribute.Extension)); } [Theory] From 2c9d3a74506312f0bc85dc27e5673c8ca9a423a4 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 11 Aug 2021 17:14:37 -0700 Subject: [PATCH 107/192] First cut of an xUnit test that runs lit tests directly --- Source/DafnyDriver.Test/DafnyTestCase.cs | 2 +- Source/DafnyDriver.Test/IntegrationTest.cs | 10 ---- .../XUnitExtensions/FileDataAttribute.cs | 21 +++++++ .../XUnitExtensions/FileDataDiscoverer.cs | 38 +++++++++++++ .../XUnitExtensions/YamlDataAttribute.cs | 11 ++-- .../XUnitExtensions/YamlDataDiscoverer.cs | 56 +++++-------------- .../XUnitExtensions/YamlDataTests.cs | 4 +- .../Conversion/ConvertingFromLitToXunit.cs | 7 +-- .../LitTestConvertor.Test.csproj | 5 +- .../LitTestRunner/LitTestDataDiscoverer.cs | 40 +++++++++++++ .../LitTestRunner/LitTests.cs | 14 +++++ 11 files changed, 141 insertions(+), 67 deletions(-) create mode 100644 Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs create mode 100644 Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs create mode 100644 Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs create mode 100644 Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index 6ff9b32eb74..f05add03219 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -27,7 +27,7 @@ private static IEnumerable OptionsToFullArguments(string sourcePath, { Dictionary optionsWithDefaults = new(defaultDafnyOptions); foreach (var (key, value) in dafnyOptions) { - optionsWithDefaults.Add(key, value); + optionsWithDefaults[key] = value; } return new[] { diff --git a/Source/DafnyDriver.Test/IntegrationTest.cs b/Source/DafnyDriver.Test/IntegrationTest.cs index 96d5544e50c..b0bca4c36f7 100644 --- a/Source/DafnyDriver.Test/IntegrationTest.cs +++ b/Source/DafnyDriver.Test/IntegrationTest.cs @@ -6,16 +6,6 @@ namespace DafnyDriver.Test { public static class DafnyTests { - // [Fact] - // public static void DafnyTestDataDiscovererDiscoversAtLeastOneTest() { - // // This test is much easier to debug than the main parameterized test - // // if the discoverer is not working correctly - // var discoverer = new DafnyTestYamlDataDiscoverer(); - // var testMethod = typeof(DafnyTests).GetMethod(nameof(Test)); - // var testData = discoverer.GetData(testMethod, false, ).ToList(); - // Assert.True(testData.Any()); - // } - [ParallelTheory] [DafnyTestData] public static void Test(CLITestCase testCase) { diff --git a/Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs new file mode 100644 index 00000000000..a47e44a544c --- /dev/null +++ b/Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Reflection; +using Xunit.Sdk; + +namespace DafnyDriver.Test.XUnitExtensions +{ + public abstract class FileDataAttribute : DataAttribute + { + public readonly string Path; + public readonly string Extension; + + public FileDataAttribute(string path = null, string extension = null) { + Path = path; + Extension = extension; + } + + public override IEnumerable GetData(MethodInfo testMethod) { + throw new System.NotImplementedException(); + } + } +} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs new file mode 100644 index 00000000000..c5a263a35c2 --- /dev/null +++ b/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace DafnyDriver.Test.XUnitExtensions +{ + public abstract class FileDataDiscoverer : IDataDiscoverer + { + public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo testMethod) { + var path = attributeInfo.GetNamedArgument(nameof(FileDataAttribute.Path)); + var extension = attributeInfo.GetNamedArgument(nameof(FileDataAttribute.Extension)); + + if (path == null) { + path = Path.Combine("TestFiles", testMethod.ToRuntimeMethod().DeclaringType.Name, testMethod.Name); + } + + if (Directory.Exists(path)) { + return Directory.EnumerateFiles(path, "*" + extension, SearchOption.AllDirectories) + .SelectMany(childPath => FileData(attributeInfo, testMethod, childPath)); + } + + var fileName = path + extension; + if (File.Exists(fileName)) { + return FileData(attributeInfo, testMethod, fileName); + } + + throw new ArgumentException("No data found for path: " + path); + } + + public abstract bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod); + + protected abstract IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName); + } +} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs index 58c676d2d7d..bfc298d2969 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs @@ -1,21 +1,20 @@ using System; using System.Collections.Generic; using System.Reflection; +using DafnyDriver.Test.XUnitExtensions; using Xunit.Sdk; namespace XUnitExtensions { [DataDiscoverer("DafnyDriver.Test.XUnitExtensions.YamlDataDiscoverer", "DafnyDriver.Test")] - public class YamlDataAttribute : DataAttribute + public class YamlDataAttribute : FileDataAttribute { public readonly bool WithParameterNames; - public readonly string Path; - public readonly string Extension; - public YamlDataAttribute(bool withParameterNames = true, string path = null, string extension = ".yml") { + public YamlDataAttribute(bool withParameterNames = true, string path = null, string extension = ".yml") + : base(path, extension) + { WithParameterNames = withParameterNames; - Path = path; - Extension = extension; } public override IEnumerable GetData(MethodInfo testMethod) { diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs index b2330574108..333eacf83c4 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs @@ -7,6 +7,7 @@ using JetBrains.Annotations; using Xunit.Abstractions; using Xunit.Sdk; +using XUnitExtensions; using YamlDotNet.Core; using YamlDotNet.Core.Events; using YamlDotNet.Serialization; @@ -15,8 +16,8 @@ using YamlDotNet.Serialization.Utilities; namespace DafnyDriver.Test.XUnitExtensions { - public class YamlDataDiscoverer : IDataDiscoverer { - public virtual IParser GetYamlParser(string fileName, Stream stream) { + public class YamlDataDiscoverer : FileDataDiscoverer { + public virtual IParser GetYamlParser(string fileName, Stream stream) { return new Parser(new StreamReader(stream)); } @@ -36,11 +37,18 @@ private static Func ForMethodInfoDeserializeFn(IDeseriali }; } - public bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { + public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { return true; } - private IEnumerable FileData(MethodInfo testMethod, string fileName, bool withParameterNames) { + protected override IEnumerable FileData(IAttributeInfo dataAttribute, IMethodInfo testMethod, string fileName) { + var withParameterNames = dataAttribute.GetNamedArgument(nameof(YamlDataAttribute.WithParameterNames)); + // YamlDotNet's deserialization framework requires runtime type information + MethodInfo methodInfo = testMethod.ToRuntimeMethod(); + if (methodInfo == null) { + return null; + } + try { using Stream stream = File.OpenRead(fileName); IParser parser = GetYamlParser(fileName, stream); @@ -55,7 +63,7 @@ private IEnumerable FileData(MethodInfo testMethod, string fileName, b if (withParameterNames) { IObjectFactory argumentsFactory = new DefaultObjectFactory(); INodeDeserializer collectionDeserializer = new CollectionNodeDeserializer(argumentsFactory); - var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, testMethod); + var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, methodInfo); if (collectionDeserializer.Deserialize(parser, typeof(List), nestedObjectDeserializer, out var value)) { List argumentses = (List) value; @@ -64,7 +72,7 @@ private IEnumerable FileData(MethodInfo testMethod, string fileName, b throw new ArgumentException(); } } else { - IEnumerable parameters = testMethod.GetParameters(); + IEnumerable parameters = methodInfo.GetParameters(); Type targetType = typeof(IEnumerable<>).MakeGenericType(parameters.Single().ParameterType); IEnumerable results = (IEnumerable) deserializer.Deserialize(parser, targetType); return results.Select(value => new[] {value}); @@ -74,42 +82,6 @@ private IEnumerable FileData(MethodInfo testMethod, string fileName, b "Exception thrown while trying to deserialize test data from file: " + fileName, e); } } - - public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo testMethod) { - // YamlDotNet's deserialization framework requires runtime type information - MethodInfo methodInfo = testMethod.ToRuntimeMethod(); - if (methodInfo == null) { - return null; - } - - List attributeArgs = attributeInfo.GetConstructorArguments().ToList(); - var withParameterNames = attributeInfo.GetNamedArgument("WithParameterNames"); - var path = attributeInfo.GetNamedArgument("Path"); - var extension = attributeInfo.GetNamedArgument("Extension"); - - return GetData(methodInfo, withParameterNames, path, extension); - } - - public IEnumerable GetData(MethodInfo methodInfo, bool withParameterNames, string path, string extension) { - if (path == null) { - path = Path.Combine("TestFiles", methodInfo.DeclaringType.Name, methodInfo.Name); - } - if (extension == null) { - extension = ".yml"; - } - - if (Directory.Exists(path)) { - return Directory.EnumerateFiles(path, "*" + extension, SearchOption.AllDirectories) - .SelectMany(childPath => FileData(methodInfo, childPath, withParameterNames)); - } - - var yamlFileName = path + ".yml"; - if (File.Exists(yamlFileName)) { - return FileData(methodInfo, yamlFileName, withParameterNames); - } - - throw new ArgumentException("No data found for path: " + path); - } } public class MethodArguments : IYamlConvertible { diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs index dc6a8e423f4..46c4e199aa4 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs @@ -16,9 +16,9 @@ public class YamlDataTests { private void AssertTheoryData(string methodName, IEnumerable expectedData) { var method = typeof(YamlDataTests).GetMethod(methodName); - var attribute = (YamlDataAttribute)Attribute.GetCustomAttribute(method!, typeof(YamlDataAttribute)); + var attribute = method!.GetCustomAttributesData().First(a => a.AttributeType == typeof(YamlDataAttribute)); var discoverer = new YamlDataDiscoverer(); - Assert.Equal(expectedData, discoverer.GetData(method, attribute!.WithParameterNames, attribute.Path, attribute.Extension)); + Assert.Equal(expectedData, discoverer.GetData(Reflector.Wrap(attribute), Reflector.Wrap(method))); } [Theory] diff --git a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs index 39b37edf0cf..36a128da8f0 100644 --- a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs +++ b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs @@ -21,10 +21,7 @@ private static IEnumerable ReadLines(StreamReader reader) { [Fact] public void HelloWorld() { var convertor = new LitTestConvertor(); - using var stream = - Assembly.GetExecutingAssembly().GetManifestResourceStream("LitTestConvertor.Test.TestFiles.HelloWorldLitTest.dfy"); - using var reader = new StreamReader(stream); - var lines = ReadLines(reader); + var lines = File.ReadLines("TestFiles/HelloWorldLitTest.dfy"); var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles/HelloWorldLitTest.dfy", lines); DafnyTestSpec spec = (DafnyTestSpec)testCases; Assert.Equal(3, spec.Compile); @@ -36,7 +33,7 @@ public void VerifyOnly() { using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("LitTestConvertor.Test.TestFiles.VerifyOnlyLitTest.dfy"); using var reader = new StreamReader(stream); - var lines = ReadLines(reader); + var lines = File.ReadLines("TestFiles/VerifyOnly.dfy"); var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles/VerifyOnly.dfy", lines); DafnyTestSpec spec = (DafnyTestSpec)testCases; Assert.Equal(0, spec.Compile); diff --git a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj index 6a01d1d0515..fefd3537aa2 100644 --- a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj +++ b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj @@ -24,7 +24,10 @@ - + + + PreserveNewest + diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs new file mode 100644 index 00000000000..6ebc059bbb9 --- /dev/null +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using DafnyDriver.Test.XUnitExtensions; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace DafnyDriver.Test { + public class LitTestDataDiscoverer : FileDataDiscoverer + { + + private readonly LitTestConvertor.LitTestConvertor convertor = new(); + + private static DafnyTestSpec SpecForFileName(string fileName) { + string filePath = fileName.Substring("TestFiles/DafnyTests/Test".Length + 1); + return new DafnyTestSpec(filePath); + } + + public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { + return true; + } + + protected override IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName) { + try { + var (testCases, _) = convertor.ConvertLitCommands(fileName, File.ReadLines(fileName)); + return testCases.Select(testCase => new[] { testCase }); + } catch (Exception e) { + // Ignore for now + return Enumerable.Empty(); + } + } + } +} + +[DataDiscoverer("DafnyDriver.Test.LitTestDataDiscoverer", "LitTestConvertor.Test")] +public class LitTestDataAttribute : FileDataAttribute { + public LitTestDataAttribute() : base(extension: ".dfy") { + } +} \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs new file mode 100644 index 00000000000..2e247e81926 --- /dev/null +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -0,0 +1,14 @@ +using DafnyDriver.Test.XUnitExtensions; +using Xunit; + +namespace LitTestConvertor.Test +{ + public class LitTestRunner + { + [Theory] + [LitTestData] + public void LitTest(CLITestCase testCase) { + testCase.Run(); + } + } +} \ No newline at end of file From 07216c9b15046ebd757eb84ab8f176db231dc274 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 23 Aug 2021 12:49:47 -0700 Subject: [PATCH 108/192] More progress on debugging lit tests directly --- Source/DafnyDriver.Test/DafnyTestCase.cs | 2 +- Source/DafnyDriver.Test/DafnyTestSpec.cs | 9 +++------ Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs | 6 +----- Source/DafnyLanguageServer/DafnyLanguageServer.csproj | 1 - Source/LitTestConverter/LitTestConvertor.cs | 5 ++++- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index f05add03219..801dfc308f1 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -7,7 +7,7 @@ namespace DafnyDriver.Test { /** * Specialization of CLITestCase that mainly exists to support a much more - * consice definition of ToString(). + * concise definition of ToString(). */ public class DafnyTestCase : CLITestCase { diff --git a/Source/DafnyDriver.Test/DafnyTestSpec.cs b/Source/DafnyDriver.Test/DafnyTestSpec.cs index 18cd5a86d3d..0247e987a3d 100644 --- a/Source/DafnyDriver.Test/DafnyTestSpec.cs +++ b/Source/DafnyDriver.Test/DafnyTestSpec.cs @@ -84,10 +84,10 @@ public int? Compile { } [YamlIgnore] - public string CompileTarget { + public object CompileTarget { get { if (DafnyArguments.TryGetValue(DAFNY_COMPILE_TARGET_OPTION, out var compileTarget)) { - return (string) compileTarget; + return compileTarget; } return null; } @@ -98,10 +98,7 @@ private IEnumerable ResolveCompile() { Compile = 3; } - if (Compile > 0 && CompileTarget == null) { - // TODO: Include c++ but flag as special case by default? - var compileTargets = new[] {"cs", "java", "go", "js"}; - + if (Compile > 0 && CompileTarget is ForEachArgumentList compileTargets) { var justVerify = new Dictionary(DafnyArguments) { [DAFNY_COMPILE_OPTION] = "0" }; diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs index 9ada976b844..30759413d3a 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs @@ -46,10 +46,6 @@ public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(SpecialCaseReason), SpecialCaseReason); } - public Expectation Adjust() { - return new Expectation(ExitCode, OutputFile == null ? null : "comp/" + OutputFile, SpecialCaseReason); - } - public override string ToString() { string result; if (ExitCode != 0) { @@ -153,7 +149,7 @@ public void Run() { Expected.ExitCode, result.ExitCode, result.StandardError)); } if (Expected.OutputFile != null) { - var expectedOutput = File.ReadAllText(Path.Combine(DafnyTestSpec.TEST_ROOT, Expected.OutputFile)); + var expectedOutput = File.ReadAllText(Expected.OutputFile); AssertWithDiff.Equal(expectedOutput, result.StandardOutput); } diff --git a/Source/DafnyLanguageServer/DafnyLanguageServer.csproj b/Source/DafnyLanguageServer/DafnyLanguageServer.csproj index f0e1d569c04..090b6a8a82a 100644 --- a/Source/DafnyLanguageServer/DafnyLanguageServer.csproj +++ b/Source/DafnyLanguageServer/DafnyLanguageServer.csproj @@ -16,7 +16,6 @@ - diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index e6ca5f290e9..15064a29eb5 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -100,7 +100,10 @@ public void ConvertLitTest(string filePath) { if (IsStandardVerifyOnly(testConfigs[0]) && testConfigs.Skip(1).All(IsStandardCompileAndRun) || testConfigs.Skip(1).All(IsStandardCompileAndRun)) { defaultCount++; - return (null, testContent); + // TODO: For now don't return a value, since we also have to split up the .expect + // files for this to work. + // return (null, testContent); + throw new ArgumentException("Default for-each-compiler test cases not yet supported"); } throw new ArgumentException("Multi-command lit tests require manual conversion"); From 24163278dc75e01f94d7fbb77905e3ec19e45ae4 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 25 Aug 2021 17:15:10 -0700 Subject: [PATCH 109/192] More progress on generalizing --- Source/DafnyDriver.Test/DafnyTestCase.cs | 27 +++-- Source/DafnyDriver.Test/DafnyTestSpec.cs | 14 --- .../XUnitExtensions/CLITestCase.cs | 99 ++++++++++++------- .../XUnitExtensions/FileDataDiscoverer.cs | 16 ++- Source/LitTestConverter/LitTestConvertor.cs | 38 +++---- .../LitTestRunner/LitTestDataDiscoverer.cs | 10 +- 6 files changed, 117 insertions(+), 87 deletions(-) diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index 801dfc308f1..bd557969712 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Reflection; using DafnyDriver.Test.XUnitExtensions; namespace DafnyDriver.Test { @@ -9,8 +11,23 @@ namespace DafnyDriver.Test { * Specialization of CLITestCase that mainly exists to support a much more * concise definition of ToString(). */ - public class DafnyTestCase : CLITestCase { + public class DafnyTestCase : CLITestCase + { + private static readonly Assembly DafnyDriverAssembly = Assembly.GetAssembly(typeof(Microsoft.Dafny.DafnyDriver)); + + private static readonly DirectoryInfo OUTPUT_ROOT = new(Directory.GetCurrentDirectory()); + + // TODO-RS: This is an ugly method of locating the project root. + // The proper fix is to run entirely out of the output directory, + // and the projects are at least partially configured to make that possible, + // but it's not quite working yet. + private static string DAFNY_ROOT = + OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; + + public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; + public static readonly string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; + private static readonly Dictionary defaultDafnyOptions = new() { ["countVerificationErrors"] = "0", @@ -30,11 +47,7 @@ private static IEnumerable OptionsToFullArguments(string sourcePath, optionsWithDefaults[key] = value; } - return new[] { - "run --no-build --project ", - DafnyTestSpec.DAFNY_PROJ, - " --" - }.Concat(OptionsToArguments(sourcePath, optionsWithDefaults, otherFiles)); + return OptionsToArguments(sourcePath, optionsWithDefaults, otherFiles); } private static IEnumerable OptionsToArguments(string sourcePath, Dictionary dafnyOptions, List otherFiles) { @@ -56,7 +69,7 @@ private static string ConfigPairToArgument(KeyValuePair pair) { public List OtherFiles = new(); public DafnyTestCase(string sourcePath, Dictionary dafnyOptions, List otherFiles, Expectation expected) - : base("dotnet", OptionsToFullArguments(sourcePath, dafnyOptions, otherFiles), expected) + : base(DafnyDriverAssembly, OptionsToFullArguments(sourcePath, dafnyOptions, otherFiles), expected) { SourcePath = sourcePath; DafnyOptions = dafnyOptions; diff --git a/Source/DafnyDriver.Test/DafnyTestSpec.cs b/Source/DafnyDriver.Test/DafnyTestSpec.cs index 0247e987a3d..48830b1db22 100644 --- a/Source/DafnyDriver.Test/DafnyTestSpec.cs +++ b/Source/DafnyDriver.Test/DafnyTestSpec.cs @@ -14,20 +14,6 @@ public class ForEachArgumentList : List { } public class DafnyTestSpec: IEnumerable { - private static readonly DirectoryInfo OUTPUT_ROOT = new(Directory.GetCurrentDirectory()); - - // TODO-RS: This is an ugly method of locating the project root. - // The proper fix is to run entirely out of the output directory, - // and the projects are at least partially configured to make that possible, - // but it's not quite working yet. - private static string DAFNY_ROOT = - OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; - - public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; - public static readonly string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - - public static readonly string DAFNY_PROJ = Path.Combine(DAFNY_ROOT, "Source/DafnyDriver/DafnyDriver.csproj"); - // Dafny options with special handling public const string DAFNY_COMPILE_OPTION = "compile"; public const string DAFNY_COMPILE_TARGET_OPTION = "compileTarget"; diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs index 30759413d3a..8213216f715 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs @@ -3,6 +3,8 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Reflection; +using System.Text; using Xunit; using Xunit.Abstractions; using Xunit.Sdk; @@ -12,8 +14,13 @@ namespace DafnyDriver.Test.XUnitExtensions { public class CLITestCase: IXunitSerializable { - public string CLIPath; - public string[] Arguments; + protected Assembly CLIAssembly; + protected string CLIAssemblyName; + + protected string[] Arguments; + protected string WorkingDirectory; + protected IEnumerable PassthroughEnvironmentVariables; + protected Expectation Expected; public class Expectation : IXunitSerializable { // May be null if the test doesn't need to check the output @@ -31,7 +38,6 @@ public Expectation(int exitCode, string outputFile, string specialCaseReason) { } public Expectation() { - } public void Deserialize(IXunitSerializationInfo info) { @@ -76,10 +82,8 @@ public CLIResult(int exitCode, string standardOutput, string standardError) { } } - public Expectation Expected; - - public CLITestCase(string cliPath, IEnumerable arguments, Expectation expected) { - CLIPath = cliPath; + public CLITestCase(Assembly cliAssembly, IEnumerable arguments, Expectation expected) { + CLIAssembly = cliAssembly; Arguments = arguments.ToArray(); Expected = expected; } @@ -89,58 +93,83 @@ public CLITestCase() { } public void Serialize(IXunitSerializationInfo info) { + info.AddValue(nameof(CLIAssemblyName), CLIAssemblyName); info.AddValue(nameof(Arguments), Arguments); info.AddValue(nameof(Expected), Expected); } public void Deserialize(IXunitSerializationInfo info) { + CLIAssemblyName = info.GetValue(nameof(CLIAssemblyName)); + CLIAssembly = AppDomain.CurrentDomain.GetAssemblies().First(a => a.FullName == CLIAssemblyName); + Arguments = info.GetValue(nameof(Arguments)); Expected = info.GetValue(nameof(Expected)); } - public CLIResult RunDafny(IEnumerable arguments) { - using (Process process = new Process()) { - process.StartInfo.FileName = CLIPath; - foreach (var argument in arguments) { - process.StartInfo.Arguments += " " + argument; - } - - process.StartInfo.UseShellExecute = false; - process.StartInfo.RedirectStandardOutput = true; - process.StartInfo.RedirectStandardError = true; - process.StartInfo.CreateNoWindow = true; - // Necessary for JS to find bignumber.js - process.StartInfo.WorkingDirectory = DafnyTestSpec.TEST_ROOT; - - // Only preserve specific whitelisted environment variables - process.StartInfo.EnvironmentVariables.Clear(); - process.StartInfo.EnvironmentVariables.Add("PATH", Environment.GetEnvironmentVariable("PATH")); - // Go requires this or GOCACHE - process.StartInfo.EnvironmentVariables.Add("HOME", Environment.GetEnvironmentVariable("HOME")); - - process.Start(); - string output = process.StandardOutput.ReadToEnd(); - string error = process.StandardError.ReadToEnd(); - process.WaitForExit(); - return new CLIResult(process.ExitCode, output, error); + public CLIResult InvokeCLI(IEnumerable arguments) { + if (Environment.GetEnvironmentVariable("XUNIT_INVOKE_CLI_DIRECTLY") == "true") { + return InvokeCLIDirectly(arguments); } + return InvokeCLIViaProcess(arguments); + } + + public CLIResult InvokeCLIDirectly(IEnumerable arguments) { + StringBuilder redirectedOut = new(); + StringBuilder redirectedErr = new(); + + Console.SetOut(new StringWriter(redirectedOut)); + Console.SetError(new StringWriter(redirectedErr)); + + int result = (int) CLIAssembly.EntryPoint.Invoke(null, new object[] { arguments }); + + return new CLIResult(result, redirectedOut.ToString(), redirectedErr.ToString()); + } + + public CLIResult InvokeCLIViaProcess(IEnumerable arguments) { + using var process = new Process(); + + process.StartInfo.FileName = "dotnet"; + process.StartInfo.Arguments = CLIAssembly.Location; + foreach (var argument in arguments) { + process.StartInfo.Arguments += " " + argument; + } + + process.StartInfo.UseShellExecute = false; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.CreateNoWindow = true; + // Necessary for JS to find bignumber.js + process.StartInfo.WorkingDirectory = WorkingDirectory; + + // Only preserve specific whitelisted environment variables + process.StartInfo.EnvironmentVariables.Clear(); + foreach(var passthrough in PassthroughEnvironmentVariables) { + process.StartInfo.EnvironmentVariables.Add(passthrough, Environment.GetEnvironmentVariable(passthrough)); + } + + process.Start(); + string output = process.StandardOutput.ReadToEnd(); + string error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + return new CLIResult(process.ExitCode, output, error); } public void Run() { CLIResult result; if (Arguments.Any(arg => arg.StartsWith("/out"))) { - result = RunDafny(Arguments); + result = InvokeCLI(Arguments); } else { // Note that the temporary directory has to be an ancestor of Test // or else Javascript won't be able to locate bignumber.js :( - using var tempDir = new TemporaryDirectory(DafnyTestSpec.OUTPUT_DIR); + using var tempDir = new TemporaryDirectory(DafnyTestCase.OUTPUT_DIR); // Add an extra component to the path to keep the files created inside the // temporary directory, since some compilers will // interpret the path as a single file basename rather than a directory. var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; var dafnyArguments = Arguments.Concat(new []{ outArgument }); - result = RunDafny(dafnyArguments); + result = InvokeCLI(dafnyArguments); } if (result.ExitCode != Expected.ExitCode) { diff --git a/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs index c5a263a35c2..f7adf0dde5f 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs @@ -10,14 +10,20 @@ namespace DafnyDriver.Test.XUnitExtensions { public abstract class FileDataDiscoverer : IDataDiscoverer { - public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo testMethod) { + protected string GetBasePath(IAttributeInfo attributeInfo, IMethodInfo testMethod) { var path = attributeInfo.GetNamedArgument(nameof(FileDataAttribute.Path)); + + if (path != null) { + return path; + } + + return Path.Combine("TestFiles", testMethod.ToRuntimeMethod().DeclaringType.Name, testMethod.Name); + } + + public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo testMethod) { + var path = GetBasePath(attributeInfo, testMethod); var extension = attributeInfo.GetNamedArgument(nameof(FileDataAttribute.Extension)); - if (path == null) { - path = Path.Combine("TestFiles", testMethod.ToRuntimeMethod().DeclaringType.Name, testMethod.Name); - } - if (Directory.Exists(path)) { return Directory.EnumerateFiles(path, "*" + extension, SearchOption.AllDirectories) .SelectMany(childPath => FileData(attributeInfo, testMethod, childPath)); diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 15064a29eb5..43b11c8204e 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -87,7 +87,7 @@ public void ConvertLitTest(string filePath) { } litCommands.RemoveAt(litCommands.Count - 1); - List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(filePath, c)).ToList(); + List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(filePath, c)).ToList(); if (testConfigs.Count == 1) { var single = testConfigs.Single(); @@ -133,9 +133,11 @@ private static string ExtractLitCommand(string line) { return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); } - private DafnyTestSpec ParseDafnyCommandArguments(string filePath, string dafnyCommand) { - var spec = new DafnyTestSpec(filePath); - + private CLITestCase ParseDafnyCommandArguments(string filePath, string dafnyCommand) { + bool includeThisFile = true; + List otherFiles = new(); + Dictionary dafnyArguments = new(); + if (!dafnyCommand.StartsWith(LIT_DAFNY)) { throw new ArgumentException("Lit command is not expected %dafny: " + dafnyCommand); } @@ -155,41 +157,39 @@ private DafnyTestSpec ParseDafnyCommandArguments(string filePath, string dafnyCo arguments.RemoveRange(arguments.Count - 2, 2); if (!arguments.Remove("\"%s\"")) { - spec.DafnyArguments[TEST_CONFIG_INCLUDE_THIS_FILE] = "no"; + includeThisFile = false; } // Check the arguments for anything non-standard foreach (var argument in arguments) { - KeyValuePair pair = ParseDafnyArgument(argument); - if (DAFNY_IGNORED_OPTIONS.Contains(pair.Key)) { + var (key, value) = ParseDafnyArgument(argument); + if (DAFNY_IGNORED_OPTIONS.Contains(key)) { continue; } - if (pair.Value.Contains("%")) { + if (value.Contains("%")) { throw new ArgumentException("Use of lit substitution (% variable) requires manual conversion: " + argument); } - if (pair.Key.Equals(TEST_CONFIG_OTHER_FILES)) { - spec.OtherFiles.Add(pair.Value); + if (key.Equals(TEST_CONFIG_OTHER_FILES)) { + otherFiles.Add(value); } else { - spec.DafnyArguments.Add(pair.Key, pair.Value); + dafnyArguments.Add(key, value); } } - return spec; + return new DafnyTestCase(); } - private static KeyValuePair ParseDafnyArgument(string argument) { + private static (string, string) ParseDafnyArgument(string argument) { if (argument.StartsWith("-") || argument.StartsWith("/")) { - argument = argument.Substring(1); + argument = argument[1..]; int colonIndex = argument.IndexOf(":"); if (colonIndex >= 0) { - return new KeyValuePair( - argument.Substring(0, colonIndex), - argument.Substring(colonIndex + 1)); + return (argument[..colonIndex], argument[(colonIndex + 1)..]); } else { - return new KeyValuePair(argument, "yes"); + return (argument, "yes"); } } else { - return new KeyValuePair(TEST_CONFIG_OTHER_FILES, argument); + return (TEST_CONFIG_OTHER_FILES, argument); } } diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs index 6ebc059bbb9..4c5721c5074 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -12,20 +12,16 @@ public class LitTestDataDiscoverer : FileDataDiscoverer private readonly LitTestConvertor.LitTestConvertor convertor = new(); - private static DafnyTestSpec SpecForFileName(string fileName) { - string filePath = fileName.Substring("TestFiles/DafnyTests/Test".Length + 1); - return new DafnyTestSpec(filePath); - } - public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { return true; } protected override IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName) { + string shortName = fileName[(GetBasePath(attributeInfo, testMethod).Length + 1)..]; try { - var (testCases, _) = convertor.ConvertLitCommands(fileName, File.ReadLines(fileName)); + var (testCases, _) = convertor.ConvertLitCommands(shortName, File.ReadLines(fileName)); return testCases.Select(testCase => new[] { testCase }); - } catch (Exception e) { + } catch (Exception) { // Ignore for now return Enumerable.Empty(); } From 7613ba5dffdfd3a8c6f65e7376f81da46c934c47 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 26 Aug 2021 15:43:06 -0700 Subject: [PATCH 110/192] More progress - can't find z3 yet --- Source/Dafny/DafnyPipeline.csproj | 3 ++ Source/DafnyDriver.Test/DafnyTestCase.cs | 32 ++++++++-------- .../XUnitExtensions/CLITestCase.cs | 38 +++++++------------ Source/LitTestConverter/LitTestConvertor.cs | 21 +++------- .../LitTestConvertor.Test.csproj | 2 +- .../LitTestRunner/LitTestDataDiscoverer.cs | 4 +- .../LitTestDataDiscovererTest.cs | 28 ++++++++++++++ .../LitTestRunner/LitTests.cs | 4 +- 8 files changed, 71 insertions(+), 61 deletions(-) create mode 100644 Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs diff --git a/Source/Dafny/DafnyPipeline.csproj b/Source/Dafny/DafnyPipeline.csproj index a333ab826e1..5cd1eda943e 100644 --- a/Source/Dafny/DafnyPipeline.csproj +++ b/Source/Dafny/DafnyPipeline.csproj @@ -34,4 +34,7 @@ + + + diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index bd557969712..cd1620d1035 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reflection; using DafnyDriver.Test.XUnitExtensions; +using Xunit.Abstractions; namespace DafnyDriver.Test { @@ -16,18 +17,6 @@ public class DafnyTestCase : CLITestCase private static readonly Assembly DafnyDriverAssembly = Assembly.GetAssembly(typeof(Microsoft.Dafny.DafnyDriver)); - private static readonly DirectoryInfo OUTPUT_ROOT = new(Directory.GetCurrentDirectory()); - - // TODO-RS: This is an ugly method of locating the project root. - // The proper fix is to run entirely out of the output directory, - // and the projects are at least partially configured to make that possible, - // but it's not quite working yet. - private static string DAFNY_ROOT = - OUTPUT_ROOT.Parent.Parent.Parent.Parent.Parent.FullName; - - public static readonly string TEST_ROOT = Path.Combine(DAFNY_ROOT, "Test") + Path.DirectorySeparatorChar; - public static readonly string OUTPUT_DIR = Path.Combine(TEST_ROOT, "Output") + Path.DirectorySeparatorChar; - private static readonly Dictionary defaultDafnyOptions = new() { ["countVerificationErrors"] = "0", @@ -63,24 +52,37 @@ private static string ConfigPairToArgument(KeyValuePair pair) { return $"/{pair.Key}:{pair.Value}"; } - public readonly string SourcePath; + public string SourcePath; public Dictionary DafnyOptions = new(); public List OtherFiles = new(); public DafnyTestCase(string sourcePath, Dictionary dafnyOptions, List otherFiles, Expectation expected) - : base(DafnyDriverAssembly, OptionsToFullArguments(sourcePath, dafnyOptions, otherFiles), expected) + : base(DafnyDriverAssembly, OptionsToFullArguments(sourcePath, dafnyOptions, otherFiles), null, new List(), expected) { SourcePath = sourcePath; DafnyOptions = dafnyOptions; OtherFiles = otherFiles; - Expected = expected; } public DafnyTestCase() { } + public void Serialize(IXunitSerializationInfo info) { + base.Serialize(info); + info.AddValue(nameof(SourcePath), SourcePath); + info.AddValue(nameof(DafnyOptions), DafnyOptions); + info.AddValue(nameof(OtherFiles), OtherFiles); + } + + public void Deserialize(IXunitSerializationInfo info) { + base.Deserialize(info); + SourcePath = info.GetValue(nameof(SourcePath)); + DafnyOptions = info.GetValue>(nameof(DafnyOptions)); + OtherFiles = info.GetValue>(nameof(OtherFiles)); + } + public override string ToString() { return String.Join(" ", OptionsToArguments(SourcePath, DafnyOptions, OtherFiles)) + " => " + Expected; } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs index 8213216f715..215d9903b06 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs @@ -82,9 +82,13 @@ public CLIResult(int exitCode, string standardOutput, string standardError) { } } - public CLITestCase(Assembly cliAssembly, IEnumerable arguments, Expectation expected) { + public CLITestCase(Assembly cliAssembly, IEnumerable arguments, + string workingDirectory, IEnumerable passthroughEnvironmentVariables, + Expectation expected) { CLIAssembly = cliAssembly; Arguments = arguments.ToArray(); + WorkingDirectory = workingDirectory; + PassthroughEnvironmentVariables = passthroughEnvironmentVariables; Expected = expected; } @@ -106,31 +110,31 @@ public void Deserialize(IXunitSerializationInfo info) { Expected = info.GetValue(nameof(Expected)); } - public CLIResult InvokeCLI(IEnumerable arguments) { + public CLIResult InvokeCLI() { if (Environment.GetEnvironmentVariable("XUNIT_INVOKE_CLI_DIRECTLY") == "true") { - return InvokeCLIDirectly(arguments); + return InvokeCLIDirectly(); } - return InvokeCLIViaProcess(arguments); + return InvokeCLIViaProcess(); } - public CLIResult InvokeCLIDirectly(IEnumerable arguments) { + public CLIResult InvokeCLIDirectly() { StringBuilder redirectedOut = new(); StringBuilder redirectedErr = new(); Console.SetOut(new StringWriter(redirectedOut)); Console.SetError(new StringWriter(redirectedErr)); - int result = (int) CLIAssembly.EntryPoint.Invoke(null, new object[] { arguments }); + int result = (int) CLIAssembly.EntryPoint.Invoke(null, new object[] { Arguments }); return new CLIResult(result, redirectedOut.ToString(), redirectedErr.ToString()); } - public CLIResult InvokeCLIViaProcess(IEnumerable arguments) { + public CLIResult InvokeCLIViaProcess() { using var process = new Process(); process.StartInfo.FileName = "dotnet"; process.StartInfo.Arguments = CLIAssembly.Location; - foreach (var argument in arguments) { + foreach (var argument in Arguments) { process.StartInfo.Arguments += " " + argument; } @@ -138,10 +142,8 @@ public CLIResult InvokeCLIViaProcess(IEnumerable arguments) { process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.StartInfo.CreateNoWindow = true; - // Necessary for JS to find bignumber.js process.StartInfo.WorkingDirectory = WorkingDirectory; - // Only preserve specific whitelisted environment variables process.StartInfo.EnvironmentVariables.Clear(); foreach(var passthrough in PassthroughEnvironmentVariables) { process.StartInfo.EnvironmentVariables.Add(passthrough, Environment.GetEnvironmentVariable(passthrough)); @@ -156,21 +158,7 @@ public CLIResult InvokeCLIViaProcess(IEnumerable arguments) { } public void Run() { - CLIResult result; - if (Arguments.Any(arg => arg.StartsWith("/out"))) { - result = InvokeCLI(Arguments); - } else { - // Note that the temporary directory has to be an ancestor of Test - // or else Javascript won't be able to locate bignumber.js :( - using var tempDir = new TemporaryDirectory(DafnyTestCase.OUTPUT_DIR); - - // Add an extra component to the path to keep the files created inside the - // temporary directory, since some compilers will - // interpret the path as a single file basename rather than a directory. - var outArgument = "/out:" + tempDir.DirInfo.FullName + "/Program"; - var dafnyArguments = Arguments.Concat(new []{ outArgument }); - result = InvokeCLI(dafnyArguments); - } + CLIResult result = InvokeCLI(); if (result.ExitCode != Expected.ExitCode) { throw new AssertActualExpectedException(Expected.ExitCode, result.ExitCode, diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 43b11c8204e..1f12d4c9459 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -90,22 +90,9 @@ public void ConvertLitTest(string filePath) { List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(filePath, c)).ToList(); if (testConfigs.Count == 1) { - var single = testConfigs.Single(); - if (IsStandardVerifyOnly(single)) { - verifyOnlyCount++; - } - return (single, testContent); + return (testConfigs, testContent); } - if (IsStandardVerifyOnly(testConfigs[0]) && testConfigs.Skip(1).All(IsStandardCompileAndRun) - || testConfigs.Skip(1).All(IsStandardCompileAndRun)) { - defaultCount++; - // TODO: For now don't return a value, since we also have to split up the .expect - // files for this to work. - // return (null, testContent); - throw new ArgumentException("Default for-each-compiler test cases not yet supported"); - } - throw new ArgumentException("Multi-command lit tests require manual conversion"); } @@ -136,7 +123,7 @@ private static string ExtractLitCommand(string line) { private CLITestCase ParseDafnyCommandArguments(string filePath, string dafnyCommand) { bool includeThisFile = true; List otherFiles = new(); - Dictionary dafnyArguments = new(); + Dictionary dafnyArguments = new(); if (!dafnyCommand.StartsWith(LIT_DAFNY)) { throw new ArgumentException("Lit command is not expected %dafny: " + dafnyCommand); @@ -176,7 +163,9 @@ private CLITestCase ParseDafnyCommandArguments(string filePath, string dafnyComm } } - return new DafnyTestCase(); + var expected = new CLITestCase.Expectation(0, filePath + ".expect", null); + + return new DafnyTestCase(filePath, dafnyArguments, otherFiles, expected); } private static (string, string) ParseDafnyArgument(string argument) { diff --git a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj index fefd3537aa2..86458e42bac 100644 --- a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj +++ b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj @@ -25,7 +25,7 @@ - + PreserveNewest diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs index 4c5721c5074..43e6929756c 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -17,9 +17,9 @@ public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, } protected override IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName) { - string shortName = fileName[(GetBasePath(attributeInfo, testMethod).Length + 1)..]; + // string shortName = fileName[(GetBasePath(attributeInfo, testMethod).Length + 1)..]; try { - var (testCases, _) = convertor.ConvertLitCommands(shortName, File.ReadLines(fileName)); + var (testCases, _) = convertor.ConvertLitCommands(fileName, File.ReadLines(fileName)); return testCases.Select(testCase => new[] { testCase }); } catch (Exception) { // Ignore for now diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs new file mode 100644 index 00000000000..ec5bbe8671c --- /dev/null +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs @@ -0,0 +1,28 @@ +using System; +using System.Linq; +using System.Reflection; +using DafnyDriver.Test; +using DafnyDriver.Test.XUnitExtensions; +using Xunit; +using Xunit.Sdk; + +namespace LitTestConvertor.Test.LitTestRunner { + public class LitTestDataDiscovererTest { + + MethodInfo GetMethodInfo(Action a) + { + return a.Method; + } + + [Fact] + public void Discoverer() { + var discoverer = new LitTestDataDiscoverer(); + var test = new LitTests(); + var methodInfo = GetMethodInfo(test.LitTest); + var method = Reflector.Wrap(GetMethodInfo(test.LitTest)); + var attribute = CustomAttributeData.GetCustomAttributes(methodInfo).First(a => a.AttributeType == typeof(LitTestDataAttribute)); + var xunitAttribute = Reflector.Wrap(attribute); + discoverer.GetData(xunitAttribute, method); + } + } +} \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index 2e247e81926..f05ace22353 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -1,9 +1,9 @@ using DafnyDriver.Test.XUnitExtensions; using Xunit; -namespace LitTestConvertor.Test +namespace LitTestConvertor.Test.LitTestRunner { - public class LitTestRunner + public class LitTests { [Theory] [LitTestData] From f7033ee323adece92986f9909c0a731cf16c082b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 27 Aug 2021 16:02:05 -0700 Subject: [PATCH 111/192] More clean up and removing code out of scope for now --- Source/DafnyDriver.Test/DafnyTestCase.cs | 30 ++-- Source/DafnyDriver.Test/DafnyTestSpec.cs | 150 ------------------ .../DafnyTestYamlDataDiscoverer.cs | 63 -------- Source/DafnyDriver.Test/IntegrationTest.cs | 15 -- .../XUnitExtensions/CLITestCase.cs | 6 +- Source/LitTestConverter/LitTestConvertor.cs | 42 +---- .../Conversion/ConvertingFromLitToXunit.cs | 8 +- .../LitTestRunner/LitTestDataDiscoverer.cs | 3 +- .../LitTestRunner/LitTests.cs | 3 +- 9 files changed, 32 insertions(+), 288 deletions(-) delete mode 100644 Source/DafnyDriver.Test/DafnyTestSpec.cs delete mode 100644 Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs delete mode 100644 Source/DafnyDriver.Test/IntegrationTest.cs diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index cd1620d1035..eaacc2c1194 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -12,10 +12,9 @@ namespace DafnyDriver.Test { * Specialization of CLITestCase that mainly exists to support a much more * concise definition of ToString(). */ - public class DafnyTestCase : CLITestCase - { + public class DafnyTestCase : CLITestCase { - private static readonly Assembly DafnyDriverAssembly = Assembly.GetAssembly(typeof(Microsoft.Dafny.DafnyDriver)); + private static readonly Assembly dafnyDriverAssembly = Assembly.GetAssembly(typeof(Microsoft.Dafny.DafnyDriver)); private static readonly Dictionary defaultDafnyOptions = new() { ["countVerificationErrors"] = "0", @@ -25,7 +24,10 @@ public class DafnyTestCase : CLITestCase // We do not want output such as "Compiled program written to Foo.cs" // from the compilers, since that changes with the target language - ["compileVerbose"] = "0" + ["compileVerbose"] = "0", + + // Hide Boogie execution traces since they are meaningless for Dafny programs + ["errorTrace"] = "0" }; private static IEnumerable OptionsToFullArguments(string sourcePath, @@ -52,15 +54,16 @@ private static string ConfigPairToArgument(KeyValuePair pair) { return $"/{pair.Key}:{pair.Value}"; } - public string SourcePath; + private string BasePath; + private string SourcePath; - public Dictionary DafnyOptions = new(); - public List OtherFiles = new(); + private Dictionary DafnyOptions = new(); + private List OtherFiles = new(); - public DafnyTestCase(string sourcePath, Dictionary dafnyOptions, List otherFiles, Expectation expected) - : base(DafnyDriverAssembly, OptionsToFullArguments(sourcePath, dafnyOptions, otherFiles), null, new List(), expected) - { - SourcePath = sourcePath; + public DafnyTestCase(string basePath, string fullSourcePath, Dictionary dafnyOptions, List otherFiles, Expectation expected) + : base(dafnyDriverAssembly, OptionsToFullArguments(fullSourcePath, dafnyOptions, otherFiles), new List(), expected) { + BasePath = basePath; + SourcePath = fullSourcePath; DafnyOptions = dafnyOptions; OtherFiles = otherFiles; } @@ -71,6 +74,7 @@ public DafnyTestCase() { public void Serialize(IXunitSerializationInfo info) { base.Serialize(info); + info.AddValue(nameof(BasePath), BasePath); info.AddValue(nameof(SourcePath), SourcePath); info.AddValue(nameof(DafnyOptions), DafnyOptions); info.AddValue(nameof(OtherFiles), OtherFiles); @@ -78,13 +82,15 @@ public void Serialize(IXunitSerializationInfo info) { public void Deserialize(IXunitSerializationInfo info) { base.Deserialize(info); + BasePath = info.GetValue(nameof(BasePath)); SourcePath = info.GetValue(nameof(SourcePath)); DafnyOptions = info.GetValue>(nameof(DafnyOptions)); OtherFiles = info.GetValue>(nameof(OtherFiles)); } public override string ToString() { - return String.Join(" ", OptionsToArguments(SourcePath, DafnyOptions, OtherFiles)) + " => " + Expected; + var relativePath = SourcePath[(BasePath.Length + 1)..]; + return String.Join(" ", OptionsToArguments(relativePath, DafnyOptions, OtherFiles)); } } } \ No newline at end of file diff --git a/Source/DafnyDriver.Test/DafnyTestSpec.cs b/Source/DafnyDriver.Test/DafnyTestSpec.cs deleted file mode 100644 index 48830b1db22..00000000000 --- a/Source/DafnyDriver.Test/DafnyTestSpec.cs +++ /dev/null @@ -1,150 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using DafnyDriver.Test.XUnitExtensions; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; -using YamlDotNet.Serialization.ObjectFactories; - -namespace DafnyDriver.Test -{ - public class ForEachArgumentList : List { } - - public class DafnyTestSpec: IEnumerable { - - // Dafny options with special handling - public const string DAFNY_COMPILE_OPTION = "compile"; - public const string DAFNY_COMPILE_TARGET_OPTION = "compileTarget"; - public const string DAFNY_NO_VERIFY_OPTION = "noVerify"; - - // Absolute file system path to the main Dafny file - [YamlIgnore] - public readonly string SourcePath; - - public Dictionary DafnyArguments = new(); - public List OtherFiles = new(); - - public Dictionary CompileTargetOverrides = new(); - - public CLITestCase.Expectation Expected; - - public DafnyTestSpec(string sourcePath) { - SourcePath = sourcePath; - } - - private DafnyTestSpec(string sourcePath, Dictionary dafnyArguments, List otherFiles, CLITestCase.Expectation expected) { - SourcePath = sourcePath; - DafnyArguments = dafnyArguments; - OtherFiles = otherFiles; - Expected = expected; - } - - public IEnumerator GetEnumerator() { - return ResolveCompile() - .SelectMany(spec => spec.ExpandArguments()) - .Select(spec => spec.ResolveExpected().ToTestCase()) - .GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() { - return GetEnumerator(); - } - - [YamlIgnore] - public int? Compile { - get { - if (DafnyArguments.TryGetValue(DAFNY_COMPILE_OPTION, out var compile)) { - return Int32.Parse((string) compile); - } - return null; - } - set { - if (value != null) { - DafnyArguments[DAFNY_COMPILE_OPTION] = value.ToString(); - } else { - DafnyArguments.Remove(DAFNY_COMPILE_OPTION); - } - } - } - - [YamlIgnore] - public object CompileTarget { - get { - if (DafnyArguments.TryGetValue(DAFNY_COMPILE_TARGET_OPTION, out var compileTarget)) { - return compileTarget; - } - return null; - } - } - - private IEnumerable ResolveCompile() { - if (Compile == null) { - Compile = 3; - } - - if (Compile > 0 && CompileTarget is ForEachArgumentList compileTargets) { - var justVerify = new Dictionary(DafnyArguments) { - [DAFNY_COMPILE_OPTION] = "0" - }; - yield return new DafnyTestSpec(SourcePath, justVerify, OtherFiles, CLITestCase.Expectation.NO_OUTPUT); - - foreach (var compileTarget in compileTargets) { - var withLanguage = new Dictionary(DafnyArguments) { - [DAFNY_NO_VERIFY_OPTION] = "yes", - [DAFNY_COMPILE_OPTION] = Compile > 2 ? "4" : "2", - [DAFNY_COMPILE_TARGET_OPTION] = compileTarget - }; - var specForLanguage = new DafnyTestSpec(SourcePath, withLanguage, OtherFiles, Expected); - if (CompileTargetOverrides.TryGetValue(compileTarget, out var compileTargetOverride)) { - yield return specForLanguage.ApplyOverride(compileTargetOverride); - } else { - yield return specForLanguage; - } - } - } else { - yield return this; - } - } - - private DafnyTestSpec ApplyOverride(DafnyTestSpec otherSpec) { - var mergedArguments = new Dictionary(DafnyArguments); - foreach (var (key, value) in otherSpec.DafnyArguments) { - mergedArguments[key] = value; - } - return new DafnyTestSpec(otherSpec.SourcePath, mergedArguments, OtherFiles.Concat(otherSpec.OtherFiles).ToList(), otherSpec.Expected); - } - - private DafnyTestSpec ResolveExpected() { - if (Expected == null) { - return new DafnyTestSpec(SourcePath, DafnyArguments, OtherFiles, - new CLITestCase.Expectation(0, SourcePath + ".expect", null)); - } - return this; - } - - private DafnyTestCase ToTestCase() { - return new DafnyTestCase(SourcePath, DafnyArguments, OtherFiles, Expected); - } - - private IEnumerable ExpandArguments() { - return DafnyArguments.Select(ExpandValue) - .CartesianProduct() - .Select(e => e.ToDictionary(p => p.Key, p => p.Value)) - .Select(args => new DafnyTestSpec(SourcePath, args, OtherFiles, Expected)); - } - - public static IEnumerable Expand(object obj) { - if (obj is ForEachArgumentList forEachArgumentList) { - return forEachArgumentList; - } else { - return new[] {(string)obj}; - } - } - - private static IEnumerable> ExpandValue(KeyValuePair pair) { - return Expand(pair.Value).Select(v => new KeyValuePair(pair.Key, v)); - } - } -} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs b/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs deleted file mode 100644 index b6e35d4e5d9..00000000000 --- a/Source/DafnyDriver.Test/DafnyTestYamlDataDiscoverer.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System.IO; -using DafnyDriver.Test.XUnitExtensions; -using Xunit.Sdk; -using XUnitExtensions; -using YamlDotNet.Core; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; -using YamlDotNet.Serialization.ObjectFactories; - -namespace DafnyDriver.Test { - public class DafnyTestYamlDataDiscoverer : YamlDataDiscoverer { - - private const string DEFAULT_CONFIG = -@"!dafnyTestSpec -dafnyArguments: {} -"; - - private static DafnyTestSpec SpecForFileName(string fileName) { - string filePath = fileName.Substring("TestFiles/DafnyTests/Test".Length + 1); - return new DafnyTestSpec(filePath); - } - - public static string GetTestCaseConfigYaml(string fullText) { - var commentStart = fullText.IndexOf("/*"); - if (commentStart >= 0) { - var commentEnd = fullText.IndexOf("*/", commentStart + 2); - if (commentEnd >= 0) { - var commentContent = fullText.Substring(commentStart + 2, commentEnd - commentStart - 2).Trim(); - if (commentContent.StartsWith("---")) { - return commentContent; - } - } - } - - return null; - } - - public override IParser GetYamlParser(string fileName, Stream stream) { - string content = GetTestCaseConfigYaml(new StreamReader(stream).ReadToEnd()) ?? DEFAULT_CONFIG; - - return new Parser(new StringReader(content)); - } - - public override IDeserializer GetDeserializer(string fileName) { - var defaultObjectFactory = new DefaultObjectFactory(); - var customObjectFactory = new LambdaObjectFactory(type => - type == typeof(DafnyTestSpec) ? SpecForFileName(fileName) : defaultObjectFactory.Create(type)); - - return new DeserializerBuilder() - .WithNamingConvention(CamelCaseNamingConvention.Instance) - .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) - .WithTagMapping("!foreach", typeof(ForEachArgumentList)) - .WithObjectFactory(customObjectFactory) - .Build(); - } - } -} - -[DataDiscoverer("DafnyDriver.Test.DafnyTestYamlDataDiscoverer", "DafnyDriver.Test")] -public class DafnyTestDataAttribute : YamlDataAttribute { - public DafnyTestDataAttribute() : base(false, null, ".dfy") { - } -} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/IntegrationTest.cs b/Source/DafnyDriver.Test/IntegrationTest.cs deleted file mode 100644 index b0bca4c36f7..00000000000 --- a/Source/DafnyDriver.Test/IntegrationTest.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System.Linq; -using DafnyDriver.Test.XUnitExtensions; -using Xunit; - -namespace DafnyDriver.Test { - - public static class DafnyTests { - - [ParallelTheory] - [DafnyTestData] - public static void Test(CLITestCase testCase) { - testCase.Run(); - } - } -} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs index 215d9903b06..0c845837387 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs @@ -83,11 +83,10 @@ public CLIResult(int exitCode, string standardOutput, string standardError) { } public CLITestCase(Assembly cliAssembly, IEnumerable arguments, - string workingDirectory, IEnumerable passthroughEnvironmentVariables, + IEnumerable passthroughEnvironmentVariables, Expectation expected) { CLIAssembly = cliAssembly; Arguments = arguments.ToArray(); - WorkingDirectory = workingDirectory; PassthroughEnvironmentVariables = passthroughEnvironmentVariables; Expected = expected; } @@ -141,8 +140,7 @@ public CLIResult InvokeCLIViaProcess() { process.StartInfo.UseShellExecute = false; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; - process.StartInfo.CreateNoWindow = true; - process.StartInfo.WorkingDirectory = WorkingDirectory; + process.StartInfo.CreateNoWindow = true; process.StartInfo.EnvironmentVariables.Clear(); foreach(var passthrough in PassthroughEnvironmentVariables) { diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 1f12d4c9459..0802a856a87 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -20,36 +20,25 @@ public class LitTestConvertor { public const string TEST_CONFIG_OTHER_FILES = "otherFiles"; public const string TEST_CONFIG_INCLUDE_THIS_FILE = "includeThisFile"; - // Arguments that are taken care of automatically. If a test is actually using the output of - // one of these as input in another command, that will be flagged as an unsupported - // use of lit substitution. - private static readonly string[] DAFNY_IGNORED_OPTIONS = { - "print", - "dprint", - "rprint" - }; - private int count = 0; private int verifyOnlyCount = 0; private int defaultCount = 0; private int alreadyConverted = 0; private int invalidCount = 0; - public void ConvertLitTest(string filePath) { + public void ConvertLitTest(string basePath, string filePath) { IEnumerable testSpec; IEnumerable testContent; if (filePath.Contains("/Inputs/")) { testSpec = Enumerable.Empty(); testContent = File.ReadAllLines(filePath); } else { - (testSpec, testContent) = ConvertLitCommands(filePath, File.ReadAllLines(filePath)); + (testSpec, testContent) = ConvertLitCommands(basePath, filePath, File.ReadAllLines(filePath)); } ISerializer serializer = new SerializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) - .WithTagMapping("!dafnyTestSpec", typeof(DafnyTestSpec)) - .WithTagMapping("!foreach", typeof(ForEachArgumentList)) .Build(); using (StreamWriter file = new StreamWriter(filePath)) { @@ -65,7 +54,7 @@ public void ConvertLitTest(string filePath) { } } - public (IEnumerable spec, IEnumerable content) ConvertLitCommands(string filePath, IEnumerable lines) { + public (IEnumerable spec, IEnumerable content) ConvertLitCommands(string basePath, string filePath, IEnumerable lines) { var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); if (!litCommands.Any()) { throw new ArgumentException("No lit commands found"); @@ -87,7 +76,7 @@ public void ConvertLitTest(string filePath) { } litCommands.RemoveAt(litCommands.Count - 1); - List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(filePath, c)).ToList(); + List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(basePath, filePath, c)).ToList(); if (testConfigs.Count == 1) { return (testConfigs, testContent); @@ -96,18 +85,6 @@ public void ConvertLitTest(string filePath) { throw new ArgumentException("Multi-command lit tests require manual conversion"); } - private static bool IsStandardCompileAndRun(DafnyTestSpec spec) { - - return spec.CompileTarget != null && - ((spec.Compile == 3 && spec.DafnyArguments.Count == 2) || - // /compile:4 might be used with /noVerify - (spec.Compile == 4 && spec.DafnyArguments.Count <= 3)); - } - - private static bool IsStandardVerifyOnly(DafnyTestSpec spec) { - return spec.Compile == 0 && spec.DafnyArguments.Count == 1; - } - private static string ExtractLitCommand(string line) { if (!line.StartsWith(DAFNY_COMMENT_PREFIX)) { return null; @@ -120,7 +97,7 @@ private static string ExtractLitCommand(string line) { return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); } - private CLITestCase ParseDafnyCommandArguments(string filePath, string dafnyCommand) { + private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, string dafnyCommand) { bool includeThisFile = true; List otherFiles = new(); Dictionary dafnyArguments = new(); @@ -150,9 +127,6 @@ private CLITestCase ParseDafnyCommandArguments(string filePath, string dafnyComm // Check the arguments for anything non-standard foreach (var argument in arguments) { var (key, value) = ParseDafnyArgument(argument); - if (DAFNY_IGNORED_OPTIONS.Contains(key)) { - continue; - } if (value.Contains("%")) { throw new ArgumentException("Use of lit substitution (% variable) requires manual conversion: " + argument); } @@ -165,7 +139,7 @@ private CLITestCase ParseDafnyCommandArguments(string filePath, string dafnyComm var expected = new CLITestCase.Expectation(0, filePath + ".expect", null); - return new DafnyTestCase(filePath, dafnyArguments, otherFiles, expected); + return new DafnyTestCase(basePath, filePath, dafnyArguments, otherFiles, expected); } private static (string, string) ParseDafnyArgument(string argument) { @@ -185,13 +159,13 @@ private static (string, string) ParseDafnyArgument(string argument) { public void Run(string root) { // TODO-RS: Search for "*.transcript" too if (!Directory.Exists(root)) { - ConvertLitTest(root); + ConvertLitTest(root, root); return; } foreach (var file in Directory.GetFiles(root, "*.dfy", SearchOption.AllDirectories)) { try { count++; - ConvertLitTest(file); + ConvertLitTest(root, file); } catch (ArgumentException e) { invalidCount++; Console.WriteLine(file + ": " + e.Message); diff --git a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs index 36a128da8f0..da278471d24 100644 --- a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs +++ b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs @@ -22,9 +22,7 @@ private static IEnumerable ReadLines(StreamReader reader) { public void HelloWorld() { var convertor = new LitTestConvertor(); var lines = File.ReadLines("TestFiles/HelloWorldLitTest.dfy"); - var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles/HelloWorldLitTest.dfy", lines); - DafnyTestSpec spec = (DafnyTestSpec)testCases; - Assert.Equal(3, spec.Compile); + var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/HelloWorldLitTest.dfy", lines); } [Fact] @@ -34,9 +32,7 @@ public void VerifyOnly() { Assembly.GetExecutingAssembly().GetManifestResourceStream("LitTestConvertor.Test.TestFiles.VerifyOnlyLitTest.dfy"); using var reader = new StreamReader(stream); var lines = File.ReadLines("TestFiles/VerifyOnly.dfy"); - var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles/VerifyOnly.dfy", lines); - DafnyTestSpec spec = (DafnyTestSpec)testCases; - Assert.Equal(0, spec.Compile); + var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/VerifyOnly.dfy", lines); } } } \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs index 43e6929756c..9a93d429a27 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -17,9 +17,8 @@ public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, } protected override IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName) { - // string shortName = fileName[(GetBasePath(attributeInfo, testMethod).Length + 1)..]; try { - var (testCases, _) = convertor.ConvertLitCommands(fileName, File.ReadLines(fileName)); + var (testCases, _) = convertor.ConvertLitCommands(GetBasePath(attributeInfo, testMethod), fileName, File.ReadLines(fileName)); return testCases.Select(testCase => new[] { testCase }); } catch (Exception) { // Ignore for now diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index f05ace22353..7335d174b66 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -1,11 +1,10 @@ using DafnyDriver.Test.XUnitExtensions; -using Xunit; namespace LitTestConvertor.Test.LitTestRunner { public class LitTests { - [Theory] + [ParallelTheory] [LitTestData] public void LitTest(CLITestCase testCase) { testCase.Run(); From 789c900f28e5f62caddc7df85a6b8d22f26fff40 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 28 Aug 2021 11:15:12 -0700 Subject: [PATCH 112/192] Add invokeDirectly parameter to test data discoverer --- Source/Dafny/Compiler-cpp.cs | 8 ++----- Source/Dafny/DafnyPipeline.csproj | 1 + Source/DafnyDriver.Test/DafnyTestCase.cs | 5 ++-- .../XUnitExtensions/CLITestCase.cs | 23 ++++++++++--------- .../CollectionPerTestCaseTheoryDiscoverer.cs | 2 +- Source/LitTestConverter/LitTestConvertor.cs | 20 ++++++++-------- .../Conversion/ConvertingFromLitToXunit.cs | 4 ++-- .../LitTestRunner/LitTestDataDiscoverer.cs | 10 +++++--- .../LitTestDataDiscovererTest.cs | 5 ++-- .../LitTestRunner/LitTests.cs | 2 +- 10 files changed, 43 insertions(+), 37 deletions(-) diff --git a/Source/Dafny/Compiler-cpp.cs b/Source/Dafny/Compiler-cpp.cs index f177eb85274..ada788eedad 100644 --- a/Source/Dafny/Compiler-cpp.cs +++ b/Source/Dafny/Compiler-cpp.cs @@ -2392,8 +2392,7 @@ public override bool CompileTargetProgram(string dafnyProgramName, string target CreateNoWindow = true, UseShellExecute = false, RedirectStandardOutput = true, - RedirectStandardError = true, - WorkingDirectory = Path.GetDirectoryName(Path.GetFullPath(targetFilename)) + RedirectStandardError = true }; var proc = Process.Start(psi); while (!proc.StandardOutput.EndOfStream) { @@ -2413,14 +2412,11 @@ public override bool CompileTargetProgram(string dafnyProgramName, string target public override bool RunTargetProgram(string dafnyProgramName, string targetProgramText, string/*?*/ callToMain, string targetFilename, ReadOnlyCollection otherFileNames, object compilationResult, TextWriter outputWriter) { var exeName = ComputeExeName(targetFilename); - var dir = Path.GetDirectoryName(Path.GetFullPath(targetFilename)); - var exePath = Path.Combine(dir, exeName); - var psi = new ProcessStartInfo(exePath) { + var psi = new ProcessStartInfo(exeName) { CreateNoWindow = true, UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, - WorkingDirectory = dir }; var proc = Process.Start(psi); while (!proc.StandardOutput.EndOfStream) { diff --git a/Source/Dafny/DafnyPipeline.csproj b/Source/Dafny/DafnyPipeline.csproj index 5cd1eda943e..696b3280b98 100644 --- a/Source/Dafny/DafnyPipeline.csproj +++ b/Source/Dafny/DafnyPipeline.csproj @@ -36,5 +36,6 @@ + diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index eaacc2c1194..2e53e180906 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -60,8 +60,9 @@ private static string ConfigPairToArgument(KeyValuePair pair) { private Dictionary DafnyOptions = new(); private List OtherFiles = new(); - public DafnyTestCase(string basePath, string fullSourcePath, Dictionary dafnyOptions, List otherFiles, Expectation expected) - : base(dafnyDriverAssembly, OptionsToFullArguments(fullSourcePath, dafnyOptions, otherFiles), new List(), expected) { + public DafnyTestCase(string basePath, string fullSourcePath, Dictionary dafnyOptions, List otherFiles, + Expectation expected, bool invokeDirectly) + : base(dafnyDriverAssembly, OptionsToFullArguments(fullSourcePath, dafnyOptions, otherFiles), new List(), expected, invokeDirectly) { BasePath = basePath; SourcePath = fullSourcePath; DafnyOptions = dafnyOptions; diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs index 0c845837387..edeb02f7ded 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs @@ -14,11 +14,11 @@ namespace DafnyDriver.Test.XUnitExtensions { public class CLITestCase: IXunitSerializable { - protected Assembly CLIAssembly; - protected string CLIAssemblyName; + protected Assembly CliAssembly; + protected string CliAssemblyName; + protected bool InvokeDirectly; protected string[] Arguments; - protected string WorkingDirectory; protected IEnumerable PassthroughEnvironmentVariables; protected Expectation Expected; @@ -84,11 +84,12 @@ public CLIResult(int exitCode, string standardOutput, string standardError) { public CLITestCase(Assembly cliAssembly, IEnumerable arguments, IEnumerable passthroughEnvironmentVariables, - Expectation expected) { - CLIAssembly = cliAssembly; + Expectation expected, bool invokeDirectly) { + CliAssembly = cliAssembly; Arguments = arguments.ToArray(); PassthroughEnvironmentVariables = passthroughEnvironmentVariables; Expected = expected; + InvokeDirectly = invokeDirectly; } public CLITestCase() { @@ -96,21 +97,21 @@ public CLITestCase() { } public void Serialize(IXunitSerializationInfo info) { - info.AddValue(nameof(CLIAssemblyName), CLIAssemblyName); + info.AddValue(nameof(CliAssemblyName), CliAssemblyName); info.AddValue(nameof(Arguments), Arguments); info.AddValue(nameof(Expected), Expected); } public void Deserialize(IXunitSerializationInfo info) { - CLIAssemblyName = info.GetValue(nameof(CLIAssemblyName)); - CLIAssembly = AppDomain.CurrentDomain.GetAssemblies().First(a => a.FullName == CLIAssemblyName); + CliAssemblyName = info.GetValue(nameof(CliAssemblyName)); + CliAssembly = AppDomain.CurrentDomain.GetAssemblies().First(a => a.FullName == CliAssemblyName); Arguments = info.GetValue(nameof(Arguments)); Expected = info.GetValue(nameof(Expected)); } public CLIResult InvokeCLI() { - if (Environment.GetEnvironmentVariable("XUNIT_INVOKE_CLI_DIRECTLY") == "true") { + if (InvokeDirectly || Environment.GetEnvironmentVariable("XUNIT_INVOKE_CLI_DIRECTLY") == "true") { return InvokeCLIDirectly(); } return InvokeCLIViaProcess(); @@ -123,7 +124,7 @@ public CLIResult InvokeCLIDirectly() { Console.SetOut(new StringWriter(redirectedOut)); Console.SetError(new StringWriter(redirectedErr)); - int result = (int) CLIAssembly.EntryPoint.Invoke(null, new object[] { Arguments }); + int result = (int) CliAssembly.EntryPoint.Invoke(null, new object[] { Arguments }); return new CLIResult(result, redirectedOut.ToString(), redirectedErr.ToString()); } @@ -132,7 +133,7 @@ public CLIResult InvokeCLIViaProcess() { using var process = new Process(); process.StartInfo.FileName = "dotnet"; - process.StartInfo.Arguments = CLIAssembly.Location; + process.StartInfo.Arguments = CliAssembly.Location; foreach (var argument in Arguments) { process.StartInfo.Arguments += " " + argument; } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index 08eca11e130..9136288a9c2 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -24,7 +24,7 @@ public IEnumerable Discover(ITestFrameworkDiscoveryOptions disco // This discoverer requires pre-enumeration in order to assign a collection to each test case. discoveryOptions.SetValue("xunit.discovery.PreEnumerateTheories", true); - IEnumerable testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); + var testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); // Select the requested fraction of the test cases if using the XUNIT_SHARD[_COUNT} environment variables. // Ideally this would be handled at a higher level so that cases from different test methods could be diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 0802a856a87..fabdc86ff4e 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -26,14 +26,14 @@ public class LitTestConvertor { private int alreadyConverted = 0; private int invalidCount = 0; - public void ConvertLitTest(string basePath, string filePath) { + public void ConvertLitTest(string basePath, string filePath, bool invokeDirectly) { IEnumerable testSpec; IEnumerable testContent; if (filePath.Contains("/Inputs/")) { testSpec = Enumerable.Empty(); testContent = File.ReadAllLines(filePath); } else { - (testSpec, testContent) = ConvertLitCommands(basePath, filePath, File.ReadAllLines(filePath)); + (testSpec, testContent) = ConvertLitCommands(basePath, filePath, invokeDirectly, File.ReadAllLines(filePath)); } ISerializer serializer = new SerializerBuilder() @@ -54,7 +54,7 @@ public void ConvertLitTest(string basePath, string filePath) { } } - public (IEnumerable spec, IEnumerable content) ConvertLitCommands(string basePath, string filePath, IEnumerable lines) { + public (IEnumerable spec, IEnumerable content) ConvertLitCommands(string basePath, string filePath, bool invokeDirectly, IEnumerable lines) { var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); if (!litCommands.Any()) { throw new ArgumentException("No lit commands found"); @@ -76,7 +76,7 @@ public void ConvertLitTest(string basePath, string filePath) { } litCommands.RemoveAt(litCommands.Count - 1); - List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(basePath, filePath, c)).ToList(); + List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(basePath, filePath, invokeDirectly, c)).ToList(); if (testConfigs.Count == 1) { return (testConfigs, testContent); @@ -97,7 +97,7 @@ private static string ExtractLitCommand(string line) { return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); } - private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, string dafnyCommand) { + private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, bool invokeDirectly, string dafnyCommand) { bool includeThisFile = true; List otherFiles = new(); Dictionary dafnyArguments = new(); @@ -131,7 +131,9 @@ private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, throw new ArgumentException("Use of lit substitution (% variable) requires manual conversion: " + argument); } if (key.Equals(TEST_CONFIG_OTHER_FILES)) { - otherFiles.Add(value); + // Lit always uses the parent directory of a test file as the current directory, + // so other file paths have to be interpreted to be relative to the output directory instead. + otherFiles.Add(Path.Join(Path.GetDirectoryName(filePath), value)); } else { dafnyArguments.Add(key, value); } @@ -139,7 +141,7 @@ private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, var expected = new CLITestCase.Expectation(0, filePath + ".expect", null); - return new DafnyTestCase(basePath, filePath, dafnyArguments, otherFiles, expected); + return new DafnyTestCase(basePath, filePath, dafnyArguments, otherFiles, expected, invokeDirectly); } private static (string, string) ParseDafnyArgument(string argument) { @@ -159,13 +161,13 @@ private static (string, string) ParseDafnyArgument(string argument) { public void Run(string root) { // TODO-RS: Search for "*.transcript" too if (!Directory.Exists(root)) { - ConvertLitTest(root, root); + ConvertLitTest(root, root, false); return; } foreach (var file in Directory.GetFiles(root, "*.dfy", SearchOption.AllDirectories)) { try { count++; - ConvertLitTest(root, file); + ConvertLitTest(root, file, false); } catch (ArgumentException e) { invalidCount++; Console.WriteLine(file + ": " + e.Message); diff --git a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs index da278471d24..c71bd21b84c 100644 --- a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs +++ b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs @@ -22,7 +22,7 @@ private static IEnumerable ReadLines(StreamReader reader) { public void HelloWorld() { var convertor = new LitTestConvertor(); var lines = File.ReadLines("TestFiles/HelloWorldLitTest.dfy"); - var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/HelloWorldLitTest.dfy", lines); + var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/HelloWorldLitTest.dfy", false, lines); } [Fact] @@ -32,7 +32,7 @@ public void VerifyOnly() { Assembly.GetExecutingAssembly().GetManifestResourceStream("LitTestConvertor.Test.TestFiles.VerifyOnlyLitTest.dfy"); using var reader = new StreamReader(stream); var lines = File.ReadLines("TestFiles/VerifyOnly.dfy"); - var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/VerifyOnly.dfy", lines); + var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/VerifyOnly.dfy", false, lines); } } } \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs index 9a93d429a27..41df99299e9 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -17,8 +17,9 @@ public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, } protected override IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName) { + bool invokeDirectly = attributeInfo.GetNamedArgument(nameof(LitTestDataAttribute.InvokeCliDirectly)); try { - var (testCases, _) = convertor.ConvertLitCommands(GetBasePath(attributeInfo, testMethod), fileName, File.ReadLines(fileName)); + var (testCases, _) = convertor.ConvertLitCommands(GetBasePath(attributeInfo, testMethod), fileName, invokeDirectly, File.ReadLines(fileName)); return testCases.Select(testCase => new[] { testCase }); } catch (Exception) { // Ignore for now @@ -29,7 +30,10 @@ protected override IEnumerable FileData(IAttributeInfo attributeInfo, } [DataDiscoverer("DafnyDriver.Test.LitTestDataDiscoverer", "LitTestConvertor.Test")] -public class LitTestDataAttribute : FileDataAttribute { - public LitTestDataAttribute() : base(extension: ".dfy") { +public class LitTestDataAttribute : FileDataAttribute +{ + public bool InvokeCliDirectly; + public LitTestDataAttribute(bool invokeCliDirectly = false) : base(extension: ".dfy") { + InvokeCliDirectly = invokeCliDirectly; } } \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs index ec5bbe8671c..916a8c1b7ee 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs @@ -19,10 +19,11 @@ public void Discoverer() { var discoverer = new LitTestDataDiscoverer(); var test = new LitTests(); var methodInfo = GetMethodInfo(test.LitTest); - var method = Reflector.Wrap(GetMethodInfo(test.LitTest)); + var method = Reflector.Wrap(methodInfo); var attribute = CustomAttributeData.GetCustomAttributes(methodInfo).First(a => a.AttributeType == typeof(LitTestDataAttribute)); var xunitAttribute = Reflector.Wrap(attribute); - discoverer.GetData(xunitAttribute, method); + var data = discoverer.GetData(xunitAttribute, method); + Assert.True(data.Any()); } } } \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index 7335d174b66..71dc0f6dda0 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -5,7 +5,7 @@ namespace LitTestConvertor.Test.LitTestRunner public class LitTests { [ParallelTheory] - [LitTestData] + [LitTestData(false)] public void LitTest(CLITestCase testCase) { testCase.Run(); } From 123937a8f8563f0487598dcf15e4dbc6fac708c1 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 30 Aug 2021 10:50:18 -0700 Subject: [PATCH 113/192] Fixing C++ test cases --- Source/DafnyDriver.Test/DafnyTestCase.cs | 2 +- .../CollectionPerTestCaseTheoryDiscoverer.cs | 5 ++-- .../XUnitExtensions/TestCaseWithCollection.cs | 23 +++++++++---------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index 2e53e180906..c29d060e38b 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -62,7 +62,7 @@ private static string ConfigPairToArgument(KeyValuePair pair) { public DafnyTestCase(string basePath, string fullSourcePath, Dictionary dafnyOptions, List otherFiles, Expectation expected, bool invokeDirectly) - : base(dafnyDriverAssembly, OptionsToFullArguments(fullSourcePath, dafnyOptions, otherFiles), new List(), expected, invokeDirectly) { + : base(dafnyDriverAssembly, OptionsToFullArguments(fullSourcePath, dafnyOptions, otherFiles), new string[] { "PATH", "HOME" } , expected, invokeDirectly) { BasePath = basePath; SourcePath = fullSourcePath; DafnyOptions = dafnyOptions; diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index 9136288a9c2..255f88d0adb 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -1,14 +1,13 @@ using System; using System.Collections.Generic; using System.Linq; -using Xunit; using Xunit.Abstractions; using Xunit.Sdk; namespace XUnitExtensions { public class CollectionPerTestCaseTheoryDiscoverer : IXunitTestCaseDiscoverer { - readonly IXunitTestCaseDiscoverer theoryDiscoverer; + private readonly IXunitTestCaseDiscoverer theoryDiscoverer; public CollectionPerTestCaseTheoryDiscoverer(IMessageSink diagnosticMessageSink) { @@ -26,7 +25,7 @@ public IEnumerable Discover(ITestFrameworkDiscoveryOptions disco var testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); - // Select the requested fraction of the test cases if using the XUNIT_SHARD[_COUNT} environment variables. + // Select the requested fraction of the test cases if using the XUNIT_SHARD[_COUNT] environment variables. // Ideally this would be handled at a higher level so that cases from different test methods could be // balanced as a whole. var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); diff --git a/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs b/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs index bf5e66e7e2b..d2de3d21cb2 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs @@ -11,14 +11,14 @@ namespace XUnitExtensions { public class TestCaseWithCollection : LongLivedMarshalByRefObject, IXunitTestCase { private IXunitTestCase testCase; - private ITestMethod testMethod; + public ITestMethod TestMethod { get; set; } public TestCaseWithCollection(IXunitTestCase testCase, ITestCollection collection) { this.testCase = testCase; var testClassWithCollection = new TestClass(collection, testCase.TestMethod.TestClass.Class); - this.testMethod = new TestMethod(testClassWithCollection, testCase.TestMethod.Method); + TestMethod = new TestMethod(testClassWithCollection, testCase.TestMethod.Method); } [Obsolete("Called by the de-serializer", error: true)] @@ -26,25 +26,24 @@ public TestCaseWithCollection() { } public void Deserialize(IXunitSerializationInfo info) { testCase = info.GetValue("InnerTestCase"); - testMethod = info.GetValue("TestMethod"); + TestMethod = info.GetValue("TestMethod"); } public void Serialize(IXunitSerializationInfo info) { info.AddValue("InnerTestCase", testCase); - info.AddValue("TestMethod", testMethod); + info.AddValue("TestMethod", TestMethod); } - public string DisplayName { get { return testCase.DisplayName; } } - public string SkipReason { get { return testCase.SkipReason; } } + public string DisplayName => testCase.DisplayName; + public string SkipReason => testCase.SkipReason; public ISourceInformation SourceInformation { - get { return testCase.SourceInformation; } - set { testCase.SourceInformation = value; } + get => testCase.SourceInformation; + set => testCase.SourceInformation = value; } - public ITestMethod TestMethod { get { return testMethod; } } - public object[] TestMethodArguments { get { return testCase.TestMethodArguments; } } - public Dictionary> Traits { get { return testCase.Traits; } } - public string UniqueID { get { return testCase.UniqueID; } } + public object[] TestMethodArguments => testCase.TestMethodArguments; + public Dictionary> Traits => testCase.Traits; + public string UniqueID => testCase.UniqueID; public Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { From b7172b539ed8df75d5b11dc22ea680b82e9c3f3b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 30 Aug 2021 10:55:35 -0700 Subject: [PATCH 114/192] Autoformat --- Source/DafnyDriver.Test/DafnyTestCase.cs | 27 +++++++------ Source/DafnyDriver.Test/TemporaryDirectory.cs | 4 +- .../XUnitExtensions/AssertWithDiff.cs | 2 +- .../XUnitExtensions/CLITestCase.cs | 39 +++++++++---------- .../CollectionPerTestCaseTheoryDiscoverer.cs | 17 ++++---- .../XUnitExtensions/EnumerableUtils.cs | 4 +- .../XUnitExtensions/FileDataAttribute.cs | 8 ++-- .../XUnitExtensions/FileDataDiscoverer.cs | 16 ++++---- .../XUnitExtensions/ForEachAttribute.cs | 2 +- .../XUnitExtensions/TestCaseWithCollection.cs | 16 ++++---- .../XUnitExtensions/YamlDataAttribute.cs | 10 ++--- .../XUnitExtensions/YamlDataDiscoverer.cs | 22 +++++------ .../XUnitExtensions/YamlDataTests.cs | 28 ++++++------- Source/LitTestConverter/LitTestConvertor.cs | 39 +++++++++---------- .../Conversion/ConvertingFromLitToXunit.cs | 12 +++--- .../LitTestRunner/LitTestDataDiscoverer.cs | 8 ++-- .../LitTestDataDiscovererTest.cs | 5 +-- .../LitTestRunner/LitTests.cs | 6 +-- 18 files changed, 123 insertions(+), 142 deletions(-) diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index c29d060e38b..61e27d78c39 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -7,7 +7,7 @@ using Xunit.Abstractions; namespace DafnyDriver.Test { - + /** * Specialization of CLITestCase that mainly exists to support a much more * concise definition of ToString(). @@ -15,7 +15,7 @@ namespace DafnyDriver.Test { public class DafnyTestCase : CLITestCase { private static readonly Assembly dafnyDriverAssembly = Assembly.GetAssembly(typeof(Microsoft.Dafny.DafnyDriver)); - + private static readonly Dictionary defaultDafnyOptions = new() { ["countVerificationErrors"] = "0", @@ -25,14 +25,13 @@ public class DafnyTestCase : CLITestCase { // We do not want output such as "Compiled program written to Foo.cs" // from the compilers, since that changes with the target language ["compileVerbose"] = "0", - + // Hide Boogie execution traces since they are meaningless for Dafny programs ["errorTrace"] = "0" }; private static IEnumerable OptionsToFullArguments(string sourcePath, - Dictionary dafnyOptions, List otherFiles) - { + Dictionary dafnyOptions, List otherFiles) { Dictionary optionsWithDefaults = new(defaultDafnyOptions); foreach (var (key, value) in dafnyOptions) { optionsWithDefaults[key] = value; @@ -42,11 +41,11 @@ private static IEnumerable OptionsToFullArguments(string sourcePath, } private static IEnumerable OptionsToArguments(string sourcePath, Dictionary dafnyOptions, List otherFiles) { - return new []{ sourcePath } + return new[] { sourcePath } .Concat(otherFiles) .Concat(dafnyOptions.Select(ConfigPairToArgument)); } - + private static string ConfigPairToArgument(KeyValuePair pair) { if (pair.Value.Equals("yes")) { return $"/{pair.Key}"; @@ -56,23 +55,23 @@ private static string ConfigPairToArgument(KeyValuePair pair) { private string BasePath; private string SourcePath; - + private Dictionary DafnyOptions = new(); private List OtherFiles = new(); public DafnyTestCase(string basePath, string fullSourcePath, Dictionary dafnyOptions, List otherFiles, Expectation expected, bool invokeDirectly) - : base(dafnyDriverAssembly, OptionsToFullArguments(fullSourcePath, dafnyOptions, otherFiles), new string[] { "PATH", "HOME" } , expected, invokeDirectly) { + : base(dafnyDriverAssembly, OptionsToFullArguments(fullSourcePath, dafnyOptions, otherFiles), new string[] { "PATH", "HOME" }, expected, invokeDirectly) { BasePath = basePath; SourcePath = fullSourcePath; DafnyOptions = dafnyOptions; OtherFiles = otherFiles; } - + public DafnyTestCase() { - + } - + public void Serialize(IXunitSerializationInfo info) { base.Serialize(info); info.AddValue(nameof(BasePath), BasePath); @@ -80,7 +79,7 @@ public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(DafnyOptions), DafnyOptions); info.AddValue(nameof(OtherFiles), OtherFiles); } - + public void Deserialize(IXunitSerializationInfo info) { base.Deserialize(info); BasePath = info.GetValue(nameof(BasePath)); @@ -88,7 +87,7 @@ public void Deserialize(IXunitSerializationInfo info) { DafnyOptions = info.GetValue>(nameof(DafnyOptions)); OtherFiles = info.GetValue>(nameof(OtherFiles)); } - + public override string ToString() { var relativePath = SourcePath[(BasePath.Length + 1)..]; return String.Join(" ", OptionsToArguments(relativePath, DafnyOptions, OtherFiles)); diff --git a/Source/DafnyDriver.Test/TemporaryDirectory.cs b/Source/DafnyDriver.Test/TemporaryDirectory.cs index 943817c81ae..f420643caec 100644 --- a/Source/DafnyDriver.Test/TemporaryDirectory.cs +++ b/Source/DafnyDriver.Test/TemporaryDirectory.cs @@ -23,11 +23,11 @@ public void Dispose() { ~TemporaryDirectory() { Dispose(false); } - + protected virtual void Dispose(bool disposing) { SafeDelete(); } - + private void SafeDelete() { try { DirInfo.Delete(true); diff --git a/Source/DafnyDriver.Test/XUnitExtensions/AssertWithDiff.cs b/Source/DafnyDriver.Test/XUnitExtensions/AssertWithDiff.cs index fabf42b95ba..7d4287a45ce 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/AssertWithDiff.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/AssertWithDiff.cs @@ -27,7 +27,7 @@ public static void Equal(string expected, string actual) { } message.AppendLine(line.Text); } - + throw new AssertActualExpectedException(expected, actual, message.ToString()); } } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs index edeb02f7ded..4eea2cc31df 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs @@ -11,13 +11,12 @@ using XUnitExtensions; namespace DafnyDriver.Test.XUnitExtensions { - - public class CLITestCase: IXunitSerializable - { + + public class CLITestCase : IXunitSerializable { protected Assembly CliAssembly; protected string CliAssemblyName; protected bool InvokeDirectly; - + protected string[] Arguments; protected IEnumerable PassthroughEnvironmentVariables; protected Expectation Expected; @@ -30,7 +29,7 @@ public class Expectation : IXunitSerializable { public string SpecialCaseReason; public static Expectation NO_OUTPUT = new Expectation(0, null, null); - + public Expectation(int exitCode, string outputFile, string specialCaseReason) { ExitCode = exitCode; OutputFile = outputFile; @@ -39,7 +38,7 @@ public Expectation(int exitCode, string outputFile, string specialCaseReason) { public Expectation() { } - + public void Deserialize(IXunitSerializationInfo info) { OutputFile = info.GetValue(nameof(OutputFile)); ExitCode = info.GetValue(nameof(ExitCode)); @@ -82,7 +81,7 @@ public CLIResult(int exitCode, string standardOutput, string standardError) { } } - public CLITestCase(Assembly cliAssembly, IEnumerable arguments, + public CLITestCase(Assembly cliAssembly, IEnumerable arguments, IEnumerable passthroughEnvironmentVariables, Expectation expected, bool invokeDirectly) { CliAssembly = cliAssembly; @@ -93,19 +92,19 @@ public CLITestCase(Assembly cliAssembly, IEnumerable arguments, } public CLITestCase() { - + } - + public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(CliAssemblyName), CliAssemblyName); info.AddValue(nameof(Arguments), Arguments); info.AddValue(nameof(Expected), Expected); } - + public void Deserialize(IXunitSerializationInfo info) { CliAssemblyName = info.GetValue(nameof(CliAssemblyName)); CliAssembly = AppDomain.CurrentDomain.GetAssemblies().First(a => a.FullName == CliAssemblyName); - + Arguments = info.GetValue(nameof(Arguments)); Expected = info.GetValue(nameof(Expected)); } @@ -116,7 +115,7 @@ public CLIResult InvokeCLI() { } return InvokeCLIViaProcess(); } - + public CLIResult InvokeCLIDirectly() { StringBuilder redirectedOut = new(); StringBuilder redirectedErr = new(); @@ -124,30 +123,30 @@ public CLIResult InvokeCLIDirectly() { Console.SetOut(new StringWriter(redirectedOut)); Console.SetError(new StringWriter(redirectedErr)); - int result = (int) CliAssembly.EntryPoint.Invoke(null, new object[] { Arguments }); - + int result = (int)CliAssembly.EntryPoint.Invoke(null, new object[] { Arguments }); + return new CLIResult(result, redirectedOut.ToString(), redirectedErr.ToString()); } - + public CLIResult InvokeCLIViaProcess() { using var process = new Process(); - + process.StartInfo.FileName = "dotnet"; process.StartInfo.Arguments = CliAssembly.Location; foreach (var argument in Arguments) { process.StartInfo.Arguments += " " + argument; } - + process.StartInfo.UseShellExecute = false; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; - process.StartInfo.CreateNoWindow = true; + process.StartInfo.CreateNoWindow = true; process.StartInfo.EnvironmentVariables.Clear(); - foreach(var passthrough in PassthroughEnvironmentVariables) { + foreach (var passthrough in PassthroughEnvironmentVariables) { process.StartInfo.EnvironmentVariables.Add(passthrough, Environment.GetEnvironmentVariable(passthrough)); } - + process.Start(); string output = process.StandardOutput.ReadToEnd(); string error = process.StandardError.ReadToEnd(); diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index 255f88d0adb..5a45bc0f54d 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -6,25 +6,24 @@ namespace XUnitExtensions { public class CollectionPerTestCaseTheoryDiscoverer : IXunitTestCaseDiscoverer { - + private readonly IXunitTestCaseDiscoverer theoryDiscoverer; - public CollectionPerTestCaseTheoryDiscoverer(IMessageSink diagnosticMessageSink) - { + public CollectionPerTestCaseTheoryDiscoverer(IMessageSink diagnosticMessageSink) { theoryDiscoverer = new SkippableTheoryDiscoverer(diagnosticMessageSink); } private TestCollection testCollectionForTestCase(IXunitTestCase testCase) { - return new TestCollection(testCase.TestMethod.TestClass.TestCollection.TestAssembly, - (ITypeInfo) null, "Test collection for " + testCase.DisplayName); + return new TestCollection(testCase.TestMethod.TestClass.TestCollection.TestAssembly, + (ITypeInfo)null, "Test collection for " + testCase.DisplayName); } public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { // This discoverer requires pre-enumeration in order to assign a collection to each test case. discoveryOptions.SetValue("xunit.discovery.PreEnumerateTheories", true); - + var testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); - + // Select the requested fraction of the test cases if using the XUNIT_SHARD[_COUNT] environment variables. // Ideally this would be handled at a higher level so that cases from different test methods could be // balanced as a whole. @@ -46,13 +45,13 @@ public IEnumerable Discover(ITestFrameworkDiscoveryOptions disco throw new ArgumentNullException( "XUNIT_SHARD must be at least 1 and at most XUNIT_SHARD_COUNT."); } - + var testCaseList = testCases.ToList(); var shardStart = (shard - 1) * testCaseList.Count / numShards; var shardEnd = shard * testCaseList.Count / numShards; testCases = testCaseList.GetRange(shardStart, shardEnd - shardStart); } - + return testCases.Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); } } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs b/Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs index 8ac8ea01da9..f461f4f8fa1 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs @@ -7,13 +7,13 @@ public static class EnumerableUtils { * Source: https://docs.microsoft.com/en-us/archive/blogs/ericlippert/computing-a-cartesian-product-with-linq */ public static IEnumerable> CartesianProduct(this IEnumerable> sequences) { - IEnumerable> emptyProduct = new[] {Enumerable.Empty()}; + IEnumerable> emptyProduct = new[] { Enumerable.Empty() }; return sequences.Aggregate( emptyProduct, (accumulator, sequence) => from accseq in accumulator from item in sequence - select accseq.Concat(new[] {item})); + select accseq.Concat(new[] { item })); } } } \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs index a47e44a544c..b87a0b3231e 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs @@ -2,10 +2,8 @@ using System.Reflection; using Xunit.Sdk; -namespace DafnyDriver.Test.XUnitExtensions -{ - public abstract class FileDataAttribute : DataAttribute - { +namespace DafnyDriver.Test.XUnitExtensions { + public abstract class FileDataAttribute : DataAttribute { public readonly string Path; public readonly string Extension; @@ -13,7 +11,7 @@ public FileDataAttribute(string path = null, string extension = null) { Path = path; Extension = extension; } - + public override IEnumerable GetData(MethodInfo testMethod) { throw new System.NotImplementedException(); } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs index f7adf0dde5f..ff7d95c454f 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs @@ -6,34 +6,32 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace DafnyDriver.Test.XUnitExtensions -{ - public abstract class FileDataDiscoverer : IDataDiscoverer - { +namespace DafnyDriver.Test.XUnitExtensions { + public abstract class FileDataDiscoverer : IDataDiscoverer { protected string GetBasePath(IAttributeInfo attributeInfo, IMethodInfo testMethod) { var path = attributeInfo.GetNamedArgument(nameof(FileDataAttribute.Path)); if (path != null) { return path; - } + } return Path.Combine("TestFiles", testMethod.ToRuntimeMethod().DeclaringType.Name, testMethod.Name); } - + public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo testMethod) { var path = GetBasePath(attributeInfo, testMethod); var extension = attributeInfo.GetNamedArgument(nameof(FileDataAttribute.Extension)); - + if (Directory.Exists(path)) { return Directory.EnumerateFiles(path, "*" + extension, SearchOption.AllDirectories) .SelectMany(childPath => FileData(attributeInfo, testMethod, childPath)); } - + var fileName = path + extension; if (File.Exists(fileName)) { return FileData(attributeInfo, testMethod, fileName); } - + throw new ArgumentException("No data found for path: " + path); } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs index 0e68ea90b52..f001aba5921 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs @@ -5,7 +5,7 @@ namespace DafnyDriver.Test.XUnitExtensions { public class ForEachAttribute : Attribute { private readonly Type EnumeratorClass; - + public ForEachAttribute(Type enumeratorClass = null) { EnumeratorClass = enumeratorClass ?? typeof(IEnumerable<>); } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs b/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs index d2de3d21cb2..7109543dda2 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs @@ -9,21 +9,20 @@ namespace XUnitExtensions { public class TestCaseWithCollection : LongLivedMarshalByRefObject, IXunitTestCase { - + private IXunitTestCase testCase; public ITestMethod TestMethod { get; set; } - public TestCaseWithCollection(IXunitTestCase testCase, ITestCollection collection) - { + public TestCaseWithCollection(IXunitTestCase testCase, ITestCollection collection) { this.testCase = testCase; - + var testClassWithCollection = new TestClass(collection, testCase.TestMethod.TestClass.Class); TestMethod = new TestMethod(testClassWithCollection, testCase.TestMethod.Method); } - + [Obsolete("Called by the de-serializer", error: true)] public TestCaseWithCollection() { } - + public void Deserialize(IXunitSerializationInfo info) { testCase = info.GetValue("InnerTestCase"); TestMethod = info.GetValue("TestMethod"); @@ -34,9 +33,8 @@ public void Serialize(IXunitSerializationInfo info) { info.AddValue("TestMethod", TestMethod); } public string DisplayName => testCase.DisplayName; - public string SkipReason => testCase.SkipReason; - public ISourceInformation SourceInformation - { + public string SkipReason => testCase.SkipReason; + public ISourceInformation SourceInformation { get => testCase.SourceInformation; set => testCase.SourceInformation = value; } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs index bfc298d2969..6b12b3e901a 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs @@ -5,18 +5,16 @@ using Xunit.Sdk; namespace XUnitExtensions { - + [DataDiscoverer("DafnyDriver.Test.XUnitExtensions.YamlDataDiscoverer", "DafnyDriver.Test")] - public class YamlDataAttribute : FileDataAttribute - { + public class YamlDataAttribute : FileDataAttribute { public readonly bool WithParameterNames; public YamlDataAttribute(bool withParameterNames = true, string path = null, string extension = ".yml") - : base(path, extension) - { + : base(path, extension) { WithParameterNames = withParameterNames; } - + public override IEnumerable GetData(MethodInfo testMethod) { // This method is not used - the YamlDataDiscoverer has all of the actual logic instead // because it exposes some methods for subclasses to customize such as GetYamlParser. diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs index 333eacf83c4..7cc87a31c01 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs @@ -36,11 +36,11 @@ private static Func ForMethodInfoDeserializeFn(IDeseriali } }; } - + public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { return true; } - + protected override IEnumerable FileData(IAttributeInfo dataAttribute, IMethodInfo testMethod, string fileName) { var withParameterNames = dataAttribute.GetNamedArgument(nameof(YamlDataAttribute.WithParameterNames)); // YamlDotNet's deserialization framework requires runtime type information @@ -48,11 +48,11 @@ protected override IEnumerable FileData(IAttributeInfo dataAttribute, if (methodInfo == null) { return null; } - + try { using Stream stream = File.OpenRead(fileName); IParser parser = GetYamlParser(fileName, stream); - if (parser == null) { + if (parser == null) { return Enumerable.Empty(); } @@ -66,7 +66,7 @@ protected override IEnumerable FileData(IAttributeInfo dataAttribute, var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, methodInfo); if (collectionDeserializer.Deserialize(parser, typeof(List), nestedObjectDeserializer, out var value)) { - List argumentses = (List) value; + List argumentses = (List)value; return argumentses.SelectMany(a => a.Combinations()); } else { throw new ArgumentException(); @@ -74,8 +74,8 @@ protected override IEnumerable FileData(IAttributeInfo dataAttribute, } else { IEnumerable parameters = methodInfo.GetParameters(); Type targetType = typeof(IEnumerable<>).MakeGenericType(parameters.Single().ParameterType); - IEnumerable results = (IEnumerable) deserializer.Deserialize(parser, targetType); - return results.Select(value => new[] {value}); + IEnumerable results = (IEnumerable)deserializer.Deserialize(parser, targetType); + return results.Select(value => new[] { value }); } } catch (Exception e) { throw new ArgumentException( @@ -97,7 +97,7 @@ public MethodArguments(MethodInfo methodInfo) { .Select((value, index) => new KeyValuePair(value.Name, index)) .ToDictionary(p => p.Key, p => p.Value); } - + public void Read(IParser parser, Type expectedType, ObjectDeserializer nestedObjectDeserializer) { if (!parser.TryConsume(out MappingStart _)) { throw new ArgumentException(); @@ -111,11 +111,11 @@ public void Read(IParser parser, Type expectedType, ObjectDeserializer nestedObj if (forEach != null) { parameterType = forEach.EnumerableTypeOf(parameterType); } - + object argument = nestedObjectDeserializer(parameterType); IValuePromise valuePromise = argument as IValuePromise; if (valuePromise != null) { - valuePromise.ValueAvailable += (Action) (v => Arguments[parameterIndex] = TypeConverter.ChangeType(v, parameterType)); + valuePromise.ValueAvailable += (Action)(v => Arguments[parameterIndex] = TypeConverter.ChangeType(v, parameterType)); } else { Arguments[parameterIndex] = TypeConverter.ChangeType(argument, parameterType); } @@ -131,7 +131,7 @@ public IEnumerable Combinations() { IEnumerable> lifted = Arguments.Select((arg, index) => { ParameterInfo parameter = Method.GetParameters()[index]; ForEachAttribute forEach = parameter.GetCustomAttribute(); - return forEach == null ? new[] {arg} : ((IEnumerable)arg).Cast(); + return forEach == null ? new[] { arg } : ((IEnumerable)arg).Cast(); }); return lifted.CartesianProduct().Select(e => e.ToArray()); } diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs index 46c4e199aa4..75b039bac95 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs +++ b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs @@ -20,7 +20,7 @@ private void AssertTheoryData(string methodName, IEnumerable expectedDat var discoverer = new YamlDataDiscoverer(); Assert.Equal(expectedData, discoverer.GetData(Reflector.Wrap(attribute), Reflector.Wrap(method))); } - + [Theory] [YamlData] public void CalculatorTest(int lhs, int rhs, int expected) { @@ -29,21 +29,21 @@ public void CalculatorTest(int lhs, int rhs, int expected) { [Fact] public void CalculatorTestData() { - AssertTheoryData(nameof(CalculatorTest), new [] { + AssertTheoryData(nameof(CalculatorTest), new[] { new object[]{ 2, 2, 4 }, new object[]{ 3, 4, 7 } }); } - + [Theory] [YamlData] public void CalculatorCombinatorialTest([ForEach] int lhs, [ForEach] int rhs) { Assert.Equal(rhs + lhs, lhs + rhs); } - + [Fact] public void CalculatorCombinatorialTestData() { - AssertTheoryData(nameof(CalculatorCombinatorialTest), new [] { + AssertTheoryData(nameof(CalculatorCombinatorialTest), new[] { new object[]{ 1, 5 }, new object[]{ 1, 10 }, new object[]{ 2, 5 }, @@ -56,16 +56,16 @@ public void CalculatorCombinatorialTestData() { new object[]{ 5, 10 }, }); } - + [Theory] [YamlData] public void MultiFileTest(int a, int b) { Assert.Equal(a + 1, b); } - + [Fact] public void MultiFileTestData() { - AssertTheoryData(nameof(MultiFileTest), new [] { + AssertTheoryData(nameof(MultiFileTest), new[] { new object[]{ 1, 2 }, new object[]{ 3, 4 }, new object[]{ 5, 6 }, @@ -78,10 +78,10 @@ public void MultiFileTestData() { public void DictionaryTest(Dictionary config) { Assert.Equal(3, config.Count); } - + [Fact] public void DictionaryTestData() { - AssertTheoryData(nameof(DictionaryTest), new [] { + AssertTheoryData(nameof(DictionaryTest), new[] { new object[]{ new Dictionary { ["one"] = "1", ["two"] = "2", @@ -100,16 +100,16 @@ public class CustomYamlDataAttribute : YamlDataAttribute { public CustomYamlDataAttribute(bool withParameterNames = true) : base(withParameterNames) { } } - + [Theory] [CustomYamlData()] public void CustomDataDiscovererTest([ForEach()] int lhs, [ForEach()] int rhs) { Assert.Equal(rhs + lhs, lhs + rhs); } - + [Fact] public void CustomDataDiscovererTestData() { - AssertTheoryData(nameof(CustomDataDiscovererTest), new [] { + AssertTheoryData(nameof(CustomDataDiscovererTest), new[] { new object[]{ 0, 0 }, new object[]{ 0, 1 }, new object[]{ 1, 0 }, @@ -123,7 +123,7 @@ public void CustomDataDiscovererTestData() { }); } } - + public class Range : IEnumerable { public int Start; diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index fabdc86ff4e..731b9da5712 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -8,18 +8,18 @@ using YamlDotNet.Serialization.NamingConventions; namespace LitTestConvertor { - + public class LitTestConvertor { private const string DAFNY_COMMENT_PREFIX = "//"; private const string LIT_COMMAND_PREFIX = "RUN:"; private const string LIT_DAFNY = "%dafny"; private const string LIT_SERVER = "%server"; - + // Fake options for which files are passed to the CLI public const string TEST_CONFIG_OTHER_FILES = "otherFiles"; public const string TEST_CONFIG_INCLUDE_THIS_FILE = "includeThisFile"; - + private int count = 0; private int verifyOnlyCount = 0; private int defaultCount = 0; @@ -35,12 +35,12 @@ public void ConvertLitTest(string basePath, string filePath, bool invokeDirectly } else { (testSpec, testContent) = ConvertLitCommands(basePath, filePath, invokeDirectly, File.ReadAllLines(filePath)); } - + ISerializer serializer = new SerializerBuilder() .WithNamingConvention(CamelCaseNamingConvention.Instance) .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) .Build(); - + using (StreamWriter file = new StreamWriter(filePath)) { if (testSpec != null) { file.WriteLine("/*"); @@ -59,7 +59,7 @@ public void ConvertLitTest(string basePath, string filePath, bool invokeDirectly if (!litCommands.Any()) { throw new ArgumentException("No lit commands found"); } - + // Make sure the commands are consecutive var testContent = lines.Skip(litCommands.Count); if (testContent.Any(line => ExtractLitCommand(line) != null)) { @@ -75,16 +75,16 @@ public void ConvertLitTest(string basePath, string filePath, bool invokeDirectly throw new ArgumentException("Last lit command is not expected %diff: " + litCommands[^1]); } litCommands.RemoveAt(litCommands.Count - 1); - + List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(basePath, filePath, invokeDirectly, c)).ToList(); if (testConfigs.Count == 1) { return (testConfigs, testContent); - } - + } + throw new ArgumentException("Multi-command lit tests require manual conversion"); } - + private static string ExtractLitCommand(string line) { if (!line.StartsWith(DAFNY_COMMENT_PREFIX)) { return null; @@ -96,26 +96,25 @@ private static string ExtractLitCommand(string line) { } return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); } - + private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, bool invokeDirectly, string dafnyCommand) { bool includeThisFile = true; List otherFiles = new(); Dictionary dafnyArguments = new(); - + if (!dafnyCommand.StartsWith(LIT_DAFNY)) { throw new ArgumentException("Lit command is not expected %dafny: " + dafnyCommand); } string argumentsString = dafnyCommand.Substring(LIT_DAFNY.Length); var arguments = argumentsString.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).ToList(); - + // Ensure the last two parts are > "%t" or >> "%t" if (arguments.Count < 3) { throw new ArgumentException("Not enough arguments to %dafny command: " + dafnyCommand); } - if (!arguments[^1].Equals("\"%t\"") - || !(arguments[^2].Equals(">") || arguments[^2].Equals(">>"))) - { + if (!arguments[^1].Equals("\"%t\"") + || !(arguments[^2].Equals(">") || arguments[^2].Equals(">>"))) { throw new ArgumentException("Non-standard output in %dafny command: " + dafnyCommand); } arguments.RemoveRange(arguments.Count - 2, 2); @@ -123,7 +122,7 @@ private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, if (!arguments.Remove("\"%s\"")) { includeThisFile = false; } - + // Check the arguments for anything non-standard foreach (var argument in arguments) { var (key, value) = ParseDafnyArgument(argument); @@ -140,7 +139,7 @@ private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, } var expected = new CLITestCase.Expectation(0, filePath + ".expect", null); - + return new DafnyTestCase(basePath, filePath, dafnyArguments, otherFiles, expected, invokeDirectly); } @@ -173,7 +172,7 @@ public void Run(string root) { Console.WriteLine(file + ": " + e.Message); } } - + Console.WriteLine(""); Console.WriteLine("Already converted: " + alreadyConverted + "/" + count); Console.WriteLine("Default: " + defaultCount + "/" + count); @@ -181,7 +180,7 @@ public void Run(string root) { Console.WriteLine("Invalid: " + invalidCount + "/" + count); } - public static void Main(string[] args) { + public static void Main(string[] args) { new LitTestConvertor().Run(args[0]); } } diff --git a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs index c71bd21b84c..908855c472a 100644 --- a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs +++ b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs @@ -5,12 +5,10 @@ using DafnyDriver.Test; using Xunit; -namespace LitTestConvertor.Test -{ - public class ConvertingFromLitToXunit - { - - +namespace LitTestConvertor.Test { + public class ConvertingFromLitToXunit { + + private static IEnumerable ReadLines(StreamReader reader) { string line; while ((line = reader.ReadLine()) != null) { @@ -24,7 +22,7 @@ public void HelloWorld() { var lines = File.ReadLines("TestFiles/HelloWorldLitTest.dfy"); var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/HelloWorldLitTest.dfy", false, lines); } - + [Fact] public void VerifyOnly() { var convertor = new LitTestConvertor(); diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs index 41df99299e9..a7340bc0aa8 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -7,11 +7,10 @@ using Xunit.Sdk; namespace DafnyDriver.Test { - public class LitTestDataDiscoverer : FileDataDiscoverer - { + public class LitTestDataDiscoverer : FileDataDiscoverer { private readonly LitTestConvertor.LitTestConvertor convertor = new(); - + public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { return true; } @@ -30,8 +29,7 @@ protected override IEnumerable FileData(IAttributeInfo attributeInfo, } [DataDiscoverer("DafnyDriver.Test.LitTestDataDiscoverer", "LitTestConvertor.Test")] -public class LitTestDataAttribute : FileDataAttribute -{ +public class LitTestDataAttribute : FileDataAttribute { public bool InvokeCliDirectly; public LitTestDataAttribute(bool invokeCliDirectly = false) : base(extension: ".dfy") { InvokeCliDirectly = invokeCliDirectly; diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs index 916a8c1b7ee..2fdae0aa86a 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs @@ -9,11 +9,10 @@ namespace LitTestConvertor.Test.LitTestRunner { public class LitTestDataDiscovererTest { - MethodInfo GetMethodInfo(Action a) - { + MethodInfo GetMethodInfo(Action a) { return a.Method; } - + [Fact] public void Discoverer() { var discoverer = new LitTestDataDiscoverer(); diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index 71dc0f6dda0..a1766fa58cf 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -1,9 +1,7 @@ using DafnyDriver.Test.XUnitExtensions; -namespace LitTestConvertor.Test.LitTestRunner -{ - public class LitTests - { +namespace LitTestConvertor.Test.LitTestRunner { + public class LitTests { [ParallelTheory] [LitTestData(false)] public void LitTest(CLITestCase testCase) { From b968ec187a344fe305ecc483825f5a1127b00c72 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 30 Aug 2021 12:02:18 -0700 Subject: [PATCH 115/192] Dumping more code we don't need yet, fixing warnings --- Source/DafnyDriver.Test/DafnyTestCase.cs | 4 +- Source/DafnyDriver.Test/TemporaryDirectory.cs | 39 ----- .../XUnitExtensions/EnumerableUtils.cs | 19 --- .../XUnitExtensions/ForEachAttribute.cs | 17 -- .../XUnitExtensions/YamlDataAttribute.cs | 24 --- .../XUnitExtensions/YamlDataDiscoverer.cs | 139 ----------------- .../XUnitExtensions/YamlDataTests.cs | 147 ------------------ Source/LitTestConverter/LitTestConvertor.cs | 3 +- 8 files changed, 3 insertions(+), 389 deletions(-) delete mode 100644 Source/DafnyDriver.Test/TemporaryDirectory.cs delete mode 100644 Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs delete mode 100644 Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs delete mode 100644 Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs delete mode 100644 Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs delete mode 100644 Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index 61e27d78c39..ef8f99700f1 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -72,7 +72,7 @@ public DafnyTestCase() { } - public void Serialize(IXunitSerializationInfo info) { + public new void Serialize(IXunitSerializationInfo info) { base.Serialize(info); info.AddValue(nameof(BasePath), BasePath); info.AddValue(nameof(SourcePath), SourcePath); @@ -80,7 +80,7 @@ public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(OtherFiles), OtherFiles); } - public void Deserialize(IXunitSerializationInfo info) { + public new void Deserialize(IXunitSerializationInfo info) { base.Deserialize(info); BasePath = info.GetValue(nameof(BasePath)); SourcePath = info.GetValue(nameof(SourcePath)); diff --git a/Source/DafnyDriver.Test/TemporaryDirectory.cs b/Source/DafnyDriver.Test/TemporaryDirectory.cs deleted file mode 100644 index f420643caec..00000000000 --- a/Source/DafnyDriver.Test/TemporaryDirectory.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.IO; - -namespace DafnyDriver.Test { - public class TemporaryDirectory : IDisposable { - public readonly DirectoryInfo DirInfo; - - public TemporaryDirectory(string parent, string prefix = "") { - string dirPath; - // Loop until we pick a random name that isn't already taken. - do { - dirPath = Path.Combine(parent, prefix + Path.GetRandomFileName()); - } while (File.Exists(dirPath) || Directory.Exists(dirPath)); - - DirInfo = Directory.CreateDirectory(dirPath); - } - - public void Dispose() { - Dispose(true); - GC.SuppressFinalize(this); - } - - ~TemporaryDirectory() { - Dispose(false); - } - - protected virtual void Dispose(bool disposing) { - SafeDelete(); - } - - private void SafeDelete() { - try { - DirInfo.Delete(true); - } catch { - // Best effort only - } - } - } -} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs b/Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs deleted file mode 100644 index f461f4f8fa1..00000000000 --- a/Source/DafnyDriver.Test/XUnitExtensions/EnumerableUtils.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System.Collections.Generic; -using System.Linq; - -namespace DafnyDriver.Test { - public static class EnumerableUtils { - /** - * Source: https://docs.microsoft.com/en-us/archive/blogs/ericlippert/computing-a-cartesian-product-with-linq - */ - public static IEnumerable> CartesianProduct(this IEnumerable> sequences) { - IEnumerable> emptyProduct = new[] { Enumerable.Empty() }; - return sequences.Aggregate( - emptyProduct, - (accumulator, sequence) => - from accseq in accumulator - from item in sequence - select accseq.Concat(new[] { item })); - } - } -} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs deleted file mode 100644 index f001aba5921..00000000000 --- a/Source/DafnyDriver.Test/XUnitExtensions/ForEachAttribute.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace DafnyDriver.Test.XUnitExtensions { - public class ForEachAttribute : Attribute { - - private readonly Type EnumeratorClass; - - public ForEachAttribute(Type enumeratorClass = null) { - EnumeratorClass = enumeratorClass ?? typeof(IEnumerable<>); - } - - public Type EnumerableTypeOf(Type elementType) { - return EnumeratorClass.MakeGenericType(elementType); - } - } -} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs deleted file mode 100644 index 6b12b3e901a..00000000000 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataAttribute.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Reflection; -using DafnyDriver.Test.XUnitExtensions; -using Xunit.Sdk; - -namespace XUnitExtensions { - - [DataDiscoverer("DafnyDriver.Test.XUnitExtensions.YamlDataDiscoverer", "DafnyDriver.Test")] - public class YamlDataAttribute : FileDataAttribute { - public readonly bool WithParameterNames; - - public YamlDataAttribute(bool withParameterNames = true, string path = null, string extension = ".yml") - : base(path, extension) { - WithParameterNames = withParameterNames; - } - - public override IEnumerable GetData(MethodInfo testMethod) { - // This method is not used - the YamlDataDiscoverer has all of the actual logic instead - // because it exposes some methods for subclasses to customize such as GetYamlParser. - throw new NotImplementedException(); - } - } -} \ No newline at end of file diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs deleted file mode 100644 index 7cc87a31c01..00000000000 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataDiscoverer.cs +++ /dev/null @@ -1,139 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using JetBrains.Annotations; -using Xunit.Abstractions; -using Xunit.Sdk; -using XUnitExtensions; -using YamlDotNet.Core; -using YamlDotNet.Core.Events; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NodeDeserializers; -using YamlDotNet.Serialization.ObjectFactories; -using YamlDotNet.Serialization.Utilities; - -namespace DafnyDriver.Test.XUnitExtensions { - public class YamlDataDiscoverer : FileDataDiscoverer { - public virtual IParser GetYamlParser(string fileName, Stream stream) { - return new Parser(new StreamReader(stream)); - } - - public virtual IDeserializer GetDeserializer(string fileName) { - return new Deserializer(); - } - - private static Func ForMethodInfoDeserializeFn(IDeserializer deserializer, MethodInfo testMethod) { - return (parser, type) => { - if (type == typeof(MethodArguments)) { - MethodArguments arguments = new MethodArguments(testMethod); - arguments.Read(parser, type, t => deserializer.Deserialize(parser, t)); - return arguments; - } else { - return deserializer.Deserialize(parser, type); - } - }; - } - - public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { - return true; - } - - protected override IEnumerable FileData(IAttributeInfo dataAttribute, IMethodInfo testMethod, string fileName) { - var withParameterNames = dataAttribute.GetNamedArgument(nameof(YamlDataAttribute.WithParameterNames)); - // YamlDotNet's deserialization framework requires runtime type information - MethodInfo methodInfo = testMethod.ToRuntimeMethod(); - if (methodInfo == null) { - return null; - } - - try { - using Stream stream = File.OpenRead(fileName); - IParser parser = GetYamlParser(fileName, stream); - if (parser == null) { - return Enumerable.Empty(); - } - - IDeserializer deserializer = GetDeserializer(fileName); - parser.Consume(); - parser.Consume(); - - if (withParameterNames) { - IObjectFactory argumentsFactory = new DefaultObjectFactory(); - INodeDeserializer collectionDeserializer = new CollectionNodeDeserializer(argumentsFactory); - var nestedObjectDeserializer = ForMethodInfoDeserializeFn(deserializer, methodInfo); - if (collectionDeserializer.Deserialize(parser, typeof(List), nestedObjectDeserializer, - out var value)) { - List argumentses = (List)value; - return argumentses.SelectMany(a => a.Combinations()); - } else { - throw new ArgumentException(); - } - } else { - IEnumerable parameters = methodInfo.GetParameters(); - Type targetType = typeof(IEnumerable<>).MakeGenericType(parameters.Single().ParameterType); - IEnumerable results = (IEnumerable)deserializer.Deserialize(parser, targetType); - return results.Select(value => new[] { value }); - } - } catch (Exception e) { - throw new ArgumentException( - "Exception thrown while trying to deserialize test data from file: " + fileName, e); - } - } - } - - public class MethodArguments : IYamlConvertible { - - private MethodInfo Method; - private Dictionary ParameterNameIndexes; - private object[] Arguments; - - public MethodArguments(MethodInfo methodInfo) { - Method = methodInfo; - Arguments = new object[Method.GetParameters().Length]; - ParameterNameIndexes = Method.GetParameters() - .Select((value, index) => new KeyValuePair(value.Name, index)) - .ToDictionary(p => p.Key, p => p.Value); - } - - public void Read(IParser parser, Type expectedType, ObjectDeserializer nestedObjectDeserializer) { - if (!parser.TryConsume(out MappingStart _)) { - throw new ArgumentException(); - } - while (!parser.TryConsume(out MappingEnd _)) { - Scalar scalar = parser.Consume(); - int parameterIndex = ParameterNameIndexes[scalar.Value]; - ParameterInfo parameter = Method.GetParameters()[parameterIndex]; - Type parameterType = parameter.ParameterType; - ForEachAttribute forEach = parameter.GetCustomAttribute(); - if (forEach != null) { - parameterType = forEach.EnumerableTypeOf(parameterType); - } - - object argument = nestedObjectDeserializer(parameterType); - IValuePromise valuePromise = argument as IValuePromise; - if (valuePromise != null) { - valuePromise.ValueAvailable += (Action)(v => Arguments[parameterIndex] = TypeConverter.ChangeType(v, parameterType)); - } else { - Arguments[parameterIndex] = TypeConverter.ChangeType(argument, parameterType); - } - } - } - - public void Write(IEmitter emitter, ObjectSerializer nestedObjectSerializer) { - throw new NotImplementedException(); - } - - public IEnumerable Combinations() { - // TODO: Optimize this away when there are no ForEach attributes - IEnumerable> lifted = Arguments.Select((arg, index) => { - ParameterInfo parameter = Method.GetParameters()[index]; - ForEachAttribute forEach = parameter.GetCustomAttribute(); - return forEach == null ? new[] { arg } : ((IEnumerable)arg).Cast(); - }); - return lifted.CartesianProduct().Select(e => e.ToArray()); - } - } -} diff --git a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs b/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs deleted file mode 100644 index 75b039bac95..00000000000 --- a/Source/DafnyDriver.Test/XUnitExtensions/YamlDataTests.cs +++ /dev/null @@ -1,147 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; -using XUnitExtensions; -using YamlDotNet.RepresentationModel; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; - -namespace DafnyDriver.Test.XUnitExtensions { - public class YamlDataTests { - - private void AssertTheoryData(string methodName, IEnumerable expectedData) { - var method = typeof(YamlDataTests).GetMethod(methodName); - var attribute = method!.GetCustomAttributesData().First(a => a.AttributeType == typeof(YamlDataAttribute)); - var discoverer = new YamlDataDiscoverer(); - Assert.Equal(expectedData, discoverer.GetData(Reflector.Wrap(attribute), Reflector.Wrap(method))); - } - - [Theory] - [YamlData] - public void CalculatorTest(int lhs, int rhs, int expected) { - Assert.Equal(expected, lhs + rhs); - } - - [Fact] - public void CalculatorTestData() { - AssertTheoryData(nameof(CalculatorTest), new[] { - new object[]{ 2, 2, 4 }, - new object[]{ 3, 4, 7 } - }); - } - - [Theory] - [YamlData] - public void CalculatorCombinatorialTest([ForEach] int lhs, [ForEach] int rhs) { - Assert.Equal(rhs + lhs, lhs + rhs); - } - - [Fact] - public void CalculatorCombinatorialTestData() { - AssertTheoryData(nameof(CalculatorCombinatorialTest), new[] { - new object[]{ 1, 5 }, - new object[]{ 1, 10 }, - new object[]{ 2, 5 }, - new object[]{ 2, 10 }, - new object[]{ 3, 5 }, - new object[]{ 3, 10 }, - new object[]{ 4, 5 }, - new object[]{ 4, 10 }, - new object[]{ 5, 5 }, - new object[]{ 5, 10 }, - }); - } - - [Theory] - [YamlData] - public void MultiFileTest(int a, int b) { - Assert.Equal(a + 1, b); - } - - [Fact] - public void MultiFileTestData() { - AssertTheoryData(nameof(MultiFileTest), new[] { - new object[]{ 1, 2 }, - new object[]{ 3, 4 }, - new object[]{ 5, 6 }, - new object[]{ 7, 8 }, - }); - } - - [Theory] - [YamlData(false)] - public void DictionaryTest(Dictionary config) { - Assert.Equal(3, config.Count); - } - - [Fact] - public void DictionaryTestData() { - AssertTheoryData(nameof(DictionaryTest), new[] { - new object[]{ new Dictionary { - ["one"] = "1", - ["two"] = "2", - ["three"] = "3" - }}, - new object[]{ new Dictionary { - ["four"] = "4", - ["five"] = "5", - ["six"] = "6" - }} - }); - } - - [DataDiscoverer("DafnyDriver.Test.XUnitExtensions.CustomDiscoverer", "DafnyDriver.Test")] - public class CustomYamlDataAttribute : YamlDataAttribute { - public CustomYamlDataAttribute(bool withParameterNames = true) : base(withParameterNames) { - } - } - - [Theory] - [CustomYamlData()] - public void CustomDataDiscovererTest([ForEach()] int lhs, [ForEach()] int rhs) { - Assert.Equal(rhs + lhs, lhs + rhs); - } - - [Fact] - public void CustomDataDiscovererTestData() { - AssertTheoryData(nameof(CustomDataDiscovererTest), new[] { - new object[]{ 0, 0 }, - new object[]{ 0, 1 }, - new object[]{ 1, 0 }, - new object[]{ 1, 1 }, - new object[]{ 2, 0 }, - new object[]{ 2, 1 }, - new object[]{ 3, 0 }, - new object[]{ 3, 1 }, - new object[]{ 4, 0 }, - new object[]{ 4, 1 }, - }); - } - } - - public class Range : IEnumerable { - - public int Start; - public int End; - - public IEnumerator GetEnumerator() { - return Enumerable.Range(Start, End).GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() { - return GetEnumerator(); - } - } - - public class CustomDiscoverer : YamlDataDiscoverer { - public override IDeserializer GetDeserializer(string manifestResourceName) { - return new DeserializerBuilder().WithTagMapping("!range", typeof(Range)) - .WithNamingConvention(CamelCaseNamingConvention.Instance).Build(); - } - } -} \ No newline at end of file diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 731b9da5712..74a4c18f5e9 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -98,7 +98,6 @@ private static string ExtractLitCommand(string line) { } private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, bool invokeDirectly, string dafnyCommand) { - bool includeThisFile = true; List otherFiles = new(); Dictionary dafnyArguments = new(); @@ -120,7 +119,7 @@ private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, arguments.RemoveRange(arguments.Count - 2, 2); if (!arguments.Remove("\"%s\"")) { - includeThisFile = false; + throw new ArgumentException("Test cases that do not include the current file are not yet supported"); } // Check the arguments for anything non-standard From 0f7a204c2aa9c942bebb5566d01413edc805292b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 30 Aug 2021 13:34:11 -0700 Subject: [PATCH 116/192] Move XUnitExtensions code to its own package --- Source/Dafny.sln | 26 +++++++++++++++++++ .../DafnyDriver.Test/DafnyDriver.Test.csproj | 9 +++++-- Source/DafnyDriver.Test/DafnyTestCase.cs | 3 +-- .../CalculatorCombinatorialTest.yml | 2 -- .../YamlDataTests/CalculatorTest.yml | 6 ----- .../CustomDataDiscovererTest.yml | 6 ----- .../YamlDataTests/DictionaryTest.yml | 6 ----- .../YamlDataTests/MultiFileTest/Data1.yml | 4 --- .../YamlDataTests/MultiFileTest/Data2.yml | 4 --- Source/LitTestConverter/LitTestConvertor.cs | 4 +-- .../Conversion/ConvertingFromLitToXunit.cs | 6 ++--- .../LitTestRunner/LitTestDataDiscoverer.cs | 4 +-- .../LitTestDataDiscovererTest.cs | 2 +- .../LitTestRunner/LitTests.cs | 2 +- .../XUnitExtensions/AssertWithDiff.cs | 0 .../XUnitExtensions/CLITestCase.cs | 3 +-- .../CollectionPerTestCaseTheoryDiscoverer.cs | 0 .../XUnitExtensions/FileDataAttribute.cs | 2 +- .../XUnitExtensions/FileDataDiscoverer.cs | 2 +- .../ParallelTheoryAttribute.cs | 0 .../XUnitExtensions/TestCaseWithCollection.cs | 0 Source/XUnitExtensions/XUnitExtensions.csproj | 24 +++++++++++++++++ 22 files changed, 69 insertions(+), 46 deletions(-) delete mode 100644 Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorCombinatorialTest.yml delete mode 100644 Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorTest.yml delete mode 100644 Source/DafnyDriver.Test/TestFiles/YamlDataTests/CustomDataDiscovererTest.yml delete mode 100644 Source/DafnyDriver.Test/TestFiles/YamlDataTests/DictionaryTest.yml delete mode 100644 Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data1.yml delete mode 100644 Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data2.yml rename Source/{DafnyDriver.Test => }/XUnitExtensions/AssertWithDiff.cs (100%) rename Source/{DafnyDriver.Test => }/XUnitExtensions/CLITestCase.cs (98%) rename Source/{DafnyDriver.Test => }/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs (100%) rename Source/{DafnyDriver.Test => }/XUnitExtensions/FileDataAttribute.cs (91%) rename Source/{DafnyDriver.Test => }/XUnitExtensions/FileDataDiscoverer.cs (97%) rename Source/{DafnyDriver.Test => }/XUnitExtensions/ParallelTheoryAttribute.cs (100%) rename Source/{DafnyDriver.Test => }/XUnitExtensions/TestCaseWithCollection.cs (100%) create mode 100644 Source/XUnitExtensions/XUnitExtensions.csproj diff --git a/Source/Dafny.sln b/Source/Dafny.sln index cc101458fe7..ec9152c0623 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -28,6 +28,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DafnyLanguageServer", "Dafn EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DafnyLanguageServer.Test", "DafnyLanguageServer.Test\DafnyLanguageServer.Test.csproj", "{320C02A3-D3E6-4AC7-A7E2-170AF86A6EEC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "XUnitExtensions", "XUnitExtensions\XUnitExtensions.csproj", "{43731A57-DBA9-43AC-BC83-A9211DA7EB77}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Checked|.NET = Checked|.NET @@ -284,6 +286,30 @@ Global {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Mixed Platforms.Build.0 = Release|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|x86.ActiveCfg = Release|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|x86.Build.0 = Release|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|.NET.ActiveCfg = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|.NET.Build.0 = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|Any CPU.ActiveCfg = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|Any CPU.Build.0 = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|Mixed Platforms.ActiveCfg = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|Mixed Platforms.Build.0 = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|x86.ActiveCfg = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|x86.Build.0 = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|.NET.ActiveCfg = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|.NET.Build.0 = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Any CPU.Build.0 = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|x86.ActiveCfg = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|x86.Build.0 = Debug|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|.NET.ActiveCfg = Release|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|.NET.Build.0 = Release|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Any CPU.ActiveCfg = Release|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Any CPU.Build.0 = Release|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Mixed Platforms.Build.0 = Release|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|x86.ActiveCfg = Release|Any CPU + {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj index 3bcb28c7960..449288dadf3 100644 --- a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj +++ b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj @@ -6,14 +6,15 @@ - + - + + @@ -27,4 +28,8 @@ PreserveNewest + + + + diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index ef8f99700f1..2a14984a2e7 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -1,10 +1,9 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Reflection; -using DafnyDriver.Test.XUnitExtensions; using Xunit.Abstractions; +using XUnitExtensions; namespace DafnyDriver.Test { diff --git a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorCombinatorialTest.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorCombinatorialTest.yml deleted file mode 100644 index 05fe6690d86..00000000000 --- a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorCombinatorialTest.yml +++ /dev/null @@ -1,2 +0,0 @@ -- lhs: [1, 2, 3, 4, 5] - rhs: [5, 10] diff --git a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorTest.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorTest.yml deleted file mode 100644 index 1f1f8294389..00000000000 --- a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CalculatorTest.yml +++ /dev/null @@ -1,6 +0,0 @@ -- lhs: 2 - rhs: 2 - expected: 4 -- lhs: 3 - rhs: 4 - expected: 7 diff --git a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CustomDataDiscovererTest.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CustomDataDiscovererTest.yml deleted file mode 100644 index 9305876ab94..00000000000 --- a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/CustomDataDiscovererTest.yml +++ /dev/null @@ -1,6 +0,0 @@ -- lhs: !range - start: 0 - end: 5 - rhs: !range - start: 0 - end: 2 diff --git a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/DictionaryTest.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/DictionaryTest.yml deleted file mode 100644 index ec67b14a12c..00000000000 --- a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/DictionaryTest.yml +++ /dev/null @@ -1,6 +0,0 @@ -- one: 1 - two: 2 - three: 3 -- four: 4 - five: 5 - six: 6 diff --git a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data1.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data1.yml deleted file mode 100644 index 344e8af8d98..00000000000 --- a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data1.yml +++ /dev/null @@ -1,4 +0,0 @@ -- a: 1 - b: 2 -- a: 3 - b: 4 \ No newline at end of file diff --git a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data2.yml b/Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data2.yml deleted file mode 100644 index 15a08ae994d..00000000000 --- a/Source/DafnyDriver.Test/TestFiles/YamlDataTests/MultiFileTest/Data2.yml +++ /dev/null @@ -1,4 +0,0 @@ -- a: 5 - b: 6 -- a: 7 - b: 8 \ No newline at end of file diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs index 74a4c18f5e9..eeeca21d363 100644 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ b/Source/LitTestConverter/LitTestConvertor.cs @@ -3,11 +3,11 @@ using System.IO; using System.Linq; using DafnyDriver.Test; -using DafnyDriver.Test.XUnitExtensions; +using XUnitExtensions; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; -namespace LitTestConvertor { +namespace LitTestConverter { public class LitTestConvertor { diff --git a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs index 908855c472a..66423f3b9c6 100644 --- a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs +++ b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs @@ -1,8 +1,6 @@ -using System; using System.Collections.Generic; using System.IO; using System.Reflection; -using DafnyDriver.Test; using Xunit; namespace LitTestConvertor.Test { @@ -18,14 +16,14 @@ private static IEnumerable ReadLines(StreamReader reader) { [Fact] public void HelloWorld() { - var convertor = new LitTestConvertor(); + var convertor = new LitTestConverter.LitTestConvertor(); var lines = File.ReadLines("TestFiles/HelloWorldLitTest.dfy"); var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/HelloWorldLitTest.dfy", false, lines); } [Fact] public void VerifyOnly() { - var convertor = new LitTestConvertor(); + var convertor = new LitTestConverter.LitTestConvertor(); using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream("LitTestConvertor.Test.TestFiles.VerifyOnlyLitTest.dfy"); using var reader = new StreamReader(stream); diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs index a7340bc0aa8..58ac5ec8aa2 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -2,14 +2,14 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using DafnyDriver.Test.XUnitExtensions; using Xunit.Abstractions; using Xunit.Sdk; +using XUnitExtensions; namespace DafnyDriver.Test { public class LitTestDataDiscoverer : FileDataDiscoverer { - private readonly LitTestConvertor.LitTestConvertor convertor = new(); + private readonly LitTestConverter.LitTestConvertor convertor = new(); public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { return true; diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs index 2fdae0aa86a..cd71273f63f 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs @@ -2,9 +2,9 @@ using System.Linq; using System.Reflection; using DafnyDriver.Test; -using DafnyDriver.Test.XUnitExtensions; using Xunit; using Xunit.Sdk; +using XUnitExtensions; namespace LitTestConvertor.Test.LitTestRunner { public class LitTestDataDiscovererTest { diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index a1766fa58cf..468fc68bdc8 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -1,4 +1,4 @@ -using DafnyDriver.Test.XUnitExtensions; +using XUnitExtensions; namespace LitTestConvertor.Test.LitTestRunner { public class LitTests { diff --git a/Source/DafnyDriver.Test/XUnitExtensions/AssertWithDiff.cs b/Source/XUnitExtensions/AssertWithDiff.cs similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/AssertWithDiff.cs rename to Source/XUnitExtensions/AssertWithDiff.cs diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs b/Source/XUnitExtensions/CLITestCase.cs similarity index 98% rename from Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs rename to Source/XUnitExtensions/CLITestCase.cs index 4eea2cc31df..6069e52538e 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/CLITestCase.cs +++ b/Source/XUnitExtensions/CLITestCase.cs @@ -8,9 +8,8 @@ using Xunit; using Xunit.Abstractions; using Xunit.Sdk; -using XUnitExtensions; -namespace DafnyDriver.Test.XUnitExtensions { +namespace XUnitExtensions { public class CLITestCase : IXunitSerializable { protected Assembly CliAssembly; diff --git a/Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs rename to Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs diff --git a/Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs b/Source/XUnitExtensions/FileDataAttribute.cs similarity index 91% rename from Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs rename to Source/XUnitExtensions/FileDataAttribute.cs index b87a0b3231e..8f375bdbb7a 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/FileDataAttribute.cs +++ b/Source/XUnitExtensions/FileDataAttribute.cs @@ -2,7 +2,7 @@ using System.Reflection; using Xunit.Sdk; -namespace DafnyDriver.Test.XUnitExtensions { +namespace XUnitExtensions { public abstract class FileDataAttribute : DataAttribute { public readonly string Path; public readonly string Extension; diff --git a/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs b/Source/XUnitExtensions/FileDataDiscoverer.cs similarity index 97% rename from Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs rename to Source/XUnitExtensions/FileDataDiscoverer.cs index ff7d95c454f..e1df405ee9c 100644 --- a/Source/DafnyDriver.Test/XUnitExtensions/FileDataDiscoverer.cs +++ b/Source/XUnitExtensions/FileDataDiscoverer.cs @@ -6,7 +6,7 @@ using Xunit.Abstractions; using Xunit.Sdk; -namespace DafnyDriver.Test.XUnitExtensions { +namespace XUnitExtensions { public abstract class FileDataDiscoverer : IDataDiscoverer { protected string GetBasePath(IAttributeInfo attributeInfo, IMethodInfo testMethod) { var path = attributeInfo.GetNamedArgument(nameof(FileDataAttribute.Path)); diff --git a/Source/DafnyDriver.Test/XUnitExtensions/ParallelTheoryAttribute.cs b/Source/XUnitExtensions/ParallelTheoryAttribute.cs similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/ParallelTheoryAttribute.cs rename to Source/XUnitExtensions/ParallelTheoryAttribute.cs diff --git a/Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs b/Source/XUnitExtensions/TestCaseWithCollection.cs similarity index 100% rename from Source/DafnyDriver.Test/XUnitExtensions/TestCaseWithCollection.cs rename to Source/XUnitExtensions/TestCaseWithCollection.cs diff --git a/Source/XUnitExtensions/XUnitExtensions.csproj b/Source/XUnitExtensions/XUnitExtensions.csproj new file mode 100644 index 00000000000..4bddf803848 --- /dev/null +++ b/Source/XUnitExtensions/XUnitExtensions.csproj @@ -0,0 +1,24 @@ + + + + net5.0 + + false + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + From 7f8c23a0c1fa37af64b35e0c2020ccc7fd10ffc9 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 30 Aug 2021 13:35:59 -0700 Subject: [PATCH 117/192] Fix assembly reference --- Source/XUnitExtensions/ParallelTheoryAttribute.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/XUnitExtensions/ParallelTheoryAttribute.cs b/Source/XUnitExtensions/ParallelTheoryAttribute.cs index 4bd67701e75..206fe54a594 100644 --- a/Source/XUnitExtensions/ParallelTheoryAttribute.cs +++ b/Source/XUnitExtensions/ParallelTheoryAttribute.cs @@ -3,5 +3,5 @@ using Xunit.Sdk; [AttributeUsage(AttributeTargets.Method)] -[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "DafnyDriver.Test")] +[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "XUnitExtensions")] public class ParallelTheoryAttribute : TheoryAttribute { } \ No newline at end of file From ecb61c510f9fbafbacaf9002438a53233697ecd2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 30 Aug 2021 16:12:53 -0700 Subject: [PATCH 118/192] Move shard filtering to a separate collection "orderer" ;) --- .../Conversion/ConvertingFromLitToXunit.cs | 8 ---- .../LitTestRunner/LitTests.cs | 3 ++ .../CollectionPerTestCaseTheoryDiscoverer.cs | 28 ------------- .../TestCollectionShardFilter.cs | 39 +++++++++++++++++++ 4 files changed, 42 insertions(+), 36 deletions(-) create mode 100644 Source/XUnitExtensions/TestCollectionShardFilter.cs diff --git a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs index 66423f3b9c6..62259a18597 100644 --- a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs +++ b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs @@ -6,14 +6,6 @@ namespace LitTestConvertor.Test { public class ConvertingFromLitToXunit { - - private static IEnumerable ReadLines(StreamReader reader) { - string line; - while ((line = reader.ReadLine()) != null) { - yield return line; - } - } - [Fact] public void HelloWorld() { var convertor = new LitTestConverter.LitTestConvertor(); diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index 468fc68bdc8..d00dcd51ab7 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -1,5 +1,8 @@ +using Xunit; using XUnitExtensions; +[assembly: TestCollectionOrderer("XUnitExtensions.TestCollectionShardFilter", "XUnitExtensions")] + namespace LitTestConvertor.Test.LitTestRunner { public class LitTests { [ParallelTheory] diff --git a/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index 5a45bc0f54d..705b4737e8f 100644 --- a/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -24,34 +24,6 @@ public IEnumerable Discover(ITestFrameworkDiscoveryOptions disco var testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); - // Select the requested fraction of the test cases if using the XUNIT_SHARD[_COUNT] environment variables. - // Ideally this would be handled at a higher level so that cases from different test methods could be - // balanced as a whole. - var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); - var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD_COUNT"); - if (shardEnvVar != null || numShardsEnvVar != null) { - if (shardEnvVar == null || numShardsEnvVar == null) { - throw new ArgumentNullException( - "The XUNIT_SHARD and XUNIT_SHARD_COUNT environment variables must both be provided."); - } - - var shard = Int32.Parse(shardEnvVar); - var numShards = Int32.Parse(numShardsEnvVar); - if (numShards <= 0) { - throw new ArgumentNullException( - "XUNIT_SHARD_COUNT must be greater than 0."); - } - if (shard <= 0 || shard > numShards) { - throw new ArgumentNullException( - "XUNIT_SHARD must be at least 1 and at most XUNIT_SHARD_COUNT."); - } - - var testCaseList = testCases.ToList(); - var shardStart = (shard - 1) * testCaseList.Count / numShards; - var shardEnd = shard * testCaseList.Count / numShards; - testCases = testCaseList.GetRange(shardStart, shardEnd - shardStart); - } - return testCases.Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); } } diff --git a/Source/XUnitExtensions/TestCollectionShardFilter.cs b/Source/XUnitExtensions/TestCollectionShardFilter.cs new file mode 100644 index 00000000000..9f9ac45b852 --- /dev/null +++ b/Source/XUnitExtensions/TestCollectionShardFilter.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +namespace XUnitExtensions { + public class TestCollectionShardFilter : ITestCollectionOrderer { + public IEnumerable OrderTestCollections(IEnumerable testCollections) { + // Select the requested fraction of the test collections if using the XUNIT_SHARD[_COUNT] environment variables. + var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); + var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD_COUNT"); + if (shardEnvVar != null || numShardsEnvVar != null) { + if (shardEnvVar == null || numShardsEnvVar == null) { + throw new InvalidOperationException( + "The XUNIT_SHARD and XUNIT_SHARD_COUNT environment variables must both be provided."); + } + + var shard = Int32.Parse(shardEnvVar); + var numShards = Int32.Parse(numShardsEnvVar); + if (numShards <= 0) { + throw new InvalidOperationException( + "XUNIT_SHARD_COUNT must be greater than 0."); + } + if (shard <= 0 || shard > numShards) { + throw new InvalidOperationException( + "XUNIT_SHARD must be at least 1 and at most XUNIT_SHARD_COUNT."); + } + + var testCaseList = testCollections.ToList(); + var shardStart = (shard - 1) * testCaseList.Count / numShards; + var shardEnd = shard * testCaseList.Count / numShards; + return testCaseList.GetRange(shardStart, shardEnd - shardStart); + } + + return testCollections; + } + } +} \ No newline at end of file From 0e486583c6060ca0cb858fc0c7b7f5756a11688b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 31 Aug 2021 15:04:04 -0700 Subject: [PATCH 119/192] Half-done refactoring to customize test cases more To add source file, traits, and skip reason, in an xUnit v3 style --- Source/XUnitExtensions/AssertWithDiff.cs | 6 +- .../CollectionPerTestCaseTheoryDiscoverer.cs | 4 +- Source/XUnitExtensions/FileTestCase.cs | 71 +++++++++++++++++++ Source/XUnitExtensions/ITheoryDataRow.cs | 29 ++++++++ .../ParallelTheoryAttribute.cs | 9 ++- .../XUnitExtensions/TestCaseWithCollection.cs | 55 -------------- 6 files changed, 111 insertions(+), 63 deletions(-) create mode 100644 Source/XUnitExtensions/FileTestCase.cs create mode 100644 Source/XUnitExtensions/ITheoryDataRow.cs delete mode 100644 Source/XUnitExtensions/TestCaseWithCollection.cs diff --git a/Source/XUnitExtensions/AssertWithDiff.cs b/Source/XUnitExtensions/AssertWithDiff.cs index 7d4287a45ce..f6a2f794860 100644 --- a/Source/XUnitExtensions/AssertWithDiff.cs +++ b/Source/XUnitExtensions/AssertWithDiff.cs @@ -16,13 +16,13 @@ public static void Equal(string expected, string actual) { foreach (var line in diff.Lines) { switch (line.Type) { case ChangeType.Inserted: - message.Append("+"); + message.Append('+'); break; case ChangeType.Deleted: - message.Append("-"); + message.Append('-'); break; default: - message.Append(" "); + message.Append(' '); break; } message.AppendLine(line.Text); diff --git a/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs index 705b4737e8f..c7412d6e477 100644 --- a/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ b/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs @@ -13,7 +13,7 @@ public CollectionPerTestCaseTheoryDiscoverer(IMessageSink diagnosticMessageSink) theoryDiscoverer = new SkippableTheoryDiscoverer(diagnosticMessageSink); } - private TestCollection testCollectionForTestCase(IXunitTestCase testCase) { + private static TestCollection TestCollectionForTestCase(ITestCase testCase) { return new TestCollection(testCase.TestMethod.TestClass.TestCollection.TestAssembly, (ITypeInfo)null, "Test collection for " + testCase.DisplayName); } @@ -24,7 +24,7 @@ public IEnumerable Discover(ITestFrameworkDiscoveryOptions disco var testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); - return testCases.Select(testCase => new TestCaseWithCollection(testCase, testCollectionForTestCase(testCase))); + return testCases.Select(testCase => new FileTestCase(testCase, TestCollectionForTestCase(testCase))); } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs new file mode 100644 index 00000000000..04ba2f506bc --- /dev/null +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Threading; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace XUnitExtensions { + public class FileTestCase : LongLivedMarshalByRefObject, IXunitTestCase { + + protected XunitTestCase innerTestCase; + + public FileTestCase(IMessageSink diagnosticMessageSink, string path, ITestMethod testMethod, ITheoryRowData data) { + innerTestCase = new XunitTestCase(diagnosticMessageSink, TestMethodDisplay.Method, testMethod, data.GetData()); + foreach (var (key, value) in data.Traits) { + innerTestCase.Traits.Add(key, value); + } + innerTestCase.SourceInformation = new SourceInformation { FileName = path }; + SkipReason = data.Skip; + } + + [Obsolete("Called by the de-serializer", error: true)] + public FileTestCase() { } + + public void Deserialize(IXunitSerializationInfo info) { + innerTestCase = info.GetValue(nameof(innerTestCase)); + SkipReason = info.GetValue(nameof(SkipReason)); + } + + public void Serialize(IXunitSerializationInfo info) { + info.AddValue(nameof(innerTestCase), innerTestCase); + info.AddValue(nameof(SkipReason), SkipReason); + } + public string DisplayName => innerTestCase.DisplayName; + public string SkipReason { + get; + protected set; + } + + public ISourceInformation SourceInformation { + get => innerTestCase.SourceInformation; + set => innerTestCase.SourceInformation = value; + } + + public ITestMethod TestMethod => innerTestCase.TestMethod; + public object[] TestMethodArguments => innerTestCase.TestMethodArguments; + public Dictionary> Traits => innerTestCase.Traits; + public string UniqueID => innerTestCase.UniqueID; + + public Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, + ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { + + return new XunitTestCaseRunner( + this, + DisplayName, + SkipReason, + constructorArguments, + TestMethodArguments, + messageBus, + aggregator, + cancellationTokenSource + ).RunAsync(); + } + + public Exception InitializationException => innerTestCase.InitializationException; + public IMethodInfo Method => innerTestCase.Method; + public int Timeout => innerTestCase.Timeout; + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/ITheoryDataRow.cs b/Source/XUnitExtensions/ITheoryDataRow.cs new file mode 100644 index 00000000000..2d571332079 --- /dev/null +++ b/Source/XUnitExtensions/ITheoryDataRow.cs @@ -0,0 +1,29 @@ +using System.Collections.Generic; +using Xunit; + +namespace XUnitExtensions { + public interface ITheoryRowData { + /// + /// Gets the reason for skipping this row of data; if null is returned, then the data + /// row isn't skipped. + /// + string? Skip { get; } + + /// + /// Gets the display name for the test (replacing the default behavior, which would be to use the DisplayName + /// from , or falling back to the class & method name). + /// + string? TestDisplayName { get; } + + /// + /// Gets the trait values associated with this theory data row. If there are none, you may either + /// return a null or empty dictionary. + /// + Dictionary>? Traits { get; } + + /// + /// Gets the theory data. + /// + object?[] GetData(); + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/ParallelTheoryAttribute.cs b/Source/XUnitExtensions/ParallelTheoryAttribute.cs index 206fe54a594..778d50355d5 100644 --- a/Source/XUnitExtensions/ParallelTheoryAttribute.cs +++ b/Source/XUnitExtensions/ParallelTheoryAttribute.cs @@ -2,6 +2,9 @@ using Xunit; using Xunit.Sdk; -[AttributeUsage(AttributeTargets.Method)] -[XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "XUnitExtensions")] -public class ParallelTheoryAttribute : TheoryAttribute { } \ No newline at end of file +namespace XUnitExtensions { + [AttributeUsage(AttributeTargets.Method)] + [XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "XUnitExtensions")] + public class ParallelTheoryAttribute : TheoryAttribute { + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/TestCaseWithCollection.cs b/Source/XUnitExtensions/TestCaseWithCollection.cs deleted file mode 100644 index 7109543dda2..00000000000 --- a/Source/XUnitExtensions/TestCaseWithCollection.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Threading; -using System.Threading.Tasks; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace XUnitExtensions { - public class TestCaseWithCollection : LongLivedMarshalByRefObject, IXunitTestCase { - - private IXunitTestCase testCase; - public ITestMethod TestMethod { get; set; } - - public TestCaseWithCollection(IXunitTestCase testCase, ITestCollection collection) { - this.testCase = testCase; - - var testClassWithCollection = new TestClass(collection, testCase.TestMethod.TestClass.Class); - TestMethod = new TestMethod(testClassWithCollection, testCase.TestMethod.Method); - } - - [Obsolete("Called by the de-serializer", error: true)] - public TestCaseWithCollection() { } - - public void Deserialize(IXunitSerializationInfo info) { - testCase = info.GetValue("InnerTestCase"); - TestMethod = info.GetValue("TestMethod"); - } - - public void Serialize(IXunitSerializationInfo info) { - info.AddValue("InnerTestCase", testCase); - info.AddValue("TestMethod", TestMethod); - } - public string DisplayName => testCase.DisplayName; - public string SkipReason => testCase.SkipReason; - public ISourceInformation SourceInformation { - get => testCase.SourceInformation; - set => testCase.SourceInformation = value; - } - - public object[] TestMethodArguments => testCase.TestMethodArguments; - public Dictionary> Traits => testCase.Traits; - public string UniqueID => testCase.UniqueID; - - public Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, - ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { - return testCase.RunAsync(diagnosticMessageSink, messageBus, constructorArguments, aggregator, cancellationTokenSource); - } - - public Exception InitializationException { get; } - public IMethodInfo Method { get; } - public int Timeout { get; } - } -} \ No newline at end of file From 206ce46bda52c93db9f491a320d2f3167a39371f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 9 Sep 2021 12:37:18 -0700 Subject: [PATCH 120/192] More progress on specialized FileTestCases --- Source/DafnyDriver.Test/DafnyTestCase.cs | 22 ++++++++---- .../LitTestRunner/LitTestDataDiscoverer.cs | 5 +-- .../LitTestDataDiscovererTest.cs | 16 ++++----- .../LitTestRunner/LitTests.cs | 4 +-- .../CollectionPerTestCaseTheoryDiscoverer.cs | 30 ---------------- Source/XUnitExtensions/FileDataAttribute.cs | 10 ++---- Source/XUnitExtensions/FileTestCase.cs | 35 +++++++++++-------- Source/XUnitExtensions/FileTheoryAttribute.cs | 10 ++++++ .../XUnitExtensions/FileTheoryDiscoverer.cs | 28 +++++++++++++++ Source/XUnitExtensions/ITheoryDataRow.cs | 11 +++++- .../ParallelTheoryAttribute.cs | 10 ------ 11 files changed, 98 insertions(+), 83 deletions(-) delete mode 100644 Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs create mode 100644 Source/XUnitExtensions/FileTheoryAttribute.cs create mode 100644 Source/XUnitExtensions/FileTheoryDiscoverer.cs delete mode 100644 Source/XUnitExtensions/ParallelTheoryAttribute.cs diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index 2a14984a2e7..25fdcd31b33 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Reflection; using Xunit.Abstractions; +using Xunit.Sdk; using XUnitExtensions; namespace DafnyDriver.Test { @@ -11,7 +12,7 @@ namespace DafnyDriver.Test { * Specialization of CLITestCase that mainly exists to support a much more * concise definition of ToString(). */ - public class DafnyTestCase : CLITestCase { + public class DafnyTestCase : CLITestCase, IFileTheoryRowData { private static readonly Assembly dafnyDriverAssembly = Assembly.GetAssembly(typeof(Microsoft.Dafny.DafnyDriver)); @@ -53,7 +54,6 @@ private static string ConfigPairToArgument(KeyValuePair pair) { } private string BasePath; - private string SourcePath; private Dictionary DafnyOptions = new(); private List OtherFiles = new(); @@ -62,7 +62,9 @@ public DafnyTestCase(string basePath, string fullSourcePath, Dictionary(nameof(BasePath)); - SourcePath = info.GetValue(nameof(SourcePath)); + SourceInformation = info.GetValue(nameof(SourceInformation)); DafnyOptions = info.GetValue>(nameof(DafnyOptions)); OtherFiles = info.GetValue>(nameof(OtherFiles)); } public override string ToString() { - var relativePath = SourcePath[(BasePath.Length + 1)..]; + var relativePath = SourceInformation.FileName[(BasePath.Length + 1)..]; return String.Join(" ", OptionsToArguments(relativePath, DafnyOptions, OtherFiles)); } + + public ISourceInformation SourceInformation { get; protected set; } + public string? Skip => null; + public string? TestDisplayName => SourceInformation.FileName; + public Dictionary>? Traits => null; + public object[] GetData() { + return new object[] { this }; + } } } \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs index 58ac5ec8aa2..829b55e823b 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -30,8 +30,5 @@ protected override IEnumerable FileData(IAttributeInfo attributeInfo, [DataDiscoverer("DafnyDriver.Test.LitTestDataDiscoverer", "LitTestConvertor.Test")] public class LitTestDataAttribute : FileDataAttribute { - public bool InvokeCliDirectly; - public LitTestDataAttribute(bool invokeCliDirectly = false) : base(extension: ".dfy") { - InvokeCliDirectly = invokeCliDirectly; - } + public bool InvokeCliDirectly { get; set; } } \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs index cd71273f63f..ce289f81dc9 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs @@ -15,14 +15,14 @@ MethodInfo GetMethodInfo(Action a) { [Fact] public void Discoverer() { - var discoverer = new LitTestDataDiscoverer(); - var test = new LitTests(); - var methodInfo = GetMethodInfo(test.LitTest); - var method = Reflector.Wrap(methodInfo); - var attribute = CustomAttributeData.GetCustomAttributes(methodInfo).First(a => a.AttributeType == typeof(LitTestDataAttribute)); - var xunitAttribute = Reflector.Wrap(attribute); - var data = discoverer.GetData(xunitAttribute, method); - Assert.True(data.Any()); + // var discoverer = new LitTestDataDiscoverer(); + // var test = new LitTests(); + // var methodInfo = GetMethodInfo(test.LitTest); + // var method = Reflector.Wrap(methodInfo); + // var attribute = CustomAttributeData.GetCustomAttributes(methodInfo).First(a => a.AttributeType == typeof(LitTestDataAttribute)); + // var xunitAttribute = Reflector.Wrap(attribute); + // var data = discoverer.GetData(xunitAttribute, method).ToList(); + // Assert.True(data.Any()); } } } \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index d00dcd51ab7..8411ba9a589 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -5,8 +5,8 @@ namespace LitTestConvertor.Test.LitTestRunner { public class LitTests { - [ParallelTheory] - [LitTestData(false)] + [FileTheory] + [LitTestData(Extension = ".dfy", InvokeCliDirectly = false)] public void LitTest(CLITestCase testCase) { testCase.Run(); } diff --git a/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs b/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs deleted file mode 100644 index c7412d6e477..00000000000 --- a/Source/XUnitExtensions/CollectionPerTestCaseTheoryDiscoverer.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace XUnitExtensions { - public class CollectionPerTestCaseTheoryDiscoverer : IXunitTestCaseDiscoverer { - - private readonly IXunitTestCaseDiscoverer theoryDiscoverer; - - public CollectionPerTestCaseTheoryDiscoverer(IMessageSink diagnosticMessageSink) { - theoryDiscoverer = new SkippableTheoryDiscoverer(diagnosticMessageSink); - } - - private static TestCollection TestCollectionForTestCase(ITestCase testCase) { - return new TestCollection(testCase.TestMethod.TestClass.TestCollection.TestAssembly, - (ITypeInfo)null, "Test collection for " + testCase.DisplayName); - } - - public IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { - // This discoverer requires pre-enumeration in order to assign a collection to each test case. - discoveryOptions.SetValue("xunit.discovery.PreEnumerateTheories", true); - - var testCases = theoryDiscoverer.Discover(discoveryOptions, testMethod, factAttribute); - - return testCases.Select(testCase => new FileTestCase(testCase, TestCollectionForTestCase(testCase))); - } - } -} \ No newline at end of file diff --git a/Source/XUnitExtensions/FileDataAttribute.cs b/Source/XUnitExtensions/FileDataAttribute.cs index 8f375bdbb7a..567bef8b3ff 100644 --- a/Source/XUnitExtensions/FileDataAttribute.cs +++ b/Source/XUnitExtensions/FileDataAttribute.cs @@ -4,13 +4,9 @@ namespace XUnitExtensions { public abstract class FileDataAttribute : DataAttribute { - public readonly string Path; - public readonly string Extension; - - public FileDataAttribute(string path = null, string extension = null) { - Path = path; - Extension = extension; - } + public virtual string Path { get; set; } + + public virtual string Extension { get; set; } public override IEnumerable GetData(MethodInfo testMethod) { throw new System.NotImplementedException(); diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index 04ba2f506bc..2497e21df06 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -12,12 +12,21 @@ public class FileTestCase : LongLivedMarshalByRefObject, IXunitTestCase { protected XunitTestCase innerTestCase; - public FileTestCase(IMessageSink diagnosticMessageSink, string path, ITestMethod testMethod, ITheoryRowData data) { - innerTestCase = new XunitTestCase(diagnosticMessageSink, TestMethodDisplay.Method, testMethod, data.GetData()); - foreach (var (key, value) in data.Traits) { - innerTestCase.Traits.Add(key, value); + public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, IFileTheoryRowData data) { + var collection = new TestCollection(testMethod.TestClass.TestCollection.TestAssembly, + (ITypeInfo)null, "Test collection for " + data.TestDisplayName); + var testClassWithCollection = new TestClass(collection, testMethod.TestClass.Class); + var testMethodWithCollection = new TestMethod(testClassWithCollection, testMethod.Method); + + innerTestCase = new XunitTestCase(diagnosticMessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.All, + testMethodWithCollection, data.GetData()); + if (data.Traits != null) { + foreach(var (key, value) in data.Traits) { + innerTestCase.Traits.Add(key, value); + } } - innerTestCase.SourceInformation = new SourceInformation { FileName = path }; + + innerTestCase.SourceInformation = data.SourceInformation; SkipReason = data.Skip; } @@ -33,22 +42,22 @@ public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(innerTestCase), innerTestCase); info.AddValue(nameof(SkipReason), SkipReason); } - public string DisplayName => innerTestCase.DisplayName; - public string SkipReason { - get; - protected set; - } + public string SkipReason { get; protected set; } public ISourceInformation SourceInformation { get => innerTestCase.SourceInformation; set => innerTestCase.SourceInformation = value; } + public string DisplayName => innerTestCase.DisplayName; public ITestMethod TestMethod => innerTestCase.TestMethod; public object[] TestMethodArguments => innerTestCase.TestMethodArguments; public Dictionary> Traits => innerTestCase.Traits; public string UniqueID => innerTestCase.UniqueID; - + public Exception InitializationException => innerTestCase.InitializationException; + public IMethodInfo Method => innerTestCase.Method; + public int Timeout => innerTestCase.Timeout; + public Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { @@ -63,9 +72,5 @@ public Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus cancellationTokenSource ).RunAsync(); } - - public Exception InitializationException => innerTestCase.InitializationException; - public IMethodInfo Method => innerTestCase.Method; - public int Timeout => innerTestCase.Timeout; } } \ No newline at end of file diff --git a/Source/XUnitExtensions/FileTheoryAttribute.cs b/Source/XUnitExtensions/FileTheoryAttribute.cs new file mode 100644 index 00000000000..daf006a0a85 --- /dev/null +++ b/Source/XUnitExtensions/FileTheoryAttribute.cs @@ -0,0 +1,10 @@ +using System; +using Xunit; +using Xunit.Sdk; + +namespace XUnitExtensions { + [AttributeUsage(AttributeTargets.Method)] + [XunitTestCaseDiscoverer("XUnitExtensions.FileTheoryDiscoverer", "XUnitExtensions")] + public class FileTheoryAttribute : TheoryAttribute { + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/FileTheoryDiscoverer.cs b/Source/XUnitExtensions/FileTheoryDiscoverer.cs new file mode 100644 index 00000000000..20e02809e7a --- /dev/null +++ b/Source/XUnitExtensions/FileTheoryDiscoverer.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace XUnitExtensions { + public class FileTheoryDiscoverer : TheoryDiscoverer { + + public FileTheoryDiscoverer(IMessageSink diagnosticMessageSink) : base(diagnosticMessageSink) { + } + + public override IEnumerable Discover(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo factAttribute) { + // This discoverer requires pre-enumeration in order to assign a collection to each test case. + discoveryOptions.SetValue("xunit.discovery.PreEnumerateTheories", true); + return base.Discover(discoveryOptions, testMethod, factAttribute); + } + + protected override IXunitTestCase CreateTestCaseForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + IAttributeInfo theoryAttribute, object[] dataRow) { + if (dataRow.Length == 1 && dataRow[0] is IFileTheoryRowData theoryRowData) { + return new FileTestCase(DiagnosticMessageSink, testMethod, theoryRowData); + } + + return base.CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); + } + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/ITheoryDataRow.cs b/Source/XUnitExtensions/ITheoryDataRow.cs index 2d571332079..4628b715bac 100644 --- a/Source/XUnitExtensions/ITheoryDataRow.cs +++ b/Source/XUnitExtensions/ITheoryDataRow.cs @@ -1,8 +1,17 @@ using System.Collections.Generic; using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; namespace XUnitExtensions { - public interface ITheoryRowData { + public interface IFileTheoryRowData { + + /// + /// Gets the source information for this row of data; if null is returned, then the location + /// of the test method is ussed. + /// + ISourceInformation? SourceInformation { get; } + /// /// Gets the reason for skipping this row of data; if null is returned, then the data /// row isn't skipped. diff --git a/Source/XUnitExtensions/ParallelTheoryAttribute.cs b/Source/XUnitExtensions/ParallelTheoryAttribute.cs deleted file mode 100644 index 778d50355d5..00000000000 --- a/Source/XUnitExtensions/ParallelTheoryAttribute.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System; -using Xunit; -using Xunit.Sdk; - -namespace XUnitExtensions { - [AttributeUsage(AttributeTargets.Method)] - [XunitTestCaseDiscoverer("XUnitExtensions.CollectionPerTestCaseTheoryDiscoverer", "XUnitExtensions")] - public class ParallelTheoryAttribute : TheoryAttribute { - } -} \ No newline at end of file From 70ba1dd6d0e00e39aa51e41b0bafa4972d751246 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 11 Sep 2021 17:27:52 -0700 Subject: [PATCH 121/192] Add skipped rows for unconvertible files --- .../LitTestRunner/LitTestDataDiscoverer.cs | 16 +++++--- Source/XUnitExtensions/FileTestCase.cs | 10 +++-- Source/XUnitExtensions/FileTheoryDataRow.cs | 41 +++++++++++++++++++ .../XUnitExtensions/FileTheoryDiscoverer.cs | 6 +-- 4 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 Source/XUnitExtensions/FileTheoryDataRow.cs diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs index 829b55e823b..6da8a0f0196 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -16,13 +16,19 @@ public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, } protected override IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName) { - bool invokeDirectly = attributeInfo.GetNamedArgument(nameof(LitTestDataAttribute.InvokeCliDirectly)); + var invokeDirectly = attributeInfo.GetNamedArgument(nameof(LitTestDataAttribute.InvokeCliDirectly)); + var basePath = GetBasePath(attributeInfo, testMethod); try { - var (testCases, _) = convertor.ConvertLitCommands(GetBasePath(attributeInfo, testMethod), fileName, invokeDirectly, File.ReadLines(fileName)); + var (testCases, _) = convertor.ConvertLitCommands(basePath, fileName, invokeDirectly, File.ReadLines(fileName)); return testCases.Select(testCase => new[] { testCase }); - } catch (Exception) { - // Ignore for now - return Enumerable.Empty(); + } catch (Exception e) { + var dummyTestCase = new DafnyTestCase(basePath, fileName, new(), new(), null, false); + var skippedCase = new FileTheoryDataRow(dummyTestCase) { + SourceInformation = new SourceInformation() { FileName = fileName, LineNumber = 0}, + TestDisplayName = basePath, + Skip = $"Exception: {e}" + }; + return new[] { new[] { skippedCase } }; } } } diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index 2497e21df06..3adb1138512 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -12,6 +12,9 @@ public class FileTestCase : LongLivedMarshalByRefObject, IXunitTestCase { protected XunitTestCase innerTestCase; + public string DisplayName { get; protected set; } + public string SkipReason { get; protected set; } + public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, IFileTheoryRowData data) { var collection = new TestCollection(testMethod.TestClass.TestCollection.TestAssembly, (ITypeInfo)null, "Test collection for " + data.TestDisplayName); @@ -27,6 +30,7 @@ public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, } innerTestCase.SourceInformation = data.SourceInformation; + DisplayName = data.SourceInformation.FileName; SkipReason = data.Skip; } @@ -35,21 +39,21 @@ public FileTestCase() { } public void Deserialize(IXunitSerializationInfo info) { innerTestCase = info.GetValue(nameof(innerTestCase)); + DisplayName = info.GetValue(nameof(DisplayName)); SkipReason = info.GetValue(nameof(SkipReason)); } public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(innerTestCase), innerTestCase); + info.AddValue(nameof(DisplayName), DisplayName); info.AddValue(nameof(SkipReason), SkipReason); } - public string SkipReason { get; protected set; } - + public ISourceInformation SourceInformation { get => innerTestCase.SourceInformation; set => innerTestCase.SourceInformation = value; } - public string DisplayName => innerTestCase.DisplayName; public ITestMethod TestMethod => innerTestCase.TestMethod; public object[] TestMethodArguments => innerTestCase.TestMethodArguments; public Dictionary> Traits => innerTestCase.Traits; diff --git a/Source/XUnitExtensions/FileTheoryDataRow.cs b/Source/XUnitExtensions/FileTheoryDataRow.cs new file mode 100644 index 00000000000..8accb98e815 --- /dev/null +++ b/Source/XUnitExtensions/FileTheoryDataRow.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using Xunit.Abstractions; + +namespace XUnitExtensions { + public class FileTheoryDataRow : IXunitSerializable, IFileTheoryRowData { + + object?[] data; + + public FileTheoryDataRow() { + + } + + public FileTheoryDataRow(params object?[] data) { + this.data = data; + } + + public ISourceInformation? SourceInformation { get; set; } + public string? Skip { get; set; } + public string? TestDisplayName { get; set; } + public Dictionary>? Traits { get; set; } + public object?[] GetData() => data; + + public void Serialize(IXunitSerializationInfo info) { + info.AddValue(nameof(SourceInformation), SourceInformation); + info.AddValue(nameof(Skip), Skip); + info.AddValue(nameof(TestDisplayName), TestDisplayName); + info.AddValue(nameof(Traits), Traits); + info.AddValue(nameof(data), data); + } + + public void Deserialize(IXunitSerializationInfo info) { + SourceInformation = info.GetValue(nameof(SourceInformation)); + Skip = info.GetValue(nameof(Skip)); + TestDisplayName = info.GetValue(nameof(TestDisplayName)); + Traits = info.GetValue>>(nameof(Traits)); + data = info.GetValue(nameof(data)); + } + + + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/FileTheoryDiscoverer.cs b/Source/XUnitExtensions/FileTheoryDiscoverer.cs index 20e02809e7a..9cf933d2fb4 100644 --- a/Source/XUnitExtensions/FileTheoryDiscoverer.cs +++ b/Source/XUnitExtensions/FileTheoryDiscoverer.cs @@ -16,13 +16,13 @@ public override IEnumerable Discover(ITestFrameworkDiscoveryOpti return base.Discover(discoveryOptions, testMethod, factAttribute); } - protected override IXunitTestCase CreateTestCaseForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, + protected override IEnumerable CreateTestCasesForDataRow(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute, object[] dataRow) { if (dataRow.Length == 1 && dataRow[0] is IFileTheoryRowData theoryRowData) { - return new FileTestCase(DiagnosticMessageSink, testMethod, theoryRowData); + return new[] { new FileTestCase(DiagnosticMessageSink, testMethod, theoryRowData) }; } - return base.CreateTestCaseForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); + return base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); } } } \ No newline at end of file From b63cc9966cfbcb303c24418cf45cc91e5f12274e Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 14 Sep 2021 13:00:48 -0700 Subject: [PATCH 122/192] Fix display names --- Source/DafnyDriver.Test/DafnyTestCase.cs | 14 +++++++++----- .../LitTestRunner/LitTestDataDiscoverer.cs | 3 ++- Source/XUnitExtensions/FileTestCase.cs | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCase.cs index 25fdcd31b33..1bf351f69a3 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCase.cs @@ -62,9 +62,10 @@ public DafnyTestCase(string basePath, string fullSourcePath, Dictionary null; - public string? TestDisplayName => SourceInformation.FileName; + public string? TestDisplayName => SourceInformation.FileName[(BasePath.Length + 1)..]; public Dictionary>? Traits => null; public object[] GetData() { return new object[] { this }; diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs index 6da8a0f0196..31a690a9810 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs @@ -22,10 +22,11 @@ protected override IEnumerable FileData(IAttributeInfo attributeInfo, var (testCases, _) = convertor.ConvertLitCommands(basePath, fileName, invokeDirectly, File.ReadLines(fileName)); return testCases.Select(testCase => new[] { testCase }); } catch (Exception e) { + var shortPath = fileName[(basePath.Length + 1)..]; var dummyTestCase = new DafnyTestCase(basePath, fileName, new(), new(), null, false); var skippedCase = new FileTheoryDataRow(dummyTestCase) { SourceInformation = new SourceInformation() { FileName = fileName, LineNumber = 0}, - TestDisplayName = basePath, + TestDisplayName = shortPath, Skip = $"Exception: {e}" }; return new[] { new[] { skippedCase } }; diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index 3adb1138512..d007a42ad67 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -30,7 +30,7 @@ public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, } innerTestCase.SourceInformation = data.SourceInformation; - DisplayName = data.SourceInformation.FileName; + DisplayName = data.TestDisplayName; SkipReason = data.Skip; } From 0cc5e485f297406689052c0e3794ac628d1ccda6 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 28 Sep 2021 16:07:22 -0700 Subject: [PATCH 123/192] Checkpointing partial work abstracting to general Lit tests --- .../DafnyDriver.Test/DafnyDriver.Test.csproj | 1 + .../{DafnyTestCase.cs => DafnyTestCommand.cs} | 6 +- Source/DafnyServer/Server.cs | 2 +- Source/LitTestConverter/LitTestConvertor.cs | 16 +- .../LitTestDataDiscovererTest.cs | 2 +- .../LitTestRunner/LitTests.cs | 20 +- Source/XUnitExtensions/CLITestCase.cs | 177 ------------------ Source/XUnitExtensions/FileTheoryDataRow.cs | 2 - .../LitSubstitutionAttribute.cs | 16 ++ Source/XUnitExtensions/LitTestCase.cs | 14 ++ Source/XUnitExtensions/LitTestCommand.cs | 146 +++++++++++++++ .../XUnitExtensions/LitTestConfiguration.cs | 13 ++ .../LitTestDataDiscoverer.cs | 5 +- 13 files changed, 221 insertions(+), 199 deletions(-) rename Source/DafnyDriver.Test/{DafnyTestCase.cs => DafnyTestCommand.cs} (94%) delete mode 100644 Source/XUnitExtensions/CLITestCase.cs create mode 100644 Source/XUnitExtensions/LitSubstitutionAttribute.cs create mode 100644 Source/XUnitExtensions/LitTestCase.cs create mode 100644 Source/XUnitExtensions/LitTestCommand.cs create mode 100644 Source/XUnitExtensions/LitTestConfiguration.cs rename Source/{LitTestConvertor.Test/LitTestRunner => XUnitExtensions}/LitTestDataDiscoverer.cs (85%) diff --git a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj index 449288dadf3..410adebeff3 100644 --- a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj +++ b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj @@ -14,6 +14,7 @@ + diff --git a/Source/DafnyDriver.Test/DafnyTestCase.cs b/Source/DafnyDriver.Test/DafnyTestCommand.cs similarity index 94% rename from Source/DafnyDriver.Test/DafnyTestCase.cs rename to Source/DafnyDriver.Test/DafnyTestCommand.cs index 1bf351f69a3..739764eeec5 100644 --- a/Source/DafnyDriver.Test/DafnyTestCase.cs +++ b/Source/DafnyDriver.Test/DafnyTestCommand.cs @@ -12,7 +12,7 @@ namespace DafnyDriver.Test { * Specialization of CLITestCase that mainly exists to support a much more * concise definition of ToString(). */ - public class DafnyTestCase : CLITestCase, IFileTheoryRowData { + public class DafnyTestCommand : LitTestCommand, IFileTheoryRowData { private static readonly Assembly dafnyDriverAssembly = Assembly.GetAssembly(typeof(Microsoft.Dafny.DafnyDriver)); @@ -58,7 +58,7 @@ private static string ConfigPairToArgument(KeyValuePair pair) { private Dictionary DafnyOptions = new(); private List OtherFiles = new(); - public DafnyTestCase(string basePath, string fullSourcePath, Dictionary dafnyOptions, List otherFiles, + public DafnyTestCommand(string basePath, string fullSourcePath, Dictionary dafnyOptions, List otherFiles, Expectation expected, bool invokeDirectly) : base(dafnyDriverAssembly, OptionsToFullArguments(fullSourcePath, dafnyOptions, otherFiles), new string[] { "PATH", "HOME" }, expected, invokeDirectly) { BasePath = basePath; @@ -70,7 +70,7 @@ public DafnyTestCase(string basePath, string fullSourcePath, Dictionary testSpec; + IEnumerable testSpec; IEnumerable testContent; if (filePath.Contains("/Inputs/")) { - testSpec = Enumerable.Empty(); + testSpec = Enumerable.Empty(); testContent = File.ReadAllLines(filePath); } else { (testSpec, testContent) = ConvertLitCommands(basePath, filePath, invokeDirectly, File.ReadAllLines(filePath)); @@ -54,7 +54,7 @@ public void ConvertLitTest(string basePath, string filePath, bool invokeDirectly } } - public (IEnumerable spec, IEnumerable content) ConvertLitCommands(string basePath, string filePath, bool invokeDirectly, IEnumerable lines) { + public (IEnumerable spec, IEnumerable content) ConvertLitCommands(string basePath, string filePath, bool invokeDirectly, IEnumerable lines) { var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); if (!litCommands.Any()) { throw new ArgumentException("No lit commands found"); @@ -68,7 +68,7 @@ public void ConvertLitTest(string basePath, string filePath, bool invokeDirectly if (litCommands.Count == 1 && litCommands.Single().StartsWith("echo")) { // This is an idiom for Dafny files used elsewhere - return (Enumerable.Empty(), testContent); + return (Enumerable.Empty(), testContent); } if (!litCommands[^1].Equals("%diff \"%s.expect\" \"%t\"")) { @@ -76,7 +76,7 @@ public void ConvertLitTest(string basePath, string filePath, bool invokeDirectly } litCommands.RemoveAt(litCommands.Count - 1); - List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(basePath, filePath, invokeDirectly, c)).ToList(); + List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(basePath, filePath, invokeDirectly, c)).ToList(); if (testConfigs.Count == 1) { return (testConfigs, testContent); @@ -97,7 +97,7 @@ private static string ExtractLitCommand(string line) { return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); } - private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, bool invokeDirectly, string dafnyCommand) { + private LitTestCommand ParseDafnyCommandArguments(string basePath, string filePath, bool invokeDirectly, string dafnyCommand) { List otherFiles = new(); Dictionary dafnyArguments = new(); @@ -137,9 +137,7 @@ private CLITestCase ParseDafnyCommandArguments(string basePath, string filePath, } } - var expected = new CLITestCase.Expectation(0, filePath + ".expect", null); - - return new DafnyTestCase(basePath, filePath, dafnyArguments, otherFiles, expected, invokeDirectly); + return new DafnyTestCommand(basePath, filePath, dafnyArguments, otherFiles, expected, invokeDirectly); } private static (string, string) ParseDafnyArgument(string argument) { diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs index ce289f81dc9..525ac3860f2 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs @@ -9,7 +9,7 @@ namespace LitTestConvertor.Test.LitTestRunner { public class LitTestDataDiscovererTest { - MethodInfo GetMethodInfo(Action a) { + MethodInfo GetMethodInfo(Action a) { return a.Method; } diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index 8411ba9a589..9a912a99918 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Dafny; using Xunit; using XUnitExtensions; @@ -7,8 +8,23 @@ namespace LitTestConvertor.Test.LitTestRunner { public class LitTests { [FileTheory] [LitTestData(Extension = ".dfy", InvokeCliDirectly = false)] - public void LitTest(CLITestCase testCase) { - testCase.Run(); + [LitSubstitution("%baredafny", MainClass = typeof(Microsoft.Dafny.DafnyDriver))] + [LitSubstitution("%dafny", MainClass = typeof(Microsoft.Dafny.DafnyDriver), Arguments = new [] { + "/countVerificationErrors:0", + + // We do not want absolute or relative paths in error messages, just the basename of the file + "/useBaseNameForFileName:yes", + + // We do not want output such as "Compiled program written to Foo.cs" + // from the compilers, since that changes with the target language + "/compileVerbose:0", + + // Hide Boogie execution traces since they are meaningless for Dafny programs + "/errorTrace:0"})] + [LitSubstitution("%server", MainClass = typeof(Server))] + [LitSubstitution("%diff", CLIPath = "diff")] + public void LitTest(string path, LitTestConfiguration configuration) { + LitTestCase.Run(path, configuration); } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/CLITestCase.cs b/Source/XUnitExtensions/CLITestCase.cs deleted file mode 100644 index 6069e52538e..00000000000 --- a/Source/XUnitExtensions/CLITestCase.cs +++ /dev/null @@ -1,177 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Text; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace XUnitExtensions { - - public class CLITestCase : IXunitSerializable { - protected Assembly CliAssembly; - protected string CliAssemblyName; - protected bool InvokeDirectly; - - protected string[] Arguments; - protected IEnumerable PassthroughEnvironmentVariables; - protected Expectation Expected; - - public class Expectation : IXunitSerializable { - // May be null if the test doesn't need to check the output - public string OutputFile; - public int ExitCode = 0; - - public string SpecialCaseReason; - - public static Expectation NO_OUTPUT = new Expectation(0, null, null); - - public Expectation(int exitCode, string outputFile, string specialCaseReason) { - ExitCode = exitCode; - OutputFile = outputFile; - SpecialCaseReason = specialCaseReason; - } - - public Expectation() { - } - - public void Deserialize(IXunitSerializationInfo info) { - OutputFile = info.GetValue(nameof(OutputFile)); - ExitCode = info.GetValue(nameof(ExitCode)); - SpecialCaseReason = info.GetValue(nameof(SpecialCaseReason)); - } - - public void Serialize(IXunitSerializationInfo info) { - info.AddValue(nameof(OutputFile), OutputFile); - info.AddValue(nameof(ExitCode), ExitCode); - info.AddValue(nameof(SpecialCaseReason), SpecialCaseReason); - } - - public override string ToString() { - string result; - if (ExitCode != 0) { - result = String.Format("", ExitCode); - } else if (OutputFile != null) { - result = OutputFile; - } else { - result = ""; - } - - if (SpecialCaseReason != null) { - result += " (special case)"; - } - return result; - } - } - - public class CLIResult { - - public int ExitCode; - public string StandardOutput; - public string StandardError; - - public CLIResult(int exitCode, string standardOutput, string standardError) { - ExitCode = exitCode; - StandardOutput = standardOutput; - StandardError = standardError; - } - } - - public CLITestCase(Assembly cliAssembly, IEnumerable arguments, - IEnumerable passthroughEnvironmentVariables, - Expectation expected, bool invokeDirectly) { - CliAssembly = cliAssembly; - Arguments = arguments.ToArray(); - PassthroughEnvironmentVariables = passthroughEnvironmentVariables; - Expected = expected; - InvokeDirectly = invokeDirectly; - } - - public CLITestCase() { - - } - - public void Serialize(IXunitSerializationInfo info) { - info.AddValue(nameof(CliAssemblyName), CliAssemblyName); - info.AddValue(nameof(Arguments), Arguments); - info.AddValue(nameof(Expected), Expected); - } - - public void Deserialize(IXunitSerializationInfo info) { - CliAssemblyName = info.GetValue(nameof(CliAssemblyName)); - CliAssembly = AppDomain.CurrentDomain.GetAssemblies().First(a => a.FullName == CliAssemblyName); - - Arguments = info.GetValue(nameof(Arguments)); - Expected = info.GetValue(nameof(Expected)); - } - - public CLIResult InvokeCLI() { - if (InvokeDirectly || Environment.GetEnvironmentVariable("XUNIT_INVOKE_CLI_DIRECTLY") == "true") { - return InvokeCLIDirectly(); - } - return InvokeCLIViaProcess(); - } - - public CLIResult InvokeCLIDirectly() { - StringBuilder redirectedOut = new(); - StringBuilder redirectedErr = new(); - - Console.SetOut(new StringWriter(redirectedOut)); - Console.SetError(new StringWriter(redirectedErr)); - - int result = (int)CliAssembly.EntryPoint.Invoke(null, new object[] { Arguments }); - - return new CLIResult(result, redirectedOut.ToString(), redirectedErr.ToString()); - } - - public CLIResult InvokeCLIViaProcess() { - using var process = new Process(); - - process.StartInfo.FileName = "dotnet"; - process.StartInfo.Arguments = CliAssembly.Location; - foreach (var argument in Arguments) { - process.StartInfo.Arguments += " " + argument; - } - - process.StartInfo.UseShellExecute = false; - process.StartInfo.RedirectStandardOutput = true; - process.StartInfo.RedirectStandardError = true; - process.StartInfo.CreateNoWindow = true; - - process.StartInfo.EnvironmentVariables.Clear(); - foreach (var passthrough in PassthroughEnvironmentVariables) { - process.StartInfo.EnvironmentVariables.Add(passthrough, Environment.GetEnvironmentVariable(passthrough)); - } - - process.Start(); - string output = process.StandardOutput.ReadToEnd(); - string error = process.StandardError.ReadToEnd(); - process.WaitForExit(); - - return new CLIResult(process.ExitCode, output, error); - } - - public void Run() { - CLIResult result = InvokeCLI(); - - if (result.ExitCode != Expected.ExitCode) { - throw new AssertActualExpectedException(Expected.ExitCode, result.ExitCode, - String.Format("Expected exit code to be {0} but was {1}. Standard error output:\n{2}", - Expected.ExitCode, result.ExitCode, result.StandardError)); - } - if (Expected.OutputFile != null) { - var expectedOutput = File.ReadAllText(Expected.OutputFile); - AssertWithDiff.Equal(expectedOutput, result.StandardOutput); - } - - Skip.If(Expected.SpecialCaseReason != null, Expected.SpecialCaseReason); - } - - public override string ToString() { - return String.Join(" ", Arguments) + " => " + Expected; - } - } -} diff --git a/Source/XUnitExtensions/FileTheoryDataRow.cs b/Source/XUnitExtensions/FileTheoryDataRow.cs index 8accb98e815..96b32168274 100644 --- a/Source/XUnitExtensions/FileTheoryDataRow.cs +++ b/Source/XUnitExtensions/FileTheoryDataRow.cs @@ -35,7 +35,5 @@ public void Deserialize(IXunitSerializationInfo info) { Traits = info.GetValue>>(nameof(Traits)); data = info.GetValue(nameof(data)); } - - } } \ No newline at end of file diff --git a/Source/XUnitExtensions/LitSubstitutionAttribute.cs b/Source/XUnitExtensions/LitSubstitutionAttribute.cs new file mode 100644 index 00000000000..73b5826e66d --- /dev/null +++ b/Source/XUnitExtensions/LitSubstitutionAttribute.cs @@ -0,0 +1,16 @@ +using System; +using System.Net.Mail; + +namespace XUnitExtensions { + [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] + public class LitSubstitutionAttribute : Attribute { + public LitSubstitutionAttribute(string key) { + + } + + public virtual Type MainClass { get; set; } + public virtual string CLIPath { get; set; } + + public virtual string[] Arguments { get; set; } + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs new file mode 100644 index 00000000000..61535100fcd --- /dev/null +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -0,0 +1,14 @@ +using System.IO; +using System.Linq; + +namespace XUnitExtensions { + public class LitTestCase { + public static void Run(string filePath, LitTestConfiguration config) { + var content = File.ReadAllLines(filePath); + var commands = content.Select(LitTestCommand.Parse).Where(c => c != null); + foreach (var command in commands) { + command.Run(config); + } + } + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/LitTestCommand.cs b/Source/XUnitExtensions/LitTestCommand.cs new file mode 100644 index 00000000000..fb5b26e2922 --- /dev/null +++ b/Source/XUnitExtensions/LitTestCommand.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using Xunit; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace XUnitExtensions { + + public class LitTestCommand { + + private const string COMMENT_PREFIX = "//"; + private const string LIT_COMMAND_PREFIX = "RUN:"; + + // This may be a symbol for substitution such as "%dafny" + private string CliPath; + private string[] Arguments; + private string OutputFile; + private bool AppendOutput; + + public LitTestCommand(string cliPath, IEnumerable arguments, string outputFile, bool appendOutput) { + CliPath = cliPath; + Arguments = arguments.ToArray(); + OutputFile = outputFile; + AppendOutput = appendOutput; + } + + public void Run(LitTestConfiguration config) { + var cliTarget = config.Substitions[CliPath]; + int result; + if (cliTarget is Assembly assembly && + (config.InvokeMainMethodsDirectly || Environment.GetEnvironmentVariable("XUNIT_INVOKE_CLI_DIRECTLY") == "true")) { + result = InvokeCLIDirectly(assembly, config); + } else { + result = InvokeCLIViaProcess(config); + } + Assert.Equal(0, result); + } + + public int InvokeCLIDirectly(Assembly assembly, LitTestConfiguration config) { + StringBuilder redirectedOut = new(); + StringBuilder redirectedErr = new(); + + Console.SetOut(new StringWriter(redirectedOut)); + Console.SetError(new StringWriter(redirectedErr)); + + return (int)assembly.EntryPoint!.Invoke(null, new object[] { ArgumentsWithSubstitutions(config) }); + } + + public int InvokeCLIViaProcess(LitTestConfiguration config) { + using var process = new Process(); + + var cliTarget = config.Substitions[CliPath]; + if (cliTarget is Assembly assembly) { + process.StartInfo.FileName = "dotnet"; + process.StartInfo.Arguments = assembly.Location; + } else { + process.StartInfo.FileName = cliTarget != null ? (string)cliTarget : CliPath; + } + + foreach (var argument in ArgumentsWithSubstitutions(config)) { + process.StartInfo.Arguments += " " + argument; + } + + process.StartInfo.UseShellExecute = false; + // process.StartInfo.RedirectStandardOutput = true; + // process.StartInfo.RedirectStandardError = true; + process.StartInfo.CreateNoWindow = true; + + process.StartInfo.EnvironmentVariables.Clear(); + foreach (var passthrough in config.PassthroughEnvironmentVariables) { + process.StartInfo.EnvironmentVariables.Add(passthrough, Environment.GetEnvironmentVariable(passthrough)); + } + + process.Start(); + process.WaitForExit(); + + return process.ExitCode; + } + + private object ApplySubstitution(string key, LitTestConfiguration config) { + if (config.Substitions.TryGetValue(key, out var value)) { + return value; + } + return key; + } + + private IEnumerable ArgumentsWithSubstitutions(LitTestConfiguration config) { + return Arguments.Select(argument => (string)ApplySubstitution(argument, config)); + } + + public override string ToString() { + var builder = new StringBuilder(); + builder.Append(CliPath); + builder.Append(' '); + builder.AppendJoin(" ", Arguments); + if (OutputFile != null) { + if (AppendOutput) { + builder.Append(" >> "); + } else { + builder.Append(" > "); + } + builder.Append(OutputFile); + } + + return builder.ToString(); + } + + public static LitTestCommand Parse(string line) { + if (!line.StartsWith(COMMENT_PREFIX)) { + return null; + } + line = line.Substring(COMMENT_PREFIX.Length).Trim(); + + if (!line.StartsWith(LIT_COMMAND_PREFIX)) { + return null; + } + line = line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); + + // TODO: Probably need to handle escaping too + var pieces = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).ToList(); + var cliPath = pieces[0]; + var arguments = pieces.GetRange(1, pieces.Count - 1); + + string outputFile = null; + bool appendOutput = false; + var redirectIndex = arguments.IndexOf(">"); + if (redirectIndex >= 0) { + outputFile = arguments[redirectIndex + 1]; + arguments.RemoveRange(redirectIndex, 2); + } + var redirectAppendIndex = arguments.IndexOf(">>"); + if (redirectAppendIndex >= 0) { + outputFile = arguments[redirectAppendIndex + 1]; + appendOutput = true; + arguments.RemoveRange(redirectAppendIndex, 2); + } + + return new LitTestCommand(cliPath, arguments, outputFile, appendOutput); + } + } +} diff --git a/Source/XUnitExtensions/LitTestConfiguration.cs b/Source/XUnitExtensions/LitTestConfiguration.cs new file mode 100644 index 00000000000..23de4fc2b83 --- /dev/null +++ b/Source/XUnitExtensions/LitTestConfiguration.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace XUnitExtensions { + public class LitTestConfiguration { + // Values are either strings or Assemblies (for our own CLI tools) + public Dictionary Substitions { get; set; } + + public bool InvokeMainMethodsDirectly { get; set; } + + public IEnumerable PassthroughEnvironmentVariables { get; set; } + + } +} \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs b/Source/XUnitExtensions/LitTestDataDiscoverer.cs similarity index 85% rename from Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs rename to Source/XUnitExtensions/LitTestDataDiscoverer.cs index 31a690a9810..80d869b7e2e 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscoverer.cs +++ b/Source/XUnitExtensions/LitTestDataDiscoverer.cs @@ -9,8 +9,6 @@ namespace DafnyDriver.Test { public class LitTestDataDiscoverer : FileDataDiscoverer { - private readonly LitTestConverter.LitTestConvertor convertor = new(); - public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { return true; } @@ -23,8 +21,7 @@ protected override IEnumerable FileData(IAttributeInfo attributeInfo, return testCases.Select(testCase => new[] { testCase }); } catch (Exception e) { var shortPath = fileName[(basePath.Length + 1)..]; - var dummyTestCase = new DafnyTestCase(basePath, fileName, new(), new(), null, false); - var skippedCase = new FileTheoryDataRow(dummyTestCase) { + var skippedCase = new FileTheoryDataRow(shortPath) { SourceInformation = new SourceInformation() { FileName = fileName, LineNumber = 0}, TestDisplayName = shortPath, Skip = $"Exception: {e}" From 1c521bd38d73e75f164b3d81780dd68ccea680d5 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 29 Sep 2021 13:00:54 -0700 Subject: [PATCH 124/192] Better model for IListTestCommands --- Source/DafnyDriver.Test/DafnyTestCommand.cs | 109 ----------- Source/LitTestConverter/LitTestConvertor.cs | 184 ------------------ .../LitTestDataDiscovererTest.cs | 28 --- .../LitTestRunner/LitTests.cs | 47 +++-- Source/XUnitExtensions/ILitCommand.cs | 36 ++++ .../LitSubstitutionAttribute.cs | 16 -- Source/XUnitExtensions/LitTestCase.cs | 12 +- Source/XUnitExtensions/LitTestCommand.cs | 146 -------------- .../XUnitExtensions/LitTestConfiguration.cs | 14 +- .../XUnitExtensions/LitTestDataDiscoverer.cs | 21 +- .../XUnitExtensions/MainMethodLitCommand.cs | 74 +++++++ Source/XUnitExtensions/ShellLitCommand.cs | 52 +++++ 12 files changed, 221 insertions(+), 518 deletions(-) delete mode 100644 Source/DafnyDriver.Test/DafnyTestCommand.cs delete mode 100644 Source/LitTestConverter/LitTestConvertor.cs delete mode 100644 Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs create mode 100644 Source/XUnitExtensions/ILitCommand.cs delete mode 100644 Source/XUnitExtensions/LitSubstitutionAttribute.cs delete mode 100644 Source/XUnitExtensions/LitTestCommand.cs create mode 100644 Source/XUnitExtensions/MainMethodLitCommand.cs create mode 100644 Source/XUnitExtensions/ShellLitCommand.cs diff --git a/Source/DafnyDriver.Test/DafnyTestCommand.cs b/Source/DafnyDriver.Test/DafnyTestCommand.cs deleted file mode 100644 index 739764eeec5..00000000000 --- a/Source/DafnyDriver.Test/DafnyTestCommand.cs +++ /dev/null @@ -1,109 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using Xunit.Abstractions; -using Xunit.Sdk; -using XUnitExtensions; - -namespace DafnyDriver.Test { - - /** - * Specialization of CLITestCase that mainly exists to support a much more - * concise definition of ToString(). - */ - public class DafnyTestCommand : LitTestCommand, IFileTheoryRowData { - - private static readonly Assembly dafnyDriverAssembly = Assembly.GetAssembly(typeof(Microsoft.Dafny.DafnyDriver)); - - private static readonly Dictionary defaultDafnyOptions = new() { - ["countVerificationErrors"] = "0", - - // We do not want absolute or relative paths in error messages, just the basename of the file - ["useBaseNameForFileName"] = "yes", - - // We do not want output such as "Compiled program written to Foo.cs" - // from the compilers, since that changes with the target language - ["compileVerbose"] = "0", - - // Hide Boogie execution traces since they are meaningless for Dafny programs - ["errorTrace"] = "0" - }; - - private static IEnumerable OptionsToFullArguments(string sourcePath, - Dictionary dafnyOptions, List otherFiles) { - Dictionary optionsWithDefaults = new(defaultDafnyOptions); - foreach (var (key, value) in dafnyOptions) { - optionsWithDefaults[key] = value; - } - - return OptionsToArguments(sourcePath, optionsWithDefaults, otherFiles); - } - - private static IEnumerable OptionsToArguments(string sourcePath, Dictionary dafnyOptions, List otherFiles) { - return new[] { sourcePath } - .Concat(otherFiles) - .Concat(dafnyOptions.Select(ConfigPairToArgument)); - } - - private static string ConfigPairToArgument(KeyValuePair pair) { - if (pair.Value.Equals("yes")) { - return $"/{pair.Key}"; - } - return $"/{pair.Key}:{pair.Value}"; - } - - private string BasePath; - - private Dictionary DafnyOptions = new(); - private List OtherFiles = new(); - - public DafnyTestCommand(string basePath, string fullSourcePath, Dictionary dafnyOptions, List otherFiles, - Expectation expected, bool invokeDirectly) - : base(dafnyDriverAssembly, OptionsToFullArguments(fullSourcePath, dafnyOptions, otherFiles), new string[] { "PATH", "HOME" }, expected, invokeDirectly) { - BasePath = basePath; - SourceInformation = new SourceInformation { - FileName = fullSourcePath, - LineNumber = 0 - }; - DafnyOptions = dafnyOptions; - OtherFiles = otherFiles; - } - - public DafnyTestCommand() { - - } - - public new void Serialize(IXunitSerializationInfo info) { - base.Serialize(info); - info.AddValue(nameof(BasePath), BasePath); - info.AddValue(nameof(SourceInformation), SourceInformation); - info.AddValue(nameof(DafnyOptions), DafnyOptions); - info.AddValue(nameof(OtherFiles), OtherFiles); - } - - public new void Deserialize(IXunitSerializationInfo info) { - base.Deserialize(info); - BasePath = info.GetValue(nameof(BasePath)); - SourceInformation = info.GetValue(nameof(SourceInformation)); - DafnyOptions = info.GetValue>(nameof(DafnyOptions)); - OtherFiles = info.GetValue>(nameof(OtherFiles)); - } - - public override string ToString() { - var relativePath = SourceInformation.FileName[(BasePath.Length + 1)..]; - return String.Join(" ", OptionsToArguments(relativePath, DafnyOptions, OtherFiles)); - } - - public ISourceInformation SourceInformation { - get; - protected set; - } - public string? Skip => null; - public string? TestDisplayName => SourceInformation.FileName[(BasePath.Length + 1)..]; - public Dictionary>? Traits => null; - public object[] GetData() { - return new object[] { this }; - } - } -} \ No newline at end of file diff --git a/Source/LitTestConverter/LitTestConvertor.cs b/Source/LitTestConverter/LitTestConvertor.cs deleted file mode 100644 index 71bcdfc39cf..00000000000 --- a/Source/LitTestConverter/LitTestConvertor.cs +++ /dev/null @@ -1,184 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using DafnyDriver.Test; -using XUnitExtensions; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; - -namespace LitTestConverter { - - public class LitTestConvertor { - - private const string DAFNY_COMMENT_PREFIX = "//"; - private const string LIT_COMMAND_PREFIX = "RUN:"; - private const string LIT_DAFNY = "%dafny"; - private const string LIT_SERVER = "%server"; - - // Fake options for which files are passed to the CLI - public const string TEST_CONFIG_OTHER_FILES = "otherFiles"; - public const string TEST_CONFIG_INCLUDE_THIS_FILE = "includeThisFile"; - - private int count = 0; - private int verifyOnlyCount = 0; - private int defaultCount = 0; - private int alreadyConverted = 0; - private int invalidCount = 0; - - public void ConvertLitTest(string basePath, string filePath, bool invokeDirectly) { - IEnumerable testSpec; - IEnumerable testContent; - if (filePath.Contains("/Inputs/")) { - testSpec = Enumerable.Empty(); - testContent = File.ReadAllLines(filePath); - } else { - (testSpec, testContent) = ConvertLitCommands(basePath, filePath, invokeDirectly, File.ReadAllLines(filePath)); - } - - ISerializer serializer = new SerializerBuilder() - .WithNamingConvention(CamelCaseNamingConvention.Instance) - .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitDefaults) - .Build(); - - using (StreamWriter file = new StreamWriter(filePath)) { - if (testSpec != null) { - file.WriteLine("/*"); - file.WriteLine("---"); - serializer.Serialize(file, testSpec); - file.WriteLine("*/"); - } - foreach (var line in testContent) { - file.WriteLine(line); - } - } - } - - public (IEnumerable spec, IEnumerable content) ConvertLitCommands(string basePath, string filePath, bool invokeDirectly, IEnumerable lines) { - var litCommands = lines.Select(ExtractLitCommand).TakeWhile(c => c != null).ToList(); - if (!litCommands.Any()) { - throw new ArgumentException("No lit commands found"); - } - - // Make sure the commands are consecutive - var testContent = lines.Skip(litCommands.Count); - if (testContent.Any(line => ExtractLitCommand(line) != null)) { - throw new ArgumentException("Lit commands are not consecutive"); - } - - if (litCommands.Count == 1 && litCommands.Single().StartsWith("echo")) { - // This is an idiom for Dafny files used elsewhere - return (Enumerable.Empty(), testContent); - } - - if (!litCommands[^1].Equals("%diff \"%s.expect\" \"%t\"")) { - throw new ArgumentException("Last lit command is not expected %diff: " + litCommands[^1]); - } - litCommands.RemoveAt(litCommands.Count - 1); - - List testConfigs = litCommands.Select(c => ParseDafnyCommandArguments(basePath, filePath, invokeDirectly, c)).ToList(); - - if (testConfigs.Count == 1) { - return (testConfigs, testContent); - } - - throw new ArgumentException("Multi-command lit tests require manual conversion"); - } - - private static string ExtractLitCommand(string line) { - if (!line.StartsWith(DAFNY_COMMENT_PREFIX)) { - return null; - } - line = line.Substring(DAFNY_COMMENT_PREFIX.Length).Trim(); - - if (!line.StartsWith(LIT_COMMAND_PREFIX)) { - return null; - } - return line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); - } - - private LitTestCommand ParseDafnyCommandArguments(string basePath, string filePath, bool invokeDirectly, string dafnyCommand) { - List otherFiles = new(); - Dictionary dafnyArguments = new(); - - if (!dafnyCommand.StartsWith(LIT_DAFNY)) { - throw new ArgumentException("Lit command is not expected %dafny: " + dafnyCommand); - } - - string argumentsString = dafnyCommand.Substring(LIT_DAFNY.Length); - var arguments = argumentsString.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).ToList(); - - // Ensure the last two parts are > "%t" or >> "%t" - if (arguments.Count < 3) { - throw new ArgumentException("Not enough arguments to %dafny command: " + dafnyCommand); - } - if (!arguments[^1].Equals("\"%t\"") - || !(arguments[^2].Equals(">") || arguments[^2].Equals(">>"))) { - throw new ArgumentException("Non-standard output in %dafny command: " + dafnyCommand); - } - arguments.RemoveRange(arguments.Count - 2, 2); - - if (!arguments.Remove("\"%s\"")) { - throw new ArgumentException("Test cases that do not include the current file are not yet supported"); - } - - // Check the arguments for anything non-standard - foreach (var argument in arguments) { - var (key, value) = ParseDafnyArgument(argument); - if (value.Contains("%")) { - throw new ArgumentException("Use of lit substitution (% variable) requires manual conversion: " + argument); - } - if (key.Equals(TEST_CONFIG_OTHER_FILES)) { - // Lit always uses the parent directory of a test file as the current directory, - // so other file paths have to be interpreted to be relative to the output directory instead. - otherFiles.Add(Path.Join(Path.GetDirectoryName(filePath), value)); - } else { - dafnyArguments.Add(key, value); - } - } - - return new DafnyTestCommand(basePath, filePath, dafnyArguments, otherFiles, expected, invokeDirectly); - } - - private static (string, string) ParseDafnyArgument(string argument) { - if (argument.StartsWith("-") || argument.StartsWith("/")) { - argument = argument[1..]; - int colonIndex = argument.IndexOf(":"); - if (colonIndex >= 0) { - return (argument[..colonIndex], argument[(colonIndex + 1)..]); - } else { - return (argument, "yes"); - } - } else { - return (TEST_CONFIG_OTHER_FILES, argument); - } - } - - public void Run(string root) { - // TODO-RS: Search for "*.transcript" too - if (!Directory.Exists(root)) { - ConvertLitTest(root, root, false); - return; - } - foreach (var file in Directory.GetFiles(root, "*.dfy", SearchOption.AllDirectories)) { - try { - count++; - ConvertLitTest(root, file, false); - } catch (ArgumentException e) { - invalidCount++; - Console.WriteLine(file + ": " + e.Message); - } - } - - Console.WriteLine(""); - Console.WriteLine("Already converted: " + alreadyConverted + "/" + count); - Console.WriteLine("Default: " + defaultCount + "/" + count); - Console.WriteLine("Verify only: " + verifyOnlyCount + "/" + count); - Console.WriteLine("Invalid: " + invalidCount + "/" + count); - } - - public static void Main(string[] args) { - new LitTestConvertor().Run(args[0]); - } - } -} \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs deleted file mode 100644 index 525ac3860f2..00000000000 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTestDataDiscovererTest.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using System.Linq; -using System.Reflection; -using DafnyDriver.Test; -using Xunit; -using Xunit.Sdk; -using XUnitExtensions; - -namespace LitTestConvertor.Test.LitTestRunner { - public class LitTestDataDiscovererTest { - - MethodInfo GetMethodInfo(Action a) { - return a.Method; - } - - [Fact] - public void Discoverer() { - // var discoverer = new LitTestDataDiscoverer(); - // var test = new LitTests(); - // var methodInfo = GetMethodInfo(test.LitTest); - // var method = Reflector.Wrap(methodInfo); - // var attribute = CustomAttributeData.GetCustomAttributes(methodInfo).First(a => a.AttributeType == typeof(LitTestDataAttribute)); - // var xunitAttribute = Reflector.Wrap(attribute); - // var data = discoverer.GetData(xunitAttribute, method).ToList(); - // Assert.True(data.Any()); - } - } -} \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index 9a912a99918..b18161c30d3 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -1,3 +1,6 @@ +using System; +using System.Collections.Generic; +using System.Reflection; using Microsoft.Dafny; using Xunit; using XUnitExtensions; @@ -6,25 +9,37 @@ namespace LitTestConvertor.Test.LitTestRunner { public class LitTests { - [FileTheory] - [LitTestData(Extension = ".dfy", InvokeCliDirectly = false)] - [LitSubstitution("%baredafny", MainClass = typeof(Microsoft.Dafny.DafnyDriver))] - [LitSubstitution("%dafny", MainClass = typeof(Microsoft.Dafny.DafnyDriver), Arguments = new [] { - "/countVerificationErrors:0", + + private static readonly LitTestConfiguration CONFIG = new () { + MainMethodSubstitutions = new Dictionary { + { "%baredafny", (typeof(Microsoft.Dafny.DafnyDriver).Assembly, Array.Empty()) }, + { "%dafny", (typeof(Microsoft.Dafny.DafnyDriver).Assembly, new [] { + "/countVerificationErrors:0", - // We do not want absolute or relative paths in error messages, just the basename of the file - "/useBaseNameForFileName:yes", + // We do not want absolute or relative paths in error messages, just the basename of the file + "/useBaseNameForFileName:yes", - // We do not want output such as "Compiled program written to Foo.cs" - // from the compilers, since that changes with the target language - "/compileVerbose:0", + // We do not want output such as "Compiled program written to Foo.cs" + // from the compilers, since that changes with the target language + "/compileVerbose:0", - // Hide Boogie execution traces since they are meaningless for Dafny programs - "/errorTrace:0"})] - [LitSubstitution("%server", MainClass = typeof(Server))] - [LitSubstitution("%diff", CLIPath = "diff")] - public void LitTest(string path, LitTestConfiguration configuration) { - LitTestCase.Run(path, configuration); + // Hide Boogie execution traces since they are meaningless for Dafny programs + "/errorTrace:0" + })}, + { "%server", (typeof(Server).Assembly, Array.Empty()) } + }, + + InvokeMainMethodsDirectly = Environment.GetEnvironmentVariable("XUNIT_INVOKE_MAIN_METHODS_DIRECTLY") == "true", + + Substitions = new Dictionary { + { "%diff", "diff" } + }, + }; + + [FileTheory] + [LitTestData(Extension = ".dfy")] + public void LitTest(string path) { + LitTestCase.Run(path, CONFIG); } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs new file mode 100644 index 00000000000..bcb57c5a142 --- /dev/null +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -0,0 +1,36 @@ +using System; +using System.Linq; +using System.Reflection; + +namespace XUnitExtensions { + public interface ILitCommand { + + private const string COMMENT_PREFIX = "//"; + private const string LIT_COMMAND_PREFIX = "RUN:"; + + public (int, string) Execute(); + + public static ILitCommand Parse(string line, LitTestConfiguration config) { + if (!line.StartsWith(COMMENT_PREFIX)) { + return null; + } + line = line[COMMENT_PREFIX.Length..].Trim(); + + if (!line.StartsWith(LIT_COMMAND_PREFIX)) { + return null; + } + line = line[LIT_COMMAND_PREFIX.Length..].Trim(); + + // TODO: Probably need to handle escaping too + var pieces = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries); + var commandSymbol = pieces[0]; + var arguments = pieces[1..]; + + if (config.MainMethodSubstitutions.TryGetValue(commandSymbol, out (Assembly, string[]) substitution)) { + return MainMethodLitCommand.Parse(substitution.Item1, substitution.Item2.Concat(arguments), config); + } + + return new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables); + } + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/LitSubstitutionAttribute.cs b/Source/XUnitExtensions/LitSubstitutionAttribute.cs deleted file mode 100644 index 73b5826e66d..00000000000 --- a/Source/XUnitExtensions/LitSubstitutionAttribute.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; -using System.Net.Mail; - -namespace XUnitExtensions { - [AttributeUsage(AttributeTargets.Method, AllowMultiple = true)] - public class LitSubstitutionAttribute : Attribute { - public LitSubstitutionAttribute(string key) { - - } - - public virtual Type MainClass { get; set; } - public virtual string CLIPath { get; set; } - - public virtual string[] Arguments { get; set; } - } -} \ No newline at end of file diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index 61535100fcd..a651b00de56 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -1,13 +1,21 @@ using System.IO; using System.Linq; +using Xunit; namespace XUnitExtensions { public class LitTestCase { public static void Run(string filePath, LitTestConfiguration config) { + // TODO: Deal with %s, and %t + // TODO: apply this for MainMethod versions of %dafny etc.: + // Lit always uses the parent directory of a test file as the current directory, + // so other file paths have to be interpreted to be relative to the output directory instead. + // otherFiles.Add(Path.Join(Path.GetDirectoryName(filePath), value)); var content = File.ReadAllLines(filePath); - var commands = content.Select(LitTestCommand.Parse).Where(c => c != null); + var commands = content.Select(line => ILitCommand.Parse(line, config)) + .Where(c => c != null); foreach (var command in commands) { - command.Run(config); + var (exitCode, output) = command.Execute(); + Assert.True(exitCode != 0, $"Command failed with output:\n{output}"); } } } diff --git a/Source/XUnitExtensions/LitTestCommand.cs b/Source/XUnitExtensions/LitTestCommand.cs deleted file mode 100644 index fb5b26e2922..00000000000 --- a/Source/XUnitExtensions/LitTestCommand.cs +++ /dev/null @@ -1,146 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Text; -using Xunit; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace XUnitExtensions { - - public class LitTestCommand { - - private const string COMMENT_PREFIX = "//"; - private const string LIT_COMMAND_PREFIX = "RUN:"; - - // This may be a symbol for substitution such as "%dafny" - private string CliPath; - private string[] Arguments; - private string OutputFile; - private bool AppendOutput; - - public LitTestCommand(string cliPath, IEnumerable arguments, string outputFile, bool appendOutput) { - CliPath = cliPath; - Arguments = arguments.ToArray(); - OutputFile = outputFile; - AppendOutput = appendOutput; - } - - public void Run(LitTestConfiguration config) { - var cliTarget = config.Substitions[CliPath]; - int result; - if (cliTarget is Assembly assembly && - (config.InvokeMainMethodsDirectly || Environment.GetEnvironmentVariable("XUNIT_INVOKE_CLI_DIRECTLY") == "true")) { - result = InvokeCLIDirectly(assembly, config); - } else { - result = InvokeCLIViaProcess(config); - } - Assert.Equal(0, result); - } - - public int InvokeCLIDirectly(Assembly assembly, LitTestConfiguration config) { - StringBuilder redirectedOut = new(); - StringBuilder redirectedErr = new(); - - Console.SetOut(new StringWriter(redirectedOut)); - Console.SetError(new StringWriter(redirectedErr)); - - return (int)assembly.EntryPoint!.Invoke(null, new object[] { ArgumentsWithSubstitutions(config) }); - } - - public int InvokeCLIViaProcess(LitTestConfiguration config) { - using var process = new Process(); - - var cliTarget = config.Substitions[CliPath]; - if (cliTarget is Assembly assembly) { - process.StartInfo.FileName = "dotnet"; - process.StartInfo.Arguments = assembly.Location; - } else { - process.StartInfo.FileName = cliTarget != null ? (string)cliTarget : CliPath; - } - - foreach (var argument in ArgumentsWithSubstitutions(config)) { - process.StartInfo.Arguments += " " + argument; - } - - process.StartInfo.UseShellExecute = false; - // process.StartInfo.RedirectStandardOutput = true; - // process.StartInfo.RedirectStandardError = true; - process.StartInfo.CreateNoWindow = true; - - process.StartInfo.EnvironmentVariables.Clear(); - foreach (var passthrough in config.PassthroughEnvironmentVariables) { - process.StartInfo.EnvironmentVariables.Add(passthrough, Environment.GetEnvironmentVariable(passthrough)); - } - - process.Start(); - process.WaitForExit(); - - return process.ExitCode; - } - - private object ApplySubstitution(string key, LitTestConfiguration config) { - if (config.Substitions.TryGetValue(key, out var value)) { - return value; - } - return key; - } - - private IEnumerable ArgumentsWithSubstitutions(LitTestConfiguration config) { - return Arguments.Select(argument => (string)ApplySubstitution(argument, config)); - } - - public override string ToString() { - var builder = new StringBuilder(); - builder.Append(CliPath); - builder.Append(' '); - builder.AppendJoin(" ", Arguments); - if (OutputFile != null) { - if (AppendOutput) { - builder.Append(" >> "); - } else { - builder.Append(" > "); - } - builder.Append(OutputFile); - } - - return builder.ToString(); - } - - public static LitTestCommand Parse(string line) { - if (!line.StartsWith(COMMENT_PREFIX)) { - return null; - } - line = line.Substring(COMMENT_PREFIX.Length).Trim(); - - if (!line.StartsWith(LIT_COMMAND_PREFIX)) { - return null; - } - line = line.Substring(LIT_COMMAND_PREFIX.Length).Trim(); - - // TODO: Probably need to handle escaping too - var pieces = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).ToList(); - var cliPath = pieces[0]; - var arguments = pieces.GetRange(1, pieces.Count - 1); - - string outputFile = null; - bool appendOutput = false; - var redirectIndex = arguments.IndexOf(">"); - if (redirectIndex >= 0) { - outputFile = arguments[redirectIndex + 1]; - arguments.RemoveRange(redirectIndex, 2); - } - var redirectAppendIndex = arguments.IndexOf(">>"); - if (redirectAppendIndex >= 0) { - outputFile = arguments[redirectAppendIndex + 1]; - appendOutput = true; - arguments.RemoveRange(redirectAppendIndex, 2); - } - - return new LitTestCommand(cliPath, arguments, outputFile, appendOutput); - } - } -} diff --git a/Source/XUnitExtensions/LitTestConfiguration.cs b/Source/XUnitExtensions/LitTestConfiguration.cs index 23de4fc2b83..82adaff8cf8 100644 --- a/Source/XUnitExtensions/LitTestConfiguration.cs +++ b/Source/XUnitExtensions/LitTestConfiguration.cs @@ -1,13 +1,21 @@ using System.Collections.Generic; +using System.Reflection; +using Xunit.Abstractions; namespace XUnitExtensions { public class LitTestConfiguration { - // Values are either strings or Assemblies (for our own CLI tools) - public Dictionary Substitions { get; set; } + public Dictionary Substitions { get; set; } + public Dictionary MainMethodSubstitutions { get; set; } public bool InvokeMainMethodsDirectly { get; set; } - + public IEnumerable PassthroughEnvironmentVariables { get; set; } + public string ApplySubstitutions(string s) { + foreach (var (key, value) in Substitions) { + s = s.Replace(key, value); + } + return s; + } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/LitTestDataDiscoverer.cs b/Source/XUnitExtensions/LitTestDataDiscoverer.cs index 80d869b7e2e..0316aa0614c 100644 --- a/Source/XUnitExtensions/LitTestDataDiscoverer.cs +++ b/Source/XUnitExtensions/LitTestDataDiscoverer.cs @@ -14,25 +14,18 @@ public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, } protected override IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName) { - var invokeDirectly = attributeInfo.GetNamedArgument(nameof(LitTestDataAttribute.InvokeCliDirectly)); var basePath = GetBasePath(attributeInfo, testMethod); - try { - var (testCases, _) = convertor.ConvertLitCommands(basePath, fileName, invokeDirectly, File.ReadLines(fileName)); - return testCases.Select(testCase => new[] { testCase }); - } catch (Exception e) { - var shortPath = fileName[(basePath.Length + 1)..]; - var skippedCase = new FileTheoryDataRow(shortPath) { - SourceInformation = new SourceInformation() { FileName = fileName, LineNumber = 0}, - TestDisplayName = shortPath, - Skip = $"Exception: {e}" - }; - return new[] { new[] { skippedCase } }; - } + var shortPath = fileName[(basePath.Length + 1)..]; + var row = new FileTheoryDataRow(fileName) { + SourceInformation = new SourceInformation() { FileName = fileName, LineNumber = 0}, + TestDisplayName = shortPath, + }; + return new[] { new[] { row } }; } } } [DataDiscoverer("DafnyDriver.Test.LitTestDataDiscoverer", "LitTestConvertor.Test")] public class LitTestDataAttribute : FileDataAttribute { - public bool InvokeCliDirectly { get; set; } + } \ No newline at end of file diff --git a/Source/XUnitExtensions/MainMethodLitCommand.cs b/Source/XUnitExtensions/MainMethodLitCommand.cs new file mode 100644 index 00000000000..2a85c7afdfe --- /dev/null +++ b/Source/XUnitExtensions/MainMethodLitCommand.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; + +namespace XUnitExtensions { + public class MainMethodLitCommand : ILitCommand { + public Assembly Assembly { get; protected set; } + public string[] Arguments { get; protected set; } + public string OutputFile { get; protected set; } + public bool AppendOutput { get; protected set; } + + public static ILitCommand Parse(Assembly assembly, IEnumerable arguments, LitTestConfiguration config) { + var result = new MainMethodLitCommand { + Assembly = assembly + }; + + var argumentsList = arguments.ToList(); + var redirectIndex = argumentsList.IndexOf(">"); + if (redirectIndex >= 0) { + result.OutputFile = argumentsList[redirectIndex + 1]; + argumentsList.RemoveRange(redirectIndex, 2); + } + var redirectAppendIndex = argumentsList.IndexOf(">>"); + if (redirectAppendIndex >= 0) { + result.OutputFile = argumentsList[redirectAppendIndex + 1]; + result.AppendOutput = true; + argumentsList.RemoveRange(redirectAppendIndex, 2); + } + + return config.InvokeMainMethodsDirectly ? result : result.ToShellCommand(config); + } + + public (int, string) Execute() { + StringBuilder redirectedErr = new(); + + if (OutputFile != null) { + Console.SetOut(new StreamWriter(OutputFile, append: AppendOutput)); + } + Console.SetError(new StringWriter(redirectedErr)); + + var exitCode = (int)Assembly.EntryPoint!.Invoke(null, new object[] { Arguments }); + + return (exitCode, redirectedErr.ToString()); + } + + public ILitCommand ToShellCommand(LitTestConfiguration config) { + var shellArguments = new[] { Assembly.Location }.Concat(Arguments); + if (OutputFile != null) { + shellArguments = shellArguments.Append(AppendOutput ? ">>" : ">").Append(OutputFile); + } + return new ShellLitCommand("dotnet", shellArguments, config.PassthroughEnvironmentVariables); + } + + public override string ToString() { + var builder = new StringBuilder(); + builder.Append(Assembly); + builder.Append(' '); + builder.AppendJoin(" ", Arguments); + if (OutputFile != null) { + if (AppendOutput) { + builder.Append(" >> "); + } else { + builder.Append(" > "); + } + builder.Append(OutputFile); + } + + return builder.ToString(); + } + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/ShellLitCommand.cs new file mode 100644 index 00000000000..0147455d593 --- /dev/null +++ b/Source/XUnitExtensions/ShellLitCommand.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; + +namespace XUnitExtensions { + public class ShellLitCommand : ILitCommand { + private string ShellCommand; + private string[] Arguments; + private string[] PassthroughEnvironmentVariables; + + public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnumerable passthroughEnvironmentVariables) { + ShellCommand = shellCommand; + Arguments = arguments.ToArray(); + PassthroughEnvironmentVariables = passthroughEnvironmentVariables.ToArray(); + } + + public (int, string) Execute() { + using var process = new Process(); + + process.StartInfo.FileName = ShellCommand; + foreach (var argument in Arguments) { + process.StartInfo.Arguments += " " + argument; + } + + process.StartInfo.UseShellExecute = false; + // process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.CreateNoWindow = true; + + process.StartInfo.EnvironmentVariables.Clear(); + foreach (var passthrough in PassthroughEnvironmentVariables) { + process.StartInfo.EnvironmentVariables.Add(passthrough, Environment.GetEnvironmentVariable(passthrough)); + } + + process.Start(); + string error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + + return (process.ExitCode, error); + } + + public override string ToString() { + var builder = new StringBuilder(); + builder.Append(ShellCommand); + builder.Append(' '); + builder.AppendJoin(" ", Arguments); + return builder.ToString(); + } + } +} \ No newline at end of file From 86d05c462aacd8d09f81d3d8f330d54c0a46caa8 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 30 Sep 2021 11:52:32 -0700 Subject: [PATCH 125/192] Various fixes - passes majority of tests now! --- Source/Dafny.sln | 26 --------- Source/Dafny/DafnyPipeline.csproj | 3 + .../LitTestConverter/LitTestConverter.csproj | 15 ----- .../Properties/AssemblyInfo.cs | 35 ------------ .../Conversion/ConvertingFromLitToXunit.cs | 14 ++--- .../LitTestRunner/LitTests.cs | 8 ++- Source/XUnitExtensions/ILitCommand.cs | 36 ++++++++++-- Source/XUnitExtensions/LitTestCase.cs | 34 +++++++++--- .../XUnitExtensions/LitTestConfiguration.cs | 13 +++++ .../XUnitExtensions/LitTestDataDiscoverer.cs | 4 +- .../XUnitExtensions/MainMethodLitCommand.cs | 28 +++------- Source/XUnitExtensions/ShellLitCommand.cs | 55 +++++++++++++------ 12 files changed, 132 insertions(+), 139 deletions(-) delete mode 100644 Source/LitTestConverter/LitTestConverter.csproj delete mode 100644 Source/LitTestConverter/Properties/AssemblyInfo.cs diff --git a/Source/Dafny.sln b/Source/Dafny.sln index ec9152c0623..8644dde97ce 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -20,8 +20,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyDriver.Test", "DafnyDriver.Test\DafnyDriver.Test.csproj", "{08A3665B-9854-4CF6-BF5F-C532DA39A043}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LitTestConverter", "LitTestConverter\LitTestConverter.csproj", "{2EA7D80A-B179-4352-BDF2-8A1599F8685A}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LitTestConvertor.Test", "LitTestConvertor.Test\LitTestConvertor.Test.csproj", "{A4BC2C77-3A48-4917-AA22-41978DFFE24E}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DafnyLanguageServer", "DafnyLanguageServer\DafnyLanguageServer.csproj", "{CD05D26C-C672-4F43-835E-7A3E1741E4D8}" @@ -166,30 +164,6 @@ Global {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|Mixed Platforms.Build.0 = Release|Any CPU {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|x86.ActiveCfg = Release|Any CPU {08A3665B-9854-4CF6-BF5F-C532DA39A043}.Release|x86.Build.0 = Release|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|.NET.ActiveCfg = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|.NET.Build.0 = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Any CPU.ActiveCfg = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Any CPU.Build.0 = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Mixed Platforms.ActiveCfg = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|Mixed Platforms.Build.0 = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|x86.ActiveCfg = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Checked|x86.Build.0 = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|.NET.ActiveCfg = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|.NET.Build.0 = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|x86.ActiveCfg = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Debug|x86.Build.0 = Debug|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|.NET.ActiveCfg = Release|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|.NET.Build.0 = Release|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Any CPU.Build.0 = Release|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|x86.ActiveCfg = Release|Any CPU - {2EA7D80A-B179-4352-BDF2-8A1599F8685A}.Release|x86.Build.0 = Release|Any CPU {CD05D26C-C672-4F43-835E-7A3E1741E4D8}.Checked|.NET.ActiveCfg = Debug|Any CPU {CD05D26C-C672-4F43-835E-7A3E1741E4D8}.Checked|.NET.Build.0 = Debug|Any CPU {CD05D26C-C672-4F43-835E-7A3E1741E4D8}.Checked|Any CPU.ActiveCfg = Debug|Any CPU diff --git a/Source/Dafny/DafnyPipeline.csproj b/Source/Dafny/DafnyPipeline.csproj index 696b3280b98..9e5849e5ebb 100644 --- a/Source/Dafny/DafnyPipeline.csproj +++ b/Source/Dafny/DafnyPipeline.csproj @@ -36,6 +36,9 @@ + + + diff --git a/Source/LitTestConverter/LitTestConverter.csproj b/Source/LitTestConverter/LitTestConverter.csproj deleted file mode 100644 index 1548e223922..00000000000 --- a/Source/LitTestConverter/LitTestConverter.csproj +++ /dev/null @@ -1,15 +0,0 @@ - - - - - Exe - net5.0 - LitTestConverter - false - - - - - - - \ No newline at end of file diff --git a/Source/LitTestConverter/Properties/AssemblyInfo.cs b/Source/LitTestConverter/Properties/AssemblyInfo.cs deleted file mode 100644 index fd9af8a2d05..00000000000 --- a/Source/LitTestConverter/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System.Reflection; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("LitTestConverter")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("LitTestConverter")] -[assembly: AssemblyCopyright("Copyright © 2020")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("2EA7D80A-B179-4352-BDF2-8A1599F8685A")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs index 62259a18597..30e47ad214d 100644 --- a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs +++ b/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs @@ -2,25 +2,21 @@ using System.IO; using System.Reflection; using Xunit; +using XUnitExtensions; namespace LitTestConvertor.Test { public class ConvertingFromLitToXunit { [Fact] public void HelloWorld() { - var convertor = new LitTestConverter.LitTestConvertor(); - var lines = File.ReadLines("TestFiles/HelloWorldLitTest.dfy"); - var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/HelloWorldLitTest.dfy", false, lines); + var config = new LitTestConfiguration(); + var program = LitTestCase.Read("TestFiles/HelloWorldLitTest.dfy", config); } [Fact] public void VerifyOnly() { - var convertor = new LitTestConverter.LitTestConvertor(); - using var stream = - Assembly.GetExecutingAssembly().GetManifestResourceStream("LitTestConvertor.Test.TestFiles.VerifyOnlyLitTest.dfy"); - using var reader = new StreamReader(stream); - var lines = File.ReadLines("TestFiles/VerifyOnly.dfy"); - var (testCases, testContent) = convertor.ConvertLitCommands("TestFiles", "TestFiles/VerifyOnly.dfy", false, lines); + var config = new LitTestConfiguration(); + var program = LitTestCase.Read("TestFiles/VerifyOnly.dfy", config); } } } \ No newline at end of file diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index b18161c30d3..435c9b947d3 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -12,12 +12,12 @@ public class LitTests { private static readonly LitTestConfiguration CONFIG = new () { MainMethodSubstitutions = new Dictionary { - { "%baredafny", (typeof(Microsoft.Dafny.DafnyDriver).Assembly, Array.Empty()) }, - { "%dafny", (typeof(Microsoft.Dafny.DafnyDriver).Assembly, new [] { + { "%baredafny", (typeof(DafnyDriver).Assembly, Array.Empty()) }, + { "%dafny", (typeof(DafnyDriver).Assembly, new [] { "/countVerificationErrors:0", // We do not want absolute or relative paths in error messages, just the basename of the file - "/useBaseNameForFileName:yes", + "/useBaseNameForFileName", // We do not want output such as "Compiled program written to Foo.cs" // from the compilers, since that changes with the target language @@ -34,6 +34,8 @@ public class LitTests { Substitions = new Dictionary { { "%diff", "diff" } }, + + PassthroughEnvironmentVariables = new []{ "PATH", "HOME" }, }; [FileTheory] diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index bcb57c5a142..9112ad9e8e6 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -1,4 +1,6 @@ using System; +using System.Collections; +using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -8,7 +10,7 @@ public interface ILitCommand { private const string COMMENT_PREFIX = "//"; private const string LIT_COMMAND_PREFIX = "RUN:"; - public (int, string) Execute(); + public (int, string, string) Execute(); public static ILitCommand Parse(string line, LitTestConfiguration config) { if (!line.StartsWith(COMMENT_PREFIX)) { @@ -22,15 +24,41 @@ public static ILitCommand Parse(string line, LitTestConfiguration config) { line = line[LIT_COMMAND_PREFIX.Length..].Trim(); // TODO: Probably need to handle escaping too - var pieces = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries); + var pieces = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).Select(StripQuotes).ToArray(); var commandSymbol = pieces[0]; - var arguments = pieces[1..]; + var arguments = pieces[1..].Select(config.ApplySubstitutions); if (config.MainMethodSubstitutions.TryGetValue(commandSymbol, out (Assembly, string[]) substitution)) { return MainMethodLitCommand.Parse(substitution.Item1, substitution.Item2.Concat(arguments), config); } - return new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables); + commandSymbol = config.ApplySubstitutions(commandSymbol); + + return ShellLitCommand.Parse(commandSymbol, arguments, config); + } + + private static string StripQuotes(string s) { + if (s.Length >= 2 && s[0] == '"' && s[^1] == '"') { + return s[1..^1]; + } + return s; + } + + public static (IEnumerable, string, bool) ExtractOutputFile(IEnumerable arguments) { + var argumentsList = arguments.ToList(); + var redirectIndex = argumentsList.IndexOf(">"); + if (redirectIndex >= 0) { + var outputFile = argumentsList[redirectIndex + 1]; + argumentsList.RemoveRange(redirectIndex, 2); + return (argumentsList, outputFile, false); + } + var redirectAppendIndex = argumentsList.IndexOf(">>"); + if (redirectAppendIndex >= 0) { + var outputFile = argumentsList[redirectAppendIndex + 1]; + argumentsList.RemoveRange(redirectAppendIndex, 2); + return (argumentsList, outputFile, true); + } + return (argumentsList, null, false); } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index a651b00de56..9375bc23763 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -1,21 +1,39 @@ +using System; +using System.Collections; +using System.Collections.Generic; using System.IO; using System.Linq; using Xunit; namespace XUnitExtensions { public class LitTestCase { + + public static IEnumerable Read(string filePath, LitTestConfiguration config) { + return File.ReadAllLines(filePath) + .Select(line => ILitCommand.Parse(line, config)) + .Where(c => c != null); + } + public static void Run(string filePath, LitTestConfiguration config) { - // TODO: Deal with %s, and %t - // TODO: apply this for MainMethod versions of %dafny etc.: + string directory = Path.GetDirectoryName(filePath); + config = config.WithSubsitutions(new Dictionary { + { "%s", filePath }, + { "%t", Path.Join(directory, "Output", $"{Path.GetFileName(filePath)}.tmp")} + }); + + // TODO: apply this for %dafny and any other commands where we recognize file arguments // Lit always uses the parent directory of a test file as the current directory, - // so other file paths have to be interpreted to be relative to the output directory instead. + // but xUnit tests will always use the .NET output directory, + // so other file paths have to be interpreted to be relative to the .NET output directory instead. // otherFiles.Add(Path.Join(Path.GetDirectoryName(filePath), value)); - var content = File.ReadAllLines(filePath); - var commands = content.Select(line => ILitCommand.Parse(line, config)) - .Where(c => c != null); + var commands = Read(filePath, config); foreach (var command in commands) { - var (exitCode, output) = command.Execute(); - Assert.True(exitCode != 0, $"Command failed with output:\n{output}"); + try { + var (exitCode, output, error) = command.Execute(); + Assert.True(exitCode == 0, $"Command failed with output:\n{output}\nError:\n{error}"); + } catch (Exception e) { + throw new Exception($"Exception thrown while executing command: {command}", e); + } } } } diff --git a/Source/XUnitExtensions/LitTestConfiguration.cs b/Source/XUnitExtensions/LitTestConfiguration.cs index 82adaff8cf8..33fd0e3e9cf 100644 --- a/Source/XUnitExtensions/LitTestConfiguration.cs +++ b/Source/XUnitExtensions/LitTestConfiguration.cs @@ -1,8 +1,12 @@ using System.Collections.Generic; +using System.Linq; using System.Reflection; +using Microsoft.Boogie; using Xunit.Abstractions; namespace XUnitExtensions { + + // TODO: Make safely immutable public class LitTestConfiguration { public Dictionary Substitions { get; set; } @@ -17,5 +21,14 @@ public string ApplySubstitutions(string s) { } return s; } + + public LitTestConfiguration WithSubsitutions(Dictionary more) { + return new LitTestConfiguration { + Substitions = Substitions.Concat(more).ToDictionary(pair => pair.Key, pair => pair.Value), + MainMethodSubstitutions = MainMethodSubstitutions, + InvokeMainMethodsDirectly = InvokeMainMethodsDirectly, + PassthroughEnvironmentVariables = PassthroughEnvironmentVariables + }; + } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/LitTestDataDiscoverer.cs b/Source/XUnitExtensions/LitTestDataDiscoverer.cs index 0316aa0614c..cdb7f8da78e 100644 --- a/Source/XUnitExtensions/LitTestDataDiscoverer.cs +++ b/Source/XUnitExtensions/LitTestDataDiscoverer.cs @@ -6,7 +6,7 @@ using Xunit.Sdk; using XUnitExtensions; -namespace DafnyDriver.Test { +namespace XUnitExtensions { public class LitTestDataDiscoverer : FileDataDiscoverer { public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { @@ -25,7 +25,7 @@ protected override IEnumerable FileData(IAttributeInfo attributeInfo, } } -[DataDiscoverer("DafnyDriver.Test.LitTestDataDiscoverer", "LitTestConvertor.Test")] +[DataDiscoverer("XUnitExtensions.LitTestDataDiscoverer", "XUnitExtensions")] public class LitTestDataAttribute : FileDataAttribute { } \ No newline at end of file diff --git a/Source/XUnitExtensions/MainMethodLitCommand.cs b/Source/XUnitExtensions/MainMethodLitCommand.cs index 2a85c7afdfe..13e3e8a340e 100644 --- a/Source/XUnitExtensions/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/MainMethodLitCommand.cs @@ -14,26 +14,17 @@ public class MainMethodLitCommand : ILitCommand { public static ILitCommand Parse(Assembly assembly, IEnumerable arguments, LitTestConfiguration config) { var result = new MainMethodLitCommand { - Assembly = assembly + Assembly = assembly, }; - var argumentsList = arguments.ToList(); - var redirectIndex = argumentsList.IndexOf(">"); - if (redirectIndex >= 0) { - result.OutputFile = argumentsList[redirectIndex + 1]; - argumentsList.RemoveRange(redirectIndex, 2); - } - var redirectAppendIndex = argumentsList.IndexOf(">>"); - if (redirectAppendIndex >= 0) { - result.OutputFile = argumentsList[redirectAppendIndex + 1]; - result.AppendOutput = true; - argumentsList.RemoveRange(redirectAppendIndex, 2); - } - + IEnumerable args; + (args, result.OutputFile, result.AppendOutput) = ILitCommand.ExtractOutputFile(arguments); + result.Arguments = args.ToArray(); + return config.InvokeMainMethodsDirectly ? result : result.ToShellCommand(config); } - public (int, string) Execute() { + public (int, string, string) Execute() { StringBuilder redirectedErr = new(); if (OutputFile != null) { @@ -43,15 +34,12 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments var exitCode = (int)Assembly.EntryPoint!.Invoke(null, new object[] { Arguments }); - return (exitCode, redirectedErr.ToString()); + return (exitCode, "", redirectedErr.ToString()); } public ILitCommand ToShellCommand(LitTestConfiguration config) { var shellArguments = new[] { Assembly.Location }.Concat(Arguments); - if (OutputFile != null) { - shellArguments = shellArguments.Append(AppendOutput ? ">>" : ">").Append(OutputFile); - } - return new ShellLitCommand("dotnet", shellArguments, config.PassthroughEnvironmentVariables); + return new ShellLitCommand("dotnet", shellArguments, OutputFile, AppendOutput, config.PassthroughEnvironmentVariables); } public override string ToString() { diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/ShellLitCommand.cs index 0147455d593..58fec5b4ee1 100644 --- a/Source/XUnitExtensions/ShellLitCommand.cs +++ b/Source/XUnitExtensions/ShellLitCommand.cs @@ -1,51 +1,72 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Linq; using System.Text; namespace XUnitExtensions { public class ShellLitCommand : ILitCommand { - private string ShellCommand; - private string[] Arguments; - private string[] PassthroughEnvironmentVariables; - - public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnumerable passthroughEnvironmentVariables) { - ShellCommand = shellCommand; - Arguments = arguments.ToArray(); - PassthroughEnvironmentVariables = passthroughEnvironmentVariables.ToArray(); + private string shellCommand; + private string[] arguments; + private string outputFile; + private bool appendOutput; + private string[] passthroughEnvironmentVariables; + + public ShellLitCommand(string shellCommand, IEnumerable arguments, string outputFile, bool appendOutput, IEnumerable passthroughEnvironmentVariables) { + this.shellCommand = shellCommand; + this.arguments = arguments.ToArray(); + this.outputFile = outputFile; + this.appendOutput = appendOutput; + this.passthroughEnvironmentVariables = passthroughEnvironmentVariables.ToArray(); } - - public (int, string) Execute() { + + public static ILitCommand Parse(string shellCommand, IEnumerable arguments, LitTestConfiguration config) { + var (args, outputFile, appendOutput) = ILitCommand.ExtractOutputFile(arguments); + return new ShellLitCommand(shellCommand, args, outputFile, appendOutput, config.PassthroughEnvironmentVariables); + } + + public (int, string, string) Execute() { using var process = new Process(); - process.StartInfo.FileName = ShellCommand; - foreach (var argument in Arguments) { + process.StartInfo.FileName = shellCommand; + foreach (var argument in arguments) { process.StartInfo.Arguments += " " + argument; } + // We avoid setting the current directory so that we maintain parity with + // MainMethodLitCommand, which can't change the current directory. + process.StartInfo.UseShellExecute = false; - // process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.StartInfo.CreateNoWindow = true; process.StartInfo.EnvironmentVariables.Clear(); - foreach (var passthrough in PassthroughEnvironmentVariables) { + foreach (var passthrough in passthroughEnvironmentVariables) { process.StartInfo.EnvironmentVariables.Add(passthrough, Environment.GetEnvironmentVariable(passthrough)); } process.Start(); + string output = process.StandardOutput.ReadToEnd(); + if (outputFile != null) { + if (appendOutput) { + File.AppendAllText(outputFile, output); + } else { + File.WriteAllText(outputFile, output); + } + } string error = process.StandardError.ReadToEnd(); process.WaitForExit(); - return (process.ExitCode, error); + return (process.ExitCode, output, error); } public override string ToString() { var builder = new StringBuilder(); - builder.Append(ShellCommand); + builder.Append(shellCommand); builder.Append(' '); - builder.AppendJoin(" ", Arguments); + builder.AppendJoin(" ", arguments); return builder.ToString(); } } From 0ca1b425508d8d4ca544cee1d6b4833435635f17 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 1 Oct 2021 14:49:08 -0700 Subject: [PATCH 126/192] Fix Javascript tests, path issues with file arguments --- .../LitTestConvertor.Test.csproj | 7 ++- .../LitTestRunner/LitTests.cs | 50 ++++++++++++------- Source/XUnitExtensions/ILitCommand.cs | 29 ++++++++--- .../XUnitExtensions/LitCommandWithOutput.cs | 31 ++++++++++++ Source/XUnitExtensions/LitTestCase.cs | 24 ++++----- .../XUnitExtensions/LitTestConfiguration.cs | 10 ++-- .../XUnitExtensions/MainMethodLitCommand.cs | 30 +++-------- Source/XUnitExtensions/ShellLitCommand.cs | 22 ++------ Test/git-issues/github-issue-1111.dfy | 0 9 files changed, 115 insertions(+), 88 deletions(-) create mode 100644 Source/XUnitExtensions/LitCommandWithOutput.cs create mode 100644 Test/git-issues/github-issue-1111.dfy diff --git a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj index 86458e42bac..cea64e18788 100644 --- a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj +++ b/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj @@ -19,15 +19,14 @@ - - - - PreserveNewest + + PreserveNewest + diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs index 435c9b947d3..811399606f8 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs @@ -1,5 +1,7 @@ using System; +using System.Collections; using System.Collections.Generic; +using System.Linq; using System.Reflection; using Microsoft.Dafny; using Xunit; @@ -9,28 +11,40 @@ namespace LitTestConvertor.Test.LitTestRunner { public class LitTests { + + private static readonly Assembly dafnyDriverAssembly = typeof(DafnyDriver).Assembly; + private static readonly Assembly dafnyServerAssembly = typeof(Server).Assembly; + + private static readonly string[] defaultDafnyArguments = new[] { + "/countVerificationErrors:0", + + // We do not want absolute or relative paths in error messages, just the basename of the file + "/useBaseNameForFileName", + + // We do not want output such as "Compiled program written to Foo.cs" + // from the compilers, since that changes with the target language + "/compileVerbose:0", + + // Hide Boogie execution traces since they are meaningless for Dafny programs + "/errorTrace:0" + }; + private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable arguments, LitTestConfiguration config, bool invokeDirectly) { + + return MainMethodLitCommand.Parse(assembly, arguments, config, invokeDirectly); + } + private static readonly LitTestConfiguration CONFIG = new () { - MainMethodSubstitutions = new Dictionary { - { "%baredafny", (typeof(DafnyDriver).Assembly, Array.Empty()) }, - { "%dafny", (typeof(DafnyDriver).Assembly, new [] { - "/countVerificationErrors:0", - - // We do not want absolute or relative paths in error messages, just the basename of the file - "/useBaseNameForFileName", - - // We do not want output such as "Compiled program written to Foo.cs" - // from the compilers, since that changes with the target language - "/compileVerbose:0", - - // Hide Boogie execution traces since they are meaningless for Dafny programs - "/errorTrace:0" - })}, - { "%server", (typeof(Server).Assembly, Array.Empty()) } + Commands = new Dictionary, LitTestConfiguration, ILitCommand>> { + { "%baredafny", (args, config) => + MainWithArguments(dafnyDriverAssembly, args, config, false) }, + { "%dafny", (args, config) => + MainWithArguments(dafnyDriverAssembly, defaultDafnyArguments.Concat(args), config, true) }, + { "%server", (args, config) => + MainWithArguments(dafnyDriverAssembly, args, config, false) }, }, - InvokeMainMethodsDirectly = Environment.GetEnvironmentVariable("XUNIT_INVOKE_MAIN_METHODS_DIRECTLY") == "true", - + // TODO: speed this up by using AssertWithDiff Substitions = new Dictionary { { "%diff", "diff" } }, diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index 9112ad9e8e6..87dc3ee37d9 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Reflection; @@ -10,9 +11,9 @@ public interface ILitCommand { private const string COMMENT_PREFIX = "//"; private const string LIT_COMMAND_PREFIX = "RUN:"; - public (int, string, string) Execute(); + public (int, string, string) Execute(TextWriter outputWriter); - public static ILitCommand Parse(string line, LitTestConfiguration config) { + public static LitCommandWithOutput Parse(string fileName, string line, LitTestConfiguration config) { if (!line.StartsWith(COMMENT_PREFIX)) { return null; } @@ -26,15 +27,22 @@ public static ILitCommand Parse(string line, LitTestConfiguration config) { // TODO: Probably need to handle escaping too var pieces = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).Select(StripQuotes).ToArray(); var commandSymbol = pieces[0]; - var arguments = pieces[1..].Select(config.ApplySubstitutions); - - if (config.MainMethodSubstitutions.TryGetValue(commandSymbol, out (Assembly, string[]) substitution)) { - return MainMethodLitCommand.Parse(substitution.Item1, substitution.Item2.Concat(arguments), config); + var basePath = Path.GetDirectoryName(fileName); + var (rawArguments, outputFile, appendOutput) = ILitCommand.ExtractOutputFile(pieces[1..]); + if (outputFile != null) { + outputFile = MakeFilePathsAbsolute(basePath, config.ApplySubstitutions(outputFile)); + } + var arguments = rawArguments + .Select(config.ApplySubstitutions) + .Select(arg => MakeFilePathsAbsolute(basePath, arg)); + + if (config.Commands.TryGetValue(commandSymbol, out var command)) { + return new LitCommandWithOutput(command(arguments, config), outputFile, appendOutput); } commandSymbol = config.ApplySubstitutions(commandSymbol); - return ShellLitCommand.Parse(commandSymbol, arguments, config); + return new LitCommandWithOutput(new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), outputFile, appendOutput); } private static string StripQuotes(string s) { @@ -43,6 +51,13 @@ private static string StripQuotes(string s) { } return s; } + + public static string MakeFilePathsAbsolute(string basePath, string s) { + if (s.StartsWith("-") || s.StartsWith("/")) { + return s; + } + return Path.Join(basePath, s); + } public static (IEnumerable, string, bool) ExtractOutputFile(IEnumerable arguments) { var argumentsList = arguments.ToList(); diff --git a/Source/XUnitExtensions/LitCommandWithOutput.cs b/Source/XUnitExtensions/LitCommandWithOutput.cs new file mode 100644 index 00000000000..e1d0f8d65f5 --- /dev/null +++ b/Source/XUnitExtensions/LitCommandWithOutput.cs @@ -0,0 +1,31 @@ +using System.IO; +using System.Text; + +namespace XUnitExtensions { + public class LitCommandWithOutput { + private ILitCommand command; + private string outputFile; + private bool append; + + public LitCommandWithOutput(ILitCommand command, string outputFile, bool append) { + this.command = command; + this.outputFile = outputFile; + this.append = append; + } + + public (int, string, string) Execute() { + var outputWriter = outputFile != null ? new StreamWriter(outputFile, append) : null; + return command.Execute(outputWriter); + } + + public override string ToString() { + var builder = new StringBuilder(); + builder.Append(command); + if (outputFile != null) { + builder.Append(append ? " >> " : ">"); + builder.Append(outputFile); + } + return builder.ToString(); + } + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index 9375bc23763..ed132078d3c 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -8,32 +8,30 @@ namespace XUnitExtensions { public class LitTestCase { - public static IEnumerable Read(string filePath, LitTestConfiguration config) { + public static IEnumerable Read(string filePath, LitTestConfiguration config) { return File.ReadAllLines(filePath) - .Select(line => ILitCommand.Parse(line, config)) + .Select(line => ILitCommand.Parse(filePath, line, config)) .Where(c => c != null); } public static void Run(string filePath, LitTestConfiguration config) { - string directory = Path.GetDirectoryName(filePath); - config = config.WithSubsitutions(new Dictionary { - { "%s", filePath }, - { "%t", Path.Join(directory, "Output", $"{Path.GetFileName(filePath)}.tmp")} + string fileName = Path.GetFileName(filePath); + config = config.WithSubstitutions(new Dictionary { + { "%s", fileName }, + { "%t", Path.Join("Output", $"{fileName}.tmp")} }); - // TODO: apply this for %dafny and any other commands where we recognize file arguments - // Lit always uses the parent directory of a test file as the current directory, - // but xUnit tests will always use the .NET output directory, - // so other file paths have to be interpreted to be relative to the .NET output directory instead. - // otherFiles.Add(Path.Join(Path.GetDirectoryName(filePath), value)); var commands = Read(filePath, config); foreach (var command in commands) { + int exitCode; + string output; + string error; try { - var (exitCode, output, error) = command.Execute(); - Assert.True(exitCode == 0, $"Command failed with output:\n{output}\nError:\n{error}"); + (exitCode, output, error) = command.Execute(); } catch (Exception e) { throw new Exception($"Exception thrown while executing command: {command}", e); } + Assert.True(exitCode == 0, $"Command failed with output:\n{output}\nError:\n{error}"); } } } diff --git a/Source/XUnitExtensions/LitTestConfiguration.cs b/Source/XUnitExtensions/LitTestConfiguration.cs index 33fd0e3e9cf..b3263cf5afa 100644 --- a/Source/XUnitExtensions/LitTestConfiguration.cs +++ b/Source/XUnitExtensions/LitTestConfiguration.cs @@ -1,7 +1,7 @@ +using System; using System.Collections.Generic; using System.Linq; using System.Reflection; -using Microsoft.Boogie; using Xunit.Abstractions; namespace XUnitExtensions { @@ -10,8 +10,7 @@ namespace XUnitExtensions { public class LitTestConfiguration { public Dictionary Substitions { get; set; } - public Dictionary MainMethodSubstitutions { get; set; } - public bool InvokeMainMethodsDirectly { get; set; } + public Dictionary, LitTestConfiguration, ILitCommand>> Commands { get; set; } public IEnumerable PassthroughEnvironmentVariables { get; set; } @@ -22,11 +21,10 @@ public string ApplySubstitutions(string s) { return s; } - public LitTestConfiguration WithSubsitutions(Dictionary more) { + public LitTestConfiguration WithSubstitutions(Dictionary more) { return new LitTestConfiguration { Substitions = Substitions.Concat(more).ToDictionary(pair => pair.Key, pair => pair.Value), - MainMethodSubstitutions = MainMethodSubstitutions, - InvokeMainMethodsDirectly = InvokeMainMethodsDirectly, + Commands = Commands, PassthroughEnvironmentVariables = PassthroughEnvironmentVariables }; } diff --git a/Source/XUnitExtensions/MainMethodLitCommand.cs b/Source/XUnitExtensions/MainMethodLitCommand.cs index 13e3e8a340e..cb0765f6d0c 100644 --- a/Source/XUnitExtensions/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/MainMethodLitCommand.cs @@ -9,26 +9,21 @@ namespace XUnitExtensions { public class MainMethodLitCommand : ILitCommand { public Assembly Assembly { get; protected set; } public string[] Arguments { get; protected set; } - public string OutputFile { get; protected set; } - public bool AppendOutput { get; protected set; } - public static ILitCommand Parse(Assembly assembly, IEnumerable arguments, LitTestConfiguration config) { + public static ILitCommand Parse(Assembly assembly, IEnumerable arguments, LitTestConfiguration config, bool invokeDirectly) { var result = new MainMethodLitCommand { Assembly = assembly, + Arguments = arguments.ToArray() }; - IEnumerable args; - (args, result.OutputFile, result.AppendOutput) = ILitCommand.ExtractOutputFile(arguments); - result.Arguments = args.ToArray(); - - return config.InvokeMainMethodsDirectly ? result : result.ToShellCommand(config); + return invokeDirectly ? result : result.ToShellCommand(config); } - public (int, string, string) Execute() { + public (int, string, string) Execute(TextWriter outputWriter) { StringBuilder redirectedErr = new(); - if (OutputFile != null) { - Console.SetOut(new StreamWriter(OutputFile, append: AppendOutput)); + if (outputWriter != null) { + Console.SetOut(outputWriter); } Console.SetError(new StringWriter(redirectedErr)); @@ -39,23 +34,14 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments public ILitCommand ToShellCommand(LitTestConfiguration config) { var shellArguments = new[] { Assembly.Location }.Concat(Arguments); - return new ShellLitCommand("dotnet", shellArguments, OutputFile, AppendOutput, config.PassthroughEnvironmentVariables); + return new ShellLitCommand("dotnet", shellArguments, config.PassthroughEnvironmentVariables); } public override string ToString() { var builder = new StringBuilder(); - builder.Append(Assembly); + builder.Append(Assembly.EntryPoint); builder.Append(' '); builder.AppendJoin(" ", Arguments); - if (OutputFile != null) { - if (AppendOutput) { - builder.Append(" >> "); - } else { - builder.Append(" > "); - } - builder.Append(OutputFile); - } - return builder.ToString(); } } diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/ShellLitCommand.cs index 58fec5b4ee1..11dafcff1f1 100644 --- a/Source/XUnitExtensions/ShellLitCommand.cs +++ b/Source/XUnitExtensions/ShellLitCommand.cs @@ -9,24 +9,15 @@ namespace XUnitExtensions { public class ShellLitCommand : ILitCommand { private string shellCommand; private string[] arguments; - private string outputFile; - private bool appendOutput; private string[] passthroughEnvironmentVariables; - public ShellLitCommand(string shellCommand, IEnumerable arguments, string outputFile, bool appendOutput, IEnumerable passthroughEnvironmentVariables) { + public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnumerable passthroughEnvironmentVariables) { this.shellCommand = shellCommand; this.arguments = arguments.ToArray(); - this.outputFile = outputFile; - this.appendOutput = appendOutput; this.passthroughEnvironmentVariables = passthroughEnvironmentVariables.ToArray(); } - public static ILitCommand Parse(string shellCommand, IEnumerable arguments, LitTestConfiguration config) { - var (args, outputFile, appendOutput) = ILitCommand.ExtractOutputFile(arguments); - return new ShellLitCommand(shellCommand, args, outputFile, appendOutput, config.PassthroughEnvironmentVariables); - } - - public (int, string, string) Execute() { + public (int, string, string) Execute(TextWriter outputWriter) { using var process = new Process(); process.StartInfo.FileName = shellCommand; @@ -49,13 +40,8 @@ public static ILitCommand Parse(string shellCommand, IEnumerable argumen process.Start(); string output = process.StandardOutput.ReadToEnd(); - if (outputFile != null) { - if (appendOutput) { - File.AppendAllText(outputFile, output); - } else { - File.WriteAllText(outputFile, output); - } - } + outputWriter?.Write(output); + outputWriter?.Flush(); string error = process.StandardError.ReadToEnd(); process.WaitForExit(); diff --git a/Test/git-issues/github-issue-1111.dfy b/Test/git-issues/github-issue-1111.dfy new file mode 100644 index 00000000000..e69de29bb2d From 6bd625080f5d69f430172292b476f08cfda10426 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 1 Oct 2021 16:13:11 -0700 Subject: [PATCH 127/192] Rename package --- Source/Dafny.sln | 2 +- .../IntegrationTests.csproj} | 6 ++++++ .../LitTestRunner}/ConvertingFromLitToXunit.cs | 2 +- .../LitTestRunner/LitTests.cs | 5 ++--- .../TestFiles/HelloWorldLitTest.dfy | 0 .../TestFiles/VerifyOnlyLitTest.dfy | 0 6 files changed, 10 insertions(+), 5 deletions(-) rename Source/{LitTestConvertor.Test/LitTestConvertor.Test.csproj => IntegrationTests/IntegrationTests.csproj} (84%) rename Source/{LitTestConvertor.Test/Conversion => IntegrationTests/LitTestRunner}/ConvertingFromLitToXunit.cs (92%) rename Source/{LitTestConvertor.Test => IntegrationTests}/LitTestRunner/LitTests.cs (93%) rename Source/{LitTestConvertor.Test => IntegrationTests}/TestFiles/HelloWorldLitTest.dfy (100%) rename Source/{LitTestConvertor.Test => IntegrationTests}/TestFiles/VerifyOnlyLitTest.dfy (100%) diff --git a/Source/Dafny.sln b/Source/Dafny.sln index 8644dde97ce..c2ec08b714e 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -20,7 +20,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyDriver.Test", "DafnyDriver.Test\DafnyDriver.Test.csproj", "{08A3665B-9854-4CF6-BF5F-C532DA39A043}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LitTestConvertor.Test", "LitTestConvertor.Test\LitTestConvertor.Test.csproj", "{A4BC2C77-3A48-4917-AA22-41978DFFE24E}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntegrationTests", "IntegrationTests\IntegrationTests.csproj", "{A4BC2C77-3A48-4917-AA22-41978DFFE24E}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DafnyLanguageServer", "DafnyLanguageServer\DafnyLanguageServer.csproj", "{CD05D26C-C672-4F43-835E-7A3E1741E4D8}" EndProject diff --git a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj b/Source/IntegrationTests/IntegrationTests.csproj similarity index 84% rename from Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj rename to Source/IntegrationTests/IntegrationTests.csproj index cea64e18788..d3a6aa34188 100644 --- a/Source/LitTestConvertor.Test/LitTestConvertor.Test.csproj +++ b/Source/IntegrationTests/IntegrationTests.csproj @@ -19,6 +19,12 @@ + + + + + + diff --git a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs b/Source/IntegrationTests/LitTestRunner/ConvertingFromLitToXunit.cs similarity index 92% rename from Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs rename to Source/IntegrationTests/LitTestRunner/ConvertingFromLitToXunit.cs index 30e47ad214d..b6116c4a319 100644 --- a/Source/LitTestConvertor.Test/Conversion/ConvertingFromLitToXunit.cs +++ b/Source/IntegrationTests/LitTestRunner/ConvertingFromLitToXunit.cs @@ -4,7 +4,7 @@ using Xunit; using XUnitExtensions; -namespace LitTestConvertor.Test { +namespace IntegrationTests.LitTestRunner { public class ConvertingFromLitToXunit { [Fact] diff --git a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs b/Source/IntegrationTests/LitTestRunner/LitTests.cs similarity index 93% rename from Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs rename to Source/IntegrationTests/LitTestRunner/LitTests.cs index 811399606f8..a0d440bc0b8 100644 --- a/Source/LitTestConvertor.Test/LitTestRunner/LitTests.cs +++ b/Source/IntegrationTests/LitTestRunner/LitTests.cs @@ -1,5 +1,4 @@ using System; -using System.Collections; using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -9,7 +8,7 @@ [assembly: TestCollectionOrderer("XUnitExtensions.TestCollectionShardFilter", "XUnitExtensions")] -namespace LitTestConvertor.Test.LitTestRunner { +namespace IntegrationTests.LitTestRunner { public class LitTests { private static readonly Assembly dafnyDriverAssembly = typeof(DafnyDriver).Assembly; @@ -41,7 +40,7 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable MainWithArguments(dafnyDriverAssembly, defaultDafnyArguments.Concat(args), config, true) }, { "%server", (args, config) => - MainWithArguments(dafnyDriverAssembly, args, config, false) }, + MainWithArguments(dafnyServerAssembly, args, config, false) }, }, // TODO: speed this up by using AssertWithDiff diff --git a/Source/LitTestConvertor.Test/TestFiles/HelloWorldLitTest.dfy b/Source/IntegrationTests/TestFiles/HelloWorldLitTest.dfy similarity index 100% rename from Source/LitTestConvertor.Test/TestFiles/HelloWorldLitTest.dfy rename to Source/IntegrationTests/TestFiles/HelloWorldLitTest.dfy diff --git a/Source/LitTestConvertor.Test/TestFiles/VerifyOnlyLitTest.dfy b/Source/IntegrationTests/TestFiles/VerifyOnlyLitTest.dfy similarity index 100% rename from Source/LitTestConvertor.Test/TestFiles/VerifyOnlyLitTest.dfy rename to Source/IntegrationTests/TestFiles/VerifyOnlyLitTest.dfy From 79d28acee25ed4cf4378748cae5d334546f32f3d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 4 Oct 2021 12:31:14 -0700 Subject: [PATCH 128/192] Fix bad merge conflict resolution --- Source/Dafny/DafnyPipeline.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Dafny/DafnyPipeline.csproj b/Source/Dafny/DafnyPipeline.csproj index 9e5849e5ebb..4bf8b7e69ac 100644 --- a/Source/Dafny/DafnyPipeline.csproj +++ b/Source/Dafny/DafnyPipeline.csproj @@ -35,7 +35,7 @@ - + From 347d1ddcc50aa1f74cee2e14179b093df6c9b2b9 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 4 Oct 2021 14:16:49 -0700 Subject: [PATCH 129/192] Cleanup --- Source/Dafny.sln | 27 +----------- .../DafnyDriver.Test/DafnyDriver.Test.csproj | 36 ---------------- Source/DafnyDriver.Test/README.md | 43 ------------------- .../DafnyLanguageServer.Test.csproj | 17 +++++++- .../LitTestRunner/LitTests.cs | 2 +- 5 files changed, 18 insertions(+), 107 deletions(-) delete mode 100644 Source/DafnyDriver.Test/DafnyDriver.Test.csproj delete mode 100644 Source/DafnyDriver.Test/README.md diff --git a/Source/Dafny.sln b/Source/Dafny.sln index a39df1b0271..65857ee9011 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -18,8 +18,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution version.cs = version.cs EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyDriver.Test", "DafnyDriver.Test\DafnyDriver.Test.csproj", "{08A3665B-9854-4CF6-BF5F-C532DA39A043}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntegrationTests", "IntegrationTests\IntegrationTests.csproj", "{A4BC2C77-3A48-4917-AA22-41978DFFE24E}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DafnyLanguageServer", "DafnyLanguageServer\DafnyLanguageServer.csproj", "{CD05D26C-C672-4F43-835E-7A3E1741E4D8}" @@ -66,34 +64,10 @@ Global {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Debug|Any CPU.Build.0 = Debug|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Any CPU.ActiveCfg = Release|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Any CPU.Build.0 = Release|Any CPU - {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|x86.ActiveCfg = Release|Any CPU - {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|x86.Build.0 = Release|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|.NET.ActiveCfg = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|.NET.Build.0 = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|Any CPU.ActiveCfg = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|Any CPU.Build.0 = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|Mixed Platforms.ActiveCfg = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|Mixed Platforms.Build.0 = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|x86.ActiveCfg = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Checked|x86.Build.0 = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|.NET.ActiveCfg = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|.NET.Build.0 = Debug|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Any CPU.Build.0 = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|x86.ActiveCfg = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|x86.Build.0 = Debug|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|.NET.ActiveCfg = Release|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|.NET.Build.0 = Release|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Any CPU.ActiveCfg = Release|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Any CPU.Build.0 = Release|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Mixed Platforms.Build.0 = Release|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|x86.ActiveCfg = Release|Any CPU - {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|x86.Build.0 = Release|Any CPU {229E0F23-2B99-4331-8BD8-C2EBCB7C9DAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {229E0F23-2B99-4331-8BD8-C2EBCB7C9DAC}.Debug|Any CPU.Build.0 = Debug|Any CPU {229E0F23-2B99-4331-8BD8-C2EBCB7C9DAC}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -102,6 +76,7 @@ Global {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Debug|Any CPU.Build.0 = Debug|Any CPU {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Release|Any CPU.ActiveCfg = Release|Any CPU {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Release|Any CPU.Build.0 = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj b/Source/DafnyDriver.Test/DafnyDriver.Test.csproj deleted file mode 100644 index 410adebeff3..00000000000 --- a/Source/DafnyDriver.Test/DafnyDriver.Test.csproj +++ /dev/null @@ -1,36 +0,0 @@ - - - - net5.0 - DafnyDriver.Test - - - - - - - - - - - - - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - - - diff --git a/Source/DafnyDriver.Test/README.md b/Source/DafnyDriver.Test/README.md deleted file mode 100644 index 410e8791240..00000000000 --- a/Source/DafnyDriver.Test/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Dafny CLI Testing - -Dafny test cases are based on single Dafny source files together with the expected output from the `dafny` CLI tool. If the source file is named `Foo.dfy`, the expected output will contained in the file named `Foo.dfy.expect`. - -The default behaviour is to assert that the source file verifies successfully and, for each currently-supported target language, can be compiled and run to produce the expected output. When the behaviour is not 100% consistent between different target languages, expected discrepancies can be recorded in additional files with the name `Foo.dfy..expect`. For example: [Test/comp/NativeNumbers.dfy.js.expect](Test/comp/NativeNumbers.dfy.js.expect). Such exceptions are automatically flagged as "known issues" and classified as a "skipped/ignored" test. - -## Test configuration syntax - -Test cases are configured and run using xUnit's support for parameterized tests, with extensions for running test cases in parallel. The sets of options passed to `dafny` can be configured using YAML embedded in the first multi-line comment in the source file. Lists of values are interpreted as multiple parameterizations. For example: [Test/dafny4/git-issue250.dfy](Test/dafny4/git-issue250.dfy). - -For details and more configuration options, see [the DafnyTests.cs source](Test/DafnyTests/DafnyTests.cs). - - -```yaml -!dafnyTestSpec -options: - compile: 3 - coverage: "-" -compileTargetOverrides: - java: - otherFiles: - - CodeCoverage.java - cs: - otherFiles: - - BranchCoverage2.cs - js: - otherFiles: - - BranchCoverage3.js - go: - otherFiles: - - BranchCoverage4.go -``` - -## TODO - -* More complete documentation about options (in this file or in the source code) -* Depend on only the project's output directory instead of the Binaries/Test directories - * This is mostly working except for errors around missing types from System.dll when compiling to C# -* Add support for regular expression matching against CLI output (needed to assert known limitations that cause errors with things like absolute paths names in them) -* Expose test case options as traits so that they can be filtered on (e.g. `dotnet test --filter compileTarget=java`) -* Convert all the existing lit-based test cases under Test/ to this format - * The `LitTestConvertor` package is able to understand common lit command patterns and convert them into the new YAML format automatically. -* Extract most of the xUnit extensions as a separate package, since most of it is generically useful for any other data-driven xUnit tests, especially CLIs. diff --git a/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj b/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj index 831771218d7..9c45bb2404c 100644 --- a/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj +++ b/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj @@ -21,7 +21,22 @@ - + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + PreserveNewest diff --git a/Source/IntegrationTests/LitTestRunner/LitTests.cs b/Source/IntegrationTests/LitTestRunner/LitTests.cs index a0d440bc0b8..ba0e7c412a1 100644 --- a/Source/IntegrationTests/LitTestRunner/LitTests.cs +++ b/Source/IntegrationTests/LitTestRunner/LitTests.cs @@ -38,7 +38,7 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable MainWithArguments(dafnyDriverAssembly, args, config, false) }, { "%dafny", (args, config) => - MainWithArguments(dafnyDriverAssembly, defaultDafnyArguments.Concat(args), config, true) }, + MainWithArguments(dafnyDriverAssembly, defaultDafnyArguments.Concat(args), config, false) }, { "%server", (args, config) => MainWithArguments(dafnyServerAssembly, args, config, false) }, }, From efe3d9bc2e24f4ceabead6d92cc49f50e1ab788c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 4 Oct 2021 14:23:38 -0700 Subject: [PATCH 130/192] More clean up, try enabling tests in CI to measure time --- .github/workflows/msbuild.yml | 8 +++---- .../LitTestRunner/ConvertingFromLitToXunit.cs | 22 ------------------- .../TestFiles/HelloWorldLitTest.dfy | 13 ----------- .../TestFiles/VerifyOnlyLitTest.dfy | 6 ----- Test/git-issues/github-issue-1111.dfy | 0 5 files changed, 4 insertions(+), 45 deletions(-) delete mode 100644 Source/IntegrationTests/LitTestRunner/ConvertingFromLitToXunit.cs delete mode 100644 Source/IntegrationTests/TestFiles/HelloWorldLitTest.dfy delete mode 100644 Source/IntegrationTests/TestFiles/VerifyOnlyLitTest.dfy delete mode 100644 Test/git-issues/github-issue-1111.dfy diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index be45e4f4091..c37f7503306 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -3,8 +3,8 @@ name: Build and Test on: push: branches: [ master ] -# pull_request: -# branches: [ master ] + pull_request: + branches: [ master ] env: dotnet-version: 5.0.x # SDK Version for building Dafny; x will use the latest version of the 5.0 channel @@ -126,5 +126,5 @@ jobs: unzip dafny/Package/CI.zip -d unzippedRelease - name: Run lit tests run: | - ## lit in this context needs the executables specified - lit --time-tests -v --num-shards=5 --run-shard=${{ matrix.shard }} --param executable=$PWD/unzippedRelease/dafny/dafny --param serverExecutable=$PWD/unzippedRelease/dafny/DafnyServer dafny/Test + # TODO: Configure $dafny to point to the released version we just create + XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 dotnet test dafny/Source/IntegrationTests/IntegrationTests.csproj diff --git a/Source/IntegrationTests/LitTestRunner/ConvertingFromLitToXunit.cs b/Source/IntegrationTests/LitTestRunner/ConvertingFromLitToXunit.cs deleted file mode 100644 index b6116c4a319..00000000000 --- a/Source/IntegrationTests/LitTestRunner/ConvertingFromLitToXunit.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using Xunit; -using XUnitExtensions; - -namespace IntegrationTests.LitTestRunner { - public class ConvertingFromLitToXunit { - - [Fact] - public void HelloWorld() { - var config = new LitTestConfiguration(); - var program = LitTestCase.Read("TestFiles/HelloWorldLitTest.dfy", config); - } - - [Fact] - public void VerifyOnly() { - var config = new LitTestConfiguration(); - var program = LitTestCase.Read("TestFiles/VerifyOnly.dfy", config); - } - } -} \ No newline at end of file diff --git a/Source/IntegrationTests/TestFiles/HelloWorldLitTest.dfy b/Source/IntegrationTests/TestFiles/HelloWorldLitTest.dfy deleted file mode 100644 index 3fca71de079..00000000000 --- a/Source/IntegrationTests/TestFiles/HelloWorldLitTest.dfy +++ /dev/null @@ -1,13 +0,0 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: %diff "%s.expect" "%t" - -method Main() { - print "Hello, World! Best, Dafny\n"; - var x := 14; - print "x is ", x, "\n"; - var y := false; - print "y is ", y, "\n"; -} diff --git a/Source/IntegrationTests/TestFiles/VerifyOnlyLitTest.dfy b/Source/IntegrationTests/TestFiles/VerifyOnlyLitTest.dfy deleted file mode 100644 index 8b2364aa0f2..00000000000 --- a/Source/IntegrationTests/TestFiles/VerifyOnlyLitTest.dfy +++ /dev/null @@ -1,6 +0,0 @@ -// RUN: %dafny /compile:0 "%s" > "%t" -// RUN: %diff "%s.expect" "%t" - -method Main() { - print "Hello, World! Best, Dafny\n"; -} diff --git a/Test/git-issues/github-issue-1111.dfy b/Test/git-issues/github-issue-1111.dfy deleted file mode 100644 index e69de29bb2d..00000000000 From 737ccdeb407f51992d4e332e994a966eca39227d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 4 Oct 2021 15:18:13 -0700 Subject: [PATCH 131/192] Move non-compiled runtime files into DafnyRuntime project --- Source/Dafny/DafnyPipeline.csproj | 3 --- Source/DafnyRuntime/DafnyRuntime.csproj | 3 +++ {Binaries => Source/DafnyRuntime}/DafnyRuntime.go | 0 {Binaries => Source/DafnyRuntime}/DafnyRuntime.h | 0 {Binaries => Source/DafnyRuntime}/DafnyRuntime.js | 0 5 files changed, 3 insertions(+), 3 deletions(-) rename {Binaries => Source/DafnyRuntime}/DafnyRuntime.go (100%) rename {Binaries => Source/DafnyRuntime}/DafnyRuntime.h (100%) rename {Binaries => Source/DafnyRuntime}/DafnyRuntime.js (100%) diff --git a/Source/Dafny/DafnyPipeline.csproj b/Source/Dafny/DafnyPipeline.csproj index 4bf8b7e69ac..631a3d8b17d 100644 --- a/Source/Dafny/DafnyPipeline.csproj +++ b/Source/Dafny/DafnyPipeline.csproj @@ -37,8 +37,5 @@ - - - diff --git a/Source/DafnyRuntime/DafnyRuntime.csproj b/Source/DafnyRuntime/DafnyRuntime.csproj index 04b202d9dbe..e9deb8edb4c 100755 --- a/Source/DafnyRuntime/DafnyRuntime.csproj +++ b/Source/DafnyRuntime/DafnyRuntime.csproj @@ -20,6 +20,9 @@ Always + + + diff --git a/Binaries/DafnyRuntime.go b/Source/DafnyRuntime/DafnyRuntime.go similarity index 100% rename from Binaries/DafnyRuntime.go rename to Source/DafnyRuntime/DafnyRuntime.go diff --git a/Binaries/DafnyRuntime.h b/Source/DafnyRuntime/DafnyRuntime.h similarity index 100% rename from Binaries/DafnyRuntime.h rename to Source/DafnyRuntime/DafnyRuntime.h diff --git a/Binaries/DafnyRuntime.js b/Source/DafnyRuntime/DafnyRuntime.js similarity index 100% rename from Binaries/DafnyRuntime.js rename to Source/DafnyRuntime/DafnyRuntime.js From 8b851c8bda024079d279094f51cdb9e0f0c4e60c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 6 Oct 2021 19:31:16 -0700 Subject: [PATCH 132/192] Fix packaging and unit test job --- .github/workflows/xunit-tests.yml | 8 ++++++-- Scripts/package.py | 4 +--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/xunit-tests.yml b/.github/workflows/xunit-tests.yml index 7b583d5c96d..542bae4e54f 100644 --- a/.github/workflows/xunit-tests.yml +++ b/.github/workflows/xunit-tests.yml @@ -62,5 +62,9 @@ jobs: chmod +x ${{env.testPath1}}/z3/bin/z3 chmod +x ${{env.testPath2}}/z3/bin/z3 chmod +x ${{env.testPath3}}/z3/bin/z3 - - name: Run Tests - run: dotnet test --no-restore --verbosity normal ${{env.solutionPath}} + - name: Run DafnyLanguageServer Tests + run: dotnet test --no-restore --verbosity normal Source/DafnyLanguageServer.Test + - name: Run DafnyPipeline Tests + run: dotnet test --no-restore --verbosity normal Source/DafnyPipeline.Test + - name: Run DafnyTestGeneration Tests + run: dotnet test --no-restore --verbosity normal Source/DafnyTestGeneration.Test diff --git a/Scripts/package.py b/Scripts/package.py index a9ffb413342..4d5e30ba037 100755 --- a/Scripts/package.py +++ b/Scripts/package.py @@ -47,8 +47,6 @@ ## On unix systems, which Dafny files should be marked as executable? (Glob syntax; Z3's permissions are preserved) UNIX_EXECUTABLES = ["dafny", "dafny-server"] -ETCs = ["DafnyPrelude.bpl", "DafnyRuntime.js", "DafnyRuntime.go", "DafnyRuntime.jar", "DafnyRuntime.h"] - # Constants THIS_FILE = path.realpath(__file__) @@ -166,7 +164,7 @@ def pack(self): lowercaseDafny = path.join(self.buildDirectory, "dafny") shutil.move(uppercaseDafny, lowercaseDafny) os.chmod(lowercaseDafny, stat.S_IEXEC| os.lstat(lowercaseDafny).st_mode) - paths = pathsInDirectory(self.buildDirectory) + list(map(lambda etc: path.join(BINARIES_DIRECTORY, etc), ETCs)) + OTHERS + paths = pathsInDirectory(self.buildDirectory) + OTHERS for fpath in paths: if os.path.isdir(fpath): continue From e2c5838326d55b7ead57b24ef7f612374d489b8c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 9 Oct 2021 10:44:52 -0700 Subject: [PATCH 133/192] Build java runtime before unit tests --- .github/workflows/xunit-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/xunit-tests.yml b/.github/workflows/xunit-tests.yml index 542bae4e54f..a515ea6c128 100644 --- a/.github/workflows/xunit-tests.yml +++ b/.github/workflows/xunit-tests.yml @@ -62,6 +62,8 @@ jobs: chmod +x ${{env.testPath1}}/z3/bin/z3 chmod +x ${{env.testPath2}}/z3/bin/z3 chmod +x ${{env.testPath3}}/z3/bin/z3 + - name: Build Java runtime + run: make --quiet runtime - name: Run DafnyLanguageServer Tests run: dotnet test --no-restore --verbosity normal Source/DafnyLanguageServer.Test - name: Run DafnyPipeline Tests From 9ca99a9b047309bbabd30d492b494ccb9335b565 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 9 Oct 2021 10:55:05 -0700 Subject: [PATCH 134/192] Move building Java runtime earlier --- .github/workflows/xunit-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/xunit-tests.yml b/.github/workflows/xunit-tests.yml index a515ea6c128..322188cd6e9 100644 --- a/.github/workflows/xunit-tests.yml +++ b/.github/workflows/xunit-tests.yml @@ -45,6 +45,8 @@ jobs: dotnet-version: 5.0.x - name: Install dependencies run: dotnet restore ${{env.solutionPath}} + - name: Build Java runtime + run: make --quiet runtime - name: Build run: dotnet build --no-restore ${{env.solutionPath}} - name: Load Z3 @@ -62,8 +64,6 @@ jobs: chmod +x ${{env.testPath1}}/z3/bin/z3 chmod +x ${{env.testPath2}}/z3/bin/z3 chmod +x ${{env.testPath3}}/z3/bin/z3 - - name: Build Java runtime - run: make --quiet runtime - name: Run DafnyLanguageServer Tests run: dotnet test --no-restore --verbosity normal Source/DafnyLanguageServer.Test - name: Run DafnyPipeline Tests From 6a91afa9283488a4b717514a4b9e13789824c832 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 9 Oct 2021 11:02:23 -0700 Subject: [PATCH 135/192] Create Output directory explicitly --- Source/XUnitExtensions/LitCommandWithOutput.cs | 2 +- Source/XUnitExtensions/LitTestCase.cs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Source/XUnitExtensions/LitCommandWithOutput.cs b/Source/XUnitExtensions/LitCommandWithOutput.cs index e1d0f8d65f5..d914f4da847 100644 --- a/Source/XUnitExtensions/LitCommandWithOutput.cs +++ b/Source/XUnitExtensions/LitCommandWithOutput.cs @@ -22,7 +22,7 @@ public override string ToString() { var builder = new StringBuilder(); builder.Append(command); if (outputFile != null) { - builder.Append(append ? " >> " : ">"); + builder.Append(append ? " >> " : " > "); builder.Append(outputFile); } return builder.ToString(); diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index ed132078d3c..caff32e1d07 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -20,6 +20,8 @@ public static void Run(string filePath, LitTestConfiguration config) { { "%s", fileName }, { "%t", Path.Join("Output", $"{fileName}.tmp")} }); + + Directory.CreateDirectory(Path.Join(Path.GetDirectoryName(filePath), "Output")); var commands = Read(filePath, config); foreach (var command in commands) { From b273512460322799968bbcb2038092fcdbe68a50 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 10 Oct 2021 07:57:21 -0700 Subject: [PATCH 136/192] Remove unnecessary namespace --- Source/IntegrationTests/{LitTestRunner => }/LitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Source/IntegrationTests/{LitTestRunner => }/LitTests.cs (98%) diff --git a/Source/IntegrationTests/LitTestRunner/LitTests.cs b/Source/IntegrationTests/LitTests.cs similarity index 98% rename from Source/IntegrationTests/LitTestRunner/LitTests.cs rename to Source/IntegrationTests/LitTests.cs index ba0e7c412a1..6bfd244e654 100644 --- a/Source/IntegrationTests/LitTestRunner/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -8,7 +8,7 @@ [assembly: TestCollectionOrderer("XUnitExtensions.TestCollectionShardFilter", "XUnitExtensions")] -namespace IntegrationTests.LitTestRunner { +namespace IntegrationTests { public class LitTests { private static readonly Assembly dafnyDriverAssembly = typeof(DafnyDriver).Assembly; From 64d8b20ca2d5ebeb58fda95336b0cc5fa3e7d2cf Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 10 Oct 2021 08:10:32 -0700 Subject: [PATCH 137/192] Environment variables for pointing to released executables --- .github/workflows/msbuild.yml | 5 ++--- Source/IntegrationTests/LitTests.cs | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index c37f7503306..5558a3996c3 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -124,7 +124,6 @@ jobs: - if: runner.os != 'Windows' run: | unzip dafny/Package/CI.zip -d unzippedRelease - - name: Run lit tests + - name: Run integration tests run: | - # TODO: Configure $dafny to point to the released version we just create - XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 dotnet test dafny/Source/IntegrationTests/IntegrationTests.csproj + XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 DAFNY_EXECUTABLE=$PWD/unzippedRelease/dafny/dafny DAFNY_SERVER_EXECUTABLE=$PWD/unzippedRelease/dafny/dafnyServer dotnet test dafny/Source/IntegrationTests/IntegrationTests.csproj diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 6bfd244e654..f4ea55cc791 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -29,7 +29,6 @@ public class LitTests { }; private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable arguments, LitTestConfiguration config, bool invokeDirectly) { - return MainMethodLitCommand.Parse(assembly, arguments, config, invokeDirectly); } @@ -50,6 +49,21 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable + new ShellLitCommand(dafnyExecutable, args, config.PassthroughEnvironmentVariables); + CONFIG.Commands["%dafny"] = (args, config) => + new ShellLitCommand(dafnyExecutable, defaultDafnyArguments.Concat(args), config.PassthroughEnvironmentVariables); + } + var dafnyServerExecutable = Environment.GetEnvironmentVariable("DAFNY_SERVER_EXECUTABLE"); + if (dafnyServerExecutable != null) { + CONFIG.Commands["%server"] = (args, config) => + new ShellLitCommand(dafnyServerExecutable, args, config.PassthroughEnvironmentVariables); + } + } [FileTheory] [LitTestData(Extension = ".dfy")] From ddb84b7191bd6d4f4691591e04f5a2aa6a5f2b1b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 12 Oct 2021 16:19:07 -0700 Subject: [PATCH 138/192] Missing solution line --- Source/Dafny.sln | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/Dafny.sln b/Source/Dafny.sln index 65857ee9011..d84bc7e3f15 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -77,6 +77,7 @@ Global {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Release|Any CPU.ActiveCfg = Release|Any CPU {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Release|Any CPU.Build.0 = Release|Any CPU {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.Build.0 = Debug|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 3627445ecc2fb82deb15fe5ae61363c882b47db0 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 12 Oct 2021 17:31:34 -0700 Subject: [PATCH 139/192] Improve exception message on non-zero exit code --- Source/XUnitExtensions/LitTestCase.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index caff32e1d07..10145462b62 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -33,7 +33,10 @@ public static void Run(string filePath, LitTestConfiguration config) { } catch (Exception e) { throw new Exception($"Exception thrown while executing command: {command}", e); } - Assert.True(exitCode == 0, $"Command failed with output:\n{output}\nError:\n{error}"); + + if (exitCode != 0) { + throw new Exception($"Command returned non-zero exit code ({exitCode}): {command}\nOutput:\n{output}\nError:\n{error}"); + } } } } From 901dcb431cad92b56d1364e14ef03a618d658bac Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 13 Oct 2021 09:01:18 -0700 Subject: [PATCH 140/192] Removed use of sed from test --- Test/git-issues/github-issue-305-b.dfy | 4 +--- Test/git-issues/github-issue-305-b.dfy.expect | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Test/git-issues/github-issue-305-b.dfy b/Test/git-issues/github-issue-305-b.dfy index 6520077a8f3..0ff3e7eb2df 100644 --- a/Test/git-issues/github-issue-305-b.dfy +++ b/Test/git-issues/github-issue-305-b.dfy @@ -1,6 +1,4 @@ -// UNSUPPORTED: windows -// RUN: %baredafny /countVerificationErrors:0 /compile:0 /spillTargetCode:2 "%s" > "%t.2" -// RUN: sed -e "s:%S:...:" -e 'sx\\x/x' < "%t.2" > "%t" +// RUN: %dafny /countVerificationErrors:0 /compile:0 /spillTargetCode:2 /useBaseNameForFileName "%s" > "%t" // RUN: %diff "%s".expect "%t" method hasNoBody() diff --git a/Test/git-issues/github-issue-305-b.dfy.expect b/Test/git-issues/github-issue-305-b.dfy.expect index 5900ae61de3..886f0b876f8 100644 --- a/Test/git-issues/github-issue-305-b.dfy.expect +++ b/Test/git-issues/github-issue-305-b.dfy.expect @@ -1,4 +1,4 @@ Dafny program verifier finished with 1 verified, 0 errors -.../github-issue-305-b.dfy(6,7): Error: Method _module._default.hasNoBody has no body +github-issue-305-b.dfy(4,7): Error: Method _module._default.hasNoBody has no body Wrote textual form of partial target program to github-issue-305-b.cs From a17ea03f48cfd532a670708208f7ff16b6a144dd Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 13 Oct 2021 11:19:49 -0700 Subject: [PATCH 141/192] Use %refmanexamples to avoid relative paths --- .../IntegrationTests/IntegrationTests.csproj | 4 +++- Source/IntegrationTests/LitTests.cs | 4 +++- Test/lit.site.cfg | 1 + Test/refman/Example-BV.dfy | 4 +--- Test/refman/Example-BV.dfy.expect | 3 +-- Test/refman/Example-BV2.dfy | 4 +--- Test/refman/Example-BV3.dfy | 4 +--- Test/refman/Example-BV3.dfy.expect | 1 - Test/refman/Example-BV3a.dfy | 4 +--- Test/refman/Example-BV3a.dfy.expect | 3 +-- Test/refman/Example-BV4.dfy | 4 +--- Test/refman/Example-BV4a.dfy | 4 +--- Test/refman/Example-BV4a.dfy.expect | 3 +-- Test/refman/Example-Fail1.dfy | 4 +--- Test/refman/Example-Fail2.dfy | 4 +--- Test/refman/Example-Old.dfy | 4 +--- Test/refman/Example-Old2.dfy | 4 +--- Test/refman/Example-Old3.dfy | 4 +--- Test/refman/Example-RM.dfy | 4 +--- Test/refman/Example-Refines1.dfy | 4 +--- Test/refman/Example-TP.dfy | 4 +--- Test/refman/Example-TP.dfy.expect | 3 +-- Test/refman/Example-TwoState.dfy | 18 +----------------- docs/DafnyRef/examples/Example-TwoState.dfy | 18 ++++++++++++++++++ 24 files changed, 44 insertions(+), 70 deletions(-) create mode 100644 docs/DafnyRef/examples/Example-TwoState.dfy diff --git a/Source/IntegrationTests/IntegrationTests.csproj b/Source/IntegrationTests/IntegrationTests.csproj index d3a6aa34188..c93abebebbb 100644 --- a/Source/IntegrationTests/IntegrationTests.csproj +++ b/Source/IntegrationTests/IntegrationTests.csproj @@ -26,10 +26,12 @@ - PreserveNewest + + PreserveNewest + PreserveNewest diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index f4ea55cc791..b1ea6ccb5e3 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Reflection; using Microsoft.Dafny; @@ -44,7 +45,8 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable { - { "%diff", "diff" } + { "%diff", "diff" }, + { "%refmanexamples", Path.Join("examples") } }, PassthroughEnvironmentVariables = new []{ "PATH", "HOME" }, diff --git a/Test/lit.site.cfg b/Test/lit.site.cfg index 79dfe84848d..0d31da08255 100644 --- a/Test/lit.site.cfg +++ b/Test/lit.site.cfg @@ -143,6 +143,7 @@ if os.name != "nt": config.substitutions.append( ('%dafny', dafnyExecutable) ) config.substitutions.append( ('%baredafny', bareDafnyExecutable) ) config.substitutions.append( ('%server', serverExecutable) ) +config.substitutions.append( ('%refmanexamples', os.path.join(repositoryRoot, 'docs', 'DafnyRef', 'examples')) ) config.substitutions.append( ('%os', os.name) ) config.substitutions.append( ('%ver', ver ) ) if os.name == "nt": diff --git a/Test/refman/Example-BV.dfy b/Test/refman/Example-BV.dfy index 3193a62ed35..3e9e45471be 100644 --- a/Test/refman/Example-BV.dfy +++ b/Test/refman/Example-BV.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-BV.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-BV.dfy" diff --git a/Test/refman/Example-BV.dfy.expect b/Test/refman/Example-BV.dfy.expect index 56c0f90a9ea..28c69a05cfa 100644 --- a/Test/refman/Example-BV.dfy.expect +++ b/Test/refman/Example-BV.dfy.expect @@ -1,3 +1,2 @@ -Example-BV.dfy(4,8): Error: the included file Example-BV.dfy contains error(s) Example-BV.dfy(3,16): Error: literal (5) is too large for the bitvector type bv2 -2 resolution/type errors detected in Example-BV.dfy +1 resolution/type errors detected in Example-BV.dfy diff --git a/Test/refman/Example-BV2.dfy b/Test/refman/Example-BV2.dfy index 33919883ffa..f66199c7cbc 100644 --- a/Test/refman/Example-BV2.dfy +++ b/Test/refman/Example-BV2.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-BV2.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-BV2.dfy" diff --git a/Test/refman/Example-BV3.dfy b/Test/refman/Example-BV3.dfy index 80ee757591e..80d51871b46 100644 --- a/Test/refman/Example-BV3.dfy +++ b/Test/refman/Example-BV3.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-BV3.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-BV3.dfy" diff --git a/Test/refman/Example-BV3.dfy.expect b/Test/refman/Example-BV3.dfy.expect index dc2aaf305b5..2ef059a5281 100644 --- a/Test/refman/Example-BV3.dfy.expect +++ b/Test/refman/Example-BV3.dfy.expect @@ -1,4 +1,3 @@ -Example-BV3.dfy(4,8): Error: the included file Example-BV3.dfy contains error(s) Example-BV3.dfy(5,15): Error: Ambiguous use of &, |, ^. Use parentheses to disambiguate. Example-BV3.dfy(5,17): Error: invalid AssertStmt 2 parse errors detected in Example-BV3.dfy diff --git a/Test/refman/Example-BV3a.dfy b/Test/refman/Example-BV3a.dfy index 73e8d30624f..0a15bdb54f2 100644 --- a/Test/refman/Example-BV3a.dfy +++ b/Test/refman/Example-BV3a.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-BV3a.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-BV3a.dfy" diff --git a/Test/refman/Example-BV3a.dfy.expect b/Test/refman/Example-BV3a.dfy.expect index fcb849d2383..3e13831ff2c 100644 --- a/Test/refman/Example-BV3a.dfy.expect +++ b/Test/refman/Example-BV3a.dfy.expect @@ -1,3 +1,2 @@ -Example-BV3a.dfy(4,8): Error: the included file Example-BV3a.dfy contains error(s) Example-BV3a.dfy(5,18): Error: arguments must have comparable types (got bv5 and bv6) -2 resolution/type errors detected in Example-BV3a.dfy +1 resolution/type errors detected in Example-BV3a.dfy diff --git a/Test/refman/Example-BV4.dfy b/Test/refman/Example-BV4.dfy index 2ea33e062b7..c43476b8a96 100644 --- a/Test/refman/Example-BV4.dfy +++ b/Test/refman/Example-BV4.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-BV4.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-BV4.dfy" diff --git a/Test/refman/Example-BV4a.dfy b/Test/refman/Example-BV4a.dfy index c9953f79088..3be99c692a9 100644 --- a/Test/refman/Example-BV4a.dfy +++ b/Test/refman/Example-BV4a.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-BV4a.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-BV4a.dfy" diff --git a/Test/refman/Example-BV4a.dfy.expect b/Test/refman/Example-BV4a.dfy.expect index 735287b4160..6f491b5f83d 100644 --- a/Test/refman/Example-BV4a.dfy.expect +++ b/Test/refman/Example-BV4a.dfy.expect @@ -1,3 +1,2 @@ -Example-BV4a.dfy(4,8): Error: the included file Example-BV4a.dfy contains error(s) Example-BV4a.dfy(4,14): Error: literal (25) is too large for the bitvector type bv4 -2 resolution/type errors detected in Example-BV4a.dfy +1 resolution/type errors detected in Example-BV4a.dfy diff --git a/Test/refman/Example-Fail1.dfy b/Test/refman/Example-Fail1.dfy index 6a554b6dc76..ffc862a059a 100644 --- a/Test/refman/Example-Fail1.dfy +++ b/Test/refman/Example-Fail1.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-Fail1.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-Fail1.dfy" diff --git a/Test/refman/Example-Fail2.dfy b/Test/refman/Example-Fail2.dfy index 2aeb6a61f50..46e02553270 100644 --- a/Test/refman/Example-Fail2.dfy +++ b/Test/refman/Example-Fail2.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-Fail2.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-Fail2.dfy" diff --git a/Test/refman/Example-Old.dfy b/Test/refman/Example-Old.dfy index c7885310028..70dc1cb3b13 100644 --- a/Test/refman/Example-Old.dfy +++ b/Test/refman/Example-Old.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-Old.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-Old.dfy" diff --git a/Test/refman/Example-Old2.dfy b/Test/refman/Example-Old2.dfy index f039906e3a1..49d5dba8fec 100644 --- a/Test/refman/Example-Old2.dfy +++ b/Test/refman/Example-Old2.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-Old2.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-Old2.dfy" diff --git a/Test/refman/Example-Old3.dfy b/Test/refman/Example-Old3.dfy index 21f3f15a2f6..3fc3d6a817c 100644 --- a/Test/refman/Example-Old3.dfy +++ b/Test/refman/Example-Old3.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-Old3.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-Old3.dfy" diff --git a/Test/refman/Example-RM.dfy b/Test/refman/Example-RM.dfy index 3ca112f5067..1868d4baa77 100644 --- a/Test/refman/Example-RM.dfy +++ b/Test/refman/Example-RM.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-RM.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-RM.dfy" diff --git a/Test/refman/Example-Refines1.dfy b/Test/refman/Example-Refines1.dfy index 803ab0cc2fc..6eea84e5f22 100644 --- a/Test/refman/Example-Refines1.dfy +++ b/Test/refman/Example-Refines1.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-Refines1.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-Refines1.dfy" diff --git a/Test/refman/Example-TP.dfy b/Test/refman/Example-TP.dfy index 436623cdc42..c60c1e9c4f1 100644 --- a/Test/refman/Example-TP.dfy +++ b/Test/refman/Example-TP.dfy @@ -1,4 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /compile:0 "%refmanexamples/Example-TP.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-TP.dfy" diff --git a/Test/refman/Example-TP.dfy.expect b/Test/refman/Example-TP.dfy.expect index 621e8e98509..fdcb26204e0 100644 --- a/Test/refman/Example-TP.dfy.expect +++ b/Test/refman/Example-TP.dfy.expect @@ -1,4 +1,3 @@ -Example-TP.dfy(4,8): Error: the included file Example-TP.dfy contains error(s) Example-TP.dfy(10,10): Error: type parameter (T) passed to type ResultN must support no references (got C) Example-TP.dfy(12,10): Error: type parameter (T) passed to type ResultN must support no references (got array) -3 resolution/type errors detected in Example-TP.dfy +2 resolution/type errors detected in Example-TP.dfy diff --git a/Test/refman/Example-TwoState.dfy b/Test/refman/Example-TwoState.dfy index 7a9cf0f3f8b..32fc5161de9 100644 --- a/Test/refman/Example-TwoState.dfy +++ b/Test/refman/Example-TwoState.dfy @@ -1,18 +1,2 @@ -// RUN: %dafny /verifyAllModules /compile:0 "%s" > "%t" +// RUN: %dafny /verifyAllModules /compile:0 "%refmanexamples/Example-TwoState.dfy" > "%t" // RUN: %diff "%s.expect" "%t" - -include "../../docs/DafnyRef/examples/Example-TwoState-Increasing.dfy" -include "../../docs/DafnyRef/examples/Example-TwoState-Caller.dfy" -include "../../docs/DafnyRef/examples/Example-TwoState-Diff.dfy" -include "../../docs/DafnyRef/examples/Example-TwoState-DiffAgain.dfy" -include "../../docs/DafnyRef/examples/Example-TwoState-SeqSum.dfy" -include "../../docs/DafnyRef/examples/Example-TwoState-EtaExample.dfy" - -class Cell { - var data: int - constructor (x: int) - ensures data == x - { - data := x; - } -} diff --git a/docs/DafnyRef/examples/Example-TwoState.dfy b/docs/DafnyRef/examples/Example-TwoState.dfy new file mode 100644 index 00000000000..98297f526d1 --- /dev/null +++ b/docs/DafnyRef/examples/Example-TwoState.dfy @@ -0,0 +1,18 @@ + +// This file only exists to include these examples as a test case. + +include "Example-TwoState-Increasing.dfy" +include "Example-TwoState-Caller.dfy" +include "Example-TwoState-Diff.dfy" +include "Example-TwoState-DiffAgain.dfy" +include "Example-TwoState-SeqSum.dfy" +include "Example-TwoState-EtaExample.dfy" + +class Cell { + var data: int + constructor (x: int) + ensures data == x + { + data := x; + } +} From 255152a5a6949a71f9b7f3deb5334de836058ffa Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 13 Oct 2021 11:59:30 -0700 Subject: [PATCH 142/192] Improved handling of current directory differences --- Source/IntegrationTests/LitTests.cs | 9 ++++++++- Source/XUnitExtensions/ILitCommand.cs | 8 +++++++- Source/XUnitExtensions/LitTestCase.cs | 12 ++++++++---- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index b1ea6ccb5e3..821cdcc7527 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -5,6 +5,7 @@ using System.Reflection; using Microsoft.Dafny; using Xunit; +using Xunit.Abstractions; using XUnitExtensions; [assembly: TestCollectionOrderer("XUnitExtensions.TestCollectionShardFilter", "XUnitExtensions")] @@ -67,10 +68,16 @@ static LitTests() { } } + private readonly ITestOutputHelper output; + + public LitTests(ITestOutputHelper output) { + this.output = output; + } + [FileTheory] [LitTestData(Extension = ".dfy")] public void LitTest(string path) { - LitTestCase.Run(path, CONFIG); + LitTestCase.Run(path, CONFIG, output); } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index 87dc3ee37d9..61dcff90671 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -56,7 +56,13 @@ public static string MakeFilePathsAbsolute(string basePath, string s) { if (s.StartsWith("-") || s.StartsWith("/")) { return s; } - return Path.Join(basePath, s); + + var absolutePath = Path.Join(basePath, s); + if (File.Exists(absolutePath)) { + return absolutePath; + } + + return s; } public static (IEnumerable, string, bool) ExtractOutputFile(IEnumerable arguments) { diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index 10145462b62..68e00dd5787 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using Xunit; +using Xunit.Abstractions; namespace XUnitExtensions { public class LitTestCase { @@ -14,14 +15,16 @@ public static IEnumerable Read(string filePath, LitTestCon .Where(c => c != null); } - public static void Run(string filePath, LitTestConfiguration config) { + public static void Run(string filePath, LitTestConfiguration config, ITestOutputHelper outputHelper) { string fileName = Path.GetFileName(filePath); + string directory = Path.GetDirectoryName(filePath); config = config.WithSubstitutions(new Dictionary { - { "%s", fileName }, - { "%t", Path.Join("Output", $"{fileName}.tmp")} + { "%s", filePath }, + { "%S", directory }, + { "%t", Path.Join(directory, "Output", $"{fileName}.tmp")} }); - Directory.CreateDirectory(Path.Join(Path.GetDirectoryName(filePath), "Output")); + Directory.CreateDirectory(Path.Join(directory, "Output")); var commands = Read(filePath, config); foreach (var command in commands) { @@ -29,6 +32,7 @@ public static void Run(string filePath, LitTestConfiguration config) { string output; string error; try { + outputHelper.WriteLine($"Executing command: {command}"); (exitCode, output, error) = command.Execute(); } catch (Exception e) { throw new Exception($"Exception thrown while executing command: {command}", e); From 82e3779387491f6e905a7acc79125019346a1e6c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 14 Oct 2021 08:59:49 -0700 Subject: [PATCH 143/192] Handle quotes in commands, fix a few more tests --- Source/XUnitExtensions/ILitCommand.cs | 30 +++++++++++++++++------ Test/dafny0/snapshots/Snapshots0.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots1.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots2.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots3.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots4.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots5.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots6.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots7.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots8.run.dfy | 2 +- Test/dafny0/snapshots/Snapshots9.run.dfy | 2 +- Test/git-issues/git-issue-801.dfy | 6 ----- Test/git-issues/git-issue-801.dfy.expect | 2 -- Test/git-issues/git-issue-801a.dfy | 6 ----- Test/git-issues/git-issue-801a.dfy.expect | 2 -- Test/git-issues/git-issue-801b.dfy | 6 ----- Test/git-issues/git-issue-801b.dfy.expect | 2 -- 17 files changed, 33 insertions(+), 41 deletions(-) delete mode 100644 Test/git-issues/git-issue-801.dfy delete mode 100644 Test/git-issues/git-issue-801.dfy.expect delete mode 100644 Test/git-issues/git-issue-801a.dfy delete mode 100644 Test/git-issues/git-issue-801a.dfy.expect delete mode 100644 Test/git-issues/git-issue-801b.dfy delete mode 100644 Test/git-issues/git-issue-801b.dfy.expect diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index 61dcff90671..8740b58a542 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Text; namespace XUnitExtensions { public interface ILitCommand { @@ -24,8 +25,7 @@ public static LitCommandWithOutput Parse(string fileName, string line, LitTestCo } line = line[LIT_COMMAND_PREFIX.Length..].Trim(); - // TODO: Probably need to handle escaping too - var pieces = line.Split((char[])null, StringSplitOptions.RemoveEmptyEntries).Select(StripQuotes).ToArray(); + var pieces = ParseArguments(line); var commandSymbol = pieces[0]; var basePath = Path.GetDirectoryName(fileName); var (rawArguments, outputFile, appendOutput) = ILitCommand.ExtractOutputFile(pieces[1..]); @@ -45,13 +45,29 @@ public static LitCommandWithOutput Parse(string fileName, string line, LitTestCo return new LitCommandWithOutput(new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), outputFile, appendOutput); } - private static string StripQuotes(string s) { - if (s.Length >= 2 && s[0] == '"' && s[^1] == '"') { - return s[1..^1]; + private static string[] ParseArguments(string line) { + var arguments = new List(); + var argument = new StringBuilder(); + var quoted = false; + for (int i = 0; i < line.Length; i++) { + var c = line[i]; + if (c == '"') { + quoted = !quoted; + } else if (Char.IsWhiteSpace(c) && !quoted) { + arguments.Add(argument.ToString()); + argument.Clear(); + } else { + argument.Append(c); + } } - return s; - } + if (argument.Length != 0) { + arguments.Add(argument.ToString()); + } + + return arguments.ToArray(); + } + public static string MakeFilePathsAbsolute(string basePath, string s) { if (s.StartsWith("-") || s.StartsWith("/")) { return s; diff --git a/Test/dafny0/snapshots/Snapshots0.run.dfy b/Test/dafny0/snapshots/Snapshots0.run.dfy index 4aeef2aae64..bd1c5c5df5e 100644 --- a/Test/dafny0/snapshots/Snapshots0.run.dfy +++ b/Test/dafny0/snapshots/Snapshots0.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots0.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 %S/Inputs/Snapshots0.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots1.run.dfy b/Test/dafny0/snapshots/Snapshots1.run.dfy index 5e8a323f8ad..439b6681fce 100644 --- a/Test/dafny0/snapshots/Snapshots1.run.dfy +++ b/Test/dafny0/snapshots/Snapshots1.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots1.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 %S/Inputs/Snapshots1.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots2.run.dfy b/Test/dafny0/snapshots/Snapshots2.run.dfy index 1658838a09e..36d8ae160af 100644 --- a/Test/dafny0/snapshots/Snapshots2.run.dfy +++ b/Test/dafny0/snapshots/Snapshots2.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots2.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 %S/Inputs/Snapshots2.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots3.run.dfy b/Test/dafny0/snapshots/Snapshots3.run.dfy index 593a13ed18a..9a052948a11 100644 --- a/Test/dafny0/snapshots/Snapshots3.run.dfy +++ b/Test/dafny0/snapshots/Snapshots3.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots3.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 %S/Inputs/Snapshots3.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots4.run.dfy b/Test/dafny0/snapshots/Snapshots4.run.dfy index 83443bca05f..8bde665646d 100644 --- a/Test/dafny0/snapshots/Snapshots4.run.dfy +++ b/Test/dafny0/snapshots/Snapshots4.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots4.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 %S/Inputs/Snapshots4.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots5.run.dfy b/Test/dafny0/snapshots/Snapshots5.run.dfy index 2ca08b01a09..3948c29772a 100644 --- a/Test/dafny0/snapshots/Snapshots5.run.dfy +++ b/Test/dafny0/snapshots/Snapshots5.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots5.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 %S/Inputs/Snapshots5.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots6.run.dfy b/Test/dafny0/snapshots/Snapshots6.run.dfy index 7eceec399be..f690cea5d0d 100644 --- a/Test/dafny0/snapshots/Snapshots6.run.dfy +++ b/Test/dafny0/snapshots/Snapshots6.run.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots6.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 %S/Inputs/Snapshots6.dfy > "%t" // RUN: %diff "%s.expect" "%t" // XFAIL: * // FIXME : Need to fix the snapshot diff --git a/Test/dafny0/snapshots/Snapshots7.run.dfy b/Test/dafny0/snapshots/Snapshots7.run.dfy index cabedf96d6e..ca6b8eed1d5 100644 --- a/Test/dafny0/snapshots/Snapshots7.run.dfy +++ b/Test/dafny0/snapshots/Snapshots7.run.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 Inputs/Snapshots7.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:2 /traceCaching:1 %S/Inputs/Snapshots7.dfy > "%t" // RUN: %diff "%s.expect" "%t" // XFAIL: * // FIXME - need to regenerate the snapshots diff --git a/Test/dafny0/snapshots/Snapshots8.run.dfy b/Test/dafny0/snapshots/Snapshots8.run.dfy index c277023df73..5d7a84f5325 100644 --- a/Test/dafny0/snapshots/Snapshots8.run.dfy +++ b/Test/dafny0/snapshots/Snapshots8.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:3 /traceCaching:1 /errorTrace:0 Inputs/Snapshots8.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:3 /traceCaching:1 /errorTrace:0 %S/Inputs/Snapshots8.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/snapshots/Snapshots9.run.dfy b/Test/dafny0/snapshots/Snapshots9.run.dfy index de77de560cc..26b6d67312a 100644 --- a/Test/dafny0/snapshots/Snapshots9.run.dfy +++ b/Test/dafny0/snapshots/Snapshots9.run.dfy @@ -1,2 +1,2 @@ -// RUN: %dafny /compile:0 /verifySnapshots:3 /traceCaching:1 Inputs/Snapshots9.dfy > "%t" +// RUN: %dafny /compile:0 /verifySnapshots:3 /traceCaching:1 %S/Inputs/Snapshots9.dfy > "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/git-issues/git-issue-801.dfy b/Test/git-issues/git-issue-801.dfy deleted file mode 100644 index 40c1d1737c7..00000000000 --- a/Test/git-issues/git-issue-801.dfy +++ /dev/null @@ -1,6 +0,0 @@ -// RUN: %dafny /compile:0 /verifyAllModules "%s" > "%t" -// RUN: %diff "%s.expect" "%t" - -// This tests that an example in the Reference Manual behaves as expected - -include "../../docs/DafnyRef/examples/Example-Old.dfy" diff --git a/Test/git-issues/git-issue-801.dfy.expect b/Test/git-issues/git-issue-801.dfy.expect deleted file mode 100644 index 823a60a105c..00000000000 --- a/Test/git-issues/git-issue-801.dfy.expect +++ /dev/null @@ -1,2 +0,0 @@ - -Dafny program verifier finished with 1 verified, 0 errors diff --git a/Test/git-issues/git-issue-801a.dfy b/Test/git-issues/git-issue-801a.dfy deleted file mode 100644 index 62db778a3c0..00000000000 --- a/Test/git-issues/git-issue-801a.dfy +++ /dev/null @@ -1,6 +0,0 @@ -// RUN: %dafny /compile:0 /verifyAllModules "%s" > "%t" -// RUN: %diff "%s.expect" "%t" - -// This tests that an example in the Reference Manual behaves as expected - -include "../../docs/DafnyRef/examples/Example-Old2.dfy" diff --git a/Test/git-issues/git-issue-801a.dfy.expect b/Test/git-issues/git-issue-801a.dfy.expect deleted file mode 100644 index ba00363fc08..00000000000 --- a/Test/git-issues/git-issue-801a.dfy.expect +++ /dev/null @@ -1,2 +0,0 @@ - -Dafny program verifier finished with 4 verified, 0 errors diff --git a/Test/git-issues/git-issue-801b.dfy b/Test/git-issues/git-issue-801b.dfy deleted file mode 100644 index fa4def880e6..00000000000 --- a/Test/git-issues/git-issue-801b.dfy +++ /dev/null @@ -1,6 +0,0 @@ -// RUN: %dafny /compile:0 /verifyAllModules "%s" > "%t" -// RUN: %diff "%s.expect" "%t" - -// This tests that an example in the Reference Manual behaves as expected - -include "../../docs/DafnyRef/examples/Example-Old3.dfy" diff --git a/Test/git-issues/git-issue-801b.dfy.expect b/Test/git-issues/git-issue-801b.dfy.expect deleted file mode 100644 index ebe2328e072..00000000000 --- a/Test/git-issues/git-issue-801b.dfy.expect +++ /dev/null @@ -1,2 +0,0 @@ - -Dafny program verifier finished with 2 verified, 0 errors From 316be19b969b96750abbff1eca3cc3f734f0ee96 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 14 Oct 2021 10:06:15 -0700 Subject: [PATCH 144/192] Remove unnecessary sed commands --- Test/dafny0/RuntimeTypeTests0.dfy | 3 +-- Test/dafny0/RuntimeTypeTests0.dfy.expect | 8 ++++---- Test/git-issues/git-issue-314.dfy | 3 +-- Test/git-issues/git-issue-374.dfy | 3 +-- Test/git-issues/git-issue-633.dfy | 3 +-- Test/git-issues/git-issue-684.dfy | 3 +-- Test/git-issues/git-issue-784.dfy | 3 +-- 7 files changed, 10 insertions(+), 16 deletions(-) diff --git a/Test/dafny0/RuntimeTypeTests0.dfy b/Test/dafny0/RuntimeTypeTests0.dfy index bbce089d55d..5fcf21f53d0 100644 --- a/Test/dafny0/RuntimeTypeTests0.dfy +++ b/Test/dafny0/RuntimeTypeTests0.dfy @@ -4,8 +4,7 @@ // RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" -// RUN: sed -e 'sx\\x/x' < "%t" > "%t"2 -// RUN: %diff "%s.expect" "%t"2 +// RUN: %diff "%s.expect" "%t" // The code in this file demonstrates complications in sorting out covariance in some // compilation target languages. Compilation support is currently spotty, where C# and // Java only supports collection types (not datatypes), and only in the "upcast" direction. diff --git a/Test/dafny0/RuntimeTypeTests0.dfy.expect b/Test/dafny0/RuntimeTypeTests0.dfy.expect index ef718faea23..f415cb9699f 100644 --- a/Test/dafny0/RuntimeTypeTests0.dfy.expect +++ b/Test/dafny0/RuntimeTypeTests0.dfy.expect @@ -2,12 +2,12 @@ Dafny program verifier finished with 4 verified, 0 errors Dafny program verifier did not attempt verification -RuntimeTypeTests0.dfy(49,9): Error: compilation does not support trait types as a type parameter (got 'object'); consider introducing a ghost -RuntimeTypeTests0.dfy(58,9): Error: compilation does not support trait types as a type parameter (got 'object'); consider introducing a ghost +RuntimeTypeTests0.dfy(48,9): Error: compilation does not support trait types as a type parameter (got 'object'); consider introducing a ghost +RuntimeTypeTests0.dfy(57,9): Error: compilation does not support trait types as a type parameter (got 'object'); consider introducing a ghost Dafny program verifier did not attempt verification -RuntimeTypeTests0.dfy(49,9): Error: compilation does not support trait types as a type parameter (got 'object'); consider introducing a ghost -RuntimeTypeTests0.dfy(58,9): Error: compilation does not support trait types as a type parameter (got 'object'); consider introducing a ghost +RuntimeTypeTests0.dfy(48,9): Error: compilation does not support trait types as a type parameter (got 'object'); consider introducing a ghost +RuntimeTypeTests0.dfy(57,9): Error: compilation does not support trait types as a type parameter (got 'object'); consider introducing a ghost Wrote textual form of partial target program to RuntimeTypeTests0-java/RuntimeTypeTests0.java Dafny program verifier did not attempt verification diff --git a/Test/git-issues/git-issue-314.dfy b/Test/git-issues/git-issue-314.dfy index 4426083f8bf..63624fcb076 100644 --- a/Test/git-issues/git-issue-314.dfy +++ b/Test/git-issues/git-issue-314.dfy @@ -3,8 +3,7 @@ // RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" -// RUN: sed -e 'sx\\x/x' < "%t" > "%t"2 -// RUN: %diff "%s.expect" "%t"2 +// RUN: %diff "%s.expect" "%t" datatype S = S(G: array) datatype T = T(F: array, ghost Repr: set) diff --git a/Test/git-issues/git-issue-374.dfy b/Test/git-issues/git-issue-374.dfy index d2df6a53f65..6357b6502b6 100644 --- a/Test/git-issues/git-issue-374.dfy +++ b/Test/git-issues/git-issue-374.dfy @@ -3,8 +3,7 @@ // RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" -// RUN: sed -e 'sx\\x/x' < "%t" > "%t"2 -// RUN: %diff "%s.expect" "%t"2 +// RUN: %diff "%s.expect" "%t" class C { constructor(ghost x: int) diff --git a/Test/git-issues/git-issue-633.dfy b/Test/git-issues/git-issue-633.dfy index b2081eb36ef..427ebfafe13 100644 --- a/Test/git-issues/git-issue-633.dfy +++ b/Test/git-issues/git-issue-633.dfy @@ -3,8 +3,7 @@ // RUN: %dafny /noVerify /compile:4 /compileTarget:js /spillTargetCode:3 "%s" git-issue-633A.dfy >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:go /spillTargetCode:3 "%s" git-issue-633A.dfy >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:java /spillTargetCode:3 "%s" git-issue-633A.dfy >> "%t" -// RUN: sed -e 'sx\\x/x' < "%t" > "%t"2 -// RUN: %diff "%s.expect" "%t"2 +// RUN: %diff "%s.expect" "%t" method m() { print "OK\n"; diff --git a/Test/git-issues/git-issue-684.dfy b/Test/git-issues/git-issue-684.dfy index 464554bbec4..43ab71ad447 100644 --- a/Test/git-issues/git-issue-684.dfy +++ b/Test/git-issues/git-issue-684.dfy @@ -3,8 +3,7 @@ // RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" -// RUN: sed -e 'sx\\x/x' < "%t" > "%t"2 -// RUN: %diff "%s.expect" "%t"2 +// RUN: %diff "%s.expect" "%t" datatype Level1 = Level1(u:int) datatype Level2 = Level2(u:int, l:Level1) diff --git a/Test/git-issues/git-issue-784.dfy b/Test/git-issues/git-issue-784.dfy index a11fba2c337..6ef11e09cce 100644 --- a/Test/git-issues/git-issue-784.dfy +++ b/Test/git-issues/git-issue-784.dfy @@ -3,8 +3,7 @@ // RUN: %dafny /noVerify /compile:4 /compileTarget:js "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:go "%s" >> "%t" // RUN: %dafny /noVerify /compile:4 /compileTarget:java "%s" >> "%t" -// RUN: sed -e 'sx\\x/x' < "%t" > "%t"2 -// RUN: %diff "%s.expect" "%t"2 +// RUN: %diff "%s.expect" "%t" datatype List = Nil | Cons(T, List) { function method App(ys: List): List { From 2e1f03e9ef53c2ed2263bcf01c8777b1fd03f8b2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 14 Oct 2021 13:13:18 -0700 Subject: [PATCH 145/192] Support input redirection, XFAIL (not working yet) --- Source/XUnitExtensions/FileTestCase.cs | 7 ++- Source/XUnitExtensions/ILitCommand.cs | 44 +++++++++++++------ .../XUnitExtensions/LitCommandWithOutput.cs | 11 +++-- Source/XUnitExtensions/LitTestCase.cs | 36 ++++++++++++--- .../XUnitExtensions/MainMethodLitCommand.cs | 5 ++- Source/XUnitExtensions/ShellLitCommand.cs | 8 +++- Source/XUnitExtensions/XFailCommand.cs | 12 +++++ 7 files changed, 96 insertions(+), 27 deletions(-) create mode 100644 Source/XUnitExtensions/XFailCommand.cs diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index d007a42ad67..08b571bfd20 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -9,7 +9,10 @@ namespace XUnitExtensions { public class FileTestCase : LongLivedMarshalByRefObject, IXunitTestCase { - + // This could use SkippableFactDiscoverer.GetSkippableExceptionNames(IAttributeInfo) + // but it doesn't seem to be worth the complexity here yet. + private static readonly string[] skippingExceptionNames = { typeof(SkipException).FullName }; + protected XunitTestCase innerTestCase; public string DisplayName { get; protected set; } @@ -21,7 +24,7 @@ public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, var testClassWithCollection = new TestClass(collection, testMethod.TestClass.Class); var testMethodWithCollection = new TestMethod(testClassWithCollection, testMethod.Method); - innerTestCase = new XunitTestCase(diagnosticMessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.All, + innerTestCase = new SkippableFactTestCase(skippingExceptionNames, diagnosticMessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.All, testMethodWithCollection, data.GetData()); if (data.Traits != null) { foreach(var (key, value) in data.Traits) { diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index 8740b58a542..73ea867f5c4 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -11,15 +11,19 @@ public interface ILitCommand { private const string COMMENT_PREFIX = "//"; private const string LIT_COMMAND_PREFIX = "RUN:"; + private const string LIT_XFAIL = "XFAIL: *"; - public (int, string, string) Execute(TextWriter outputWriter); + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter); - public static LitCommandWithOutput Parse(string fileName, string line, LitTestConfiguration config) { + public static ILitCommand Parse(string fileName, string line, LitTestConfiguration config) { if (!line.StartsWith(COMMENT_PREFIX)) { return null; } line = line[COMMENT_PREFIX.Length..].Trim(); + if (line.Equals(LIT_XFAIL)) { + return new XFailCommand(); + } if (!line.StartsWith(LIT_COMMAND_PREFIX)) { return null; } @@ -28,7 +32,10 @@ public static LitCommandWithOutput Parse(string fileName, string line, LitTestCo var pieces = ParseArguments(line); var commandSymbol = pieces[0]; var basePath = Path.GetDirectoryName(fileName); - var (rawArguments, outputFile, appendOutput) = ILitCommand.ExtractOutputFile(pieces[1..]); + var (rawArguments, inputFile, outputFile, appendOutput) = ILitCommand.ExtractRedirections(pieces[1..]); + if (inputFile != null) { + inputFile = MakeFilePathsAbsolute(basePath, config.ApplySubstitutions(inputFile)); + } if (outputFile != null) { outputFile = MakeFilePathsAbsolute(basePath, config.ApplySubstitutions(outputFile)); } @@ -37,12 +44,14 @@ public static LitCommandWithOutput Parse(string fileName, string line, LitTestCo .Select(arg => MakeFilePathsAbsolute(basePath, arg)); if (config.Commands.TryGetValue(commandSymbol, out var command)) { - return new LitCommandWithOutput(command(arguments, config), outputFile, appendOutput); + return new LitCommandWithOutput(command(arguments, config), inputFile, outputFile, appendOutput); } commandSymbol = config.ApplySubstitutions(commandSymbol); - return new LitCommandWithOutput(new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), outputFile, appendOutput); + return new LitCommandWithOutput( + new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), + inputFile, outputFile, appendOutput); } private static string[] ParseArguments(string line) { @@ -81,21 +90,28 @@ public static string MakeFilePathsAbsolute(string basePath, string s) { return s; } - public static (IEnumerable, string, bool) ExtractOutputFile(IEnumerable arguments) { + public static (IEnumerable, string, string, bool) ExtractRedirections(IEnumerable arguments) { var argumentsList = arguments.ToList(); - var redirectIndex = argumentsList.IndexOf(">"); - if (redirectIndex >= 0) { - var outputFile = argumentsList[redirectIndex + 1]; - argumentsList.RemoveRange(redirectIndex, 2); - return (argumentsList, outputFile, false); + string inputFile = null; + string outputFile = null; + bool appendOutput = false; + var redirectInIndex = argumentsList.IndexOf("<"); + if (redirectInIndex >= 0) { + inputFile = argumentsList[redirectInIndex + 1]; + argumentsList.RemoveRange(redirectInIndex, 2); + } + var redirectOutIndex = argumentsList.IndexOf(">"); + if (redirectOutIndex >= 0) { + outputFile = argumentsList[redirectOutIndex + 1]; + argumentsList.RemoveRange(redirectOutIndex, 2); } var redirectAppendIndex = argumentsList.IndexOf(">>"); if (redirectAppendIndex >= 0) { - var outputFile = argumentsList[redirectAppendIndex + 1]; + outputFile = argumentsList[redirectAppendIndex + 1]; + appendOutput = true; argumentsList.RemoveRange(redirectAppendIndex, 2); - return (argumentsList, outputFile, true); } - return (argumentsList, null, false); + return (argumentsList, inputFile, outputFile, appendOutput); } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/LitCommandWithOutput.cs b/Source/XUnitExtensions/LitCommandWithOutput.cs index d914f4da847..63337517042 100644 --- a/Source/XUnitExtensions/LitCommandWithOutput.cs +++ b/Source/XUnitExtensions/LitCommandWithOutput.cs @@ -2,20 +2,23 @@ using System.Text; namespace XUnitExtensions { - public class LitCommandWithOutput { + public class LitCommandWithOutput : ILitCommand { private ILitCommand command; + private string inputFile; private string outputFile; private bool append; - public LitCommandWithOutput(ILitCommand command, string outputFile, bool append) { + public LitCommandWithOutput(ILitCommand command, string inputFile, string outputFile, bool append) { this.command = command; + this.inputFile = inputFile; this.outputFile = outputFile; this.append = append; } - public (int, string, string) Execute() { + public (int, string, string) Execute(TextReader inReader, TextWriter outWriter) { + var inputReader = inputFile != null ? new StreamReader(inputFile) : null; var outputWriter = outputFile != null ? new StreamWriter(outputFile, append) : null; - return command.Execute(outputWriter); + return command.Execute(inputReader, outputWriter); } public override string ToString() { diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index 68e00dd5787..927501b4aab 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -9,10 +9,16 @@ namespace XUnitExtensions { public class LitTestCase { - public static IEnumerable Read(string filePath, LitTestConfiguration config) { - return File.ReadAllLines(filePath) + private string filePath; + private IEnumerable commands; + private bool expectFailure; + + public static LitTestCase Read(string filePath, LitTestConfiguration config) { + var commands = File.ReadAllLines(filePath) .Select(line => ILitCommand.Parse(filePath, line, config)) .Where(c => c != null); + var xfail = commands.Any(c => c is XFailCommand); + return new LitTestCase(filePath, commands, xfail); } public static void Run(string filePath, LitTestConfiguration config, ITestOutputHelper outputHelper) { @@ -24,24 +30,44 @@ public static void Run(string filePath, LitTestConfiguration config, ITestOutput { "%t", Path.Join(directory, "Output", $"{fileName}.tmp")} }); - Directory.CreateDirectory(Path.Join(directory, "Output")); + var testCase = Read(filePath, config); + testCase.Execute(outputHelper); + } + + public LitTestCase(string filePath, IEnumerable commands, bool expectFailure) { + this.filePath = filePath; + this.commands = commands; + this.expectFailure = expectFailure; + } + + public void Execute(ITestOutputHelper outputHelper) { + Directory.CreateDirectory(Path.Join(Path.GetDirectoryName(filePath), "Output")); - var commands = Read(filePath, config); foreach (var command in commands) { int exitCode; string output; string error; try { outputHelper.WriteLine($"Executing command: {command}"); - (exitCode, output, error) = command.Execute(); + (exitCode, output, error) = command.Execute(null, null); } catch (Exception e) { throw new Exception($"Exception thrown while executing command: {command}", e); } + if (expectFailure) { + if (exitCode != 0) { + throw new SkipException($"Command returned non-zero exit code ({exitCode}): {command}\nOutput:\n{output}\nError:\n{error}"); + } + } + if (exitCode != 0) { throw new Exception($"Command returned non-zero exit code ({exitCode}): {command}\nOutput:\n{output}\nError:\n{error}"); } } + + if (expectFailure) { + throw new Exception($"Test case passed but expected to fail: {filePath}"); + } } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/MainMethodLitCommand.cs b/Source/XUnitExtensions/MainMethodLitCommand.cs index cb0765f6d0c..29f1b06573b 100644 --- a/Source/XUnitExtensions/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/MainMethodLitCommand.cs @@ -19,9 +19,12 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments return invokeDirectly ? result : result.ToShellCommand(config); } - public (int, string, string) Execute(TextWriter outputWriter) { + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter) { StringBuilder redirectedErr = new(); + if (inputReader != null) { + Console.SetIn(inputReader); + } if (outputWriter != null) { Console.SetOut(outputWriter); } diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/ShellLitCommand.cs index 11dafcff1f1..0626ced0e9f 100644 --- a/Source/XUnitExtensions/ShellLitCommand.cs +++ b/Source/XUnitExtensions/ShellLitCommand.cs @@ -17,7 +17,7 @@ public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnum this.passthroughEnvironmentVariables = passthroughEnvironmentVariables.ToArray(); } - public (int, string, string) Execute(TextWriter outputWriter) { + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter) { using var process = new Process(); process.StartInfo.FileName = shellCommand; @@ -29,6 +29,7 @@ public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnum // MainMethodLitCommand, which can't change the current directory. process.StartInfo.UseShellExecute = false; + process.StartInfo.RedirectStandardInput = true; process.StartInfo.RedirectStandardOutput = true; process.StartInfo.RedirectStandardError = true; process.StartInfo.CreateNoWindow = true; @@ -39,6 +40,11 @@ public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnum } process.Start(); + if (inputReader != null) { + string input = inputReader.ReadToEnd(); + process.StandardInput.Write(input); + process.StandardInput.Close(); + } string output = process.StandardOutput.ReadToEnd(); outputWriter?.Write(output); outputWriter?.Flush(); diff --git a/Source/XUnitExtensions/XFailCommand.cs b/Source/XUnitExtensions/XFailCommand.cs new file mode 100644 index 00000000000..9e427441e8b --- /dev/null +++ b/Source/XUnitExtensions/XFailCommand.cs @@ -0,0 +1,12 @@ +using System.IO; + +namespace XUnitExtensions { + public class XFailCommand : ILitCommand { + + public XFailCommand() {} + + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter) { + return (0, "", ""); + } + } +} \ No newline at end of file From 0906cb1aaa1b0c82e00456ab39de326b1ec3bda6 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 14 Oct 2021 18:42:27 -0700 Subject: [PATCH 146/192] Fix more tests using %S, trying without absolute path magic --- Source/IntegrationTests/LitTests.cs | 4 +++- Source/XUnitExtensions/ILitCommand.cs | 8 ++++---- Test/comp/compile1quiet/CompileRunQuietly.dfy | 10 +++++----- Test/comp/compile1verbose/CompileAndThenRun.dfy | 10 +++++----- Test/lit.site.cfg | 3 ++- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 821cdcc7527..f141e661c50 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -44,9 +44,10 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable { + // TODO: speed this up by using AssertWithDiff { "%diff", "diff" }, + { "%binaries", "." }, { "%refmanexamples", Path.Join("examples") } }, @@ -54,6 +55,7 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index 73ea867f5c4..7d5d4f3ccaf 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -82,10 +82,10 @@ public static string MakeFilePathsAbsolute(string basePath, string s) { return s; } - var absolutePath = Path.Join(basePath, s); - if (File.Exists(absolutePath)) { - return absolutePath; - } + // var absolutePath = Path.Join(basePath, s); + // if (File.Exists(absolutePath)) { + // return absolutePath; + // } return s; } diff --git a/Test/comp/compile1quiet/CompileRunQuietly.dfy b/Test/comp/compile1quiet/CompileRunQuietly.dfy index e8b4149cea4..5393f9008fd 100644 --- a/Test/comp/compile1quiet/CompileRunQuietly.dfy +++ b/Test/comp/compile1quiet/CompileRunQuietly.dfy @@ -1,17 +1,17 @@ // RUN: %dafny /compileTarget:cs "%s" > "%t" -// RUN: dotnet CompileRunQuietly.dll >> "%t" +// RUN: dotnet %S/CompileRunQuietly.dll >> "%t" // RUN: %dafny /compileTarget:js "%s" >> "%t" -// RUN: node CompileRunQuietly.js >> "%t" +// RUN: node %S/CompileRunQuietly.js >> "%t" // RUN: %dafny /compileTarget:go "%s" >> "%t" -// RUN: ./CompileRunQuietly >> "%t" +// RUN: %S/CompileRunQuietly >> "%t" // RUN: %dafny /compileTarget:java "%s" >> "%t" -// RUN: java CompileRunQuietly >> "%t" +// RUN: java -cp %binaries/DafnyRuntime.jar:%S/CompileRunQuietly-java CompileRunQuietly >> "%t" // RUN: %dafny /compileTarget:cpp "%s" >> "%t" -// RUN: ./CompileRunQuietly.exe >> "%t" +// RUN: %S/CompileRunQuietly.exe >> "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/comp/compile1verbose/CompileAndThenRun.dfy b/Test/comp/compile1verbose/CompileAndThenRun.dfy index c14e80e257b..b0f3b62ad3d 100644 --- a/Test/comp/compile1verbose/CompileAndThenRun.dfy +++ b/Test/comp/compile1verbose/CompileAndThenRun.dfy @@ -1,17 +1,17 @@ // RUN: %dafny /compileVerbose:1 /compileTarget:cs "%s" > "%t" -// RUN: dotnet CompileAndThenRun.dll >> "%t" +// RUN: dotnet %S/CompileAndThenRun.dll >> "%t" // RUN: %dafny /compileVerbose:1 /compileTarget:js "%s" >> "%t" -// RUN: node CompileAndThenRun.js >> "%t" +// RUN: node %S/CompileAndThenRun.js >> "%t" // RUN: %dafny /compileVerbose:1 /compileTarget:go "%s" >> "%t" -// RUN: ./CompileAndThenRun >> "%t" +// RUN: %S/CompileAndThenRun >> "%t" // RUN: %dafny /compileVerbose:1 /compileTarget:java "%s" >> "%t" -// RUN: java CompileAndThenRun >> "%t" +// RUN: java -cp %binaries/DafnyRuntime.jar:%S/CompileAndThenRun-java CompileAndThenRun >> "%t" // RUN: %dafny /compileVerbose:1 /compileTarget:cpp "%s" >> "%t" -// RUN: ./CompileAndThenRun.exe >> "%t" +// RUN: %S/CompileAndThenRun.exe >> "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/lit.site.cfg b/Test/lit.site.cfg index 0d31da08255..18f0481a6f5 100644 --- a/Test/lit.site.cfg +++ b/Test/lit.site.cfg @@ -139,7 +139,8 @@ lit_config.note('%baredafny will expand to {}\n'.format(bareDafnyExecutable)) ver = "0" if os.name != "nt": ver = os.uname().version - + +config.substitutions.append( ('%binaries`, binaryDir) ) config.substitutions.append( ('%dafny', dafnyExecutable) ) config.substitutions.append( ('%baredafny', bareDafnyExecutable) ) config.substitutions.append( ('%server', serverExecutable) ) From deae7f5e5186bf6277c8e3c309a26287be5a06c9 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 14 Oct 2021 20:35:55 -0700 Subject: [PATCH 147/192] timeLimit --- Source/IntegrationTests/LitTests.cs | 5 ++++- Test/lit.site.cfg | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index f141e661c50..a8232c182cf 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -27,7 +27,10 @@ public class LitTests { "/compileVerbose:0", // Hide Boogie execution traces since they are meaningless for Dafny programs - "/errorTrace:0" + "/errorTrace:0", + + // Set a default time limit, to catch cases where verification time runs off the rails + "/timeLimit:300" }; private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable arguments, LitTestConfiguration config, bool invokeDirectly) { diff --git a/Test/lit.site.cfg b/Test/lit.site.cfg index 18f0481a6f5..ad9b1ff409f 100644 --- a/Test/lit.site.cfg +++ b/Test/lit.site.cfg @@ -140,7 +140,7 @@ ver = "0" if os.name != "nt": ver = os.uname().version -config.substitutions.append( ('%binaries`, binaryDir) ) +config.substitutions.append( ('%binaries', binaryDir) ) config.substitutions.append( ('%dafny', dafnyExecutable) ) config.substitutions.append( ('%baredafny', bareDafnyExecutable) ) config.substitutions.append( ('%server', serverExecutable) ) From 3e30b039f66e72004e76745df1d648484d5dc37c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 14 Oct 2021 20:45:26 -0700 Subject: [PATCH 148/192] Fixed XFAIL handling --- Source/XUnitExtensions/FileTestCase.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index 08b571bfd20..b0d01d98669 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -65,19 +65,23 @@ public ISourceInformation SourceInformation { public IMethodInfo Method => innerTestCase.Method; public int Timeout => innerTestCase.Timeout; - public Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, + public async Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { - return new XunitTestCaseRunner( + var messageBusInterceptor = new SkippableTestMessageBus(messageBus, skippingExceptionNames); + RunSummary result = await new XunitTestCaseRunner( this, DisplayName, SkipReason, constructorArguments, TestMethodArguments, - messageBus, + messageBusInterceptor, aggregator, cancellationTokenSource ).RunAsync(); + result.Failed -= messageBusInterceptor.SkippedCount; + result.Skipped += messageBusInterceptor.SkippedCount; + return result; } } } \ No newline at end of file From b19782f588245eedf4996330d6ee9cb8743fed41 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 14 Oct 2021 22:59:52 -0700 Subject: [PATCH 149/192] Added %S as needed --- Source/IntegrationTests/LitTests.cs | 2 +- Test/comp/BranchCoverage.dfy | 8 ++++---- Test/comp/CSharpStyling.dfy | 2 +- Test/comp/Extern.dfy | 8 ++++---- Test/comp/ExternCtors.dfy | 4 ++-- Test/comp/manualcompile/ManualCompile.dfy | 10 +++++----- Test/dafny0/DafnyLibClient.dfy | 4 ++-- Test/dafny0/Extern.dfy | 2 +- Test/dafny0/ExternCopyFromTrait.dfy | 2 +- Test/git-issues/git-issue-1100.dfy | 2 +- Test/git-issues/git-issue-1151.dfy | 2 +- Test/git-issues/git-issue-1199.dfy | 2 +- Test/git-issues/git-issue-1229.dfy | 2 +- Test/git-issues/git-issue-289.dfy | 6 +++--- Test/git-issues/git-issue-633.dfy | 10 +++++----- Test/git-issues/git-issue-845.dfy | 2 +- Test/git-issues/git-issue-950.dfy | 4 ++-- 17 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index a8232c182cf..b5deda58ca8 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -51,7 +51,7 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable "%t" -// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:js BranchCoverage3.js "%s" >> "%t" -// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:go BranchCoverage4.go "%s" >> "%t" -// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:java CodeCoverage.java "%s" >> "%t" +// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:cs %S/BranchCoverage2.cs "%s" > "%t" +// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:js %S/BranchCoverage3.js "%s" >> "%t" +// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:go %S/BranchCoverage4.go "%s" >> "%t" +// RUN: %dafny /compile:3 /coverage:- /spillTargetCode:1 /compileTarget:java %S/CodeCoverage.java "%s" >> "%t" // RUN: %diff "%s.expect" "%t" // The Main method is at the end of this file, because that makes it easier to maintain diff --git a/Test/comp/CSharpStyling.dfy b/Test/comp/CSharpStyling.dfy index 038eac22aef..584eac0b887 100644 --- a/Test/comp/CSharpStyling.dfy +++ b/Test/comp/CSharpStyling.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" CSharpStyling2.cs > "%t" +// RUN: %dafny /compile:3 /spillTargetCode:2 /compileTarget:cs "%s" %S/CSharpStyling2.cs > "%t" // RUN: %diff "%s.expect" "%t" method Main() { diff --git a/Test/comp/Extern.dfy b/Test/comp/Extern.dfy index f375937fe0a..52225810420 100644 --- a/Test/comp/Extern.dfy +++ b/Test/comp/Extern.dfy @@ -1,7 +1,7 @@ -// RUN: %dafny /compile:3 /compileTarget:cs "%s" Extern2.cs > "%t" -// RUN: %dafny /compile:3 /compileTarget:js "%s" Extern3.js >> "%t" -// RUN: %dafny /compile:3 /compileTarget:go "%s" Extern4.go >> "%t" -// RUN: %dafny /compile:3 /compileTarget:java "%s" LibClass.java OtherClass.java AllDafny.java Mixed.java AllExtern.java >> "%t" +// RUN: %dafny /compile:3 /compileTarget:cs "%s" %S/Extern2.cs > "%t" +// RUN: %dafny /compile:3 /compileTarget:js "%s" %S/Extern3.js >> "%t" +// RUN: %dafny /compile:3 /compileTarget:go "%s" %S/Extern4.go >> "%t" +// RUN: %dafny /compile:3 /compileTarget:java "%s" %S/LibClass.java %S/OtherClass.java %S/AllDafny.java %S/Mixed.java %S/AllExtern.java >> "%t" // RUN: %diff "%s.expect" "%t" method Main() { diff --git a/Test/comp/ExternCtors.dfy b/Test/comp/ExternCtors.dfy index 35b6112bab6..a9a0f9a897e 100644 --- a/Test/comp/ExternCtors.dfy +++ b/Test/comp/ExternCtors.dfy @@ -1,5 +1,5 @@ -// RUN: %dafny /compile:3 /compileTarget:cs "%s" ExternCtors-externs/Library.cs > "%t" -// RUN: %dafny /compile:3 /compileTarget:java "%s" ExternCtors-externs/Class.java >> "%t" +// RUN: %dafny /compile:3 /compileTarget:cs "%s" %S/ExternCtors-externs/Library.cs > "%t" +// RUN: %dafny /compile:3 /compileTarget:java "%s" %S/ExternCtors-externs/Class.java >> "%t" // RUN: %diff "%s.expect" "%t" // FIXME: Extern constructors are currently broken in Go and JavaScript, diff --git a/Test/comp/manualcompile/ManualCompile.dfy b/Test/comp/manualcompile/ManualCompile.dfy index 1d194078622..e699b4fa720 100644 --- a/Test/comp/manualcompile/ManualCompile.dfy +++ b/Test/comp/manualcompile/ManualCompile.dfy @@ -1,12 +1,12 @@ // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:cs "%s" > "%t" -// RUN: dotnet build ManualCompile.csproj -// RUN: dotnet bin/Debug/net5.0/ManualCompile.dll >> "%t" +// RUN: dotnet build %S/ManualCompile.csproj +// RUN: dotnet %S/bin/Debug/net5.0/ManualCompile.dll >> "%t" // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:js "%s" >> "%t" -// RUN: node ManualCompile.js >> "%t" +// RUN: node %S/ManualCompile.js >> "%t" // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: go run ManualCompile-go/src/ManualCompile.go >> "%t" +// RUN: GOPATH=%S/ManualCompile-go go run %S/ManualCompile-go/src/ManualCompile.go >> "%t" // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" // RUN: javac ManualCompile-java/ManualCompile.java ManualCompile-java/*/*.java @@ -14,7 +14,7 @@ // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:cpp "%s" >> "%t" // RUN: g++ -g -Wall -Wextra -Wpedantic -Wno-unused-variable -std=c++17 -I %binaryDir -o ManualCompile.exe ManualCompile.cpp -// RUN: ./ManualCompile.exe >> "%t" +// RUN: %S/ManualCompile.exe >> "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/dafny0/DafnyLibClient.dfy b/Test/dafny0/DafnyLibClient.dfy index 1a32d8cabc1..00e70143620 100644 --- a/Test/dafny0/DafnyLibClient.dfy +++ b/Test/dafny0/DafnyLibClient.dfy @@ -1,5 +1,5 @@ -// RUN: %dafny /spillTargetCode:1 DafnyLib.dfy DafnyLib-extern.cs /useRuntimeLib /out:Output/DafnyLib.dll /compileTarget:cs > "%t" -// RUN: %dafny /spillTargetCode:1 /compile:3 "%s" Output/DafnyLib.dll >> "%t" +// RUN: %dafny /spillTargetCode:1 %S/DafnyLib.dfy %S/DafnyLib-extern.cs /useRuntimeLib /out:%S/Output/DafnyLib.dll /compileTarget:cs > "%t" +// RUN: %dafny /spillTargetCode:1 /compile:3 "%s" %S/Output/DafnyLib.dll >> "%t" // RUN: %diff "%s.expect" "%t" module Client { diff --git a/Test/dafny0/Extern.dfy b/Test/dafny0/Extern.dfy index f5e66f416e3..b56ffaf35f4 100644 --- a/Test/dafny0/Extern.dfy +++ b/Test/dafny0/Extern.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:1 /compileTarget:cs /print:"%t.print" /dprint:"%t.dprint" "%s" Extern2.cs ExternHelloLibrary.dll > "%t" +// RUN: %dafny /compile:1 /compileTarget:cs /print:"%t.print" /dprint:"%t.dprint" "%s" %S/Extern2.cs %S/ExternHelloLibrary.dll > "%t" // RUN: %diff "%s.expect" "%t" method Main() { diff --git a/Test/dafny0/ExternCopyFromTrait.dfy b/Test/dafny0/ExternCopyFromTrait.dfy index 8ead0b18c8c..df15cf31368 100644 --- a/Test/dafny0/ExternCopyFromTrait.dfy +++ b/Test/dafny0/ExternCopyFromTrait.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:1 /compileTarget:cs /print:"%t.print" /dprint:"%t.dprint" "%s" ExternCopyFromTraitLibrary.cs > "%t" +// RUN: %dafny /compile:1 /compileTarget:cs /print:"%t.print" /dprint:"%t.dprint" "%s" %S/ExternCopyFromTraitLibrary.cs > "%t" // RUN: %diff "%s.expect" "%t" module {:extern "M"} M { trait {:extern} T1 { diff --git a/Test/git-issues/git-issue-1100.dfy b/Test/git-issues/git-issue-1100.dfy index 47af222b432..08126c15550 100644 --- a/Test/git-issues/git-issue-1100.dfy +++ b/Test/git-issues/git-issue-1100.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:3 /compileTarget:cpp ../c++/arrays.dfy > "%t" +// RUN: %dafny /compile:3 /compileTarget:cpp %S/../c++/arrays.dfy > "%t" // RUN: %diff "%s.expect" "%t" // Test compilation of a file in another directory diff --git a/Test/git-issues/git-issue-1151.dfy b/Test/git-issues/git-issue-1151.dfy index a2ef5b9a45b..b58b4f21d12 100644 --- a/Test/git-issues/git-issue-1151.dfy +++ b/Test/git-issues/git-issue-1151.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:3 "%s" git-issue-1151-concrete.cs > "%t" +// RUN: %dafny /compile:3 "%s" %S/git-issue-1151-concrete.cs > "%t" // RUN: %diff "%s.expect" "%t" module {:extern "M"} M { diff --git a/Test/git-issues/git-issue-1199.dfy b/Test/git-issues/git-issue-1199.dfy index a6b5ec650b4..50c1a5c92ce 100644 --- a/Test/git-issues/git-issue-1199.dfy +++ b/Test/git-issues/git-issue-1199.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:1 /compileTarget:cs "%s" git-issue-1199-extern.cs > "%t" +// RUN: %dafny /compile:1 /compileTarget:cs "%s" %S/git-issue-1199-extern.cs > "%t" // RUN: %diff "%s.expect" "%t" module {:extern "Microsoft"} Microsoft { diff --git a/Test/git-issues/git-issue-1229.dfy b/Test/git-issues/git-issue-1229.dfy index a0f0f48b264..192f14ae3cd 100644 --- a/Test/git-issues/git-issue-1229.dfy +++ b/Test/git-issues/git-issue-1229.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:1 /compileTarget:cs "%s" git-issue-1229-extern.cs > "%t" +// RUN: %dafny /compile:1 /compileTarget:cs "%s" %S/git-issue-1229-extern.cs > "%t" // RUN: %diff "%s.expect" "%t" module {:extern "MyModule2"} MyModule2 { diff --git a/Test/git-issues/git-issue-289.dfy b/Test/git-issues/git-issue-289.dfy index 7dd84e07637..19c3d3076e9 100644 --- a/Test/git-issues/git-issue-289.dfy +++ b/Test/git-issues/git-issue-289.dfy @@ -1,6 +1,6 @@ -// RUN: %dafny /compile:1 "%s" > "%t" -// RUN: %dafny /compile:1 "%s" "git-issue-289b.dfy" >> "%t" -// RUN: %dafny /compile:1 "git-issue-289b.dfy" >> "%t" +// RUN: %dafny /compile:1 "%s" > "%t" +// RUN: %dafny /compile:1 "%s" %S/"git-issue-289b.dfy" >> "%t" +// RUN: %dafny /compile:1 %S/"git-issue-289b.dfy" >> "%t" // RUN: %diff "%s.expect" "%t" include "git-issue-289b.dfy" diff --git a/Test/git-issues/git-issue-633.dfy b/Test/git-issues/git-issue-633.dfy index 427ebfafe13..5d9b5aecf58 100644 --- a/Test/git-issues/git-issue-633.dfy +++ b/Test/git-issues/git-issue-633.dfy @@ -1,8 +1,8 @@ -// RUN: %dafny /compile:0 "%s" git-issue-633A.dfy > "%t" -// RUN: %dafny /noVerify /compile:4 /compileTarget:cs /spillTargetCode:3 "%s" git-issue-633A.dfy >> "%t" -// RUN: %dafny /noVerify /compile:4 /compileTarget:js /spillTargetCode:3 "%s" git-issue-633A.dfy >> "%t" -// RUN: %dafny /noVerify /compile:4 /compileTarget:go /spillTargetCode:3 "%s" git-issue-633A.dfy >> "%t" -// RUN: %dafny /noVerify /compile:4 /compileTarget:java /spillTargetCode:3 "%s" git-issue-633A.dfy >> "%t" +// RUN: %dafny /compile:0 "%s" %S/git-issue-633A.dfy > "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:cs /spillTargetCode:3 "%s" %S/git-issue-633A.dfy >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:js /spillTargetCode:3 "%s" %S/git-issue-633A.dfy >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:go /spillTargetCode:3 "%s" %S/git-issue-633A.dfy >> "%t" +// RUN: %dafny /noVerify /compile:4 /compileTarget:java /spillTargetCode:3 "%s" %S/git-issue-633A.dfy >> "%t" // RUN: %diff "%s.expect" "%t" method m() { diff --git a/Test/git-issues/git-issue-845.dfy b/Test/git-issues/git-issue-845.dfy index 0a3fe4963d1..3fe9a576338 100644 --- a/Test/git-issues/git-issue-845.dfy +++ b/Test/git-issues/git-issue-845.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /useBaseNameForFileName /compile:0 git-issue-845.dfy > "%t" +// RUN: %dafny /useBaseNameForFileName /compile:0 %S/git-issue-845.dfy > "%t" // RUN: %diff "%s.expect" "%t" /* blah blah /* blah */ diff --git a/Test/git-issues/git-issue-950.dfy b/Test/git-issues/git-issue-950.dfy index 12f3eada5c3..c16a1487d5b 100644 --- a/Test/git-issues/git-issue-950.dfy +++ b/Test/git-issues/git-issue-950.dfy @@ -1,6 +1,6 @@ // RUN: %dafny /compile:0 "%s" "%s" > "%t" -// RUN: %dafny /compile:0 "%s" git-issue-950.dfy >> "%t" -// RUN: %dafny /compile:0 "%s" ../git-issues/git-issue-950.dfy >> "%t" +// RUN: %dafny /compile:0 "%s" %S/git-issue-950.dfy >> "%t" +// RUN: %dafny /compile:0 "%s" %S/../git-issues/git-issue-950.dfy >> "%t" // RUN: %diff "%s.expect" "%t" module M { From ca9c15ecd32465e1fef6f63077468de31c1af978 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 15 Oct 2021 11:44:25 -0700 Subject: [PATCH 150/192] Adding || support, fixing legit bug, etc. --- Source/Dafny/Dafny.atg | 5 +- Source/IntegrationTests/LitTests.cs | 1 + Source/XUnitExtensions/ILitCommand.cs | 86 +++++-------------- .../XUnitExtensions/LitCommandWithOutput.cs | 34 -------- .../LitCommandWithRedirection.cs | 84 ++++++++++++++++++ Source/XUnitExtensions/OrCommand.cs | 22 +++++ .../we-should-always-print-tooltips.dfy | 2 +- 7 files changed, 133 insertions(+), 101 deletions(-) delete mode 100644 Source/XUnitExtensions/LitCommandWithOutput.cs create mode 100644 Source/XUnitExtensions/LitCommandWithRedirection.cs create mode 100644 Source/XUnitExtensions/OrCommand.cs diff --git a/Source/Dafny/Dafny.atg b/Source/Dafny/Dafny.atg index e75695e9157..3d1dbdf7b77 100644 --- a/Source/Dafny/Dafny.atg +++ b/Source/Dafny/Dafny.atg @@ -875,7 +875,10 @@ Dafny { TopDecl } (. // find the default class in the default module, then append membersDefaultClass to its member list if (membersDefaultClass.Count == 0 && defaultModule.Includes.Count == 0 && defaultModule.TopLevelDecls.Count == 0) { - errors.Warning(defaultModule.tok, "File contains no code: " + Path.GetRelativePath(Directory.GetCurrentDirectory(), scanner.FullFilename)); + var fileName = DafnyOptions.Clo.UseBaseNameForFileName + ? Path.GetFileName(scanner.FullFilename) + : Path.GetRelativePath(Directory.GetCurrentDirectory(), scanner.FullFilename); + errors.Warning(defaultModule.tok, "File contains no code: " + fileName); } DefaultClassDecl defaultClass = null; foreach (TopLevelDecl topleveldecl in defaultModule.TopLevelDecls) { diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index b5deda58ca8..8582c3e61c4 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -51,6 +51,7 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable MakeFilePathsAbsolute(basePath, arg)); - - if (config.Commands.TryGetValue(commandSymbol, out var command)) { - return new LitCommandWithOutput(command(arguments, config), inputFile, outputFile, appendOutput); - } - - commandSymbol = config.ApplySubstitutions(commandSymbol); - - return new LitCommandWithOutput( - new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), - inputFile, outputFile, appendOutput); + var tokens = Tokenize(line); + return ParseRunCommand(tokens, config); } - private static string[] ParseArguments(string line) { + public static ILitCommand ParseRunCommand(string[] tokens, LitTestConfiguration config) { + // Just supporting || for now since it's a precise way to ignore an exit code + var seqOperatorIndex = Array.IndexOf(tokens, "||"); + if (seqOperatorIndex >= 0) { + var lhs = LitCommandWithRedirection.Parse(tokens[0..seqOperatorIndex], config); + var rhs = ParseRunCommand(tokens[(seqOperatorIndex + 1)..], config); + return new OrCommand(lhs, rhs); + } + return LitCommandWithRedirection.Parse(tokens, config); + } + + private static string[] Tokenize(string line) { var arguments = new List(); var argument = new StringBuilder(); - var quoted = false; + var singleQuoted = false; + var doubleQuoted = false; for (int i = 0; i < line.Length; i++) { var c = line[i]; - if (c == '"') { - quoted = !quoted; - } else if (Char.IsWhiteSpace(c) && !quoted) { + if (c == '\'') { + singleQuoted = !singleQuoted; + } else if (c == '"') { + doubleQuoted = !doubleQuoted; + } else if (Char.IsWhiteSpace(c) && !(singleQuoted || doubleQuoted)) { arguments.Add(argument.ToString()); argument.Clear(); } else { @@ -76,42 +69,5 @@ private static string[] ParseArguments(string line) { return arguments.ToArray(); } - - public static string MakeFilePathsAbsolute(string basePath, string s) { - if (s.StartsWith("-") || s.StartsWith("/")) { - return s; - } - - // var absolutePath = Path.Join(basePath, s); - // if (File.Exists(absolutePath)) { - // return absolutePath; - // } - - return s; - } - - public static (IEnumerable, string, string, bool) ExtractRedirections(IEnumerable arguments) { - var argumentsList = arguments.ToList(); - string inputFile = null; - string outputFile = null; - bool appendOutput = false; - var redirectInIndex = argumentsList.IndexOf("<"); - if (redirectInIndex >= 0) { - inputFile = argumentsList[redirectInIndex + 1]; - argumentsList.RemoveRange(redirectInIndex, 2); - } - var redirectOutIndex = argumentsList.IndexOf(">"); - if (redirectOutIndex >= 0) { - outputFile = argumentsList[redirectOutIndex + 1]; - argumentsList.RemoveRange(redirectOutIndex, 2); - } - var redirectAppendIndex = argumentsList.IndexOf(">>"); - if (redirectAppendIndex >= 0) { - outputFile = argumentsList[redirectAppendIndex + 1]; - appendOutput = true; - argumentsList.RemoveRange(redirectAppendIndex, 2); - } - return (argumentsList, inputFile, outputFile, appendOutput); - } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/LitCommandWithOutput.cs b/Source/XUnitExtensions/LitCommandWithOutput.cs deleted file mode 100644 index 63337517042..00000000000 --- a/Source/XUnitExtensions/LitCommandWithOutput.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System.IO; -using System.Text; - -namespace XUnitExtensions { - public class LitCommandWithOutput : ILitCommand { - private ILitCommand command; - private string inputFile; - private string outputFile; - private bool append; - - public LitCommandWithOutput(ILitCommand command, string inputFile, string outputFile, bool append) { - this.command = command; - this.inputFile = inputFile; - this.outputFile = outputFile; - this.append = append; - } - - public (int, string, string) Execute(TextReader inReader, TextWriter outWriter) { - var inputReader = inputFile != null ? new StreamReader(inputFile) : null; - var outputWriter = outputFile != null ? new StreamWriter(outputFile, append) : null; - return command.Execute(inputReader, outputWriter); - } - - public override string ToString() { - var builder = new StringBuilder(); - builder.Append(command); - if (outputFile != null) { - builder.Append(append ? " >> " : " > "); - builder.Append(outputFile); - } - return builder.ToString(); - } - } -} \ No newline at end of file diff --git a/Source/XUnitExtensions/LitCommandWithRedirection.cs b/Source/XUnitExtensions/LitCommandWithRedirection.cs new file mode 100644 index 00000000000..7e4bc770ccb --- /dev/null +++ b/Source/XUnitExtensions/LitCommandWithRedirection.cs @@ -0,0 +1,84 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace XUnitExtensions { + public class LitCommandWithRedirection : ILitCommand { + + public static LitCommandWithRedirection Parse(string[] tokens, LitTestConfiguration config) { + var commandSymbol = tokens[0]; + var (rawArguments, inputFile, outputFile, appendOutput) = ExtractRedirections(tokens[1..]); + if (inputFile != null) { + inputFile = config.ApplySubstitutions(inputFile); + } + if (outputFile != null) { + outputFile = config.ApplySubstitutions(outputFile); + } + + var arguments = rawArguments.Select(config.ApplySubstitutions); + + if (config.Commands.TryGetValue(commandSymbol, out var command)) { + return new LitCommandWithRedirection(command(arguments, config), inputFile, outputFile, appendOutput); + } + + commandSymbol = config.ApplySubstitutions(commandSymbol); + + return new LitCommandWithRedirection( + new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), + inputFile, outputFile, appendOutput); + } + + public static (IEnumerable, string, string, bool) ExtractRedirections(IEnumerable arguments) { + var argumentsList = arguments.ToList(); + string inputFile = null; + string outputFile = null; + bool appendOutput = false; + var redirectInIndex = argumentsList.IndexOf("<"); + if (redirectInIndex >= 0) { + inputFile = argumentsList[redirectInIndex + 1]; + argumentsList.RemoveRange(redirectInIndex, 2); + } + var redirectOutIndex = argumentsList.IndexOf(">"); + if (redirectOutIndex >= 0) { + outputFile = argumentsList[redirectOutIndex + 1]; + argumentsList.RemoveRange(redirectOutIndex, 2); + } + var redirectAppendIndex = argumentsList.IndexOf(">>"); + if (redirectAppendIndex >= 0) { + outputFile = argumentsList[redirectAppendIndex + 1]; + appendOutput = true; + argumentsList.RemoveRange(redirectAppendIndex, 2); + } + return (argumentsList, inputFile, outputFile, appendOutput); + } + + private ILitCommand command; + private string inputFile; + private string outputFile; + private bool append; + + public LitCommandWithRedirection(ILitCommand command, string inputFile, string outputFile, bool append) { + this.command = command; + this.inputFile = inputFile; + this.outputFile = outputFile; + this.append = append; + } + + public (int, string, string) Execute(TextReader inReader, TextWriter outWriter) { + var inputReader = inputFile != null ? new StreamReader(inputFile) : null; + var outputWriter = outputFile != null ? new StreamWriter(outputFile, append) : null; + return command.Execute(inputReader, outputWriter); + } + + public override string ToString() { + var builder = new StringBuilder(); + builder.Append(command); + if (outputFile != null) { + builder.Append(append ? " >> " : " > "); + builder.Append(outputFile); + } + return builder.ToString(); + } + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/OrCommand.cs b/Source/XUnitExtensions/OrCommand.cs new file mode 100644 index 00000000000..3c6777aef27 --- /dev/null +++ b/Source/XUnitExtensions/OrCommand.cs @@ -0,0 +1,22 @@ +using System.IO; + +namespace XUnitExtensions { + public class OrCommand : ILitCommand { + private readonly ILitCommand lhs; + private readonly ILitCommand rhs; + + public OrCommand(ILitCommand lhs, ILitCommand rhs) { + this.lhs = lhs; + this.rhs = rhs; + } + + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter) { + var (exitCode, output, error) = lhs.Execute(inputReader, outputWriter); + if (exitCode == 0) { + return (exitCode, output, error); + } else { + return rhs.Execute(inputReader, outputWriter); + } + } + } +} \ No newline at end of file diff --git a/Test/wishlist/we-should-always-print-tooltips.dfy b/Test/wishlist/we-should-always-print-tooltips.dfy index e70f612062f..50540907a10 100644 --- a/Test/wishlist/we-should-always-print-tooltips.dfy +++ b/Test/wishlist/we-should-always-print-tooltips.dfy @@ -1,4 +1,4 @@ -// RUN: %dafny /compile:0 /print:"%t.print" /dprint:"%t.dprint" /printTooltips we-should-always-print-tooltips.dfy > "%t" +// RUN: %dafny /compile:0 /print:"%t.print" /dprint:"%t.dprint" /printTooltips %S/we-should-always-print-tooltips.dfy > "%t" // RUN: %diff "%s.expect" "%t" // WISH it would be great to add /printTooltips to all tests From b1d24c8309b807f8d0295c2cb9ce32879a8dee81 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 15 Oct 2021 12:28:32 -0700 Subject: [PATCH 151/192] Partial progress on TestGeneration.dfy --- Source/XUnitExtensions/OrCommand.cs | 10 +++++++--- Test/DafnyTestGeneration/TestGeneration.dfy | 8 ++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Source/XUnitExtensions/OrCommand.cs b/Source/XUnitExtensions/OrCommand.cs index 3c6777aef27..26948169557 100644 --- a/Source/XUnitExtensions/OrCommand.cs +++ b/Source/XUnitExtensions/OrCommand.cs @@ -9,14 +9,18 @@ public OrCommand(ILitCommand lhs, ILitCommand rhs) { this.lhs = lhs; this.rhs = rhs; } - + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter) { var (exitCode, output, error) = lhs.Execute(inputReader, outputWriter); if (exitCode == 0) { return (exitCode, output, error); - } else { - return rhs.Execute(inputReader, outputWriter); } + + return rhs.Execute(inputReader, outputWriter); + } + + public override string ToString() { + return $"{lhs} || {rhs}"; } } } \ No newline at end of file diff --git a/Test/DafnyTestGeneration/TestGeneration.dfy b/Test/DafnyTestGeneration/TestGeneration.dfy index 1b405bf2b9c..df376f9fdfa 100644 --- a/Test/DafnyTestGeneration/TestGeneration.dfy +++ b/Test/DafnyTestGeneration/TestGeneration.dfy @@ -1,17 +1,17 @@ // Generating tests: -// RUN: cp TestGeneration.dfy %t.dfy +// RUN: cp %S/TestGeneration.dfy %t.dfy // RUN: %dafny /definiteAssignment:3 /generateTestMode:Block %t.dfy > %t-tests.dfy // Compiling test to java: -// RUN: cd Output; %dafny /compileTarget:java %t-tests.dfy; cd ../ +// RUN: %dafny /compileTarget:java /out:Output %t-tests.dfy // Adding reflection code that allows running the tests: // RUN: perl -pe 's/import M_Compile.*;/`cat import.txt`/ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java // RUN: perl -pe 's/public class __default \{/`cat reflectionCode.txt`/ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java // Compiling to bytecode and running the tests -// RUN: javac -cp %t-tests-java:../../Binaries/DafnyRuntime.jar %t-tests-java/TestGenerationUnitTests_Compile/__default.java -// RUN: java -cp %t-tests-java:../../Binaries/DafnyRuntime.jar TestGenerationUnitTests_Compile/__default > %t +// RUN: javac -cp %t-tests-java:%binaries/DafnyRuntime.jar %t-tests-java/TestGenerationUnitTests_Compile/__default.java +// RUN: java -cp %t-tests-java:%binaries/DafnyRuntime.jar TestGenerationUnitTests_Compile/__default > %t // RUN: %diff "%s.expect" "%t" module M { From 642ece03dfed34fd95b95bfd9afa5a41cd097c01 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 15 Oct 2021 15:48:07 -0700 Subject: [PATCH 152/192] Clean up pointing at released dafny --- .github/workflows/msbuild.yml | 2 +- Source/IntegrationTests/LitTests.cs | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index 5558a3996c3..29be441cf7b 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -126,4 +126,4 @@ jobs: unzip dafny/Package/CI.zip -d unzippedRelease - name: Run integration tests run: | - XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 DAFNY_EXECUTABLE=$PWD/unzippedRelease/dafny/dafny DAFNY_SERVER_EXECUTABLE=$PWD/unzippedRelease/dafny/dafnyServer dotnet test dafny/Source/IntegrationTests/IntegrationTests.csproj + XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 DAFNY_RELEASE=$PWD/unzippedRelease/dafny dotnet test dafny/Source/IntegrationTests/IntegrationTests.csproj diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 8582c3e61c4..4e7f2d7d804 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -51,7 +51,7 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable - new ShellLitCommand(dafnyExecutable, args, config.PassthroughEnvironmentVariables); + new ShellLitCommand(Path.Join(dafnyReleaseDir, "dafny"), args, config.PassthroughEnvironmentVariables); CONFIG.Commands["%dafny"] = (args, config) => - new ShellLitCommand(dafnyExecutable, defaultDafnyArguments.Concat(args), config.PassthroughEnvironmentVariables); - } - var dafnyServerExecutable = Environment.GetEnvironmentVariable("DAFNY_SERVER_EXECUTABLE"); - if (dafnyServerExecutable != null) { + new ShellLitCommand(Path.Join(dafnyReleaseDir, "dafny"), defaultDafnyArguments.Concat(args), config.PassthroughEnvironmentVariables); CONFIG.Commands["%server"] = (args, config) => - new ShellLitCommand(dafnyServerExecutable, args, config.PassthroughEnvironmentVariables); - } + new ShellLitCommand(Path.Join(dafnyReleaseDir, "DafnyServer"), args, config.PassthroughEnvironmentVariables); + CONFIG.Substitions["%z3"] = Path.Join(dafnyReleaseDir, "z3", "bin", "z3"); + } } private readonly ITestOutputHelper output; From cdf6a00a8eb8dcc9dc6c1e9a6574ec044fd534ec Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 15 Oct 2021 16:23:19 -0700 Subject: [PATCH 153/192] Add 2> support to fix TestAttribute.dfy --- Source/XUnitExtensions/ILitCommand.cs | 2 +- .../LitCommandWithRedirection.cs | 58 +++++++++---------- Source/XUnitExtensions/LitTestCase.cs | 2 +- .../XUnitExtensions/MainMethodLitCommand.cs | 12 ++-- Source/XUnitExtensions/OrCommand.cs | 7 +-- Source/XUnitExtensions/ShellLitCommand.cs | 4 +- Source/XUnitExtensions/XFailCommand.cs | 2 +- Test/DafnyTests/TestAttribute.dfy | 2 +- 8 files changed, 44 insertions(+), 45 deletions(-) diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index b2676286720..ac94acf5556 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -13,7 +13,7 @@ public interface ILitCommand { private const string LIT_COMMAND_PREFIX = "RUN:"; private const string LIT_XFAIL = "XFAIL: *"; - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter); + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter); public static ILitCommand Parse(string fileName, string line, LitTestConfiguration config) { if (!line.StartsWith(COMMENT_PREFIX)) { diff --git a/Source/XUnitExtensions/LitCommandWithRedirection.cs b/Source/XUnitExtensions/LitCommandWithRedirection.cs index 7e4bc770ccb..96ae9d9ad6e 100644 --- a/Source/XUnitExtensions/LitCommandWithRedirection.cs +++ b/Source/XUnitExtensions/LitCommandWithRedirection.cs @@ -8,67 +8,65 @@ public class LitCommandWithRedirection : ILitCommand { public static LitCommandWithRedirection Parse(string[] tokens, LitTestConfiguration config) { var commandSymbol = tokens[0]; - var (rawArguments, inputFile, outputFile, appendOutput) = ExtractRedirections(tokens[1..]); - if (inputFile != null) { - inputFile = config.ApplySubstitutions(inputFile); - } - if (outputFile != null) { - outputFile = config.ApplySubstitutions(outputFile); - } - - var arguments = rawArguments.Select(config.ApplySubstitutions); - - if (config.Commands.TryGetValue(commandSymbol, out var command)) { - return new LitCommandWithRedirection(command(arguments, config), inputFile, outputFile, appendOutput); - } - - commandSymbol = config.ApplySubstitutions(commandSymbol); - - return new LitCommandWithRedirection( - new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), - inputFile, outputFile, appendOutput); - } - - public static (IEnumerable, string, string, bool) ExtractRedirections(IEnumerable arguments) { - var argumentsList = arguments.ToList(); + var argumentsList = tokens[1..].ToList(); string inputFile = null; string outputFile = null; bool appendOutput = false; + string errorFile = null; var redirectInIndex = argumentsList.IndexOf("<"); if (redirectInIndex >= 0) { - inputFile = argumentsList[redirectInIndex + 1]; + inputFile = config.ApplySubstitutions(argumentsList[redirectInIndex + 1]); argumentsList.RemoveRange(redirectInIndex, 2); } var redirectOutIndex = argumentsList.IndexOf(">"); if (redirectOutIndex >= 0) { - outputFile = argumentsList[redirectOutIndex + 1]; + outputFile = config.ApplySubstitutions(argumentsList[redirectOutIndex + 1]); argumentsList.RemoveRange(redirectOutIndex, 2); } var redirectAppendIndex = argumentsList.IndexOf(">>"); if (redirectAppendIndex >= 0) { - outputFile = argumentsList[redirectAppendIndex + 1]; + outputFile = config.ApplySubstitutions(argumentsList[redirectAppendIndex + 1]); appendOutput = true; argumentsList.RemoveRange(redirectAppendIndex, 2); } - return (argumentsList, inputFile, outputFile, appendOutput); + var redirectErrorIndex = argumentsList.IndexOf("2>"); + if (redirectErrorIndex >= 0) { + errorFile = config.ApplySubstitutions(argumentsList[redirectErrorIndex + 1]); + argumentsList.RemoveRange(redirectErrorIndex, 2); + } + + var arguments = argumentsList.Select(config.ApplySubstitutions); + + if (config.Commands.TryGetValue(commandSymbol, out var command)) { + return new LitCommandWithRedirection(command(arguments, config), inputFile, outputFile, appendOutput, errorFile); + } + + commandSymbol = config.ApplySubstitutions(commandSymbol); + + return new LitCommandWithRedirection( + new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), + inputFile, outputFile, appendOutput, errorFile); } private ILitCommand command; private string inputFile; private string outputFile; private bool append; + private string errorFile; - public LitCommandWithRedirection(ILitCommand command, string inputFile, string outputFile, bool append) { + public LitCommandWithRedirection(ILitCommand command, string inputFile, string outputFile, bool append, string errorFile) { this.command = command; this.inputFile = inputFile; this.outputFile = outputFile; this.append = append; + this.errorFile = errorFile; } - public (int, string, string) Execute(TextReader inReader, TextWriter outWriter) { + public (int, string, string) Execute(TextReader inReader, TextWriter outWriter, TextWriter errWriter) { var inputReader = inputFile != null ? new StreamReader(inputFile) : null; var outputWriter = outputFile != null ? new StreamWriter(outputFile, append) : null; - return command.Execute(inputReader, outputWriter); + var errorWriter = errorFile != null ? new StreamWriter(errorFile, false) : null; + return command.Execute(inputReader, outputWriter, errorWriter); } public override string ToString() { diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index 927501b4aab..b70e1ff022b 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -49,7 +49,7 @@ public void Execute(ITestOutputHelper outputHelper) { string error; try { outputHelper.WriteLine($"Executing command: {command}"); - (exitCode, output, error) = command.Execute(null, null); + (exitCode, output, error) = command.Execute(null, null, null); } catch (Exception e) { throw new Exception($"Exception thrown while executing command: {command}", e); } diff --git a/Source/XUnitExtensions/MainMethodLitCommand.cs b/Source/XUnitExtensions/MainMethodLitCommand.cs index 29f1b06573b..efecc347e27 100644 --- a/Source/XUnitExtensions/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/MainMethodLitCommand.cs @@ -19,20 +19,20 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments return invokeDirectly ? result : result.ToShellCommand(config); } - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter) { - StringBuilder redirectedErr = new(); - + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { if (inputReader != null) { Console.SetIn(inputReader); } if (outputWriter != null) { Console.SetOut(outputWriter); } - Console.SetError(new StringWriter(redirectedErr)); - + if (errorWriter != null) { + Console.SetError(errorWriter); + } + var exitCode = (int)Assembly.EntryPoint!.Invoke(null, new object[] { Arguments }); - return (exitCode, "", redirectedErr.ToString()); + return (exitCode, "", ""); } public ILitCommand ToShellCommand(LitTestConfiguration config) { diff --git a/Source/XUnitExtensions/OrCommand.cs b/Source/XUnitExtensions/OrCommand.cs index 26948169557..282eb1a64d0 100644 --- a/Source/XUnitExtensions/OrCommand.cs +++ b/Source/XUnitExtensions/OrCommand.cs @@ -10,13 +10,12 @@ public OrCommand(ILitCommand lhs, ILitCommand rhs) { this.rhs = rhs; } - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter) { - var (exitCode, output, error) = lhs.Execute(inputReader, outputWriter); + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + var (exitCode, output, error) = lhs.Execute(inputReader, outputWriter, errorWriter); if (exitCode == 0) { return (exitCode, output, error); } - - return rhs.Execute(inputReader, outputWriter); + return rhs.Execute(inputReader, outputWriter, errorWriter); } public override string ToString() { diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/ShellLitCommand.cs index 0626ced0e9f..a05c5858f59 100644 --- a/Source/XUnitExtensions/ShellLitCommand.cs +++ b/Source/XUnitExtensions/ShellLitCommand.cs @@ -17,7 +17,7 @@ public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnum this.passthroughEnvironmentVariables = passthroughEnvironmentVariables.ToArray(); } - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter) { + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { using var process = new Process(); process.StartInfo.FileName = shellCommand; @@ -49,6 +49,8 @@ public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnum outputWriter?.Write(output); outputWriter?.Flush(); string error = process.StandardError.ReadToEnd(); + errorWriter?.Write(error); + errorWriter?.Flush(); process.WaitForExit(); return (process.ExitCode, output, error); diff --git a/Source/XUnitExtensions/XFailCommand.cs b/Source/XUnitExtensions/XFailCommand.cs index 9e427441e8b..b754a703ef9 100644 --- a/Source/XUnitExtensions/XFailCommand.cs +++ b/Source/XUnitExtensions/XFailCommand.cs @@ -5,7 +5,7 @@ public class XFailCommand : ILitCommand { public XFailCommand() {} - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter) { + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { return (0, "", ""); } } diff --git a/Test/DafnyTests/TestAttribute.dfy b/Test/DafnyTests/TestAttribute.dfy index 0f1858d9f80..6e9bd0ad658 100644 --- a/Test/DafnyTests/TestAttribute.dfy +++ b/Test/DafnyTests/TestAttribute.dfy @@ -1,5 +1,5 @@ // RUN: %dafny /out:Output/DafnyMain.cs TestAttribute.dfy /compile:0 /spillTargetCode:3 /noVerify -// RUN: dotnet test -v:q -noLogo 2>%t.raw || true +// RUN: dotnet test -v:q -noLogo %S 2> %t.raw || true // Remove the timestamp prefixes on the expected errors // RUN: sed 's/[^]]*\]//' "%t".raw > "%t" // RUN: %diff "%s.expect" "%t" From 33dd9499054c05b7abbb22590952528fe459a206 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 15 Oct 2021 16:28:41 -0700 Subject: [PATCH 154/192] Formatting --- Source/IntegrationTests/LitTests.cs | 20 +++++++++---------- Source/XUnitExtensions/FileDataAttribute.cs | 2 +- Source/XUnitExtensions/FileTestCase.cs | 10 +++++----- Source/XUnitExtensions/FileTheoryDataRow.cs | 18 ++++++++--------- .../XUnitExtensions/FileTheoryDiscoverer.cs | 2 +- Source/XUnitExtensions/ILitCommand.cs | 10 +++++----- Source/XUnitExtensions/ITheoryDataRow.cs | 2 +- .../LitCommandWithRedirection.cs | 8 ++++---- Source/XUnitExtensions/LitTestCase.cs | 12 +++++------ .../XUnitExtensions/LitTestConfiguration.cs | 4 ++-- .../XUnitExtensions/LitTestDataDiscoverer.cs | 4 ++-- .../XUnitExtensions/MainMethodLitCommand.cs | 6 +++--- Source/XUnitExtensions/ShellLitCommand.cs | 4 ++-- Source/XUnitExtensions/XFailCommand.cs | 4 ++-- 14 files changed, 53 insertions(+), 53 deletions(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 4e7f2d7d804..d0c7fbb755e 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -32,21 +32,21 @@ public class LitTests { // Set a default time limit, to catch cases where verification time runs off the rails "/timeLimit:300" }; - + private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable arguments, LitTestConfiguration config, bool invokeDirectly) { return MainMethodLitCommand.Parse(assembly, arguments, config, invokeDirectly); } - private static readonly LitTestConfiguration CONFIG = new () { + private static readonly LitTestConfiguration CONFIG = new() { Commands = new Dictionary, LitTestConfiguration, ILitCommand>> { - { "%baredafny", (args, config) => + { "%baredafny", (args, config) => MainWithArguments(dafnyDriverAssembly, args, config, false) }, - { "%dafny", (args, config) => + { "%dafny", (args, config) => MainWithArguments(dafnyDriverAssembly, defaultDafnyArguments.Concat(args), config, false) }, - { "%server", (args, config) => + { "%server", (args, config) => MainWithArguments(dafnyServerAssembly, args, config, false) }, }, - + Substitions = new Dictionary { // TODO: speed this up by using AssertWithDiff { "%diff", "diff" }, @@ -54,8 +54,8 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable GetData(MethodInfo testMethod) { diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index b0d01d98669..8f5d1b1eb90 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -12,7 +12,7 @@ public class FileTestCase : LongLivedMarshalByRefObject, IXunitTestCase { // This could use SkippableFactDiscoverer.GetSkippableExceptionNames(IAttributeInfo) // but it doesn't seem to be worth the complexity here yet. private static readonly string[] skippingExceptionNames = { typeof(SkipException).FullName }; - + protected XunitTestCase innerTestCase; public string DisplayName { get; protected set; } @@ -23,11 +23,11 @@ public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, (ITypeInfo)null, "Test collection for " + data.TestDisplayName); var testClassWithCollection = new TestClass(collection, testMethod.TestClass.Class); var testMethodWithCollection = new TestMethod(testClassWithCollection, testMethod.Method); - + innerTestCase = new SkippableFactTestCase(skippingExceptionNames, diagnosticMessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.All, testMethodWithCollection, data.GetData()); if (data.Traits != null) { - foreach(var (key, value) in data.Traits) { + foreach (var (key, value) in data.Traits) { innerTestCase.Traits.Add(key, value); } } @@ -51,7 +51,7 @@ public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(DisplayName), DisplayName); info.AddValue(nameof(SkipReason), SkipReason); } - + public ISourceInformation SourceInformation { get => innerTestCase.SourceInformation; set => innerTestCase.SourceInformation = value; @@ -64,7 +64,7 @@ public ISourceInformation SourceInformation { public Exception InitializationException => innerTestCase.InitializationException; public IMethodInfo Method => innerTestCase.Method; public int Timeout => innerTestCase.Timeout; - + public async Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { diff --git a/Source/XUnitExtensions/FileTheoryDataRow.cs b/Source/XUnitExtensions/FileTheoryDataRow.cs index 96b32168274..da847da132d 100644 --- a/Source/XUnitExtensions/FileTheoryDataRow.cs +++ b/Source/XUnitExtensions/FileTheoryDataRow.cs @@ -3,23 +3,23 @@ namespace XUnitExtensions { public class FileTheoryDataRow : IXunitSerializable, IFileTheoryRowData { - + object?[] data; public FileTheoryDataRow() { - + } - - public FileTheoryDataRow(params object?[] data) { + + public FileTheoryDataRow(params object?[] data) { this.data = data; } - - public ISourceInformation? SourceInformation { get; set; } + + public ISourceInformation? SourceInformation { get; set; } public string? Skip { get; set; } - public string? TestDisplayName { get; set; } + public string? TestDisplayName { get; set; } public Dictionary>? Traits { get; set; } public object?[] GetData() => data; - + public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(SourceInformation), SourceInformation); info.AddValue(nameof(Skip), Skip); @@ -27,7 +27,7 @@ public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(Traits), Traits); info.AddValue(nameof(data), data); } - + public void Deserialize(IXunitSerializationInfo info) { SourceInformation = info.GetValue(nameof(SourceInformation)); Skip = info.GetValue(nameof(Skip)); diff --git a/Source/XUnitExtensions/FileTheoryDiscoverer.cs b/Source/XUnitExtensions/FileTheoryDiscoverer.cs index 9cf933d2fb4..09eebd2d668 100644 --- a/Source/XUnitExtensions/FileTheoryDiscoverer.cs +++ b/Source/XUnitExtensions/FileTheoryDiscoverer.cs @@ -21,7 +21,7 @@ protected override IEnumerable CreateTestCasesForDataRow(ITestFr if (dataRow.Length == 1 && dataRow[0] is IFileTheoryRowData theoryRowData) { return new[] { new FileTestCase(DiagnosticMessageSink, testMethod, theoryRowData) }; } - + return base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); } } diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index ac94acf5556..bb53e2fb9c9 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -8,13 +8,13 @@ namespace XUnitExtensions { public interface ILitCommand { - + private const string COMMENT_PREFIX = "//"; private const string LIT_COMMAND_PREFIX = "RUN:"; private const string LIT_XFAIL = "XFAIL: *"; public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter); - + public static ILitCommand Parse(string fileName, string line, LitTestConfiguration config) { if (!line.StartsWith(COMMENT_PREFIX)) { return null; @@ -28,7 +28,7 @@ public static ILitCommand Parse(string fileName, string line, LitTestConfigurati return null; } line = line[LIT_COMMAND_PREFIX.Length..].Trim(); - + var tokens = Tokenize(line); return ParseRunCommand(tokens, config); } @@ -43,7 +43,7 @@ public static ILitCommand ParseRunCommand(string[] tokens, LitTestConfiguration } return LitCommandWithRedirection.Parse(tokens, config); } - + private static string[] Tokenize(string line) { var arguments = new List(); var argument = new StringBuilder(); @@ -66,7 +66,7 @@ private static string[] Tokenize(string line) { if (argument.Length != 0) { arguments.Add(argument.ToString()); } - + return arguments.ToArray(); } } diff --git a/Source/XUnitExtensions/ITheoryDataRow.cs b/Source/XUnitExtensions/ITheoryDataRow.cs index 4628b715bac..709a1971e4b 100644 --- a/Source/XUnitExtensions/ITheoryDataRow.cs +++ b/Source/XUnitExtensions/ITheoryDataRow.cs @@ -11,7 +11,7 @@ public interface IFileTheoryRowData { /// of the test method is ussed. /// ISourceInformation? SourceInformation { get; } - + /// /// Gets the reason for skipping this row of data; if null is returned, then the data /// row isn't skipped. diff --git a/Source/XUnitExtensions/LitCommandWithRedirection.cs b/Source/XUnitExtensions/LitCommandWithRedirection.cs index 96ae9d9ad6e..0ffde63a749 100644 --- a/Source/XUnitExtensions/LitCommandWithRedirection.cs +++ b/Source/XUnitExtensions/LitCommandWithRedirection.cs @@ -34,9 +34,9 @@ public static LitCommandWithRedirection Parse(string[] tokens, LitTestConfigurat errorFile = config.ApplySubstitutions(argumentsList[redirectErrorIndex + 1]); argumentsList.RemoveRange(redirectErrorIndex, 2); } - + var arguments = argumentsList.Select(config.ApplySubstitutions); - + if (config.Commands.TryGetValue(commandSymbol, out var command)) { return new LitCommandWithRedirection(command(arguments, config), inputFile, outputFile, appendOutput, errorFile); } @@ -44,10 +44,10 @@ public static LitCommandWithRedirection Parse(string[] tokens, LitTestConfigurat commandSymbol = config.ApplySubstitutions(commandSymbol); return new LitCommandWithRedirection( - new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), + new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), inputFile, outputFile, appendOutput, errorFile); } - + private ILitCommand command; private string inputFile; private string outputFile; diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index b70e1ff022b..d54497f3c2f 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -12,7 +12,7 @@ public class LitTestCase { private string filePath; private IEnumerable commands; private bool expectFailure; - + public static LitTestCase Read(string filePath, LitTestConfiguration config) { var commands = File.ReadAllLines(filePath) .Select(line => ILitCommand.Parse(filePath, line, config)) @@ -20,7 +20,7 @@ public static LitTestCase Read(string filePath, LitTestConfiguration config) { var xfail = commands.Any(c => c is XFailCommand); return new LitTestCase(filePath, commands, xfail); } - + public static void Run(string filePath, LitTestConfiguration config, ITestOutputHelper outputHelper) { string fileName = Path.GetFileName(filePath); string directory = Path.GetDirectoryName(filePath); @@ -39,10 +39,10 @@ public LitTestCase(string filePath, IEnumerable commands, bool expe this.commands = commands; this.expectFailure = expectFailure; } - + public void Execute(ITestOutputHelper outputHelper) { Directory.CreateDirectory(Path.Join(Path.GetDirectoryName(filePath), "Output")); - + foreach (var command in commands) { int exitCode; string output; @@ -59,12 +59,12 @@ public void Execute(ITestOutputHelper outputHelper) { throw new SkipException($"Command returned non-zero exit code ({exitCode}): {command}\nOutput:\n{output}\nError:\n{error}"); } } - + if (exitCode != 0) { throw new Exception($"Command returned non-zero exit code ({exitCode}): {command}\nOutput:\n{output}\nError:\n{error}"); } } - + if (expectFailure) { throw new Exception($"Test case passed but expected to fail: {filePath}"); } diff --git a/Source/XUnitExtensions/LitTestConfiguration.cs b/Source/XUnitExtensions/LitTestConfiguration.cs index b3263cf5afa..8c2b6078b89 100644 --- a/Source/XUnitExtensions/LitTestConfiguration.cs +++ b/Source/XUnitExtensions/LitTestConfiguration.cs @@ -5,11 +5,11 @@ using Xunit.Abstractions; namespace XUnitExtensions { - + // TODO: Make safely immutable public class LitTestConfiguration { public Dictionary Substitions { get; set; } - + public Dictionary, LitTestConfiguration, ILitCommand>> Commands { get; set; } public IEnumerable PassthroughEnvironmentVariables { get; set; } diff --git a/Source/XUnitExtensions/LitTestDataDiscoverer.cs b/Source/XUnitExtensions/LitTestDataDiscoverer.cs index cdb7f8da78e..cb60673ca43 100644 --- a/Source/XUnitExtensions/LitTestDataDiscoverer.cs +++ b/Source/XUnitExtensions/LitTestDataDiscoverer.cs @@ -17,7 +17,7 @@ protected override IEnumerable FileData(IAttributeInfo attributeInfo, var basePath = GetBasePath(attributeInfo, testMethod); var shortPath = fileName[(basePath.Length + 1)..]; var row = new FileTheoryDataRow(fileName) { - SourceInformation = new SourceInformation() { FileName = fileName, LineNumber = 0}, + SourceInformation = new SourceInformation() { FileName = fileName, LineNumber = 0 }, TestDisplayName = shortPath, }; return new[] { new[] { row } }; @@ -27,5 +27,5 @@ protected override IEnumerable FileData(IAttributeInfo attributeInfo, [DataDiscoverer("XUnitExtensions.LitTestDataDiscoverer", "XUnitExtensions")] public class LitTestDataAttribute : FileDataAttribute { - + } \ No newline at end of file diff --git a/Source/XUnitExtensions/MainMethodLitCommand.cs b/Source/XUnitExtensions/MainMethodLitCommand.cs index efecc347e27..b3417cbaf6d 100644 --- a/Source/XUnitExtensions/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/MainMethodLitCommand.cs @@ -29,9 +29,9 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments if (errorWriter != null) { Console.SetError(errorWriter); } - + var exitCode = (int)Assembly.EntryPoint!.Invoke(null, new object[] { Arguments }); - + return (exitCode, "", ""); } @@ -39,7 +39,7 @@ public ILitCommand ToShellCommand(LitTestConfiguration config) { var shellArguments = new[] { Assembly.Location }.Concat(Arguments); return new ShellLitCommand("dotnet", shellArguments, config.PassthroughEnvironmentVariables); } - + public override string ToString() { var builder = new StringBuilder(); builder.Append(Assembly.EntryPoint); diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/ShellLitCommand.cs index a05c5858f59..407610f2981 100644 --- a/Source/XUnitExtensions/ShellLitCommand.cs +++ b/Source/XUnitExtensions/ShellLitCommand.cs @@ -27,7 +27,7 @@ public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnum // We avoid setting the current directory so that we maintain parity with // MainMethodLitCommand, which can't change the current directory. - + process.StartInfo.UseShellExecute = false; process.StartInfo.RedirectStandardInput = true; process.StartInfo.RedirectStandardOutput = true; @@ -55,7 +55,7 @@ public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnum return (process.ExitCode, output, error); } - + public override string ToString() { var builder = new StringBuilder(); builder.Append(shellCommand); diff --git a/Source/XUnitExtensions/XFailCommand.cs b/Source/XUnitExtensions/XFailCommand.cs index b754a703ef9..8f8f4948def 100644 --- a/Source/XUnitExtensions/XFailCommand.cs +++ b/Source/XUnitExtensions/XFailCommand.cs @@ -3,8 +3,8 @@ namespace XUnitExtensions { public class XFailCommand : ILitCommand { - public XFailCommand() {} - + public XFailCommand() { } + public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { return (0, "", ""); } From 04b7daadf927f598e7b385edff050045829fac87 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 15 Oct 2021 16:30:50 -0700 Subject: [PATCH 155/192] Build Java runtime before build in refman.yml --- .github/workflows/refman.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/refman.yml b/.github/workflows/refman.yml index cc8b3bd593b..4a68077096b 100644 --- a/.github/workflows/refman.yml +++ b/.github/workflows/refman.yml @@ -29,6 +29,9 @@ jobs: with: submodules: recursive path: dafny + - name: Build Java runtime + working-directory: dafny + run: make --quiet runtime - name: Build Dafny run: dotnet build dafny/Source/Dafny.sln - name: Install latex pandoc - Linux From 210aca7b27a643c38cd0a1f780ec62309e537c50 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Fri, 15 Oct 2021 16:47:35 -0700 Subject: [PATCH 156/192] Remove over-zealous %S uses --- Test/comp/compile1quiet/CompileRunQuietly.dfy | 4 ++-- Test/comp/compile1verbose/CompileAndThenRun.dfy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Test/comp/compile1quiet/CompileRunQuietly.dfy b/Test/comp/compile1quiet/CompileRunQuietly.dfy index 5393f9008fd..9ac8092b542 100644 --- a/Test/comp/compile1quiet/CompileRunQuietly.dfy +++ b/Test/comp/compile1quiet/CompileRunQuietly.dfy @@ -1,5 +1,5 @@ // RUN: %dafny /compileTarget:cs "%s" > "%t" -// RUN: dotnet %S/CompileRunQuietly.dll >> "%t" +// RUN: dotnet CompileRunQuietly.dll >> "%t" // RUN: %dafny /compileTarget:js "%s" >> "%t" // RUN: node %S/CompileRunQuietly.js >> "%t" @@ -11,7 +11,7 @@ // RUN: java -cp %binaries/DafnyRuntime.jar:%S/CompileRunQuietly-java CompileRunQuietly >> "%t" // RUN: %dafny /compileTarget:cpp "%s" >> "%t" -// RUN: %S/CompileRunQuietly.exe >> "%t" +// RUN: CompileRunQuietly.exe >> "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/comp/compile1verbose/CompileAndThenRun.dfy b/Test/comp/compile1verbose/CompileAndThenRun.dfy index b0f3b62ad3d..9fa4140bcf5 100644 --- a/Test/comp/compile1verbose/CompileAndThenRun.dfy +++ b/Test/comp/compile1verbose/CompileAndThenRun.dfy @@ -1,5 +1,5 @@ // RUN: %dafny /compileVerbose:1 /compileTarget:cs "%s" > "%t" -// RUN: dotnet %S/CompileAndThenRun.dll >> "%t" +// RUN: dotnet CompileAndThenRun.dll >> "%t" // RUN: %dafny /compileVerbose:1 /compileTarget:js "%s" >> "%t" // RUN: node %S/CompileAndThenRun.js >> "%t" @@ -11,7 +11,7 @@ // RUN: java -cp %binaries/DafnyRuntime.jar:%S/CompileAndThenRun-java CompileAndThenRun >> "%t" // RUN: %dafny /compileVerbose:1 /compileTarget:cpp "%s" >> "%t" -// RUN: %S/CompileAndThenRun.exe >> "%t" +// RUN: CompileAndThenRun.exe >> "%t" // RUN: %diff "%s.expect" "%t" From b41e4d14d5fdb31546f678cedcd2ad08957c86dc Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sat, 16 Oct 2021 10:56:33 -0700 Subject: [PATCH 157/192] Add globbing support, continue working on backticks --- Source/IntegrationTests/LitTests.cs | 9 ++-- Source/XUnitExtensions/ILitCommand.cs | 46 ++++++++++++++++++- .../LitCommandWithRedirection.cs | 10 ++-- Source/XUnitExtensions/LitTestCase.cs | 4 +- .../XUnitExtensions/MainMethodLitCommand.cs | 7 ++- Source/XUnitExtensions/OrCommand.cs | 7 +-- Source/XUnitExtensions/ShellLitCommand.cs | 11 +++-- Source/XUnitExtensions/XFailCommand.cs | 3 +- Source/XUnitExtensions/XUnitExtensions.csproj | 1 + Test/DafnyTestGeneration/TestGeneration.dfy | 10 ++-- Test/comp/compile1quiet/CompileRunQuietly.dfy | 2 +- .../compile1verbose/CompileAndThenRun.dfy | 2 +- Test/comp/manualcompile/ManualCompile.dfy | 10 ++-- Test/comp/manualcompile/lit.local.cfg | 3 -- Test/lit.site.cfg | 2 +- 15 files changed, 91 insertions(+), 36 deletions(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index d0c7fbb755e..3db7d004683 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -50,7 +50,7 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable { // TODO: speed this up by using AssertWithDiff { "%diff", "diff" }, - { "%binaries", "." }, + { "%binaryDir", "." }, { "%z3", Path.Join("z3", "bin", "z3") }, { "%refmanexamples", Path.Join("TestFiles", "LitTests", "LitTest", "refman", "examples") } }, @@ -59,15 +59,14 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable - new ShellLitCommand(Path.Join(dafnyReleaseDir, "dafny"), args, config.PassthroughEnvironmentVariables); + new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "dafny"), args, config.PassthroughEnvironmentVariables); CONFIG.Commands["%dafny"] = (args, config) => - new ShellLitCommand(Path.Join(dafnyReleaseDir, "dafny"), defaultDafnyArguments.Concat(args), config.PassthroughEnvironmentVariables); + new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "dafny"), defaultDafnyArguments.Concat(args), config.PassthroughEnvironmentVariables); CONFIG.Commands["%server"] = (args, config) => - new ShellLitCommand(Path.Join(dafnyReleaseDir, "DafnyServer"), args, config.PassthroughEnvironmentVariables); + new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "DafnyServer"), args, config.PassthroughEnvironmentVariables); CONFIG.Substitions["%z3"] = Path.Join(dafnyReleaseDir, "z3", "bin", "z3"); } } diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index bb53e2fb9c9..2fc38023cd1 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -5,6 +5,10 @@ using System.Linq; using System.Reflection; using System.Text; +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; +using Xunit; +using Xunit.Abstractions; namespace XUnitExtensions { public interface ILitCommand { @@ -13,8 +17,27 @@ public interface ILitCommand { private const string LIT_COMMAND_PREFIX = "RUN:"; private const string LIT_XFAIL = "XFAIL: *"; - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter); + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter); + public void ExecuteWithExpectation(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter, bool expectFailure = false) { + try { + outputHelper.WriteLine($"Executing command: {this}"); + var (exitCode, output, error) = Execute(outputHelper, inputReader, outputWriter, errorWriter); + + if (expectFailure) { + if (exitCode != 0) { + throw new SkipException($"Command returned non-zero exit code ({exitCode}): {this}\nOutput:\n{output}\nError:\n{error}"); + } + } + + if (exitCode != 0) { + throw new Exception($"Command returned non-zero exit code ({exitCode}): {this}\nOutput:\n{output}\nError:\n{error}"); + } + } catch (Exception e) { + throw new Exception($"Exception thrown while executing command: {this}", e); + } + } + public static ILitCommand Parse(string fileName, string line, LitTestConfiguration config) { if (!line.StartsWith(COMMENT_PREFIX)) { return null; @@ -53,6 +76,7 @@ private static string[] Tokenize(string line) { var c = line[i]; if (c == '\'') { singleQuoted = !singleQuoted; + argument.Append(c); } else if (c == '"') { doubleQuoted = !doubleQuoted; } else if (Char.IsWhiteSpace(c) && !(singleQuoted || doubleQuoted)) { @@ -69,5 +93,25 @@ private static string[] Tokenize(string line) { return arguments.ToArray(); } + + protected static IEnumerable ExpandGlobsAndBackticks(string chunk, LitTestConfiguration config, ITestOutputHelper outputHelper) { + if (chunk.Contains('*') || chunk.Contains('?')) { + var matcher = new Matcher(); + matcher.AddInclude(chunk); + var result = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo("."))); + return result.Files.Select(f => f.Path); + } + + var firstTickIndex = chunk.IndexOf('`'); + if (firstTickIndex >= 0) { + var secondTickIndex = chunk.IndexOf('`', firstTickIndex + 1); + var toParse = chunk[(firstTickIndex + 1)..(secondTickIndex)]; + ILitCommand command = LitCommandWithRedirection.Parse(Tokenize(toParse), config); + var output = new StringBuilder(); + command.ExecuteWithExpectation(outputHelper, null, new StringWriter(output), null); + return new[] { chunk[..firstTickIndex] + output + chunk[(secondTickIndex + 1)..] }; + } + return new[] { chunk }; + } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/LitCommandWithRedirection.cs b/Source/XUnitExtensions/LitCommandWithRedirection.cs index 0ffde63a749..5e126bfba03 100644 --- a/Source/XUnitExtensions/LitCommandWithRedirection.cs +++ b/Source/XUnitExtensions/LitCommandWithRedirection.cs @@ -2,6 +2,10 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.RegularExpressions; +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; +using Xunit.Abstractions; namespace XUnitExtensions { public class LitCommandWithRedirection : ILitCommand { @@ -44,7 +48,7 @@ public static LitCommandWithRedirection Parse(string[] tokens, LitTestConfigurat commandSymbol = config.ApplySubstitutions(commandSymbol); return new LitCommandWithRedirection( - new ShellLitCommand(commandSymbol, arguments, config.PassthroughEnvironmentVariables), + new ShellLitCommand(config, commandSymbol, arguments, config.PassthroughEnvironmentVariables), inputFile, outputFile, appendOutput, errorFile); } @@ -62,11 +66,11 @@ public LitCommandWithRedirection(ILitCommand command, string inputFile, string o this.errorFile = errorFile; } - public (int, string, string) Execute(TextReader inReader, TextWriter outWriter, TextWriter errWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inReader, TextWriter outWriter, TextWriter errWriter) { var inputReader = inputFile != null ? new StreamReader(inputFile) : null; var outputWriter = outputFile != null ? new StreamWriter(outputFile, append) : null; var errorWriter = errorFile != null ? new StreamWriter(errorFile, false) : null; - return command.Execute(inputReader, outputWriter, errorWriter); + return command.Execute(outputHelper, inputReader, outputWriter, errorWriter); } public override string ToString() { diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index d54497f3c2f..64b3a69a72d 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -23,7 +23,7 @@ public static LitTestCase Read(string filePath, LitTestConfiguration config) { public static void Run(string filePath, LitTestConfiguration config, ITestOutputHelper outputHelper) { string fileName = Path.GetFileName(filePath); - string directory = Path.GetDirectoryName(filePath); + string directory = Path.GetFullPath(Path.GetDirectoryName(filePath)); config = config.WithSubstitutions(new Dictionary { { "%s", filePath }, { "%S", directory }, @@ -49,7 +49,7 @@ public void Execute(ITestOutputHelper outputHelper) { string error; try { outputHelper.WriteLine($"Executing command: {command}"); - (exitCode, output, error) = command.Execute(null, null, null); + (exitCode, output, error) = command.Execute(outputHelper, null, null, null); } catch (Exception e) { throw new Exception($"Exception thrown while executing command: {command}", e); } diff --git a/Source/XUnitExtensions/MainMethodLitCommand.cs b/Source/XUnitExtensions/MainMethodLitCommand.cs index b3417cbaf6d..98e3bbc02cb 100644 --- a/Source/XUnitExtensions/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/MainMethodLitCommand.cs @@ -4,14 +4,17 @@ using System.Linq; using System.Reflection; using System.Text; +using Xunit.Abstractions; namespace XUnitExtensions { public class MainMethodLitCommand : ILitCommand { + public LitTestConfiguration Config { get; protected set; } public Assembly Assembly { get; protected set; } public string[] Arguments { get; protected set; } public static ILitCommand Parse(Assembly assembly, IEnumerable arguments, LitTestConfiguration config, bool invokeDirectly) { var result = new MainMethodLitCommand { + Config = config, Assembly = assembly, Arguments = arguments.ToArray() }; @@ -19,7 +22,7 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments return invokeDirectly ? result : result.ToShellCommand(config); } - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { if (inputReader != null) { Console.SetIn(inputReader); } @@ -37,7 +40,7 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments public ILitCommand ToShellCommand(LitTestConfiguration config) { var shellArguments = new[] { Assembly.Location }.Concat(Arguments); - return new ShellLitCommand("dotnet", shellArguments, config.PassthroughEnvironmentVariables); + return new ShellLitCommand(config, "dotnet", shellArguments, config.PassthroughEnvironmentVariables); } public override string ToString() { diff --git a/Source/XUnitExtensions/OrCommand.cs b/Source/XUnitExtensions/OrCommand.cs index 282eb1a64d0..8c6097b3496 100644 --- a/Source/XUnitExtensions/OrCommand.cs +++ b/Source/XUnitExtensions/OrCommand.cs @@ -1,4 +1,5 @@ using System.IO; +using Xunit.Abstractions; namespace XUnitExtensions { public class OrCommand : ILitCommand { @@ -10,12 +11,12 @@ public OrCommand(ILitCommand lhs, ILitCommand rhs) { this.rhs = rhs; } - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { - var (exitCode, output, error) = lhs.Execute(inputReader, outputWriter, errorWriter); + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + var (exitCode, output, error) = lhs.Execute(outputHelper, inputReader, outputWriter, errorWriter); if (exitCode == 0) { return (exitCode, output, error); } - return rhs.Execute(inputReader, outputWriter, errorWriter); + return rhs.Execute(outputHelper, inputReader, outputWriter, errorWriter); } public override string ToString() { diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/ShellLitCommand.cs index 407610f2981..69de2048bb2 100644 --- a/Source/XUnitExtensions/ShellLitCommand.cs +++ b/Source/XUnitExtensions/ShellLitCommand.cs @@ -4,24 +4,29 @@ using System.IO; using System.Linq; using System.Text; +using Xunit.Abstractions; namespace XUnitExtensions { public class ShellLitCommand : ILitCommand { + private LitTestConfiguration config; private string shellCommand; private string[] arguments; private string[] passthroughEnvironmentVariables; - public ShellLitCommand(string shellCommand, IEnumerable arguments, IEnumerable passthroughEnvironmentVariables) { + public ShellLitCommand(LitTestConfiguration config, string shellCommand, IEnumerable arguments, IEnumerable passthroughEnvironmentVariables) { + this.config = config; this.shellCommand = shellCommand; this.arguments = arguments.ToArray(); this.passthroughEnvironmentVariables = passthroughEnvironmentVariables.ToArray(); } - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { using var process = new Process(); + var expandedArguments = arguments.SelectMany(arg => ILitCommand.ExpandGlobsAndBackticks(arg, config, outputHelper)); + process.StartInfo.FileName = shellCommand; - foreach (var argument in arguments) { + foreach (var argument in expandedArguments) { process.StartInfo.Arguments += " " + argument; } diff --git a/Source/XUnitExtensions/XFailCommand.cs b/Source/XUnitExtensions/XFailCommand.cs index 8f8f4948def..0515e728860 100644 --- a/Source/XUnitExtensions/XFailCommand.cs +++ b/Source/XUnitExtensions/XFailCommand.cs @@ -1,11 +1,12 @@ using System.IO; +using Xunit.Abstractions; namespace XUnitExtensions { public class XFailCommand : ILitCommand { public XFailCommand() { } - public (int, string, string) Execute(TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { return (0, "", ""); } } diff --git a/Source/XUnitExtensions/XUnitExtensions.csproj b/Source/XUnitExtensions/XUnitExtensions.csproj index 4bddf803848..583a13e4c5a 100644 --- a/Source/XUnitExtensions/XUnitExtensions.csproj +++ b/Source/XUnitExtensions/XUnitExtensions.csproj @@ -8,6 +8,7 @@ + diff --git a/Test/DafnyTestGeneration/TestGeneration.dfy b/Test/DafnyTestGeneration/TestGeneration.dfy index df376f9fdfa..8dcd434f02a 100644 --- a/Test/DafnyTestGeneration/TestGeneration.dfy +++ b/Test/DafnyTestGeneration/TestGeneration.dfy @@ -3,15 +3,15 @@ // RUN: %dafny /definiteAssignment:3 /generateTestMode:Block %t.dfy > %t-tests.dfy // Compiling test to java: -// RUN: %dafny /compileTarget:java /out:Output %t-tests.dfy +// RUN: %dafny /compileTarget:java /out:%t-tests.dfy %t-tests.dfy // Adding reflection code that allows running the tests: -// RUN: perl -pe 's/import M_Compile.*;/`cat import.txt`/ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java -// RUN: perl -pe 's/public class __default \{/`cat reflectionCode.txt`/ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java +// RUN: perl -pe 's/import M_Compile.*;/`cat %S/import.txt`/ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java +// RUN: perl -pe 's/public class __default \{/`cat %S/reflectionCode.txt`/ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java // Compiling to bytecode and running the tests -// RUN: javac -cp %t-tests-java:%binaries/DafnyRuntime.jar %t-tests-java/TestGenerationUnitTests_Compile/__default.java -// RUN: java -cp %t-tests-java:%binaries/DafnyRuntime.jar TestGenerationUnitTests_Compile/__default > %t +// RUN: javac -cp %t-tests-java:%binaryDir/DafnyRuntime.jar %t-tests-java/TestGenerationUnitTests_Compile/__default.java +// RUN: java -cp %t-tests-java:%binaryDir/DafnyRuntime.jar TestGenerationUnitTests_Compile/__default > %t // RUN: %diff "%s.expect" "%t" module M { diff --git a/Test/comp/compile1quiet/CompileRunQuietly.dfy b/Test/comp/compile1quiet/CompileRunQuietly.dfy index 9ac8092b542..95b489e25fd 100644 --- a/Test/comp/compile1quiet/CompileRunQuietly.dfy +++ b/Test/comp/compile1quiet/CompileRunQuietly.dfy @@ -8,7 +8,7 @@ // RUN: %S/CompileRunQuietly >> "%t" // RUN: %dafny /compileTarget:java "%s" >> "%t" -// RUN: java -cp %binaries/DafnyRuntime.jar:%S/CompileRunQuietly-java CompileRunQuietly >> "%t" +// RUN: java -cp %binaryDir/DafnyRuntime.jar:%S/CompileRunQuietly-java CompileRunQuietly >> "%t" // RUN: %dafny /compileTarget:cpp "%s" >> "%t" // RUN: CompileRunQuietly.exe >> "%t" diff --git a/Test/comp/compile1verbose/CompileAndThenRun.dfy b/Test/comp/compile1verbose/CompileAndThenRun.dfy index 9fa4140bcf5..0db655f9129 100644 --- a/Test/comp/compile1verbose/CompileAndThenRun.dfy +++ b/Test/comp/compile1verbose/CompileAndThenRun.dfy @@ -8,7 +8,7 @@ // RUN: %S/CompileAndThenRun >> "%t" // RUN: %dafny /compileVerbose:1 /compileTarget:java "%s" >> "%t" -// RUN: java -cp %binaries/DafnyRuntime.jar:%S/CompileAndThenRun-java CompileAndThenRun >> "%t" +// RUN: java -cp %binaryDir/DafnyRuntime.jar:%S/CompileAndThenRun-java CompileAndThenRun >> "%t" // RUN: %dafny /compileVerbose:1 /compileTarget:cpp "%s" >> "%t" // RUN: CompileAndThenRun.exe >> "%t" diff --git a/Test/comp/manualcompile/ManualCompile.dfy b/Test/comp/manualcompile/ManualCompile.dfy index e699b4fa720..83f0f672fe2 100644 --- a/Test/comp/manualcompile/ManualCompile.dfy +++ b/Test/comp/manualcompile/ManualCompile.dfy @@ -6,15 +6,15 @@ // RUN: node %S/ManualCompile.js >> "%t" // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:go "%s" >> "%t" -// RUN: GOPATH=%S/ManualCompile-go go run %S/ManualCompile-go/src/ManualCompile.go >> "%t" +// RUN: env GOPATH=%S/ManualCompile-go go run %S/ManualCompile-go/src/ManualCompile.go >> "%t" // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: javac ManualCompile-java/ManualCompile.java ManualCompile-java/*/*.java -// RUN: java ManualCompile >> "%t" +// RUN: javac -cp %binaryDir/DafnyRuntime.jar:%S/ManualCompile-java %S/ManualCompile-java/ManualCompile.java ManualCompile-java/*/*.java +// RUN: java -cp %binaryDir/DafnyRuntime.jar:%S/ManualCompile-java ManualCompile >> "%t" // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:cpp "%s" >> "%t" -// RUN: g++ -g -Wall -Wextra -Wpedantic -Wno-unused-variable -std=c++17 -I %binaryDir -o ManualCompile.exe ManualCompile.cpp -// RUN: %S/ManualCompile.exe >> "%t" +// RUN: g++ -g -Wall -Wextra -Wpedantic -Wno-unused-variable -std=c++17 -I %binaryDir -o ManualCompile.exe %S/ManualCompile.cpp +// RUN: ManualCompile.exe >> "%t" // RUN: %diff "%s.expect" "%t" diff --git a/Test/comp/manualcompile/lit.local.cfg b/Test/comp/manualcompile/lit.local.cfg index f093c5c1e0b..27d608ecb02 100644 --- a/Test/comp/manualcompile/lit.local.cfg +++ b/Test/comp/manualcompile/lit.local.cfg @@ -20,9 +20,6 @@ if lit.util.which('g++') == None: lit_config.note('g++ is unavailable, so cross-compilation tests are skipped\n') config.unsupported = True -binaryDir = config.dafnyBinaryDir -config.substitutions.append( ('%binaryDir', binaryDir) ) - current_dir = os.path.join(config.test_source_root, 'comp', 'manualcompile') config.environment['GOPATH'] = os.path.join(current_dir, 'ManualCompile-go') diff --git a/Test/lit.site.cfg b/Test/lit.site.cfg index ad9b1ff409f..6a4c61f5504 100644 --- a/Test/lit.site.cfg +++ b/Test/lit.site.cfg @@ -140,7 +140,7 @@ ver = "0" if os.name != "nt": ver = os.uname().version -config.substitutions.append( ('%binaries', binaryDir) ) +config.substitutions.append( ('%binaryDir', binaryDir) ) config.substitutions.append( ('%dafny', dafnyExecutable) ) config.substitutions.append( ('%baredafny', bareDafnyExecutable) ) config.substitutions.append( ('%server', serverExecutable) ) From cb01db40876260cbd0464f11c376236b7e07d9d5 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 17 Oct 2021 09:45:10 -0700 Subject: [PATCH 158/192] Fix broken globbing, remove backtick support, fix last test (perl is interpreting the backticks, not the shell) --- Source/XUnitExtensions/ILitCommand.cs | 41 ++++++++++----------- Source/XUnitExtensions/ShellLitCommand.cs | 6 +-- Test/DafnyTestGeneration/TestGeneration.dfy | 4 +- Test/comp/manualcompile/ManualCompile.dfy | 2 +- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index 2fc38023cd1..e7143f7d4d3 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -72,46 +72,45 @@ private static string[] Tokenize(string line) { var argument = new StringBuilder(); var singleQuoted = false; var doubleQuoted = false; + var hasGlobCharacters = false; for (int i = 0; i < line.Length; i++) { var c = line[i]; if (c == '\'') { singleQuoted = !singleQuoted; - argument.Append(c); } else if (c == '"') { doubleQuoted = !doubleQuoted; } else if (Char.IsWhiteSpace(c) && !(singleQuoted || doubleQuoted)) { - arguments.Add(argument.ToString()); + if (hasGlobCharacters) { + arguments.AddRange(ExpandGlobs(argument.ToString())); + } else { + arguments.Add(argument.ToString()); + } argument.Clear(); + hasGlobCharacters = false; } else { + if (c is '*' or '?' && !singleQuoted) { + hasGlobCharacters = true; + } argument.Append(c); } } if (argument.Length != 0) { - arguments.Add(argument.ToString()); + if (hasGlobCharacters) { + arguments.AddRange(ExpandGlobs(argument.ToString())); + } else { + arguments.Add(argument.ToString()); + } } return arguments.ToArray(); } - protected static IEnumerable ExpandGlobsAndBackticks(string chunk, LitTestConfiguration config, ITestOutputHelper outputHelper) { - if (chunk.Contains('*') || chunk.Contains('?')) { - var matcher = new Matcher(); - matcher.AddInclude(chunk); - var result = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo("."))); - return result.Files.Select(f => f.Path); - } - - var firstTickIndex = chunk.IndexOf('`'); - if (firstTickIndex >= 0) { - var secondTickIndex = chunk.IndexOf('`', firstTickIndex + 1); - var toParse = chunk[(firstTickIndex + 1)..(secondTickIndex)]; - ILitCommand command = LitCommandWithRedirection.Parse(Tokenize(toParse), config); - var output = new StringBuilder(); - command.ExecuteWithExpectation(outputHelper, null, new StringWriter(output), null); - return new[] { chunk[..firstTickIndex] + output + chunk[(secondTickIndex + 1)..] }; - } - return new[] { chunk }; + protected static IEnumerable ExpandGlobs(string chunk ) { + var matcher = new Matcher(); + matcher.AddInclude(chunk); + var result = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo("."))); + return result.Files.Select(f => f.Path); } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/ShellLitCommand.cs index 69de2048bb2..1ce9029a790 100644 --- a/Source/XUnitExtensions/ShellLitCommand.cs +++ b/Source/XUnitExtensions/ShellLitCommand.cs @@ -22,12 +22,10 @@ public ShellLitCommand(LitTestConfiguration config, string shellCommand, IEnumer public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { using var process = new Process(); - - var expandedArguments = arguments.SelectMany(arg => ILitCommand.ExpandGlobsAndBackticks(arg, config, outputHelper)); process.StartInfo.FileName = shellCommand; - foreach (var argument in expandedArguments) { - process.StartInfo.Arguments += " " + argument; + foreach (var argument in arguments) { + process.StartInfo.ArgumentList.Add(argument); } // We avoid setting the current directory so that we maintain parity with diff --git a/Test/DafnyTestGeneration/TestGeneration.dfy b/Test/DafnyTestGeneration/TestGeneration.dfy index 8dcd434f02a..9a979576227 100644 --- a/Test/DafnyTestGeneration/TestGeneration.dfy +++ b/Test/DafnyTestGeneration/TestGeneration.dfy @@ -6,8 +6,8 @@ // RUN: %dafny /compileTarget:java /out:%t-tests.dfy %t-tests.dfy // Adding reflection code that allows running the tests: -// RUN: perl -pe 's/import M_Compile.*;/`cat %S/import.txt`/ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java -// RUN: perl -pe 's/public class __default \{/`cat %S/reflectionCode.txt`/ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java +// RUN: perl -pe 's|import M_Compile.*;|`cat %S/import.txt`|ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java +// RUN: perl -pe 's|public class __default \{|`cat %S/reflectionCode.txt`|ge' -i %t-tests-java/TestGenerationUnitTests_Compile/__default.java // Compiling to bytecode and running the tests // RUN: javac -cp %t-tests-java:%binaryDir/DafnyRuntime.jar %t-tests-java/TestGenerationUnitTests_Compile/__default.java diff --git a/Test/comp/manualcompile/ManualCompile.dfy b/Test/comp/manualcompile/ManualCompile.dfy index 83f0f672fe2..68255ca6431 100644 --- a/Test/comp/manualcompile/ManualCompile.dfy +++ b/Test/comp/manualcompile/ManualCompile.dfy @@ -9,7 +9,7 @@ // RUN: env GOPATH=%S/ManualCompile-go go run %S/ManualCompile-go/src/ManualCompile.go >> "%t" // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:java "%s" >> "%t" -// RUN: javac -cp %binaryDir/DafnyRuntime.jar:%S/ManualCompile-java %S/ManualCompile-java/ManualCompile.java ManualCompile-java/*/*.java +// RUN: javac -cp %binaryDir/DafnyRuntime.jar:%S/ManualCompile-java %S/ManualCompile-java/ManualCompile.java %S/ManualCompile-java/*/*.java // RUN: java -cp %binaryDir/DafnyRuntime.jar:%S/ManualCompile-java ManualCompile >> "%t" // RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:2 /compileTarget:cpp "%s" >> "%t" From d03b1be659e174a9665ca40f6a0f7a30f138430b Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 17 Oct 2021 10:47:54 -0700 Subject: [PATCH 159/192] Another lexer bug fix --- Source/XUnitExtensions/ILitCommand.cs | 15 ++++++++------ Source/XUnitExtensions/LitTestCase.cs | 20 +------------------ .../TestCollectionShardFilter.cs | 8 ++++++-- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index e7143f7d4d3..806a7ee06b3 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -80,13 +80,16 @@ private static string[] Tokenize(string line) { } else if (c == '"') { doubleQuoted = !doubleQuoted; } else if (Char.IsWhiteSpace(c) && !(singleQuoted || doubleQuoted)) { - if (hasGlobCharacters) { - arguments.AddRange(ExpandGlobs(argument.ToString())); - } else { - arguments.Add(argument.ToString()); + if (argument.Length != 0) { + if (hasGlobCharacters) { + arguments.AddRange(ExpandGlobs(argument.ToString())); + } else { + arguments.Add(argument.ToString()); + } + + argument.Clear(); + hasGlobCharacters = false; } - argument.Clear(); - hasGlobCharacters = false; } else { if (c is '*' or '?' && !singleQuoted) { hasGlobCharacters = true; diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index 64b3a69a72d..4791d265ebc 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -44,25 +44,7 @@ public void Execute(ITestOutputHelper outputHelper) { Directory.CreateDirectory(Path.Join(Path.GetDirectoryName(filePath), "Output")); foreach (var command in commands) { - int exitCode; - string output; - string error; - try { - outputHelper.WriteLine($"Executing command: {command}"); - (exitCode, output, error) = command.Execute(outputHelper, null, null, null); - } catch (Exception e) { - throw new Exception($"Exception thrown while executing command: {command}", e); - } - - if (expectFailure) { - if (exitCode != 0) { - throw new SkipException($"Command returned non-zero exit code ({exitCode}): {command}\nOutput:\n{output}\nError:\n{error}"); - } - } - - if (exitCode != 0) { - throw new Exception($"Command returned non-zero exit code ({exitCode}): {command}\nOutput:\n{output}\nError:\n{error}"); - } + command.ExecuteWithExpectation(outputHelper, null, null, null, expectFailure); } if (expectFailure) { diff --git a/Source/XUnitExtensions/TestCollectionShardFilter.cs b/Source/XUnitExtensions/TestCollectionShardFilter.cs index 9f9ac45b852..4ff11ad983c 100644 --- a/Source/XUnitExtensions/TestCollectionShardFilter.cs +++ b/Source/XUnitExtensions/TestCollectionShardFilter.cs @@ -7,6 +7,8 @@ namespace XUnitExtensions { public class TestCollectionShardFilter : ITestCollectionOrderer { public IEnumerable OrderTestCollections(IEnumerable testCollections) { + var sorted = testCollections.OrderBy(c => c.DisplayName); + // Select the requested fraction of the test collections if using the XUNIT_SHARD[_COUNT] environment variables. var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD_COUNT"); @@ -27,13 +29,15 @@ public IEnumerable OrderTestCollections(IEnumerable Date: Sun, 17 Oct 2021 10:59:55 -0700 Subject: [PATCH 160/192] Fix XFAIL handling --- Source/XUnitExtensions/ILitCommand.cs | 19 ------------------- Source/XUnitExtensions/LitTestCase.cs | 20 +++++++++++++++++++- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index 806a7ee06b3..381301f2358 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -19,25 +19,6 @@ public interface ILitCommand { public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter); - public void ExecuteWithExpectation(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter, bool expectFailure = false) { - try { - outputHelper.WriteLine($"Executing command: {this}"); - var (exitCode, output, error) = Execute(outputHelper, inputReader, outputWriter, errorWriter); - - if (expectFailure) { - if (exitCode != 0) { - throw new SkipException($"Command returned non-zero exit code ({exitCode}): {this}\nOutput:\n{output}\nError:\n{error}"); - } - } - - if (exitCode != 0) { - throw new Exception($"Command returned non-zero exit code ({exitCode}): {this}\nOutput:\n{output}\nError:\n{error}"); - } - } catch (Exception e) { - throw new Exception($"Exception thrown while executing command: {this}", e); - } - } - public static ILitCommand Parse(string fileName, string line, LitTestConfiguration config) { if (!line.StartsWith(COMMENT_PREFIX)) { return null; diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index 4791d265ebc..64b3a69a72d 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -44,7 +44,25 @@ public void Execute(ITestOutputHelper outputHelper) { Directory.CreateDirectory(Path.Join(Path.GetDirectoryName(filePath), "Output")); foreach (var command in commands) { - command.ExecuteWithExpectation(outputHelper, null, null, null, expectFailure); + int exitCode; + string output; + string error; + try { + outputHelper.WriteLine($"Executing command: {command}"); + (exitCode, output, error) = command.Execute(outputHelper, null, null, null); + } catch (Exception e) { + throw new Exception($"Exception thrown while executing command: {command}", e); + } + + if (expectFailure) { + if (exitCode != 0) { + throw new SkipException($"Command returned non-zero exit code ({exitCode}): {command}\nOutput:\n{output}\nError:\n{error}"); + } + } + + if (exitCode != 0) { + throw new Exception($"Command returned non-zero exit code ({exitCode}): {command}\nOutput:\n{output}\nError:\n{error}"); + } } if (expectFailure) { From eaa736434a49c53ce6b2aa68a0b4a35464d39429 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Sun, 17 Oct 2021 20:49:40 -0700 Subject: [PATCH 161/192] Add UNSUPPORTED and features --- Source/IntegrationTests/LitTests.cs | 11 ++++++++ Source/XUnitExtensions/ILitCommand.cs | 8 ++++-- Source/XUnitExtensions/LitTestCase.cs | 7 +++++ .../XUnitExtensions/LitTestConfiguration.cs | 5 +++- Source/XUnitExtensions/UnsupportedCommand.cs | 26 +++++++++++++++++++ 5 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 Source/XUnitExtensions/UnsupportedCommand.cs diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 3db7d004683..1418b5de04f 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Runtime.InteropServices; using Microsoft.Dafny; using Xunit; using Xunit.Abstractions; @@ -69,6 +70,16 @@ static LitTests() { new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "DafnyServer"), args, config.PassthroughEnvironmentVariables); CONFIG.Substitions["%z3"] = Path.Join(dafnyReleaseDir, "z3", "bin", "z3"); } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { + CONFIG.Features = new[] { "ubuntu", "posix" }; + } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { + CONFIG.Features = new[] { "windows" }; + } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { + CONFIG.Features = new[] { "macosx", "posix" }; + } else { + throw new Exception($"Unsupported OS: {RuntimeInformation.OSDescription}"); + } } private readonly ITestOutputHelper output; diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index 381301f2358..f03db0de65b 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -15,7 +15,8 @@ public interface ILitCommand { private const string COMMENT_PREFIX = "//"; private const string LIT_COMMAND_PREFIX = "RUN:"; - private const string LIT_XFAIL = "XFAIL: *"; + private const string LIT_UNSUPPORTED = "UNSUPPORTED"; + private const string LIT_XFAIL_ALL = "XFAIL: *"; public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter); @@ -25,9 +26,12 @@ public static ILitCommand Parse(string fileName, string line, LitTestConfigurati } line = line[COMMENT_PREFIX.Length..].Trim(); - if (line.Equals(LIT_XFAIL)) { + if (line.Equals(LIT_XFAIL_ALL)) { return new XFailCommand(); } + if (line.StartsWith(LIT_UNSUPPORTED)) { + return UnsupportedCommand.Parse(line[LIT_UNSUPPORTED.Length..].Trim()); + } if (!line.StartsWith(LIT_COMMAND_PREFIX)) { return null; } diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index 64b3a69a72d..9d996fa4ca2 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -18,6 +18,13 @@ public static LitTestCase Read(string filePath, LitTestConfiguration config) { .Select(line => ILitCommand.Parse(filePath, line, config)) .Where(c => c != null); var xfail = commands.Any(c => c is XFailCommand); + foreach (var unsupported in commands.OfType()) { + foreach (var feature in config.Features) { + if (unsupported.Features.Contains(feature)) { + throw new SkipException($"Test case not supported: {feature}"); + } + } + } return new LitTestCase(filePath, commands, xfail); } diff --git a/Source/XUnitExtensions/LitTestConfiguration.cs b/Source/XUnitExtensions/LitTestConfiguration.cs index 8c2b6078b89..1ec65cf5026 100644 --- a/Source/XUnitExtensions/LitTestConfiguration.cs +++ b/Source/XUnitExtensions/LitTestConfiguration.cs @@ -13,6 +13,8 @@ public class LitTestConfiguration { public Dictionary, LitTestConfiguration, ILitCommand>> Commands { get; set; } public IEnumerable PassthroughEnvironmentVariables { get; set; } + + public string[] Features { get; set; } public string ApplySubstitutions(string s) { foreach (var (key, value) in Substitions) { @@ -25,7 +27,8 @@ public LitTestConfiguration WithSubstitutions(Dictionary more) { return new LitTestConfiguration { Substitions = Substitions.Concat(more).ToDictionary(pair => pair.Key, pair => pair.Value), Commands = Commands, - PassthroughEnvironmentVariables = PassthroughEnvironmentVariables + PassthroughEnvironmentVariables = PassthroughEnvironmentVariables, + Features = Features }; } } diff --git a/Source/XUnitExtensions/UnsupportedCommand.cs b/Source/XUnitExtensions/UnsupportedCommand.cs new file mode 100644 index 00000000000..0eefb447ace --- /dev/null +++ b/Source/XUnitExtensions/UnsupportedCommand.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Xunit.Abstractions; + +namespace XUnitExtensions { + public class UnsupportedCommand : ILitCommand { + + public static UnsupportedCommand Parse(string line) { + var features = line.Split(",", StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()); + return new UnsupportedCommand(features); + } + + public IEnumerable Features { get; protected set; } + + public UnsupportedCommand(IEnumerable features) { + this.Features = features; + } + + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + return (0, "", ""); + } + } +} \ No newline at end of file From dce7168912c7693878d7cb78d627fa98d16ab04d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 18 Oct 2021 10:10:19 -0700 Subject: [PATCH 162/192] Fix ShowSnippets.dfy, debug TestAttribute.dfy --- .github/workflows/msbuild.yml | 2 +- .../LitCommandWithRedirection.cs | 8 ++++++++ Test/dafny0/ShowSnippets.dfy | 8 +++++++- Test/dafny0/ShowSnippets.dfy.expect | 20 +++++++++---------- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index 29be441cf7b..853cab6f99e 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -126,4 +126,4 @@ jobs: unzip dafny/Package/CI.zip -d unzippedRelease - name: Run integration tests run: | - XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 DAFNY_RELEASE=$PWD/unzippedRelease/dafny dotnet test dafny/Source/IntegrationTests/IntegrationTests.csproj + XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 DAFNY_RELEASE=$PWD/unzippedRelease/dafny dotnet test -v:n dafny/Source/IntegrationTests/IntegrationTests.csproj diff --git a/Source/XUnitExtensions/LitCommandWithRedirection.cs b/Source/XUnitExtensions/LitCommandWithRedirection.cs index 5e126bfba03..8e475e127a9 100644 --- a/Source/XUnitExtensions/LitCommandWithRedirection.cs +++ b/Source/XUnitExtensions/LitCommandWithRedirection.cs @@ -76,10 +76,18 @@ public LitCommandWithRedirection(ILitCommand command, string inputFile, string o public override string ToString() { var builder = new StringBuilder(); builder.Append(command); + if (inputFile != null) { + builder.Append(" < "); + builder.Append(inputFile); + } if (outputFile != null) { builder.Append(append ? " >> " : " > "); builder.Append(outputFile); } + if (errorFile != null) { + builder.Append(" 2> "); + builder.Append(errorFile); + } return builder.ToString(); } } diff --git a/Test/dafny0/ShowSnippets.dfy b/Test/dafny0/ShowSnippets.dfy index 9cf036cea60..c0a9f1f2ae5 100644 --- a/Test/dafny0/ShowSnippets.dfy +++ b/Test/dafny0/ShowSnippets.dfy @@ -1,4 +1,10 @@ -// RUN: %dafny /compile:0 /showSnippets:1 "%s" > "%t" +// There is a known bug in this feature where it can't find the original file if +// /useBaseNameForFileName is used and the file isn't in the current directory: +// https://github.com/dafny-lang/dafny/issues/1518. +// To work around this, we don't pass /useBaseNameForFileName and instead manually +// truncate the source paths to their base names using sed. +// RUN: %baredafny /countVerificationErrors:0 /errorTrace:0 /compile:0 /showSnippets:1 "%s" > "%t".raw +// RUN: sed 's/^.*\///' "%t".raw > "%t" // RUN: %diff "%s.expect" "%t" method Never() requires true && false {} diff --git a/Test/dafny0/ShowSnippets.dfy.expect b/Test/dafny0/ShowSnippets.dfy.expect index d3d72e1095d..a916e715643 100644 --- a/Test/dafny0/ShowSnippets.dfy.expect +++ b/Test/dafny0/ShowSnippets.dfy.expect @@ -1,17 +1,17 @@ -ShowSnippets.dfy(7,9): Error: assertion violation - | -7 | assert false; - | ^ here +ShowSnippets.dfy(13,9): Error: assertion violation + | +13 | assert false; + | ^ here -ShowSnippets.dfy(11,7): Error: A precondition for this call might not hold. +ShowSnippets.dfy(17,7): Error: A precondition for this call might not hold. | -11 | Never(); +17 | Never(); | ^ here -ShowSnippets.dfy(4,32): Related location: This is the precondition that might not hold. - | -4 | method Never() requires true && false {} - | ^ here +ShowSnippets.dfy(10,32): Related location: This is the precondition that might not hold. + | +10 | method Never() requires true && false {} + | ^ here Dafny program verifier finished with 0 verified, 2 errors From c85142558f34394b842be5e35fb20e80f45d85c2 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 18 Oct 2021 10:28:25 -0700 Subject: [PATCH 163/192] Way better load balancing of shards (same as lit's) --- Source/XUnitExtensions/TestCollectionShardFilter.cs | 7 +------ Test/comp/manualcompile/lit.local.cfg | 5 ----- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/Source/XUnitExtensions/TestCollectionShardFilter.cs b/Source/XUnitExtensions/TestCollectionShardFilter.cs index 4ff11ad983c..1b37e063165 100644 --- a/Source/XUnitExtensions/TestCollectionShardFilter.cs +++ b/Source/XUnitExtensions/TestCollectionShardFilter.cs @@ -29,12 +29,7 @@ public IEnumerable OrderTestCollections(IEnumerable index % numShards == shard); } return sorted; diff --git a/Test/comp/manualcompile/lit.local.cfg b/Test/comp/manualcompile/lit.local.cfg index 27d608ecb02..8f63d146ca3 100644 --- a/Test/comp/manualcompile/lit.local.cfg +++ b/Test/comp/manualcompile/lit.local.cfg @@ -21,8 +21,3 @@ if lit.util.which('g++') == None: config.unsupported = True current_dir = os.path.join(config.test_source_root, 'comp', 'manualcompile') - -config.environment['GOPATH'] = os.path.join(current_dir, 'ManualCompile-go') -config.environment['GO111MODULE'] = 'auto' - -config.environment['CLASSPATH'] = os.path.join(current_dir, 'ManualCompile-java') + ":" + os.path.join( binaryDir, "DafnyRuntime.jar" ) From 52c9f89a26c6a7cdb5777ad46468ac144a7eaae1 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 18 Oct 2021 10:32:49 -0700 Subject: [PATCH 164/192] Debugging TestAttribute.dfy issues on GHA --- Test/DafnyTests/TestAttribute.dfy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Test/DafnyTests/TestAttribute.dfy b/Test/DafnyTests/TestAttribute.dfy index 6e9bd0ad658..b4294a8cb9d 100644 --- a/Test/DafnyTests/TestAttribute.dfy +++ b/Test/DafnyTests/TestAttribute.dfy @@ -1,5 +1,5 @@ // RUN: %dafny /out:Output/DafnyMain.cs TestAttribute.dfy /compile:0 /spillTargetCode:3 /noVerify -// RUN: dotnet test -v:q -noLogo %S 2> %t.raw || true +// RUN: dotnet test -v:q -noLogo %S 2> %t.raw // Remove the timestamp prefixes on the expected errors // RUN: sed 's/[^]]*\]//' "%t".raw > "%t" // RUN: %diff "%s.expect" "%t" From dd21ef19fcde3c8baae6581769d16f26aac78e1c Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 18 Oct 2021 10:41:06 -0700 Subject: [PATCH 165/192] Only two hard problems in computer science... :) --- Source/XUnitExtensions/TestCollectionShardFilter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/XUnitExtensions/TestCollectionShardFilter.cs b/Source/XUnitExtensions/TestCollectionShardFilter.cs index 1b37e063165..8ca6cc2b253 100644 --- a/Source/XUnitExtensions/TestCollectionShardFilter.cs +++ b/Source/XUnitExtensions/TestCollectionShardFilter.cs @@ -29,7 +29,7 @@ public IEnumerable OrderTestCollections(IEnumerable index % numShards == shard); + return sorted.Where((_, index) => index % numShards == shard - 1); } return sorted; From 6fa230ef187edc3aba8308cdd6bc732c968c00c4 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 18 Oct 2021 12:48:05 -0700 Subject: [PATCH 166/192] Cleanup and debugging --- .github/workflows/msbuild.yml | 2 +- Source/XUnitExtensions/FileDataAttribute.cs | 4 ++-- Source/XUnitExtensions/FileTestCase.cs | 4 ++-- Source/XUnitExtensions/FileTheoryDataRow.cs | 2 +- Source/XUnitExtensions/ILitCommand.cs | 4 ++-- .../LitCommandWithRedirection.cs | 22 +++++++++---------- Source/XUnitExtensions/LitTestCase.cs | 11 +++++++--- .../XUnitExtensions/MainMethodLitCommand.cs | 2 +- Source/XUnitExtensions/OrCommand.cs | 2 +- Source/XUnitExtensions/ShellLitCommand.cs | 2 +- Source/XUnitExtensions/UnsupportedCommand.cs | 2 +- Source/XUnitExtensions/XFailCommand.cs | 2 +- Source/XUnitExtensions/XUnitExtensions.csproj | 2 +- 13 files changed, 33 insertions(+), 28 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index 853cab6f99e..fb7d3b524bd 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -126,4 +126,4 @@ jobs: unzip dafny/Package/CI.zip -d unzippedRelease - name: Run integration tests run: | - XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 DAFNY_RELEASE=$PWD/unzippedRelease/dafny dotnet test -v:n dafny/Source/IntegrationTests/IntegrationTests.csproj + XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 DAFNY_RELEASE=$PWD/unzippedRelease/dafny dotnet test -v:n dafny/Source/IntegrationTests/IntegrationTests.csproj --filter DisplayName~TestAttribute.dfy diff --git a/Source/XUnitExtensions/FileDataAttribute.cs b/Source/XUnitExtensions/FileDataAttribute.cs index 372bc79e4a6..d06373a5f16 100644 --- a/Source/XUnitExtensions/FileDataAttribute.cs +++ b/Source/XUnitExtensions/FileDataAttribute.cs @@ -4,9 +4,9 @@ namespace XUnitExtensions { public abstract class FileDataAttribute : DataAttribute { - public virtual string Path { get; set; } + public virtual string? Path { get; set; } - public virtual string Extension { get; set; } + public virtual string? Extension { get; set; } public override IEnumerable GetData(MethodInfo testMethod) { throw new System.NotImplementedException(); diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index 8f5d1b1eb90..55565eb8ccd 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -15,8 +15,8 @@ public class FileTestCase : LongLivedMarshalByRefObject, IXunitTestCase { protected XunitTestCase innerTestCase; - public string DisplayName { get; protected set; } - public string SkipReason { get; protected set; } + public string? DisplayName { get; protected set; } + public string? SkipReason { get; protected set; } public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, IFileTheoryRowData data) { var collection = new TestCollection(testMethod.TestClass.TestCollection.TestAssembly, diff --git a/Source/XUnitExtensions/FileTheoryDataRow.cs b/Source/XUnitExtensions/FileTheoryDataRow.cs index da847da132d..f9ca50c5469 100644 --- a/Source/XUnitExtensions/FileTheoryDataRow.cs +++ b/Source/XUnitExtensions/FileTheoryDataRow.cs @@ -4,7 +4,7 @@ namespace XUnitExtensions { public class FileTheoryDataRow : IXunitSerializable, IFileTheoryRowData { - object?[] data; + private object?[] data; public FileTheoryDataRow() { diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/ILitCommand.cs index f03db0de65b..e1a1accafa3 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/ILitCommand.cs @@ -18,9 +18,9 @@ public interface ILitCommand { private const string LIT_UNSUPPORTED = "UNSUPPORTED"; private const string LIT_XFAIL_ALL = "XFAIL: *"; - public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter); + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter); - public static ILitCommand Parse(string fileName, string line, LitTestConfiguration config) { + public static ILitCommand? Parse(string fileName, string line, LitTestConfiguration config) { if (!line.StartsWith(COMMENT_PREFIX)) { return null; } diff --git a/Source/XUnitExtensions/LitCommandWithRedirection.cs b/Source/XUnitExtensions/LitCommandWithRedirection.cs index 8e475e127a9..b79ba3598c0 100644 --- a/Source/XUnitExtensions/LitCommandWithRedirection.cs +++ b/Source/XUnitExtensions/LitCommandWithRedirection.cs @@ -13,10 +13,10 @@ public class LitCommandWithRedirection : ILitCommand { public static LitCommandWithRedirection Parse(string[] tokens, LitTestConfiguration config) { var commandSymbol = tokens[0]; var argumentsList = tokens[1..].ToList(); - string inputFile = null; - string outputFile = null; - bool appendOutput = false; - string errorFile = null; + string? inputFile = null; + string? outputFile = null; + var appendOutput = false; + string? errorFile = null; var redirectInIndex = argumentsList.IndexOf("<"); if (redirectInIndex >= 0) { inputFile = config.ApplySubstitutions(argumentsList[redirectInIndex + 1]); @@ -52,13 +52,13 @@ public static LitCommandWithRedirection Parse(string[] tokens, LitTestConfigurat inputFile, outputFile, appendOutput, errorFile); } - private ILitCommand command; - private string inputFile; - private string outputFile; - private bool append; - private string errorFile; + private readonly ILitCommand command; + private readonly string? inputFile; + private readonly string? outputFile; + private readonly bool append; + private readonly string? errorFile; - public LitCommandWithRedirection(ILitCommand command, string inputFile, string outputFile, bool append, string errorFile) { + public LitCommandWithRedirection(ILitCommand command, string? inputFile, string? outputFile, bool append, string? errorFile) { this.command = command; this.inputFile = inputFile; this.outputFile = outputFile; @@ -66,7 +66,7 @@ public LitCommandWithRedirection(ILitCommand command, string inputFile, string o this.errorFile = errorFile; } - public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inReader, TextWriter outWriter, TextWriter errWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inReader, TextWriter? outWriter, TextWriter? errWriter) { var inputReader = inputFile != null ? new StreamReader(inputFile) : null; var outputWriter = outputFile != null ? new StreamWriter(outputFile, append) : null; var errorWriter = errorFile != null ? new StreamWriter(errorFile, false) : null; diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index 9d996fa4ca2..c736353b7af 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -30,11 +30,15 @@ public static LitTestCase Read(string filePath, LitTestConfiguration config) { public static void Run(string filePath, LitTestConfiguration config, ITestOutputHelper outputHelper) { string fileName = Path.GetFileName(filePath); - string directory = Path.GetFullPath(Path.GetDirectoryName(filePath)); + string? directory = Path.GetDirectoryName(filePath); + if (directory == null) { + throw new ArgumentException("Couldn't get directory name for path: {}"); + } + string fullDirectoryPath = Path.GetFullPath(directory); config = config.WithSubstitutions(new Dictionary { { "%s", filePath }, - { "%S", directory }, - { "%t", Path.Join(directory, "Output", $"{fileName}.tmp")} + { "%S", fullDirectoryPath }, + { "%t", Path.Join(fullDirectoryPath, "Output", $"{fileName}.tmp")} }); var testCase = Read(filePath, config); @@ -57,6 +61,7 @@ public void Execute(ITestOutputHelper outputHelper) { try { outputHelper.WriteLine($"Executing command: {command}"); (exitCode, output, error) = command.Execute(outputHelper, null, null, null); + outputHelper.WriteLine($"Exit code: {exitCode}\n{output}\nError:\n{error}"); } catch (Exception e) { throw new Exception($"Exception thrown while executing command: {command}", e); } diff --git a/Source/XUnitExtensions/MainMethodLitCommand.cs b/Source/XUnitExtensions/MainMethodLitCommand.cs index 98e3bbc02cb..8656f290e8f 100644 --- a/Source/XUnitExtensions/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/MainMethodLitCommand.cs @@ -22,7 +22,7 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments return invokeDirectly ? result : result.ToShellCommand(config); } - public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { if (inputReader != null) { Console.SetIn(inputReader); } diff --git a/Source/XUnitExtensions/OrCommand.cs b/Source/XUnitExtensions/OrCommand.cs index 8c6097b3496..d1b1356a2b1 100644 --- a/Source/XUnitExtensions/OrCommand.cs +++ b/Source/XUnitExtensions/OrCommand.cs @@ -11,7 +11,7 @@ public OrCommand(ILitCommand lhs, ILitCommand rhs) { this.rhs = rhs; } - public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { var (exitCode, output, error) = lhs.Execute(outputHelper, inputReader, outputWriter, errorWriter); if (exitCode == 0) { return (exitCode, output, error); diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/ShellLitCommand.cs index 1ce9029a790..27afa1643e2 100644 --- a/Source/XUnitExtensions/ShellLitCommand.cs +++ b/Source/XUnitExtensions/ShellLitCommand.cs @@ -20,7 +20,7 @@ public ShellLitCommand(LitTestConfiguration config, string shellCommand, IEnumer this.passthroughEnvironmentVariables = passthroughEnvironmentVariables.ToArray(); } - public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { using var process = new Process(); process.StartInfo.FileName = shellCommand; diff --git a/Source/XUnitExtensions/UnsupportedCommand.cs b/Source/XUnitExtensions/UnsupportedCommand.cs index 0eefb447ace..30bbb7b93bf 100644 --- a/Source/XUnitExtensions/UnsupportedCommand.cs +++ b/Source/XUnitExtensions/UnsupportedCommand.cs @@ -19,7 +19,7 @@ public UnsupportedCommand(IEnumerable features) { this.Features = features; } - public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { return (0, "", ""); } } diff --git a/Source/XUnitExtensions/XFailCommand.cs b/Source/XUnitExtensions/XFailCommand.cs index 0515e728860..21cf83c81dd 100644 --- a/Source/XUnitExtensions/XFailCommand.cs +++ b/Source/XUnitExtensions/XFailCommand.cs @@ -6,7 +6,7 @@ public class XFailCommand : ILitCommand { public XFailCommand() { } - public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader inputReader, TextWriter outputWriter, TextWriter errorWriter) { + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { return (0, "", ""); } } diff --git a/Source/XUnitExtensions/XUnitExtensions.csproj b/Source/XUnitExtensions/XUnitExtensions.csproj index 583a13e4c5a..c499739c8ee 100644 --- a/Source/XUnitExtensions/XUnitExtensions.csproj +++ b/Source/XUnitExtensions/XUnitExtensions.csproj @@ -2,8 +2,8 @@ net5.0 - false + enable From 946548587a4834e35c794c4ba538958dd771f737 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 18 Oct 2021 13:54:23 -0700 Subject: [PATCH 167/192] Fixed TestAttribute.dfy! --- Source/XUnitExtensions/LitTestCase.cs | 1 - Test/DafnyTests/TestAttribute.dfy | 6 +++--- Test/DafnyTests/TestAttribute.dfy.expect | 3 +++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/LitTestCase.cs index c736353b7af..b55ba681207 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/LitTestCase.cs @@ -61,7 +61,6 @@ public void Execute(ITestOutputHelper outputHelper) { try { outputHelper.WriteLine($"Executing command: {command}"); (exitCode, output, error) = command.Execute(outputHelper, null, null, null); - outputHelper.WriteLine($"Exit code: {exitCode}\n{output}\nError:\n{error}"); } catch (Exception e) { throw new Exception($"Exception thrown while executing command: {command}", e); } diff --git a/Test/DafnyTests/TestAttribute.dfy b/Test/DafnyTests/TestAttribute.dfy index b4294a8cb9d..9c08a160017 100644 --- a/Test/DafnyTests/TestAttribute.dfy +++ b/Test/DafnyTests/TestAttribute.dfy @@ -1,7 +1,7 @@ -// RUN: %dafny /out:Output/DafnyMain.cs TestAttribute.dfy /compile:0 /spillTargetCode:3 /noVerify -// RUN: dotnet test -v:q -noLogo %S 2> %t.raw +// RUN: %dafny /compileVerbose:1 /compile:0 /spillTargetCode:3 /noVerify "%s" > "%t" +// RUN: dotnet test -v:q -noLogo %S 2> %t.testresults.raw || true // Remove the timestamp prefixes on the expected errors -// RUN: sed 's/[^]]*\]//' "%t".raw > "%t" +// RUN: sed 's/[^]]*\]//' "%t".testresults.raw >> "%t" // RUN: %diff "%s.expect" "%t" include "../exceptions/VoidOutcomeDt.dfy" diff --git a/Test/DafnyTests/TestAttribute.dfy.expect b/Test/DafnyTests/TestAttribute.dfy.expect index 09066a94aa9..18bfb24f835 100644 --- a/Test/DafnyTests/TestAttribute.dfy.expect +++ b/Test/DafnyTests/TestAttribute.dfy.expect @@ -1,3 +1,6 @@ + +Dafny program verifier did not attempt verification +Wrote textual form of target program to TestAttribute.cs _module.__default.FailingTest_CheckForFailureForXunit [FAIL] _module.__default.FailingTestUsingExpectWithMessage [FAIL] _module.__default.FailingTestUsingExpect [FAIL] From c0853763407650c1bb3909ca2c5e7bca7b41c69f Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 18 Oct 2021 14:07:20 -0700 Subject: [PATCH 168/192] Restore full test suite --- .github/workflows/msbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index fb7d3b524bd..ddad1097389 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -126,4 +126,4 @@ jobs: unzip dafny/Package/CI.zip -d unzippedRelease - name: Run integration tests run: | - XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 DAFNY_RELEASE=$PWD/unzippedRelease/dafny dotnet test -v:n dafny/Source/IntegrationTests/IntegrationTests.csproj --filter DisplayName~TestAttribute.dfy + XUNIT_SHARD=${{ matrix.shard }} XUNIT_SHARD_COUNT=5 DAFNY_RELEASE=$PWD/unzippedRelease/dafny dotnet test -v:n dafny/Source/IntegrationTests/IntegrationTests.csproj From 008e0284427796753b4b61a28e0214f54eadaa94 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 18 Oct 2021 20:55:54 -0700 Subject: [PATCH 169/192] Move Lit-related classes into separate namespace --- Source/IntegrationTests/LitTests.cs | 1 + Source/XUnitExtensions/{ => Lit}/ILitCommand.cs | 2 +- Source/XUnitExtensions/{ => Lit}/LitCommandWithRedirection.cs | 2 +- Source/XUnitExtensions/{ => Lit}/LitTestCase.cs | 2 +- Source/XUnitExtensions/{ => Lit}/LitTestConfiguration.cs | 2 +- Source/XUnitExtensions/{ => Lit}/LitTestDataDiscoverer.cs | 2 +- Source/XUnitExtensions/{ => Lit}/MainMethodLitCommand.cs | 2 +- Source/XUnitExtensions/{ => Lit}/OrCommand.cs | 2 +- Source/XUnitExtensions/{ => Lit}/ShellLitCommand.cs | 2 +- Source/XUnitExtensions/{ => Lit}/UnsupportedCommand.cs | 2 +- Source/XUnitExtensions/{ => Lit}/XFailCommand.cs | 2 +- 11 files changed, 11 insertions(+), 10 deletions(-) rename Source/XUnitExtensions/{ => Lit}/ILitCommand.cs (99%) rename Source/XUnitExtensions/{ => Lit}/LitCommandWithRedirection.cs (99%) rename Source/XUnitExtensions/{ => Lit}/LitTestCase.cs (98%) rename Source/XUnitExtensions/{ => Lit}/LitTestConfiguration.cs (97%) rename Source/XUnitExtensions/{ => Lit}/LitTestDataDiscoverer.cs (96%) rename Source/XUnitExtensions/{ => Lit}/MainMethodLitCommand.cs (98%) rename Source/XUnitExtensions/{ => Lit}/OrCommand.cs (96%) rename Source/XUnitExtensions/{ => Lit}/ShellLitCommand.cs (98%) rename Source/XUnitExtensions/{ => Lit}/UnsupportedCommand.cs (95%) rename Source/XUnitExtensions/{ => Lit}/XFailCommand.cs (90%) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 1418b5de04f..bfa2fce27db 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -8,6 +8,7 @@ using Xunit; using Xunit.Abstractions; using XUnitExtensions; +using XUnitExtensions.Lit; [assembly: TestCollectionOrderer("XUnitExtensions.TestCollectionShardFilter", "XUnitExtensions")] diff --git a/Source/XUnitExtensions/ILitCommand.cs b/Source/XUnitExtensions/Lit/ILitCommand.cs similarity index 99% rename from Source/XUnitExtensions/ILitCommand.cs rename to Source/XUnitExtensions/Lit/ILitCommand.cs index e1a1accafa3..294802c1e09 100644 --- a/Source/XUnitExtensions/ILitCommand.cs +++ b/Source/XUnitExtensions/Lit/ILitCommand.cs @@ -10,7 +10,7 @@ using Xunit; using Xunit.Abstractions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { public interface ILitCommand { private const string COMMENT_PREFIX = "//"; diff --git a/Source/XUnitExtensions/LitCommandWithRedirection.cs b/Source/XUnitExtensions/Lit/LitCommandWithRedirection.cs similarity index 99% rename from Source/XUnitExtensions/LitCommandWithRedirection.cs rename to Source/XUnitExtensions/Lit/LitCommandWithRedirection.cs index b79ba3598c0..be8b788d2f1 100644 --- a/Source/XUnitExtensions/LitCommandWithRedirection.cs +++ b/Source/XUnitExtensions/Lit/LitCommandWithRedirection.cs @@ -7,7 +7,7 @@ using Microsoft.Extensions.FileSystemGlobbing.Abstractions; using Xunit.Abstractions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { public class LitCommandWithRedirection : ILitCommand { public static LitCommandWithRedirection Parse(string[] tokens, LitTestConfiguration config) { diff --git a/Source/XUnitExtensions/LitTestCase.cs b/Source/XUnitExtensions/Lit/LitTestCase.cs similarity index 98% rename from Source/XUnitExtensions/LitTestCase.cs rename to Source/XUnitExtensions/Lit/LitTestCase.cs index b55ba681207..1279d8447db 100644 --- a/Source/XUnitExtensions/LitTestCase.cs +++ b/Source/XUnitExtensions/Lit/LitTestCase.cs @@ -6,7 +6,7 @@ using Xunit; using Xunit.Abstractions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { public class LitTestCase { private string filePath; diff --git a/Source/XUnitExtensions/LitTestConfiguration.cs b/Source/XUnitExtensions/Lit/LitTestConfiguration.cs similarity index 97% rename from Source/XUnitExtensions/LitTestConfiguration.cs rename to Source/XUnitExtensions/Lit/LitTestConfiguration.cs index 1ec65cf5026..44553465e7c 100644 --- a/Source/XUnitExtensions/LitTestConfiguration.cs +++ b/Source/XUnitExtensions/Lit/LitTestConfiguration.cs @@ -4,7 +4,7 @@ using System.Reflection; using Xunit.Abstractions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { // TODO: Make safely immutable public class LitTestConfiguration { diff --git a/Source/XUnitExtensions/LitTestDataDiscoverer.cs b/Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs similarity index 96% rename from Source/XUnitExtensions/LitTestDataDiscoverer.cs rename to Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs index cb60673ca43..b852f3066ce 100644 --- a/Source/XUnitExtensions/LitTestDataDiscoverer.cs +++ b/Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs @@ -6,7 +6,7 @@ using Xunit.Sdk; using XUnitExtensions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { public class LitTestDataDiscoverer : FileDataDiscoverer { public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { diff --git a/Source/XUnitExtensions/MainMethodLitCommand.cs b/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs similarity index 98% rename from Source/XUnitExtensions/MainMethodLitCommand.cs rename to Source/XUnitExtensions/Lit/MainMethodLitCommand.cs index 8656f290e8f..473608ef1ae 100644 --- a/Source/XUnitExtensions/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs @@ -6,7 +6,7 @@ using System.Text; using Xunit.Abstractions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { public class MainMethodLitCommand : ILitCommand { public LitTestConfiguration Config { get; protected set; } public Assembly Assembly { get; protected set; } diff --git a/Source/XUnitExtensions/OrCommand.cs b/Source/XUnitExtensions/Lit/OrCommand.cs similarity index 96% rename from Source/XUnitExtensions/OrCommand.cs rename to Source/XUnitExtensions/Lit/OrCommand.cs index d1b1356a2b1..885a8f75cf2 100644 --- a/Source/XUnitExtensions/OrCommand.cs +++ b/Source/XUnitExtensions/Lit/OrCommand.cs @@ -1,7 +1,7 @@ using System.IO; using Xunit.Abstractions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { public class OrCommand : ILitCommand { private readonly ILitCommand lhs; private readonly ILitCommand rhs; diff --git a/Source/XUnitExtensions/ShellLitCommand.cs b/Source/XUnitExtensions/Lit/ShellLitCommand.cs similarity index 98% rename from Source/XUnitExtensions/ShellLitCommand.cs rename to Source/XUnitExtensions/Lit/ShellLitCommand.cs index 27afa1643e2..fbb8ae54c66 100644 --- a/Source/XUnitExtensions/ShellLitCommand.cs +++ b/Source/XUnitExtensions/Lit/ShellLitCommand.cs @@ -6,7 +6,7 @@ using System.Text; using Xunit.Abstractions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { public class ShellLitCommand : ILitCommand { private LitTestConfiguration config; private string shellCommand; diff --git a/Source/XUnitExtensions/UnsupportedCommand.cs b/Source/XUnitExtensions/Lit/UnsupportedCommand.cs similarity index 95% rename from Source/XUnitExtensions/UnsupportedCommand.cs rename to Source/XUnitExtensions/Lit/UnsupportedCommand.cs index 30bbb7b93bf..d84b9649778 100644 --- a/Source/XUnitExtensions/UnsupportedCommand.cs +++ b/Source/XUnitExtensions/Lit/UnsupportedCommand.cs @@ -5,7 +5,7 @@ using System.Linq; using Xunit.Abstractions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { public class UnsupportedCommand : ILitCommand { public static UnsupportedCommand Parse(string line) { diff --git a/Source/XUnitExtensions/XFailCommand.cs b/Source/XUnitExtensions/Lit/XFailCommand.cs similarity index 90% rename from Source/XUnitExtensions/XFailCommand.cs rename to Source/XUnitExtensions/Lit/XFailCommand.cs index 21cf83c81dd..2d8034b926f 100644 --- a/Source/XUnitExtensions/XFailCommand.cs +++ b/Source/XUnitExtensions/Lit/XFailCommand.cs @@ -1,7 +1,7 @@ using System.IO; using Xunit.Abstractions; -namespace XUnitExtensions { +namespace XUnitExtensions.Lit { public class XFailCommand : ILitCommand { public XFailCommand() { } From 062871d534710b428f5e26b4369a55d14e2a8877 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Mon, 18 Oct 2021 21:36:54 -0700 Subject: [PATCH 170/192] Add DiffCommand --- .../IntegrationTests/IntegrationTests.csproj | 3 ++ Source/IntegrationTests/LitTests.cs | 4 +-- Source/XUnitExtensions/Lit/DiffCommand.cs | 33 +++++++++++++++++++ Source/XUnitExtensions/Lit/LitTestCase.cs | 9 ++--- .../Lit/LitTestDataDiscoverer.cs | 2 +- 5 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 Source/XUnitExtensions/Lit/DiffCommand.cs diff --git a/Source/IntegrationTests/IntegrationTests.csproj b/Source/IntegrationTests/IntegrationTests.csproj index c93abebebbb..cbe809c73ca 100644 --- a/Source/IntegrationTests/IntegrationTests.csproj +++ b/Source/IntegrationTests/IntegrationTests.csproj @@ -26,6 +26,9 @@ + + PreserveNewest + PreserveNewest diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index bfa2fce27db..533fa709a27 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -47,11 +47,11 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable MainWithArguments(dafnyServerAssembly, args, config, false) }, + { "%diff", (args, config) => + DiffCommand.Parse(args.ToArray()) }, }, Substitions = new Dictionary { - // TODO: speed this up by using AssertWithDiff - { "%diff", "diff" }, { "%binaryDir", "." }, { "%z3", Path.Join("z3", "bin", "z3") }, { "%refmanexamples", Path.Join("TestFiles", "LitTests", "LitTest", "refman", "examples") } diff --git a/Source/XUnitExtensions/Lit/DiffCommand.cs b/Source/XUnitExtensions/Lit/DiffCommand.cs new file mode 100644 index 00000000000..8bdc32063e0 --- /dev/null +++ b/Source/XUnitExtensions/Lit/DiffCommand.cs @@ -0,0 +1,33 @@ +using System.IO; +using Xunit; +using Xunit.Abstractions; + +namespace XUnitExtensions.Lit { + public class DiffCommand : ILitCommand { + + private readonly string expectedPath; + private readonly string actualPath; + + private DiffCommand(string expectedPath, string actualPath) { + this.expectedPath = expectedPath; + this.actualPath = actualPath; + } + + public static ILitCommand Parse(string[] args) { + var expectedPath = args[0]; + var actualPath = args[1]; + return new DiffCommand(expectedPath, actualPath); + } + + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { + var expected = File.ReadAllText(expectedPath); + var actual = File.ReadAllText(actualPath); + AssertWithDiff.Equal(expected, actual); + return (0, "", ""); + } + + public override string ToString() { + return $"DiffCommand {expectedPath} {actualPath}"; + } + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/Lit/LitTestCase.cs b/Source/XUnitExtensions/Lit/LitTestCase.cs index 1279d8447db..70ab9c55bab 100644 --- a/Source/XUnitExtensions/Lit/LitTestCase.cs +++ b/Source/XUnitExtensions/Lit/LitTestCase.cs @@ -9,14 +9,15 @@ namespace XUnitExtensions.Lit { public class LitTestCase { - private string filePath; - private IEnumerable commands; - private bool expectFailure; + private readonly string filePath; + private readonly IEnumerable commands; + private readonly bool expectFailure; public static LitTestCase Read(string filePath, LitTestConfiguration config) { var commands = File.ReadAllLines(filePath) .Select(line => ILitCommand.Parse(filePath, line, config)) - .Where(c => c != null); + .Where(c => c != null) + .ToArray(); var xfail = commands.Any(c => c is XFailCommand); foreach (var unsupported in commands.OfType()) { foreach (var feature in config.Features) { diff --git a/Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs b/Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs index b852f3066ce..6eb2ac96e6e 100644 --- a/Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs +++ b/Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs @@ -25,7 +25,7 @@ protected override IEnumerable FileData(IAttributeInfo attributeInfo, } } -[DataDiscoverer("XUnitExtensions.LitTestDataDiscoverer", "XUnitExtensions")] +[DataDiscoverer("XUnitExtensions.Lit.LitTestDataDiscoverer", "XUnitExtensions")] public class LitTestDataAttribute : FileDataAttribute { } \ No newline at end of file From c027ae9a2a9cd19e915d9c4cf39f952a8414b6ec Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 10:07:16 -0700 Subject: [PATCH 171/192] Revert DiffCommand, generalize command parsing to support .transcript too --- Source/IntegrationTests/LitTests.cs | 9 +++- Source/XUnitExtensions/AssertWithDiff.cs | 35 ------------ Source/XUnitExtensions/Lit/DiffCommand.cs | 33 ------------ Source/XUnitExtensions/Lit/ILitCommand.cs | 54 +++++++------------ Source/XUnitExtensions/Lit/LitTestCase.cs | 5 +- Source/XUnitExtensions/Lit/RunCommand.cs | 22 ++++++++ .../XUnitExtensions/Lit/UnsupportedCommand.cs | 6 +-- Source/XUnitExtensions/Lit/XFailCommand.cs | 10 +++- 8 files changed, 63 insertions(+), 111 deletions(-) delete mode 100644 Source/XUnitExtensions/AssertWithDiff.cs delete mode 100644 Source/XUnitExtensions/Lit/DiffCommand.cs create mode 100644 Source/XUnitExtensions/Lit/RunCommand.cs diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 533fa709a27..17f5dde1e6b 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -47,11 +47,10 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable MainWithArguments(dafnyServerAssembly, args, config, false) }, - { "%diff", (args, config) => - DiffCommand.Parse(args.ToArray()) }, }, Substitions = new Dictionary { + { "%diff", "diff" }, { "%binaryDir", "." }, { "%z3", Path.Join("z3", "bin", "z3") }, { "%refmanexamples", Path.Join("TestFiles", "LitTests", "LitTest", "refman", "examples") } @@ -94,5 +93,11 @@ public LitTests(ITestOutputHelper output) { public void LitTest(string path) { LitTestCase.Run(path, CONFIG, output); } + + [FileTheory] + [LitTestData(Path = "TestFiles/LitTests/LitTest", Extension = ".transcript")] + public void DafnyServerTest(string path) { + LitTestCase.Run(path, CONFIG, output); + } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/AssertWithDiff.cs b/Source/XUnitExtensions/AssertWithDiff.cs deleted file mode 100644 index f6a2f794860..00000000000 --- a/Source/XUnitExtensions/AssertWithDiff.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System.Text; -using DiffPlex; -using DiffPlex.DiffBuilder; -using DiffPlex.DiffBuilder.Model; -using Xunit.Sdk; - -namespace XUnitExtensions { - public class AssertWithDiff { - public static void Equal(string expected, string actual) { - if (expected != actual) { - var diff = InlineDiffBuilder.Instance.BuildDiffModel(expected, actual); - - var message = new StringBuilder(); - message.AppendLine("AssertEqualWithDiff() Failure"); - message.AppendLine("Diff (changing expected into actual):"); - foreach (var line in diff.Lines) { - switch (line.Type) { - case ChangeType.Inserted: - message.Append('+'); - break; - case ChangeType.Deleted: - message.Append('-'); - break; - default: - message.Append(' '); - break; - } - message.AppendLine(line.Text); - } - - throw new AssertActualExpectedException(expected, actual, message.ToString()); - } - } - } -} \ No newline at end of file diff --git a/Source/XUnitExtensions/Lit/DiffCommand.cs b/Source/XUnitExtensions/Lit/DiffCommand.cs deleted file mode 100644 index 8bdc32063e0..00000000000 --- a/Source/XUnitExtensions/Lit/DiffCommand.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System.IO; -using Xunit; -using Xunit.Abstractions; - -namespace XUnitExtensions.Lit { - public class DiffCommand : ILitCommand { - - private readonly string expectedPath; - private readonly string actualPath; - - private DiffCommand(string expectedPath, string actualPath) { - this.expectedPath = expectedPath; - this.actualPath = actualPath; - } - - public static ILitCommand Parse(string[] args) { - var expectedPath = args[0]; - var actualPath = args[1]; - return new DiffCommand(expectedPath, actualPath); - } - - public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { - var expected = File.ReadAllText(expectedPath); - var actual = File.ReadAllText(actualPath); - AssertWithDiff.Equal(expected, actual); - return (0, "", ""); - } - - public override string ToString() { - return $"DiffCommand {expectedPath} {actualPath}"; - } - } -} \ No newline at end of file diff --git a/Source/XUnitExtensions/Lit/ILitCommand.cs b/Source/XUnitExtensions/Lit/ILitCommand.cs index 294802c1e09..61fabf6cac0 100644 --- a/Source/XUnitExtensions/Lit/ILitCommand.cs +++ b/Source/XUnitExtensions/Lit/ILitCommand.cs @@ -13,46 +13,26 @@ namespace XUnitExtensions.Lit { public interface ILitCommand { - private const string COMMENT_PREFIX = "//"; - private const string LIT_COMMAND_PREFIX = "RUN:"; - private const string LIT_UNSUPPORTED = "UNSUPPORTED"; - private const string LIT_XFAIL_ALL = "XFAIL: *"; - - public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter); - - public static ILitCommand? Parse(string fileName, string line, LitTestConfiguration config) { - if (!line.StartsWith(COMMENT_PREFIX)) { - return null; - } - line = line[COMMENT_PREFIX.Length..].Trim(); - - if (line.Equals(LIT_XFAIL_ALL)) { - return new XFailCommand(); - } - if (line.StartsWith(LIT_UNSUPPORTED)) { - return UnsupportedCommand.Parse(line[LIT_UNSUPPORTED.Length..].Trim()); - } - if (!line.StartsWith(LIT_COMMAND_PREFIX)) { - return null; - } - line = line[LIT_COMMAND_PREFIX.Length..].Trim(); - - var tokens = Tokenize(line); - return ParseRunCommand(tokens, config); + private static readonly Dictionary CommandClasses = new(); + static ILitCommand() { + CommandClasses.Add("RUN:", typeof(RunCommand)); + CommandClasses.Add("UNSUPPORTED:", typeof(UnsupportedCommand)); + CommandClasses.Add("XFAIL:", typeof(XFailCommand)); } - - public static ILitCommand ParseRunCommand(string[] tokens, LitTestConfiguration config) { - // Just supporting || for now since it's a precise way to ignore an exit code - var seqOperatorIndex = Array.IndexOf(tokens, "||"); - if (seqOperatorIndex >= 0) { - var lhs = LitCommandWithRedirection.Parse(tokens[0..seqOperatorIndex], config); - var rhs = ParseRunCommand(tokens[(seqOperatorIndex + 1)..], config); - return new OrCommand(lhs, rhs); + + public static ILitCommand? Parse(string line, LitTestConfiguration config) { + foreach (var (keyword, type) in CommandClasses) { + var index = line.IndexOf(keyword); + if (index >= 0) { + var arguments = line[(index + keyword.Length)..].Trim(); + var parseMethod = type.GetMethod("Parse", new[] { typeof(string), typeof(LitTestConfiguration) }); + return (ILitCommand)parseMethod!.Invoke(null, new object[] { arguments, config }); + } } - return LitCommandWithRedirection.Parse(tokens, config); + return null; } - private static string[] Tokenize(string line) { + public static string[] Tokenize(string line) { var arguments = new List(); var argument = new StringBuilder(); var singleQuoted = false; @@ -100,5 +80,7 @@ protected static IEnumerable ExpandGlobs(string chunk ) { var result = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo("."))); return result.Files.Select(f => f.Path); } + + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter); } } \ No newline at end of file diff --git a/Source/XUnitExtensions/Lit/LitTestCase.cs b/Source/XUnitExtensions/Lit/LitTestCase.cs index 70ab9c55bab..c85cdef8e7d 100644 --- a/Source/XUnitExtensions/Lit/LitTestCase.cs +++ b/Source/XUnitExtensions/Lit/LitTestCase.cs @@ -15,9 +15,12 @@ public class LitTestCase { public static LitTestCase Read(string filePath, LitTestConfiguration config) { var commands = File.ReadAllLines(filePath) - .Select(line => ILitCommand.Parse(filePath, line, config)) + .Select(line => ILitCommand.Parse(line, config)) .Where(c => c != null) .ToArray(); + if (commands.Length == 0) { + throw new ArgumentException($"No lit commands found in test file: {filePath}"); + } var xfail = commands.Any(c => c is XFailCommand); foreach (var unsupported in commands.OfType()) { foreach (var feature in config.Features) { diff --git a/Source/XUnitExtensions/Lit/RunCommand.cs b/Source/XUnitExtensions/Lit/RunCommand.cs new file mode 100644 index 00000000000..5afbbdc74a3 --- /dev/null +++ b/Source/XUnitExtensions/Lit/RunCommand.cs @@ -0,0 +1,22 @@ +using System; +using System.IO; +using Xunit.Abstractions; + +namespace XUnitExtensions.Lit { + public abstract class RunCommand { + public static ILitCommand Parse(string line, LitTestConfiguration config) { + return ParseArguments(ILitCommand.Tokenize(line), config); + } + + private static ILitCommand ParseArguments(string[] tokens, LitTestConfiguration config) { + // Just supporting || for now since it's a precise way to ignore an exit code + var seqOperatorIndex = Array.IndexOf(tokens, "||"); + if (seqOperatorIndex >= 0) { + var lhs = LitCommandWithRedirection.Parse(tokens[0..seqOperatorIndex], config); + var rhs = ParseArguments(tokens[(seqOperatorIndex + 1)..], config); + return new OrCommand(lhs, rhs); + } + return LitCommandWithRedirection.Parse(tokens, config); + } + } +} \ No newline at end of file diff --git a/Source/XUnitExtensions/Lit/UnsupportedCommand.cs b/Source/XUnitExtensions/Lit/UnsupportedCommand.cs index d84b9649778..a90fbfa9630 100644 --- a/Source/XUnitExtensions/Lit/UnsupportedCommand.cs +++ b/Source/XUnitExtensions/Lit/UnsupportedCommand.cs @@ -8,15 +8,15 @@ namespace XUnitExtensions.Lit { public class UnsupportedCommand : ILitCommand { - public static UnsupportedCommand Parse(string line) { + public static UnsupportedCommand Parse(string line, LitTestConfiguration config) { var features = line.Split(",", StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()); return new UnsupportedCommand(features); } - public IEnumerable Features { get; protected set; } + public IEnumerable Features { get; } public UnsupportedCommand(IEnumerable features) { - this.Features = features; + Features = features; } public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { diff --git a/Source/XUnitExtensions/Lit/XFailCommand.cs b/Source/XUnitExtensions/Lit/XFailCommand.cs index 2d8034b926f..7c438171036 100644 --- a/Source/XUnitExtensions/Lit/XFailCommand.cs +++ b/Source/XUnitExtensions/Lit/XFailCommand.cs @@ -1,10 +1,18 @@ +using System; using System.IO; using Xunit.Abstractions; namespace XUnitExtensions.Lit { public class XFailCommand : ILitCommand { - public XFailCommand() { } + public static ILitCommand Parse(string line, LitTestConfiguration config) { + // Only supporting * for now + if (line.Equals("*")) { + return new XFailCommand(); + } + + throw new ArgumentException($"Unrecognized arguments to XFAIL: {line}"); + } public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { return (0, "", ""); From 16668f9cf865b793e00877b04e84424fcfba7b18 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 10:18:49 -0700 Subject: [PATCH 172/192] Formatting --- Source/IntegrationTests/LitTests.cs | 4 ++-- Source/XUnitExtensions/Lit/ILitCommand.cs | 8 ++++---- Source/XUnitExtensions/Lit/LitTestConfiguration.cs | 2 +- Source/XUnitExtensions/Lit/ShellLitCommand.cs | 2 +- Source/XUnitExtensions/Lit/UnsupportedCommand.cs | 4 ++-- Source/XUnitExtensions/TestCollectionShardFilter.cs | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 17f5dde1e6b..ff2c946ad83 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -70,7 +70,7 @@ static LitTests() { new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "DafnyServer"), args, config.PassthroughEnvironmentVariables); CONFIG.Substitions["%z3"] = Path.Join(dafnyReleaseDir, "z3", "bin", "z3"); } - + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { CONFIG.Features = new[] { "ubuntu", "posix" }; } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { @@ -93,7 +93,7 @@ public LitTests(ITestOutputHelper output) { public void LitTest(string path) { LitTestCase.Run(path, CONFIG, output); } - + [FileTheory] [LitTestData(Path = "TestFiles/LitTests/LitTest", Extension = ".transcript")] public void DafnyServerTest(string path) { diff --git a/Source/XUnitExtensions/Lit/ILitCommand.cs b/Source/XUnitExtensions/Lit/ILitCommand.cs index 61fabf6cac0..0b4de109ea9 100644 --- a/Source/XUnitExtensions/Lit/ILitCommand.cs +++ b/Source/XUnitExtensions/Lit/ILitCommand.cs @@ -19,7 +19,7 @@ static ILitCommand() { CommandClasses.Add("UNSUPPORTED:", typeof(UnsupportedCommand)); CommandClasses.Add("XFAIL:", typeof(XFailCommand)); } - + public static ILitCommand? Parse(string line, LitTestConfiguration config) { foreach (var (keyword, type) in CommandClasses) { var index = line.IndexOf(keyword); @@ -73,14 +73,14 @@ public static string[] Tokenize(string line) { return arguments.ToArray(); } - - protected static IEnumerable ExpandGlobs(string chunk ) { + + protected static IEnumerable ExpandGlobs(string chunk) { var matcher = new Matcher(); matcher.AddInclude(chunk); var result = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo("."))); return result.Files.Select(f => f.Path); } - + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter); } } \ No newline at end of file diff --git a/Source/XUnitExtensions/Lit/LitTestConfiguration.cs b/Source/XUnitExtensions/Lit/LitTestConfiguration.cs index 44553465e7c..da8ac895691 100644 --- a/Source/XUnitExtensions/Lit/LitTestConfiguration.cs +++ b/Source/XUnitExtensions/Lit/LitTestConfiguration.cs @@ -13,7 +13,7 @@ public class LitTestConfiguration { public Dictionary, LitTestConfiguration, ILitCommand>> Commands { get; set; } public IEnumerable PassthroughEnvironmentVariables { get; set; } - + public string[] Features { get; set; } public string ApplySubstitutions(string s) { diff --git a/Source/XUnitExtensions/Lit/ShellLitCommand.cs b/Source/XUnitExtensions/Lit/ShellLitCommand.cs index fbb8ae54c66..46da67974f8 100644 --- a/Source/XUnitExtensions/Lit/ShellLitCommand.cs +++ b/Source/XUnitExtensions/Lit/ShellLitCommand.cs @@ -22,7 +22,7 @@ public ShellLitCommand(LitTestConfiguration config, string shellCommand, IEnumer public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { using var process = new Process(); - + process.StartInfo.FileName = shellCommand; foreach (var argument in arguments) { process.StartInfo.ArgumentList.Add(argument); diff --git a/Source/XUnitExtensions/Lit/UnsupportedCommand.cs b/Source/XUnitExtensions/Lit/UnsupportedCommand.cs index a90fbfa9630..b9d392cb99d 100644 --- a/Source/XUnitExtensions/Lit/UnsupportedCommand.cs +++ b/Source/XUnitExtensions/Lit/UnsupportedCommand.cs @@ -12,13 +12,13 @@ public static UnsupportedCommand Parse(string line, LitTestConfiguration config) var features = line.Split(",", StringSplitOptions.RemoveEmptyEntries).Select(s => s.Trim()); return new UnsupportedCommand(features); } - + public IEnumerable Features { get; } public UnsupportedCommand(IEnumerable features) { Features = features; } - + public (int, string, string) Execute(ITestOutputHelper outputHelper, TextReader? inputReader, TextWriter? outputWriter, TextWriter? errorWriter) { return (0, "", ""); } diff --git a/Source/XUnitExtensions/TestCollectionShardFilter.cs b/Source/XUnitExtensions/TestCollectionShardFilter.cs index 8ca6cc2b253..cf7c3c3f4c3 100644 --- a/Source/XUnitExtensions/TestCollectionShardFilter.cs +++ b/Source/XUnitExtensions/TestCollectionShardFilter.cs @@ -8,7 +8,7 @@ namespace XUnitExtensions { public class TestCollectionShardFilter : ITestCollectionOrderer { public IEnumerable OrderTestCollections(IEnumerable testCollections) { var sorted = testCollections.OrderBy(c => c.DisplayName); - + // Select the requested fraction of the test collections if using the XUNIT_SHARD[_COUNT] environment variables. var shardEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD"); var numShardsEnvVar = Environment.GetEnvironmentVariable("XUNIT_SHARD_COUNT"); From 87852d8976f99617e3d26ff0394e4d829fb78313 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 16:25:00 -0700 Subject: [PATCH 173/192] Simplify file data attributes, exclude Inputs Also simplify handling z3 for the other 3 test projects in the same way as IntegrationTests --- .github/workflows/xunit-tests.yml | 26 ++++------- .../DafnyLanguageServer.Test.csproj | 4 ++ .../DafnyPipeline.Test.csproj | 6 +++ .../DafnyTestGeneration.Test.csproj | 5 +++ Source/IntegrationTests/LitTests.cs | 9 +--- Source/XUnitExtensions/FileDataAttribute.cs | 43 +++++++++++++++++-- Source/XUnitExtensions/FileDataDiscoverer.cs | 42 ------------------ Source/XUnitExtensions/FileTestCase.cs | 2 + .../Lit/LitTestDataDiscoverer.cs | 31 ------------- 9 files changed, 67 insertions(+), 101 deletions(-) delete mode 100644 Source/XUnitExtensions/FileDataDiscoverer.cs delete mode 100644 Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs diff --git a/.github/workflows/xunit-tests.yml b/.github/workflows/xunit-tests.yml index 322188cd6e9..f2c17b55126 100644 --- a/.github/workflows/xunit-tests.yml +++ b/.github/workflows/xunit-tests.yml @@ -30,9 +30,6 @@ jobs: chmod: true coverage: false env: - testPath1: Source/DafnyLanguageServer.Test/bin/Debug/net5.0 - testPath2: Source/DafnyPipeline.Test/bin/Debug/net5.0 - testPath3: Source/DafnyTestGeneration.Test/bin/Debug/net5.0 solutionPath: Source/Dafny.sln z3BaseUri: https://github.com/Z3Prover/z3/releases/download/Z3-4.8.5 steps: @@ -47,23 +44,18 @@ jobs: run: dotnet restore ${{env.solutionPath}} - name: Build Java runtime run: make --quiet runtime - - name: Build - run: dotnet build --no-restore ${{env.solutionPath}} - name: Load Z3 - shell: pwsh - run: | - Invoke-WebRequest ${{env.z3BaseUri}}/${{matrix.z3}}.zip -OutFile z3.zip - Expand-Archive z3.zip . - Remove-Item z3.zip - Copy-Item ${{matrix.z3}} ${{env.testPath1}}/z3 -Recurse - Copy-Item ${{matrix.z3}} ${{env.testPath2}}/z3 -Recurse - Copy-Item ${{matrix.z3}} ${{env.testPath3}}/z3 -Recurse + shell: pwsh + run: | + Invoke-WebRequest ${{env.z3BaseUri}}/${{matrix.z3}}.zip -OutFile z3.zip + Expand-Archive z3.zip Binaries + Remove-Item z3.zip - name: Set Z3 Permissions if: ${{matrix.chmod}} - run: | - chmod +x ${{env.testPath1}}/z3/bin/z3 - chmod +x ${{env.testPath2}}/z3/bin/z3 - chmod +x ${{env.testPath3}}/z3/bin/z3 + run: | + chmod +x Binaries/z3/bin/z3 + - name: Build + run: dotnet build --no-restore ${{env.solutionPath}} - name: Run DafnyLanguageServer Tests run: dotnet test --no-restore --verbosity normal Source/DafnyLanguageServer.Test - name: Run DafnyPipeline Tests diff --git a/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj b/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj index 9c45bb2404c..bc0988475e2 100644 --- a/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj +++ b/Source/DafnyLanguageServer.Test/DafnyLanguageServer.Test.csproj @@ -21,6 +21,10 @@ + + PreserveNewest + + PreserveNewest diff --git a/Source/DafnyPipeline.Test/DafnyPipeline.Test.csproj b/Source/DafnyPipeline.Test/DafnyPipeline.Test.csproj index 675331b71f0..fac62db3754 100644 --- a/Source/DafnyPipeline.Test/DafnyPipeline.Test.csproj +++ b/Source/DafnyPipeline.Test/DafnyPipeline.Test.csproj @@ -24,4 +24,10 @@ + + + PreserveNewest + + + diff --git a/Source/DafnyTestGeneration.Test/DafnyTestGeneration.Test.csproj b/Source/DafnyTestGeneration.Test/DafnyTestGeneration.Test.csproj index 222a855a3bb..e8c1199552e 100644 --- a/Source/DafnyTestGeneration.Test/DafnyTestGeneration.Test.csproj +++ b/Source/DafnyTestGeneration.Test/DafnyTestGeneration.Test.csproj @@ -17,4 +17,9 @@ + + + PreserveNewest + + diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index ff2c946ad83..e2810a48208 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -89,15 +89,10 @@ public LitTests(ITestOutputHelper output) { } [FileTheory] - [LitTestData(Extension = ".dfy")] + [FileData(Includes = new [] { "**/*.dfy", "**/*.transcript" }, + Excludes = new [] { "**/Inputs/**/*" })] public void LitTest(string path) { LitTestCase.Run(path, CONFIG, output); } - - [FileTheory] - [LitTestData(Path = "TestFiles/LitTests/LitTest", Extension = ".transcript")] - public void DafnyServerTest(string path) { - LitTestCase.Run(path, CONFIG, output); - } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/FileDataAttribute.cs b/Source/XUnitExtensions/FileDataAttribute.cs index d06373a5f16..1eab9716330 100644 --- a/Source/XUnitExtensions/FileDataAttribute.cs +++ b/Source/XUnitExtensions/FileDataAttribute.cs @@ -1,15 +1,50 @@ +using System; using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Reflection; +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; +using Xunit.Abstractions; using Xunit.Sdk; namespace XUnitExtensions { - public abstract class FileDataAttribute : DataAttribute { - public virtual string? Path { get; set; } + public class FileDataAttribute : DataAttribute { + public string? Directory { get; set; } + public string[] Includes { get; set; } + public string[] Excludes { get; set; } - public virtual string? Extension { get; set; } + protected string GetBasePath(MethodInfo testMethod) { + if (Directory != null) { + return Directory; + } + return Path.Combine("TestFiles", testMethod.DeclaringType.Name, testMethod.Name); + } public override IEnumerable GetData(MethodInfo testMethod) { - throw new System.NotImplementedException(); + var matcher = new Matcher(); + + foreach(var include in Includes) { + matcher.AddInclude(include); + } + foreach(var exclude in Excludes) { + matcher.AddExclude(exclude); + } + + var basePath = GetBasePath(testMethod); + var result = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo(basePath))); + if (!result.HasMatches) { + throw new ArgumentException("No matching files found: " + this); + } + + return result.Files.Select(file => { + var fullPath = Path.Join(basePath, file.Stem); + var row = new FileTheoryDataRow(fullPath) { + SourceInformation = new SourceInformation() { FileName = fullPath, LineNumber = 0 }, + TestDisplayName = file.Stem, + }; + return new[] { row }; + }); } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/FileDataDiscoverer.cs b/Source/XUnitExtensions/FileDataDiscoverer.cs deleted file mode 100644 index e1df405ee9c..00000000000 --- a/Source/XUnitExtensions/FileDataDiscoverer.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using Xunit.Abstractions; -using Xunit.Sdk; - -namespace XUnitExtensions { - public abstract class FileDataDiscoverer : IDataDiscoverer { - protected string GetBasePath(IAttributeInfo attributeInfo, IMethodInfo testMethod) { - var path = attributeInfo.GetNamedArgument(nameof(FileDataAttribute.Path)); - - if (path != null) { - return path; - } - - return Path.Combine("TestFiles", testMethod.ToRuntimeMethod().DeclaringType.Name, testMethod.Name); - } - - public IEnumerable GetData(IAttributeInfo attributeInfo, IMethodInfo testMethod) { - var path = GetBasePath(attributeInfo, testMethod); - var extension = attributeInfo.GetNamedArgument(nameof(FileDataAttribute.Extension)); - - if (Directory.Exists(path)) { - return Directory.EnumerateFiles(path, "*" + extension, SearchOption.AllDirectories) - .SelectMany(childPath => FileData(attributeInfo, testMethod, childPath)); - } - - var fileName = path + extension; - if (File.Exists(fileName)) { - return FileData(attributeInfo, testMethod, fileName); - } - - throw new ArgumentException("No data found for path: " + path); - } - - public abstract bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod); - - protected abstract IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName); - } -} \ No newline at end of file diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index 55565eb8ccd..58f1ff62341 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -68,6 +68,8 @@ public ISourceInformation SourceInformation { public async Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { + // This is the same thing SkippableFactTestCase.RunAsync does. + // support for dynamic test skipping is much cleaner in xUnit 3. var messageBusInterceptor = new SkippableTestMessageBus(messageBus, skippingExceptionNames); RunSummary result = await new XunitTestCaseRunner( this, diff --git a/Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs b/Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs deleted file mode 100644 index 6eb2ac96e6e..00000000000 --- a/Source/XUnitExtensions/Lit/LitTestDataDiscoverer.cs +++ /dev/null @@ -1,31 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Xunit.Abstractions; -using Xunit.Sdk; -using XUnitExtensions; - -namespace XUnitExtensions.Lit { - public class LitTestDataDiscoverer : FileDataDiscoverer { - - public override bool SupportsDiscoveryEnumeration(IAttributeInfo dataAttribute, IMethodInfo testMethod) { - return true; - } - - protected override IEnumerable FileData(IAttributeInfo attributeInfo, IMethodInfo testMethod, string fileName) { - var basePath = GetBasePath(attributeInfo, testMethod); - var shortPath = fileName[(basePath.Length + 1)..]; - var row = new FileTheoryDataRow(fileName) { - SourceInformation = new SourceInformation() { FileName = fileName, LineNumber = 0 }, - TestDisplayName = shortPath, - }; - return new[] { new[] { row } }; - } - } -} - -[DataDiscoverer("XUnitExtensions.Lit.LitTestDataDiscoverer", "XUnitExtensions")] -public class LitTestDataAttribute : FileDataAttribute { - -} \ No newline at end of file From 59543accd57779a09d30b84b75230d262e85546d Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 16:27:31 -0700 Subject: [PATCH 174/192] Formatting --- Source/IntegrationTests/LitTests.cs | 4 ++-- Source/XUnitExtensions/FileDataAttribute.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index e2810a48208..d6a4325d8d5 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -89,8 +89,8 @@ public LitTests(ITestOutputHelper output) { } [FileTheory] - [FileData(Includes = new [] { "**/*.dfy", "**/*.transcript" }, - Excludes = new [] { "**/Inputs/**/*" })] + [FileData(Includes = new[] { "**/*.dfy", "**/*.transcript" }, + Excludes = new[] { "**/Inputs/**/*" })] public void LitTest(string path) { LitTestCase.Run(path, CONFIG, output); } diff --git a/Source/XUnitExtensions/FileDataAttribute.cs b/Source/XUnitExtensions/FileDataAttribute.cs index 1eab9716330..e89bfb88147 100644 --- a/Source/XUnitExtensions/FileDataAttribute.cs +++ b/Source/XUnitExtensions/FileDataAttribute.cs @@ -24,10 +24,10 @@ protected string GetBasePath(MethodInfo testMethod) { public override IEnumerable GetData(MethodInfo testMethod) { var matcher = new Matcher(); - foreach(var include in Includes) { + foreach (var include in Includes) { matcher.AddInclude(include); } - foreach(var exclude in Excludes) { + foreach (var exclude in Excludes) { matcher.AddExclude(exclude); } @@ -44,7 +44,7 @@ public override IEnumerable GetData(MethodInfo testMethod) { TestDisplayName = file.Stem, }; return new[] { row }; - }); + }); } } } \ No newline at end of file From 71ee9929a87d8a53c3d6433ffe0955c99bad7bbc Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 16:53:46 -0700 Subject: [PATCH 175/192] Fix xunit-tests.yml indentation, excludes refman/examples --- .github/workflows/xunit-tests.yml | 10 +++++----- Source/IntegrationTests/LitTests.cs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/xunit-tests.yml b/.github/workflows/xunit-tests.yml index f2c17b55126..28d956e27f4 100644 --- a/.github/workflows/xunit-tests.yml +++ b/.github/workflows/xunit-tests.yml @@ -45,11 +45,11 @@ jobs: - name: Build Java runtime run: make --quiet runtime - name: Load Z3 - shell: pwsh - run: | - Invoke-WebRequest ${{env.z3BaseUri}}/${{matrix.z3}}.zip -OutFile z3.zip - Expand-Archive z3.zip Binaries - Remove-Item z3.zip + shell: pwsh + run: | + Invoke-WebRequest ${{env.z3BaseUri}}/${{matrix.z3}}.zip -OutFile z3.zip + Expand-Archive z3.zip Binaries + Remove-Item z3.zip - name: Set Z3 Permissions if: ${{matrix.chmod}} run: | diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index d6a4325d8d5..c0ab7f26a86 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -90,7 +90,7 @@ public LitTests(ITestOutputHelper output) { [FileTheory] [FileData(Includes = new[] { "**/*.dfy", "**/*.transcript" }, - Excludes = new[] { "**/Inputs/**/*" })] + Excludes = new[] { "**/Inputs/**/*", "refman/examples/**/*" })] public void LitTest(string path) { LitTestCase.Run(path, CONFIG, output); } From 5766472091e70e85a20d52a327a611eeab768cd4 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 18:23:57 -0700 Subject: [PATCH 176/192] Fix z3 unzipped directory name --- .github/workflows/xunit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/xunit-tests.yml b/.github/workflows/xunit-tests.yml index 28d956e27f4..938e7403bea 100644 --- a/.github/workflows/xunit-tests.yml +++ b/.github/workflows/xunit-tests.yml @@ -48,7 +48,7 @@ jobs: shell: pwsh run: | Invoke-WebRequest ${{env.z3BaseUri}}/${{matrix.z3}}.zip -OutFile z3.zip - Expand-Archive z3.zip Binaries + Expand-Archive z3.zip Binaries/z3 Remove-Item z3.zip - name: Set Z3 Permissions if: ${{matrix.chmod}} From ec03867c8df0dd1954de74803674506732a521e8 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 18:28:55 -0700 Subject: [PATCH 177/192] One more try before I give up --- .github/workflows/xunit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/xunit-tests.yml b/.github/workflows/xunit-tests.yml index 938e7403bea..8c3305d81f5 100644 --- a/.github/workflows/xunit-tests.yml +++ b/.github/workflows/xunit-tests.yml @@ -48,7 +48,7 @@ jobs: shell: pwsh run: | Invoke-WebRequest ${{env.z3BaseUri}}/${{matrix.z3}}.zip -OutFile z3.zip - Expand-Archive z3.zip Binaries/z3 + Expand-Archive z3.zip ./Binaries/z3 Remove-Item z3.zip - name: Set Z3 Permissions if: ${{matrix.chmod}} From 9b4e75beae97208f62cfb4bd169cbc6710e686a1 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 18:45:56 -0700 Subject: [PATCH 178/192] Okay I lied --- .github/workflows/xunit-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/xunit-tests.yml b/.github/workflows/xunit-tests.yml index 8c3305d81f5..9fb3037b322 100644 --- a/.github/workflows/xunit-tests.yml +++ b/.github/workflows/xunit-tests.yml @@ -48,8 +48,9 @@ jobs: shell: pwsh run: | Invoke-WebRequest ${{env.z3BaseUri}}/${{matrix.z3}}.zip -OutFile z3.zip - Expand-Archive z3.zip ./Binaries/z3 + Expand-Archive z3.zip . Remove-Item z3.zip + Copy-Item ${{matrix.z3}} Binaries/z3 -Recurse - name: Set Z3 Permissions if: ${{matrix.chmod}} run: | From 123b5c98145dd27c9da20d6fba0f1631e0a283e5 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 21:23:03 -0700 Subject: [PATCH 179/192] Fix customBoogie.patch, misc. cleanup --- .github/workflows/msbuild.yml | 2 +- .gitignore | 1 - Source/Dafny.sln | 6 ++++-- customBoogie.patch | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index ddad1097389..abf0591c51a 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -104,7 +104,7 @@ jobs: java-version: 1.8 - name: Upgrade outdated pip run: python -m pip install --upgrade pip - - name: Install lit + - name: Install lit run: pip install lit OutputCheck pyyaml - uses: actions/setup-node@v1 - run: npm install bignumber.js diff --git a/.gitignore b/.gitignore index 32a7520704b..eecb6d98b8b 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ Package/ Source/*/bin/ Source/*/obj/ -Source/*/bin_core/ Source/*/obj_core/ Source/packages diff --git a/Source/Dafny.sln b/Source/Dafny.sln index d84bc7e3f15..05949878fbb 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -64,6 +64,10 @@ Global {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Debug|Any CPU.Build.0 = Debug|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Any CPU.ActiveCfg = Release|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Any CPU.Build.0 = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {896E7F24-FD59-4B34-A1BF-41978DFFE24E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {896E7F24-FD59-4B34-A1BF-41978DFFE24E}.Release|Any CPU.Build.0 = Release|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Any CPU.Build.0 = Debug|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -76,8 +80,6 @@ Global {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Debug|Any CPU.Build.0 = Debug|Any CPU {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Release|Any CPU.ActiveCfg = Release|Any CPU {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Release|Any CPU.Build.0 = Release|Any CPU - {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.Build.0 = Debug|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/customBoogie.patch b/customBoogie.patch index 2e3eff57ac5..a6f8e5c3b1b 100644 --- a/customBoogie.patch +++ b/customBoogie.patch @@ -2,7 +2,7 @@ diff --git a/Source/Dafny.sln b/Source/Dafny.sln index 3cc1738d..789c04ca 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln -@@ -26,6 +26,30 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTestGeneration", "Dafn +@@ -30,6 +30,30 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTestGeneration", "Dafn EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DafnyTestGeneration.Test", "DafnyTestGeneration.Test\DafnyTestGeneration.Test.csproj", "{896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}" EndProject @@ -33,7 +33,7 @@ index 3cc1738d..789c04ca 100644 Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU -@@ -68,6 +92,64 @@ Global +@@ -80,6 +104,64 @@ Global {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Debug|Any CPU.Build.0 = Debug|Any CPU {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Release|Any CPU.ActiveCfg = Release|Any CPU {896E7F24-FD59-4B34-A1BF-53C51DDBC9E9}.Release|Any CPU.Build.0 = Release|Any CPU From 9b58828a0c84e25309e3af665c7b8485255668ba Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 21:59:08 -0700 Subject: [PATCH 180/192] Fixing most nullability warnings --- Source/XUnitExtensions/FileDataAttribute.cs | 26 +++++++++---------- Source/XUnitExtensions/FileTestCase.cs | 4 +-- Source/XUnitExtensions/FileTheoryDataRow.cs | 6 ++--- Source/XUnitExtensions/Lit/ILitCommand.cs | 2 +- Source/XUnitExtensions/Lit/LitTestCase.cs | 3 ++- .../Lit/MainMethodLitCommand.cs | 4 +-- 6 files changed, 21 insertions(+), 24 deletions(-) diff --git a/Source/XUnitExtensions/FileDataAttribute.cs b/Source/XUnitExtensions/FileDataAttribute.cs index e89bfb88147..26cfd280b3e 100644 --- a/Source/XUnitExtensions/FileDataAttribute.cs +++ b/Source/XUnitExtensions/FileDataAttribute.cs @@ -11,27 +11,25 @@ namespace XUnitExtensions { public class FileDataAttribute : DataAttribute { public string? Directory { get; set; } - public string[] Includes { get; set; } - public string[] Excludes { get; set; } - - protected string GetBasePath(MethodInfo testMethod) { - if (Directory != null) { - return Directory; - } - return Path.Combine("TestFiles", testMethod.DeclaringType.Name, testMethod.Name); - } + public string[]? Includes { get; set; } + public string[]? Excludes { get; set; } public override IEnumerable GetData(MethodInfo testMethod) { var matcher = new Matcher(); - foreach (var include in Includes) { - matcher.AddInclude(include); + if (Includes != null) { + foreach (var include in Includes) { + matcher.AddInclude(include); + } } - foreach (var exclude in Excludes) { - matcher.AddExclude(exclude); + + if (Excludes != null) { + foreach(var exclude in Excludes) { + matcher.AddExclude(exclude); + } } - var basePath = GetBasePath(testMethod); + var basePath = Directory ?? Path.Combine("TestFiles", testMethod.DeclaringType!.Name, testMethod.Name); var result = matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo(basePath))); if (!result.HasMatches) { throw new ArgumentException("No matching files found: " + this); diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index 58f1ff62341..85f6522ff1f 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -11,7 +11,7 @@ namespace XUnitExtensions { public class FileTestCase : LongLivedMarshalByRefObject, IXunitTestCase { // This could use SkippableFactDiscoverer.GetSkippableExceptionNames(IAttributeInfo) // but it doesn't seem to be worth the complexity here yet. - private static readonly string[] skippingExceptionNames = { typeof(SkipException).FullName }; + private static readonly string[] skippingExceptionNames = { typeof(SkipException).FullName! }; protected XunitTestCase innerTestCase; @@ -20,7 +20,7 @@ public class FileTestCase : LongLivedMarshalByRefObject, IXunitTestCase { public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, IFileTheoryRowData data) { var collection = new TestCollection(testMethod.TestClass.TestCollection.TestAssembly, - (ITypeInfo)null, "Test collection for " + data.TestDisplayName); + testMethod.TestClass.Class, "Test collection for " + data.TestDisplayName); var testClassWithCollection = new TestClass(collection, testMethod.TestClass.Class); var testMethodWithCollection = new TestMethod(testClassWithCollection, testMethod.Method); diff --git a/Source/XUnitExtensions/FileTheoryDataRow.cs b/Source/XUnitExtensions/FileTheoryDataRow.cs index f9ca50c5469..67a71cf4b4c 100644 --- a/Source/XUnitExtensions/FileTheoryDataRow.cs +++ b/Source/XUnitExtensions/FileTheoryDataRow.cs @@ -4,13 +4,13 @@ namespace XUnitExtensions { public class FileTheoryDataRow : IXunitSerializable, IFileTheoryRowData { - private object?[] data; + private object[]? data; public FileTheoryDataRow() { } - public FileTheoryDataRow(params object?[] data) { + public FileTheoryDataRow(params object[] data) { this.data = data; } @@ -18,7 +18,7 @@ public FileTheoryDataRow(params object?[] data) { public string? Skip { get; set; } public string? TestDisplayName { get; set; } public Dictionary>? Traits { get; set; } - public object?[] GetData() => data; + public object?[] GetData() => data!; public void Serialize(IXunitSerializationInfo info) { info.AddValue(nameof(SourceInformation), SourceInformation); diff --git a/Source/XUnitExtensions/Lit/ILitCommand.cs b/Source/XUnitExtensions/Lit/ILitCommand.cs index 0b4de109ea9..ef76c2a3c4f 100644 --- a/Source/XUnitExtensions/Lit/ILitCommand.cs +++ b/Source/XUnitExtensions/Lit/ILitCommand.cs @@ -26,7 +26,7 @@ static ILitCommand() { if (index >= 0) { var arguments = line[(index + keyword.Length)..].Trim(); var parseMethod = type.GetMethod("Parse", new[] { typeof(string), typeof(LitTestConfiguration) }); - return (ILitCommand)parseMethod!.Invoke(null, new object[] { arguments, config }); + return (ILitCommand)parseMethod!.Invoke(null, new object[] { arguments, config })!; } } return null; diff --git a/Source/XUnitExtensions/Lit/LitTestCase.cs b/Source/XUnitExtensions/Lit/LitTestCase.cs index c85cdef8e7d..a60e2a04a18 100644 --- a/Source/XUnitExtensions/Lit/LitTestCase.cs +++ b/Source/XUnitExtensions/Lit/LitTestCase.cs @@ -14,9 +14,10 @@ public class LitTestCase { private readonly bool expectFailure; public static LitTestCase Read(string filePath, LitTestConfiguration config) { - var commands = File.ReadAllLines(filePath) + ILitCommand[] commands = File.ReadAllLines(filePath) .Select(line => ILitCommand.Parse(line, config)) .Where(c => c != null) + .Select(e => e!) .ToArray(); if (commands.Length == 0) { throw new ArgumentException($"No lit commands found in test file: {filePath}"); diff --git a/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs b/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs index 473608ef1ae..4faea54f286 100644 --- a/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs @@ -8,13 +8,11 @@ namespace XUnitExtensions.Lit { public class MainMethodLitCommand : ILitCommand { - public LitTestConfiguration Config { get; protected set; } public Assembly Assembly { get; protected set; } public string[] Arguments { get; protected set; } public static ILitCommand Parse(Assembly assembly, IEnumerable arguments, LitTestConfiguration config, bool invokeDirectly) { var result = new MainMethodLitCommand { - Config = config, Assembly = assembly, Arguments = arguments.ToArray() }; @@ -33,7 +31,7 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments Console.SetError(errorWriter); } - var exitCode = (int)Assembly.EntryPoint!.Invoke(null, new object[] { Arguments }); + var exitCode = (int)Assembly.EntryPoint!.Invoke(null, new object[] { Arguments })!; return (exitCode, "", ""); } From 3aaed5855d1b3b8a11b82640ba928feb93c24bbf Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 22:08:19 -0700 Subject: [PATCH 181/192] Formatting --- Source/XUnitExtensions/FileDataAttribute.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/XUnitExtensions/FileDataAttribute.cs b/Source/XUnitExtensions/FileDataAttribute.cs index 26cfd280b3e..17752e13ef5 100644 --- a/Source/XUnitExtensions/FileDataAttribute.cs +++ b/Source/XUnitExtensions/FileDataAttribute.cs @@ -24,7 +24,7 @@ public override IEnumerable GetData(MethodInfo testMethod) { } if (Excludes != null) { - foreach(var exclude in Excludes) { + foreach (var exclude in Excludes) { matcher.AddExclude(exclude); } } From 4f6bd1e0816e3be0e84286810c449a6c60e59bce Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Tue, 19 Oct 2021 22:38:37 -0700 Subject: [PATCH 182/192] Misc cleanup --- .github/workflows/msbuild.yml | 2 ++ Source/Dafny.sln | 8 ++++---- Source/Dafny/Dafny.atg | 2 +- Source/Dafny/DafnyOptions.cs | 1 - 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index abf0591c51a..fa33cbb0366 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -47,6 +47,8 @@ jobs: repository: boogie-org/boogie path: boogie ref: v${{ steps.regex-match.outputs.group1 }} + - name: Build Java runtime + run: make --quiet runtime - name: Build Dafny with local Boogie working-directory: dafny run: dotnet build Source/Dafny.sln diff --git a/Source/Dafny.sln b/Source/Dafny.sln index 05949878fbb..36bb4d93baa 100644 --- a/Source/Dafny.sln +++ b/Source/Dafny.sln @@ -64,10 +64,10 @@ Global {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Debug|Any CPU.Build.0 = Debug|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Any CPU.ActiveCfg = Release|Any CPU {FA2A3A73-3035-497B-B9D7-B6BC4888A058}.Release|Any CPU.Build.0 = Release|Any CPU - {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {896E7F24-FD59-4B34-A1BF-41978DFFE24E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {896E7F24-FD59-4B34-A1BF-41978DFFE24E}.Release|Any CPU.Build.0 = Release|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A4BC2C77-3A48-4917-AA22-41978DFFE24E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {896E7F24-FD59-4B34-A1BF-41978DFFE24E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {896E7F24-FD59-4B34-A1BF-41978DFFE24E}.Release|Any CPU.Build.0 = Release|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Debug|Any CPU.Build.0 = Debug|Any CPU {43731A57-DBA9-43AC-BC83-A9211DA7EB77}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/Source/Dafny/Dafny.atg b/Source/Dafny/Dafny.atg index 3d1dbdf7b77..48a9c994afe 100644 --- a/Source/Dafny/Dafny.atg +++ b/Source/Dafny/Dafny.atg @@ -878,7 +878,7 @@ Dafny var fileName = DafnyOptions.Clo.UseBaseNameForFileName ? Path.GetFileName(scanner.FullFilename) : Path.GetRelativePath(Directory.GetCurrentDirectory(), scanner.FullFilename); - errors.Warning(defaultModule.tok, "File contains no code: " + fileName); + errors.Warning(defaultModule.tok, "File contains no code: " + fileName); } DefaultClassDecl defaultClass = null; foreach (TopLevelDecl topleveldecl in defaultModule.TopLevelDecls) { diff --git a/Source/Dafny/DafnyOptions.cs b/Source/Dafny/DafnyOptions.cs index 7b2d9decc00..c7f5fa7751f 100644 --- a/Source/Dafny/DafnyOptions.cs +++ b/Source/Dafny/DafnyOptions.cs @@ -57,7 +57,6 @@ public static void Install(DafnyOptions options) { public enum PrintModes { Everything, DllEmbed, NoIncludes, NoGhost }; public PrintModes PrintMode = PrintModes.Everything; // Default to printing everything public bool DafnyVerify = true; - public bool VerifyVerbose = true; public string DafnyPrintResolvedFile = null; public List DafnyPrintExportedViews = new List(); public bool Compile = true; From e473dd5f025e154837267e797bf7ec7655fc8701 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 20 Oct 2021 11:51:23 -0700 Subject: [PATCH 183/192] Eliminate parsing reflection, partial work on building jar automatically --- Source/Dafny/DafnyPipeline.csproj | 1 - Source/DafnyRuntime/DafnyRuntime.csproj | 22 +++++++++++++++++++ .../DafnyRuntimeJava/build.gradle | 10 --------- Source/XUnitExtensions/Lit/ILitCommand.cs | 16 +++++--------- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/Source/Dafny/DafnyPipeline.csproj b/Source/Dafny/DafnyPipeline.csproj index 631a3d8b17d..0e383f93139 100644 --- a/Source/Dafny/DafnyPipeline.csproj +++ b/Source/Dafny/DafnyPipeline.csproj @@ -36,6 +36,5 @@ - diff --git a/Source/DafnyRuntime/DafnyRuntime.csproj b/Source/DafnyRuntime/DafnyRuntime.csproj index e9deb8edb4c..9f0d0cf5abd 100755 --- a/Source/DafnyRuntime/DafnyRuntime.csproj +++ b/Source/DafnyRuntime/DafnyRuntime.csproj @@ -23,6 +23,28 @@ + + + + + + + + + + + DafnyRuntimeJava/build/libs/DafnyRuntime.jar + + + + + + + + + + + diff --git a/Source/DafnyRuntime/DafnyRuntimeJava/build.gradle b/Source/DafnyRuntime/DafnyRuntimeJava/build.gradle index a516b1e9ffe..cc41949815b 100644 --- a/Source/DafnyRuntime/DafnyRuntimeJava/build.gradle +++ b/Source/DafnyRuntime/DafnyRuntimeJava/build.gradle @@ -26,16 +26,6 @@ test { useJUnitPlatform() } -task copyJarToBinaries { - dependsOn build - doLast { - copy { - from 'build/libs/DafnyRuntime.jar' - into '../../../Binaries' - } - } -} - clean { delete '../../../Binaries/DafnyRuntime.jar' } diff --git a/Source/XUnitExtensions/Lit/ILitCommand.cs b/Source/XUnitExtensions/Lit/ILitCommand.cs index ef76c2a3c4f..ac9aee86bc0 100644 --- a/Source/XUnitExtensions/Lit/ILitCommand.cs +++ b/Source/XUnitExtensions/Lit/ILitCommand.cs @@ -1,32 +1,28 @@ using System; -using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Reflection; using System.Text; using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.FileSystemGlobbing.Abstractions; -using Xunit; using Xunit.Abstractions; namespace XUnitExtensions.Lit { public interface ILitCommand { - private static readonly Dictionary CommandClasses = new(); + private static readonly Dictionary> CommandParsers = new(); static ILitCommand() { - CommandClasses.Add("RUN:", typeof(RunCommand)); - CommandClasses.Add("UNSUPPORTED:", typeof(UnsupportedCommand)); - CommandClasses.Add("XFAIL:", typeof(XFailCommand)); + CommandParsers.Add("RUN:", RunCommand.Parse); + CommandParsers.Add("UNSUPPORTED:", UnsupportedCommand.Parse); + CommandParsers.Add("XFAIL:", XFailCommand.Parse); } public static ILitCommand? Parse(string line, LitTestConfiguration config) { - foreach (var (keyword, type) in CommandClasses) { + foreach (var (keyword, parser) in CommandParsers) { var index = line.IndexOf(keyword); if (index >= 0) { var arguments = line[(index + keyword.Length)..].Trim(); - var parseMethod = type.GetMethod("Parse", new[] { typeof(string), typeof(LitTestConfiguration) }); - return (ILitCommand)parseMethod!.Invoke(null, new object[] { arguments, config })!; + return parser(arguments, config); } } return null; From f6e1a01155ee0a175b88fd42af3d02f046abac60 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 20 Oct 2021 15:54:55 -0700 Subject: [PATCH 184/192] Fix automatic build of DafnyRuntime.jar, remove unnecessary CI steps --- .github/workflows/msbuild.yml | 2 -- .github/workflows/refman.yml | 3 --- .github/workflows/xunit-tests.yml | 2 -- Source/DafnyRuntime/DafnyRuntime.csproj | 2 +- 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/msbuild.yml b/.github/workflows/msbuild.yml index fa33cbb0366..abf0591c51a 100644 --- a/.github/workflows/msbuild.yml +++ b/.github/workflows/msbuild.yml @@ -47,8 +47,6 @@ jobs: repository: boogie-org/boogie path: boogie ref: v${{ steps.regex-match.outputs.group1 }} - - name: Build Java runtime - run: make --quiet runtime - name: Build Dafny with local Boogie working-directory: dafny run: dotnet build Source/Dafny.sln diff --git a/.github/workflows/refman.yml b/.github/workflows/refman.yml index 4a68077096b..cc8b3bd593b 100644 --- a/.github/workflows/refman.yml +++ b/.github/workflows/refman.yml @@ -29,9 +29,6 @@ jobs: with: submodules: recursive path: dafny - - name: Build Java runtime - working-directory: dafny - run: make --quiet runtime - name: Build Dafny run: dotnet build dafny/Source/Dafny.sln - name: Install latex pandoc - Linux diff --git a/.github/workflows/xunit-tests.yml b/.github/workflows/xunit-tests.yml index 9fb3037b322..d7fd8dc3652 100644 --- a/.github/workflows/xunit-tests.yml +++ b/.github/workflows/xunit-tests.yml @@ -42,8 +42,6 @@ jobs: dotnet-version: 5.0.x - name: Install dependencies run: dotnet restore ${{env.solutionPath}} - - name: Build Java runtime - run: make --quiet runtime - name: Load Z3 shell: pwsh run: | diff --git a/Source/DafnyRuntime/DafnyRuntime.csproj b/Source/DafnyRuntime/DafnyRuntime.csproj index 9f0d0cf5abd..64f4fd0371c 100755 --- a/Source/DafnyRuntime/DafnyRuntime.csproj +++ b/Source/DafnyRuntime/DafnyRuntime.csproj @@ -23,7 +23,7 @@ - + From ebd302ba7eeca9ba3d9eaf06b873996d01c85be5 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 20 Oct 2021 16:16:02 -0700 Subject: [PATCH 185/192] Missed a call to build the Java runtime --- Scripts/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Scripts/package.py b/Scripts/package.py index 4d5e30ba037..00c820e81d6 100755 --- a/Scripts/package.py +++ b/Scripts/package.py @@ -123,7 +123,6 @@ def build(self): if path.exists(self.buildDirectory): shutil.rmtree(self.buildDirectory) run(["make", "--quiet", "clean"]) - run(["make", "--quiet", "runtime"]) run(["dotnet", "publish", path.join(SOURCE_DIRECTORY, "DafnyLanguageServer", "DafnyLanguageServer.csproj"), "--nologo", "-f", "net5.0", From b5da09807abbb3d7356114460b1047d7c35931ed Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 20 Oct 2021 17:14:59 -0700 Subject: [PATCH 186/192] Eliminate a few more nullability warnings --- .../Lit/MainMethodLitCommand.cs | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs b/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs index 4faea54f286..baf1bff8e32 100644 --- a/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs +++ b/Source/XUnitExtensions/Lit/MainMethodLitCommand.cs @@ -8,15 +8,16 @@ namespace XUnitExtensions.Lit { public class MainMethodLitCommand : ILitCommand { - public Assembly Assembly { get; protected set; } - public string[] Arguments { get; protected set; } + private readonly Assembly assembly; + private readonly string[] arguments; - public static ILitCommand Parse(Assembly assembly, IEnumerable arguments, LitTestConfiguration config, bool invokeDirectly) { - var result = new MainMethodLitCommand { - Assembly = assembly, - Arguments = arguments.ToArray() - }; + private MainMethodLitCommand(Assembly assembly, string[] arguments) { + this.assembly = assembly; + this.arguments = arguments; + } + public static ILitCommand Parse(Assembly assembly, IEnumerable arguments, LitTestConfiguration config, bool invokeDirectly) { + var result = new MainMethodLitCommand(assembly, arguments.ToArray()); return invokeDirectly ? result : result.ToShellCommand(config); } @@ -31,21 +32,21 @@ public static ILitCommand Parse(Assembly assembly, IEnumerable arguments Console.SetError(errorWriter); } - var exitCode = (int)Assembly.EntryPoint!.Invoke(null, new object[] { Arguments })!; + var exitCode = (int)assembly.EntryPoint!.Invoke(null, new object[] { arguments })!; return (exitCode, "", ""); } public ILitCommand ToShellCommand(LitTestConfiguration config) { - var shellArguments = new[] { Assembly.Location }.Concat(Arguments); + var shellArguments = new[] { assembly.Location }.Concat(arguments); return new ShellLitCommand(config, "dotnet", shellArguments, config.PassthroughEnvironmentVariables); } public override string ToString() { var builder = new StringBuilder(); - builder.Append(Assembly.EntryPoint); + builder.Append(assembly.EntryPoint); builder.Append(' '); - builder.AppendJoin(" ", Arguments); + builder.AppendJoin(" ", arguments); return builder.ToString(); } } From 9d4aef0aca69d77a0cea9aa5c229af1b34e09066 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 20 Oct 2021 20:48:01 -0700 Subject: [PATCH 187/192] Exclude Output files, more nullability fixes --- Source/IntegrationTests/LitTests.cs | 70 ++++++++++--------- .../Lit/LitTestConfiguration.cs | 30 +++++--- 2 files changed, 58 insertions(+), 42 deletions(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index c0ab7f26a86..619c1cf6e01 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -39,47 +39,53 @@ private static ILitCommand MainWithArguments(Assembly assembly, IEnumerable, LitTestConfiguration, ILitCommand>> { - { "%baredafny", (args, config) => - MainWithArguments(dafnyDriverAssembly, args, config, false) }, - { "%dafny", (args, config) => - MainWithArguments(dafnyDriverAssembly, defaultDafnyArguments.Concat(args), config, false) }, - { "%server", (args, config) => - MainWithArguments(dafnyServerAssembly, args, config, false) }, - }, - - Substitions = new Dictionary { + private static readonly LitTestConfiguration Config; + static LitTests() { + var substitutions = new Dictionary { { "%diff", "diff" }, { "%binaryDir", "." }, { "%z3", Path.Join("z3", "bin", "z3") }, { "%refmanexamples", Path.Join("TestFiles", "LitTests", "LitTest", "refman", "examples") } - }, - - PassthroughEnvironmentVariables = new[] { "PATH", "HOME" }, - }; + }; + + var commands = new Dictionary, LitTestConfiguration, ILitCommand>> { + { + "%baredafny", (args, config) => + MainWithArguments(dafnyDriverAssembly, args, config, false) + }, { + "%dafny", (args, config) => + MainWithArguments(dafnyDriverAssembly, defaultDafnyArguments.Concat(args), config, false) + }, { + "%server", (args, config) => + MainWithArguments(dafnyServerAssembly, args, config, false) + } + }; + + var passthroughEnvironmentVariables = new[] { "PATH", "HOME" }; + + string[] features; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { + features = new[] { "ubuntu", "posix" }; + } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { + features = new[] { "windows" }; + } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { + features = new[] { "macosx", "posix" }; + } else { + throw new Exception($"Unsupported OS: {RuntimeInformation.OSDescription}"); + } - static LitTests() { var dafnyReleaseDir = Environment.GetEnvironmentVariable("DAFNY_RELEASE"); if (dafnyReleaseDir != null) { - CONFIG.Commands["%baredafny"] = (args, config) => + commands["%baredafny"] = (args, config) => new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "dafny"), args, config.PassthroughEnvironmentVariables); - CONFIG.Commands["%dafny"] = (args, config) => + commands["%dafny"] = (args, config) => new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "dafny"), defaultDafnyArguments.Concat(args), config.PassthroughEnvironmentVariables); - CONFIG.Commands["%server"] = (args, config) => + commands["%server"] = (args, config) => new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "DafnyServer"), args, config.PassthroughEnvironmentVariables); - CONFIG.Substitions["%z3"] = Path.Join(dafnyReleaseDir, "z3", "bin", "z3"); - } - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { - CONFIG.Features = new[] { "ubuntu", "posix" }; - } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - CONFIG.Features = new[] { "windows" }; - } else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - CONFIG.Features = new[] { "macosx", "posix" }; - } else { - throw new Exception($"Unsupported OS: {RuntimeInformation.OSDescription}"); + substitutions["%z3"] = Path.Join(dafnyReleaseDir, "z3", "bin", "z3"); } + + Config = new LitTestConfiguration(substitutions, commands, features, passthroughEnvironmentVariables); } private readonly ITestOutputHelper output; @@ -90,9 +96,9 @@ public LitTests(ITestOutputHelper output) { [FileTheory] [FileData(Includes = new[] { "**/*.dfy", "**/*.transcript" }, - Excludes = new[] { "**/Inputs/**/*", "refman/examples/**/*" })] + Excludes = new[] { "**/Inputs/**/*", "**/Output/**/*", "refman/examples/**/*" })] public void LitTest(string path) { - LitTestCase.Run(path, CONFIG, output); + LitTestCase.Run(path, Config, output); } } } \ No newline at end of file diff --git a/Source/XUnitExtensions/Lit/LitTestConfiguration.cs b/Source/XUnitExtensions/Lit/LitTestConfiguration.cs index da8ac895691..46a6361d5a8 100644 --- a/Source/XUnitExtensions/Lit/LitTestConfiguration.cs +++ b/Source/XUnitExtensions/Lit/LitTestConfiguration.cs @@ -8,13 +8,23 @@ namespace XUnitExtensions.Lit { // TODO: Make safely immutable public class LitTestConfiguration { - public Dictionary Substitions { get; set; } + public readonly Dictionary Substitions; - public Dictionary, LitTestConfiguration, ILitCommand>> Commands { get; set; } + public readonly Dictionary, LitTestConfiguration, ILitCommand>> Commands; - public IEnumerable PassthroughEnvironmentVariables { get; set; } + public readonly string[] Features; - public string[] Features { get; set; } + public readonly IEnumerable PassthroughEnvironmentVariables; + + public LitTestConfiguration(Dictionary substitions, + Dictionary, LitTestConfiguration, ILitCommand>> commands, + string[] features, + IEnumerable passthroughEnvironmentVariables) { + Substitions = substitions; + Commands = commands; + Features = features; + PassthroughEnvironmentVariables = passthroughEnvironmentVariables; + } public string ApplySubstitutions(string s) { foreach (var (key, value) in Substitions) { @@ -24,12 +34,12 @@ public string ApplySubstitutions(string s) { } public LitTestConfiguration WithSubstitutions(Dictionary more) { - return new LitTestConfiguration { - Substitions = Substitions.Concat(more).ToDictionary(pair => pair.Key, pair => pair.Value), - Commands = Commands, - PassthroughEnvironmentVariables = PassthroughEnvironmentVariables, - Features = Features - }; + return new LitTestConfiguration( + Substitions.Concat(more).ToDictionary(pair => pair.Key, pair => pair.Value), + Commands, + Features, + PassthroughEnvironmentVariables + ); } } } \ No newline at end of file From c99017b0722fa73f3682ddc5607994f55f4790dd Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 20 Oct 2021 21:23:00 -0700 Subject: [PATCH 188/192] Fix remaining nullability warnings, flip TreatWarningsAsErrors --- .../IntegrationTests/IntegrationTests.csproj | 2 + Source/IntegrationTests/LitTests.cs | 14 +++---- Source/XUnitExtensions/FileTestCase.cs | 40 ++++++++++--------- Source/XUnitExtensions/XUnitExtensions.csproj | 1 + 4 files changed, 32 insertions(+), 25 deletions(-) diff --git a/Source/IntegrationTests/IntegrationTests.csproj b/Source/IntegrationTests/IntegrationTests.csproj index cbe809c73ca..79e6b080063 100644 --- a/Source/IntegrationTests/IntegrationTests.csproj +++ b/Source/IntegrationTests/IntegrationTests.csproj @@ -4,6 +4,8 @@ net5.0 true false + enable + true diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 619c1cf6e01..5796ff8657c 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -15,10 +15,10 @@ namespace IntegrationTests { public class LitTests { - private static readonly Assembly dafnyDriverAssembly = typeof(DafnyDriver).Assembly; - private static readonly Assembly dafnyServerAssembly = typeof(Server).Assembly; + private static readonly Assembly DafnyDriverAssembly = typeof(DafnyDriver).Assembly; + private static readonly Assembly DafnyServerAssembly = typeof(Server).Assembly; - private static readonly string[] defaultDafnyArguments = new[] { + private static readonly string[] DefaultDafnyArguments = new[] { "/countVerificationErrors:0", // We do not want absolute or relative paths in error messages, just the basename of the file @@ -51,13 +51,13 @@ static LitTests() { var commands = new Dictionary, LitTestConfiguration, ILitCommand>> { { "%baredafny", (args, config) => - MainWithArguments(dafnyDriverAssembly, args, config, false) + MainWithArguments(DafnyDriverAssembly, args, config, false) }, { "%dafny", (args, config) => - MainWithArguments(dafnyDriverAssembly, defaultDafnyArguments.Concat(args), config, false) + MainWithArguments(DafnyDriverAssembly, DefaultDafnyArguments.Concat(args), config, false) }, { "%server", (args, config) => - MainWithArguments(dafnyServerAssembly, args, config, false) + MainWithArguments(DafnyServerAssembly, args, config, false) } }; @@ -79,7 +79,7 @@ static LitTests() { commands["%baredafny"] = (args, config) => new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "dafny"), args, config.PassthroughEnvironmentVariables); commands["%dafny"] = (args, config) => - new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "dafny"), defaultDafnyArguments.Concat(args), config.PassthroughEnvironmentVariables); + new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "dafny"), DefaultDafnyArguments.Concat(args), config.PassthroughEnvironmentVariables); commands["%server"] = (args, config) => new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "DafnyServer"), args, config.PassthroughEnvironmentVariables); substitutions["%z3"] = Path.Join(dafnyReleaseDir, "z3", "bin", "z3"); diff --git a/Source/XUnitExtensions/FileTestCase.cs b/Source/XUnitExtensions/FileTestCase.cs index 85f6522ff1f..cf0d94cdc85 100644 --- a/Source/XUnitExtensions/FileTestCase.cs +++ b/Source/XUnitExtensions/FileTestCase.cs @@ -13,28 +13,32 @@ public class FileTestCase : LongLivedMarshalByRefObject, IXunitTestCase { // but it doesn't seem to be worth the complexity here yet. private static readonly string[] skippingExceptionNames = { typeof(SkipException).FullName! }; - protected XunitTestCase innerTestCase; + // Only nullable for the sake of the deserialization constructor + private XunitTestCase? innerTestCase; public string? DisplayName { get; protected set; } public string? SkipReason { get; protected set; } - public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, IFileTheoryRowData data) { + public FileTestCase(IMessageSink diagnosticMessageSink, ITestMethod testMethod, IFileTheoryRowData rowData) { var collection = new TestCollection(testMethod.TestClass.TestCollection.TestAssembly, - testMethod.TestClass.Class, "Test collection for " + data.TestDisplayName); + testMethod.TestClass.Class, "Test collection for " + rowData.TestDisplayName); var testClassWithCollection = new TestClass(collection, testMethod.TestClass.Class); var testMethodWithCollection = new TestMethod(testClassWithCollection, testMethod.Method); + // This unsafe cast is necessary because the signature of the constructor we're passing arguments to + // is wrong: the type should be object?[] because arbitrary test method arguments can absolutely be null! + object[] data = (object[])rowData.GetData(); innerTestCase = new SkippableFactTestCase(skippingExceptionNames, diagnosticMessageSink, TestMethodDisplay.Method, TestMethodDisplayOptions.All, - testMethodWithCollection, data.GetData()); - if (data.Traits != null) { - foreach (var (key, value) in data.Traits) { + testMethodWithCollection, data); + if (rowData.Traits != null) { + foreach (var (key, value) in rowData.Traits) { innerTestCase.Traits.Add(key, value); } } - innerTestCase.SourceInformation = data.SourceInformation; - DisplayName = data.TestDisplayName; - SkipReason = data.Skip; + innerTestCase.SourceInformation = rowData.SourceInformation; + DisplayName = rowData.TestDisplayName; + SkipReason = rowData.Skip; } [Obsolete("Called by the de-serializer", error: true)] @@ -53,17 +57,17 @@ public void Serialize(IXunitSerializationInfo info) { } public ISourceInformation SourceInformation { - get => innerTestCase.SourceInformation; - set => innerTestCase.SourceInformation = value; + get => innerTestCase!.SourceInformation; + set => innerTestCase!.SourceInformation = value; } - public ITestMethod TestMethod => innerTestCase.TestMethod; - public object[] TestMethodArguments => innerTestCase.TestMethodArguments; - public Dictionary> Traits => innerTestCase.Traits; - public string UniqueID => innerTestCase.UniqueID; - public Exception InitializationException => innerTestCase.InitializationException; - public IMethodInfo Method => innerTestCase.Method; - public int Timeout => innerTestCase.Timeout; + public ITestMethod TestMethod => innerTestCase!.TestMethod; + public object[] TestMethodArguments => innerTestCase!.TestMethodArguments; + public Dictionary> Traits => innerTestCase!.Traits; + public string UniqueID => innerTestCase!.UniqueID; + public Exception InitializationException => innerTestCase!.InitializationException; + public IMethodInfo Method => innerTestCase!.Method; + public int Timeout => innerTestCase!.Timeout; public async Task RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource) { diff --git a/Source/XUnitExtensions/XUnitExtensions.csproj b/Source/XUnitExtensions/XUnitExtensions.csproj index c499739c8ee..ca4daca3e94 100644 --- a/Source/XUnitExtensions/XUnitExtensions.csproj +++ b/Source/XUnitExtensions/XUnitExtensions.csproj @@ -4,6 +4,7 @@ net5.0 false enable + true From 65827fa4ed456e20ad1724e3ba409ed5453f7c98 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Wed, 20 Oct 2021 21:32:57 -0700 Subject: [PATCH 189/192] Formatting --- Source/IntegrationTests/LitTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/IntegrationTests/LitTests.cs b/Source/IntegrationTests/LitTests.cs index 5796ff8657c..50af64270bf 100644 --- a/Source/IntegrationTests/LitTests.cs +++ b/Source/IntegrationTests/LitTests.cs @@ -84,7 +84,7 @@ static LitTests() { new ShellLitCommand(config, Path.Join(dafnyReleaseDir, "DafnyServer"), args, config.PassthroughEnvironmentVariables); substitutions["%z3"] = Path.Join(dafnyReleaseDir, "z3", "bin", "z3"); } - + Config = new LitTestConfiguration(substitutions, commands, features, passthroughEnvironmentVariables); } From 25a65a917bcb973eeac2931175e6201ca9e7e211 Mon Sep 17 00:00:00 2001 From: Remy Willems <> Date: Thu, 21 Oct 2021 12:31:37 +0200 Subject: [PATCH 190/192] Remove treat warnings as errors for now --- Source/IntegrationTests/IntegrationTests.csproj | 1 - Source/XUnitExtensions/XUnitExtensions.csproj | 1 - 2 files changed, 2 deletions(-) diff --git a/Source/IntegrationTests/IntegrationTests.csproj b/Source/IntegrationTests/IntegrationTests.csproj index 79e6b080063..db1c9c3be9d 100644 --- a/Source/IntegrationTests/IntegrationTests.csproj +++ b/Source/IntegrationTests/IntegrationTests.csproj @@ -5,7 +5,6 @@ true false enable - true diff --git a/Source/XUnitExtensions/XUnitExtensions.csproj b/Source/XUnitExtensions/XUnitExtensions.csproj index ca4daca3e94..c499739c8ee 100644 --- a/Source/XUnitExtensions/XUnitExtensions.csproj +++ b/Source/XUnitExtensions/XUnitExtensions.csproj @@ -4,7 +4,6 @@ net5.0 false enable - true From 167d2c19433e7a3d62c559bc094d700d14c4dfe9 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 21 Oct 2021 07:52:35 -0700 Subject: [PATCH 191/192] chore: xUnit-based lit test runner Empty commit just to make the Semantic Pull Request app happy. From 702321277d15dc6e3964d045ba49961b87d23162 Mon Sep 17 00:00:00 2001 From: Robin Salkeld Date: Thu, 21 Oct 2021 09:39:24 -0700 Subject: [PATCH 192/192] Poke CI