Skip to content

Commit

Permalink
Don't run neg tests in groups (#2012)
Browse files Browse the repository at this point in the history
  • Loading branch information
@forki @KevinRansom
forki authored and KevinRansom committed Dec 15, 2016
1 parent 08cb87f commit 688dbcd
Show file tree
Hide file tree
Showing 5 changed files with 447 additions and 385 deletions.
1 change: 1 addition & 0 deletions tests/fsharp/FSharp.Tests.FSharpSuite.fsproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
<Link>NunitHelpers.fs</Link>
</Compile>
<Compile Include="single-test.fs" />
<Compile Include="TypeProviderTests.fs" />
<Compile Include="tests.fs" />
</ItemGroup>
<ItemGroup>
Expand Down
302 changes: 302 additions & 0 deletions tests/fsharp/TypeProviderTests.fs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,302 @@
// Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

#if INTERACTIVE
//#r @"../../release/net40/bin/FSharp.Compiler.dll"
#r @"../../packages/NUnit.3.5.0/lib/net45/nunit.framework.dll"
#load "../../src/scripts/scriptlib.fsx"
#load "test-framework.fs"
#load "single-test.fs"
#else
module FSharp.Test.FSharpSuite.TypeProviderTests
#endif

open System
open System.IO
open System.Reflection
open NUnit.Framework
open TestFramework
open Scripting
open SingleTest

#if FSHARP_SUITE_DRIVES_CORECLR_TESTS
// Use these lines if you want to test CoreCLR
let FSC_BASIC = FSC_CORECLR
let FSI_BASIC = FSI_CORECLR
#else
let FSC_BASIC = FSC_OPT_PLUS_DEBUG
let FSI_BASIC = FSI_FILE
#endif

[<Test>]
let diamondAssembly () =
let cfg = testConfig "typeProviders/diamondAssembly"

rm cfg "provider.dll"

fsc cfg "%s" "--out:provided.dll -a" [".." ++ "helloWorld" ++ "provided.fs"]

fsc cfg "%s" "--out:provider.dll -a" [".." ++ "helloWorld" ++ "provider.fsx"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test1.dll -a" cfg.fsc_flags ["test1.fsx"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test2a.dll -a -r:test1.dll" cfg.fsc_flags ["test2a.fsx"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test2b.dll -a -r:test1.dll" cfg.fsc_flags ["test2b.fsx"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test3.exe -r:test1.dll -r:test2a.dll -r:test2b.dll" cfg.fsc_flags ["test3.fsx"]

peverify cfg "test1.dll"

peverify cfg "test2a.dll"

peverify cfg "test2b.dll"

peverify cfg "test3.exe"

exec cfg ("." ++ "test3.exe") ""

use testOkFile = fileguard cfg "test.ok"

fsi cfg "%s" cfg.fsi_flags ["test3.fsx"]

testOkFile.CheckExists()

[<Test>]
let globalNamespace () =
let cfg = testConfig "typeProviders/globalNamespace"

csc cfg """/out:globalNamespaceTP.dll /debug+ /target:library /r:"%s" """ cfg.FSCOREDLLPATH ["globalNamespaceTP.cs"]

fsc cfg "%s /debug+ /r:globalNamespaceTP.dll /optimize-" cfg.fsc_flags ["test.fsx"]

let helloWorld p =
let cfg = testConfig "typeProviders/helloWorld"

fsc cfg "%s" "--out:provided1.dll -g -a" [".." ++ "helloWorld" ++ "provided.fs"]

fsc cfg "%s" "--out:provided2.dll -g -a" [".." ++ "helloWorld" ++ "provided.fs"]

fsc cfg "%s" "--out:provided3.dll -g -a" [".." ++ "helloWorld" ++ "provided.fs"]

fsc cfg "%s" "--out:provided4.dll -g -a" [".." ++ "helloWorld" ++ "provided.fs"]

fsc cfg "%s" "--out:providedJ.dll -g -a" [".." ++ "helloWorld" ++ "providedJ.fs"]

fsc cfg "%s" "--out:providedK.dll -g -a" [".." ++ "helloWorld" ++ "providedK.fs"]

fsc cfg "%s" "--out:providedNullAssemblyName.dll -g -a" [".." ++ "helloWorld" ++ "providedNullAssemblyName.fsx"]

fsc cfg "--out:provided.dll -a" [".." ++ "helloWorld" ++ "provided.fs"]

fsc cfg "--out:providedJ.dll -a" [".." ++ "helloWorld" ++ "providedJ.fs"]

fsc cfg "--out:providedK.dll -a" [".." ++ "helloWorld" ++ "providedK.fs"]

fsc cfg "--out:provider.dll -a" ["provider.fsx"]

SingleTest.singleTestBuildAndRunAux cfg p


rm cfg "provider_with_binary_compat_changes.dll"

mkdir cfg "bincompat1"

log "pushd bincompat1"
let bincompat1 = getfullpath cfg "bincompat1"

Directory.EnumerateFiles(bincompat1 ++ "..", "*.dll")
|> Seq.iter (fun from -> Commands.copy_y bincompat1 from ("." ++ Path.GetFileName(from)) |> ignore)

fscIn cfg bincompat1 "%s" "-g -a -o:test_lib.dll -r:provider.dll" [".." ++ "test.fsx"]

fscIn cfg bincompat1 "%s" "-r:test_lib.dll -r:provider.dll" [".." ++ "testlib_client.fsx"]

log "popd"

mkdir cfg "bincompat2"

log "pushd bincompat2"
let bincompat2 = getfullpath cfg "bincompat2"

Directory.EnumerateFiles(bincompat2 ++ ".." ++ "bincompat1", "*.dll")
|> Seq.iter (fun from -> Commands.copy_y bincompat2 from ("." ++ Path.GetFileName(from)) |> ignore)

fscIn cfg bincompat2 "%s" "--define:ADD_AN_OPTIONAL_STATIC_PARAMETER --define:USE_IMPLICIT_ITypeProvider2 --out:provider.dll -g -a" [".." ++ "provider.fsx"]

fscIn cfg bincompat2 "-g -a -o:test_lib_recompiled.dll -r:provider.dll" [".." ++ "test.fsx"]

fscIn cfg bincompat2 "%s" "--define:ADD_AN_OPTIONAL_STATIC_PARAMETER -r:test_lib.dll -r:provider.dll" [".." ++ "testlib_client.fsx"]

peverify cfg (bincompat2 ++ "provider.dll")

peverify cfg (bincompat2 ++ "test_lib.dll")

peverify cfg (bincompat2 ++ "test_lib_recompiled.dll")

peverify cfg (bincompat2 ++ "testlib_client.exe")

[<Test>]
let ``helloWorld fsc`` () = helloWorld FSC_BASIC

[<Test>]
let ``helloWorld fsi`` () = helloWorld FSI_STDIN


[<Test>]
let helloWorldCSharp () =
let cfg = testConfig "typeProviders/helloWorldCSharp"

rm cfg "magic.dll"

fsc cfg "%s" "--out:magic.dll -a --keyfile:magic.snk" ["magic.fs "]

rm cfg "provider.dll"

csc cfg """/out:provider.dll /target:library "/r:%s" /r:magic.dll""" cfg.FSCOREDLLPATH ["provider.cs"]

fsc cfg "%s /debug+ /r:provider.dll /optimize-" cfg.fsc_flags ["test.fsx"]

peverify cfg "magic.dll"

peverify cfg "provider.dll"

peverify cfg "test.exe"

exec cfg ("." ++ "test.exe") ""


[<TestCase("neg1")>]

This comment has been minimized.

Sign in to view

Copy link
@dsyme

dsyme Dec 28, 2016

Contributor

@forki Please don't use the TestCase attribute - jut define a new test for each case. This lets us run and develop the tests iteratively with F# Interactive (load-whole-fie, then run body of test). Basically each test should just be a function let test() = ...

thanks

This comment has been minimized.

Sign in to view

Copy link
@forki

forki Dec 28, 2016

Author Contributor

Ok will send another PR tomorrow

This comment has been minimized.

Sign in to view

Copy link
@dsyme

dsyme Dec 28, 2016

Contributor

(I'll adjust this in a future checkin)
[<TestCase("neg2")>]
[<TestCase("neg2c")>]
[<TestCase("neg2e")>]
[<TestCase("neg2g")>]
[<TestCase("neg2h")>]
[<TestCase("neg4")>]
[<TestCase("neg6")>]
[<TestCase("InvalidInvokerExpression")>]
[<TestCase("providerAttributeErrorConsume")>]
[<TestCase("ProviderAttribute_EmptyConsume")>]
[<TestCase("EVIL_PROVIDER_GetNestedNamespaces_Exception")>]
[<TestCase("EVIL_PROVIDER_NamespaceName_Exception")>]
[<TestCase("EVIL_PROVIDER_NamespaceName_Empty")>]
[<TestCase("EVIL_PROVIDER_GetTypes_Exception")>]
[<TestCase("EVIL_PROVIDER_ResolveTypeName_Exception")>]
[<TestCase("EVIL_PROVIDER_GetNamespaces_Exception")>]
[<TestCase("EVIL_PROVIDER_GetStaticParameters_Exception")>]
[<TestCase("EVIL_PROVIDER_GetInvokerExpression_Exception")>]
[<TestCase("EVIL_PROVIDER_GetTypes_Null")>]
[<TestCase("EVIL_PROVIDER_ResolveTypeName_Null")>]
[<TestCase("EVIL_PROVIDER_GetNamespaces_Null")>]
[<TestCase("EVIL_PROVIDER_GetStaticParameters_Null")>]
[<TestCase("EVIL_PROVIDER_GetInvokerExpression_Null")>]
[<TestCase("EVIL_PROVIDER_DoesNotHaveConstructor")>]
[<TestCase("EVIL_PROVIDER_ConstructorThrows")>]
[<TestCase("EVIL_PROVIDER_ReturnsTypeWithIncorrectNameFromApplyStaticArguments")>]
let ``negative type provider tests`` (name:string) =
let cfg = testConfig "typeProviders/negTests"
let dir = cfg.Directory

if requireENCulture () then

let fileExists = Commands.fileExists dir >> Option.isSome

rm cfg "provided.dll"

fsc cfg "--out:provided.dll -a" [".." ++ "helloWorld" ++ "provided.fs"]

rm cfg "providedJ.dll"

fsc cfg "--out:providedJ.dll -a" [".." ++ "helloWorld" ++ "providedJ.fs"]

rm cfg "providedK.dll"

fsc cfg "--out:providedK.dll -a" [".." ++ "helloWorld" ++ "providedK.fs"]

rm cfg "provider.dll"

fsc cfg "--out:provider.dll -a" ["provider.fsx"]

fsc cfg "--out:provider_providerAttributeErrorConsume.dll -a" ["providerAttributeError.fsx"]

fsc cfg "--out:provider_ProviderAttribute_EmptyConsume.dll -a" ["providerAttribute_Empty.fsx"]

rm cfg "helloWorldProvider.dll"

fsc cfg "--out:helloWorldProvider.dll -a" [".." ++ "helloWorld" ++ "provider.fsx"]

rm cfg "MostBasicProvider.dll"

fsc cfg "--out:MostBasicProvider.dll -a" ["MostBasicProvider.fsx"]

let preprocess name pref =
let dirp = (dir |> Commands.pathAddBackslash)
do
File.ReadAllText(sprintf "%s%s.%sbslpp" dirp name pref)
.Replace("<ASSEMBLY>", getfullpath cfg (sprintf "provider_%s.dll" name))
.Replace("<URIPATH>",sprintf "file:///%s" dirp)
|> fun txt -> File.WriteAllText(sprintf "%s%s.%sbsl" dirp name pref,txt)

if name = "ProviderAttribute_EmptyConsume" || name = "providerAttributeErrorConsume" then ()
else fsc cfg "--define:%s --out:provider_%s.dll -a" name name ["provider.fsx"]

if fileExists (sprintf "%s.bslpp" name) then preprocess name ""

if fileExists (sprintf "%s.vsbslpp" name) then preprocess name "vs"

SingleTest.singleNegTest cfg name

[<Test>]
let splitAssembly () =
let cfg = testConfig "typeProviders/splitAssembly"

fsc cfg "--out:provider.dll -a" ["provider.fs"]

fsc cfg "--out:providerDesigner.dll -a" ["providerDesigner.fsx"]

SingleTest.singleTestBuildAndRunAux cfg FSC_BASIC

[<Test>]
let wedgeAssembly () =
let cfg = testConfig "typeProviders/wedgeAssembly"

rm cfg "provider.dll"

rm cfg "provided.dll"

fsc cfg "%s" "--out:provided.dll -a" [".." ++ "helloWorld" ++ "provided.fs"]

rm cfg "providedJ.dll"

fsc cfg "%s" "--out:providedJ.dll -a" [".." ++ "helloWorld" ++ "providedJ.fs"]

rm cfg "providedK.dll"

fsc cfg "%s" "--out:providedK.dll -a" [".." ++ "helloWorld" ++ "providedK.fs"]

fsc cfg "%s" "--out:provider.dll -a" [".." ++ "helloWorld" ++ "provider.fsx"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test2a.dll -a" cfg.fsc_flags ["test2a.fs"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test2b.dll -a" cfg.fsc_flags ["test2b.fs"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test3.exe" cfg.fsc_flags ["test3.fsx"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test2a-with-sig.dll -a" cfg.fsc_flags ["test2a.fsi"; "test2a.fs"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test2b-with-sig.dll -a" cfg.fsc_flags ["test2b.fsi"; "test2b.fs"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test3-with-sig.exe --define:SIGS" cfg.fsc_flags ["test3.fsx"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test2a-with-sig-restricted.dll -a" cfg.fsc_flags ["test2a-restricted.fsi"; "test2a.fs"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test2b-with-sig-restricted.dll -a"cfg.fsc_flags ["test2b-restricted.fsi"; "test2b.fs"]

fsc cfg "%s --debug+ -r:provider.dll --optimize- -o:test3-with-sig-restricted.exe --define:SIGS_RESTRICTED" cfg.fsc_flags ["test3.fsx"]

peverify cfg "test2a.dll"

peverify cfg "test2b.dll"

peverify cfg "test3.exe"

exec cfg ("." ++ "test3.exe") ""
31 changes: 15 additions & 16 deletions tests/fsharp/single-test.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,25 +182,24 @@ let singleNegTest (cfg: TestConfig) testname =

fscAppendErrExpectFail cfg (sprintf "%s.err" testname) """%s --vserrors --warnaserror --nologo --maxerrors:10000 -a -o:%s.dll""" cfg.fsc_flags testname sources

let testnameDiff = fsdiff cfg (sprintf "%s.err" testname) (sprintf "%s.bsl" testname)

match testnameDiff with
| "" -> ()
| l ->
log "***** %s.err %s.bsl differed: a bug or baseline may need updating" testname testname
failwithf "%s.err %s.bsl differ; %A" testname testname l

log "Good, output %s.err matched %s.bsl" testname testname
let diff = fsdiff cfg (sprintf "%s.err" testname) (sprintf "%s.bsl" testname)

fscAppendErrExpectFail cfg (sprintf "%s.vserr" testname) "%s --test:ContinueAfterParseFailure --vserrors --warnaserror --nologo --maxerrors:10000 -a -o:%s.dll" cfg.fsc_flags testname sources

let testnameDiff = fsdiff cfg (sprintf "%s.vserr" testname) VSBSLFILE
let vbslDiff = fsdiff cfg (sprintf "%s.vserr" testname) VSBSLFILE

match testnameDiff with
| "" -> ()
| l ->
match diff,vbslDiff with
| "","" ->
log "Good, output %s.err matched %s.bsl" testname testname
log "Good, output %s.vserr matched %s" testname VSBSLFILE
| l,"" ->
log "***** %s.err %s.bsl differed: a bug or baseline may need updating" testname testname
failwithf "%s.err %s.bsl differ; %A" testname testname l
| "",l ->
log "Good, output %s.err matched %s.bsl" testname testname
log "***** %s.vserr %s differed: a bug or baseline may need updating" testname VSBSLFILE
failwithf "%s.vserr %s differ; %A" testname VSBSLFILE l

log "Good, output %s.vserr matched %s" testname VSBSLFILE

| l1,l2 ->
log "***** %s.err %s.bsl differed: a bug or baseline may need updating" testname testname
log "***** %s.vserr %s differed: a bug or baseline may need updating" testname VSBSLFILE
failwithf "%s.err %s.bsl differ; %A; %s.vserr %s differ; %A" testname testname l1 testname VSBSLFILE l2
2 changes: 1 addition & 1 deletion tests/fsharp/test-framework.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,7 @@ let fsdiff cfg a b =
let expectedFile = System.IO.Path.Combine(cfg.Directory, b)
let errorText = System.IO.File.ReadAllText (System.IO.Path.Combine(cfg.Directory, a))

let result = diff false actualFile expectedFile
let result = diff false expectedFile actualFile
if result <> "" then
log "%s" result
log "New error file:"
Expand Down
Loading

0 comments on commit 688dbcd

Please sign in to comment.