diff --git a/.gitignore b/.gitignore index e60f09fbfd7..4c2cb5c3dac 100644 --- a/.gitignore +++ b/.gitignore @@ -45,12 +45,18 @@ src/fsharp/FSharp.Compiler/ilpars.fsi src/fsharp/FSharp.Compiler/lex.fs src/fsharp/FSharp.Compiler/pars.fs src/fsharp/FSharp.Compiler/pars.fsi +src/fsharp/FSharp.Compiler/pplex.fs +src/fsharp/FSharp.Compiler/pppars.fs +src/fsharp/FSharp.Compiler/pppars.fsi src/fsharp/FSharp.Compiler-proto/illex.fs src/fsharp/FSharp.Compiler-proto/ilpars.fs src/fsharp/FSharp.Compiler-proto/ilpars.fsi src/fsharp/FSharp.Compiler-proto/lex.fs src/fsharp/FSharp.Compiler-proto/pars.fs src/fsharp/FSharp.Compiler-proto/pars.fsi +src/fsharp/FSharp.Compiler-proto/pplex.fs +src/fsharp/FSharp.Compiler-proto/pppars.fs +src/fsharp/FSharp.Compiler-proto/pppars.fsi *~ tests/projects/Sample_VS2012_FSharp_ConsoleApp_net45_with_resource/Sample_VS2012_FSharp_ConsoleApp_net45/Sample_VS2012_FSharp_ConsoleApp_net45.sln tests/projects/Sample_VS2012_FSharp_ConsoleApp_net45_with_resource/Sample_VS2012_FSharp_ConsoleApp_net45/Sample_VS2012_FSharp_ConsoleApp_net45.userprefs @@ -68,6 +74,9 @@ src/fsharp/FSharp.LanguageService.Compiler/illex.* src/fsharp/FSharp.LanguageService.Compiler/ilpars.* src/fsharp/FSharp.LanguageService.Compiler/lex.* src/fsharp/FSharp.LanguageService.Compiler/pars.* +src/fsharp/FSharp.LanguageService.Compiler/pplex.fs +src/fsharp/FSharp.LanguageService.Compiler/pppars.fs +src/fsharp/FSharp.LanguageService.Compiler/pppars.fsi tests/fsharp/typecheck/sigs/*.dll tests/fsharp/typecheck/sigs/*.exe vsintegration/src/unittests/Unittests.fsi diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 76b5050efe7..ff8ca11166a 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -45,6 +45,7 @@ Prior to a **Debug** test run, you need to complete **all** of these steps: msbuild src/fsharp-library-build.proj msbuild src/fsharp-compiler-build.proj msbuild src/fsharp-typeproviders-build.proj +msbuild src/fsharp-compiler-unittests-build.proj msbuild src/fsharp-library-build.proj /p:TargetFramework=net20 msbuild src/fsharp-library-build.proj /p:TargetFramework=portable47 msbuild src/fsharp-library-build.proj /p:TargetFramework=portable7 @@ -72,6 +73,7 @@ Prior to a **Release** test run, you need to do **all** of these: msbuild src/fsharp-library-build.proj /p:Configuration=Release msbuild src/fsharp-compiler-build.proj /p:Configuration=Release msbuild src/fsharp-typeproviders-build.proj /p:Configuration=Release +msbuild src/fsharp-compiler-unittests-build.proj /p:Configuration=Release msbuild src/fsharp-library-build.proj /p:TargetFramework=net20 /p:Configuration=Release msbuild src/fsharp-library-build.proj /p:TargetFramework=portable47 /p:Configuration=Release msbuild src/fsharp-library-build.proj /p:TargetFramework=portable7 /p:Configuration=Release diff --git a/TESTGUIDE.md b/TESTGUIDE.md index e91bd009389..844c4704894 100644 --- a/TESTGUIDE.md +++ b/TESTGUIDE.md @@ -11,8 +11,8 @@ The test cases for this suite reside under `tests\fsharp`. This suite dates back The test cases for this suite reside under `tests\fsharpqa\source`. This suite was first created when F# 2.0 was being added to Visual Studio 2010. Tests for this suite are driven by the "RunAll" framework, implemented in Perl. This suite is rather large and has broad and deep coverage of a variety of compiler, runtime, and syntax scenarios. -### Compiler and Library Core Unit Test Suite -The test cases for this suite reside next to the F# core library code, at `src\fsharp\FSharp.Core.Unittests`. This suite is a set of standard NUnit test cases, implemented in F#. This suite focuses on validation of the core F# types and the public surface area of `FSharp.Core.dll`. +### Compiler and Library Core Unit Test Suites +The test cases for these suites reside next to the F# core library code, at `src\fsharp\FSharp.Core.Unittests` and `src\fsharp\FSharp.Compiler.Unittests`. These suites are standard NUnit test cases, implemented in F#. The FSharp.Core.Unittests suite focuses on validation of the core F# types and the public surface area of `FSharp.Core.dll`, and the FSharp.Compiler.Unittests suite focuses on validation of compiler internals. ### Visual F# Tools IDE Unit Test Suite The test cases for this suite reside next to the Visual F# Tools code, at `vsintegration\src\unittests`. This suite is a set of standard NUnit test cases, implemented in F#. This suite exercises a wide range of behaviors in the F# Visual Studio project system and language service. @@ -39,6 +39,7 @@ The script `tests\RunTests.cmd` has been provided to make execution of the above ``` RunTests.cmd fsharp [tags to run] [tags not to run] RunTests.cmd fsharpqa [tags to run] [tags not to run] +RunTests.cmd compilerunit RunTests.cmd coreunit RunTests.cmd coreunitportable47 RunTests.cmd coreunitportable7 @@ -49,7 +50,7 @@ RunTests.cmd ideunit `RunTests.cmd` sets a handful of environment variables which allow for the tests to work, then puts together and executes the appropriate command line to start the specified test suite. -All test execution logs and result files will be dropped into the `tests\TestResults` folder, and have file names matching `FSharp_*.*`, `FSharpQA_*.*`, `CoreUnit_*.*`, `IDEUnit_*.*`, e.g. `FSharpQA_Results.log` or `FSharp_Failures.log`. +All test execution logs and result files will be dropped into the `tests\TestResults` folder, and have file names matching `FSharp_*.*`, `FSharpQA_*.*`, `CompilerUnit_*.*`, `CoreUnit_*.*`, `IDEUnit_*.*`, e.g. `FSharpQA_Results.log` or `FSharp_Failures.log`. For the FSharp and FSharpQA suites, the list of test areas and their associated "tags" is stored at @@ -94,9 +95,14 @@ Test area directories in this suite will contain a number of source code files a Test cases will run an optional "pre command," compile some set of source files using some set of flags, optionally run the resulting binary, then optionally run a final "post command." If all of these steps complete without issue, the test is considered to have passed. -### Core Unit Test Suite +### FSharp.Compiler and FSharp.Core Unit Test Suites -To build the unit test binary, call `msbuild fsharp-library-unittests-build.proj` from the `src` directory. Tests are contained in the binary `FSharp.Core.Unittests.dll`. +To build these unit test binaries, from the `src` directory call + +- `msbuild fsharp-compiler-unittests-build.proj` + - Output binary is `FSharp.Compiler.Unittests.dll` +- `msbuild fsharp-library-unittests-build.proj` + - Output binary is `FSharp.Core.Unittests.dll` You can execute and re-run these tests using any standard NUnit approach - via graphical `nunit.exe` or on the command line via `nunit-console.exe`. diff --git a/src/assemblyinfo/assemblyinfo.FSharp.Compiler.dll.fs b/src/assemblyinfo/assemblyinfo.FSharp.Compiler.dll.fs index 957bb6ca5fa..990a59b48d7 100644 --- a/src/assemblyinfo/assemblyinfo.FSharp.Compiler.dll.fs +++ b/src/assemblyinfo/assemblyinfo.FSharp.Compiler.dll.fs @@ -22,6 +22,7 @@ open System.Reflection // Note: internals visible to unit test DLLs in Retail (and all) builds. [] +[] [] [] [] @@ -45,6 +46,7 @@ open System.Reflection [] [] [] +[] #endif #if STRONG_NAME_FSHARP_COMPILER_WITH_TEST_KEY @@ -62,6 +64,7 @@ open System.Reflection [] [] [] +[] #endif diff --git a/src/fsharp-compiler-unittests-build.proj b/src/fsharp-compiler-unittests-build.proj new file mode 100644 index 00000000000..200c883d9a0 --- /dev/null +++ b/src/fsharp-compiler-unittests-build.proj @@ -0,0 +1,18 @@ + + + + + net40 + + + + + + + + + + + + diff --git a/src/fsharp.sln b/src/fsharp.sln index 5c92f68583f..0e9eb059b59 100644 --- a/src/fsharp.sln +++ b/src/fsharp.sln @@ -31,6 +31,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Data.TypeProviders", EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FsiAnyCPU", "fsharp\fsiAnyCpu\FsiAnyCPU.fsproj", "{8B3E283D-B5FE-4055-9D80-7E3A32F3967B}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Compiler.Unittests", "fsharp\FSharp.Compiler.Unittests\FSharp.Compiler.Unittests.fsproj", "{A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -75,6 +77,10 @@ Global {CB7D20C4-6506-406D-9144-5342C3595F03}.Release|Any CPU.Build.0 = Release|Any CPU {8B3E283D-B5FE-4055-9D80-7E3A32F3967B}.Debug|Any CPU.ActiveCfg = Debug|x86 {8B3E283D-B5FE-4055-9D80-7E3A32F3967B}.Release|Any CPU.ActiveCfg = Release|x86 + {A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A8D9641A-9170-4CF4-8FE0-6DB8C134E1B5}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/fsharp/FSComp.txt b/src/fsharp/FSComp.txt index 7999bcaafc2..71c16082eef 100644 --- a/src/fsharp/FSComp.txt +++ b/src/fsharp/FSComp.txt @@ -1064,7 +1064,7 @@ lexHashEndingNoMatchingIf,"#endif has no matching #if" 1169,lexHashIfMustHaveIdent,"#if directive should be immediately followed by an identifier" 1170,lexWrongNestedHashEndif,"Syntax error. Wrong nested #endif, unexpected tokens before it." lexHashBangMustBeFirstInFile,"#! may only appear as the first line at the start of a file." -1171,lexExpectedSingleLineComment,"Expected single line comment or end of line" +1171,pplexExpectedSingleLineComment,"Expected single line comment or end of line" 1172,memberOperatorDefinitionWithNoArguments,"Infix operator member '%s' has no arguments. Expected a tuple of 2 arguments, e.g. static member (+) (x,y) = ..." 1173,memberOperatorDefinitionWithNonPairArgument,"Infix operator member '%s' has %d initial argument(s). Expected a tuple of 2 arguments, e.g. static member (+) (x,y) = ..." 1174,memberOperatorDefinitionWithCurriedArguments,"Infix operator member '%s' has extra curried arguments. Expected a tuple of 2 arguments, e.g. static member (+) (x,y) = ..." @@ -1336,3 +1336,7 @@ descriptionUnavailable,"(description unavailable...)" 3180,abImplicitHeapAllocation,"The mutable local '%s' is implicitly allocated as a reference cell because it has been captured by a closure. This warning is for informational purposes only to indicate where implicit allocations are performed." estApplyStaticArgumentsForMethodNotImplemented,"A type provider implemented GetStaticParametersForMethod, but ApplyStaticArgumentsForMethod was not implemented or invalid" 3181,etErrorApplyingStaticArgumentsToMethod,"An error occured applying the static arguments to a provided method" +3182,pplexUnexpectedChar,"Unexpected character '%s' in preprocessor expression" +3183,ppparsUnexpectedToken,"Unexpected token '%s' in preprocessor expression" +3184,ppparsIncompleteExpression,"Incomplete preprocessor expression" +3185,ppparsMissingToken,"Missing token '%s' in preprocessor expression" diff --git a/src/fsharp/FSharp.Compiler-proto/FSharp.Compiler-proto.fsproj b/src/fsharp/FSharp.Compiler-proto/FSharp.Compiler-proto.fsproj index 4f2d5186657..747e4aaa64b 100644 --- a/src/fsharp/FSharp.Compiler-proto/FSharp.Compiler-proto.fsproj +++ b/src/fsharp/FSharp.Compiler-proto/FSharp.Compiler-proto.fsproj @@ -22,6 +22,16 @@ FSComp.txt + + --lexlib Internal.Utilities.Text.Lexing + pplex.fsl + + + Microsoft.FSharp.Compiler.PPParser + Microsoft.FSharp.Compiler + --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing + pppars.fsy + --lexlib Internal.Utilities.Text.Lexing lex.fsl @@ -250,6 +260,7 @@ ast.fs + lexhelp.fsi @@ -257,6 +268,7 @@ lexhelp.fs + sreflect.fsi diff --git a/src/fsharp/FSharp.Compiler.Unittests/FSharp.Compiler.Unittests.fsproj b/src/fsharp/FSharp.Compiler.Unittests/FSharp.Compiler.Unittests.fsproj new file mode 100644 index 00000000000..f48ce0fcbf3 --- /dev/null +++ b/src/fsharp/FSharp.Compiler.Unittests/FSharp.Compiler.Unittests.fsproj @@ -0,0 +1,82 @@ + + + + + ..\.. + {a8d9641a-9170-4cf4-8fe0-6db8c134e1b5} + + + + Debug + AnyCPU + 2.0 + true + true + Library + FSharp.Compiler.Unittests + v3.5 + SystematicUnitTests + + false + false + netcore + + + {CandidateAssemblyFiles}; + {TargetFrameworkDirectory}; + {Registry:Software\Microsoft\.NETFramework,v4.5,AssemblyFoldersEx}; + + + + $(DefineConstants);SILVERLIGHT + $(DefineConstants);EXTENSIONTYPING + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 3 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 3 + + + + + true + + + + + + + + + + + + + + + + + + FSharp.Compiler + {2e4d67b4-522d-4cf7-97e4-ba940f0b18f3} + + + FSharp.Core + {ded3bbd7-53f4-428a-8c9f-27968e768605} + True + + + + \ No newline at end of file diff --git a/src/fsharp/FSharp.Compiler.Unittests/HashIfExpression.fs b/src/fsharp/FSharp.Compiler.Unittests/HashIfExpression.fs new file mode 100644 index 00000000000..cf70addb330 --- /dev/null +++ b/src/fsharp/FSharp.Compiler.Unittests/HashIfExpression.fs @@ -0,0 +1,268 @@ +// Copyright (c) Microsoft Open Technologies, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace FSharp.Compiler.Unittests + +open System +open System.Text + +open NUnit.Framework + +open Internal.Utilities.Text.Lexing +open Microsoft.FSharp.Compiler +open Microsoft.FSharp.Compiler.Lexer +open Microsoft.FSharp.Compiler.Lexhelp +open Microsoft.FSharp.Compiler.ErrorLogger +open Microsoft.FSharp.Compiler.ErrorLogger +open Microsoft.FSharp.Compiler.Ast + +[] +type HashIfExpression() = + + let preludes = [|"#if "; "#elif "|] + let epilogues = [|""; " // Testing"|] + + let ONE = IfdefId "ONE" + let TWO = IfdefId "TWO" + let THREE = IfdefId "THREE" + + let isSet l r = (l &&& r) <> 0 + + let (!!) e = IfdefNot(e) + let (&&&) l r = IfdefAnd(l,r) + let (|||) l r = IfdefOr(l,r) + + let mutable tearDown = fun () -> () + + let exprAsString (e : LexerIfdefExpression) : string = + let sb = StringBuilder() + let append (s : string) = ignore <| sb.Append s + let rec build (e : LexerIfdefExpression) : unit = + match e with + | IfdefAnd (l,r)-> append "("; build l; append " && "; build r; append ")" + | IfdefOr (l,r) -> append "("; build l; append " || "; build r; append ")" + | IfdefNot ee -> append "!"; build ee + | IfdefId nm -> append nm + + build e + + sb.ToString () + + let createParser () = + let errors = ResizeArray() + let warnings = ResizeArray() + + let errorLogger = + { + new ErrorLogger("TestErrorLogger") with + member x.WarnSinkImpl(e) = warnings.Add e + member x.ErrorSinkImpl(e) = errors.Add e + member x.ErrorCount = errors.Count + } + + let stack : LexerIfdefStack = ref [] + let lightSyntax = LightSyntaxStatus(true, false) + let resourceManager = LexResourceManager () + let defines = [] + let startPos = Position.Empty + let args = mkLexargs ("dummy", defines, lightSyntax, resourceManager, stack, errorLogger) + + CompileThreadStatic.ErrorLogger <- errorLogger + + let parser (s : string) = + let lexbuf = LexBuffer.FromChars (s.ToCharArray ()) + lexbuf.StartPos <- startPos + lexbuf.EndPos <- startPos + let tokenStream = PPLexer.tokenstream args + + PPParser.start tokenStream lexbuf + + errors, warnings, parser + + [] + member this.Setup() = + let el = CompileThreadStatic.ErrorLogger + tearDown <- + fun () -> + CompileThreadStatic.BuildPhase <- BuildPhase.DefaultPhase + CompileThreadStatic.ErrorLogger <- el + + CompileThreadStatic.BuildPhase <- BuildPhase.Compile + + [] + member this.TearDown() = + tearDown () + + [] + member this.PositiveParserTestCases()= + + let errors, warnings, parser = createParser () + + let positiveTestCases = + [| + "ONE" , ONE + "ONE//" , ONE + "ONE // Comment" , ONE + "!ONE" , !!ONE + "!!ONE" , !! (!!ONE) + "DEBUG" , (IfdefId "DEBUG") + "!DEBUG" , !! (IfdefId "DEBUG") + "O_s1" , IfdefId "O_s1" + "(ONE)" , (ONE) + "ONE&&TWO" , ONE &&& TWO + "ONE||TWO" , ONE ||| TWO + "( ONE && TWO )" , ONE &&& TWO + "ONE && TWO && THREE" , (ONE &&& TWO) &&& THREE + "ONE || TWO || THREE" , (ONE ||| TWO) ||| THREE + "ONE || TWO && THREE" , ONE ||| (TWO &&& THREE) + "ONE && TWO || THREE" , (ONE &&& TWO) ||| THREE + "ONE || (TWO && THREE)" , ONE ||| (TWO &&& THREE) + "ONE && (TWO || THREE)" , ONE &&& (TWO ||| THREE) + "!ONE || TWO && THREE" , (!!ONE) ||| (TWO &&& THREE) + "ONE && !TWO || THREE" , (ONE &&& (!!TWO)) ||| THREE + "ONE || !(TWO && THREE)" , ONE ||| (!!(TWO &&& THREE)) + "true" , IfdefId "true" + "false" , IfdefId "false" + |] + + let failures = ResizeArray () + let fail = failures.Add + + for test,expected in positiveTestCases do + for prelude in preludes do + let test = prelude + test + for epilogue in epilogues do + let test = test + epilogue + try + let expr = parser test + + if expected <> expr then + fail <| sprintf "'%s', expected %A, actual %A" test (exprAsString expected) (exprAsString expr) + with + | e -> fail <| sprintf "'%s', expected %A, actual %s,%A" test (exprAsString expected) (e.GetType().Name) e.Message + + + let fs = + failures + |> Seq.append (errors |> Seq.map (fun pe -> pe.DebugDisplay ())) + |> Seq.append (warnings |> Seq.map (fun pe -> pe.DebugDisplay ())) + |> Seq.toArray + + let failure = String.Join ("\n", fs) + + Assert.AreEqual("", failure) + + () + + [] + member this.NegativeParserTestCases()= + + let errors, warnings, parser = createParser () + + let negativeTests = + [| + "" + "!" + "&&" + "||" + "@" + "ONE ONE" + "ONE@" + "@ONE" + "$" + "ONE$" + "$ONE" + "ONE!" + "(ONE" + "ONE)" + // TODO: Investigate why this raises a parse failure + // "(ONE ||)" + "ONE&&" + "ONE ||" + "&& ONE" + "||ONE" + "ONE TWO" + "ONE(* Comment" + "ONE(* Comment *)" + "ONE(**)" + "ONE (* Comment" + "ONE (* Comment *)" + "ONE (**)" + "ONE )(@$&%*@^#%#!$)" + |] + + let failures = ResizeArray () + let fail = failures.Add + + for test in negativeTests do + for prelude in preludes do + let test = prelude + test + for epilogue in epilogues do + let test = test + epilogue + try + let bec = errors.Count + let expr = parser test + let aec = errors.Count + + if bec = aec then // No new errors discovered + fail <| sprintf "'%s', expected 'parse error', actual %A" test (exprAsString expr) + with + | e -> fail <| sprintf "'%s', expected 'parse error', actual %s,%A" test (e.GetType().Name) e.Message + + let fs = failures |> Seq.toArray + + let fails = String.Join ("\n", fs) + + Assert.AreEqual("", fails) + + [] + member this.LexerIfdefEvalTestCases()= + + let failures = ResizeArray () + let fail = failures.Add + + for i in 0..7 do + let one = isSet i 1 + let two = isSet i 2 + let three = isSet i 4 + + let lookup s = + match s with + | "ONE" -> one + | "TWO" -> two + | "THREE" -> three + | _ -> false + + let testCases = + [| + ONE , one + !!ONE , not one + !! (!!ONE) , not (not one) + TWO , two + !!TWO , not two + !! (!!TWO) , not (not two) + ONE &&& TWO , one && two + ONE ||| TWO , one || two + (ONE &&& TWO) &&& THREE , (one && two) && three + (ONE ||| TWO) ||| THREE , (one || two) || three + ONE ||| (TWO &&& THREE) , one || (two && three) + (ONE &&& TWO) ||| THREE , (one && two) || three + ONE ||| (TWO &&& THREE) , one || (two && three) + ONE &&& (TWO ||| THREE) , one && (two || three) + (!!ONE) ||| (TWO &&& THREE) , (not one) || (two && three) + (ONE &&& (!!TWO)) ||| THREE , (one && (not two)) || three + ONE ||| (!!(TWO &&& THREE)) , one || (not (two && three)) + |] + + let eval = LexerIfdefEval lookup + for expr, expected in testCases do + let actual = eval expr + + if actual <> expected then + fail <| sprintf "For ONE=%A, TWO=%A, THREE=%A the expression %A is expected to be %A but was %A" one two three (exprAsString expr) expected actual + + + let fs = failures |> Seq.toArray + + let fails = String.Join ("\n", fs) + + Assert.AreEqual("", fails) diff --git a/src/fsharp/FSharp.Compiler/FSharp.Compiler.fsproj b/src/fsharp/FSharp.Compiler/FSharp.Compiler.fsproj index 41f5c106d69..b2e381c0022 100644 --- a/src/fsharp/FSharp.Compiler/FSharp.Compiler.fsproj +++ b/src/fsharp/FSharp.Compiler/FSharp.Compiler.fsproj @@ -247,6 +247,16 @@ ILXErase\cu_erase.fs + + --lexlib Internal.Utilities.Text.Lexing + ParserAndUntypedAST\pplex.fsl + + + Microsoft.FSharp.Compiler.PPParser + Microsoft.FSharp.Compiler + --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing + ParserAndUntypedAST\pppars.fsy + --lexlib Internal.Utilities.Text.Lexing ParserAndUntypedAST\lex.fsl @@ -272,6 +282,9 @@ ParserAndUntypedAST\ast.fs + + ParserAndUntypedAST\pppars.fs + ParserAndUntypedAST\pars.fs @@ -281,6 +294,9 @@ ParserAndUntypedAST\lexhelp.fs + + ParserAndUntypedAST\pplex.fs + ParserAndUntypedAST\lex.fs diff --git a/src/fsharp/FSharp.Compiler/InternalsVisibleTo.fs b/src/fsharp/FSharp.Compiler/InternalsVisibleTo.fs index b77e12a0a12..f0d5e127e65 100644 --- a/src/fsharp/FSharp.Compiler/InternalsVisibleTo.fs +++ b/src/fsharp/FSharp.Compiler/InternalsVisibleTo.fs @@ -18,6 +18,7 @@ open System.Reflection [] [] [] +[] do() diff --git a/src/fsharp/FSharp.LanguageService.Compiler/FSharp.LanguageService.Compiler.fsproj b/src/fsharp/FSharp.LanguageService.Compiler/FSharp.LanguageService.Compiler.fsproj index 3309752329d..8bc00ff3d9f 100644 --- a/src/fsharp/FSharp.LanguageService.Compiler/FSharp.LanguageService.Compiler.fsproj +++ b/src/fsharp/FSharp.LanguageService.Compiler/FSharp.LanguageService.Compiler.fsproj @@ -36,6 +36,16 @@ assemblyinfo.FSharp.Compiler.dll.fs + + --lexlib Internal.Utilities.Text.Lexing + pplex.fsl + + + Microsoft.FSharp.Compiler.PPParser + Microsoft.FSharp.Compiler + --internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing + pppars.fsy + --lexlib Internal.Utilities.Text.Lexing lex.fsl @@ -267,6 +277,8 @@ lexhelp.fs + + sreflect.fsi @@ -459,7 +471,7 @@ - + diff --git a/src/fsharp/ast.fs b/src/fsharp/ast.fs index 4222e3b6698..3af95cf757c 100644 --- a/src/fsharp/ast.fs +++ b/src/fsharp/ast.fs @@ -2016,7 +2016,19 @@ type LexerEndlineContinuation = match x with | LexerEndlineContinuation.Token(ifd) | LexerEndlineContinuation.Skip(ifd, _, _) -> ifd - + +type LexerIfdefExpression = + | IfdefAnd of LexerIfdefExpression*LexerIfdefExpression + | IfdefOr of LexerIfdefExpression*LexerIfdefExpression + | IfdefNot of LexerIfdefExpression + | IfdefId of string + +let rec LexerIfdefEval (lookup : string -> bool) = function + | IfdefAnd (l,r) -> (LexerIfdefEval lookup l) && (LexerIfdefEval lookup r) + | IfdefOr (l,r) -> (LexerIfdefEval lookup l) || (LexerIfdefEval lookup r) + | IfdefNot e -> not (LexerIfdefEval lookup e) + | IfdefId id -> lookup id + /// The parser defines a number of tokens for whitespace and /// comments eliminated by the lexer. These carry a specification of /// a continuation for the lexer for continued processing after we've dealt with diff --git a/src/fsharp/lex.fsl b/src/fsharp/lex.fsl index 36234663702..2c37ee00d93 100644 --- a/src/fsharp/lex.fsl +++ b/src/fsharp/lex.fsl @@ -149,15 +149,17 @@ let shouldStartFile args lexbuf (m:range) err tok = if (m.StartColumn <> 0 || m.StartLine <> 1) then fail args lexbuf err tok else tok -let extractIdentFromHashIf (lexed:string) = - // Skip the '#if' token, then trim whitespace, then find the end of the identifier - let lexed = lexed.Trim() - let trimIf = lexed.Substring(3).Trim() - let identEnd = trimIf.IndexOfAny([| ' '; '\t'; '/' |]) - let identEnd = (if identEnd = -1 then trimIf.Length else identEnd) - trimIf.Substring(0, identEnd) -} +let evalIfDefExpression startPos args (lookup:string->bool) (lexed:string) = + let lexbuf = LexBuffer.FromChars (lexed.ToCharArray ()) + lexbuf.StartPos <- startPos + lexbuf.EndPos <- startPos + let tokenStream = Microsoft.FSharp.Compiler.PPLexer.tokenstream args + let expr = Microsoft.FSharp.Compiler.PPParser.start tokenStream lexbuf + + LexerIfdefEval lookup expr + +} let letter = '\Lu' | '\Ll' | '\Lt' | '\Lm' | '\Lo' | '\Nl' let surrogateChar = '\Cs' let digit = '\Nd' @@ -165,6 +167,8 @@ let hex = ['0'-'9'] | ['A'-'F'] | ['a'-'f'] let truewhite = [' '] let offwhite = ['\t'] let anywhite = truewhite | offwhite +let anychar = [^'\n''\r'] +let anystring = anychar* let op_char = '!'|'$'|'%'|'&'|'*'|'+'|'-'|'.'|'/'|'<'|'='|'>'|'?'|'@'|'^'|'|'|'~'|':' let ignored_op_char = '.' | '$' | '?' let xinteger = @@ -597,15 +601,16 @@ rule token args skip = parse mlCompatWarning (FSComp.SR.lexIndentOffForML()) lexbuf.LexemeRange; if not skip then (HASH_LIGHT (LexCont.Token !args.ifdefStack)) else token args skip lexbuf } - | anywhite* "#if" anywhite+ ident anywhite* ("//" [^'\n''\r']*)? + | anywhite* "#if" anywhite+ anystring { let m = lexbuf.LexemeRange + let lookup id = List.mem id args.defines let lexed = lexeme lexbuf - let id = extractIdentFromHashIf lexed + let isTrue = evalIfDefExpression lexbuf.StartPos args lookup lexed args.ifdefStack := (IfDefIf,m) :: !(args.ifdefStack); // Get the token; make sure it starts at zero position & return let cont, f = - ( if List.mem id args.defines then (LexCont.EndLine(LexerEndlineContinuation.Token(!args.ifdefStack)), endline (LexerEndlineContinuation.Token !args.ifdefStack) args skip) + ( if isTrue then (LexCont.EndLine(LexerEndlineContinuation.Token(!args.ifdefStack)), endline (LexerEndlineContinuation.Token !args.ifdefStack) args skip) else (LexCont.EndLine(LexerEndlineContinuation.Skip(!args.ifdefStack,0,m)), endline (LexerEndlineContinuation.Skip(!args.ifdefStack,0,m)) args skip) ) let tok = shouldStartLine args lexbuf m (FSComp.SR.lexHashIfMustBeFirst()) (HASH_IF(m,lexed,cont)) if not skip then tok else f lexbuf } @@ -646,9 +651,8 @@ rule token args skip = parse // Skips INACTIVE code until if finds #else / #endif matching with the #if or #else and ifdefSkip n m args skip = parse - | anywhite* "#if" anywhite+ ident anywhite* ("//" [^'\n''\r']*)? + | anywhite* "#if" anywhite+ anystring { let m = lexbuf.LexemeRange - let _id = extractIdentFromHashIf (lexeme lexbuf) // If #if is the first thing on the line then increase depth, otherwise skip, because it is invalid (e.g. "(**) #if ...") if (m.StartColumn <> 0) then @@ -721,7 +725,7 @@ and endline cont args skip = parse } | [^'\r' '\n']+ | _ - { let tok = fail args lexbuf (FSComp.SR.lexExpectedSingleLineComment()) (WHITESPACE (LexCont.Token !args.ifdefStack)) + { let tok = fail args lexbuf (FSComp.SR.pplexExpectedSingleLineComment()) (WHITESPACE (LexCont.Token !args.ifdefStack)) if not skip then tok else token args skip lexbuf } and string sargs skip = parse diff --git a/src/fsharp/pplex.fsl b/src/fsharp/pplex.fsl new file mode 100644 index 00000000000..8fe85ee7dea --- /dev/null +++ b/src/fsharp/pplex.fsl @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Open Technologies, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +{ + +module internal Microsoft.FSharp.Compiler.PPLexer + +open System + +open Microsoft.FSharp.Compiler.Ast +open Microsoft.FSharp.Compiler.ErrorLogger +open Microsoft.FSharp.Compiler.Lexhelp + +open Internal.Utilities.Text.Lexing + +let lexeme (lexbuf : UnicodeLexing.Lexbuf) = UnicodeLexing.Lexbuf.LexemeString lexbuf + +let fail (args : lexargs) (lexbuf:UnicodeLexing.Lexbuf) e = + let m = lexbuf.LexemeRange + args.errorLogger.ErrorR(Error(e,m)) + PPParser.EOF +} + +let letter = '\Lu' | '\Ll' | '\Lt' | '\Lm' | '\Lo' | '\Nl' +let digit = '\Nd' +let connecting_char = '\Pc' +let combining_char = '\Mn' | '\Mc' +let formatting_char = '\Cf' + +let ident_start_char = + letter | '_' + +let ident_char = + letter + | connecting_char + | combining_char + | formatting_char + | digit + | ['\''] + +let ident = ident_start_char ident_char* +let comment = "//" _* +let mcomment = "(*" _* +let whitespace = [' ' '\t'] + +rule tokenstream args = parse +// -------------------------- +| "#if" { PPParser.PRELUDE } +| "#elif" { PPParser.PRELUDE } +| ident { PPParser.ID(lexeme lexbuf) } +// -------------------------- +| "!" { PPParser.OP_NOT } +| "&&" { PPParser.OP_AND } +| "||" { PPParser.OP_OR } +| "(" { PPParser.LPAREN } +| ")" { PPParser.RPAREN } +// -------------------------- +| whitespace { tokenstream args lexbuf } +// -------------------------- +| comment { PPParser.EOF } +| mcomment { fail args lexbuf (FSComp.SR.pplexExpectedSingleLineComment()) } +| _ { + let lex = lexeme lexbuf + let _ = rest lexbuf + fail args lexbuf (FSComp.SR.pplexUnexpectedChar(lex)) + } +| eof { PPParser.EOF } +// -------------------------- +and rest = parse +| _ { rest lexbuf } +| eof { () } diff --git a/src/fsharp/pppars.fsy b/src/fsharp/pppars.fsy new file mode 100644 index 00000000000..0f76836c366 --- /dev/null +++ b/src/fsharp/pppars.fsy @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Open Technologies, Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +%{ +open Microsoft.FSharp.Compiler.Ast +open Microsoft.FSharp.Compiler.ErrorLogger + +let dummy = IfdefId("DUMMY") + +let doNothing _ dflt= + dflt + +let fail (ps : Internal.Utilities.Text.Parsing.IParseState) i e = + let f,t = ps.InputRange i + let m = mkSynRange f t + errorR(Error(e,m)) + dummy +%} + + +%start start + +%token ID +%token OP_NOT OP_AND OP_OR LPAREN RPAREN PRELUDE EOF + +%nonassoc RPAREN +%nonassoc PRELUDE +%left OP_OR +%left OP_AND +%left OP_NOT +%nonassoc LPAREN +%nonassoc ID + +%type < LexerIfdefExpression > start + +%% + +start: Full { $1 } + +Recover: + | error { doNothing parseState () } + +Full: + | PRELUDE Expr EOF { $2 } + | Recover { fail parseState 1 (FSComp.SR.ppparsMissingToken("#if/#elif")) } + +Expr: + | LPAREN Expr RPAREN { $2 } + | ID { IfdefId($1) } + | OP_NOT Expr { IfdefNot($2) } + | Expr OP_AND Expr { IfdefAnd($1,$3) } + | Expr OP_OR Expr { IfdefOr($1,$3) } + + | OP_AND Recover { fail parseState 1 (FSComp.SR.ppparsUnexpectedToken("&&")) } + | OP_OR Recover { fail parseState 1 (FSComp.SR.ppparsUnexpectedToken("||")) } + | OP_NOT Recover { fail parseState 1 (FSComp.SR.ppparsUnexpectedToken("!")) } + | LPAREN error RPAREN { doNothing parseState dummy } + | LPAREN Expr Recover { fail parseState 3 (FSComp.SR.ppparsMissingToken(")")) } + | LPAREN Recover { fail parseState 2 (FSComp.SR.ppparsIncompleteExpression()) } + | RPAREN Recover { fail parseState 1 (FSComp.SR.ppparsUnexpectedToken(")")) } + | Expr Recover { fail parseState 2 (FSComp.SR.ppparsIncompleteExpression()) } + | EOF { fail parseState 1 (FSComp.SR.ppparsIncompleteExpression()) } diff --git a/tests/RunTests.cmd b/tests/RunTests.cmd index 908b2fd8ba8..0321f74167a 100644 --- a/tests/RunTests.cmd +++ b/tests/RunTests.cmd @@ -39,6 +39,10 @@ if not exist "%RESULTSDIR%" (mkdir "%RESULTSDIR%") if /I "%2" == "fsharp" (goto :FSHARP) if /I "%2" == "fsharpqa" (goto :FSHARPQA) +if /I "%2" == "compilerunit" ( + set compilerunitsuffix=net40 + goto :COMPILERUNIT +) if /I "%2" == "coreunit" ( set coreunitsuffix=net40 goto :COREUNIT @@ -65,7 +69,7 @@ if /I "%2" == "ideunit" (goto :IDEUNIT) echo Usage: echo. -echo RunTests.cmd ^ ^ [TagToRun^|"Tags,To,Run"] [TagNotToRun^|"Tags,Not,To,Run"] +echo RunTests.cmd ^ ^ [TagToRun^|"Tags,To,Run"] [TagNotToRun^|"Tags,Not,To,Run"] echo. exit /b 1 @@ -178,6 +182,22 @@ echo nunit-console.exe /nologo /result=%XMLFILE% /output=%OUTPUTFILE% /err=%ERRO goto :EOF +:COMPILERUNIT + +set XMLFILE=ComplierUnit_%compilerunitsuffix%_Xml.xml +set OUTPUTFILE=ComplierUnit_%compilerunitsuffix%_Output.log +set ERRORFILE=ComplierUnit_%compilerunitsuffix%_Error.log + +where.exe nunit-console.exe > NUL 2> NUL +if errorlevel 1 ( + echo Error: nunit-console.exe is not in the PATH + exit /b 1 +) +echo nunit-console.exe /nologo /result=%XMLFILE% /output=%OUTPUTFILE% /err=%ERRORFILE% /work=%RESULTSDIR% %FSCBINPATH%\..\..\%compilerunitsuffix%\bin\FSharp.Compiler.Unittests.dll + nunit-console.exe /nologo /result=%XMLFILE% /output=%OUTPUTFILE% /err=%ERRORFILE% /work=%RESULTSDIR% %FSCBINPATH%\..\..\%compilerunitsuffix%\bin\FSharp.Compiler.Unittests.dll + +goto :EOF + :IDEUNIT set XMLFILE=IDEUnit_Xml.xml diff --git a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/E_MustBeIdent01.fs b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/E_MustBeIdent01.fs index 1a37d88be02..7c9d079bbfa 100644 --- a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/E_MustBeIdent01.fs +++ b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/E_MustBeIdent01.fs @@ -1,6 +1,7 @@ -// #Regression #Conformance #LexicalAnalysis +// #Regression #Conformance #LexicalAnalysis // Verify error if preprocessor directive isn't a valid identifier -//#if directive should be immediately followed by an identifier +//Unexpected character '\*' in preprocessor expression +//Incomplete preprocessor expression #if *COMPILED* exit 0 diff --git a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/ExtendedIfGrammar.fs b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/ExtendedIfGrammar.fs new file mode 100644 index 00000000000..947c71b7617 --- /dev/null +++ b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/ExtendedIfGrammar.fs @@ -0,0 +1,170 @@ +let e0 : int = +#if DEFINED + 0 +#else + failwith "e0" +#endif + +let e1 : int = +#if UNDEFINED + failwith "e1" +#else + 0 +#endif + +let e2 : int = +#if DEFINED && UNDEFINED + failwith "e2" +#else + 0 +#endif + +let e3 : int = +#if UNDEFINED && DEFINED + failwith "e3" +#else + 0 +#endif + +let e4 : int = +#if DEFINED || UNDEFINED + 0 +#else + failwith "e4" +#endif + +let e5 : int = +#if UNDEFINED || DEFINED + 0 +#else + failwith "e5" +#endif + +let e6 : int = +#if !UNDEFINED + 0 +#else + failwith "e6" +#endif + +let e7 :int = +#if !DEFINED + failwith "e7" +#else + 0 +#endif + +let e8 : int = +#if !UNDEFINED || DEFINED + 0 +#else + failwith "e8" +#endif + +let e9 : int = +#if !DEFINED && DEFINED + failwith "e9" +#else + 0 +#endif + +let e10 : int = +#if DEFINED && DEFINED && UNDEFINED + failwith "e10" +#else + 0 +#endif + +let e11 : int = +#if UNDEFINED || UNDEFINED || DEFINED + 0 +#else + failwith "e11" +#endif + +let e12 : int = +#if DEFINED || DEFINED && UNDEFINED + 0 +#else + failwith "e12" +#endif + +let e13 : int = +#if UNDEFINED && DEFINED || DEFINED + 0 +#else + failwith "e13" +#endif + +let e14 : int = +#if (DEFINED) + 0 +#else + failwith "e14" +#endif + +let e15 : int = +#if (DEFINED || DEFINED) && UNDEFINED + failwith "e15" +#else + 0 +#endif + +let e16 : int = +#if UNDEFINED && (DEFINED || DEFINED) + failwith "e16" +#else + 0 +#endif + +let e17 : int = +#if DEFINED // A test comment + 0 +#else + failwith "e17" +#endif + +// When it comes to #if true/false are seen as identifiers not values +let e18 : int = +#if true + failwith "e18" +#else + 0 +#endif + +let e19 : int = +#if false + failwith "e19" +#else + 0 +#endif + +let e20 : int = +#if !!DEFINED + 0 +#else + failwith "e20" +#endif + +let e21 : int = +#if !!!DEFINED + failwith "e21" +#else + 0 +#endif + +let e22 : int = +#if !!UNDEFINED + failwith "e22" +#else + 0 +#endif + +let e23 : int = +#if !!!UNDEFINED + 0 +#else + failwith "e23" +#endif + +exit 0 \ No newline at end of file diff --git a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/env.lst b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/env.lst index db60a5ae6ce..8ab90626e01 100644 --- a/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/env.lst +++ b/tests/fsharpqa/Source/Conformance/LexicalAnalysis/ConditionalCompilation/env.lst @@ -19,4 +19,4 @@ SOURCE=FSharp01.fs # FSharp01.fs SOURCE=FSharp02.fs # FSharp02.fs SOURCE=OCaml01.fs # OCaml01.fs - + SOURCE=ExtendedIfGrammar.fs SCFLAGS="--define:DEFINED" # ExtendedIfGrammar.fs diff --git a/vsintegration/src/unittests/Tests.LanguageService.Colorizer.fs b/vsintegration/src/unittests/Tests.LanguageService.Colorizer.fs index e0238c328f9..83e7c65b18e 100644 --- a/vsintegration/src/unittests/Tests.LanguageService.Colorizer.fs +++ b/vsintegration/src/unittests/Tests.LanguageService.Colorizer.fs @@ -985,7 +985,35 @@ let z = __LINE__(*Test3*) check "(*If*)" TokenType.Comment check "(*Else*)" TokenType.Comment check "(*Endif*)" TokenType.Comment - + + /// FEATURE: Preprocessor extended grammar basic check. + /// FEATURE: More extensive grammar test is done in compiler unit tests + [] + member public this.``Preprocessor.ExtendedIfGrammar.Basic01``() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED || !UNDEFINED // Extended #if + let x = "activeCode" + #else + let x = "inactiveCode" + #endif + """, + marker = "activeCode", + tokenType = TokenType.String) + + [] + member public this.``Preprocessor.ExtendedIfGrammar.Basic02``() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED || !UNDEFINED // Extended #if + let x = "activeCode" + #else + let x = "inactiveCode" + #endif + """, + marker = "inactiveCode", + tokenType = TokenType.InactiveCode) + /// #else / #endif in multiline strings is ignored [] member public this.``Preprocessor.DirectivesInString``() = @@ -1040,7 +1068,7 @@ let z = __LINE__(*Test3*) "#endif"] let (_, _, file) = this.CreateSingleFileProject(code) MoveCursorToStartOfMarker(file, "!!COMPILED") - AssertEqual(TokenType.Operator, GetTokenTypeAtCursor(file)) + AssertEqual(TokenType.Identifier, GetTokenTypeAtCursor(file)) // This was an off-by-one bug in the replacement Colorizer diff --git a/vsintegration/src/unittests/Tests.LanguageService.ErrorList.fs b/vsintegration/src/unittests/Tests.LanguageService.ErrorList.fs index e300da13935..f67faabda08 100644 --- a/vsintegration/src/unittests/Tests.LanguageService.ErrorList.fs +++ b/vsintegration/src/unittests/Tests.LanguageService.ErrorList.fs @@ -579,14 +579,6 @@ but here has type TakeCoffeeBreak(this.VS) // Wait for the background compiler to catch up. VerifyErrorListContainedExpectedStr("nonexistent",project) - [] - member public this.``ErrorReporting.WrongPreprocessorIf``() = - let fileContent = """ - #light - #if !!!!COMPILED - #endif""" - this.VerifyErrorListContainedExpectedString(fileContent,"#if") - [] member public this.``BackgroundComplier``() = this.VerifyErrorListCountAtOpenProject( diff --git a/vsintegration/src/unittests/Tests.LanguageService.Squiggles.fs b/vsintegration/src/unittests/Tests.LanguageService.Squiggles.fs index a70d4d8b033..2c5db4199be 100644 --- a/vsintegration/src/unittests/Tests.LanguageService.Squiggles.fs +++ b/vsintegration/src/unittests/Tests.LanguageService.Squiggles.fs @@ -854,17 +854,13 @@ type X() = member public this.``Squiggles.HashIfWithoutIdent``() = this.TestSquiggle true [ "#if"; "#endif" ] "if" "#if directive should be immediately followed by an identifier" - [] - member public this.``Squiggles.HashIfWrongExpr``() = - this.TestSquiggle true [ "#if !IDENT"; "#endif" ] "if" "#if directive should be immediately followed by an identifier" - [] member public this.``Squiggles.HashIfWithMultilineComment``() = this.TestSquiggle true [ "#if IDENT (* aaa *)"; "#endif" ] "(* aaa" "Expected single line comment or end of line" [] member public this.``Squiggles.HashIfWithUnexpected``() = - this.TestSquiggle true [ "#if IDENT whatever"; "#endif" ] "whatever" "Expected single line comment or end of line" + this.TestSquiggle true [ "#if IDENT whatever"; "#endif" ] "whatever" "Incomplete preprocessor expression" // FEATURE: Touching a depended-upon file will cause a intellisense to update in the currently edited file. []