From eacfa15b35f1c71efbe86326faaf708211b8e65c Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Sun, 29 Sep 2024 18:29:44 +0000 Subject: [PATCH 01/10] ifdef test added --- .../CompilerDirectives/Ifdef.fs | 27 +++++++++++++++++++ .../FSharp.Compiler.ComponentTests.fsproj | 1 + 2 files changed, 28 insertions(+) create mode 100644 tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs b/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs new file mode 100644 index 00000000000..1ddd455960f --- /dev/null +++ b/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs @@ -0,0 +1,27 @@ +namespace CompilerDirectives + +open Xunit +open FSharp.Test.Compiler + +module Ifdef = + + let ifdefSource = """ +[] +let main _ = + #if MYDEFINE1 + printf "1" + #else + printf "2" + #endif + 0 +""" + + [] + [] + [] + let ifdefTest (mydefine, expectedOutput) = + + FSharp ifdefSource + |> withDefines [mydefine] + |> compileExeAndRun + |> verifyOutput expectedOutput diff --git a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj index 03ff28e096a..bbd4fe0ac4d 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj +++ b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj @@ -32,6 +32,7 @@ FsUnit.fs + From 84021bf862adb7efe0e67f0b4866f3904880eb34 Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Sun, 29 Sep 2024 18:54:26 +0000 Subject: [PATCH 02/10] simplified #if lexing --- src/Compiler/lex.fsl | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/Compiler/lex.fsl b/src/Compiler/lex.fsl index fd95cd3ebce..ff900ee41fe 100644 --- a/src/Compiler/lex.fsl +++ b/src/Compiler/lex.fsl @@ -1030,25 +1030,15 @@ rule token (args: LexArgs) (skip: bool) = parse | anywhite* "#if" anywhite+ anystring { let m = lexbuf.LexemeRange + shouldStartLine args lexbuf m (FSComp.SR.lexHashIfMustBeFirst()) () let lookup id = List.contains id args.conditionalDefines let lexed = lexeme lexbuf let isTrue, expr = evalIfDefExpression lexbuf.StartPos lexbuf.ReportLibraryOnlyFeatures lexbuf.LanguageVersion lexbuf.StrictIndentation args lookup lexed args.ifdefStack <- (IfDefIf,m) :: args.ifdefStack LexbufIfdefStore.SaveIfHash(lexbuf, lexed, expr, m) - - // Get the token; make sure it starts at zero position & return - let cont, f = - if isTrue then - let cont = LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Token) - let f = endline LexerEndlineContinuation.Token args skip - cont, f - else - let cont = LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(0, m)) - let f = endline (LexerEndlineContinuation.Skip(0, m)) args skip - cont, f - - let tok = shouldStartLine args lexbuf m (FSComp.SR.lexHashIfMustBeFirst()) (HASH_IF(m,lexed,cont)) - if not skip then tok else f lexbuf } + let contCase = if isTrue then LexerEndlineContinuation.Token else LexerEndlineContinuation.Skip(0, m) + let tok = HASH_IF(m, lexed, LexCont.EndLine(args.ifdefStack, args.stringNest, contCase)) + if skip then endline contCase args skip lexbuf else tok } | anywhite* "#else" anywhite* ("//" [^'\n''\r']*)? { let lexed = (lexeme lexbuf) From 64afff09f8d52c75f3c50d1b023c650bb3038be4 Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Sun, 29 Sep 2024 20:36:09 +0000 Subject: [PATCH 03/10] simplified #else, #endif --- src/Compiler/lex.fsl | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/Compiler/lex.fsl b/src/Compiler/lex.fsl index ff900ee41fe..d8d346c4c7e 100644 --- a/src/Compiler/lex.fsl +++ b/src/Compiler/lex.fsl @@ -1047,36 +1047,28 @@ rule token (args: LexArgs) (skip: bool) = parse | (IfDefElse,_) :: _rest -> LEX_FAILURE (FSComp.SR.lexHashEndifRequiredForElse()) | (IfDefIf,_) :: rest -> let m = lexbuf.LexemeRange + shouldStartLine args lexbuf m (FSComp.SR.lexHashElseMustBeFirst()) () args.ifdefStack <- (IfDefElse,m) :: rest LexbufIfdefStore.SaveElseHash(lexbuf, lexed, m) let tok = HASH_ELSE(m, lexed, LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(0, m))) - let tok = shouldStartLine args lexbuf m (FSComp.SR.lexHashElseMustBeFirst()) tok - if not skip then tok else endline (LexerEndlineContinuation.Skip(0, m)) args skip lexbuf } + if skip then endline (LexerEndlineContinuation.Skip(0, m)) args skip lexbuf else tok } | anywhite* "#endif" anywhite* ("//" [^'\n''\r']*)? { let lexed = (lexeme lexbuf) let m = lexbuf.LexemeRange + shouldStartLine args lexbuf m (FSComp.SR.lexHashEndifMustBeFirst()) () match args.ifdefStack with | []-> LEX_FAILURE (FSComp.SR.lexHashEndingNoMatchingIf()) | _ :: rest -> args.ifdefStack <- rest LexbufIfdefStore.SaveEndIfHash(lexbuf, lexed, m) let tok = HASH_ENDIF(m,lexed,LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Token)) - let tok = shouldStartLine args lexbuf m (FSComp.SR.lexHashEndifMustBeFirst()) tok if not skip then tok else endline LexerEndlineContinuation.Token args skip lexbuf } | "#if" { let tok = WHITESPACE (LexCont.Token (args.ifdefStack, args.stringNest)) let tok = fail args lexbuf (FSComp.SR.lexHashIfMustHaveIdent()) tok - if not skip then tok else token args skip lexbuf } - - | anywhite* "#if" ident_char+ - | anywhite* "#else" ident_char+ - | anywhite* "#endif" ident_char+ - | anywhite* "#light" ident_char+ - { let n = (lexeme lexbuf).IndexOf('#') - lexbuf.StartPos <- lexbuf.StartPos.ShiftColumnBy(n) - HASH_IDENT(lexemeTrimLeft lexbuf (n+1)) } + if skip then token args skip lexbuf else tok } | surrogateChar surrogateChar @@ -1094,15 +1086,15 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse // If #if is the first thing on the line then increase depth, otherwise skip, because it is invalid (e.g. "(**) #if ...") if (m.StartColumn <> 0) then - if not skip then INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack, args.stringNest, n, m)) - else ifdefSkip n m args skip lexbuf + if skip then ifdefSkip n m args skip lexbuf + else INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack, args.stringNest, n, m)) else let lexed = lexeme lexbuf let lookup id = List.contains id args.conditionalDefines let _, expr = evalIfDefExpression lexbuf.StartPos lexbuf.ReportLibraryOnlyFeatures lexbuf.LanguageVersion lexbuf.StrictIndentation args lookup lexed LexbufIfdefStore.SaveIfHash(lexbuf, lexed, expr, m) let tok = INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(n+1, m))) - if not skip then tok else endline (LexerEndlineContinuation.Skip(n+1, m)) args skip lexbuf } + if skip then endline (LexerEndlineContinuation.Skip(n+1, m)) args skip lexbuf else tok } | anywhite* "#else" anywhite* ("//" [^'\n''\r']*)? { let lexed = (lexeme lexbuf) From f5ba0259c3958133ffd0adb45355d6d38682c16d Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Mon, 30 Sep 2024 07:17:35 +0000 Subject: [PATCH 04/10] no need to thread token through shouldStartLine --- src/Compiler/lex.fsl | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Compiler/lex.fsl b/src/Compiler/lex.fsl index d8d346c4c7e..9e407921e1e 100644 --- a/src/Compiler/lex.fsl +++ b/src/Compiler/lex.fsl @@ -194,9 +194,8 @@ let tryAppendXmlDoc (buff: (range * StringBuilder) option) (s:string) = // Utilities for parsing #if/#else/#endif -let shouldStartLine args lexbuf (m:range) err tok = - if (m.StartColumn <> 0) then fail args lexbuf err tok - else tok +let shouldStartLine args lexbuf (m:range) err = + if (m.StartColumn <> 0) then fail args lexbuf err () let shouldStartFile args lexbuf (m:range) err tok = if (m.StartColumn <> 0 || m.StartLine <> 1) then fail args lexbuf err tok @@ -1030,7 +1029,7 @@ rule token (args: LexArgs) (skip: bool) = parse | anywhite* "#if" anywhite+ anystring { let m = lexbuf.LexemeRange - shouldStartLine args lexbuf m (FSComp.SR.lexHashIfMustBeFirst()) () + shouldStartLine args lexbuf m (FSComp.SR.lexHashIfMustBeFirst()) let lookup id = List.contains id args.conditionalDefines let lexed = lexeme lexbuf let isTrue, expr = evalIfDefExpression lexbuf.StartPos lexbuf.ReportLibraryOnlyFeatures lexbuf.LanguageVersion lexbuf.StrictIndentation args lookup lexed @@ -1047,7 +1046,7 @@ rule token (args: LexArgs) (skip: bool) = parse | (IfDefElse,_) :: _rest -> LEX_FAILURE (FSComp.SR.lexHashEndifRequiredForElse()) | (IfDefIf,_) :: rest -> let m = lexbuf.LexemeRange - shouldStartLine args lexbuf m (FSComp.SR.lexHashElseMustBeFirst()) () + shouldStartLine args lexbuf m (FSComp.SR.lexHashElseMustBeFirst()) args.ifdefStack <- (IfDefElse,m) :: rest LexbufIfdefStore.SaveElseHash(lexbuf, lexed, m) let tok = HASH_ELSE(m, lexed, LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(0, m))) @@ -1056,7 +1055,7 @@ rule token (args: LexArgs) (skip: bool) = parse | anywhite* "#endif" anywhite* ("//" [^'\n''\r']*)? { let lexed = (lexeme lexbuf) let m = lexbuf.LexemeRange - shouldStartLine args lexbuf m (FSComp.SR.lexHashEndifMustBeFirst()) () + shouldStartLine args lexbuf m (FSComp.SR.lexHashEndifMustBeFirst()) match args.ifdefStack with | []-> LEX_FAILURE (FSComp.SR.lexHashEndingNoMatchingIf()) | _ :: rest -> @@ -1114,7 +1113,7 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse args.ifdefStack <- (IfDefElse,m) :: rest if not skip then HASH_ELSE(m,lexed,LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Token)) else endline LexerEndlineContinuation.Token args skip lexbuf - else + else LexbufIfdefStore.SaveElseHash(lexbuf, lexed, m) if not skip then INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(n, m))) else endline (LexerEndlineContinuation.Skip(n, m)) args skip lexbuf } @@ -1136,9 +1135,9 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse if not skip then HASH_ENDIF(m,lexed,LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Token)) else endline LexerEndlineContinuation.Token args skip lexbuf else + shouldStartLine args lexbuf m (FSComp.SR.lexWrongNestedHashEndif()) LexbufIfdefStore.SaveEndIfHash(lexbuf, lexed, m) let tok = INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(n-1, m))) - let tok = shouldStartLine args lexbuf m (FSComp.SR.lexWrongNestedHashEndif()) tok if not skip then tok else endline (LexerEndlineContinuation.Skip(n-1, m)) args skip lexbuf } | newline From f467c026fa815898e0808815910febba2fca5b61 Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Mon, 30 Sep 2024 07:41:37 +0000 Subject: [PATCH 05/10] removed HASH_IDENT, some renaming --- src/Compiler/Service/ServiceLexing.fs | 8 ++----- src/Compiler/SyntaxTree/LexFilter.fs | 6 ----- src/Compiler/SyntaxTree/ParseHelpers.fs | 3 ++- src/Compiler/lex.fsl | 32 +++++++++++++------------ src/Compiler/pars.fsy | 1 - 5 files changed, 21 insertions(+), 29 deletions(-) diff --git a/src/Compiler/Service/ServiceLexing.fs b/src/Compiler/Service/ServiceLexing.fs index 37990b026d1..0c4a832e660 100644 --- a/src/Compiler/Service/ServiceLexing.fs +++ b/src/Compiler/Service/ServiceLexing.fs @@ -171,7 +171,6 @@ module internal TokenClassifications = let tokenInfo token = match token with - | HASH_IDENT s | IDENT s -> if s.Length > 0 && Char.ToUpperInvariant s[0] = s[0] then (FSharpTokenColorKind.UpperIdentifier, FSharpTokenCharKind.Identifier, FSharpTokenTriggerClass.None) @@ -734,7 +733,7 @@ module internal LexerStateEncoding = ) | LexCont.EndLine(ifdefs, stringNest, econt) -> match econt with - | LexerEndlineContinuation.Skip(n, m) -> + | LexerEndlineContinuation.IfdefSkip(n, m) -> encodeLexCont ( FSharpTokenizerColorState.EndLineThenSkip, int64 n, @@ -834,7 +833,7 @@ module internal LexerStateEncoding = | FSharpTokenizerColorState.ExtendedInterpolatedString -> LexCont.String(ifdefs, stringNest, LexerStringStyle.ExtendedInterpolated, stringKind, delimLen, mkRange "file" p1 p1) | FSharpTokenizerColorState.EndLineThenSkip -> - LexCont.EndLine(ifdefs, stringNest, LexerEndlineContinuation.Skip(n1, mkRange "file" p1 p1)) + LexCont.EndLine(ifdefs, stringNest, LexerEndlineContinuation.IfdefSkip(n1, mkRange "file" p1 p1)) | FSharpTokenizerColorState.EndLineThenToken -> LexCont.EndLine(ifdefs, stringNest, LexerEndlineContinuation.Token) | _ -> LexCont.Token([], stringNest) @@ -1035,9 +1034,6 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi | HASH_IF(m, lineStr, cont) when lineStr <> "" -> false, processHashIfLine m.StartColumn lineStr cont | HASH_ELSE(m, lineStr, cont) when lineStr <> "" -> false, processHashEndElse m.StartColumn lineStr 4 cont | HASH_ENDIF(m, lineStr, cont) when lineStr <> "" -> false, processHashEndElse m.StartColumn lineStr 5 cont - | HASH_IDENT(ident) -> - delayToken (IDENT ident, leftc + 1, rightc) - false, (HASH, leftc, leftc) | RQUOTE_DOT(s, raw) -> delayToken (DOT, rightc, rightc) false, (RQUOTE(s, raw), leftc, rightc - 1) diff --git a/src/Compiler/SyntaxTree/LexFilter.fs b/src/Compiler/SyntaxTree/LexFilter.fs index 19f7be5d31b..ec136e4035a 100644 --- a/src/Compiler/SyntaxTree/LexFilter.fs +++ b/src/Compiler/SyntaxTree/LexFilter.fs @@ -2513,12 +2513,6 @@ type LexFilterImpl ( and rulesForBothSoftWhiteAndHardWhite(tokenTup: TokenTup) = match tokenTup.Token with - | HASH_IDENT ident -> - let hashPos = LexbufState(tokenTup.StartPos, tokenTup.StartPos.ShiftColumnBy(1), false) - let identPos = LexbufState(tokenTup.StartPos.ShiftColumnBy(1), tokenTup.EndPos, false) - delayToken(TokenTup(IDENT(ident), identPos, tokenTup.LastTokenPos)) - delayToken(TokenTup(HASH, hashPos, tokenTup.LastTokenPos)) - true // Insert HIGH_PRECEDENCE_BRACK_APP if needed // ident[3] diff --git a/src/Compiler/SyntaxTree/ParseHelpers.fs b/src/Compiler/SyntaxTree/ParseHelpers.fs index 22c27eeb9b0..77258fb17e8 100644 --- a/src/Compiler/SyntaxTree/ParseHelpers.fs +++ b/src/Compiler/SyntaxTree/ParseHelpers.fs @@ -165,6 +165,7 @@ type LexerIfdefStack = LexerIfdefStackEntries /// Specifies how the 'endline' function in the lexer should continue after /// it reaches end of line or eof. The options are to continue with 'token' function /// or to continue with 'skip' function. +[] type LexerEndlineContinuation = | Token | Skip of int * range: range @@ -967,7 +968,7 @@ let checkEndOfFileError t = | LexCont.MLOnly(_, _, m) -> reportParseErrorAt m (FSComp.SR.parsEofInIfOcaml ()) - | LexCont.EndLine(_, _, LexerEndlineContinuation.Skip(_, m)) -> reportParseErrorAt m (FSComp.SR.parsEofInDirective ()) + | LexCont.EndLine(_, _, LexerEndlineContinuation.IfdefSkip(_, m)) -> reportParseErrorAt m (FSComp.SR.parsEofInDirective ()) | LexCont.EndLine(endifs, nesting, LexerEndlineContinuation.Token) | LexCont.Token(endifs, nesting) -> diff --git a/src/Compiler/lex.fsl b/src/Compiler/lex.fsl index 9e407921e1e..dd53ef3ca39 100644 --- a/src/Compiler/lex.fsl +++ b/src/Compiler/lex.fsl @@ -333,6 +333,8 @@ let ident_char = let ident = ident_start_char ident_char* +// skip = true: skip whitespace (used by compiler, fsi etc) +// skip = false: send artificial tokens for whitespace (used by VS) rule token (args: LexArgs) (skip: bool) = parse | ident { Keywords.KeywordOrIdentifierToken args lexbuf (lexeme lexbuf) } @@ -1035,11 +1037,11 @@ rule token (args: LexArgs) (skip: bool) = parse let isTrue, expr = evalIfDefExpression lexbuf.StartPos lexbuf.ReportLibraryOnlyFeatures lexbuf.LanguageVersion lexbuf.StrictIndentation args lookup lexed args.ifdefStack <- (IfDefIf,m) :: args.ifdefStack LexbufIfdefStore.SaveIfHash(lexbuf, lexed, expr, m) - let contCase = if isTrue then LexerEndlineContinuation.Token else LexerEndlineContinuation.Skip(0, m) + let contCase = if isTrue then LexerEndlineContinuation.Token else LexerEndlineContinuation.IfdefSkip(0, m) let tok = HASH_IF(m, lexed, LexCont.EndLine(args.ifdefStack, args.stringNest, contCase)) if skip then endline contCase args skip lexbuf else tok } - | anywhite* "#else" anywhite* ("//" [^'\n''\r']*)? + | anywhite* "#else" anywhite* ("//" anystring)? { let lexed = (lexeme lexbuf) match args.ifdefStack with | [] -> LEX_FAILURE (FSComp.SR.lexHashElseNoMatchingIf()) @@ -1049,10 +1051,10 @@ rule token (args: LexArgs) (skip: bool) = parse shouldStartLine args lexbuf m (FSComp.SR.lexHashElseMustBeFirst()) args.ifdefStack <- (IfDefElse,m) :: rest LexbufIfdefStore.SaveElseHash(lexbuf, lexed, m) - let tok = HASH_ELSE(m, lexed, LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(0, m))) - if skip then endline (LexerEndlineContinuation.Skip(0, m)) args skip lexbuf else tok } + let tok = HASH_ELSE(m, lexed, LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.IfdefSkip(0, m))) + if skip then endline (LexerEndlineContinuation.IfdefSkip(0, m)) args skip lexbuf else tok } - | anywhite* "#endif" anywhite* ("//" [^'\n''\r']*)? + | anywhite* "#endif" anywhite* ("//" anystring)? { let lexed = (lexeme lexbuf) let m = lexbuf.LexemeRange shouldStartLine args lexbuf m (FSComp.SR.lexHashEndifMustBeFirst()) @@ -1092,10 +1094,10 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse let lookup id = List.contains id args.conditionalDefines let _, expr = evalIfDefExpression lexbuf.StartPos lexbuf.ReportLibraryOnlyFeatures lexbuf.LanguageVersion lexbuf.StrictIndentation args lookup lexed LexbufIfdefStore.SaveIfHash(lexbuf, lexed, expr, m) - let tok = INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(n+1, m))) - if skip then endline (LexerEndlineContinuation.Skip(n+1, m)) args skip lexbuf else tok } + let tok = INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.IfdefSkip(n+1, m))) + if skip then endline (LexerEndlineContinuation.IfdefSkip(n+1, m)) args skip lexbuf else tok } - | anywhite* "#else" anywhite* ("//" [^'\n''\r']*)? + | anywhite* "#else" anywhite* ("//" anystring)? { let lexed = (lexeme lexbuf) let m = lexbuf.LexemeRange @@ -1115,10 +1117,10 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse else endline LexerEndlineContinuation.Token args skip lexbuf else LexbufIfdefStore.SaveElseHash(lexbuf, lexed, m) - if not skip then INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(n, m))) - else endline (LexerEndlineContinuation.Skip(n, m)) args skip lexbuf } + if not skip then INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.IfdefSkip(n, m))) + else endline (LexerEndlineContinuation.IfdefSkip(n, m)) args skip lexbuf } - | anywhite* "#endif" anywhite* ("//" [^'\n''\r']*)? + | anywhite* "#endif" anywhite* ("//" anystring)? { let lexed = lexeme lexbuf let m = lexbuf.LexemeRange @@ -1137,8 +1139,8 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse else shouldStartLine args lexbuf m (FSComp.SR.lexWrongNestedHashEndif()) LexbufIfdefStore.SaveEndIfHash(lexbuf, lexed, m) - let tok = INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Skip(n-1, m))) - if not skip then tok else endline (LexerEndlineContinuation.Skip(n-1, m)) args skip lexbuf } + let tok = INACTIVECODE(LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.IfdefSkip(n-1, m))) + if not skip then tok else endline (LexerEndlineContinuation.IfdefSkip(n-1, m)) args skip lexbuf } | newline { newline lexbuf; ifdefSkip n m args skip lexbuf } @@ -1167,7 +1169,7 @@ and endline (cont: LexerEndlineContinuation) (args: LexArgs) (skip: bool) = pars if not skip then WHITESPACE(LexCont.Token (args.ifdefStack, args.stringNest)) else token args skip lexbuf - | LexerEndlineContinuation.Skip(n, m) -> + | LexerEndlineContinuation.IfdefSkip(n, m) -> if not skip then INACTIVECODE (LexCont.IfDefSkip(args.ifdefStack, args.stringNest, n, m)) else ifdefSkip n m args skip lexbuf } @@ -1176,7 +1178,7 @@ and endline (cont: LexerEndlineContinuation) (args: LexArgs) (skip: bool) = pars { match cont with | LexerEndlineContinuation.Token -> EOF(LexCont.Token(args.ifdefStack, args.stringNest)) - | LexerEndlineContinuation.Skip(n, m) -> + | LexerEndlineContinuation.IfdefSkip(n, m) -> EOF(LexCont.IfDefSkip(args.ifdefStack, args.stringNest, n, m)) } diff --git a/src/Compiler/pars.fsy b/src/Compiler/pars.fsy index 656b75c39ff..9aac40a4e90 100644 --- a/src/Compiler/pars.fsy +++ b/src/Compiler/pars.fsy @@ -45,7 +45,6 @@ let parse_error_rich = Some(fun (ctxt: ParseErrorContext<_>) -> %token KEYWORD_STRING // Like __SOURCE_DIRECTORY__ %token IDENT -%token HASH_IDENT %token INFIX_STAR_STAR_OP %token INFIX_COMPARE_OP %token INFIX_AT_HAT_OP From 480c670f7ae7d38568c07d14d86bfa6187c6ce46 Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Mon, 30 Sep 2024 08:12:05 +0000 Subject: [PATCH 06/10] renamed the function 'newline' to 'incrLine' to differenciate it from the production 'newline' --- src/Compiler/SyntaxTree/LexHelpers.fs | 2 +- src/Compiler/SyntaxTree/LexHelpers.fsi | 2 +- src/Compiler/SyntaxTree/ParseHelpers.fs | 2 +- src/Compiler/SyntaxTree/ParseHelpers.fsi | 2 +- src/Compiler/lex.fsl | 34 ++++++++++++------------ 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Compiler/SyntaxTree/LexHelpers.fs b/src/Compiler/SyntaxTree/LexHelpers.fs index 736e4be04f5..96b4a9a570b 100644 --- a/src/Compiler/SyntaxTree/LexHelpers.fs +++ b/src/Compiler/SyntaxTree/LexHelpers.fs @@ -249,7 +249,7 @@ let errorsInByteStringBuffer (buf: ByteBuffer) = else None -let newline (lexbuf: LexBuffer<_>) = lexbuf.EndPos <- lexbuf.EndPos.NextLine +let incrLine (lexbuf: LexBuffer<_>) = lexbuf.EndPos <- lexbuf.EndPos.NextLine let advanceColumnBy (lexbuf: LexBuffer<_>) n = lexbuf.EndPos <- lexbuf.EndPos.ShiftColumnBy(n) diff --git a/src/Compiler/SyntaxTree/LexHelpers.fsi b/src/Compiler/SyntaxTree/LexHelpers.fsi index 0ba901a05e0..63997f08988 100644 --- a/src/Compiler/SyntaxTree/LexHelpers.fsi +++ b/src/Compiler/SyntaxTree/LexHelpers.fsi @@ -101,7 +101,7 @@ type LargerThanOneByte = int type LargerThan127ButInsideByte = int val errorsInByteStringBuffer: ByteBuffer -> Option -val newline: Lexing.LexBuffer<'a> -> unit +val incrLine: Lexing.LexBuffer<'a> -> unit val advanceColumnBy: Lexing.LexBuffer<'a> -> n: int -> unit diff --git a/src/Compiler/SyntaxTree/ParseHelpers.fs b/src/Compiler/SyntaxTree/ParseHelpers.fs index 77258fb17e8..a40d2a36de1 100644 --- a/src/Compiler/SyntaxTree/ParseHelpers.fs +++ b/src/Compiler/SyntaxTree/ParseHelpers.fs @@ -168,7 +168,7 @@ type LexerIfdefStack = LexerIfdefStackEntries [] type LexerEndlineContinuation = | Token - | Skip of int * range: range + | IfdefSkip of int * range: range type LexerIfdefExpression = | IfdefAnd of LexerIfdefExpression * LexerIfdefExpression diff --git a/src/Compiler/SyntaxTree/ParseHelpers.fsi b/src/Compiler/SyntaxTree/ParseHelpers.fsi index 52f4257d4c2..c98e7897a32 100644 --- a/src/Compiler/SyntaxTree/ParseHelpers.fsi +++ b/src/Compiler/SyntaxTree/ParseHelpers.fsi @@ -67,7 +67,7 @@ type LexerIfdefStack = LexerIfdefStackEntries type LexerEndlineContinuation = | Token - | Skip of int * range: range + | IfdefSkip of int * range: range type LexerIfdefExpression = | IfdefAnd of LexerIfdefExpression * LexerIfdefExpression diff --git a/src/Compiler/lex.fsl b/src/Compiler/lex.fsl index dd53ef3ca39..a61125645e0 100644 --- a/src/Compiler/lex.fsl +++ b/src/Compiler/lex.fsl @@ -753,7 +753,7 @@ rule token (args: LexArgs) (skip: bool) = parse else singleLineComment (None,1,m,m,args) skip lexbuf } | newline - { newline lexbuf + { incrLine lexbuf if not skip then WHITESPACE (LexCont.Token(args.ifdefStack, args.stringNest)) else token args skip lexbuf } @@ -820,12 +820,12 @@ rule token (args: LexArgs) (skip: bool) = parse lexbuf.EndPos <- pos.ApplyLineDirective((match file with Some f -> FileIndex.fileIndexOfFile f | None -> pos.FileIndex), line) else // add a newline when we don't apply a directive since we consumed a newline getting here - newline lexbuf + incrLine lexbuf token args skip lexbuf else // add a newline when we don't apply a directive since we consumed a newline getting here - newline lexbuf + incrLine lexbuf HASH_LINE (LexCont.Token (args.ifdefStack, args.stringNest)) } @@ -1143,7 +1143,7 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse if not skip then tok else endline (LexerEndlineContinuation.IfdefSkip(n-1, m)) args skip lexbuf } | newline - { newline lexbuf; ifdefSkip n m args skip lexbuf } + { incrLine lexbuf; ifdefSkip n m args skip lexbuf } | [^ ' ' '\n' '\r' ]+ @@ -1163,7 +1163,7 @@ and ifdefSkip (n: int) (m: range) (args: LexArgs) (skip: bool) = parse // or end of file and then calls the lexing function specified by 'cont' - either token or ifdefSkip and endline (cont: LexerEndlineContinuation) (args: LexArgs) (skip: bool) = parse | newline - { newline lexbuf + { incrLine lexbuf match cont with | LexerEndlineContinuation.Token -> if not skip then WHITESPACE(LexCont.Token (args.ifdefStack, args.stringNest)) @@ -1192,7 +1192,7 @@ and endline (cont: LexerEndlineContinuation) (args: LexArgs) (skip: bool) = pars and singleQuoteString (sargs: LexerStringArgs) (skip: bool) = parse | '\\' newline anywhite* { let (_buf, _fin, m, kind, args) = sargs - newline lexbuf + incrLine lexbuf let text = lexeme lexbuf let text2 = text |> String.filter (fun c -> c <> ' ' && c <> '\t') advanceColumnBy lexbuf (text.Length - text2.Length) @@ -1321,7 +1321,7 @@ and singleQuoteString (sargs: LexerStringArgs) (skip: bool) = parse | newline { let (buf, _fin, m, kind, args) = sargs - newline lexbuf + incrLine lexbuf addUnicodeString buf (lexeme lexbuf) if not skip then STRING_TEXT (LexCont.String(args.ifdefStack, args.stringNest, LexerStringStyle.SingleQuote, kind, args.interpolationDelimiterLength, m)) @@ -1390,7 +1390,7 @@ and verbatimString (sargs: LexerStringArgs) (skip: bool) = parse | newline { let (buf, _fin, m, kind, args) = sargs - newline lexbuf + incrLine lexbuf addUnicodeString buf (lexeme lexbuf) if not skip then STRING_TEXT (LexCont.String(args.ifdefStack, args.stringNest, LexerStringStyle.Verbatim, kind, args.interpolationDelimiterLength, m)) @@ -1483,7 +1483,7 @@ and tripleQuoteString (sargs: LexerStringArgs) (skip: bool) = parse | newline { let (buf, _fin, m, kind, args) = sargs - newline lexbuf + incrLine lexbuf addUnicodeString buf (lexeme lexbuf) if not skip then STRING_TEXT (LexCont.String(args.ifdefStack, args.stringNest, LexerStringStyle.TripleQuote, kind, args.interpolationDelimiterLength, m)) @@ -1577,7 +1577,7 @@ and extendedInterpolatedString (sargs: LexerStringArgs) (skip: bool) = parse | newline { let (buf, _fin, m, kind, args) = sargs - newline lexbuf + incrLine lexbuf addUnicodeString buf (lexeme lexbuf) if not skip then STRING_TEXT (LexCont.String(args.ifdefStack, args.stringNest, LexerStringStyle.ExtendedInterpolated, kind, args.interpolationDelimiterLength, m)) @@ -1694,7 +1694,7 @@ and singleLineComment (cargs: SingleLineCommentArgs) (skip: bool) = parse | newline { let buff,_n, mStart, mEnd, args = cargs trySaveXmlDoc lexbuf buff - newline lexbuf + incrLine lexbuf // Saves the documentation (if we're collecting any) into a buffer-local variable. if not skip then LINE_COMMENT (LexCont.Token(args.ifdefStack, args.stringNest)) else @@ -1756,7 +1756,7 @@ and comment (cargs: BlockCommentArgs) (skip: bool) = parse | newline { let n, m, args = cargs - newline lexbuf + incrLine lexbuf if not skip then COMMENT (LexCont.Comment(args.ifdefStack, args.stringNest, n, m)) else comment cargs skip lexbuf } | "*)" @@ -1790,7 +1790,7 @@ and comment (cargs: BlockCommentArgs) (skip: bool) = parse and stringInComment (n: int) (m: range) (args: LexArgs) (skip: bool) = parse // Follow string lexing, skipping tokens until it finishes | '\\' newline anywhite* - { newline lexbuf + { incrLine lexbuf if not skip then COMMENT (LexCont.StringInComment(args.ifdefStack, args.stringNest, LexerStringStyle.SingleQuote, n, m)) else stringInComment n m args skip lexbuf } @@ -1812,7 +1812,7 @@ and stringInComment (n: int) (m: range) (args: LexArgs) (skip: bool) = parse else comment (n, m, args) skip lexbuf } | newline - { newline lexbuf + { incrLine lexbuf if not skip then COMMENT (LexCont.StringInComment(args.ifdefStack, args.stringNest, LexerStringStyle.SingleQuote, n, m)) else stringInComment n m args skip lexbuf } @@ -1842,7 +1842,7 @@ and verbatimStringInComment (n: int) (m: range) (args: LexArgs) (skip: bool) = p else verbatimStringInComment n m args skip lexbuf } | newline - { newline lexbuf + { incrLine lexbuf if not skip then COMMENT (LexCont.StringInComment(args.ifdefStack, args.stringNest, LexerStringStyle.Verbatim, n, m)) else verbatimStringInComment n m args skip lexbuf } @@ -1868,7 +1868,7 @@ and tripleQuoteStringInComment (n: int) (m: range) (args: LexArgs) (skip: bool) else tripleQuoteStringInComment n m args skip lexbuf } | newline - { newline lexbuf + { incrLine lexbuf if not skip then COMMENT (LexCont.StringInComment(args.ifdefStack, args.stringNest, LexerStringStyle.TripleQuote, n, m)) else tripleQuoteStringInComment n m args skip lexbuf } @@ -1890,7 +1890,7 @@ and mlOnly (m: range) (args: LexArgs) (skip: bool) = parse else mlOnly m args skip lexbuf } | newline - { newline lexbuf + { incrLine lexbuf if not skip then COMMENT (LexCont.MLOnly(args.ifdefStack, args.stringNest, m)) else mlOnly m args skip lexbuf } From 2979611923abedcbc0164fa4a4ffd82ff7988b02 Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:55:43 +0000 Subject: [PATCH 07/10] reverting removal of HASH_IDENT --- src/Compiler/Service/ServiceLexing.fs | 4 ++++ src/Compiler/SyntaxTree/LexFilter.fs | 6 ++++++ src/Compiler/lex.fsl | 9 +++++++++ src/Compiler/pars.fsy | 1 + 4 files changed, 20 insertions(+) diff --git a/src/Compiler/Service/ServiceLexing.fs b/src/Compiler/Service/ServiceLexing.fs index 0c4a832e660..083bae1d6d8 100644 --- a/src/Compiler/Service/ServiceLexing.fs +++ b/src/Compiler/Service/ServiceLexing.fs @@ -171,6 +171,7 @@ module internal TokenClassifications = let tokenInfo token = match token with + | HASH_IDENT s | IDENT s -> if s.Length > 0 && Char.ToUpperInvariant s[0] = s[0] then (FSharpTokenColorKind.UpperIdentifier, FSharpTokenCharKind.Identifier, FSharpTokenTriggerClass.None) @@ -1034,6 +1035,9 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi | HASH_IF(m, lineStr, cont) when lineStr <> "" -> false, processHashIfLine m.StartColumn lineStr cont | HASH_ELSE(m, lineStr, cont) when lineStr <> "" -> false, processHashEndElse m.StartColumn lineStr 4 cont | HASH_ENDIF(m, lineStr, cont) when lineStr <> "" -> false, processHashEndElse m.StartColumn lineStr 5 cont + | HASH_IDENT(ident) -> + delayToken (IDENT ident, leftc + 1, rightc) + false, (HASH, leftc, leftc) | RQUOTE_DOT(s, raw) -> delayToken (DOT, rightc, rightc) false, (RQUOTE(s, raw), leftc, rightc - 1) diff --git a/src/Compiler/SyntaxTree/LexFilter.fs b/src/Compiler/SyntaxTree/LexFilter.fs index ec136e4035a..19f7be5d31b 100644 --- a/src/Compiler/SyntaxTree/LexFilter.fs +++ b/src/Compiler/SyntaxTree/LexFilter.fs @@ -2513,6 +2513,12 @@ type LexFilterImpl ( and rulesForBothSoftWhiteAndHardWhite(tokenTup: TokenTup) = match tokenTup.Token with + | HASH_IDENT ident -> + let hashPos = LexbufState(tokenTup.StartPos, tokenTup.StartPos.ShiftColumnBy(1), false) + let identPos = LexbufState(tokenTup.StartPos.ShiftColumnBy(1), tokenTup.EndPos, false) + delayToken(TokenTup(IDENT(ident), identPos, tokenTup.LastTokenPos)) + delayToken(TokenTup(HASH, hashPos, tokenTup.LastTokenPos)) + true // Insert HIGH_PRECEDENCE_BRACK_APP if needed // ident[3] diff --git a/src/Compiler/lex.fsl b/src/Compiler/lex.fsl index a61125645e0..992dad59ce4 100644 --- a/src/Compiler/lex.fsl +++ b/src/Compiler/lex.fsl @@ -1071,6 +1071,15 @@ rule token (args: LexArgs) (skip: bool) = parse let tok = fail args lexbuf (FSComp.SR.lexHashIfMustHaveIdent()) tok if skip then token args skip lexbuf else tok } + // Let the parser deal with these invalid directives + | anywhite* "#if" ident_char+ + | anywhite* "#else" ident_char+ + | anywhite* "#endif" ident_char+ + | anywhite* "#light" ident_char+ + { let n = (lexeme lexbuf).IndexOf('#') + lexbuf.StartPos <- lexbuf.StartPos.ShiftColumnBy(n) + HASH_IDENT(lexemeTrimLeft lexbuf (n+1)) } + | surrogateChar surrogateChar | _ diff --git a/src/Compiler/pars.fsy b/src/Compiler/pars.fsy index 9aac40a4e90..656b75c39ff 100644 --- a/src/Compiler/pars.fsy +++ b/src/Compiler/pars.fsy @@ -45,6 +45,7 @@ let parse_error_rich = Some(fun (ctxt: ParseErrorContext<_>) -> %token KEYWORD_STRING // Like __SOURCE_DIRECTORY__ %token IDENT +%token HASH_IDENT %token INFIX_STAR_STAR_OP %token INFIX_COMPARE_OP %token INFIX_AT_HAT_OP From 6c533845673e5168efd88ead556c70ec8c67cc54 Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Mon, 30 Sep 2024 12:22:06 +0000 Subject: [PATCH 08/10] Ensure no change in behavior. Add test for it. --- src/Compiler/lex.fsl | 2 +- .../CompilerDirectives/Ifdef.fs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Compiler/lex.fsl b/src/Compiler/lex.fsl index 992dad59ce4..9de91ebab74 100644 --- a/src/Compiler/lex.fsl +++ b/src/Compiler/lex.fsl @@ -1057,10 +1057,10 @@ rule token (args: LexArgs) (skip: bool) = parse | anywhite* "#endif" anywhite* ("//" anystring)? { let lexed = (lexeme lexbuf) let m = lexbuf.LexemeRange - shouldStartLine args lexbuf m (FSComp.SR.lexHashEndifMustBeFirst()) match args.ifdefStack with | []-> LEX_FAILURE (FSComp.SR.lexHashEndingNoMatchingIf()) | _ :: rest -> + shouldStartLine args lexbuf m (FSComp.SR.lexHashEndifMustBeFirst()) args.ifdefStack <- rest LexbufIfdefStore.SaveEndIfHash(lexbuf, lexed, m) let tok = HASH_ENDIF(m,lexed,LexCont.EndLine(args.ifdefStack, args.stringNest, LexerEndlineContinuation.Token)) diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs b/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs index 1ddd455960f..5781d4111bc 100644 --- a/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs +++ b/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs @@ -25,3 +25,20 @@ let main _ = |> withDefines [mydefine] |> compileExeAndRun |> verifyOutput expectedOutput + + let ifdefSourceExtraEndif = """ +#if MYDEFINE1 +printf "1" +#endif +(**)#endif(**) +0 +""" + + [] + let extraEndif () = + + FSharp ifdefSourceExtraEndif + |> withDefines ["MYDEFINE1"] + |> asExe + |> compile + |> withDiagnosticMessage "#endif has no matching #if in implementation file" From 5a8daa15f54fc76b6fd919ef52c418b38fa9bf6a Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Tue, 1 Oct 2024 08:12:20 +0000 Subject: [PATCH 09/10] Added a test that will fail if ever somebody else will try again to remove HASH_IDENT --- .../CompilerDirectives/Ifdef.fs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs b/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs index 5781d4111bc..50393ff9850 100644 --- a/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs +++ b/tests/FSharp.Compiler.ComponentTests/CompilerDirectives/Ifdef.fs @@ -26,7 +26,7 @@ let main _ = |> compileExeAndRun |> verifyOutput expectedOutput - let ifdefSourceExtraEndif = """ + let sourceExtraEndif = """ #if MYDEFINE1 printf "1" #endif @@ -37,8 +37,21 @@ printf "1" [] let extraEndif () = - FSharp ifdefSourceExtraEndif + FSharp sourceExtraEndif |> withDefines ["MYDEFINE1"] |> asExe |> compile |> withDiagnosticMessage "#endif has no matching #if in implementation file" + + let sourceUnknownHash = """ +module A +#ifxx +#abc +""" + + [] + let unknownHashDirectiveIsIgnored () = + + FSharp sourceUnknownHash + |> compile + |> shouldSucceed From 905f4cc4a0b93111e50716b0a647f1ad4e7d2397 Mon Sep 17 00:00:00 2001 From: Martin521 <29605222+Martin521@users.noreply.github.com> Date: Wed, 9 Oct 2024 13:58:29 +0000 Subject: [PATCH 10/10] resolve merge issue --- .../FSharp.Compiler.ComponentTests.fsproj | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj index 2d26f092a9b..029ec14b0b4 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj +++ b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj @@ -34,7 +34,6 @@ -