Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Readability of lex.fsl #17817

Merged
merged 14 commits into from
Oct 10, 2024
Merged

Readability of lex.fsl #17817

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
eacfa15
ifdef test added
Martin521 Sep 29, 2024
84021bf
simplified #if lexing
Martin521 Sep 29, 2024
64afff0
simplified #else, #endif
Martin521 Sep 29, 2024
f5ba025
no need to thread token through shouldStartLine
Martin521 Sep 30, 2024
f467c02
removed HASH_IDENT, some renaming
Martin521 Sep 30, 2024
480c670
renamed the function 'newline' to 'incrLine' to differenciate it from…
Martin521 Sep 30, 2024
f6bbacf
Merge branch 'main' into lexing-deobfuscation
Martin521 Sep 30, 2024
2979611
reverting removal of HASH_IDENT
Martin521 Sep 30, 2024
e174b65
Merge branch 'main' into lexing-deobfuscation
Martin521 Sep 30, 2024
6c53384
Ensure no change in behavior. Add test for it.
Martin521 Sep 30, 2024
5a8daa1
Added a test that will fail if ever somebody else will try again to r…
Martin521 Oct 1, 2024
8b6cedb
Merge remote-tracking branch 'upstream/main' into lexing-deobfuscation
Martin521 Oct 9, 2024
905f4cc
resolve merge issue
Martin521 Oct 9, 2024
05be765
Merge branch 'main' into lexing-deobfuscation
Martin521 Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions src/Compiler/Service/ServiceLexing.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,6 @@ module internal TokenClassifications =

let tokenInfo token =
match token with
| HASH_IDENT s
| IDENT s ->
if s.Length > 0 && Char.ToUpperInvariant s[0] = s[0] then
(FSharpTokenColorKind.UpperIdentifier, FSharpTokenCharKind.Identifier, FSharpTokenTriggerClass.None)
Expand Down Expand Up @@ -734,7 +733,7 @@ module internal LexerStateEncoding =
)
| LexCont.EndLine(ifdefs, stringNest, econt) ->
match econt with
| LexerEndlineContinuation.Skip(n, m) ->
| LexerEndlineContinuation.IfdefSkip(n, m) ->
encodeLexCont (
FSharpTokenizerColorState.EndLineThenSkip,
int64 n,
Expand Down Expand Up @@ -834,7 +833,7 @@ module internal LexerStateEncoding =
| FSharpTokenizerColorState.ExtendedInterpolatedString ->
LexCont.String(ifdefs, stringNest, LexerStringStyle.ExtendedInterpolated, stringKind, delimLen, mkRange "file" p1 p1)
| FSharpTokenizerColorState.EndLineThenSkip ->
LexCont.EndLine(ifdefs, stringNest, LexerEndlineContinuation.Skip(n1, mkRange "file" p1 p1))
LexCont.EndLine(ifdefs, stringNest, LexerEndlineContinuation.IfdefSkip(n1, mkRange "file" p1 p1))
| FSharpTokenizerColorState.EndLineThenToken -> LexCont.EndLine(ifdefs, stringNest, LexerEndlineContinuation.Token)
| _ -> LexCont.Token([], stringNest)

Expand Down Expand Up @@ -1035,9 +1034,6 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi
| HASH_IF(m, lineStr, cont) when lineStr <> "" -> false, processHashIfLine m.StartColumn lineStr cont
| HASH_ELSE(m, lineStr, cont) when lineStr <> "" -> false, processHashEndElse m.StartColumn lineStr 4 cont
| HASH_ENDIF(m, lineStr, cont) when lineStr <> "" -> false, processHashEndElse m.StartColumn lineStr 5 cont
| HASH_IDENT(ident) ->
delayToken (IDENT ident, leftc + 1, rightc)
false, (HASH, leftc, leftc)
| RQUOTE_DOT(s, raw) ->
delayToken (DOT, rightc, rightc)
false, (RQUOTE(s, raw), leftc, rightc - 1)
Expand Down
6 changes: 0 additions & 6 deletions src/Compiler/SyntaxTree/LexFilter.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2513,12 +2513,6 @@ type LexFilterImpl (

and rulesForBothSoftWhiteAndHardWhite(tokenTup: TokenTup) =
match tokenTup.Token with
| HASH_IDENT ident ->
let hashPos = LexbufState(tokenTup.StartPos, tokenTup.StartPos.ShiftColumnBy(1), false)
let identPos = LexbufState(tokenTup.StartPos.ShiftColumnBy(1), tokenTup.EndPos, false)
delayToken(TokenTup(IDENT(ident), identPos, tokenTup.LastTokenPos))
delayToken(TokenTup(HASH, hashPos, tokenTup.LastTokenPos))
true

// Insert HIGH_PRECEDENCE_BRACK_APP if needed
// ident[3]
Expand Down
2 changes: 1 addition & 1 deletion src/Compiler/SyntaxTree/LexHelpers.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ let errorsInByteStringBuffer (buf: ByteBuffer) =
else
None

let newline (lexbuf: LexBuffer<_>) = lexbuf.EndPos <- lexbuf.EndPos.NextLine
let incrLine (lexbuf: LexBuffer<_>) = lexbuf.EndPos <- lexbuf.EndPos.NextLine

let advanceColumnBy (lexbuf: LexBuffer<_>) n =
lexbuf.EndPos <- lexbuf.EndPos.ShiftColumnBy(n)
Expand Down
2 changes: 1 addition & 1 deletion src/Compiler/SyntaxTree/LexHelpers.fsi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ type LargerThanOneByte = int
type LargerThan127ButInsideByte = int
val errorsInByteStringBuffer: ByteBuffer -> Option<LargerThanOneByte * LargerThan127ButInsideByte>

val newline: Lexing.LexBuffer<'a> -> unit
val incrLine: Lexing.LexBuffer<'a> -> unit

val advanceColumnBy: Lexing.LexBuffer<'a> -> n: int -> unit

Expand Down
5 changes: 3 additions & 2 deletions src/Compiler/SyntaxTree/ParseHelpers.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,9 +165,10 @@ type LexerIfdefStack = LexerIfdefStackEntries
/// Specifies how the 'endline' function in the lexer should continue after
/// it reaches end of line or eof. The options are to continue with 'token' function
/// or to continue with 'skip' function.
[<RequireQualifiedAccess>]
type LexerEndlineContinuation =
| Token
| Skip of int * range: range
| IfdefSkip of int * range: range

type LexerIfdefExpression =
| IfdefAnd of LexerIfdefExpression * LexerIfdefExpression
Expand Down Expand Up @@ -967,7 +968,7 @@ let checkEndOfFileError t =

| LexCont.MLOnly(_, _, m) -> reportParseErrorAt m (FSComp.SR.parsEofInIfOcaml ())

| LexCont.EndLine(_, _, LexerEndlineContinuation.Skip(_, m)) -> reportParseErrorAt m (FSComp.SR.parsEofInDirective ())
| LexCont.EndLine(_, _, LexerEndlineContinuation.IfdefSkip(_, m)) -> reportParseErrorAt m (FSComp.SR.parsEofInDirective ())

| LexCont.EndLine(endifs, nesting, LexerEndlineContinuation.Token)
| LexCont.Token(endifs, nesting) ->
Expand Down
2 changes: 1 addition & 1 deletion src/Compiler/SyntaxTree/ParseHelpers.fsi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ type LexerIfdefStack = LexerIfdefStackEntries

type LexerEndlineContinuation =
| Token
| Skip of int * range: range
| IfdefSkip of int * range: range

type LexerIfdefExpression =
| IfdefAnd of LexerIfdefExpression * LexerIfdefExpression
Expand Down
Loading
Loading