Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clean lexer #1263

Merged
merged 4 commits into from
Jun 20, 2016
Merged

Clean lexer #1263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0143a7a
Remove unused permitFsharpKeywords switch
forki Jun 15, 2016
540a0fe
Remove deprecated condition from lexer
forki Jun 15, 2016
9d00338
use pattern matching in lexer
forki Jun 15, 2016
6a60bd2
Remove unused permitFsharpKeywords switch
forki Jun 15, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/fsharp/CompileOptions.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -891,7 +891,7 @@ let compilingFsLib20Flag (tcConfigB : TcConfigBuilder) =
let compilingFsLib40Flag (tcConfigB : TcConfigBuilder) =
CompilerOption("compiling-fslib-40", tagNone, OptionUnit (fun () -> tcConfigB.compilingFslib40 <- true; ), Some(InternalCommandLineOption("--compiling-fslib-40", rangeCmdArgs)), None)
let mlKeywordsFlag =
CompilerOption("ml-keywords", tagNone, OptionUnit (fun () -> Lexhelp.Keywords.permitFsharpKeywords <- false), Some(DeprecatedCommandLineOptionNoDescription("--ml-keywords", rangeCmdArgs)), None)
CompilerOption("ml-keywords", tagNone, OptionUnit (fun () -> ()), Some(DeprecatedCommandLineOptionNoDescription("--ml-keywords", rangeCmdArgs)), None)

let gnuStyleErrorsFlag tcConfigB =
CompilerOption("gnu-style-errors", tagNone, OptionUnit (fun () -> tcConfigB.errorStyle <- ErrorStyle.EmacsErrors), Some(DeprecatedCommandLineOptionNoDescription("--gnu-style-errors", rangeCmdArgs)), None)
Expand Down
44 changes: 20 additions & 24 deletions src/fsharp/lexhelp.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,48 +305,44 @@ module Keywords =
keywordList |> List.map (fun (_, w, _) -> w)

let keywordTable =
// TODO: this doesn't need to be a multi-map, a dictionary will do
let tab = System.Collections.Generic.Dictionary<string,token>(100)
for (_mode,keyword,token) in keywordList do tab.Add(keyword,token)
for _,keyword,token in keywordList do
tab.Add(keyword,token)
tab

let KeywordToken s = keywordTable.[s]

/// ++GLOBAL MUTABLE STATE. Note this is a deprecated, undocumented command line option anyway, we can ignore it.
let mutable permitFsharpKeywords = true

let IdentifierToken args (lexbuf:UnicodeLexing.Lexbuf) (s:string) =
if IsCompilerGeneratedName s then
warning(Error(FSComp.SR.lexhlpIdentifiersContainingAtSymbolReserved(), lexbuf.LexemeRange));
args.resourceManager.InternIdentifierToken s

let KeywordOrIdentifierToken args (lexbuf:UnicodeLexing.Lexbuf) s =
if not permitFsharpKeywords && List.contains s unreserveWords then
// You can assume this condition never fires - this is a deprecated, undocumented command line option anyway, we can ignore it.
IdentifierToken args lexbuf s
else
let mutable v = Unchecked.defaultof<_>
if keywordTable.TryGetValue(s, &v) then
if (match v with RESERVED -> true | _ -> false) then
warning(ReservedKeyword(FSComp.SR.lexhlpIdentifierReserved(s), lexbuf.LexemeRange));
IdentifierToken args lexbuf s
else v
else
match keywordTable.TryGetValue s with
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please double check this return-a-tuple-and-match-on-it formulation produces code at least as good as before. I still tend to write the mutable by hand in performance-critical code but I know there is an optimization that should give us the right code here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

  internal static Parser.token KeywordOrIdentifierToken(Lexhelp.lexargs args, LexBuffer<char> lexbuf, string s)
  {
    Parser.token token1 = (Parser.token) null;
    bool flag = Lexhelp.Keywords.keywordTable.TryGetValue(s, out token1);
    Parser.token token2 = token1;
    if (flag)
    {
         ...
    }
  }

| true,v ->
match v with
| RESERVED ->
warning(ReservedKeyword(FSComp.SR.lexhlpIdentifierReserved(s), lexbuf.LexemeRange));
IdentifierToken args lexbuf s
| _ -> v
| _ ->
match s with
| "__SOURCE_DIRECTORY__" ->
let filename = fileOfFileIndex lexbuf.StartPos.FileIndex
let dirname = if filename = stdinMockFilename then
System.IO.Directory.GetCurrentDirectory()
else
filename |> FileSystem.GetFullPathShim (* asserts that path is already absolute *)
|> System.IO.Path.GetDirectoryName
let dirname =
if filename = stdinMockFilename then
System.IO.Directory.GetCurrentDirectory()
else
filename
|> FileSystem.GetFullPathShim (* asserts that path is already absolute *)
|> System.IO.Path.GetDirectoryName
KEYWORD_STRING dirname
| "__SOURCE_FILE__" ->
KEYWORD_STRING (System.IO.Path.GetFileName((fileOfFileIndex lexbuf.StartPos.FileIndex)))
KEYWORD_STRING (System.IO.Path.GetFileName((fileOfFileIndex lexbuf.StartPos.FileIndex)))
| "__LINE__" ->
KEYWORD_STRING (string lexbuf.StartPos.Line)
KEYWORD_STRING (string lexbuf.StartPos.Line)
| _ ->
IdentifierToken args lexbuf s
IdentifierToken args lexbuf s

/// A utility to help determine if an identifier needs to be quoted
let QuoteIdentifierIfNeeded (s : string) : string =
Expand Down
1 change: 0 additions & 1 deletion src/fsharp/lexhelp.fsi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,5 +67,4 @@ module Keywords =
val KeywordOrIdentifierToken : lexargs -> UnicodeLexing.Lexbuf -> string -> Parser.token
val IdentifierToken : lexargs -> UnicodeLexing.Lexbuf -> string -> Parser.token
val QuoteIdentifierIfNeeded : string -> string
val mutable permitFsharpKeywords : bool
val keywordNames : string list