Skip to content

Commit 8dfc02f

Browse files
dsymeKevinRansom
dsyme
authored and
KevinRansom
committed
update FsLexYacc to 8.0.1 source (by diret copy) (#6355)
1 parent f1aee5f commit 8dfc02f

File tree

4 files changed

+70
-74
lines changed

4 files changed

+70
-74
lines changed

‎src/buildtools/fslex/fslex.fs

Lines changed: 56 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ open System.Collections.Generic
1313
open System.IO
1414

1515
//------------------------------------------------------------------
16-
// This code is duplicated from FSharp.Compiler.UnicodeLexing
16+
// This code is duplicated from Microsoft.FSharp.Compiler.UnicodeLexing
1717

1818
type Lexbuf = LexBuffer<char>
1919

@@ -32,7 +32,7 @@ let UnicodeFileAsLexbuf (filename,codePage : int option) : FileStream * StreamRe
3232
| None -> new StreamReader(stream,true)
3333
| Some n -> new StreamReader(stream,System.Text.Encoding.GetEncoding(n))
3434
let lexbuf = LexBuffer.FromFunction(reader.Read)
35-
lexbuf.EndPos <- Position.FirstLine(filename);
35+
lexbuf.EndPos <- Position.FirstLine(filename)
3636
stream, reader, lexbuf
3737

3838
//------------------------------------------------------------------
@@ -43,32 +43,31 @@ let out = ref None
4343
let inputCodePage = ref None
4444
let light = ref None
4545

46-
let mutable lexlib = "Microsoft.FSharp.Text.Lexing"
46+
let mutable lexlib = "FSharp.Text.Lexing"
4747

4848
let usage =
49-
[ ArgInfo ("-o", ArgType.String (fun s -> out := Some s), "Name the output file.");
50-
ArgInfo ("--codepage", ArgType.Int (fun i -> inputCodePage := Some i), "Assume input lexer specification file is encoded with the given codepage.");
51-
ArgInfo ("--light", ArgType.Unit (fun () -> light := Some true), "(ignored)");
52-
ArgInfo ("--light-off", ArgType.Unit (fun () -> light := Some false), "Add #light \"off\" to the top of the generated file");
53-
ArgInfo ("--lexlib", ArgType.String (fun s -> lexlib <- s), "Specify the namespace for the implementation of the lexer table interpreter (default Microsoft.FSharp.Text.Lexing)");
54-
ArgInfo ("--unicode", ArgType.Set unicode, "Produce a lexer for use with 16-bit unicode characters.");
49+
[ ArgInfo ("-o", ArgType.String (fun s -> out := Some s), "Name the output file.")
50+
ArgInfo ("--codepage", ArgType.Int (fun i -> inputCodePage := Some i), "Assume input lexer specification file is encoded with the given codepage.")
51+
ArgInfo ("--light", ArgType.Unit (fun () -> light := Some true), "(ignored)")
52+
ArgInfo ("--light-off", ArgType.Unit (fun () -> light := Some false), "Add #light \"off\" to the top of the generated file")
53+
ArgInfo ("--lexlib", ArgType.String (fun s -> lexlib <- s), "Specify the namespace for the implementation of the lexer table interpreter (default FSharp.Text.Lexing)")
54+
ArgInfo ("--unicode", ArgType.Set unicode, "Produce a lexer for use with 16-bit unicode characters.")
5555
]
5656

5757
let _ = ArgParser.Parse(usage, (fun x -> match !input with Some _ -> failwith "more than one input given" | None -> input := Some x), "fslex <filename>")
5858

5959
let outputInt (os: TextWriter) (n:int) = os.Write(string n)
6060

6161
let outputCodedUInt16 (os: #TextWriter) (n:int) =
62-
os.Write n;
63-
os.Write "us; ";
62+
os.Write n
63+
os.Write "us; "
6464

6565
let sentinel = 255 * 256 + 255
6666

6767
let lineCount = ref 0
6868
let cfprintfn (os: #TextWriter) fmt = Printf.kfprintf (fun () -> incr lineCount; os.WriteLine()) os fmt
6969

70-
[<EntryPoint>]
71-
let main(args: string[]) =
70+
let main() =
7271
try
7372
let filename = (match !input with Some x -> x | None -> failwith "no input given")
7473
let domain = if !unicode then "Unicode" else "Ascii"
@@ -82,14 +81,14 @@ let main(args: string[]) =
8281
eprintf "%s(%d,%d): error: %s" filename lexbuf.StartPos.Line lexbuf.StartPos.Column
8382
(match e with
8483
| Failure s -> s
85-
| _ -> e.Message);
84+
| _ -> e.Message)
8685
exit 1
87-
printfn "compiling to dfas (can take a while...)";
86+
printfn "compiling to dfas (can take a while...)"
8887
let perRuleData, dfaNodes = AST.Compile spec
8988
let dfaNodes = dfaNodes |> List.sortBy (fun n -> n.Id)
9089

91-
printfn "%d states" dfaNodes.Length;
92-
printfn "writing output";
90+
printfn "%d states" dfaNodes.Length
91+
printfn "writing output"
9392

9493
let output =
9594
match !out with
@@ -99,21 +98,21 @@ let main(args: string[]) =
9998
use os = System.IO.File.CreateText output
10099

101100
if (!light = Some(false)) || (!light = None && (Path.HasExtension(output) && Path.GetExtension(output) = ".ml")) then
102-
cfprintfn os "#light \"off\"";
101+
cfprintfn os "#light \"off\""
103102

104103
let printLinesIfCodeDefined (code,pos:Position) =
105104
if pos <> Position.Empty // If bottom code is unspecified, then position is empty.
106105
then
107-
cfprintfn os "# %d \"%s\"" pos.Line pos.FileName;
108-
cfprintfn os "%s" code;
106+
cfprintfn os "# %d \"%s\"" pos.Line pos.FileName
107+
cfprintfn os "%s" code
109108

110109
printLinesIfCodeDefined spec.TopCode
111110
let code = fst spec.TopCode
112-
lineCount := !lineCount + code.Replace("\r","").Split([| '\n' |]).Length;
113-
cfprintfn os "# %d \"%s\"" !lineCount output;
111+
lineCount := !lineCount + code.Replace("\r","").Split([| '\n' |]).Length
112+
cfprintfn os "# %d \"%s\"" !lineCount output
114113

115-
cfprintfn os "let trans : uint16[] array = ";
116-
cfprintfn os " [| ";
114+
cfprintfn os "let trans : uint16[] array = "
115+
cfprintfn os " [| "
117116
if !unicode then
118117
let specificUnicodeChars = GetSpecificUnicodeChars()
119118
// This emits a (numLowUnicodeChars+NumUnicodeCategories+(2*#specificUnicodeChars)+1) * #states array of encoded UInt16 values
@@ -128,8 +127,8 @@ let main(args: string[]) =
128127
//
129128
// For the SpecificUnicodeChars the entries are char/next-state pairs.
130129
for state in dfaNodes do
131-
cfprintfn os " (* State %d *)" state.Id;
132-
fprintf os " [| ";
130+
cfprintfn os " (* State %d *)" state.Id
131+
fprintf os " [| "
133132
let trans =
134133
let dict = new Dictionary<_,_>()
135134
state.Transitions |> List.iter dict.Add
@@ -141,15 +140,15 @@ let main(args: string[]) =
141140
outputCodedUInt16 os sentinel
142141
for i = 0 to numLowUnicodeChars-1 do
143142
let c = char i
144-
emit (EncodeChar c);
143+
emit (EncodeChar c)
145144
for c in specificUnicodeChars do
146-
outputCodedUInt16 os (int c);
147-
emit (EncodeChar c);
145+
outputCodedUInt16 os (int c)
146+
emit (EncodeChar c)
148147
for i = 0 to NumUnicodeCategories-1 do
149-
emit (EncodeUnicodeCategoryIndex i);
150-
emit Eof;
148+
emit (EncodeUnicodeCategoryIndex i)
149+
emit Eof
151150
cfprintfn os "|];"
152-
done;
151+
done
153152

154153
else
155154
// Each row for the ASCII table has format
@@ -160,8 +159,8 @@ let main(args: string[]) =
160159

161160
// This emits a (256+1) * #states array of encoded UInt16 values
162161
for state in dfaNodes do
163-
cfprintfn os " (* State %d *)" state.Id;
164-
fprintf os " [|";
162+
cfprintfn os " (* State %d *)" state.Id
163+
fprintf os " [|"
165164
let trans =
166165
let dict = new Dictionary<_,_>()
167166
state.Transitions |> List.iter dict.Add
@@ -173,52 +172,51 @@ let main(args: string[]) =
173172
outputCodedUInt16 os sentinel
174173
for i = 0 to 255 do
175174
let c = char i
176-
emit (EncodeChar c);
177-
emit Eof;
175+
emit (EncodeChar c)
176+
emit Eof
178177
cfprintfn os "|];"
179-
done;
178+
done
180179

181-
cfprintfn os " |] ";
180+
cfprintfn os " |] "
182181

183-
fprintf os "let actions : uint16[] = [|";
182+
fprintf os "let actions : uint16[] = [|"
184183
for state in dfaNodes do
185184
if state.Accepted.Length > 0 then
186185
outputCodedUInt16 os (snd state.Accepted.Head)
187186
else
188187
outputCodedUInt16 os sentinel
189-
done;
190-
cfprintfn os "|]";
191-
cfprintfn os "let _fslex_tables = %s.%sTables.Create(trans,actions)" lexlib domain;
188+
done
189+
cfprintfn os "|]"
190+
cfprintfn os "let _fslex_tables = %s.%sTables.Create(trans,actions)" lexlib domain
192191

193-
cfprintfn os "let rec _fslex_dummy () = _fslex_dummy() ";
192+
cfprintfn os "let rec _fslex_dummy () = _fslex_dummy() "
194193

195194
// These actions push the additional start state and come first, because they are then typically inlined into later
196195
// rules. This means more tailcalls are taken as direct branches, increasing efficiency and
197196
// improving stack usage on platforms that do not take tailcalls.
198197
for ((startNode, actions),(ident,args,_)) in List.zip perRuleData spec.Rules do
199-
cfprintfn os "(* Rule %s *)" ident;
200-
cfprintfn os "and %s %s (lexbuf : %s.LexBuffer<_>) = _fslex_%s %s %d lexbuf" ident (String.Join(" ",Array.ofList args)) lexlib ident (String.Join(" ",Array.ofList args)) startNode.Id;
201-
for ((startNode, actions),(ident,args,_)) in List.zip perRuleData spec.Rules do
202-
cfprintfn os "(* Rule %s *)" ident;
203-
cfprintfn os "and _fslex_%s %s _fslex_state lexbuf =" ident (String.Join(" ",Array.ofList args));
204-
cfprintfn os " match _fslex_tables.Interpret(_fslex_state,lexbuf) with" ;
205-
actions |> Seq.iteri (fun i (code,pos) ->
206-
cfprintfn os " | %d -> ( " i;
207-
cfprintfn os "# %d \"%s\"" pos.Line pos.FileName;
198+
cfprintfn os "// Rule %s" ident
199+
cfprintfn os "and %s %s lexbuf =" ident (String.Join(" ",Array.ofList args))
200+
cfprintfn os " match _fslex_tables.Interpret(%d,lexbuf) with" startNode.Id
201+
actions |> Seq.iteri (fun i (code:string, pos) ->
202+
cfprintfn os " | %d -> ( " i
203+
cfprintfn os "# %d \"%s\"" pos.Line pos.FileName
208204
let lines = code.Split([| '\r'; '\n' |], StringSplitOptions.RemoveEmptyEntries)
209205
for line in lines do
210-
cfprintfn os " %s" line;
211-
cfprintfn os "# %d \"%s\"" !lineCount output;
206+
cfprintfn os " %s" line
207+
cfprintfn os "# %d \"%s\"" !lineCount output
212208
cfprintfn os " )")
213209
cfprintfn os " | _ -> failwith \"%s\"" ident
214210

215211

216-
cfprintfn os "";
212+
cfprintfn os ""
217213

218214
printLinesIfCodeDefined spec.BottomCode
219-
cfprintfn os "# 3000000 \"%s\"" output;
220-
0
215+
cfprintfn os "# 3000000 \"%s\"" output
221216

222217
with e ->
223-
eprintf "FSLEX: error FSL000: %s" (match e with Failure s -> s | e -> e.ToString());
218+
eprintf "FSLEX: error FSL000: %s" (match e with Failure s -> s | e -> e.ToString())
224219
exit 1
220+
221+
222+
let result = main()

‎src/buildtools/fslex/fslexast.fs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
module internal FsLexYacc.FsLex.AST
44

55
open System.Collections.Generic
6-
open Microsoft.FSharp.Text
6+
open FSharp.Text
77
open Microsoft.FSharp.Collections
88
open Internal.Utilities
99
open Internal.Utilities.Text.Lexing

‎src/buildtools/fsyacc/fsyacc.fs

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ open FsLexYacc.FsYacc
1212
open FsLexYacc.FsYacc.AST
1313

1414
//------------------------------------------------------------------
15-
// This code is duplicated from FSharp.Compiler.UnicodeLexing
15+
// This code is duplicated from Microsoft.FSharp.Compiler.UnicodeLexing
1616

1717
type Lexbuf = LexBuffer<char>
1818

@@ -47,8 +47,8 @@ let compat = ref false
4747
let log = ref false
4848
let light = ref None
4949
let inputCodePage = ref None
50-
let mutable lexlib = "Microsoft.FSharp.Text.Lexing"
51-
let mutable parslib = "Microsoft.FSharp.Text.Parsing"
50+
let mutable lexlib = "FSharp.Text.Lexing"
51+
let mutable parslib = "FSharp.Text.Parsing"
5252

5353
let usage =
5454
[ ArgInfo("-o", ArgType.String (fun s -> out := Some s), "Name the output file.");
@@ -60,8 +60,8 @@ let usage =
6060
ArgInfo("--light-off", ArgType.Unit (fun () -> light := Some false), "Add #light \"off\" to the top of the generated file");
6161
ArgInfo("--ml-compatibility", ArgType.Set compat, "Support the use of the global state from the 'Parsing' module in FSharp.PowerPack.dll.");
6262
ArgInfo("--tokens", ArgType.Set tokenize, "Simply tokenize the specification file itself.");
63-
ArgInfo("--lexlib", ArgType.String (fun s -> lexlib <- s), "Specify the namespace for the implementation of the lexer (default: Microsoft.FSharp.Text.Lexing)");
64-
ArgInfo("--parslib", ArgType.String (fun s -> parslib <- s), "Specify the namespace for the implementation of the parser table interpreter (default: Microsoft.FSharp.Text.Parsing)");
63+
ArgInfo("--lexlib", ArgType.String (fun s -> lexlib <- s), "Specify the namespace for the implementation of the lexer (default: FSharp.Text.Lexing)");
64+
ArgInfo("--parslib", ArgType.String (fun s -> parslib <- s), "Specify the namespace for the implementation of the parser table interpreter (default: FSharp.Text.Parsing)");
6565
ArgInfo("--codepage", ArgType.Int (fun i -> inputCodePage := Some i), "Assume input lexer specification file is encoded with the given codepage."); ]
6666

6767
let _ = ArgParser.Parse(usage,(fun x -> match !input with Some _ -> failwith "more than one input given" | None -> input := Some x),"fsyacc <filename>")
@@ -523,11 +523,9 @@ let main() =
523523

524524
logf (fun oso -> oso.Close())
525525

526-
[<EntryPoint>]
527-
let result(args: string[]) =
528-
try
529-
main()
530-
0
526+
let result =
527+
try main()
531528
with e ->
532-
eprintf "FSYACC: error FSY000: %s" (match e with Failure s -> s | e -> e.Message);
533-
1
529+
eprintf "FSYACC: error FSY000: %s" (match e with Failure s -> s | e -> e.Message);
530+
exit 1
531+

‎src/buildtools/fsyacc/fsyaccast.fs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ type Symbols = Symbol list
3939
//---------------------------------------------------------------------
4040
// Output Raw Parser Spec AST
4141

42-
let StringOfSym sym = match sym with Terminal s -> "'" + s + "'" | NonTerminal s -> s
42+
let StringOfSym sym = match sym with Terminal s -> "'" ^ s ^ "'" | NonTerminal s -> s
4343

4444
let OutputSym os sym = fprintf os "%s" (StringOfSym sym)
4545

@@ -353,7 +353,7 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec) =
353353
stopWatch.Start()
354354

355355
// Augment the grammar
356-
let fakeStartNonTerminals = spec.StartSymbols |> List.map(fun nt -> "_start" + nt)
356+
let fakeStartNonTerminals = spec.StartSymbols |> List.map(fun nt -> "_start"^nt)
357357
let nonTerminals = fakeStartNonTerminals@spec.NonTerminals
358358
let endOfInputTerminal = "$$"
359359
let dummyLookahead = "#"
@@ -466,7 +466,7 @@ let CompilerLalrParserSpec logf (spec : ProcessedParserSpec) =
466466
let IsStartItem item0 = fakeStartNonTerminalsSet.Contains(ntIdx_of_item0 item0)
467467
let IsKernelItem item0 = (IsStartItem item0 || dotIdx_of_item0 item0 <> 0)
468468

469-
let StringOfSym sym = match sym with PTerminal s -> "'" + termTab.OfIndex s + "'" | PNonTerminal s -> ntTab.OfIndex s
469+
let StringOfSym sym = match sym with PTerminal s -> "'" ^ termTab.OfIndex s ^ "'" | PNonTerminal s -> ntTab.OfIndex s
470470

471471
let OutputSym os sym = fprintf os "%s" (StringOfSym sym)
472472

0 commit comments

Comments
 (0)