From 50f37dbba434be083701fc704d047e678f9eaeee Mon Sep 17 00:00:00 2001 From: James Hugard Date: Thu, 13 Oct 2016 09:11:42 -0700 Subject: [PATCH 1/6] WIP - start implementing import processing. Handles string import of normal and public, but doesn't yet handle weak. --- Parser.Test/TestParser.fs | 92 +++++++++++++++++++++++++++++++++++++++ Parser/Ast.fs | 10 ++--- Parser/Parser.fs | 55 +++++++++++++++++++++++ 3 files changed, 152 insertions(+), 5 deletions(-) diff --git a/Parser.Test/TestParser.fs b/Parser.Test/TestParser.fs index c044f37..43549cc 100644 --- a/Parser.Test/TestParser.fs +++ b/Parser.Test/TestParser.fs @@ -1043,3 +1043,95 @@ module Proto2 = } """ |> ignore |> should not' (throw typeof) + +[] +module Import = + + [] + let ``Resolve Import Statement`` () = + let src = """ + syntax = "proto2"; + + import "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + + let fetch = function + | "import.proto" -> + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + | s -> raise <| System.ArgumentOutOfRangeException(s) + + let ast = + src + |> Parse.fromString + |> fun ast -> ("test.proto", ast) + |> Parse.resolveImports Parse.fromString fetch + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TImport ("import.proto", TNormal) + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]); + ("import.proto", [ + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + ]) + ]) + + [] + let ``Resolve Public Import Statement`` () = + let src = """ + syntax = "proto2"; + + import public "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + + let fetch = function + | "import.proto" -> + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + | s -> raise <| System.ArgumentOutOfRangeException(s) + + let ast = + src + |> Parse.fromString + |> fun ast -> ("test.proto", ast) + |> Parse.resolveImports Parse.fromString fetch + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]) + ]) diff --git a/Parser/Ast.fs b/Parser/Ast.fs index 33ad65d..3c3446a 100644 --- a/Parser/Ast.fs +++ b/Parser/Ast.fs @@ -24,14 +24,14 @@ and PStatement = | TSyntax s -> sprintf "TSyntax %A" s | TImport (s,v) -> sprintf "TImport (\"%s\",%A)" s v | TPackage id -> sprintf "TPackage \"%s\"" id - | TOption (n,v) -> sprintf "TOption (%s,%A)" n v + | TOption (n,v) -> sprintf "TOption (\"%s\",%A)" n v | TMessage (id, xs) -> - sprintf "TMessage (%s,[%s]" id + sprintf "TMessage (\"%s\",[%s]" id ( xs |> List.map (sprintf "%A") |> List.reduce (sprintf "%s;%s") ) - | TEnum (id, xs) -> sprintf "TEnum (%s,%A)" id xs - | TExtend (id,xs) -> sprintf "TEnum (%s,%A)" id xs - | TService (id,xs) -> sprintf "TService (%s,%A)" id xs + | TEnum (id, xs) -> sprintf "TEnum (\"%s\",%A)" id xs + | TExtend (id,xs) -> sprintf "TEnum (\"%s\",%A)" id xs + | TService (id,xs) -> sprintf "TService (\"%s\",%A)" id xs // TSyntax and PSyntax = diff --git a/Parser/Parser.fs b/Parser/Parser.fs index 946a7fe..cb5cc75 100644 --- a/Parser/Parser.fs +++ b/Parser/Parser.fs @@ -720,3 +720,58 @@ module Parse = /// Parse proto from a file. Throws System.FormatException on failure. let fromFile fileName = fromFileWithParser Parsers.pProto fileName + + /// Resolve imports using provided `fetch` function, each parsed with supplied `parse` function. + /// Returns list of (filename, ast) tuples. + /// Public imports are replaced in-line. + /// Weak imports ignore failure to fetch the import. + /// TODO: Should `import weak "filename"` ignore ALL errors? Or just FileNotFound? + let rec resolveImports + (parse:'a -> Ast.PProto) + (fetch: string -> 'a) + (name:string,source:Ast.PProto) + : (string * Ast.PProto) list = + + // First, resolve public imports + let rec insertPublic (xs:Ast.PProto) : Ast.PProto = + let processStatement x acc = + match x with + | Ast.TImport( name, Ast.TPublic ) -> + let imp = + name + |> fetch + |> parse + |> insertPublic + imp @ acc + | statement -> + statement :: acc + List.foldBack processStatement xs [] + + // Next, try to resovle weak imports + + // TODO: implement - same as loadImports, but ignore fetch exceptions + + // Finally, resolve normal imports + let loadImports parse (xs:Ast.PProto) = + xs + |> List.choose (function + | Ast.TImport (name, Ast.TNormal) -> Some(name) + | _ -> None + ) + |> List.map (fun name -> + name + |> fetch + |> parse + |> fun ast -> (name, ast) + |> resolveImports parse fetch + ) + |> List.concat + + let source = + source + |> insertPublic + + source + |> loadImports parse + |> fun imports -> (name, source) :: imports + From d6ff68ddd087778925d78ba55059e874ae743623 Mon Sep 17 00:00:00 2001 From: James Hugard Date: Thu, 13 Oct 2016 23:16:08 -0700 Subject: [PATCH 2/6] WIP - implement normal, public, and weak imports; properly support recursive imports. Add unit tests. --- Parser.Test/TestParser.fs | 254 ++++++++++++++++++++++++++++++++++---- Parser/Parser.fs | 67 ++++++---- 2 files changed, 273 insertions(+), 48 deletions(-) diff --git a/Parser.Test/TestParser.fs b/Parser.Test/TestParser.fs index 43549cc..388aeb4 100644 --- a/Parser.Test/TestParser.fs +++ b/Parser.Test/TestParser.fs @@ -1059,26 +1059,30 @@ module Import = } """ - let fetch = function - | "import.proto" -> - """ - enum MyEnum { - DEFAULT = 0; - ONE = 1; - } - """ - | s -> raise <| System.ArgumentOutOfRangeException(s) + let fetch name = + let aux = function + | "import.proto" -> + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + | s -> raise <| System.IO.FileNotFoundException(s) + (name, aux name) + + let parse (name, s) = + (name, Parse.fromString s) let ast = src |> Parse.fromString |> fun ast -> ("test.proto", ast) - |> Parse.resolveImports Parse.fromString fetch + |> Parse.resolveImports fetch parse ast |> should equal ( [ ("test.proto", [ TSyntax TProto2 - TImport ("import.proto", TNormal) TMessage ("Test", [ TField ("a", TOptional, TIdent("MyEnum"), 1u, []) @@ -1093,6 +1097,83 @@ module Import = ]) ]) + [] + let ``Resolve Recursive Import Statements`` () = + let src = """ + syntax = "proto2"; + + import "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + + let fetch name = + let aux = function + | "import.proto" -> + """ + import "inner.proto"; + """ + | "inner.proto" -> + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + | s -> raise <| System.IO.FileNotFoundException(s) + (name, aux name) + + let parse (name, s) = + (name, Parse.fromString s) + + let ast = + src + |> Parse.fromString + |> fun ast -> ("test.proto", ast) + |> Parse.resolveImports fetch parse + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]); + ("import.proto", []); + ("inner.proto", [ + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + ]) + ]) + + [] + let ``Missing import throws`` () = + let src = """ + import "missing.proto"; + """ + + let fetch name = + let aux = function + | s -> raise <| System.IO.FileNotFoundException(s) + (name, aux name) + + let parse (name, s) = + (name, Parse.fromString s) + + fun () -> + src + |> Parse.fromString + |> fun ast -> ("test.proto", ast) + |> Parse.resolveImports fetch parse + |> ignore + |> should throw typeof + [] let ``Resolve Public Import Statement`` () = let src = """ @@ -1105,21 +1186,26 @@ module Import = } """ - let fetch = function - | "import.proto" -> - """ - enum MyEnum { - DEFAULT = 0; - ONE = 1; - } - """ - | s -> raise <| System.ArgumentOutOfRangeException(s) + let fetch name = + let aux = function + | "import.proto" -> + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + | s -> raise <| System.IO.FileNotFoundException(s) + (name, aux name) + + let parse (name, s) = + (name, Parse.fromString s) let ast = src |> Parse.fromString |> fun ast -> ("test.proto", ast) - |> Parse.resolveImports Parse.fromString fetch + |> Parse.resolveImports fetch parse ast |> should equal ( [ ("test.proto", [ @@ -1135,3 +1221,129 @@ module Import = ]) ]) ]) + + [] + let ``Resolve recursive Public Import Statement`` () = + let src = """ + syntax = "proto2"; + + import public "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + + let fetch name = + let aux = function + | "import.proto" -> + """ + import public "inner.proto"; + """ + | "inner.proto" -> + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + | s -> raise <| System.IO.FileNotFoundException(s) + (name, aux name) + + let parse (name, s) = + (name, Parse.fromString s) + + let ast = + src + |> Parse.fromString + |> fun ast -> ("test.proto", ast) + |> Parse.resolveImports fetch parse + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]) + ]) + + [] + let ``Missing public import throws`` () = + let src = """ + import "missing.proto"; + """ + + let fetch name = + let aux = function + | s -> raise <| System.IO.FileNotFoundException(s) + (name, aux name) + + let parse (name, s) = + (name, Parse.fromString s) + + fun () -> + src + |> Parse.fromString + |> fun ast -> ("test.proto", ast) + |> Parse.resolveImports fetch parse + |> ignore + |> should throw typeof + + + [] + let ``Resolve Weak Import Statement and ignore missing weak import`` () = + let src = """ + syntax = "proto2"; + + import weak "import.proto"; + import weak "missing.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + + let fetch name = + let aux = function + | "import.proto" -> + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + | s -> raise <| System.IO.FileNotFoundException(s) + (name, aux name) + + let parse (name, s) = + (name, Parse.fromString s) + + let ast = + src + |> Parse.fromString + |> fun ast -> ("test.proto", ast) + |> Parse.resolveImports fetch parse + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]); + ("import.proto", [ + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + ]) + ]) diff --git a/Parser/Parser.fs b/Parser/Parser.fs index cb5cc75..eb21423 100644 --- a/Parser/Parser.fs +++ b/Parser/Parser.fs @@ -727,51 +727,64 @@ module Parse = /// Weak imports ignore failure to fetch the import. /// TODO: Should `import weak "filename"` ignore ALL errors? Or just FileNotFound? let rec resolveImports - (parse:'a -> Ast.PProto) - (fetch: string -> 'a) - (name:string,source:Ast.PProto) + (fetch: string -> string * 'a) + (parse: string * 'a -> string * Ast.PProto) + (name:string, source:Ast.PProto) : (string * Ast.PProto) list = - // First, resolve public imports - let rec insertPublic (xs:Ast.PProto) : Ast.PProto = + // Recursively inline public imports; imported file replaces import statement + // TODO: What should be done with the `syntax` line in an import public? + // TODO: Invalid? Override? Honor in a scope? + let rec inlineImports (name:string, xs:Ast.PProto) : (string * Ast.PProto) = let processStatement x acc = match x with | Ast.TImport( name, Ast.TPublic ) -> - let imp = + let _, imp = name |> fetch |> parse - |> insertPublic + |> inlineImports imp @ acc | statement -> statement :: acc - List.foldBack processStatement xs [] + (name, List.foldBack processStatement xs []) - // Next, try to resovle weak imports + // Wrap fetcher to ignore FNF exception on weak import + let selectFetcher = function + | Ast.TNormal -> fun source -> Some(fetch source) + | Ast.TPublic -> fun _ -> None + | Ast.TWeak -> fun source -> try Some(fetch source) with :? System.IO.FileNotFoundException -> None - // TODO: implement - same as loadImports, but ignore fetch exceptions + // Filter imports + let filterImports (name,ast) = + ast + |> List.choose (function + | Ast.TImport _ -> None + | s -> Some(s) + ) + |> fun ast -> (name,ast) - // Finally, resolve normal imports - let loadImports parse (xs:Ast.PProto) = + // Load normal imports + // Returns list with original source, followed by of parsed imports + // Recursively calls resolveImports + let loadImports (name, xs:Ast.PProto) : (string*Ast.PProto) list = xs - |> List.choose (function - | Ast.TImport (name, Ast.TNormal) -> Some(name) + |> Seq.ofList + |> Seq.choose (function + | Ast.TImport (name, visibility) -> Some(selectFetcher visibility, name) | _ -> None ) - |> List.map (fun name -> + |> Seq.choose (fun (fetcher, name) -> name - |> fetch - |> parse - |> fun ast -> (name, ast) - |> resolveImports parse fetch + |> fetcher + |> Option.map parse + |> Option.map (resolveImports fetch parse) ) - |> List.concat - - let source = - source - |> insertPublic + |> Seq.concat + |> List.ofSeq + |> fun imports -> filterImports (name,xs) :: imports - source - |> loadImports parse - |> fun imports -> (name, source) :: imports + (name, source) + |> inlineImports + |> loadImports From de5ff90f3c507bcec9d50abbce0d2cfa0b0351e0 Mon Sep 17 00:00:00 2001 From: James Hugard Date: Thu, 13 Oct 2016 23:30:15 -0700 Subject: [PATCH 3/6] Refactor import unit tests. --- Parser.Test/TestParser.fs | 153 ++++++++++++++++++-------------------- 1 file changed, 73 insertions(+), 80 deletions(-) diff --git a/Parser.Test/TestParser.fs b/Parser.Test/TestParser.fs index 388aeb4..2b2cb46 100644 --- a/Parser.Test/TestParser.fs +++ b/Parser.Test/TestParser.fs @@ -1049,18 +1049,18 @@ module Import = [] let ``Resolve Import Statement`` () = - let src = """ - syntax = "proto2"; - - import "import.proto"; - - message Test { - optional MyEnum a = 1; - } - """ - let fetch name = let aux = function + | "test.proto" -> + """ + syntax = "proto2"; + + import "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ | "import.proto" -> """ enum MyEnum { @@ -1075,9 +1075,8 @@ module Import = (name, Parse.fromString s) let ast = - src - |> Parse.fromString - |> fun ast -> ("test.proto", ast) + fetch "test.proto" + |> parse |> Parse.resolveImports fetch parse ast |> should equal ( @@ -1099,18 +1098,18 @@ module Import = [] let ``Resolve Recursive Import Statements`` () = - let src = """ - syntax = "proto2"; - - import "import.proto"; - - message Test { - optional MyEnum a = 1; - } - """ - let fetch name = let aux = function + | "test.proto" -> + """ + syntax = "proto2"; + + import "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ | "import.proto" -> """ import "inner.proto"; @@ -1129,9 +1128,8 @@ module Import = (name, Parse.fromString s) let ast = - src - |> Parse.fromString - |> fun ast -> ("test.proto", ast) + fetch "test.proto" + |> parse |> Parse.resolveImports fetch parse ast |> should equal ( @@ -1154,12 +1152,12 @@ module Import = [] let ``Missing import throws`` () = - let src = """ - import "missing.proto"; - """ - let fetch name = let aux = function + | "test.proto" -> + """ + import "missing.proto"; + """ | s -> raise <| System.IO.FileNotFoundException(s) (name, aux name) @@ -1167,27 +1165,26 @@ module Import = (name, Parse.fromString s) fun () -> - src - |> Parse.fromString - |> fun ast -> ("test.proto", ast) + fetch "test.proto" + |> parse |> Parse.resolveImports fetch parse |> ignore |> should throw typeof [] let ``Resolve Public Import Statement`` () = - let src = """ - syntax = "proto2"; - - import public "import.proto"; - - message Test { - optional MyEnum a = 1; - } - """ - let fetch name = let aux = function + | "test.proto" -> + """ + syntax = "proto2"; + + import public "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ | "import.proto" -> """ enum MyEnum { @@ -1202,9 +1199,8 @@ module Import = (name, Parse.fromString s) let ast = - src - |> Parse.fromString - |> fun ast -> ("test.proto", ast) + fetch "test.proto" + |> parse |> Parse.resolveImports fetch parse ast |> should equal ( @@ -1224,18 +1220,18 @@ module Import = [] let ``Resolve recursive Public Import Statement`` () = - let src = """ - syntax = "proto2"; - - import public "import.proto"; - - message Test { - optional MyEnum a = 1; - } - """ - let fetch name = let aux = function + | "test.proto" -> + """ + syntax = "proto2"; + + import public "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ | "import.proto" -> """ import public "inner.proto"; @@ -1254,9 +1250,8 @@ module Import = (name, Parse.fromString s) let ast = - src - |> Parse.fromString - |> fun ast -> ("test.proto", ast) + fetch "test.proto" + |> parse |> Parse.resolveImports fetch parse ast |> should equal ( @@ -1276,12 +1271,12 @@ module Import = [] let ``Missing public import throws`` () = - let src = """ - import "missing.proto"; - """ - let fetch name = let aux = function + | "test.proto" -> + """ + import "missing.proto"; + """ | s -> raise <| System.IO.FileNotFoundException(s) (name, aux name) @@ -1289,9 +1284,8 @@ module Import = (name, Parse.fromString s) fun () -> - src - |> Parse.fromString - |> fun ast -> ("test.proto", ast) + fetch "test.proto" + |> parse |> Parse.resolveImports fetch parse |> ignore |> should throw typeof @@ -1299,19 +1293,19 @@ module Import = [] let ``Resolve Weak Import Statement and ignore missing weak import`` () = - let src = """ - syntax = "proto2"; - - import weak "import.proto"; - import weak "missing.proto"; - - message Test { - optional MyEnum a = 1; - } - """ - let fetch name = let aux = function + | "test.proto" -> + """ + syntax = "proto2"; + + import weak "import.proto"; + import weak "missing.proto"; + + message Test { + optional MyEnum a = 1; + } + """ | "import.proto" -> """ enum MyEnum { @@ -1326,9 +1320,8 @@ module Import = (name, Parse.fromString s) let ast = - src - |> Parse.fromString - |> fun ast -> ("test.proto", ast) + fetch "test.proto" + |> parse |> Parse.resolveImports fetch parse ast |> should equal ( From 2f0a57848ae9f1471a7e842514d9231c0caf8fd4 Mon Sep 17 00:00:00 2001 From: James Hugard Date: Thu, 13 Oct 2016 23:33:50 -0700 Subject: [PATCH 4/6] Correct 'missing public import' test. --- Parser.Test/TestParser.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser.Test/TestParser.fs b/Parser.Test/TestParser.fs index 2b2cb46..20ec116 100644 --- a/Parser.Test/TestParser.fs +++ b/Parser.Test/TestParser.fs @@ -1275,7 +1275,7 @@ module Import = let aux = function | "test.proto" -> """ - import "missing.proto"; + import public "missing.proto"; """ | s -> raise <| System.IO.FileNotFoundException(s) (name, aux name) From 70e1208e9d122ee0708595f47f8d75549efd26a2 Mon Sep 17 00:00:00 2001 From: jhugard Date: Fri, 14 Oct 2016 09:02:31 -0700 Subject: [PATCH 5/6] WIP - add Parse.loadFromString and loadFromFile . Refactor unit tests. --- Parser.Test/TestParser.fs | 223 +++++++++++++++----------------------- Parser/Parser.fs | 34 +++++- 2 files changed, 123 insertions(+), 134 deletions(-) diff --git a/Parser.Test/TestParser.fs b/Parser.Test/TestParser.fs index 20ec116..97ceb66 100644 --- a/Parser.Test/TestParser.fs +++ b/Parser.Test/TestParser.fs @@ -1049,9 +1049,9 @@ module Import = [] let ``Resolve Import Statement`` () = - let fetch name = - let aux = function - | "test.proto" -> + let files = + [ + "test.proto", """ syntax = "proto2"; @@ -1061,24 +1061,18 @@ module Import = optional MyEnum a = 1; } """ - | "import.proto" -> + + "import.proto", """ enum MyEnum { DEFAULT = 0; ONE = 1; } """ - | s -> raise <| System.IO.FileNotFoundException(s) - (name, aux name) - - let parse (name, s) = - (name, Parse.fromString s) - - let ast = - fetch "test.proto" - |> parse - |> Parse.resolveImports fetch parse - + ] |> Map.ofList + + let ast = files |> Parse.loadFromString "test.proto" + ast |> should equal ( [ ("test.proto", [ TSyntax TProto2 @@ -1098,9 +1092,9 @@ module Import = [] let ``Resolve Recursive Import Statements`` () = - let fetch name = - let aux = function - | "test.proto" -> + let files = + [ + "test.proto", """ syntax = "proto2"; @@ -1110,27 +1104,20 @@ module Import = optional MyEnum a = 1; } """ - | "import.proto" -> + "import.proto", """ import "inner.proto"; """ - | "inner.proto" -> + "inner.proto", """ enum MyEnum { DEFAULT = 0; ONE = 1; } """ - | s -> raise <| System.IO.FileNotFoundException(s) - (name, aux name) - - let parse (name, s) = - (name, Parse.fromString s) + ] |> Map.ofList - let ast = - fetch "test.proto" - |> parse - |> Parse.resolveImports fetch parse + let ast = files |> Parse.loadFromString "test.proto" ast |> should equal ( [ ("test.proto", [ @@ -1152,56 +1139,45 @@ module Import = [] let ``Missing import throws`` () = - let fetch name = - let aux = function - | "test.proto" -> + + let files = + [ + "test.proto", """ import "missing.proto"; """ - | s -> raise <| System.IO.FileNotFoundException(s) - (name, aux name) - - let parse (name, s) = - (name, Parse.fromString s) + ] |> Map.ofList fun () -> - fetch "test.proto" - |> parse - |> Parse.resolveImports fetch parse + files + |> Parse.loadFromString "test.proto" |> ignore |> should throw typeof [] let ``Resolve Public Import Statement`` () = - let fetch name = - let aux = function - | "test.proto" -> - """ - syntax = "proto2"; - - import public "import.proto"; + let files = + [ + "test.proto", + """ + syntax = "proto2"; - message Test { - optional MyEnum a = 1; - } - """ - | "import.proto" -> - """ - enum MyEnum { - DEFAULT = 0; - ONE = 1; - } - """ - | s -> raise <| System.IO.FileNotFoundException(s) - (name, aux name) + import public "import.proto"; - let parse (name, s) = - (name, Parse.fromString s) + message Test { + optional MyEnum a = 1; + } + """ + "import.proto", + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + ] |> Map.ofList - let ast = - fetch "test.proto" - |> parse - |> Parse.resolveImports fetch parse + let ast = files |> Parse.loadFromString "test.proto" ast |> should equal ( [ ("test.proto", [ @@ -1220,39 +1196,32 @@ module Import = [] let ``Resolve recursive Public Import Statement`` () = - let fetch name = - let aux = function - | "test.proto" -> - """ - syntax = "proto2"; + let files = + [ + "test.proto", + """ + syntax = "proto2"; - import public "import.proto"; + import public "import.proto"; - message Test { - optional MyEnum a = 1; - } - """ - | "import.proto" -> - """ - import public "inner.proto"; - """ - | "inner.proto" -> - """ - enum MyEnum { - DEFAULT = 0; - ONE = 1; - } + message Test { + optional MyEnum a = 1; + } + """ + "import.proto", + """ + import public "inner.proto"; + """ + "inner.proto", + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } """ - | s -> raise <| System.IO.FileNotFoundException(s) - (name, aux name) + ] |> Map.ofList - let parse (name, s) = - (name, Parse.fromString s) - - let ast = - fetch "test.proto" - |> parse - |> Parse.resolveImports fetch parse + let ast = files |> Parse.loadFromString "test.proto" ast |> should equal ( [ ("test.proto", [ @@ -1271,58 +1240,46 @@ module Import = [] let ``Missing public import throws`` () = - let fetch name = - let aux = function - | "test.proto" -> + let files = + [ + "test.proto", """ import public "missing.proto"; """ - | s -> raise <| System.IO.FileNotFoundException(s) - (name, aux name) - - let parse (name, s) = - (name, Parse.fromString s) + ] |> Map.ofList fun () -> - fetch "test.proto" - |> parse - |> Parse.resolveImports fetch parse + files + |> Parse.loadFromString "test.proto" |> ignore |> should throw typeof [] let ``Resolve Weak Import Statement and ignore missing weak import`` () = - let fetch name = - let aux = function - | "test.proto" -> - """ - syntax = "proto2"; + let files = + [ + "test.proto", + """ + syntax = "proto2"; - import weak "import.proto"; - import weak "missing.proto"; + import weak "import.proto"; + import weak "missing.proto"; - message Test { - optional MyEnum a = 1; - } - """ - | "import.proto" -> - """ - enum MyEnum { - DEFAULT = 0; - ONE = 1; - } - """ - | s -> raise <| System.IO.FileNotFoundException(s) - (name, aux name) - - let parse (name, s) = - (name, Parse.fromString s) + message Test { + optional MyEnum a = 1; + } + """ + "import.proto", + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + ] |> Map.ofList - let ast = - fetch "test.proto" - |> parse - |> Parse.resolveImports fetch parse + let ast = files |> Parse.loadFromString "test.proto" ast |> should equal ( [ ("test.proto", [ diff --git a/Parser/Parser.fs b/Parser/Parser.fs index eb21423..004cdca 100644 --- a/Parser/Parser.fs +++ b/Parser/Parser.fs @@ -751,7 +751,7 @@ module Parse = // Wrap fetcher to ignore FNF exception on weak import let selectFetcher = function - | Ast.TNormal -> fun source -> Some(fetch source) + | Ast.TNormal -> fetch >> Some | Ast.TPublic -> fun _ -> None | Ast.TWeak -> fun source -> try Some(fetch source) with :? System.IO.FileNotFoundException -> None @@ -788,3 +788,35 @@ module Parse = |> inlineImports |> loadImports + let fetchFromMap filesMap = + fun filename -> + match filesMap |> Map.tryFind filename with + | Some(source) -> (filename, source) + | None -> raise <| System.IO.FileNotFoundException(filename) + + let fetchFromFile dirsList = + fun filename -> + let optFullPath = + dirsList + |> Seq.map (fun dir -> System.IO.Path.Combine(dir,filename)) + |> Seq.tryFind (fun fn -> System.IO.File.Exists(fn)) + match optFullPath with + | Some(fullPath) -> (fullPath,System.IO.File.OpenRead(fullPath)) + | None -> raise <| System.IO.FileNotFoundException(filename) + + let parseFromString (filename,string) = + (filename, fromString string) + + let parseFromStream (filename,stream) = + (filename, fromStream filename stream) + + let loadFromString fileName filesMap = + fetchFromMap filesMap fileName + |> parseFromString + |> resolveImports (fetchFromMap filesMap) parseFromString + + let loadFromFile fileName dirsList = + fetchFromFile dirsList fileName + |> parseFromStream + |> resolveImports (fetchFromFile dirsList) parseFromStream + From 97163028b1daf23838d456438b2bf5725da22daa Mon Sep 17 00:00:00 2001 From: jhugard Date: Fri, 14 Oct 2016 22:22:40 -0700 Subject: [PATCH 6/6] Add import file unit test. --- Parser.Test/TestParser.fs | 49 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/Parser.Test/TestParser.fs b/Parser.Test/TestParser.fs index 97ceb66..01faac1 100644 --- a/Parser.Test/TestParser.fs +++ b/Parser.Test/TestParser.fs @@ -1045,7 +1045,7 @@ module Proto2 = |> should not' (throw typeof) [] -module Import = +module StringImport = [] let ``Resolve Import Statement`` () = @@ -1297,3 +1297,50 @@ module Import = ]) ]) ]) + + +[] +module FileImport = + + open System + open System.IO + + let isMono = System.Type.GetType "Mono.Runtime" |> isNull |> not + + /// gets the path for a test file based on the relative path from the executing assembly + let testDir = + let solutionPath = + if isMono then + "../../../" + else + let codeBase = Reflection.Assembly.GetExecutingAssembly().CodeBase + let assemblyPath = DirectoryInfo (Uri codeBase).LocalPath + (assemblyPath.Parent.Parent.Parent.Parent).FullName + Path.Combine(solutionPath, "test") + + [] + let ``Resolve File Import`` () = + let dirs = + [ + testDir + ] + + let ast = + dirs |> Parse.loadFromFile "riak_kv.proto" + + // riak_kv.proto includes riak.proto + ast + |> List.length + |> should equal 2 + + // riak.proto contains a message definition for RpbGetServerInfoResp + let _, riakAst = ast.[1] + riakAst + |> List.exists(function + | TMessage( "RpbGetServerInfoResp", _ ) -> true + | _ -> false + ) + |> should equal true + + +