diff --git a/Parser.Test/TestParser.fs b/Parser.Test/TestParser.fs index 7f5c8d2..e26757d 100644 --- a/Parser.Test/TestParser.fs +++ b/Parser.Test/TestParser.fs @@ -1056,6 +1056,305 @@ module Proto2 = |> should not' (throw typeof) [] +module StringImport = + + [] + let ``Resolve Import Statement`` () = + let files = + [ + "test.proto", + """ + syntax = "proto2"; + + import "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + + "import.proto", + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + ] |> Map.ofList + + let ast = files |> Parse.loadFromString "test.proto" + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]); + ("import.proto", [ + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + ]) + ]) + + [] + let ``Resolve Recursive Import Statements`` () = + let files = + [ + "test.proto", + """ + syntax = "proto2"; + + import "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + "import.proto", + """ + import "inner.proto"; + """ + "inner.proto", + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + ] |> Map.ofList + + let ast = files |> Parse.loadFromString "test.proto" + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]); + ("import.proto", []); + ("inner.proto", [ + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + ]) + ]) + + [] + let ``Missing import throws`` () = + + let files = + [ + "test.proto", + """ + import "missing.proto"; + """ + ] |> Map.ofList + + fun () -> + files + |> Parse.loadFromString "test.proto" + |> ignore + |> should throw typeof + + [] + let ``Resolve Public Import Statement`` () = + let files = + [ + "test.proto", + """ + syntax = "proto2"; + + import public "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + "import.proto", + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + ] |> Map.ofList + + let ast = files |> Parse.loadFromString "test.proto" + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]) + ]) + + [] + let ``Resolve recursive Public Import Statement`` () = + let files = + [ + "test.proto", + """ + syntax = "proto2"; + + import public "import.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + "import.proto", + """ + import public "inner.proto"; + """ + "inner.proto", + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + ] |> Map.ofList + + let ast = files |> Parse.loadFromString "test.proto" + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]) + ]) + + [] + let ``Missing public import throws`` () = + let files = + [ + "test.proto", + """ + import public "missing.proto"; + """ + ] |> Map.ofList + + fun () -> + files + |> Parse.loadFromString "test.proto" + |> ignore + |> should throw typeof + + + [] + let ``Resolve Weak Import Statement and ignore missing weak import`` () = + let files = + [ + "test.proto", + """ + syntax = "proto2"; + + import weak "import.proto"; + import weak "missing.proto"; + + message Test { + optional MyEnum a = 1; + } + """ + "import.proto", + """ + enum MyEnum { + DEFAULT = 0; + ONE = 1; + } + """ + ] |> Map.ofList + + let ast = files |> Parse.loadFromString "test.proto" + + ast |> should equal ( + [ ("test.proto", [ + TSyntax TProto2 + TMessage ("Test", + [ + TField ("a", TOptional, TIdent("MyEnum"), 1u, []) + ]) + ]); + ("import.proto", [ + TEnum ("MyEnum", + [ + TEnumField ("DEFAULT", 0, []) + TEnumField ("ONE", 1, []) + ]) + ]) + ]) + + +[] +module FileImport = + + open System + open System.IO + + let isMono = System.Type.GetType "Mono.Runtime" |> isNull |> not + + /// gets the path for a test file based on the relative path from the executing assembly + let testDir = + let solutionPath = + if isMono then + "../../../" + else + let codeBase = Reflection.Assembly.GetExecutingAssembly().CodeBase + let assemblyPath = DirectoryInfo (Uri codeBase).LocalPath + (assemblyPath.Parent.Parent.Parent.Parent).FullName + Path.Combine(solutionPath, "test") + + [] + let ``Resolve File Import`` () = + let dirs = + [ + testDir + ] + + let ast = + dirs |> Parse.loadFromFile "riak_kv.proto" + + // riak_kv.proto includes riak.proto + ast + |> List.length + |> should equal 2 + + // riak.proto contains a message definition for RpbGetServerInfoResp + let _, riakAst = ast.[1] + riakAst + |> List.exists(function + | TMessage( "RpbGetServerInfoResp", _ ) -> true + | _ -> false + ) + |> should equal true + + + module RegressionTests = [] diff --git a/Parser/Ast.fs b/Parser/Ast.fs index 33ad65d..3c3446a 100644 --- a/Parser/Ast.fs +++ b/Parser/Ast.fs @@ -24,14 +24,14 @@ and PStatement = | TSyntax s -> sprintf "TSyntax %A" s | TImport (s,v) -> sprintf "TImport (\"%s\",%A)" s v | TPackage id -> sprintf "TPackage \"%s\"" id - | TOption (n,v) -> sprintf "TOption (%s,%A)" n v + | TOption (n,v) -> sprintf "TOption (\"%s\",%A)" n v | TMessage (id, xs) -> - sprintf "TMessage (%s,[%s]" id + sprintf "TMessage (\"%s\",[%s]" id ( xs |> List.map (sprintf "%A") |> List.reduce (sprintf "%s;%s") ) - | TEnum (id, xs) -> sprintf "TEnum (%s,%A)" id xs - | TExtend (id,xs) -> sprintf "TEnum (%s,%A)" id xs - | TService (id,xs) -> sprintf "TService (%s,%A)" id xs + | TEnum (id, xs) -> sprintf "TEnum (\"%s\",%A)" id xs + | TExtend (id,xs) -> sprintf "TEnum (\"%s\",%A)" id xs + | TService (id,xs) -> sprintf "TService (\"%s\",%A)" id xs // TSyntax and PSyntax = diff --git a/Parser/Parser.fs b/Parser/Parser.fs index ab86dd1..1675102 100644 --- a/Parser/Parser.fs +++ b/Parser/Parser.fs @@ -720,3 +720,103 @@ module Parse = /// Parse proto from a file. Throws System.FormatException on failure. let fromFile fileName = fromFileWithParser Parsers.pProto fileName + + /// Resolve imports using provided `fetch` function, each parsed with supplied `parse` function. + /// Returns list of (filename, ast) tuples. + /// Public imports are replaced in-line. + /// Weak imports ignore failure to fetch the import. + /// TODO: Should `import weak "filename"` ignore ALL errors? Or just FileNotFound? + let rec resolveImports + (fetch: string -> string * 'a) + (parse: string * 'a -> string * Ast.PProto) + (name:string, source:Ast.PProto) + : (string * Ast.PProto) list = + + // Recursively inline public imports; imported file replaces import statement + // TODO: What should be done with the `syntax` line in an import public? + // TODO: Invalid? Override? Honor in a scope? + let rec inlineImports (name:string, xs:Ast.PProto) : (string * Ast.PProto) = + let processStatement x acc = + match x with + | Ast.TImport( name, Ast.TPublic ) -> + let _, imp = + name + |> fetch + |> parse + |> inlineImports + imp @ acc + | statement -> + statement :: acc + (name, List.foldBack processStatement xs []) + + // Wrap fetcher to ignore FNF exception on weak import + let selectFetcher = function + | Ast.TNormal -> fetch >> Some + | Ast.TPublic -> fun _ -> None + | Ast.TWeak -> fun source -> try Some(fetch source) with :? System.IO.FileNotFoundException -> None + + // Filter imports + let filterImports (name,ast) = + ast + |> List.choose (function + | Ast.TImport _ -> None + | s -> Some(s) + ) + |> fun ast -> (name,ast) + + // Load normal imports + // Returns list with original source, followed by of parsed imports + // Recursively calls resolveImports + let loadImports (name, xs:Ast.PProto) : (string*Ast.PProto) list = + xs + |> Seq.ofList + |> Seq.choose (function + | Ast.TImport (name, visibility) -> Some(selectFetcher visibility, name) + | _ -> None + ) + |> Seq.choose (fun (fetcher, name) -> + name + |> fetcher + |> Option.map parse + |> Option.map (resolveImports fetch parse) + ) + |> Seq.concat + |> List.ofSeq + |> fun imports -> filterImports (name,xs) :: imports + + (name, source) + |> inlineImports + |> loadImports + + let fetchFromMap filesMap = + fun filename -> + match filesMap |> Map.tryFind filename with + | Some(source) -> (filename, source) + | None -> raise <| System.IO.FileNotFoundException(filename) + + let fetchFromFile dirsList = + fun filename -> + let optFullPath = + dirsList + |> Seq.map (fun dir -> System.IO.Path.Combine(dir,filename)) + |> Seq.tryFind (fun fn -> System.IO.File.Exists(fn)) + match optFullPath with + | Some(fullPath) -> (fullPath,System.IO.File.OpenRead(fullPath)) + | None -> raise <| System.IO.FileNotFoundException(filename) + + let parseFromString (filename,string) = + (filename, fromString string) + + let parseFromStream (filename,stream) = + (filename, fromStream filename stream) + + let loadFromString fileName filesMap = + fetchFromMap filesMap fileName + |> parseFromString + |> resolveImports (fetchFromMap filesMap) parseFromString + + let loadFromFile fileName dirsList = + fetchFromFile dirsList fileName + |> parseFromStream + |> resolveImports (fetchFromFile dirsList) parseFromStream +