From 7574f166008bb45c0df97315aae7907ac25f8602 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Mon, 4 Jan 2021 14:00:41 -0500 Subject: [PATCH] [generator] Add `generator --with-javadoc-xml=FILE` support. (#687) Fixes: https://github.com/xamarin/java.interop/issues/642 Context: https://github.com/xamarin/xamarin-android/issues/4789 Context: https://github.com/xamarin/xamarin-android/issues/5200 Commit 69e1b80a added `tools/java-source-utils`, which along with b588ef50, can parse Java source code and extract Javadoc comments from Android API-30 sources into an XML file: into an XML file: # API-30 `sources` package contains both `Object.java` and `Object.annotated.java`; # Skip the `.annotated.java` files $ find $HOME/android-toolchain/sdk/platforms/android-30/src/{android,java,javax,org} -iname \*.java \ | grep -v '\.annotated\.' > sources.txt $ java -jar java-source-utils.jar -v \ --source "$HOME/android-toolchain/sdk/platforms/android-30/src" \ --output-javadoc android-javadoc.xml \ @sources.txt What can we *do* with the generated `android-javadoc.xml`? `android-javadoc.xml` contains parameter names, and thus can be used with `class-parse --docspath`; see commit 806082f2. What we *really* want to do is make the Javadoc information *useful* to consumers of the binding assembly. This means that we want a C# XML Documentation file for the binding assembly. The most straightforward way to get a C# XML Documentation File is to emit [C# XML Documentation Comments][0] into the binding source code! Add a new `generator --with-javadoc-xml=FILE` option. When specified, `FILE` will be treated as an XML file containing the output from `java-source-utils.jar --output-javadoc` (69e1b80a), and all `` elements within the XML file will be associated with C# types and members to emit, based on the `//@jni-signature` and `//@name` attributes, as appropriate. When the bindings are written to disk, the Javadoc comments will be translated to C# XML Documentation comments, in a "best effort" basis. (THIS WILL BE INCOMPLETE.) To perform the Javadoc-to-C# XML Documentation comments conversion, add a new Irony-based grammar to `Java.Interop.Tools.JavaSource.dll`, in the new `Java.Interop.Tools.JavaSource.SourceJavadocToXmldocParser` type, which parses the Javadoc content and translates to XML. In addition to transforming the Javadoc comments into C# XML documentation comments, we *also* want to provide "upstream information" in the form of: 1. A URL to the corresponding online Javadoc HTML documentation. 2. A copyright notice disclaimer. Allow provision of this information by updating `java-source-utils.jar` to support the new options: --doc-copyright FILE Copyright information for Javadoc. Should be in mdoc(5) XML, to be held within . Stored in //javadoc-metadata/copyright. --doc-url-prefix URL Base URL for links to documentation. Stored in //javadoc-metadata/link/@prefix. --doc-url-style STYLE STYLE of URLs to generate for member links. Stored in //javadoc-metadata/link/@style. Supported styles include: - developer.android.com/reference@2020-Nov The new `/api/javadoc-metadata/link@prefix` and `/api/javadoc-metadata/link@style` XML attributes, stored within `java-source-utils.jar --output-javadoc` XML output, allow construction of a URL to the Java member. For example, given: java -jar java-source-utils.jar \ --doc-url-prefix https://developer.android.com/reference \ --doc-url-style developer.android.com/reference@2020-Nov Then `generator` can emit the C# documentation comment for `java.lang.Object.equals(Object)`: /// /// Java documentation for java.lang.Object.equals(java.lang.Object). /// The copyright notice disclaimer is supported by `java-source-utils.jar --doc-copyright FILE`; the contents of `FILE` are inserted into the `/api/javadoc-metadata/copyright` element, and will be copied into the output of every C# XML documentation block. Example output is at: * Unfortunately, converting Javadoc to C# XML Documentation Comments is not a "zero cost" operation. Add a new `generator --doc-comment-verbosity=STYLE` option to control how "complete" the generated documentation comments are: --doc-comment-verbosity=STYLE STYLE of C# documentation comments to emit. Defaults to `full`. STYLE may be: * `intellisense`: emit , , , . * `full`: plus , , ... Using `--doc-comment-verbosity=full` will *most* impact build times. TODO: * `SourceJavadocToXmldocParser` doesn't support many constructs. [0]: https://docs.microsoft.com/en-us/dotnet/csharp/codedoc --- .../IronyExtensions.cs | 28 ++ .../JavadocInfo.cs | 31 ++ ...avadocToXmldocGrammar.BlockTagsBnfTerms.cs | 239 ++++++++++++ ...urceJavadocToXmldocGrammar.HtmlBnfTerms.cs | 326 +++++++++++++++++ ...vadocToXmldocGrammar.InlineTagsBnfTerms.cs | 113 ++++++ .../SourceJavadocToXmldocGrammar.cs | 134 +++++++ .../SourceJavadocToXmldocParser.cs | 164 +++++++++ src/Java.Interop.Tools.JavaSource/README.md | 22 ++ ...cToXmldocGrammar.BlockTagsBnfTermsTests.cs | 137 +++++++ ...avadocToXmldocGrammar.HtmlBnfTermsTests.cs | 51 +++ ...ToXmldocGrammar.InlineTagsBnfTermsTests.cs | 92 +++++ .../SourceJavadocToXmldocGrammarFixture.cs | 64 ++++ .../SourceJavadocToXmldocParserTests.cs | 175 +++++++++ tools/generator/CodeGenerator.cs | 2 + tools/generator/CodeGeneratorOptions.cs | 25 ++ .../XmlApiImporter.cs | 1 + .../Field.cs | 3 + .../GenBase.cs | 2 + .../Javadoc.cs | 79 ++++ .../JavadocInfo.cs | 344 ++++++++++++++++++ .../MethodBase.cs | 2 + .../JavadocFixups.cs | 108 ++++++ tools/generator/SourceWriters/BoundClass.cs | 1 + .../SourceWriters/BoundConstructor.cs | 1 + tools/generator/SourceWriters/BoundField.cs | 1 + .../SourceWriters/BoundFieldAsProperty.cs | 1 + .../generator/SourceWriters/BoundInterface.cs | 1 + .../BoundInterfaceMethodDeclaration.cs | 2 + tools/generator/SourceWriters/BoundMethod.cs | 2 + .../BoundMethodAbstractDeclaration.cs | 2 + .../BoundMethodStringOverload.cs | 2 + .../generator/SourceWriters/BoundProperty.cs | 59 +++ tools/generator/generator.csproj | 2 + .../main/java/com/microsoft/android/App.java | 7 +- .../android/JavaSourceUtilsOptions.java | 62 +++- .../android/JavadocXmlGenerator.java | 86 ++++- .../android/JavadocXmlGeneratorTest.java | 5 +- 37 files changed, 2354 insertions(+), 22 deletions(-) create mode 100644 src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/IronyExtensions.cs create mode 100644 src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/JavadocInfo.cs create mode 100644 src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs create mode 100644 src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs create mode 100644 src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs create mode 100644 src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.cs create mode 100644 src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs create mode 100644 src/Java.Interop.Tools.JavaSource/README.md create mode 100644 tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs create mode 100644 tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs create mode 100644 tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.InlineTagsBnfTermsTests.cs create mode 100644 tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammarFixture.cs create mode 100644 tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs create mode 100644 tools/generator/Java.Interop.Tools.Generator.ObjectModel/Javadoc.cs create mode 100644 tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs create mode 100644 tools/generator/Java.Interop.Tools.Generator.Transformation/JavadocFixups.cs diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/IronyExtensions.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/IronyExtensions.cs new file mode 100644 index 000000000..b4260935d --- /dev/null +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/IronyExtensions.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Irony.Ast; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource { + + static class IronyExtensions { + + public static void MakePlusRule (this NonTerminal star, Grammar grammar, BnfTerm delimiter) + { + star.Rule = grammar.MakePlusRule (star, delimiter); + } + + public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm delimiter, BnfTerm of) + { + star.Rule = grammar.MakeStarRule (star, delimiter, of); + } + + public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm of) + { + star.Rule = grammar.MakeStarRule (star, of); + } + } +} diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/JavadocInfo.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/JavadocInfo.cs new file mode 100644 index 000000000..675193a09 --- /dev/null +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/JavadocInfo.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +using Irony.Ast; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource { + + sealed class JavadocInfo { + public readonly ICollection Exceptions = new Collection (); + public readonly ICollection Extra = new Collection (); + public readonly ICollection Remarks = new Collection (); + public readonly ICollection Parameters = new Collection (); + public readonly ICollection Returns = new Collection (); + + public override string ToString () + { + return new XElement ("Javadoc", + new XElement (nameof (Parameters), Parameters), + new XElement (nameof (Remarks), Remarks), + new XElement (nameof (Returns), Returns), + new XElement (nameof (Exceptions), Exceptions), + new XElement (nameof (Extra), Extra)) + .ToString (); + } + } +} diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs new file mode 100644 index 000000000..642e02da6 --- /dev/null +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs @@ -0,0 +1,239 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +using Irony.Ast; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource { + + public partial class SourceJavadocToXmldocGrammar { + + // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#javadoctags + public class BlockTagsBnfTerms { + + internal BlockTagsBnfTerms () + { + } + + internal void CreateRules (SourceJavadocToXmldocGrammar grammar) + { + AllBlockTerms.Rule = AuthorDeclaration + | ApiSinceDeclaration + | DeprecatedDeclaration + | DeprecatedSinceDeclaration + | ExceptionDeclaration + | ParamDeclaration + | ReturnDeclaration + | SeeDeclaration + | SerialDataDeclaration + | SerialFieldDeclaration + | SinceDeclaration + | ThrowsDeclaration + | UnknownTagDeclaration + | VersionDeclaration + ; + BlockValue.Rule = grammar.HtmlTerms.ParsedCharacterData + | grammar.HtmlTerms.InlineDeclaration + ; + BlockValues.MakePlusRule (grammar, BlockValue); + + AuthorDeclaration.Rule = "@author" + BlockValues; + AuthorDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.AuthorTag)) + return; + // Ignore; not sure how best to convert to Xmldoc + FinishParse (context, parseNode); + }; + + ApiSinceDeclaration.Rule = "@apiSince" + BlockValues; + ApiSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.SinceTag)) { + return; + } + var p = new XElement ("para", "Added in API level ", AstNodeToXmlContent (parseNode.ChildNodes [1]), "."); + FinishParse (context, parseNode).Remarks.Add (p); + parseNode.AstNode = p; + }; + + DeprecatedDeclaration.Rule = "@deprecated" + BlockValues; + DeprecatedDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.DeprecatedTag)) { + return; + } + var p = new XElement ("para", "This member is deprecated. ", AstNodeToXmlContent (parseNode.ChildNodes [1])); + FinishParse (context, parseNode).Remarks.Add (p); + parseNode.AstNode = p; + }; + + DeprecatedSinceDeclaration.Rule = "@deprecatedSince" + BlockValues; + DeprecatedSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.DeprecatedTag)) { + return; + } + var p = new XElement ("para", "This member was deprecated in API level ", AstNodeToXmlContent (parseNode.ChildNodes [1]), "."); + FinishParse (context, parseNode).Remarks.Add (p); + parseNode.AstNode = p; + }; + + var nonSpaceTerm = new RegexBasedTerminal ("[^ ]", "[^ ]+") { + AstConfig = new AstNodeConfig { + NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value, + }, + }; + + ExceptionDeclaration.Rule = "@exception" + nonSpaceTerm + BlockValues; + ExceptionDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.ExceptionTag)) { + return; + } + // TODO: convert `nonSpaceTerm` into a proper CREF + var e = new XElement ("exception", + new XAttribute ("cref", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))), + AstNodeToXmlContent (parseNode.ChildNodes [2])); + FinishParse (context, parseNode).Exceptions.Add (e); + parseNode.AstNode = e; + }; + + ParamDeclaration.Rule = "@param" + nonSpaceTerm + BlockValues; + ParamDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.ParamTag)) { + return; + } + var p = new XElement ("param", + new XAttribute ("name", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))), + AstNodeToXmlContent (parseNode.ChildNodes [2])); + FinishParse (context, parseNode).Parameters.Add (p); + parseNode.AstNode = p; + }; + + ReturnDeclaration.Rule = "@return" + BlockValues; + ReturnDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.ReturnTag)) { + return; + } + var r = new XElement ("returns", + AstNodeToXmlContent (parseNode.ChildNodes [1])); + FinishParse (context, parseNode).Returns.Add (r); + parseNode.AstNode = r; + }; + + SeeDeclaration.Rule = "@see" + BlockValues; + SeeDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.SeeTag)) { + return; + } + // TODO: @see supports multiple forms; see: https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#see + var e = new XElement ("seealso", + new XAttribute ("cref", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1])))); + FinishParse (context, parseNode).Extra.Add (e); + parseNode.AstNode = e; + }; + + SinceDeclaration.Rule = "@since" + BlockValues; + SinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.SinceTag)) { + return; + } + var p = new XElement ("para", "Added in ", AstNodeToXmlContent (parseNode.ChildNodes [1]), "."); + FinishParse (context, parseNode).Remarks.Add (p); + parseNode.AstNode = p; + }; + + ThrowsDeclaration.Rule = "@throws" + nonSpaceTerm + BlockValues; + ThrowsDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.ExceptionTag)) { + return; + } + // TODO: convert `nonSpaceTerm` into a proper CREF + var e = new XElement ("exception", + new XAttribute ("cref", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))), + AstNodeToXmlContent (parseNode.ChildNodes [2])); + FinishParse (context, parseNode).Exceptions.Add (e); + parseNode.AstNode = e; + }; + + // Ignore serialization informatino + SerialDeclaration.Rule = "@serial" + BlockValues; + SerialDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.SerialTag)) { + return; + } + FinishParse (context, parseNode); + }; + + SerialDataDeclaration.Rule = "@serialData" + BlockValues; + SerialDataDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.SerialTag)) { + return; + } + FinishParse (context, parseNode); + }; + + SerialFieldDeclaration.Rule = "@serialField" + BlockValues; + SerialFieldDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.SerialTag)) { + return; + } + FinishParse (context, parseNode); + }; + + var unknownTagTerminal = new RegexBasedTerminal ("@[unknown]", @"@\S+") { + Priority = TerminalPriority.Low, + }; + unknownTagTerminal.AstConfig.NodeCreator = (context, parseNode) => + parseNode.AstNode = parseNode.Token.Value.ToString (); + + + UnknownTagDeclaration.Rule = unknownTagTerminal + BlockValues; + UnknownTagDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.Remarks)) { + return; + } + Console.WriteLine ($"# Unsupported @block-tag value: {parseNode.ChildNodes [0].AstNode}"); + FinishParse (context, parseNode); + }; + + // Ignore Version + VersionDeclaration.Rule = "@version" + BlockValues; + VersionDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.VersionTag)) { + return; + } + FinishParse (context, parseNode); + }; + } + + public readonly NonTerminal AllBlockTerms = new NonTerminal (nameof (AllBlockTerms), ConcatChildNodes); + + public readonly Terminal Cdata = new CharacterDataTerminal ("#CDATA", preserveLeadingWhitespace: true); +/* + public readonly Terminal Cdata = new RegexBasedTerminal (nameof (BlockValue), "[^<]*") { + AstConfig = new AstNodeConfig { + NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (), + }, + }; + */ + + public readonly NonTerminal BlockValue = new NonTerminal (nameof (BlockValue), ConcatChildNodes); + public readonly NonTerminal BlockValues = new NonTerminal (nameof (BlockValues), ConcatChildNodes); + public readonly NonTerminal AuthorDeclaration = new NonTerminal (nameof (AuthorDeclaration)); + public readonly NonTerminal ApiSinceDeclaration = new NonTerminal (nameof (ApiSinceDeclaration)); + public readonly NonTerminal DeprecatedDeclaration = new NonTerminal (nameof (DeprecatedDeclaration)); + public readonly NonTerminal DeprecatedSinceDeclaration = new NonTerminal (nameof (DeprecatedSinceDeclaration)); + public readonly NonTerminal ExceptionDeclaration = new NonTerminal (nameof (ExceptionDeclaration)); + public readonly NonTerminal ParamDeclaration = new NonTerminal (nameof (ParamDeclaration)); + public readonly NonTerminal ReturnDeclaration = new NonTerminal (nameof (ReturnDeclaration)); + public readonly NonTerminal SeeDeclaration = new NonTerminal (nameof (SeeDeclaration)); + public readonly NonTerminal SerialDeclaration = new NonTerminal (nameof (SerialDeclaration)); + public readonly NonTerminal SerialDataDeclaration = new NonTerminal (nameof (SerialDataDeclaration)); + public readonly NonTerminal SerialFieldDeclaration = new NonTerminal (nameof (SerialFieldDeclaration)); + public readonly NonTerminal SinceDeclaration = new NonTerminal (nameof (SinceDeclaration)); + public readonly NonTerminal ThrowsDeclaration = new NonTerminal (nameof (ThrowsDeclaration)); + public readonly NonTerminal UnknownTagDeclaration = new NonTerminal (nameof (UnknownTagDeclaration)); + public readonly NonTerminal VersionDeclaration = new NonTerminal (nameof (VersionDeclaration)); + } + } +} diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs new file mode 100644 index 000000000..87259a1e0 --- /dev/null +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs @@ -0,0 +1,326 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +using Irony.Ast; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource { + + using static IronyExtensions; + + public partial class SourceJavadocToXmldocGrammar { + + public class HtmlBnfTerms { + internal HtmlBnfTerms () + { + } + + internal void CreateRules (SourceJavadocToXmldocGrammar grammar) + { + AllHtmlTerms.Rule = TopLevelInlineDeclaration + | PBlockDeclaration + | PreBlockDeclaration + ; + + var inlineDeclaration = new NonTerminal ("", ConcatChildNodes) { + Rule = ParsedCharacterData + | FontStyleDeclaration + /* + | PhraseDeclaration + | SpecialDeclaration + | FormCtrlDeclaration + */ + | grammar.InlineTagsTerms.AllInlineTerms + | UnknownHtmlElementStart + , + }; + var inlineDeclarations = new NonTerminal ("*", ConcatChildNodes); + inlineDeclarations.MakePlusRule (grammar, inlineDeclaration); + + InlineDeclaration.Rule = inlineDeclaration; + InlineDeclarations.MakeStarRule (grammar, InlineDeclaration); + + TopLevelInlineDeclaration.Rule = inlineDeclarations; + TopLevelInlineDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + var remarks = FinishParse (context, parseNode).Remarks; + var addRemarks = grammar.ShouldImport (ImportJavadoc.Remarks) || + (grammar.ShouldImport (ImportJavadoc.Summary) && remarks.Count == 0); + if (!addRemarks) { + parseNode.AstNode = ""; + return; + } + foreach (var p in GetParagraphs (parseNode.ChildNodes)) { + remarks.Add (p); + } + parseNode.AstNode = ""; + }; + + var fontstyle_tt = CreateHtmlToCrefElement (grammar, "tt", "c", InlineDeclarations); + var fontstyle_i = CreateHtmlToCrefElement (grammar, "i", "i", InlineDeclarations); + + var preText = new PreBlockDeclarationBodyTerminal (); + PreBlockDeclaration.Rule = CreateStartElement ("pre", grammar) + preText + CreateEndElement ("pre", grammar); + PreBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (!grammar.ShouldImport (ImportJavadoc.Remarks)) { + parseNode.AstNode = ""; + return; + } + var c = new XElement ("code", + new XAttribute ("lang", "text/java"), + parseNode.ChildNodes [1].Token.Value); + FinishParse (context, parseNode).Remarks.Add (c); + parseNode.AstNode = c; + }; + + FontStyleDeclaration.Rule = fontstyle_tt | fontstyle_i; + + PBlockDeclaration.Rule = + CreateStartElement ("p", grammar) + InlineDeclarations + CreateEndElement ("p", grammar, optional:true) + ; + PBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + var remarks = FinishParse (context, parseNode).Remarks; + var addRemarks = grammar.ShouldImport (ImportJavadoc.Remarks) || + (grammar.ShouldImport (ImportJavadoc.Summary) && remarks.Count == 0); + if (!addRemarks) { + parseNode.AstNode = ""; + return; + } + var p = new XElement ("para", + parseNode.ChildNodes + .Select (c => AstNodeToXmlContent (c))); + FinishParse (context, parseNode).Remarks.Add (p); + parseNode.AstNode = p; + }; + } + + static IEnumerable GetParagraphs (ParseTreeNodeList children) + { + var items = new List (); + foreach (var child in children) { + var s = child.AstNode as string; + if (s == null || (!s.Contains ("\n\n") && !s.Contains ("\r\n\r\n"))) { + items.Add (child.AstNode); + continue; + } + + const string UnixParagraph = "\n\n"; + const string DosParagraph = "\r\n\r\n"; + for (int i = 0; i < s.Length; ) { + int len = 0; + int n = -1; + + if ((n = s.IndexOf (UnixParagraph, i)) >= 0) { + len = UnixParagraph.Length; + } + else if ((n = s.IndexOf (DosParagraph, i)) >= 0) { + len = DosParagraph.Length; + } + + if (n <= 0) { + items.Add (s.Substring (i)); + break; + } + + var c = s.Substring (i, n-i); + items.Add (c); + i = n + len; + yield return new XElement ("para", items.Select (v => ToXmlContent (v))); + items.Clear (); + } + } + if (items.Count > 0) { + yield return new XElement ("para", items.Select (v => ToXmlContent (v))); + } + } + + public readonly NonTerminal AllHtmlTerms = new NonTerminal (nameof (AllHtmlTerms), ConcatChildNodes); + + public readonly NonTerminal TopLevelInlineDeclaration = new NonTerminal (nameof (TopLevelInlineDeclaration), ConcatChildNodes); + + + // https://www.w3.org/TR/html401/struct/global.html#h-7.5.3 +// public readonly Terminal ParsedCharacterData = new RegexBasedTerminal (nameof (ParsedCharacterData), "[^<{@}]*") { +// public readonly Terminal ParsedCharacterData = new WikiTextTerminal (nameof (ParsedCharacterData)) {* + public readonly Terminal ParsedCharacterData = new CharacterDataTerminal ("#PCDATA", preserveLeadingWhitespace:true); + + // https://www.w3.org/TR/html4/sgml/dtd.html#inline + public readonly NonTerminal InlineDeclaration = new NonTerminal (nameof (InlineDeclaration), ConcatChildNodes); + public readonly NonTerminal InlineDeclarations = new NonTerminal (nameof (InlineDeclarations), ConcatChildNodes); + // https://www.w3.org/TR/html4/sgml/dtd.html#fontstyle + public readonly NonTerminal FontStyleDeclaration = new NonTerminal (nameof (FontStyleDeclaration), ConcatChildNodes); + // https://www.w3.org/TR/html4/sgml/dtd.html#phrase + public readonly NonTerminal PhraseDeclaration = new NonTerminal (nameof (PhraseDeclaration), ConcatChildNodes); + // https://www.w3.org/TR/html4/sgml/dtd.html#special + public readonly NonTerminal SpecialDeclaration = new NonTerminal (nameof (SpecialDeclaration), ConcatChildNodes); + // https://www.w3.org/TR/html4/sgml/dtd.html#formctrl + public readonly NonTerminal FormCtrlDeclaration = new NonTerminal (nameof (FormCtrlDeclaration), ConcatChildNodes); + // https://www.w3.org/TR/html4/sgml/dtd.html#block + public readonly NonTerminal BlockDeclaration = new NonTerminal (nameof (BlockDeclaration), ConcatChildNodes); + public readonly NonTerminal PBlockDeclaration = new NonTerminal (nameof (PBlockDeclaration), ConcatChildNodes); + public readonly NonTerminal PreBlockDeclaration = new NonTerminal (nameof (PreBlockDeclaration), ConcatChildNodes); + + public readonly Terminal UnknownHtmlElementStart = new UnknownHtmlElementStartTerminal (nameof (UnknownHtmlElementStart)) { + AstConfig = new AstNodeConfig { + NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (), + }, + }; + + static NonTerminal CreateHtmlToCrefElement (Grammar grammar, string htmlElement, string crefElement, BnfTerm body, bool optionalEnd = false) + { + var start = CreateStartElement (htmlElement, grammar); + var end = CreateEndElement (htmlElement, grammar, optionalEnd); + var nonTerminal = new NonTerminal ("<" + htmlElement + ">", ConcatChildNodes) { + Rule = start + body + end, + AstConfig = { + NodeCreator = (context, parseNode) => { + var n = new XElement (crefElement, + parseNode.ChildNodes.Select (c => c.AstNode ?? "")); + parseNode.AstNode = n; + }, + } + }; + return nonTerminal; + } + + static NonTerminal CreateStartElement (string startElement, Grammar grammar) + { + var start = new NonTerminal ("<" + startElement + ">", nodeCreator: (context, parseNode) => parseNode.AstNode = "") { + Rule = grammar.ToTerm ("<" + startElement + ">") | "<" + startElement.ToUpperInvariant () + ">", + }; + return start; + } + + static NonTerminal CreateEndElement (string endElement, Grammar grammar, bool optional = false) + { + var end = new NonTerminal (endElement, nodeCreator: (context, parseNode) => parseNode.AstNode = "") { + Rule = grammar.ToTerm ("") | "", + }; + if (optional) { + end.Rule |= grammar.Empty; + } + return end; + } + } + } + + // Based in part on WikiTextTerminal + class CharacterDataTerminal : Terminal { + + char[]? _stopChars; + + bool preserveLeadingWhitespace; + + public CharacterDataTerminal (string name, bool preserveLeadingWhitespace) + : base (name) + { + base.Priority = TerminalPriority.Low; + + this.preserveLeadingWhitespace = preserveLeadingWhitespace; + + this.AstConfig.NodeCreator = (context, parseNode) => + parseNode.AstNode = parseNode.Token.Value.ToString (); + } + + public override void Init (GrammarData grammarData) + { + base.Init (grammarData); + var stopCharSet = new Irony.CharHashSet (); + foreach(var term in grammarData.Terminals) { + var firsts = term.GetFirsts (); + if (firsts == null) + continue; + foreach (var first in firsts) { + if (string.IsNullOrEmpty (first)) + continue; + stopCharSet.Add (first [0]); + } + } + _stopChars = stopCharSet.ToArray(); + } + + public override Token? TryMatch (ParsingContext context, ISourceStream source) + { + var stopIndex = source.Text.IndexOfAny (_stopChars, source.Location.Position); + if (stopIndex == source.Location.Position) + return null; + if (stopIndex < 0) + stopIndex = source.Text.Length; + source.PreviewPosition = stopIndex; + + // preserve leading whitespace, if present. + int start = source.Location.Position; + if (preserveLeadingWhitespace) { + while (start > 0 && char.IsWhiteSpace (source.Text, start-1)) { + start--; + } + } + var content = source.Text.Substring (start, stopIndex - start); + + return source.CreateToken (this.OutputTerminal, content); + } + } + + class PreBlockDeclarationBodyTerminal : Terminal { + + public PreBlockDeclarationBodyTerminal () + : base ("
 body")
+		{
+			this.AstConfig.NodeCreator = (context, parseNode) =>
+				parseNode.AstNode = parseNode.Token.Value.ToString ();
+		}
+
+		public override void Init (GrammarData grammarData)
+		{
+			base.Init (grammarData);
+		}
+
+		public override Token? TryMatch (ParsingContext context, ISourceStream source)
+		{
+			int startIndex  = source.Location.Position;
+			var stopIndex   = source.Text.IndexOf ("
", source.Location.Position, StringComparison.OrdinalIgnoreCase); + if (stopIndex < 0) + stopIndex = source.Text.Length; + source.PreviewPosition = stopIndex; + + var content = source.Text.Substring (startIndex, stopIndex - startIndex); + + return source.CreateToken (this.OutputTerminal, content); + } + } + + class UnknownHtmlElementStartTerminal : Terminal { + + bool addingRemarks; + + public UnknownHtmlElementStartTerminal (string name) + : base (name) + { + base.Priority = TerminalPriority.Low-1; + } + + public override void Init (GrammarData grammarData) + { + base.Init (grammarData); + var g = grammarData.Grammar as SourceJavadocToXmldocGrammar; + addingRemarks = g?.ShouldImport (ImportJavadoc.Remarks) ?? false; + } + + public override Token? TryMatch (ParsingContext context, ISourceStream source) + { + if (source.Text [source.Location.Position] != '<') + return null; + source.PreviewPosition += 1; + int start = source.Location.Position; + int stop = start; + while (source.Text [stop] != '>' && stop < source.Text.Length) + stop++; + if (addingRemarks) { + Console.Error.WriteLine ($"# Unsupported HTML element: {source.Text.Substring (start, stop - start)}"); + } + return source.CreateToken (this.OutputTerminal, "<"); + } + } +} diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs new file mode 100644 index 000000000..7eef65c1b --- /dev/null +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +using Irony.Ast; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource { + + public partial class SourceJavadocToXmldocGrammar { + + // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#javadoctags + public class InlineTagsBnfTerms { + + public InlineTagsBnfTerms () + { + } + + internal void CreateRules (SourceJavadocToXmldocGrammar grammar) + { + AllInlineTerms.Rule = CodeDeclaration + | DocRootDeclaration + | InheritDocDeclaration + | LinkDeclaration + | LinkplainDeclaration + | LiteralDeclaration + | ValueDeclaration + ; + + CodeDeclaration.Rule = grammar.ToTerm ("{@code") + InlineValue + "}"; + CodeDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + parseNode.AstNode = new XElement ("c", parseNode.ChildNodes [1].AstNode.ToString ().Trim ()); + }; + + DocRootDeclaration.Rule = grammar.ToTerm ("{@docRoot}"); + DocRootDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + parseNode.AstNode = new XText ("[TODO: @docRoot]"); + }; + + InheritDocDeclaration.Rule = grammar.ToTerm ("{@inheritDoc}"); + InheritDocDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + parseNode.AstNode = new XText ("[TODO: @inheritDoc]"); + }; + + LinkDeclaration.Rule = grammar.ToTerm ("{@link") + InlineValue + "}"; + LinkDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + // TODO: *everything*; {@link target label}, but target can contain spaces! + // Also need to convert to appropriate CREF value + var target = parseNode.ChildNodes [1].AstNode; + var x = new XElement ("c"); + parseNode.AstNode = new XElement ("c", new XElement ("see", new XAttribute ("cref", target))); + }; + + LinkplainDeclaration.Rule = grammar.ToTerm ("{@linkplain") + InlineValue + "}"; + LinkplainDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + // TODO: *everything*; {@link target label}, but target can contain spaces! + // Also need to convert to appropriate CREF value + var target = parseNode.ChildNodes [1].AstNode; + parseNode.AstNode = new XElement ("see", new XAttribute ("cref", target)); + }; + + LiteralDeclaration.Rule = grammar.ToTerm ("{@literal") + InlineValue + "}"; + LiteralDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + var content = parseNode.ChildNodes [1].AstNode.ToString (); + parseNode.AstNode = new XText (content); + }; + + ValueDeclaration.Rule = grammar.ToTerm ("{@value}") + | grammar.ToTerm ("{@value") + InlineValue + "}"; + ValueDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + if (parseNode.ChildNodes.Count > 1) { + var field = parseNode.ChildNodes [1].AstNode.ToString (); + parseNode.AstNode = new XText ($"[TODO: @value for `{field}`]"); + } + else { + parseNode.AstNode = new XText ("[TODO: @value]"); + } + }; + } + + public readonly NonTerminal AllInlineTerms = new NonTerminal (nameof (AllInlineTerms), ConcatChildNodes); + + public readonly Terminal InlineValue = new RegexBasedTerminal (nameof (InlineValue), "[^}]*") { + AstConfig = new AstNodeConfig { + NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value, + }, + }; + + // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#code + public readonly NonTerminal CodeDeclaration = new NonTerminal (nameof (CodeDeclaration)); + + // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#docRoot + public readonly NonTerminal DocRootDeclaration = new NonTerminal (nameof (DocRootDeclaration)); + + // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#inheritDoc + public readonly NonTerminal InheritDocDeclaration = new NonTerminal (nameof (InheritDocDeclaration)); + + // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#link + public readonly NonTerminal LinkDeclaration = new NonTerminal (nameof (LinkDeclaration)); + + // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#linkplain + public readonly NonTerminal LinkplainDeclaration = new NonTerminal (nameof (LinkplainDeclaration)); + + // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#literal + public readonly NonTerminal LiteralDeclaration = new NonTerminal (nameof (LinkplainDeclaration)); + + // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#value + public readonly NonTerminal ValueDeclaration = new NonTerminal (nameof (ValueDeclaration)); + } + } +} diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.cs new file mode 100644 index 000000000..a2be927b5 --- /dev/null +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +using Irony.Ast; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource { + + [Language ("SourceJavadocToXmldoc", "0.1", "Convert Javadoc within Java source code, sans comment delimiter, to CSC /doc XML.")] + public partial class SourceJavadocToXmldocGrammar : Grammar { + + public readonly BlockTagsBnfTerms BlockTagsTerms; + public readonly InlineTagsBnfTerms InlineTagsTerms; + public readonly HtmlBnfTerms HtmlTerms; + + public readonly XmldocStyle XmldocStyle; + + public SourceJavadocToXmldocGrammar (XmldocStyle style) + { + BlockTagsTerms = new BlockTagsBnfTerms (); + InlineTagsTerms = new InlineTagsBnfTerms (); + HtmlTerms = new HtmlBnfTerms (); + + XmldocStyle = style; + + BlockTagsTerms.CreateRules (this); + InlineTagsTerms.CreateRules (this); + HtmlTerms.CreateRules (this); + + var remark = new NonTerminal ("", ConcatChildNodes) { + Rule = HtmlTerms.AllHtmlTerms, + }; + var remarks = new NonTerminal ("*", ConcatChildNodes); + remarks.MakeStarRule (this, remark); + + var block = new NonTerminal ("@block", ConcatChildNodes) { + Rule = BlockTagsTerms.AllBlockTerms, + }; + var blocks = new NonTerminal ("@blocks", ConcatChildNodes); + blocks.MakeStarRule (this, block); + + var root = new NonTerminal ("", ConcatChildNodes) { + Rule = remarks + blocks, + }; + + root.AstConfig.NodeCreator = (context, parseNode) => { + FinishParse (context, parseNode); + }; + + this.Root = root; + } + + internal bool ShouldImport (ImportJavadoc value) + { + var v = (ImportJavadoc) XmldocStyle; + return v.HasFlag (value); + } + + internal static void ConcatChildNodes (AstContext context, ParseTreeNode parseNode) + { + switch (parseNode.ChildNodes.Count) { + case 0: + parseNode.AstNode = ""; + break; + case 1: + parseNode.AstNode = parseNode.ChildNodes [0].AstNode ?? ""; + break; + default: { + parseNode.AstNode = parseNode.ChildNodes + .Select (c => c.AstNode ?? "") + .ToArray (); + break; + } + } + } + + internal static IEnumerable AstNodeToXmlContent (ParseTreeNode node) + { + return ToXmlContent (node.AstNode); + } + + // Trim leading & trailing whitespace from `value`, which could be: + // * a string + // * an object[] + // * Anything else (XElement, etc.) + internal static IEnumerable ToXmlContent (object? value) + { + if (value == null) + yield break; + if (value is string s) { + yield return s.Trim (); + } + else if (value is IEnumerable nested) { + object? first = null; + object? last = null; + foreach (var n in nested) { + if (first != null) { + if (last != null) + yield return last; + last = n; + continue; + } + first = n; + if (first is string s1) { + yield return s1.TrimStart (); + } + else + yield return ToXmlContent (first); + } + if (last != null) { + if (last is string l) + yield return l.TrimEnd (); + else + yield return ToXmlContent (last); + } + } + else + yield return value; + } + + internal static JavadocInfo FinishParse (AstContext context, ParseTreeNode parseNode) + { + const string key = ".__JavadocInfo"; + if (!context.Values.TryGetValue (key, out var r)) { + context.Values.Add (key, r = new JavadocInfo ()); + } + parseNode.Tag = r; + return (JavadocInfo) r; + } + } +} diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs new file mode 100644 index 000000000..5a0d79ce3 --- /dev/null +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs @@ -0,0 +1,164 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using System.Text; + +using Irony; +using Irony.Ast; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource { + + [Flags] + internal enum ImportJavadoc { + None, + Summary = 1 << 0, + Remarks = 1 << 1, + AuthorTag = 1 << 2, + DeprecatedTag = 1 << 3, + ExceptionTag = 1 << 4, + ParamTag = 1 << 5, + ReturnTag = 1 << 6, + SeeTag = 1 << 7, + SerialTag = 1 << 8, + SinceTag = 1 << 9, + VersionTag = 1 << 10, + } + + [Flags] + public enum XmldocStyle { + None, + Full = ImportJavadoc.Summary + | ImportJavadoc.Remarks + | ImportJavadoc.AuthorTag + | ImportJavadoc.DeprecatedTag + | ImportJavadoc.ExceptionTag + | ImportJavadoc.ParamTag + | ImportJavadoc.ReturnTag + | ImportJavadoc.SeeTag + | ImportJavadoc.SerialTag + | ImportJavadoc.SinceTag + | ImportJavadoc.VersionTag + , + IntelliSense = ImportJavadoc.Summary + | ImportJavadoc.ExceptionTag + | ImportJavadoc.ParamTag + | ImportJavadoc.ReturnTag + , + } + + public class SourceJavadocToXmldocParser : Irony.Parsing.Parser { + + public SourceJavadocToXmldocParser (XmldocStyle style = XmldocStyle.Full) + : base (CreateGrammar (style)) + { + XmldocStyle = style; + } + + public XmldocStyle XmldocStyle { get; } + + public XElement[]? ExtraRemarks { get; set; } + + static Grammar CreateGrammar (XmldocStyle style) + { + return new SourceJavadocToXmldocGrammar (style) { + LanguageFlags = LanguageFlags.Default | LanguageFlags.CreateAst, + }; + } + + public IEnumerable TryParse (string javadoc, string? fileName = null, Action? onError = null) + { + onError = onError ?? DumpMessages; + + ParseTree parseTree; + var r = TryParse (javadoc, fileName, out parseTree); + if (parseTree.HasErrors ()) { + onError (parseTree); + } + return r; + } + + public IEnumerable TryParse (string javadoc, string? fileName, out ParseTree parseTree) + { + parseTree = base.Parse (javadoc, fileName); + if (parseTree.HasErrors ()) { + return Array.Empty(); + } + return CreateParseIterator (parseTree); + } + + IEnumerable CreateParseIterator (ParseTree parseTree) + { + if (parseTree.Root.Tag is JavadocInfo info) { + foreach (var n in info.Parameters) + yield return n; + var summary = CreateSummaryNode (info); + if (summary != null) + yield return summary; + var style = (ImportJavadoc) XmldocStyle; + if (style.HasFlag (ImportJavadoc.Remarks) && + (info.Remarks.Count > 0 || ExtraRemarks?.Length > 0)) { + yield return new XElement ("remarks", info.Remarks, ExtraRemarks); + } + foreach (var n in info.Returns) { + yield return n; + } + foreach (var n in info.Exceptions) { + yield return n; + } + foreach (var n in info.Extra) { + yield return n; + } + yield break; + } + var ast = parseTree.Root.AstNode; + if (ast is XNode node) { + yield return node; + } + else { + yield return new XCData (ast?.ToString ()); + } + } + + static XElement? CreateSummaryNode (JavadocInfo info) + { + var summaryNode = info.Remarks.FirstOrDefault (); + if (summaryNode == null) + return null; + + if (summaryNode is XElement p) { + var summaryItems = new List (); + for (var n = p.FirstNode; n != null; n = n.NextNode) { + if (n is XText text) { + var tdot = text.Value.IndexOf ('.'); + if (tdot < 0) { + summaryItems.Add (n); + continue; + } + summaryItems.Add (text.Value.Substring (0, tdot+1)); + break; + } + summaryItems.Add (n); + } + return new XElement ("summary", summaryItems); + } + var content = summaryNode.ToString (); + if (string.IsNullOrWhiteSpace (content)) + return null; + + var dot = content.IndexOf ('.'); + if (dot <= 0) + return new XElement ("summary", content); + return new XElement ("summary", content.Substring (0, dot+1)); + } + + static void DumpMessages (ParseTree parseTree) + { + foreach (var m in parseTree.ParserMessages) { + Console.Error.WriteLine ($"{m.Level} {m.Location}: {m.Message}"); + } + } + } +} diff --git a/src/Java.Interop.Tools.JavaSource/README.md b/src/Java.Interop.Tools.JavaSource/README.md new file mode 100644 index 000000000..5465daf8a --- /dev/null +++ b/src/Java.Interop.Tools.JavaSource/README.md @@ -0,0 +1,22 @@ +# Java.Interop.Tools.JavaSource + +Utilities for processing Java source code. + +## SourceJavadocToXmldocGrammar & SourceJavadocToXmldocParser + +`SourceJavadocToXmldocParser` parses Javadoc comments, as found in +`java-source-utils.jar` output (commit 69e1b80a), and converts it into +C# /doc XML via the Irony `SourceJavadocToXmldocGrammar` grammar. + +Multiple Javadoc+HTML language constructs are not yet supported: + + * Member lookup: + `@see #hashCode` is currently translated into + ``; no translation is performed. + This should be turned into + ``, but requires + additional plumbing so that `SourceJavadocToXmldocGrammar` can "know" + about all the possible types & members, and how to map them to C# names. + + * The following HTML elements need to be (better) supported: + `ul`, `li`. diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs new file mode 100644 index 000000000..f5cde828a --- /dev/null +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs @@ -0,0 +1,137 @@ +using System; +using System.Linq; +using System.Xml.Linq; + +using NUnit.Framework; + +using Java.Interop.Tools.JavaSource; + +using Irony; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource.Tests +{ + [TestFixture] + public class SourceJavadocToXmldocGrammarBlockTagsBnfTermsTests : SourceJavadocToXmldocGrammarFixture { + + [Test] + public void ApiSinceDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.ApiSinceDeclaration); + + var r = p.Parse ("@apiSince 3\n"); + Assert.IsFalse (r.HasErrors (), "@apiSince: " + DumpMessages (r, p)); + Assert.AreEqual ("Added in API level 3.", r.Root.AstNode.ToString ()); + } + + [Test] + public void DeprecatedDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.DeprecatedDeclaration); + + var r = p.Parse ("@deprecated Insert reason here.\n"); + Assert.IsFalse (r.HasErrors (), "@deprecated: " + DumpMessages (r, p)); + Assert.AreEqual ("This member is deprecated. Insert reason here.", r.Root.AstNode.ToString ()); + } + + [Test] + public void DeprecatedSinceDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.DeprecatedSinceDeclaration); + + var r = p.Parse ("@deprecatedSince 3\n"); + Assert.IsFalse (r.HasErrors (), "@deprecatedSince: " + DumpMessages (r, p)); + Assert.AreEqual ("This member was deprecated in API level 3.", r.Root.AstNode.ToString ()); + } + + [Test] + public void ExceptionDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.ExceptionDeclaration); + + var r = p.Parse ("@exception Throwable Just Because.\n"); + Assert.IsFalse (r.HasErrors (), "@exception: " + DumpMessages (r, p)); + Assert.AreEqual ("Just Because.", r.Root.AstNode.ToString ()); + } + + [Test] + public void ParamDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.ParamDeclaration); + + var r = p.Parse ("@param a Insert description here\n"); + Assert.IsFalse (r.HasErrors (), "@param: " + DumpMessages (r, p)); + Assert.AreEqual ("Insert description here", r.Root.AstNode.ToString ()); + } + + [Test] + public void ReturnDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.ReturnDeclaration); + + var r = p.Parse ("@return insert description here"); + Assert.IsFalse (r.HasErrors (), "single-line @return: " + DumpMessages (r, p)); + Assert.AreEqual ("insert description here", r.Root.AstNode.ToString ()); + + r = p.Parse ("@return line 1\n\tline two"); + Assert.IsFalse (r.HasErrors (), "multi-line @return: " + DumpMessages (r, p)); + Assert.AreEqual ("line 1\n\tline two".Replace ("\n", Environment.NewLine), + r.Root.AstNode.ToString ()); + } + + [Test] + public void ReturnDeclaration_WithInlineTags () + { + var p = CreateParser (g => g.BlockTagsTerms.ReturnDeclaration); + + var r = p.Parse ("@return {@code text} here."); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("\n text here.".Replace ("\n", Environment.NewLine), + r.Root.AstNode.ToString ()); + } + + [Test] + public void SeeDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.SeeDeclaration); + + var r = p.Parse ("@see \"Insert Book Name Here\""); + Assert.IsFalse (r.HasErrors (), "@see: " + DumpMessages (r, p)); + Assert.AreEqual ("", r.Root.AstNode.ToString ()); + } + + [Test] + public void SinceDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.SinceDeclaration); + + var r = p.Parse ("@since Insert Version Here"); + Assert.IsFalse (r.HasErrors (), "@since: " + DumpMessages (r, p)); + Assert.AreEqual ("Added in Insert Version Here.", r.Root.AstNode.ToString ()); + } + + [Test] + public void ThrowsDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.ThrowsDeclaration); + + var r = p.Parse ("@throws Throwable the {@code Exception} raised by this method"); + Assert.IsFalse (r.HasErrors (), "@throws: " + DumpMessages (r, p)); + Assert.AreEqual ("the Exception raised by this method", r.Root.AstNode.ToString ()); + + r = p.Parse ("@throws Throwable something or other!"); + Assert.IsFalse (r.HasErrors (), "@throws: " + DumpMessages (r, p)); + Assert.AreEqual ("something or other!", r.Root.AstNode.ToString ()); + } + + [Test] + public void UnknownTagDeclaration () + { + var p = CreateParser (g => g.BlockTagsTerms.UnknownTagDeclaration); + + var r = p.Parse ("@this-is-not-supported something {@code foo} else."); + Assert.IsFalse (r.HasErrors (), "@this-is-not-supported: " + DumpMessages (r, p)); + Assert.AreEqual (null, r.Root.AstNode); + } + } +} diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs new file mode 100644 index 000000000..575a2ce8f --- /dev/null +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using System.Text; + +using NUnit.Framework; + +using Java.Interop.Tools.JavaSource; + +using Irony; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource.Tests +{ + [TestFixture] + public class SourceJavadocToXmldocGrammarHtmlBnfTermsTests : SourceJavadocToXmldocGrammarFixture { + + [Test] + public void PBlockDeclaration () + { + var p = CreateParser (g => g.HtmlTerms.PBlockDeclaration); + + var r = p.Parse ("

paragraph text\nand more!"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("paragraph text\nand more!".Replace ("\n", Environment.NewLine), + r.Root.AstNode.ToString ()); + + r = p.Parse ("

r= {@code Object} and following {@literal AC} text

"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("r= Object and following A<B>C text", r.Root.AstNode.ToString ()); + + r = p.Parse("

r= unknown text"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("r= <em>unknown</em> text", r.Root.AstNode.ToString ()); + } + + [Test] + public void PreBlockDeclaration () + { + var p = CreateParser (g => g.HtmlTerms.PreBlockDeclaration); + + var r = p.Parse ("

this @contains  text.
"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("this @contains <arbitrary/> text.", + r.Root.AstNode.ToString ()); + + } + } +} diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.InlineTagsBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.InlineTagsBnfTermsTests.cs new file mode 100644 index 000000000..f59bb058d --- /dev/null +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.InlineTagsBnfTermsTests.cs @@ -0,0 +1,92 @@ +using System; +using System.Linq; +using System.Xml.Linq; + +using NUnit.Framework; + +using Java.Interop.Tools.JavaSource; + +using Irony; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource.Tests +{ + [TestFixture] + public class SourceJavadocToXmldocGrammarInlineTagsBnfTermsTests : SourceJavadocToXmldocGrammarFixture { + + [Test] + public void CodeDeclaration () + { + var p = CreateParser (g => g.InlineTagsTerms.CodeDeclaration); + + var r = p.Parse ("{@code Object}"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("Object", r.Root.AstNode.ToString ()); + } + + [Test] + public void DocRootDeclaration () + { + var p = CreateParser (g => g.InlineTagsTerms.DocRootDeclaration); + + var r = p.Parse ("{@docRoot}"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("[TODO: @docRoot]", r.Root.AstNode.ToString ()); + } + + [Test] + public void InheritDocDeclaration () + { + var p = CreateParser (g => g.InlineTagsTerms.InheritDocDeclaration); + + var r = p.Parse ("{@inheritDoc}"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("[TODO: @inheritDoc]", r.Root.AstNode.ToString ()); + } + + [Test] + public void LinkDeclaration () + { + var p = CreateParser (g => g.InlineTagsTerms.LinkDeclaration); + + var r = p.Parse ("{@link #ctor}"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + var c = (XElement) r.Root.AstNode; + Assert.AreEqual ("", c.ToString (SaveOptions.DisableFormatting)); + } + + [Test] + public void LinkplainDeclaration () + { + var p = CreateParser (g => g.InlineTagsTerms.LinkplainDeclaration); + + var r = p.Parse ("{@linkplain #ctor}"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("", r.Root.AstNode.ToString ()); + } + + [Test] + public void LiteralDeclaration () + { + var p = CreateParser (g => g.InlineTagsTerms.LiteralDeclaration); + + var r = p.Parse ("{@literal AC}"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("A<B>C", r.Root.AstNode.ToString ()); + } + + [Test] + public void ValueDeclaration () + { + var p = CreateParser (g => g.InlineTagsTerms.ValueDeclaration); + + var r = p.Parse ("{@value}"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("[TODO: @value]", r.Root.AstNode.ToString ()); + + r = p.Parse ("{@value #field}"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("[TODO: @value for `#field`]", r.Root.AstNode.ToString ()); + } + } +} diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammarFixture.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammarFixture.cs new file mode 100644 index 000000000..c8e52448a --- /dev/null +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammarFixture.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using System.Text; + +using NUnit.Framework; + +using Java.Interop.Tools.JavaSource; + +using Irony; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource.Tests +{ + [TestFixture] + public class SourceJavadocToXmldocGrammarFixture { + + public static Parser CreateParser (Func root) + { + var g = new SourceJavadocToXmldocGrammar (XmldocStyle.Full) { + LanguageFlags = LanguageFlags.Default | LanguageFlags.CreateAst, + }; + g.Root = root (g); + return new Parser (g) { + Context = { + TracingEnabled = true, + } + }; + } + + public static string DumpMessages (ParseTree tree, Parser parser) + { + var lines = GetLines (tree.SourceText); + var message = new StringBuilder (); + message.AppendLine ("ParserMessages:"); + foreach (var m in tree.ParserMessages) { + message.AppendLine ($" {m.Level} {m.Location}: {m.Message}"); + message.AppendLine (lines [m.Location.Line]); + message.Append (" "); + message.Append (new string (' ', m.Location.Column)); + message.Append ("^"); + message.AppendLine (); + } + message.AppendLine ("ParserTrace:"); + foreach (var t in parser.Context.ParserTrace) { + message.AppendLine ($" input=`{t.Input}`; error? {t.IsError}; message={t.Message}"); + } + return message.ToString (); + } + + static List GetLines (string text) + { + var lines = new List(); + var reader = new StringReader (text); + string line; + while ((line = reader.ReadLine()) != null) { + lines.Add (line); + } + return lines; + } + } +} diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs new file mode 100644 index 000000000..a5cf7ddcc --- /dev/null +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs @@ -0,0 +1,175 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using System.Text; + +using NUnit.Framework; + +using Java.Interop.Tools.JavaSource; + +using Irony; +using Irony.Parsing; + +namespace Java.Interop.Tools.JavaSource.Tests +{ + [TestFixture] + public class SourceJavadocToXmldocParserTests : SourceJavadocToXmldocGrammarFixture { + + [Test] + public void TryParse () + { + foreach (var values in TryParse_Success) { + ParseTree parseTree; + var p = new SourceJavadocToXmldocParser (XmldocStyle.Full); + var n = p.TryParse (values.Javadoc, null, out parseTree); + Assert.IsFalse (parseTree.HasErrors (), DumpMessages (parseTree, p)); + Assert.AreEqual (values.FullXml, GetMemberXml (n), $"while parsing input: ```{values.Javadoc}```"); + + p = new SourceJavadocToXmldocParser (XmldocStyle.IntelliSense); + n = p.TryParse (values.Javadoc, null, out parseTree); + Assert.IsFalse (parseTree.HasErrors (), DumpMessages (parseTree, p)); + Assert.AreEqual (values.IntelliSenseXml, GetMemberXml (n), $"while parsing input: ```{values.Javadoc}```"); + } + } + + static string GetMemberXml (IEnumerable members) + { + var e = new XElement ("member", members); + return e.ToString (); + } + + static readonly ParseResult[] TryParse_Success = new ParseResult[]{ + new ParseResult { + Javadoc = "Summary.\n\nP2.\n\n

Hello!

", + FullXml = @" + Summary. + + Summary. + P2. + Hello! + +", + IntelliSenseXml = @" + Summary. +", + }, + new ParseResult { + Javadoc = "The inline {@code code} tag should work for summary info.", + FullXml = @" + The inline code tag should work for summary info. + + The inline code tag should work for summary info. + +", + IntelliSenseXml = @" + The inline code tag should work for summary info. +", + }, + new ParseResult { + Javadoc = "@return {@code true} if something\n or other; otherwise {@code false}.", + FullXml = @" + + true if something + or other; otherwise false. +", + IntelliSenseXml = @" + + true if something + or other; otherwise false. +", + }, + new ParseResult { + Javadoc = @"This is the summary sentence. Insert +more description here. + +What about soft paragraphs? + +

What about hard paragraphs? + +@param a something +@see #method() +@apiSince 1 +", + FullXml = @" + something +

This is the summary sentence. + + This is the summary sentence. Insert +more description here. + What about soft paragraphs? + What about hard paragraphs? + Added in API level 1. + + +", + IntelliSenseXml = @" + something + This is the summary sentence. +", + }, + new ParseResult { + Javadoc = "Summary.\n\n

Paragraph.

foo @bar baz
", + FullXml = @" + Summary. + + Summary. + Paragraph. + foo @bar baz + +", + IntelliSenseXml = @" + Summary. +", + }, + new ParseResult { + Javadoc = "Something {@link #method}: description, \"declaration\" or \"another declaration\".\n\n@apiSince 1\n", + FullXml = @" + Something : description, ""<code>declaration</code>"" or ""<code>another declaration</code>"". + + Something : description, ""<code>declaration</code>"" or ""<code>another declaration</code>"". + Added in API level 1. + +", + IntelliSenseXml = @" + Something : description, ""<code>declaration</code>"" or ""<code>another declaration</code>"". +", + }, + new ParseResult { + // @jls is currently not supported; should be handled by @unknown-tag & ignored. + Javadoc = "Summary.\n\n@jls 1.2\n", + FullXml = @" + Summary. + + Summary. + +", + IntelliSenseXml = @" + Summary. +", + }, + new ParseResult { + // @jls is currently not supported; should be handled by @unknown-tag & ignored. + Javadoc = "Summary.\n\n@throws Throwable insert description here.\n", + FullXml = @" + Summary. + + Summary. + + insert description here. +", + IntelliSenseXml = @" + Summary. + insert description here. +", + }, + }; + + class ParseResult { + public string Javadoc; + public string FullXml; + public string IntelliSenseXml; + } + } +} diff --git a/tools/generator/CodeGenerator.cs b/tools/generator/CodeGenerator.cs index d88daaafd..720582879 100644 --- a/tools/generator/CodeGenerator.cs +++ b/tools/generator/CodeGenerator.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Xml; +using System.Xml.Linq; using Mono.Cecil; using MonoDroid.Generation; using Xamarin.AndroidTools.AnnotationSupport; @@ -179,6 +180,7 @@ static void Run (CodeGeneratorOptions options, DirectoryAssemblyResolver resolve SealedProtectedFixups.Fixup (gens); GenerateAnnotationAttributes (gens, annotations_zips); + JavadocFixups.Fixup (gens, options); //SymbolTable.Dump (); diff --git a/tools/generator/CodeGeneratorOptions.cs b/tools/generator/CodeGeneratorOptions.cs index b1bb8d283..cdb4e254f 100644 --- a/tools/generator/CodeGeneratorOptions.cs +++ b/tools/generator/CodeGeneratorOptions.cs @@ -2,6 +2,9 @@ using System.Collections.ObjectModel; using Mono.Cecil; using Mono.Options; + +using Java.Interop.Tools.JavaSource; + using MonoDroid.Generation; namespace Xamarin.Android.Binder @@ -15,6 +18,7 @@ public CodeGeneratorOptions () FixupFiles = new Collection (); LibraryPaths = new Collection (); AnnotationsZipFiles = new Collection (); + JavadocXmlFiles = new Collection (); } public string ApiLevel {get; set;} @@ -24,6 +28,7 @@ public CodeGeneratorOptions () public Collection AssemblyReferences {get; private set;} public Collection FixupFiles {get; private set;} public Collection LibraryPaths {get; private set;} + public Collection JavadocXmlFiles {get; private set;} public bool GlobalTypeNames {get; set;} public bool OnlyBindPublicTypes {get; set;} public string ApiDescriptionFile {get; set;} @@ -47,6 +52,8 @@ public CodeGeneratorOptions () public bool SupportNestedInterfaceTypes { get; set; } public bool SupportNullableReferenceTypes { get; set; } + public XmldocStyle XmldocStyle { get; set; } = XmldocStyle.IntelliSense; + public static CodeGeneratorOptions Parse (string[] args) { var opts = new CodeGeneratorOptions (); @@ -125,6 +132,18 @@ public static CodeGeneratorOptions Parse (string[] args) "Show this message and exit.", v => show_help = v != null }, "", + "Javadoc to C# Documentation Comments Support:", + { "doc-comment-verbosity=", + "{STYLE} of C# documentation comments to emit.\n" + + "Defaults to `full`. {STYLE} may be:\n" + + " * `intellisense`: emit , ,\n" + + " , .\n" + + " * `full`: plus , , ...", + v => opts.XmldocStyle = ParseXmldocStyle (v) }, + { "with-javadoc-xml=", + "{PATH} to `api.xml` containing Javadoc docs in\n`` elements", + v => opts.JavadocXmlFiles.Add (v) }, + "", "C# Enumeration Support:", { "enumdir=", "{DIRECTORY} to write enumeration declarations.", @@ -189,5 +208,11 @@ static CodeGenerationTarget ParseCodeGenerationTarget (string value) } throw new NotSupportedException ($"Don't know how to convert '{value}' to a CodeGenerationTarget value!"); } + + static XmldocStyle ParseXmldocStyle (string style) => style?.ToLowerInvariant () switch { + "intellisense" => XmldocStyle.IntelliSense, + "full" => XmldocStyle.Full, + _ => XmldocStyle.Full, + }; } } diff --git a/tools/generator/Java.Interop.Tools.Generator.Importers/XmlApiImporter.cs b/tools/generator/Java.Interop.Tools.Generator.Importers/XmlApiImporter.cs index ebb58aa6d..dac3593c2 100644 --- a/tools/generator/Java.Interop.Tools.Generator.Importers/XmlApiImporter.cs +++ b/tools/generator/Java.Interop.Tools.Generator.Importers/XmlApiImporter.cs @@ -113,6 +113,7 @@ public static Field CreateField (GenBase declaringType, XElement elem, CodeGener IsFinal = elem.XGetAttribute ("final") == "true", IsStatic = elem.XGetAttribute ("static") == "true", JavaName = elem.XGetAttribute ("name"), + JniSignature = elem.XGetAttribute ("jni-signature"), NotNull = elem.XGetAttribute ("not-null") == "true", SetterParameter = CreateParameter (elem, options), TypeName = elem.XGetAttribute ("type"), diff --git a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/Field.cs b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/Field.cs index 7df4d8348..a3ded000b 100644 --- a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/Field.cs +++ b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/Field.cs @@ -24,6 +24,9 @@ public class Field : ApiVersionsSupport.IApiAvailability, ISourceLineInfo public string TypeName { get; set; } public string Value { get; set; } public string Visibility { get; set; } + public string JniSignature { get; set; } + + public JavadocInfo JavadocInfo { get; set; } public int LineNumber { get; set; } = -1; public int LinePosition { get; set; } = -1; diff --git a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/GenBase.cs b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/GenBase.cs index 2fa22ee43..75003763f 100644 --- a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/GenBase.cs +++ b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/GenBase.cs @@ -39,6 +39,8 @@ protected GenBase (GenBaseSupport support) public string ReturnCast => string.Empty; + public JavadocInfo JavadocInfo { get; set; } + // This means Ctors/Methods/Properties/Fields has not been populated yet. // If this type is retrieved from the SymbolTable, it will call PopulateAction // to fill in members before returning it to the user. diff --git a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/Javadoc.cs b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/Javadoc.cs new file mode 100644 index 000000000..28c2702db --- /dev/null +++ b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/Javadoc.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Xml.Linq; + +using Irony.Parsing; + +using Java.Interop.Tools.JavaSource; + +namespace MonoDroid.Generation +{ + public static class Javadoc { + + public static void AddJavadocs (ICollection comments, string javadoc) + { + if (string.IsNullOrWhiteSpace (javadoc)) + return; + + javadoc = javadoc.Trim (); + + ParseTree tree = null; + + try { + var parser = new SourceJavadocToXmldocParser (); + var nodes = parser.TryParse (javadoc, fileName: null, out tree); + foreach (var node in (nodes ?? new XNode [0])) { + AddNode (comments, node); + } + } + catch (Exception e) { + Console.Error.WriteLine ($"## Exception translating remarks: {e.ToString ()}"); + } + + if (tree != null && tree.HasErrors ()) { + Console.Error.WriteLine ($"## Unable to translate remarks:"); + Console.Error.WriteLine ("```"); + Console.Error.WriteLine (javadoc); + Console.Error.WriteLine ("```"); + PrintMessages (tree, Console.Error); + Console.Error.WriteLine (); + } + } + + static void AddNode (ICollection comments, XNode node) + { + if (node == null) + return; + var contents = node.ToString (); + + var lines = new StringReader (contents); + string line; + while ((line = lines.ReadLine ()) != null) { + comments.Add ($"/// {line}"); + } + } + + static void PrintMessages (ParseTree tree, TextWriter writer) + { + var lines = GetLines (tree.SourceText); + foreach (var m in tree.ParserMessages) { + writer.WriteLine ($"{m.Level} {m.Location}: {m.Message}"); + writer.WriteLine (lines [m.Location.Line]); + writer.Write (new string (' ', m.Location.Column)); + writer.WriteLine ("^"); + } + } + + static List GetLines (string text) + { + var lines = new List(); + var reader = new StringReader (text); + string line; + while ((line = reader.ReadLine()) != null) { + lines.Add (line); + } + return lines; + } + } +} diff --git a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs new file mode 100644 index 000000000..cfcdb4fc5 --- /dev/null +++ b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs @@ -0,0 +1,344 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +using Irony.Parsing; + +using Java.Interop.Tools.JavaSource; + +namespace MonoDroid.Generation +{ + enum ApiLinkStyle { + None, + DeveloperAndroidComReference_2020Nov, + } + + public sealed class JavadocInfo { + + public string Javadoc { get; set; } + + public XElement[] ExtraRemarks { get; set; } + + public XmldocStyle XmldocStyle { get; set; } + + string MemberDescription; + + public static JavadocInfo CreateInfo (XElement element, XmldocStyle style) + { + if (element == null) { + return null; + } + + string javadoc = element.Element ("javadoc")?.Value; + + var desc = GetMemberDescription (element); + string declaringJniType = desc.DeclaringJniType; + string declaringMemberName = desc.DeclaringMemberName; + var declaringMemberJniSignature = desc.DeclaringMemberJniSignature; + + XElement[] extra = GetExtra (element, style, declaringJniType, declaringMemberName, declaringMemberJniSignature); + + if (string.IsNullOrEmpty (javadoc) && extra == null) + return null; + + var info = new JavadocInfo () { + ExtraRemarks = extra, + Javadoc = javadoc, + MemberDescription = declaringMemberName == null + ? declaringJniType + : $"{declaringJniType}.{declaringMemberName}.{declaringMemberJniSignature}", + XmldocStyle = style, + }; + return info; + } + + static (string DeclaringJniType, string DeclaringMemberName, string DeclaringMemberJniSignature) GetMemberDescription (XElement element) + { + bool isType = element.Name.LocalName == "class" || + element.Name.LocalName == "interface"; + + string declaringJniType = isType + ? (string) element.Attribute ("jni-signature") + : (string) element.Parent.Attribute ("jni-signature"); + if (declaringJniType.StartsWith ("L", StringComparison.Ordinal) && + declaringJniType.EndsWith (";", StringComparison.Ordinal)) { + declaringJniType = declaringJniType.Substring (1, declaringJniType.Length-2); + } + + string declaringMemberName = isType + ? null + : (string) element.Attribute ("name") ?? declaringJniType.Substring (declaringJniType.LastIndexOf ('/')+1); + string declaringMemberJniSignature = isType + ? null + : (string) element.Attribute ("jni-signature"); + + return (declaringJniType, declaringMemberName, declaringMemberJniSignature); + } + + static XElement[] GetExtra (XElement element, XmldocStyle style, string declaringJniType, string declaringMemberName, string declaringMemberJniSignature) + { + if (!style.HasFlag (XmldocStyle.Full)) + return null; + + XElement javadocMetadata = null; + while (element != null) { + javadocMetadata = element.Element ("javadoc-metadata"); + if (javadocMetadata != null) { + break; + } + element = element.Parent; + } + + List extra = null; + if (javadocMetadata != null) { + var link = javadocMetadata.Element ("link"); + var urlPrefix = (string) link.Attribute ("prefix"); + var linkStyle = (string) link.Attribute ("style"); + var kind = ParseApiLinkStyle (linkStyle); + + XElement docLink = null; + if (!string.IsNullOrEmpty (urlPrefix)) { + docLink = CreateDocLinkUrl (kind, urlPrefix, declaringJniType, declaringMemberName, declaringMemberJniSignature); + } + extra = new List (); + extra.Add (docLink); + extra.AddRange (javadocMetadata.Element ("copyright").Elements ()); + } + return extra?.ToArray (); + } + + static ApiLinkStyle ParseApiLinkStyle (string style) + { + switch (style) { + case "developer.android.com/reference@2020-Nov": + return ApiLinkStyle.DeveloperAndroidComReference_2020Nov; + default: + return ApiLinkStyle.None; + } + } + + + public void AddJavadocs (ICollection comments) + { + var nodes = ParseJavadoc (); + AddComments (comments, nodes); + } + + public IEnumerable ParseJavadoc () + { + if (string.IsNullOrWhiteSpace (Javadoc)) + return Enumerable.Empty (); + + Javadoc = Javadoc.Trim (); + + ParseTree tree = null; + IEnumerable nodes = null; + + try { + var parser = new SourceJavadocToXmldocParser (XmldocStyle) { + ExtraRemarks = ExtraRemarks, + }; + nodes = parser.TryParse (Javadoc, fileName: null, out tree); + } + catch (Exception e) { + Console.Error.WriteLine ($"## Exception translating remarks: {e.ToString ()}"); + } + + if (tree != null && tree.HasErrors ()) { + Console.Error.WriteLine ($"## Unable to translate remarks for {MemberDescription}:"); + Console.Error.WriteLine ("```"); + Console.Error.WriteLine (Javadoc); + Console.Error.WriteLine ("```"); + PrintMessages (tree, Console.Error); + Console.Error.WriteLine (); + } + + return nodes; + } + + public static void AddComments (ICollection comments, IEnumerable nodes) + { + if (nodes == null) + return; + + foreach (var node in nodes) { + AddNode (comments, node); + } + } + + static void AddNode (ICollection comments, XNode node) + { + if (node == null) + return; + var contents = node.ToString (); + + var lines = new StringReader (contents); + string line; + while ((line = lines.ReadLine ()) != null) { + comments.Add ($"/// {line}"); + } + } + + static void PrintMessages (ParseTree tree, TextWriter writer) + { + var lines = GetLines (tree.SourceText); + foreach (var m in tree.ParserMessages) { + writer.WriteLine ($"{m.Level} {m.Location}: {m.Message}"); + writer.WriteLine (lines [m.Location.Line]); + writer.Write (new string (' ', m.Location.Column)); + writer.WriteLine ("^"); + } + } + + static List GetLines (string text) + { + var lines = new List(); + var reader = new StringReader (text); + string line; + while ((line = reader.ReadLine()) != null) { + lines.Add (line); + } + return lines; + } + + static Dictionary> UrlCreators = new Dictionary> { + [ApiLinkStyle.DeveloperAndroidComReference_2020Nov] = CreateAndroidDocLinkUri, + }; + + static XElement CreateDocLinkUrl (ApiLinkStyle style, string prefix, string declaringJniType, string declaringMemberName, string declaringMemberJniSignature) + { + ; + if (style == ApiLinkStyle.None || prefix == null || declaringJniType == null) + return null; + if (UrlCreators.TryGetValue (style, out var creator)) { + return creator (prefix, declaringJniType, declaringMemberName, declaringMemberJniSignature); + } + return null; + } + + static XElement CreateAndroidDocLinkUri (string prefix, string declaringJniType, string declaringMemberName, string declaringMemberJniSignature) + { + // URL is: + // * {prefix} + // * declaring type in JNI format + // * when `declaringJniMemberName` != null, `#{declaringJniMemberName}` + // * for methods & constructors, a `(`, the arguments in *Java* syntax -- separated by `, ` -- and `)` + // + // Example: https://developer.android.com/reference/android/app/Application#registerOnProvideAssistDataListener(android.app.Application.OnProvideAssistDataListener) + + var java = new StringBuilder (declaringJniType) + .Replace ("/", ".") + .Replace ("$", "."); + var url = new StringBuilder (prefix); + if (!prefix.EndsWith ("/")) { + url.Append ("/"); + } + url.Append (declaringJniType); + + if (declaringMemberName != null) { + java.Append (".").Append (declaringMemberName); + url.Append ("#").Append (declaringMemberName); + if (declaringMemberJniSignature?.StartsWith ("(", StringComparison.Ordinal) ?? false) { + java.Append ("("); + url.Append ("("); + AppendJavaParameterTypes (java, declaringMemberJniSignature); + AppendJavaParameterTypes (url, declaringMemberJniSignature); + java.Append (")"); + url.Append (")"); + } + } + var format = new XElement ("format", + new XAttribute ("type", "text/html"), + new XElement ("a", + new XAttribute ("href", new Uri (url.ToString ()).AbsoluteUri), + "Java documentation for ", + new XElement ("tt", java.ToString ()), + ".")); + return new XElement ("para", format); + } + + static StringBuilder AppendJavaParameterTypes (StringBuilder builder, string declaringMemberJniSignature) + { + if (string.IsNullOrEmpty (declaringMemberJniSignature) || declaringMemberJniSignature [0] != '(') + return builder; + + int startLen = builder.Length; + + for (int i = 1; i < declaringMemberJniSignature.Length; ++i) { + if (declaringMemberJniSignature [i] == ')') + break; + AppendComma (); + AppendJavaParameterType (builder, declaringMemberJniSignature, ref i); + } + + return builder; + + void AppendComma () + { + if (startLen == builder.Length) + return; + builder.Append (", "); + } + } + + static void AppendJavaParameterType (StringBuilder builder, string declaringMemberJniSignature, ref int i) + { + switch (declaringMemberJniSignature [i]) { + case '[': { + ++i; + AppendJavaParameterType (builder, declaringMemberJniSignature, ref i); + builder.Append ("[]"); + break; + } + case 'B': { + builder.Append ("byte"); + break; + } + case 'C': { + builder.Append ("char"); + break; + } + case 'D': { + builder.Append ("double"); + break; + } + case 'F': { + builder.Append ("float"); + break; + } + case 'I': { + builder.Append ("int"); + break; + } + case 'J': { + builder.Append ("long"); + break; + } + case 'L': { + int end = declaringMemberJniSignature.IndexOf (';', i); + if (end < 0) + throw new InvalidOperationException ($"INTERNAL ERROR: Invalid JNI signature '{declaringMemberJniSignature}': no ';' to end 'L' at index {i}!"); + var type = declaringMemberJniSignature.Substring (i+1, end - i - 1) + .Replace ('/', '.') + .Replace ('$', '.'); + builder.Append (type); + i = end; + break; + } + case 'S': { + builder.Append ("short"); + break; + } + case 'Z': { + builder.Append ("boolean"); + break; + } + default: + throw new NotSupportedException ($"INTERNAL ERROR: Don't know what to do with '{declaringMemberJniSignature [i]}' in '{declaringMemberJniSignature}'!"); + } + } + } +} diff --git a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/MethodBase.cs b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/MethodBase.cs index e79b7c78d..b43a2cea8 100644 --- a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/MethodBase.cs +++ b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/MethodBase.cs @@ -28,6 +28,8 @@ protected MethodBase (GenBase declaringType) public int LinePosition { get; set; } = -1; public string SourceFile { get; set; } + public JavadocInfo JavadocInfo { get; set; } + public string [] AutoDetectEnumifiedOverrideParameters (AncestorDescendantCache cache) { if (Parameters.All (p => p.Type != "int")) diff --git a/tools/generator/Java.Interop.Tools.Generator.Transformation/JavadocFixups.cs b/tools/generator/Java.Interop.Tools.Generator.Transformation/JavadocFixups.cs new file mode 100644 index 000000000..fb751e455 --- /dev/null +++ b/tools/generator/Java.Interop.Tools.Generator.Transformation/JavadocFixups.cs @@ -0,0 +1,108 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +using Java.Interop.Tools.JavaSource; + +using MonoDroid.Generation; +using MonoDroid.Utils; + +using Xamarin.Android.Binder; + +namespace Java.Interop.Tools.Generator.Transformation +{ + public static class JavadocFixups + { + public static void Fixup (List gens, CodeGeneratorOptions options) + { + if (options.JavadocXmlFiles == null || options.JavadocXmlFiles.Count == 0) + return; + + var typeJavadocs = new Dictionary (); + + foreach (var path in options.JavadocXmlFiles) { + XDocument doc = null; + try { + doc = XDocument.Load (path); + } + catch (Exception e) { + Report.LogCodedWarning (0, Report.WarningInvalidXmlFile, e, path, e.Message); + continue; + } + + var types = doc.Elements ("api") + .Elements ("package") + .Elements (); + foreach (var typeXml in types) { + var typeJniSig = (string) typeXml.Attribute ("jni-signature"); + if (string.IsNullOrEmpty (typeJniSig)) + continue; + if (!typeJavadocs.TryGetValue (typeJniSig, out _)) + typeJavadocs.Add (typeJniSig, typeXml); + } + } + + foreach (var type in gens) { + AddJavadoc (type, typeJavadocs, options.XmldocStyle); + } + } + + static void AddJavadoc (GenBase type, Dictionary typeJavadocs, XmldocStyle style) + { + if (!typeJavadocs.TryGetValue (type.JniName, out XElement typeJavadoc)) + return; + if (typeJavadoc == null) + return; + + if (type.JavadocInfo == null) { + type.JavadocInfo = JavadocInfo.CreateInfo (typeJavadoc, style); + } + + foreach (var method in type.Methods) { + if (method.JavadocInfo != null) + continue; + var methodJavadoc = GetMemberJavadoc (typeJavadoc, "method", method.JavaName, method.JniSignature); + method.JavadocInfo = JavadocInfo.CreateInfo (methodJavadoc?.Parent, style); + } + + foreach (var property in type.Properties) { + if (property.Getter != null && property.Getter.JavadocInfo == null) { + var getterJavadoc = GetMemberJavadoc (typeJavadoc, "method", property.Getter.JavaName, property.Getter.JniSignature); + property.Getter.JavadocInfo = JavadocInfo.CreateInfo (getterJavadoc?.Parent, style); + } + if (property.Setter != null && property.Setter.JavadocInfo == null) { + var setterJavadoc = GetMemberJavadoc (typeJavadoc, "method", property.Setter.JavaName, property.Setter.JniSignature); + property.Setter.JavadocInfo = JavadocInfo.CreateInfo (setterJavadoc?.Parent, style); + } + } + + foreach (var field in type.Fields) { + if (field.JavadocInfo != null) + continue; + var fieldJavadoc = GetMemberJavadoc (typeJavadoc, "field", field.JavaName, field.JniSignature); + field.JavadocInfo = JavadocInfo.CreateInfo (fieldJavadoc?.Parent, style); + } + + if (type is ClassGen @class) { + foreach (var ctor in @class.Ctors) { + if (ctor.JavadocInfo != null) + continue; + var ctorJavadoc = GetMemberJavadoc (typeJavadoc, "constructor", null, ctor.JniSignature); + ctor.JavadocInfo = JavadocInfo.CreateInfo (ctorJavadoc?.Parent, style); + } + } + } + + static XElement GetMemberJavadoc (XElement typeJavadoc, string elementName, string name, string jniSignature) + { + return typeJavadoc + .Elements (elementName) + .Where (e => jniSignature == (string) e.Attribute ("jni-signature") && + name == null ? true : name == (string) e.Attribute ("name")) + .Elements ("javadoc") + .FirstOrDefault (); + } + } +} diff --git a/tools/generator/SourceWriters/BoundClass.cs b/tools/generator/SourceWriters/BoundClass.cs index afce639e8..d4d317099 100644 --- a/tools/generator/SourceWriters/BoundClass.cs +++ b/tools/generator/SourceWriters/BoundClass.cs @@ -39,6 +39,7 @@ public BoundClass (ClassGen klass, CodeGenerationOptions opt, CodeGeneratorConte AddImplementedInterfaces (klass); + klass.JavadocInfo?.AddJavadocs (Comments); Comments.Add ($"// Metadata.xml XPath class reference: path=\"{klass.MetadataXPathReference}\""); if (klass.IsDeprecated) diff --git a/tools/generator/SourceWriters/BoundConstructor.cs b/tools/generator/SourceWriters/BoundConstructor.cs index 5ec69de81..025c5f8c0 100644 --- a/tools/generator/SourceWriters/BoundConstructor.cs +++ b/tools/generator/SourceWriters/BoundConstructor.cs @@ -24,6 +24,7 @@ public BoundConstructor (ClassGen klass, Ctor constructor, bool useBase, CodeGen Name = klass.Name; + constructor.JavadocInfo?.AddJavadocs (Comments); Comments.Add (string.Format ("// Metadata.xml XPath constructor reference: path=\"{0}/constructor[@name='{1}'{2}]\"", klass.MetadataXPathReference, klass.JavaSimpleName, constructor.Parameters.GetMethodXPathPredicate ())); Attributes.Add (new RegisterAttr (".ctor", constructor.JniSignature, string.Empty, additionalProperties: constructor.AdditionalAttributeString ())); diff --git a/tools/generator/SourceWriters/BoundField.cs b/tools/generator/SourceWriters/BoundField.cs index fb89716b1..a8f80bea7 100644 --- a/tools/generator/SourceWriters/BoundField.cs +++ b/tools/generator/SourceWriters/BoundField.cs @@ -19,6 +19,7 @@ public BoundField (GenBase type, Field field, CodeGenerationOptions opt) Name = field.Name; Type = new TypeReferenceWriter (opt.GetOutputName (field.Symbol.FullName)); + field.JavadocInfo?.AddJavadocs (Comments); Comments.Add ($"// Metadata.xml XPath field reference: path=\"{type.MetadataXPathReference}/field[@name='{field.JavaName}']\""); Attributes.Add (new RegisterAttr (field.JavaName, additionalProperties: field.AdditionalAttributeString ())); diff --git a/tools/generator/SourceWriters/BoundFieldAsProperty.cs b/tools/generator/SourceWriters/BoundFieldAsProperty.cs index dcb332fae..a05fdd3e4 100644 --- a/tools/generator/SourceWriters/BoundFieldAsProperty.cs +++ b/tools/generator/SourceWriters/BoundFieldAsProperty.cs @@ -25,6 +25,7 @@ public BoundFieldAsProperty (GenBase type, Field field, CodeGenerationOptions op var fieldType = field.Symbol.IsArray ? "IList<" + field.Symbol.ElementType + ">" + opt.NullableOperator : opt.GetTypeReferenceName (field); PropertyType = new TypeReferenceWriter (fieldType); + field.JavadocInfo?.AddJavadocs (Comments); Comments.Add ($"// Metadata.xml XPath field reference: path=\"{type.MetadataXPathReference}/field[@name='{field.JavaName}']\""); if (field.IsEnumified) diff --git a/tools/generator/SourceWriters/BoundInterface.cs b/tools/generator/SourceWriters/BoundInterface.cs index c0439ca0e..003dd9caf 100644 --- a/tools/generator/SourceWriters/BoundInterface.cs +++ b/tools/generator/SourceWriters/BoundInterface.cs @@ -37,6 +37,7 @@ public BoundInterface (InterfaceGen iface, CodeGenerationOptions opt, CodeGenera SetVisibility (iface.Visibility); + iface.JavadocInfo?.AddJavadocs (Comments); Comments.Add ($"// Metadata.xml XPath interface reference: path=\"{iface.MetadataXPathReference}\""); if (iface.IsDeprecated) diff --git a/tools/generator/SourceWriters/BoundInterfaceMethodDeclaration.cs b/tools/generator/SourceWriters/BoundInterfaceMethodDeclaration.cs index 98817006f..e4e71b8b4 100644 --- a/tools/generator/SourceWriters/BoundInterfaceMethodDeclaration.cs +++ b/tools/generator/SourceWriters/BoundInterfaceMethodDeclaration.cs @@ -33,6 +33,8 @@ public BoundInterfaceMethodDeclaration (Method method, string adapter, CodeGener Attributes.Add (new RegisterAttr (method.JavaName, method.JniSignature, method.ConnectorName + ":" + method.GetAdapterName (opt, adapter), additionalProperties: method.AdditionalAttributeString ())); + method.JavadocInfo?.AddJavadocs (Comments); + SourceWriterExtensions.AddMethodCustomAttributes (Attributes, method); this.AddMethodParameters (method.Parameters, opt); } diff --git a/tools/generator/SourceWriters/BoundMethod.cs b/tools/generator/SourceWriters/BoundMethod.cs index 0525b31e4..e30bd83e3 100644 --- a/tools/generator/SourceWriters/BoundMethod.cs +++ b/tools/generator/SourceWriters/BoundMethod.cs @@ -59,6 +59,8 @@ public BoundMethod (GenBase type, Method method, CodeGenerationOptions opt, bool ReturnType = new TypeReferenceWriter (opt.GetTypeReferenceName (method.RetVal)); + method.JavadocInfo?.AddJavadocs (Comments); + if (method.DeclaringType.IsGeneratable) Comments.Add ($"// Metadata.xml XPath method reference: path=\"{method.GetMetadataXPathReference (method.DeclaringType)}\""); diff --git a/tools/generator/SourceWriters/BoundMethodAbstractDeclaration.cs b/tools/generator/SourceWriters/BoundMethodAbstractDeclaration.cs index 03a060666..66a3f25d4 100644 --- a/tools/generator/SourceWriters/BoundMethodAbstractDeclaration.cs +++ b/tools/generator/SourceWriters/BoundMethodAbstractDeclaration.cs @@ -41,6 +41,8 @@ public BoundMethodAbstractDeclaration (GenBase gen, Method method, CodeGeneratio method_callback = new MethodCallback (impl, method, opt, null, method.IsReturnCharSequence); + method.JavadocInfo?.AddJavadocs (Comments); + if (method.DeclaringType.IsGeneratable) Comments.Add ($"// Metadata.xml XPath method reference: path=\"{method.GetMetadataXPathReference (method.DeclaringType)}\""); diff --git a/tools/generator/SourceWriters/BoundMethodStringOverload.cs b/tools/generator/SourceWriters/BoundMethodStringOverload.cs index 47dea8e1d..168e55ff7 100644 --- a/tools/generator/SourceWriters/BoundMethodStringOverload.cs +++ b/tools/generator/SourceWriters/BoundMethodStringOverload.cs @@ -27,6 +27,8 @@ public BoundMethodStringOverload (Method method, CodeGenerationOptions opt) if (method.Deprecated != null) Attributes.Add (new ObsoleteAttr (method.Deprecated.Replace ("\"", "\"\"").Trim ())); + method.JavadocInfo?.AddJavadocs (Comments); + this.AddMethodParametersStringOverloads (method.Parameters, opt); } diff --git a/tools/generator/SourceWriters/BoundProperty.cs b/tools/generator/SourceWriters/BoundProperty.cs index fbcac7150..6f708aa82 100644 --- a/tools/generator/SourceWriters/BoundProperty.cs +++ b/tools/generator/SourceWriters/BoundProperty.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using System.Xml.Linq; using MonoDroid.Generation; using Xamarin.SourceWriter; @@ -97,6 +98,8 @@ public BoundProperty (GenBase gen, Property property, CodeGenerationOptions opt, SetterComments.Add ("// This is a dispatching setter"); SetBody.Add ($"Set{property.Name} (value);"); } + + AddJavadocs (property); } public override void Write (CodeWriter writer) @@ -144,5 +147,61 @@ bool ShouldForceOverride (Property property) return false; } + + void AddJavadocs (Property property) + { + if (property.Getter?.JavadocInfo == null && property.Setter?.JavadocInfo == null) + return; + + var memberDocs = new XElement ("member"); + + if (property.Getter?.JavadocInfo != null) { + memberDocs.Add (property.Getter.JavadocInfo.ParseJavadoc ()); + } + + if (property.Setter?.JavadocInfo != null) { + var setterDocs = new XElement ("member", property.Setter.JavadocInfo.ParseJavadoc ()); + + MergeSummary (memberDocs, setterDocs); + MergeRemarks (memberDocs, setterDocs); + + memberDocs.Add (setterDocs.DescendantNodes ()); + } + + JavadocInfo.AddComments (Comments, memberDocs.Elements ()); + } + + static void MergeSummary (XElement mergeInto, XElement mergeFrom) + { + var toContent = mergeInto.Element ("summary"); + var fromContent = mergeFrom.Element ("summary"); + + if (toContent == null && fromContent != null) { + fromContent.Remove (); + mergeInto.Add (fromContent); + } + else if (toContent != null && fromContent != null) { + fromContent.Remove (); + toContent.Add (" -or- "); + toContent.Add (fromContent.DescendantNodes ()); + } + } + + static void MergeRemarks (XElement mergeInto, XElement mergeFrom) + { + var toContent = mergeInto.Element ("remarks"); + var fromContent = mergeFrom.Element ("remarks"); + + if (toContent == null && fromContent != null) { + fromContent.Remove (); + mergeInto.Add (fromContent); + } + else if (toContent != null && fromContent != null) { + fromContent.Remove (); + toContent.AddFirst (new XElement ("para", "Property getter documentation:")); + toContent.Add (new XElement ("para", "Property setter documentation:")); + toContent.Add (fromContent.DescendantNodes ()); + } + } } } diff --git a/tools/generator/generator.csproj b/tools/generator/generator.csproj index 88489255d..80bcf2b0f 100644 --- a/tools/generator/generator.csproj +++ b/tools/generator/generator.csproj @@ -30,6 +30,7 @@ + @@ -48,6 +49,7 @@ + diff --git a/tools/java-source-utils/src/main/java/com/microsoft/android/App.java b/tools/java-source-utils/src/main/java/com/microsoft/android/App.java index b67d58c02..f8f449feb 100644 --- a/tools/java-source-utils/src/main/java/com/microsoft/android/App.java +++ b/tools/java-source-utils/src/main/java/com/microsoft/android/App.java @@ -40,7 +40,7 @@ public static void main (final String[] args) throws Throwable { if ((options.outputParamsTxt = Parameter.normalize(options.outputParamsTxt, "")).length() > 0) { generateParamsTxt(options.outputParamsTxt, packages); } - generateXml(options.outputJavadocXml, packages); + generateXml(options, packages); options.close(); } catch (Throwable t) { @@ -66,8 +66,9 @@ static void generateParamsTxt(String filename, JniPackagesInfo packages) throws } } - static void generateXml(String filename, JniPackagesInfo packages) throws Throwable { - try (final JavadocXmlGenerator javadocXmlGen = new JavadocXmlGenerator(filename)) { + static void generateXml(JavaSourceUtilsOptions options, JniPackagesInfo packages) throws Throwable { + try (final JavadocXmlGenerator javadocXmlGen = new JavadocXmlGenerator(options.outputJavadocXml)) { + javadocXmlGen.writeCopyrightInfo(options.docCopyrightFile, options.docUrlPrefix, options.docUrlStyle); javadocXmlGen.writePackages(packages); } } diff --git a/tools/java-source-utils/src/main/java/com/microsoft/android/JavaSourceUtilsOptions.java b/tools/java-source-utils/src/main/java/com/microsoft/android/JavaSourceUtilsOptions.java index 6e36de3a8..448101f07 100644 --- a/tools/java-source-utils/src/main/java/com/microsoft/android/JavaSourceUtilsOptions.java +++ b/tools/java-source-utils/src/main/java/com/microsoft/android/JavaSourceUtilsOptions.java @@ -40,10 +40,45 @@ public class JavaSourceUtilsOptions implements AutoCloseable { - public static final String HELP_STRING = "[-v] [<-a|--aar> AAR]* [<-j|--jar> JAR]* [<-s|--source> DIRS]*\n" + + public static final String HELP_STRING = + "[-v] [<-a|--aar> AAR]* [<-j|--jar> JAR]* [<-s|--source> DIRS]*\n" + "\t[--bootclasspath CLASSPATH]\n" + "\t[<-P|--output-params> OUT.params.txt] [<-D|--output-javadoc> OUT.xml]\n" + - "\t[@RESPONSE-FILE]* FILES"; + "\t[--doc-copyright FILE] [--doc-url-prefix URL] [--doc-url-style STYLE]\n" + + "\t[@RESPONSE-FILE]* FILES\n" + + "\n" + + "Options:\n" + + " @RESPONSE-FILE Additional options to parse, one option per line.\n" + + " FILES .java files to parse.\n" + + " -v Verbose output; show diagnostic information.\n" + + " -h, -?, --help Show this message and exit.\n" + + "\n" + + "Java type resolution options:\n" + + " --bootclasspath CLASSPATH\n" + + " '" + File.pathSeparator + "'-separated list of .jar files to use\n" + + " for type resolution.\n" + + " -a, --aar FILE .aar file to use for type resolution.\n" + + " -j, --jar FILE .jar file to use for type resolution.\n" + + " -s, --source DIR Directory containing .java files for type\n" + + " resolution purposes. DOES NOT parse all files.\n" + + "\n" + + "Documentation copyright file options:\n" + + " Results in an additional '/api/javadoc-metadata' element when using\n" + + " --output-javadoc.\n" + + " --doc-copyright FILE Copyright information for Javadoc. Should be in\n" + + " mdoc(5) XML, to be held within .\n" + + " Stored in //javadoc-metadata/copyright.\n" + + " --doc-url-prefix URL Base URL for links to documentation.\n" + + " Stored in //javadoc-metadata/link/@prefix.\n" + + " --doc-url-style STYLE STYLE of URLs to generate for member links.\n" + + " Stored in //javadoc-metadata/link/@style.\n" + + " Supported styles include:\n" + + " - developer.android.com/reference@2020-Nov\n" + + "\n" + + "Output file options:\n" + + " -P, --output-params FILE Write method parameter names to FILE.\n" + + " -D, --output-javadoc FILE Write Javadoc within XML container to FILE.\n" + + ""; public static boolean verboseOutput; @@ -56,6 +91,10 @@ public class JavaSourceUtilsOptions implements AutoCloseable { public String outputParamsTxt; public String outputJavadocXml; + public File docCopyrightFile; + public String docUrlPrefix; + public String docUrlStyle; + private final Collection sourceDirectoryFiles = new ArrayList(); private File extractedTempDir; @@ -147,6 +186,24 @@ private final JavaSourceUtilsOptions parse(Iterator args) throws IOExcep aarFiles.add(file); break; } + case "--doc-copyright": { + final File file = getNextOptionFile(args, arg); + if (file == null) { + break; + } + docCopyrightFile = file; + break; + } + case "--doc-url-prefix": { + final String prefix = getNextOptionValue(args, arg); + docUrlPrefix = prefix; + break; + } + case "--doc-url-style": { + final String style = getNextOptionValue(args, arg); + docUrlStyle = style; + break; + } case "-j": case "--jar": { final File file = getNextOptionFile(args, arg); @@ -180,6 +237,7 @@ private final JavaSourceUtilsOptions parse(Iterator args) throws IOExcep break; } case "-h": + case "-?": case "--help": { return null; } diff --git a/tools/java-source-utils/src/main/java/com/microsoft/android/JavadocXmlGenerator.java b/tools/java-source-utils/src/main/java/com/microsoft/android/JavadocXmlGenerator.java index 41ad934d0..d600c1fd8 100644 --- a/tools/java-source-utils/src/main/java/com/microsoft/android/JavadocXmlGenerator.java +++ b/tools/java-source-utils/src/main/java/com/microsoft/android/JavadocXmlGenerator.java @@ -2,9 +2,13 @@ import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import java.io.PrintStream; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.OutputKeys; @@ -16,6 +20,7 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.w3c.dom.NodeList; import com.microsoft.android.ast.*; import com.microsoft.android.util.Parameter; @@ -24,7 +29,10 @@ public final class JavadocXmlGenerator implements AutoCloseable { final PrintStream output; - public JavadocXmlGenerator(final String output) throws FileNotFoundException, UnsupportedEncodingException { + Document document; + Element api; + + public JavadocXmlGenerator(final String output) throws FileNotFoundException, ParserConfigurationException, UnsupportedEncodingException { if (output == null) this.output = System.out; else { @@ -35,40 +43,86 @@ public JavadocXmlGenerator(final String output) throws FileNotFoundException, Un } this.output = new PrintStream(file, "UTF-8"); } + + startApi(); } - public JavadocXmlGenerator(final PrintStream output) { + public JavadocXmlGenerator(final PrintStream output) throws ParserConfigurationException { Parameter.requireNotNull("output", output); this.output = output; + + startApi(); + } + + private void startApi() throws ParserConfigurationException { + document = DocumentBuilderFactory.newInstance () + .newDocumentBuilder() + .newDocument(); + api = document.createElement("api"); + api.setAttribute("api-source", "java-source-utils"); + document.appendChild(api); } - public void close() { + public void close() throws TransformerException { + Transformer transformer = TransformerFactory.newInstance() + .newTransformer(); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); + transformer.transform(new DOMSource(document), new StreamResult(output)); + if (output != System.out) { output.flush(); output.close(); } } + public final void writeCopyrightInfo(final File copyright, final String urlPrefix, final String urlStyle) throws IOException, ParserConfigurationException { + final Element info = document.createElement("javadoc-metadata"); + if (copyright != null) { + final Element blurb = document.createElement("copyright"); + final NodeList contents = readXmlFile(copyright); + if (contents == null) { + final byte[] data = Files.readAllBytes(copyright.toPath()); + blurb.appendChild(document.createCDATASection(new String(data, StandardCharsets.UTF_8))); + } else { + final int len = contents.getLength(); + for (int i = 0; i < len; ++i) + blurb.appendChild(document.importNode(contents.item(i),true)); + } + info.appendChild(blurb); + } + if (urlPrefix != null && urlStyle != null) { + final Element link = document.createElement("link"); + link.setAttribute("prefix", urlPrefix); + link.setAttribute("style", urlStyle); + + info.appendChild(link); + } + + if (info.hasChildNodes()) { + api.appendChild(info); + } + } + + final NodeList readXmlFile(final File file) throws ParserConfigurationException { + final DocumentBuilder builder = DocumentBuilderFactory.newInstance () + .newDocumentBuilder(); + try { + final Document contents = builder.parse(file); + return contents.getChildNodes(); + } + catch (Throwable t) { + return null; + } + } + public final void writePackages(final JniPackagesInfo packages) throws ParserConfigurationException, TransformerException { Parameter.requireNotNull("packages", packages); - final Document document = DocumentBuilderFactory.newInstance () - .newDocumentBuilder() - .newDocument(); - final Element api = document.createElement("api"); - api.setAttribute("api-source", "java-source-utils"); - document.appendChild(api); - for (JniPackageInfo packageInfo : packages.getSortedPackages()) { writePackage(document, api, packageInfo); } - - Transformer transformer = TransformerFactory.newInstance() - .newTransformer(); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); - transformer.transform(new DOMSource(document), new StreamResult(output)); } private static final void writePackage(final Document document, final Element api, final JniPackageInfo packageInfo) { diff --git a/tools/java-source-utils/src/test/java/com/microsoft/android/JavadocXmlGeneratorTest.java b/tools/java-source-utils/src/test/java/com/microsoft/android/JavadocXmlGeneratorTest.java index 28107fc68..11c4c5e97 100644 --- a/tools/java-source-utils/src/test/java/com/microsoft/android/JavadocXmlGeneratorTest.java +++ b/tools/java-source-utils/src/test/java/com/microsoft/android/JavadocXmlGeneratorTest.java @@ -18,7 +18,7 @@ public final class JavadocXmlGeneratorTest { @Test(expected = FileNotFoundException.class) - public void init_invalidFileThrows() throws FileNotFoundException, UnsupportedEncodingException { + public void init_invalidFileThrows() throws FileNotFoundException, ParserConfigurationException, TransformerException, UnsupportedEncodingException { try (JavadocXmlGenerator g = new JavadocXmlGenerator("/this/file/does/not/exist")) { } } @@ -38,6 +38,7 @@ public void testWritePackages_noPackages() throws ParserConfigurationException, JniPackagesInfo packages = new JniPackagesInfo(); generator.writePackages(packages); + generator.close(); final String expected = "\n" + @@ -96,6 +97,7 @@ public void testWritePackages_demo() throws ParserConfigurationException, Transf "\n"; generator.writePackages(packages); + generator.close(); assertEquals("global package + example packages", expected, bytes.toString()); } @@ -126,6 +128,7 @@ private static void testWritePackages(final String resourceJava, final String re final String expected = JniPackagesInfoTest.getResourceContents(resourceXml); generator.writePackages(packagesInfo); + generator.close(); // try (FileOutputStream o = new FileOutputStream(resourceXml + "-jonp.xml")) { // bytes.writeTo(o); // }