@(JavaSourceJar) & JDK11

[jonpryor]

@(JavaSourceJar) can be is a way to extract parameter name information from Javadoc files, and use that parameter name information within a .jar binding.

See e.g. https://github.com/xamarin/xamarin-android/blob/a111f6704d640481a6a85234ec4aa86ddbc18c3a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs#L434-L451

With the introduction of JDK 11 support #4567, @(JavaSourceJar) support is unfortunately broken, in part because JDK 11's javadoc utility emits different HTML, which isn't currently supported.

Steps to Reproduce

1. Revert the JavaSourceJar-related changes from PR JetBrains OpenJDK 11 #4567
2. Run the BindingBuildTest.JavaSourceJar() unit test
3. Watch it fail.

Expected Behavior

The unit test should work!

Actual Behavior

It doesn't.

Version Information

Log File

[jonpryor]

In-progress (and incomplete) xamarin-android patch to fix javadoc-to-mdoc.exe:

diff --git a/src/Xamarin.Android.Build.Tasks/MSBuild/Xamarin/Android/Xamarin.Android.Bindings.Documentation.targets b/src/Xamarin.Android.Build.Tasks/MSBuild/Xamarin/Android/Xamarin.Android.Bindings.Documentation.targets
index ad1e824a..c31086f5 100644
--- a/src/Xamarin.Android.Build.Tasks/MSBuild/Xamarin/Android/Xamarin.Android.Bindings.Documentation.targets
+++ b/src/Xamarin.Android.Build.Tasks/MSBuild/Xamarin/Android/Xamarin.Android.Bindings.Documentation.targets
@@ -16,12 +16,7 @@ This file is only used by binding projects. .NET 5 can eventually use it, once `
   <UsingTask TaskName="Xamarin.Android.Tasks.MDoc"          AssemblyFile="Xamarin.Android.Build.Tasks.dll" />
   <UsingTask TaskName="Xamarin.Android.Tasks.Unzip"         AssemblyFile="Xamarin.Android.Build.Tasks.dll" />
 
-  <PropertyGroup>
-    <_JavadocSupported Condition=" $(_JdkVersion.StartsWith ('1.8')) ">True</_JavadocSupported>
-  </PropertyGroup>
-
   <Target Name="_ExtractJavaDocJars"
-      Condition=" '$(_JavadocSupported)' == 'True' "
       Inputs="@(JavaDocJar)"
       Outputs="@(JavaDocJar->'$(IntermediateOutputPath)javadocs\%(FileName).stamp')">
     <Unzip
@@ -35,7 +30,6 @@ This file is only used by binding projects. .NET 5 can eventually use it, once `
   </Target>
 
   <Target Name="_GenerateJavaDocFromSourceJars"
-      Condition=" '$(_JavadocSupported)' == 'True' "
       Inputs="@(JavaSourceJar)"
       Outputs="@(JavaSourceJar->'$(IntermediateOutputPath)javadocs\%(FileName).stamp')">
     <Unzip
@@ -57,9 +51,9 @@ This file is only used by binding projects. .NET 5 can eventually use it, once `
   </Target>
 
   <Target Name="BuildDocumentation"
-      Condition=" '@(JavaDocIndex)' != '' And '$(_JavadocSupported)' == 'True' "
       Inputs="@(JavaDocIndex);@(IntermediateAssembly->'$(IntermediateOutputPath)%(filename).xml')"
-      Outputs="@(IntermediateAssembly->'$(OutputPath)%(filename).xml')">
+      Outputs="@(IntermediateAssembly->'$(OutputPath)%(filename).xml')"
+      Condition=" '@(JavaDocIndex)' != '' ">
     <MDoc
         References="@(ReferencePath);@(ReferenceDependencyPaths)"
         TargetAssembly="@(IntermediateAssembly)"
diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/JavaDoc.cs b/src/Xamarin.Android.Build.Tasks/Tasks/JavaDoc.cs
index a139415e..8e79e613 100644
--- a/src/Xamarin.Android.Build.Tasks/Tasks/JavaDoc.cs
+++ b/src/Xamarin.Android.Build.Tasks/Tasks/JavaDoc.cs
@@ -40,12 +40,19 @@ namespace Xamarin.Android.Tasks
 				context_src = pair.Source;
 				context_dst = pair.Destination;
 				base.RunTask ();
+				try {
+					File.Delete (context_responseFile);
+				}
+				catch (Exception e) {
+					Log.LogDebugMessage ($"Could not delete temporary file `{context_responseFile}`: {e.ToString ()}");
+				}
 			}
-			return true;
+			return !Log.HasLoggedErrors;
 		}
 
 		string context_src;
 		string context_dst;
+		string context_responseFile;
 
 		protected override string GenerateCommandLineCommands ()
 		{
@@ -55,8 +62,6 @@ namespace Xamarin.Android.Tasks
 			cmd.AppendFileNameIfNotNull (context_dst);
 			cmd.AppendSwitch ("-sourcepath");
 			cmd.AppendFileNameIfNotNull (context_src);
-			cmd.AppendSwitch ("-subpackages");
-			cmd.AppendSwitch (".");
 			var cps = ReferenceJars?.ToList () ?? new List<string> ();
 			if (JavaPlatformJar != null)
 				cps.Add (JavaPlatformJar);
@@ -70,6 +75,18 @@ namespace Xamarin.Android.Tasks
 				foreach (var extraArg in ExtraArgs)
 					cmd.AppendSwitch (extraArg);
 
+			context_responseFile    = Path.GetTempFileName ();
+			cmd.AppendSwitch ($"\"@{context_responseFile}\"");
+
+			var javaSources = Directory.EnumerateFiles (context_src, "*.java", SearchOption.AllDirectories)
+				.Select (file => Path.GetFullPath (file))
+				.ToList ();
+			File.AppendAllLines (context_responseFile, javaSources, MonoAndroidHelper.UTF8withoutBOM);
+			Log.LogDebugMessage ($"Response file `{context_responseFile}` contains: ");
+			foreach (var file in javaSources) {
+				Log.LogDebugMessage ($"  {file}");
+			}
+
 			return cmd.ToString ();
 		}
 
diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs
index 51ca9f7e..58db90e6 100644
--- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs
+++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs
@@ -445,38 +445,11 @@ namespace Foo {
 					BinaryContent = () => Convert.FromBase64String (InlineData.JavaSourcesJarBase64)
 				});
 				Assert.IsTrue (bindingBuilder.Build (binding), "binding build should have succeeded");
-				var jdkVersion = GetJdkVersion ();
-				if (jdkVersion > new Version (9, 0)) {
-					Assert.Ignore ("JDK 11 and @(JavaSourceJar) don't currently mix.");
-					return;
-				}
 				string xml = bindingBuilder.Output.GetIntermediaryAsText ("docs/Com.Xamarin.Android.Test.Msbuildtest/JavaSourceJarTest.xml");
 				Assert.IsTrue (xml.Contains ("<param name=\"name\"> - name to display.</param>"), "missing doc");
 			}
 		}
 
-		static Version GetJdkVersion ()
-		{
-			var jdkPath     = AndroidSdkResolver.GetJavaSdkPath ();
-			var releasePath = Path.Combine (jdkPath, "release");
-			if (!File.Exists (releasePath))
-				return null;
-			foreach (var line in File.ReadLines (releasePath)) {
-				const string JavaVersionStart = "JAVA_VERSION=\"";
-				if (!line.StartsWith (JavaVersionStart, StringComparison.OrdinalIgnoreCase))
-					continue;
-				var value   = line.Substring (JavaVersionStart.Length, line.Length - JavaVersionStart.Length - 1);
-				int last    = 0;
-				for (last = 0; last < value.Length; ++last) {
-					if (char.IsDigit (value, last) || value [last] == '.')
-						continue;
-					break;
-				}
-				return Version.Parse (last == value.Length ? value : value.Substring (0, last));
-			}
-			return null;
-		}
-
 		[Test]
 		[TestCaseSource (nameof (ClassParseOptions))]
 		public void DesignTimeBuild (string classParser)
diff --git a/src/Xamarin.Android.Tools.JavadocImporter/DocumentElement.cs b/src/Xamarin.Android.Tools.JavadocImporter/DocumentElement.cs
index 962ea487..819075df 100644
--- a/src/Xamarin.Android.Tools.JavadocImporter/DocumentElement.cs
+++ b/src/Xamarin.Android.Tools.JavadocImporter/DocumentElement.cs
@@ -410,4 +410,35 @@ namespace Xamarin.Android.Tools.JavaDocToMdoc
 			});
 		}
 	}
+
+	class JavaDoc11DocumentElement : JavaDoc7DocumentElement
+	{
+		public JavaDoc11DocumentElement (MdocHelper mdoc, XElement documentElement)
+			: base (mdoc, documentElement)
+		{
+		}
+
+		public override DocumentSection CreateSection (XElement sectionAnchor, string anchor, RegisterAttribute mregister)
+		{
+			return new JavaDoc11DocumentSection (sectionAnchor) {
+					Anchor              = anchor,
+					Mdoc                = Mdoc,
+					RegisterAttribute   = mregister,
+			};
+		}
+
+		public override XElement GetSectionAnchor (string localAnchor)
+		{
+			localAnchor = localAnchor.Replace (" ", string.Empty);
+			while (localAnchor.IndexOf ('<') >= 0 && localAnchor.IndexOf ('>') > localAnchor.IndexOf ('<'))
+				localAnchor = localAnchor.Substring (0, localAnchor.IndexOf ('<')) + localAnchor.Substring (localAnchor.IndexOf ('>') + 1);
+			localAnchor = localAnchor;
+			return Element.Descendants ("a").FirstOrDefault (a => {
+				var h = (string) a.Attribute ("id");
+				if (h == null || h != localAnchor)
+					return false;
+				return true;
+			});
+		}
+	}
 }
diff --git a/src/Xamarin.Android.Tools.JavadocImporter/DocumentSection.cs b/src/Xamarin.Android.Tools.JavadocImporter/DocumentSection.cs
index d858cf82..40e394ce 100644
--- a/src/Xamarin.Android.Tools.JavadocImporter/DocumentSection.cs
+++ b/src/Xamarin.Android.Tools.JavadocImporter/DocumentSection.cs
@@ -480,4 +480,24 @@ namespace Xamarin.Android.Tools.JavaDocToMdoc
 				.Select (n => n.NodeType == XmlNodeType.Text ? new XElement ("p", n) : n);
 		}
 	}
+
+	class JavaDoc11DocumentSection : JavaDocDocumentSection
+	{
+		public JavaDoc11DocumentSection (XElement anchorNode)
+			: base (anchorNode.ElementsAfterSelf ("ul").Elements ("li").FirstOrDefault ())
+		{
+		}
+
+		public override string SectionNameWrapperTag {
+			get { return "span"; }
+		}
+
+		public override IEnumerable<XNode> GetSummaryNodes ()
+		{
+			var div = section_node.Elements ("div").FirstOrDefault ();
+			return div == null ? Enumerable.Empty<XNode> () : div.Nodes ()
+				.Where (n => n.NodeType != XmlNodeType.Text || ((XText)n).Value.Length > 0)
+				.Select (n => n.NodeType == XmlNodeType.Text ? new XElement ("p", n) : n);
+		}
+	}
 }
diff --git a/src/Xamarin.Android.Tools.JavadocImporter/HtmlLoader.cs b/src/Xamarin.Android.Tools.JavadocImporter/HtmlLoader.cs
index 28a39fab..9d03e539 100644
--- a/src/Xamarin.Android.Tools.JavadocImporter/HtmlLoader.cs
+++ b/src/Xamarin.Android.Tools.JavadocImporter/HtmlLoader.cs
@@ -53,9 +53,11 @@ namespace Xamarin.Android.Tools.JavaDocToMdoc
 		{
 			DroidDoc,
 			DroidDoc2,
+			Java,
 			Java6,
 			Java7,
-			Java8
+			Java8,
+			Java11,
 		}
 
 		Sgml.SgmlDtd HtmlDtd;
@@ -87,6 +89,9 @@ namespace Xamarin.Android.Tools.JavaDocToMdoc
 				return new JavaDoc7DocumentElement (new JavaDocMdocHelper (), jd);
 			case JavaDocKind.Java8:
 				return new JavaDoc8DocumentElement (new JavaDocMdocHelper (), jd);
+			case JavaDocKind.Java:
+			case JavaDocKind.Java11:
+				return new JavaDoc11DocumentElement (new JavaDocMdocHelper (), jd);
 			}
 			throw new InvalidOperationException ();
 		}
@@ -115,10 +120,15 @@ namespace Xamarin.Android.Tools.JavaDocToMdoc
 				kind = JavaDocKind.DroidDoc2;
 			if (rawHTML.Substring (0, Math.Min (rawHTML.Length, 500)).IndexOf ("Generated by javadoc (build 1.6", StringComparison.Ordinal) > 0)
 				kind = JavaDocKind.Java6;
-			if (rawHTML.Substring (0, Math.Min (rawHTML.Length, 500)).IndexOf ("Generated by javadoc (version 1.7", StringComparison.Ordinal) > 0)
+			else if (rawHTML.Substring (0, Math.Min (rawHTML.Length, 500)).IndexOf ("Generated by javadoc (version 1.7", StringComparison.Ordinal) > 0)
 				kind = JavaDocKind.Java7;
-			if (rawHTML.Substring (0, Math.Min (rawHTML.Length, 500)).IndexOf ("Generated by javadoc (1.8", StringComparison.Ordinal) > 0)
+			else if (rawHTML.Substring (0, Math.Min (rawHTML.Length, 500)).IndexOf ("Generated by javadoc (1.8", StringComparison.Ordinal) > 0)
 				kind = JavaDocKind.Java8;
+			else if (rawHTML.Substring (0, Math.Min (rawHTML.Length, 500)).IndexOf ("Generated by javadoc (11.", StringComparison.Ordinal) > 0)
+				kind = JavaDocKind.Java11;
+			else if (rawHTML.Substring (0, Math.Min (rawHTML.Length, 500)).IndexOf ("Generated by javadoc ", StringComparison.Ordinal) > 0)
+				kind = JavaDocKind.Java;
+
 			if (kind == JavaDocKind.DroidDoc) {
 				throw new NotSupportedException ("Old DroidDoc is not supported anymore.");
 			} else {

[jonpryor]

In-progress (and incomplete) changes to Java.Interop allow reading of JDK11 Javadoc:

commit 280b9ce7f399f50aea1499fa37b40f482356c93c
Author: Jonathan Pryor <jonpryor@vt.edu>
Date:   Thu May 28 18:54:40 2020 -0400

    Javadoc 11 support?

diff --git a/external/xamarin-android-tools b/external/xamarin-android-tools
index 23c4fe0..f5fcb9f 160000
--- a/external/xamarin-android-tools
+++ b/external/xamarin-android-tools
@@ -1 +1 @@
-Subproject commit 23c4fe0a989f8d0887786fdee6db635387597c13
+Subproject commit f5fcb9fd1583ebe928734e8525303a0dbe93c79c
diff --git a/src/Xamarin.Android.Tools.Bytecode/ClassPath.cs b/src/Xamarin.Android.Tools.Bytecode/ClassPath.cs
index 62adb9a..5909c4a 100644
--- a/src/Xamarin.Android.Tools.Bytecode/ClassPath.cs
+++ b/src/Xamarin.Android.Tools.Bytecode/ClassPath.cs
@@ -19,6 +19,8 @@ namespace Xamarin.Android.Tools.Bytecode {
 		Java8,
 		_ApiXml,
 		JavaApiParameterNamesXml,
+		Java,
+		Java11,
 	}
 
 	public class ClassPath {
@@ -247,9 +249,11 @@ namespace Xamarin.Android.Tools.Bytecode {
 			switch (JavaMethodParameterNameProvider.GetDocletType (src)) {
 			default: return new DroidDoc2Scraper (src);
 			case JavaDocletType.DroidDoc: return new DroidDocScraper (src);
-			case JavaDocletType.Java6: return new JavaDocScraper (src);
+			case JavaDocletType.Java: return new JavaDocScraper (src);
+			case JavaDocletType.Java6: return new Java6DocScraper (src);
 			case JavaDocletType.Java7: return new Java7DocScraper (src);
 			case JavaDocletType.Java8: return new Java8DocScraper (src);
+			case JavaDocletType.Java11: return new Java11DocScraper (src);
 			case JavaDocletType._ApiXml: return new ApiXmlDocScraper (src);
 			case JavaDocletType.JavaApiParameterNamesXml: return new JavaParameterNamesLoader (src);
 			}
diff --git a/src/Xamarin.Android.Tools.Bytecode/JavaDocumentScraper.cs b/src/Xamarin.Android.Tools.Bytecode/JavaDocumentScraper.cs
index 3adb738..a52e164 100644
--- a/src/Xamarin.Android.Tools.Bytecode/JavaDocumentScraper.cs
+++ b/src/Xamarin.Android.Tools.Bytecode/JavaDocumentScraper.cs
@@ -83,13 +83,13 @@ namespace Xamarin.Android.Tools.Bytecode
 		}
 	}
 
-	class JavaDocScraper : AndroidDocScraper
+	class Java6DocScraper : AndroidDocScraper
 	{
 		const String pattern_head_javadoc = "<TD><CODE><B><A HREF=\"[./]*"; // I'm not sure how path could be specified... (./ , ../ , or even /)
 		const String reset_pattern_head_javadoc = "<TD><CODE>";
 		const String parameter_pair_splitter_javadoc = "&nbsp;";
 	
-		public JavaDocScraper (string dir)
+		public Java6DocScraper (string dir)
 			: base (dir, pattern_head_javadoc, reset_pattern_head_javadoc, parameter_pair_splitter_javadoc, false)
 		{
 		}
@@ -129,6 +129,35 @@ namespace Xamarin.Android.Tools.Bytecode
 		}
 	}
 
+	class Java11DocScraper : AndroidDocScraper
+	{
+		const String pattern_head_javadoc = "<td class=\"col.+\" scope=\"row\"><code><span class=\"memberNameLink\"><a href=\"[./]*";
+		const String reset_pattern_head_javadoc = "<td><code>";
+		const String parameter_pair_splitter_javadoc = "&nbsp;";
+
+		public Java11DocScraper (string dir)
+			: base (dir, pattern_head_javadoc, reset_pattern_head_javadoc, parameter_pair_splitter_javadoc, true, "\\-", "\\-", "\\-", null)
+		{
+			ShouldAlterArraySpec = true;
+			ShouldEliminateGenericArguments = true;
+		}
+
+		protected override string StripTagsFromParameters (string value)
+		{
+			// Java8 javadoc contains possibly linked types with <a> tags, so remove all of them.
+			while (value.IndexOf ('<') >= 0 && value.IndexOf ('>') > value.IndexOf ('<'))
+				value = value.Substring (0, value.IndexOf ('<')) + value.Substring (value.IndexOf ('>') + 1);
+			return value;
+		}
+	}
+
+	class JavaDocScraper : Java11DocScraper {
+		public JavaDocScraper (string dir)
+			: base (dir)
+		{
+		}
+	}
+
 	public abstract class AndroidDocScraper : IJavaMethodParameterNameProvider
 	{
 		readonly String pattern_head;
@@ -163,8 +192,14 @@ namespace Xamarin.Android.Tools.Bytecode
 				throw new Exception ("Directory '" + dir + "' does not exist");
 	
 			root = dir;
+
+			var packageFiles = new[]{
+				Path.Combine (dir, "package-list"),
+				Path.Combine (dir, "packages.html"),
+				Path.Combine (dir, "allpackages-index.html"),
+			};
 	
-			if (!File.Exists (Path.Combine (dir, "package-list")) && !File.Exists (Path.Combine (dir, "packages.html")))
+			if (!packageFiles.Any (File.Exists))
 				throw new ArgumentException ("Directory '" + dir + "' does not appear to be an android doc reference directory.");
 			
 			//foreach (var f in Directory.GetFiles (dir, "*.html", SearchOption.AllDirectories))
@@ -319,6 +354,10 @@ namespace Xamarin.Android.Tools.Bytecode
 					kind = JavaDocletType.Java7;
 				else if (rawHTML.Contains ("Generated by javadoc (1.8"))
 					kind = JavaDocletType.Java8;
+				else if (rawHTML.Contains ("Generated by javadoc (11."))
+					kind = JavaDocletType.Java11;
+				else if (rawHTML.Contains ("Generated by javadoc "))
+					kind = JavaDocletType.Java;
 			}
 
 			// Check to see if it's an api.xml formatted doc

[jonpryor]

Better solution: dotnet/java-interop#687.