[class-parse] really support Java8 and other various doclet types. (#125)

atsushieno · jonpryor · commit f291722c08ec · 2017-02-14T10:15:39.000-05:00
ClassPath class had a red-herring property named "DocletType", which was
NEVER assigned and therefore never worked. Its existence is optimistic
so that it premised that any consuming code knows which doclet type its
argument javadoc is generated with.

So, just kill it and implement working code here.

Now that we get a working doclet parser, a handful of Java8 scraper issues
were discovered, so fixed them all. Namely:

- Arrays are replaced as :A, not mere [].
  - the new DroidDoc actually uses [] as is, and I believe it used to be
    so too in Javadocs. On the other hand, [] may show up only within
    droiddocs, so skip that string.Replace() for Java8.
- Java8 doc parsers simply passed '-' which actually needed to be escaped.
  It caused errors when argument names were simple like 'x-y' (which was
  treated as character range specification in regex).
- regex matching for "anything except for '('" should actually be
  "... except for '(' and ')'" and then they should have been escaped too.
diff --git a/src/Xamarin.Android.Tools.Bytecode/ClassPath.cs b/src/Xamarin.Android.Tools.Bytecode/ClassPath.cs
@@ -26,7 +26,7 @@ public class ClassPath {
 
 		public string ApiSource { get; set; }
 
-		public JavaDocletType DocletType { get; set; }
+		public JavaDocletType? DocletType { get; set; }
 
 		public IEnumerable<string> DocumentationPaths { get; set; }
 
@@ -224,9 +224,28 @@ void FixupParametersFromDocs (XElement api)
 			}
 		}
 
+		JavaDocletType GetDocletType (string path)
+		{
+			var kind = JavaDocletType.DroidDoc;
+			char [] buf = new char [500];
+			string packagesHtml = Path.Combine (path, "packages.html");
+			if (File.Exists (packagesHtml) && File.ReadAllText (packagesHtml).Contains ("<body class=\"gc-documentation develop reference api "))
+				kind = JavaDocletType.DroidDoc2;
+			using (var reader = File.OpenText (Path.Combine (path, "index.html")))
+				reader.ReadBlock (buf, 0, buf.Length);
+			string rawHTML = new string (buf);
+			if (rawHTML.Contains ("Generated by javadoc (build 1.6"))
+				kind = JavaDocletType.Java6;
+			else if (rawHTML.Contains ("Generated by javadoc (version 1.7"))
+				kind = JavaDocletType.Java7;
+			else if (rawHTML.Contains ("Generated by javadoc (1.8"))
+				kind = JavaDocletType.Java8;
+			return kind;
+		}
+
 		IAndroidDocScraper CreateDocScraper (string src)
 		{
-			switch (DocletType) {
+			switch (DocletType ?? GetDocletType (src)) {
 			default: return new DroidDoc2Scraper (src);
 			case JavaDocletType.DroidDoc: return new DroidDocScraper (src);
 			case JavaDocletType.Java6: return new JavaDocScraper (src);
diff --git a/src/Xamarin.Android.Tools.Bytecode/JavaDocumentScraper.cs b/src/Xamarin.Android.Tools.Bytecode/JavaDocumentScraper.cs
@@ -33,21 +33,14 @@
 
 namespace Xamarin.Android.Tools.Bytecode
 {
-	enum JavaDocKind {
-		DroidDoc,
-		DroidDoc2,
-		Java6,
-		Java7,
-		Java8
-	}
-	
 	class DroidDocScraper : AndroidDocScraper
 	{
 		const String pattern_head_droiddoc = "<span class=\"sympad\"><a href=\".*";
 
 		public DroidDocScraper (string dir)
 			: base (dir, pattern_head_droiddoc, null, " ", false)
 		{
+			ShouldEscapeBrackets = true;
 		}
 	}
 	
@@ -59,6 +52,7 @@ class DroidDoc2Scraper : AndroidDocScraper
 		public DroidDoc2Scraper (string dir)
 			: base (dir, pattern_head_droiddoc, reset_pattern_head, " ", true, "\\(", ", ", "\\)", "\\s*</code>")
 		{
+			ShouldEscapeBrackets = true;
 		}
 
 		string prev_path;
@@ -115,13 +109,15 @@ public Java7DocScraper (string dir)
 
 	class Java8DocScraper : AndroidDocScraper
 	{
-		const String pattern_head_javadoc = "<td class=\"col.+\"><code><strong><a href=\"[./]*"; // I'm not sure how path could be specified... (./ , ../ , or even /)
+		const String pattern_head_javadoc = "<td class=\"col.+\"><code><span class=\"memberNameLink\"><a href=\"[./]*"; // I'm not sure how path could be specified... (./ , ../ , or even /)
 		const String reset_pattern_head_javadoc = "<td><code>";
 		const String parameter_pair_splitter_javadoc = "&nbsp;";
 
 		public Java8DocScraper (string dir)
-			: base (dir, pattern_head_javadoc, reset_pattern_head_javadoc, parameter_pair_splitter_javadoc, true, "-", "-", "-", null)
+			: base (dir, pattern_head_javadoc, reset_pattern_head_javadoc, parameter_pair_splitter_javadoc, true, "\\-", "\\-", "\\-", null)
 		{
+			ShouldAlterArraySpec = true;
+			ShouldEliminateGenericArguments = true;
 		}
 	}
 
@@ -167,6 +163,10 @@ protected AndroidDocScraper (string dir, String patternHead, String resetPattern
 			//	LoadDocument (f.Substring (dir.Length + 1), f);
 		}
 
+		protected bool ShouldEscapeBrackets { get; set; }
+		protected bool ShouldAlterArraySpec { get; set; }
+		protected bool ShouldEliminateGenericArguments { get; set; }
+
 		protected virtual IEnumerable<string> GetContentLines (string path)
 		{
 			return File.ReadAllText (path).Split ('\n');
@@ -197,11 +197,18 @@ public virtual String[] GetParameterNames (string package, string type, string m
 			for (int i = 0; i < ptypes.Length; i++) {
 				if (i != 0)
 					buffer.Append (param_sep);
-				buffer.Append (ptypes [i].Replace ('$', '.'));
+				var ptype = ptypes [i];
+				if (ShouldEliminateGenericArguments)
+					while (ptype.IndexOf ('<') > 0 && ptype.IndexOf ('>') > ptype.IndexOf ('<'))
+						ptype = ptype.Substring (0, ptype.IndexOf ('<')) + ptype.Substring (ptype.IndexOf ('>') + 1);
+				buffer.Append (ptype.Replace ('$', '.'));
 			}
-			buffer.Replace ("[", "\\[").Replace ("]", "\\]");
+			if (ShouldEscapeBrackets)
+				buffer.Replace ("[", "\\[").Replace ("]", "\\]");
+			if (ShouldAlterArraySpec)
+				buffer.Replace ("[]", ":A");
 			buffer.Append (close_method);
-			buffer.Append ("\".*\\(([^(]*)\\)");
+			buffer.Append ("\".*\\(([^\\(\\)]*)\\)");
 			buffer.Append (post_close_method_parens);
 			buffer.Replace ("?", "\\?");
 			Regex pattern = new Regex (buffer.ToString (), RegexOptions.Multiline);

Original file line number	Diff line number	Diff line change
`@@ -33,21 +33,14 @@`
`33`	`33`
`34`	`34`	`namespace Xamarin.Android.Tools.Bytecode`
`35`	`35`	`{`
`36`		`- enum JavaDocKind {`
`37`		`- DroidDoc,`
`38`		`- DroidDoc2,`
`39`		`- Java6,`
`40`		`- Java7,`
`41`		`- Java8`
`42`		`- }`
`43`		`-`
`44`	`36`	`class DroidDocScraper : AndroidDocScraper`
`45`	`37`	`{`
`46`	`38`	`const String pattern_head_droiddoc = "<span class=\"sympad\"><a href=\".*";`
`47`	`39`
`48`	`40`	`public DroidDocScraper (string dir)`
`49`	`41`	`: base (dir, pattern_head_droiddoc, null, " ", false)`
`50`	`42`	`{`
	`43`	`+ ShouldEscapeBrackets = true;`
`51`	`44`	`}`
`52`	`45`	`}`
`53`	`46`
`@@ -59,6 +52,7 @@ class DroidDoc2Scraper : AndroidDocScraper`
`59`	`52`	`public DroidDoc2Scraper (string dir)`
`60`	`53`	`: base (dir, pattern_head_droiddoc, reset_pattern_head, " ", true, "\\(", ", ", "\\)", "\\s*</code>")`
`61`	`54`	`{`
	`55`	`+ ShouldEscapeBrackets = true;`
`62`	`56`	`}`
`63`	`57`
`64`	`58`	`string prev_path;`
`@@ -115,13 +109,15 @@ public Java7DocScraper (string dir)`
`115`	`109`
`116`	`110`	`class Java8DocScraper : AndroidDocScraper`
`117`	`111`	`{`
`118`		`- const String pattern_head_javadoc = "<td class=\"col.+\"><code><strong><a href=\"[./]*"; // I'm not sure how path could be specified... (./ , ../ , or even /)`
	`112`	`+ const String pattern_head_javadoc = "<td class=\"col.+\"><code><span class=\"memberNameLink\"><a href=\"[./]*"; // I'm not sure how path could be specified... (./ , ../ , or even /)`
`119`	`113`	`const String reset_pattern_head_javadoc = "<td><code>";`
`120`	`114`	`const String parameter_pair_splitter_javadoc = " ";`
`121`	`115`
`122`	`116`	`public Java8DocScraper (string dir)`
`123`		`- : base (dir, pattern_head_javadoc, reset_pattern_head_javadoc, parameter_pair_splitter_javadoc, true, "-", "-", "-", null)`
	`117`	`+ : base (dir, pattern_head_javadoc, reset_pattern_head_javadoc, parameter_pair_splitter_javadoc, true, "\\-", "\\-", "\\-", null)`
`124`	`118`	`{`
	`119`	`+ ShouldAlterArraySpec = true;`
	`120`	`+ ShouldEliminateGenericArguments = true;`
`125`	`121`	`}`
`126`	`122`	`}`
`127`	`123`
`@@ -167,6 +163,10 @@ protected AndroidDocScraper (string dir, String patternHead, String resetPattern`
`167`	`163`	`// LoadDocument (f.Substring (dir.Length + 1), f);`
`168`	`164`	`}`
`169`	`165`
	`166`	`+ protected bool ShouldEscapeBrackets { get; set; }`
	`167`	`+ protected bool ShouldAlterArraySpec { get; set; }`
	`168`	`+ protected bool ShouldEliminateGenericArguments { get; set; }`
	`169`	`+`
`170`	`170`	`protected virtual IEnumerable<string> GetContentLines (string path)`
`171`	`171`	`{`
`172`	`172`	`return File.ReadAllText (path).Split ('\n');`
`@@ -197,11 +197,18 @@ public virtual String[] GetParameterNames (string package, string type, string m`
`197`	`197`	`for (int i = 0; i < ptypes.Length; i++) {`
`198`	`198`	`if (i != 0)`
`199`	`199`	`buffer.Append (param_sep);`
`200`		`- buffer.Append (ptypes [i].Replace ('$', '.'));`
	`200`	`+ var ptype = ptypes [i];`
	`201`	`+ if (ShouldEliminateGenericArguments)`
	`202`	`+ while (ptype.IndexOf ('<') > 0 && ptype.IndexOf ('>') > ptype.IndexOf ('<'))`
	`203`	`+ ptype = ptype.Substring (0, ptype.IndexOf ('<')) + ptype.Substring (ptype.IndexOf ('>') + 1);`
	`204`	`+ buffer.Append (ptype.Replace ('$', '.'));`
`201`	`205`	`}`
`202`		`- buffer.Replace ("[", "\\[").Replace ("]", "\\]");`
	`206`	`+ if (ShouldEscapeBrackets)`
	`207`	`+ buffer.Replace ("[", "\\[").Replace ("]", "\\]");`
	`208`	`+ if (ShouldAlterArraySpec)`
	`209`	`+ buffer.Replace ("[]", ":A");`
`203`	`210`	`buffer.Append (close_method);`
`204`		`- buffer.Append ("\".\\(([^(])\\)");`
	`211`	`+ buffer.Append ("\".\\(([^\\(\\)])\\)");`
`205`	`212`	`buffer.Append (post_close_method_parens);`
`206`	`213`	`buffer.Replace ("?", "\\?");`
`207`	`214`	`Regex pattern = new Regex (buffer.ToString (), RegexOptions.Multiline);`