Remove DroidDoc dependencies from enumification helpers.

api-*.xml.in can be generated dynamically (up to [*1]) but enumification
helper tools still depended on DroidDoc, meaning that we cannot remove
package download dependencies. This fixes the situation by eliminating
the dependencies. Especially it is easier to just parse api-*.xml.in to
extract int consts from Java API.

... thought so? It turnd out to not be that easy.

Unlike DroidDoc, api-*.xml.in does not contain "api-since" information,
so we need to parse ALL the API XML. And they cannot be **precise**
because we only have a subset of the API definitions. For example,
"since API Level 1" now becomes "since API Level 10" because we don't
have api-1.xml.in (not even api-4.xml.in). So they will become non-precise.

(A slightly better possibility to "fix" this is to parse
`android-sdk-whatever/platform-tools/api/api-versions.xml`, but it won't
contain "already vanished" constants so it will be incomplete either.
I didn't bother to do "better" so far. We don't need that for enumification.)

Therefore, there is a lot of changes in `map.csv` to reflect those
"insignificant" changes to remap "API since" column (e.g. from "1" to "10").

Regeneration from api-LEVEL.xml.in instead of DroidDoc has some other
side effects; some consts are back to the list so that we don't have to
manually copy existing mapped consts that could not be generated from
the latest DroidDocs (due to disappearance) anymore. Instead we have to
remove manually-mapped constants (either from the sources or `map.csv`).
Therefore, a handful of heading lines in `map.csv` were removed, and
some definitions in `enum-conversion-mappings.xml` are back.

[*1] #1038

Author: atsushieno

build-tools/enumification-helpers/Makefile

@@ -1,12 +1,11 @@
 topdir = ../..
 
-API_LEVEL = 26
-API_LEVEL_NAME = 26
+API_LEVEL = 27
+API_LEVEL_NAME = 27
 
 CSCOMPILE = csc -debug:portable
 
 CONFIGURATION = Debug
-DOCS_DIR=~/android-toolchain/docs/docs-api-$(API_LEVEL_NAME)
 
 CONST_MAPPER_SOURCES = \
 	generate-const-mapping.cs \
@@ -15,20 +14,11 @@ CONST_MAPPER_SOURCES = \
 
 all:: map.ext.csv remaining-int-consts.txt remaining-int-methods-filtered.txt
 
-#enum mappings
+generate-const-list-2.exe: generate-const-list-2.cs
+	$(CSCOMPILE) generate-const-list-2.cs
 
-generate-const-list.exe: generate-const-list.cs
-	$(CSCOMPILE) generate-const-list.cs
-
-const-list-$(API_LEVEL).xml: generate-const-list.exe
-	mono --debug generate-const-list.exe $(DOCS_DIR)/reference/ -v > const-list-$(API_LEVEL).xml || rm const-list-$(API_LEVEL).xml
-
-generate-intermediary-enum-candidates.exe : generate-intermediary-enum-candidates.cs
-	$(CSCOMPILE) generate-intermediary-enum-candidates.cs
-
-#it is obsolete
-intermediary-enum-candidates.xml: generate-intermediary-enum-candidates.exe
-	mono --debug generate-intermediary-enum-candidates.exe $(topdir)/src/Mono.Android/api-$(API_LEVEL).xml.in > intermediary-enum-candidates.xml || rm intermediary-enum-candidates.xml
+const-list-$(API_LEVEL).xml: generate-const-list-2.exe ../../src/Mono.Android/Profiles/*.xml.in
+	mono --debug generate-const-list-2.exe ../../src/Mono.Android/Profiles > const-list-$(API_LEVEL).xml || rm const-list-$(API_LEVEL).xml
 
 generate-const-mapping.exe: $(CONST_MAPPER_SOURCES)
 	$(CSCOMPILE) $(CONST_MAPPER_SOURCES)
@@ -47,28 +37,6 @@ remaining-int-consts.txt: ../../src/Mono.Android/obj/$(CONFIGURATION)/android-$(
 		| sed -e 's/android-[0-9]*/android-XXX/' \
 		> remaining-int-consts.txt
 
-# method mappings
-generate-intermediary-doc-enum-mapping.exe: generate-intermediary-doc-enum-mapping.cs
-	$(CSCOMPILE) generate-intermediary-doc-enum-mapping.cs
-
-generate-enumlist-to-query.exe : generate-enumlist-to-query.cs
-	$(CSCOMPILE) generate-enumlist-to-query.cs
-
-generate-intermediary-method-mapping.exe: generate-intermediary-method-mapping.cs
-	$(CSCOMPILE) generate-intermediary-method-mapping.cs
-
-intermediary-enum-list.txt: generate-enumlist-to-query.exe
-	mono --debug generate-enumlist-to-query.exe const-list-$(API_LEVEL).xml > intermediary-enum-list.txt || rm intermediary-enum-list.txt
-
-intermediary-doc-enum-mapping.tsv: generate-intermediary-doc-enum-mapping.exe intermediary-enum-list.txt
-	mono --debug generate-intermediary-doc-enum-mapping.exe $(DOCS_DIR)/reference intermediary-enum-list.txt > intermediary-doc-enum-mapping.tsv || rm intermediary-doc-enum-mapping.tsv
-
-intermediary-method-mapping.txt: generate-intermediary-method-mapping.exe
-	mono --debug generate-intermediary-method-mapping.exe intermediary-doc-enum-mapping.tsv const-list-$(API_LEVEL).xml $(DOCS_DIR)/reference > intermediary-method-mapping.txt || rm intermediary-method-mapping.txt
-
-intermediary-field-candidates.txt: intermediary-method-mapping.txt
-	grep "#[a-z]" intermediary-method-mapping.txt  | grep -v '('  > intermediary-field-candidates.txt
-
 remaining-int-methods-filtered.txt: remaining-int-methods.txt reduction_rules.txt
 	vi remaining-int-methods.txt -s reduction_rules.txt
 	grep "int[\[ ]" remaining-int-methods.txt > remaining-int-methods-filtered.txt
@@ -78,8 +46,6 @@ remaining-int-methods.txt: $(topdir)/src/Mono.Android/obj/$(CONFIGURATION)/andro
 
 clean::
 	-rm -rf tmp.xml const-list-$(API_LEVEL).xml \
-		generate-const-list.exe generate-const-list.exe.mdb \
-		generate-intermediary-enum-candidates.exe generate-intermediary-enum-candidates.exe.mdb \
+		generate-const-list-2.exe generate-const-list-2.exe.mdb \
 		generate-const-mapping.exe generate-const-mapping.exe.mdb \
 		map.ext.csv intermediate-enum-candidates.xml remaining-int-consts.txt \
-		generate-intermediary-doc-enum-mapping.exe generate-intermediary-doc-enum-mapping.exe.mdb

build-tools/enumification-helpers/README …ld-tools/enumification-helpers/README.mdbuild-tools/enumification-helpers/README renamed to build-tools/enumification-helpers/README.md build-tools/enumification-helpers/README renamed to build-tools/enumification-helpers/README.md

@@ -1,4 +1,70 @@
 
+# Enumification
+
+This directory contains a couple of "helper" tools for API int-to-enum 
+conversion. You need these tools only when you are going to make changes
+to existing enum mappings in Mono.Android.dll.
+
+This work is partly done by both parsing API XML definition (`api-*.xml.in`) and by human.
+
+The outcomes here are saved as "map.ext.csv" and "methodmap.ext.csv"
+that are to be appended to "map.csv" and "methodmap.csv" in Mono.Android source.
+
+Historically this enumification work was done by scraping Android API
+References (DroidDocs) until 2017, but things get simplified nowadays.
+
+
+## Const lookup
+
+Const lookup is simpler and easier to deal with (it still needs manual work). Here are the steps to generate new mappings:
+
+- Run "make" here.
+  - It generates a list of consts as `const-list-*.xml` from `api-*.xml.in` using `generate-const-list-2.exe`.
+- Read `const-list-*.xml` changes from the local git repo and find "better
+  to be enumified" consts, then manually add new entries in
+  `enum-conversion-mappings.xml`.
+- Run "make" here (again)
+  - it runs `generate-const-mapping.exe` which generates `map.ext.csv` from
+  `enum-conversion-mappings.xml`.
+- Replace "generated" part of entries in `map.csv` with the entire contents
+  of `map.ext.csv`.
+- Run "make" at top level, or something that triggers Mono.Android.dll builds 
+  - Actually dll doesn't matter, we only need the
+  updated `api-*.xml.in`.
+- come back to this directory and iterate from the top of this list.
+  - Run "make" and it will result in the updated `remaining-int-consts.txt`,
+    which helps you finding remaining int consts that are possible to be mapped.
+    - `git diff remaining-int-consts.txt` is helpful.
+
+
+## Method/constructor int arguments lookup
+
+As of 2017 we basically rely on git history diff on `remaining-int-methods-filtered.txt` from the past results.
+
+After running "make" here, methods that takes one or more int arguments
+are listed on `remaining-int-methods.txt`, and `remaining-int-methods-filtered.txt` for "saner" results with some filtering.
+Filtering is done for argument names e.g. "x", "y", "width" and "height"
+are most unlikely enums, and they are based on `reduction-rules.txt`
+which are just list of `sed` replacements.
+
+When a new API has arrived, this `remaining-int-methods-filtered.txt`
+come up with a lot of additions to the existing file. So, pick up them
+and add more `methodmap.ext.txt` for further filtering, and add it
+`methodmap.txt`.
+
+(It was done historically to distinguish those new lines from the
+ancient ages, but we most unlikely need to maintain two lists anymore...)
+
+
+
+
+## Obsoleted
+
+but sometimes contain useful hints so we still keep this...
+
+
+<pre>
+
 This directory contains a couple of "helper" tools for API int-to-enum conversion.
 
 This is partly done by scraping Android API reference, but it's mostly done by human.
@@ -273,3 +339,5 @@ The results are to be saved as "map.ext.csv" and "methodmap.ext.csv" that are to
 
 	intermediary-enum-candidates.txt is not in use. Not useful for the
 	same reason as intermediary-enum-list.txt.
+
+</pre>

build-tools/enumification-helpers/enum-conversion-mappings.xml

@@ -645,11 +645,7 @@
   <map package="android.accessibilityservice" class="AccessibilityService" fields="" prefix="GESTURE_" enum-name="AccessibilityGesture" api-level="16" />
   <map package="android.accessibilityservice" class="AccessibilityService" fields="" prefix="GLOBAL_ACTION_" enum-name="GlobalAction" api-level="16" />
   <map package='android.app' class='Notification' fields='' prefix='PRIORITY_' enum-name="NotificationPriority" api-level="16" />
-  <!--
-  This had to be explicitly removed from auto-generation. see map.csv for the reason.
-  -->
-  <!--
-  map package="android.content.pm" class="PackageInfo" fields="" prefix="REQUESTED_PERMISSION_" enum-name="RequestedPermission" api-level="16" / -->
+  <map package="android.content.pm" class="PackageInfo" fields="" prefix="REQUESTED_PERMISSION_" enum-name="RequestedPermission" api-level="16" />
   <map package='android.media' class='MediaCodecInfo$CodecCapabilities' fields='' prefix='COLOR_' enum-name="MediaCodecCapabilities" api-level="16" />
   <!-- for those constants category, see http://tools.oesf.biz/android-4.1.1_r1.0/xref/frameworks/base/media/java/android/media/MediaCodecInfo.java -->
   <map package='android.media' class='MediaCodecInfo$CodecProfileLevel' fields='.*Level.*' enum-name="MediaCodecProfileLevel" api-level="16" />
@@ -940,7 +936,6 @@
   -->
   <map package='android.provider' class='Browser' prefix='HISTORY_PROJECTION_' enum-name='HistoryProjection' is-transient='false' />
   <map package='android.provider' class='Browser' prefix='SEARCHES_PROJECTION_' enum-name='SearchesProjection' is-transient='false' />
-  <map package='android.provider' class='Browser' prefix='SEARCHES_PROJECTION_' enum-name='HistoryProjection' is-transient='false' />
   <map package='android.provider' class='ContactsContract.ProviderStatus' prefix='STATUS_' enum-name='ContactsProviderStatus' is-transient='false' />
 
   <!-- New in API Level 24 -->

build-tools/enumification-helpers/generate-const-list-2.cs

@@ -0,0 +1,94 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Xml;
+using System.Xml.Linq;
+
+
+public class Driver
+{
+	class Constant {
+		public string Package, ParentType, Level, FieldType, Name, Value;
+		public bool IsTypeInterface;
+	}
+	
+	static string GetApiLevel (string file)
+	{
+		Console.Error.WriteLine ($"[{file}]");
+		return file.Substring (0, file.Length - ".xml.in".Length).Substring ("api-".Length);
+	}
+	
+	public static void Main (string [] args)
+	{
+		var docs = args.SelectMany (a => Directory.GetFiles (a, "api-*.xml.in"))
+			.Select (file => XDocument.Load (file, LoadOptions.SetBaseUri));
+		Dictionary<string,string> levels = docs.Select (doc => new { Doc = doc.BaseUri, File = Path.GetFileName (doc.BaseUri) })
+			.Select (p => new { Doc = p.Doc, Level = GetApiLevel (p.File) })
+			.ToDictionary (p => p.Doc, p => p.Level);
+		var results = docs.Select (doc => doc.Root.Elements ("package"))
+			.SelectMany (p => p.Elements ())
+			.SelectMany (t => t.Elements ("field"))
+			.Where (f => f.Attribute ("type")?.Value == "int")
+			.Where (f => f.Attribute ("final")?.Value == "true" && f.Attribute ("value") != null)
+			.ToArray ();
+		var consts = results.Select (f => new Constant {
+			Package = f.Parent.Parent.Attribute ("name").Value,
+			ParentType = f.Parent.Attribute ("name").Value,
+			IsTypeInterface = f.Parent.Name.LocalName == "interface",
+			Name = f.Attribute ("name").Value,
+			FieldType = f.Attribute ("type").Value,
+			Value = f.Attribute ("value").Value,
+			Level = levels [f.Document.BaseUri] 
+			})
+			.OrderBy (c => c.Package)
+			.ThenBy (c => c.ParentType)
+			.ThenBy (c => c.Name)
+			.ThenBy (c => c.Level)
+			.ToArray ();
+
+		for (int i = 1; i < consts.Length; i++) {
+			int x = 1;
+			while (consts [i - x].Name == consts [i].Name && consts [i - x].ParentType == consts [i].ParentType && consts [i - x].Package == consts [i].Package && consts [i - x].Value != consts [i].Value) {
+				Console.Error.WriteLine ("Overwrite field value: {0}.{1}.{2}: {3} (at {4}) -> {5} (at {6})", consts [i - x].Package, consts [i - x].ParentType, consts [i - x].Name, consts [i - x].Value, consts [i - x].Level, consts [i].Value, consts [i].Level);
+				consts [i - x].Value = consts [i].Value;
+				x++;
+			}
+		}
+
+		var fields = new List<string> ();
+		string package = null, type = null;
+		var writer = XmlWriter.Create (Console.Out, new XmlWriterSettings { Indent = true });
+		writer.WriteStartElement ("enums");
+		foreach (var c in consts) {
+			if (c.Package != package) {
+				if (package != null) {
+					writer.WriteEndElement (); // type
+					type = null;
+					writer.WriteEndElement (); // package
+				}
+				package = c.Package;
+				writer.WriteStartElement ("package");
+				writer.WriteAttributeString ("name", package);
+			}
+			if (c.ParentType != type) {
+				if (type != null)
+					writer.WriteEndElement ();
+				type = c.ParentType;
+				writer.WriteStartElement (c.IsTypeInterface ? "interface" : "class");
+				writer.WriteAttributeString ("name", c.ParentType);
+				fields.Clear ();
+			}
+			if (fields.Contains (c.Name))
+				continue;
+			fields.Add (c.Name);
+			writer.WriteStartElement ("const");
+			writer.WriteAttributeString ("type", c.FieldType);
+			writer.WriteAttributeString ("name", c.Name);
+			writer.WriteAttributeString ("api-level", c.Level);
+			writer.WriteString (c.Value);
+			writer.WriteEndElement ();
+		}
+		writer.Close ();
+	}
+}