[Xamarin.Android.Tools.Bytecode] Hide Kotlin nested types inside (#723)

Fixes: #682

Context: dotnet/android#4955

We added support for reading Kotlin-provided metadata in commit
439bd83.  Part of this was recognizing when a type was
"Kotlin-internal" and marking it as `private` or "package-private" to
prevent `generator` from attempting to bind it. 

However, we didn't consider *nested* types of `private` or
"package-private" types in 439bd83, leaving the nested type
visibility unchanged.  This could result in a warning when binding
such nested types:

	warning BG8604: top ancestor ParentType not found for nested type ParentType.ChildEnum 

Note: this doesn't prevent building -- unless `csc /warnaserror` is
used -- and the type is not bound, so we are doing the "correct" thing.
This warning *does* create a "less-than-desirable" user experience,
possibly confusing users.

Update `generator` to change the visibility of nested types within
non-`public` types so that `generator` doesn't attempt to bind the
nested types.  This avoids the BG8604 warning.

Author: jpobst

src/Xamarin.Android.Tools.Bytecode/Kotlin/KotlinFixups.cs

@@ -67,21 +67,41 @@ static void FixupClassVisibility (ClassFile klass, KotlinClass metadata)
 			// Hide class if it isn't Public/Protected
 			if (klass.AccessFlags.IsPubliclyVisible () && !metadata.Visibility.IsPubliclyVisible ()) {
 
-				// Interfaces should be set to "package-private", which is "no visibility flags"
+				// Interfaces should be set to "package-private"
 				if (klass.AccessFlags.HasFlag (ClassAccessFlags.Interface)) {
-					Log.Debug ($"Kotlin: Setting interface {klass.ThisClass.Name.Value} to package-private");
-					klass.AccessFlags = (klass.AccessFlags ^ ClassAccessFlags.Public) & klass.AccessFlags;
-					klass.AccessFlags = (klass.AccessFlags ^ ClassAccessFlags.Protected) & klass.AccessFlags;
-					klass.AccessFlags = (klass.AccessFlags ^ ClassAccessFlags.Private) & klass.AccessFlags;
+					Log.Debug ($"Kotlin: Setting internal interface {klass.ThisClass.Name.Value} to package-private");
+					klass.AccessFlags = SetPackagePrivate (klass.AccessFlags);
+
+					foreach (var ic in klass.InnerClasses) {
+						Log.Debug ($"Kotlin: Setting nested type {ic.InnerClass.Name.Value} in an internal interface to package-private");
+						ic.InnerClassAccessFlags = SetPackagePrivate (ic.InnerClassAccessFlags);
+					}
+
 					return;
 				}
 
 				Log.Debug ($"Kotlin: Hiding internal class {klass.ThisClass.Name.Value}");
 				klass.AccessFlags = ClassAccessFlags.Private;
+
+				foreach (var ic in klass.InnerClasses) {
+					Log.Debug ($"Kotlin: Hiding nested internal type {ic.InnerClass.Name.Value}");
+					ic.InnerClassAccessFlags = ClassAccessFlags.Private;
+				}
+
 				return;
 			}
 		}
 
+		static ClassAccessFlags SetPackagePrivate (ClassAccessFlags flags)
+		{
+			// Package-private is stored as "no visibility flags"
+			flags = (flags ^ ClassAccessFlags.Public) & flags;
+			flags = (flags ^ ClassAccessFlags.Protected) & flags;
+			flags = (flags ^ ClassAccessFlags.Private) & flags;
+
+			return flags;
+		}
+
 		static void FixupJavaMethods (Methods methods)
 		{
 			// We do the following method level fixups here because we can operate on all methods,

tests/Xamarin.Android.Tools.Bytecode-Tests/KotlinFixupsTests.cs

@@ -15,27 +15,36 @@ public class KotlinFixupsTests : ClassFileFixture
 		public void HideInternalClass ()
 		{
 			var klass = LoadClassFile ("InternalClass.class");
+			var inner_class = klass.InnerClasses.First ();
 
 			Assert.True (klass.AccessFlags.HasFlag (ClassAccessFlags.Public));
+			Assert.True (inner_class.InnerClassAccessFlags.HasFlag (ClassAccessFlags.Public));
 
 			KotlinFixups.Fixup (new [] { klass });
 
 			Assert.False (klass.AccessFlags.HasFlag (ClassAccessFlags.Public));
+			Assert.False (inner_class.InnerClassAccessFlags.HasFlag (ClassAccessFlags.Public));
 		}
 
 		[Test]
 		public void MakeInternalInterfacePackagePrivate ()
 		{
 			var klass = LoadClassFile ("InternalInterface.class");
+			var inner_class = klass.InnerClasses.First ();
 
 			Assert.True (klass.AccessFlags.HasFlag (ClassAccessFlags.Public));
+			Assert.True (inner_class.InnerClassAccessFlags.HasFlag (ClassAccessFlags.Public));
 
 			KotlinFixups.Fixup (new [] { klass });
 
 			// "package-private" is denoted as no visibility flags
 			Assert.False (klass.AccessFlags.HasFlag (ClassAccessFlags.Public));
 			Assert.False (klass.AccessFlags.HasFlag (ClassAccessFlags.Protected));
 			Assert.False (klass.AccessFlags.HasFlag (ClassAccessFlags.Private));
+
+			Assert.False (inner_class.InnerClassAccessFlags.HasFlag (ClassAccessFlags.Public));
+			Assert.False (inner_class.InnerClassAccessFlags.HasFlag (ClassAccessFlags.Protected));
+			Assert.False (inner_class.InnerClassAccessFlags.HasFlag (ClassAccessFlags.Private));
 		}
 
 		[Test]

tests/Xamarin.Android.Tools.Bytecode-Tests/kotlin/InternalClass.class

@@ -1 +1,3 @@
-internal class InternalClass
+internal class InternalClass {
+    enum class ChildEnum { VALUE1, VALUE2 }
+}

tests/Xamarin.Android.Tools.Bytecode-Tests/kotlin/InternalClass.kt

@@ -1 +1,3 @@
-internal interface InternalInterface
+internal interface InternalInterface {
+    enum class ChildEnum { VALUE1, VALUE2 }
+}