[class-parse] Use annotated-visibility

Context: #1097 (comment)
Context: #1097 (comment)
Context: b274a67

During code review, two questions came to light:

 1. Will this cause API breakage in AndroidX/etc., because
    `module-info.class` will make types "disappear"?
    (See also b274a67).

 2. `class-parse` can accept *multiple* `.jar` files and `.class`
    files on the command-line.  Should `module-info.class` processing
    apply to *everything*?

Address the first concern by keeping `//*/@visibility` as `public`,
and instead if we think the type should be "internal" we instead add
an `//*/@annotated-visibility` attribute value of `module-info`.
If there is *already* an `//*/@annotated-visibility` value, then we
*append* ` module-info` to the attribute value.

Address the second concern by re-working `class-parse` command-line
parsing.  There is now a "global `ClassPath`", which will be used
to hold `.class` files provided on the command-line.  `.jar` and
`.jmod` files provided on the command-line will be given their own
`ClassPath` instances, and `module-info.class`-based visibility
fixups are specific to each `ClassPath` instance.  Global files are
processed together.  There is thus no way for `module-info.class`
visibility changes from `a.jar` to impact `b.jar`.  After
visibilities are fixed up, we then merge everything into the "global"
`ClassPath` instance before writing transforming to XML.

Additionally, `class-parse --dump` can now accept `.jar` files,
and will dump out *all* `.class` filers within the `.jar` file.
To make this output easier, each "entry" starts with a "header" of
`-- Begin {ClassFile.FullJniName}`, and a blank link will be printed
between each entry.

Author: jonpryor

src/Xamarin.Android.Tools.Bytecode/ClassPath.cs

@@ -108,6 +108,14 @@ public void Add (ClassFile classFile)
 			classFiles.Add (classFile);
 		}
 
+		public void Add (ClassPath classPath, bool removeModules = true)
+		{
+			classPath.FixupModuleVisibility (removeModules);
+			foreach (var c in classPath.classFiles) {
+				Add (c);
+			}
+		}
+
 		public ReadOnlyDictionary<string, List<ClassFile>> GetPackages ()
 		{
 			return new ReadOnlyDictionary<string, List<ClassFile>> (classFiles
@@ -357,7 +365,7 @@ public XElement ToXElement ()
 				FixUpParametersFromClasses ();
 
 			KotlinFixups.Fixup (classFiles);
-			FixupModuleVisibility ();
+			FixupModuleVisibility (removeModules: true);
 
 			var packagesDictionary = GetPackages ();
 			var api = new XElement ("api",
@@ -373,7 +381,7 @@ packagesDictionary [p].OrderBy (c => c.ThisClass.Name.Value, StringComparer.Ordi
 			return api;
 		}
 
-		void FixupModuleVisibility ()
+		public void FixupModuleVisibility (bool removeModules)
 		{
 			var publicPackages  = new HashSet<string> ();
 
@@ -383,7 +391,9 @@ void FixupModuleVisibility ()
 				return;
 			}
 			foreach (var moduleFile in moduleFiles) {
-				classFiles.Remove (moduleFile);
+				if (removeModules) {
+					classFiles.Remove (moduleFile);
+				}
 				foreach (var moduleAttr in moduleFile.Attributes.OfType<ModuleAttribute> ()) {
 					foreach (var export in moduleAttr.Exports) {
 						publicPackages.Add (export.Exports);
@@ -427,5 +437,7 @@ public void SaveXmlDescription (TextWriter textWriter)
 				contents.Save (writer);
 			textWriter.WriteLine ();
 		}
+
+		public IEnumerable<ClassFile> GetClassFiles () => classFiles;
 	}
 }

src/Xamarin.Android.Tools.Bytecode/XmlClassDeclarationBuilder.cs

@@ -40,7 +40,7 @@ public XElement ToXElement ()
 					GetSourceFile (),
 					new XAttribute ("static",                   classFile.IsStatic),
 					new XAttribute ("visibility",               GetVisibility (classFile.Visibility)),
-					GetAnnotatedVisibility (classFile.Attributes),
+					GetAnnotatedVisibility (classFile.Visibility, classFile.Attributes),
 					GetTypeParmeters (signature == null ? null : signature.TypeParameters),
 					GetImplementedInterfaces (),
 					GetConstructors (),
@@ -111,7 +111,7 @@ static string GetVisibility (ClassAccessFlags accessFlags)
 			if ((accessFlags & ClassAccessFlags.Private) != 0)
 				return "private";
 			if (accessFlags.HasFlag (ClassAccessFlags.Internal))
-				return "kotlin-internal";
+				return "public";    // TODO: `kotlin-internal` at some point?  See also GetAnnotatedVisibility()
 			return "";
 		}
 
@@ -431,6 +431,19 @@ IEnumerable<XElement> GetExceptions (MethodInfo method)
 			}
 		}
 
+		static XAttribute? GetAnnotatedVisibility (ClassAccessFlags flags, AttributeCollection attributes)
+		{
+			var attr = GetAnnotatedVisibility (attributes);
+			if (flags.HasFlag (ClassAccessFlags.Internal)) {
+				if (attr == null) {
+					attr = new XAttribute ("annotated-visibility", "module-info");
+				} else {
+					attr.Value += " module-info";
+				}
+			}
+			return attr;
+		}
+
 		static XAttribute? GetAnnotatedVisibility (AttributeCollection attributes)
 		{
 			var annotations = attributes?.OfType<RuntimeInvisibleAnnotationsAttribute> ().FirstOrDefault ()?.Annotations;

tools/class-parse/Program.cs

@@ -71,59 +71,63 @@ public static void Main (string[] args)
 			Log.OnLog = (t, v, m, a) => {
 				Console.Error.WriteLine(m, a);
 			};
-			var classPath = new ClassPath () {
-				ApiSource         = "class-parse",
-				AndroidFrameworkPlatform = platform,
-				DocumentationPaths  = docsPaths.Count == 0 ? null : docsPaths,
-				AutoRename = autorename
-			};
+			var globalClassPath = CreateClassPath (platform, docsPaths, autorename);
+			var classPaths      = new List<ClassPath> ();
 			foreach (var file in files) {
 				try {
-					if (dump) {
-						DumpClassFile (file, output);
+					if (ClassPath.IsJmodFile (file) || ClassPath.IsJarFile (file)) {
+						var cp = CreateClassPath (platform, docsPaths, autorename);
+						cp.Load (file);
+						classPaths.Add (cp);
 						continue;
 					}
-					DumpFileToXml (classPath, file);
+					using (var s = File.OpenRead (file)) {
+						if (!ClassFile.IsClassFile (s)) {
+							Console.Error.WriteLine ("class-parse: Unable to read file '{0}': Unknown file format.");
+							Environment.ExitCode    = 1;
+							continue;
+						}
+						globalClassPath.Add (new ClassFile (s));
+					}
 				} catch (Exception e) {
 					Console.Error.WriteLine ("class-parse: Unable to read file '{0}': {1}",
 							file, verbosity == 0 ? e.Message : e.ToString ());
 					Environment.ExitCode    = 1;
 				}
 			}
-			if (!dump)
-				classPath.SaveXmlDescription (output);
-			if (outputFile != null)
-				output.Close ();
-		}
-
-		static void DumpFileToXml (ClassPath jar, string file)
-		{
-			using (var s = File.OpenRead (file)) {
-				if (ClassFile.IsClassFile (s)) {
-					s.Position = 0;
-					var c = new ClassFile (s);
-					jar.Add (c);
-					return;
-				}
+			globalClassPath.FixupModuleVisibility (removeModules: !dump);
+			foreach (var cp in classPaths) {
+				globalClassPath.Add (cp, removeModules: !dump);
 			}
-			if (ClassPath.IsJmodFile (file) || ClassPath.IsJarFile (file)) {
-				jar.Load (file);
-				return;
+			if (!dump) {
+				globalClassPath.SaveXmlDescription (output);
+			} else {
+				bool first = true;
+				foreach (var c in globalClassPath.GetClassFiles ()) {
+					if (!first) {
+						output.WriteLine ();
+					}
+					first = false;
+					DumpClassFile (c, output);
+				}
 			}
-			Console.Error.WriteLine ("class-parse: Unable to read file '{0}': Unknown file format.");
-			Environment.ExitCode    = 1;
+			if (outputFile != null)
+				output.Close ();
 		}
 
-		static ClassFile LoadClassFile (string file)
+		static ClassPath CreateClassPath (string platform, List<string> docsPaths, bool autoRename)
 		{
-			using (var s = File.OpenRead (file)) {
-				return new ClassFile (s);
-			}
+			return new ClassPath () {
+				ApiSource                   = "class-parse",
+				AndroidFrameworkPlatform    = platform,
+				DocumentationPaths          = docsPaths.Count == 0 ? null : docsPaths,
+				AutoRename                  = autoRename
+			};
 		}
 
-		static void DumpClassFile (string file, TextWriter output)
+		static void DumpClassFile (ClassFile c, TextWriter output)
 		{
-			var c   = LoadClassFile (file);
+			output.WriteLine ($"-- Begin {c.FullJniName} --");
 			output.WriteLine (".class version: {0}.{1}", c.MajorVersion, c.MinorVersion);
 			output.WriteLine ("ConstantPool Count: {0}", c.ConstantPool.Count);
 			for (int i = 0; i < c.ConstantPool.Count; ++i) {