Add support for structured output

Summary: When running in automation, we'll want to report analysis results to our performance database. To find analysis results (such as number of live bytes on the heap), we don't want our automation drivers to have to parse the textual output of MemorySnapshotAnalyzer. Therefore, this diff adds a command line option that makes MemorySnapshotAnalyzer write out, before it exits, everything it did as a JSON file ("structured output"). This JSON file will be easier to interpret by out automation than textual (human-readable, aka "display string") output. Differential Revision: D54330870 fbshipit-source-id: 7b09c4a0b50181da141c608d7062f430ae41dc5d
facebookexperimental · Mar 1, 2024 · c5aaaf3 · c5aaaf3
1 parent f9eae5e
commit c5aaaf3
Show file tree

Hide file tree

Showing 53 changed files with 1,261 additions and 269 deletions.
diff --git a/AbstractMemorySnapshot/HeapSegment.cs b/AbstractMemorySnapshot/HeapSegment.cs
@@ -1,10 +1,12 @@
-/*
+/*
  * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
 
+using System.Text;
+
 namespace MemorySnapshotAnalyzer.AbstractMemorySnapshot
 {
     public sealed class HeapSegment
@@ -30,6 +32,17 @@ public HeapSegment(NativeWord startAddress, MemoryView memoryView, bool isRuntim
 
         public bool IsRuntimeTypeInformation => m_isRuntimeTypeInformation;
 
+        public void Describe(IStructuredOutput output, StringBuilder sb)
+        {
+            output.AddProperty("startAddress", m_startAddress.ToString());
+            output.AddProperty("isRtti", m_isRuntimeTypeInformation.ToString());
+            sb.AppendFormat("{0} segment at {1} (",
+                m_isRuntimeTypeInformation ? "runtime type information" : "managed heap",
+                m_startAddress);
+            m_memoryView.Describe(output, sb);
+            sb.Append(')');
+        }
+
         public override string ToString()
         {
             return string.Format("{0} segment at {1} ({2})",

diff --git a/AbstractMemorySnapshot/IStructuredOutput.cs b/AbstractMemorySnapshot/IStructuredOutput.cs
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+namespace MemorySnapshotAnalyzer.AbstractMemorySnapshot
+{
+    public interface IStructuredOutput
+    {
+        void AddProperty(string key, string value);
+
+        void AddProperty(string key, long value);
+
+        void AddDisplayString(string message);
+
+        void AddDisplayString(string format, params object[] args);
+
+        void AddDisplayStringLine(string message);
+
+        void AddDisplayStringLine(string format, params object[] args);
+
+        void AddDisplayStringLineIndented(int indent, string format, params object[] args);
+
+        void BeginArray(string key);
+
+        void BeginElement();
+
+        void EndElement();
+
+        void EndArray();
+
+        void BeginChild(string key);
+
+        void EndChild();
+    }
+}
diff --git a/AbstractMemorySnapshot/MemoryView.cs b/AbstractMemorySnapshot/MemoryView.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * This source code is licensed under the MIT license found in the
@@ -29,6 +29,18 @@ public override string ToString()
             return $"{m_memoryAccessor} offset {m_offset} size 0x{m_size:X} ({m_size})";
         }
 
+        public void Describe(IStructuredOutput output, StringBuilder sb)
+        {
+            output.AddProperty("memoryAccessor", m_memoryAccessor.ToString()!);
+            output.AddProperty("offset", m_offset);
+            output.AddProperty("size", m_size);
+            sb.AppendFormat("{0} offset {1} size 0x{2:X} ({3})",
+                m_memoryAccessor,
+                m_offset,
+                m_size,
+                m_size);
+        }
+
         public long Size => m_size;
 
         public bool IsValid => m_size > 0;

diff --git a/AbstractMemorySnapshot/TraceableHeap.cs b/AbstractMemorySnapshot/TraceableHeap.cs
@@ -49,7 +49,7 @@ public TraceableHeap(TypeSystem typeSystem)
 
         public abstract bool ContainsAddress(NativeWord address);
 
-        public abstract string? DescribeAddress(NativeWord address);
+        public abstract string? DescribeAddress(NativeWord address, IStructuredOutput output);
 
         public abstract SegmentedHeap? SegmentedHeapOpt { get; }
     }

diff --git a/AbstractMemorySnapshot/TypeSystem.cs b/AbstractMemorySnapshot/TypeSystem.cs
@@ -97,6 +97,11 @@ static int ComputeArity(ReadOnlySpan<char> genericArguments)
 
         public abstract bool IsArray(int typeIndex);
 
+        public string Kind(int typeIndex)
+        {
+            return IsValueType(typeIndex) ? "value" : IsArray(typeIndex) ? "array" : "object";
+        }
+
         public abstract int Rank(int typeIndex);
 
         public abstract int NumberOfFields(int typeIndex);
@@ -123,7 +128,18 @@ static int ComputeArity(ReadOnlySpan<char> genericArguments)
 
         public abstract int SystemVoidStarTypeIndex { get; }
 
-        public abstract IEnumerable<string> DumpStats();
+        public abstract IEnumerable<string> DumpStats(IStructuredOutput output);
+
+        public void OutputType(IStructuredOutput output, string key, int typeIndex)
+        {
+            output.BeginChild(key);
+
+            output.AddProperty("assembly", Assembly(typeIndex));
+            output.AddProperty("qualifiedName", QualifiedName(typeIndex));
+            output.AddProperty("typeIndex", typeIndex);
+
+            output.EndChild();
+        }
 
         ValueTuple<int, int> EnsurePointerOffsets(int typeIndex)
         {

diff --git a/AbstractMemorySnapshotTests/TestSegmentedTraceableHeap.cs b/AbstractMemorySnapshotTests/TestSegmentedTraceableHeap.cs
@@ -66,11 +66,13 @@ public override int GetObjectSize(NativeWord objectAddress, int typeIndex, bool
             throw new System.NotImplementedException();
         }
 
-        public override string? DescribeAddress(NativeWord address)
+        public override string? DescribeAddress(NativeWord address, IStructuredOutput output)
         {
             int typeInfoIndex = TypeInfoAddressToIndex(address);
             if (typeInfoIndex != -1)
             {
+                output.AddProperty("addressTargetKind", "vtable");
+                TypeSystem.OutputType(output, "vtableType", typeInfoIndex);
                 return string.Format("VTable[{0}, type index {1}]",
                     TypeSystem.QualifiedName(typeInfoIndex),
                     typeInfoIndex);

diff --git a/AbstractMemorySnapshotTests/TestTypeSystem.cs b/AbstractMemorySnapshotTests/TestTypeSystem.cs
@@ -417,7 +417,7 @@ public override int GetArrayElementSize(int elementTypeIndex)
 
         public override int SystemVoidStarTypeIndex => throw new NotImplementedException();
 
-        public override IEnumerable<string> DumpStats()
+        public override IEnumerable<string> DumpStats(IStructuredOutput output)
         {
             throw new NotImplementedException();
         }

diff --git a/AbstractMemorySnapshotTests/TypeSystemTest.cs b/AbstractMemorySnapshotTests/TypeSystemTest.cs
@@ -43,7 +43,7 @@ public void TestQualifiedGenericNameWithArity()
                 Is.EqualTo("GenericTypeWithNesting`2"));
 
             Assert.That(m_typeSystem!.QualifiedGenericNameWithArity((int)TestTypeIndex.EmptyTypeNameCornerCase),
-                Is.EqualTo(""));
+                Is.EqualTo(string.Empty));
         }
 
         [Test]
@@ -317,7 +317,7 @@ public void TestBindSelector()
                 Assert.That(selector.StaticPrefix, Has.Exactly(0).Items);
                 Assert.That(selector.DynamicTail, Is.Null);
 
-                Assert.That(selector.Stringify(m_typeSystem!, pathIndex: 0, inStaticPrefix: true), Is.EqualTo(""));
+                Assert.That(selector.Stringify(m_typeSystem!, pathIndex: 0, inStaticPrefix: true), Is.EqualTo(string.Empty));
             });
 
             // Base case: single field

diff --git a/Analysis/Backtracer.cs b/Analysis/Backtracer.cs
@@ -97,41 +97,50 @@ public int PostorderIndexToNodeIndex(int postorderIndex)
             return postorderIndex;
         }
 
-        public string DescribeNodeIndex(int nodeIndex, bool fullyQualified)
+        public string DescribeNodeIndex(int nodeIndex, IStructuredOutput output, bool fullyQualified)
         {
+            output.AddProperty("nodeIndex", nodeIndex);
+
             if (nodeIndex == m_rootNodeIndex)
             {
+                output.AddProperty("nodeKind", "process");
                 return "Process";
             }
             else if (nodeIndex == m_unreachableNodeIndex)
             {
+                output.AddProperty("nodeKind", "unreachable");
                 return "Unreachable";
             }
 
             int postorderIndex = NodeIndexToPostorderIndex(nodeIndex);
             int typeIndex = m_tracedHeap.PostorderTypeIndexOrSentinel(postorderIndex);
             if (typeIndex == -1)
             {
+                output.AddProperty("nodeKind", "root");
                 List<(int rootIndex, PointerInfo<NativeWord> pointerFlags)> rootInfos = m_tracedHeap.PostorderRootIndices(nodeIndex);
                 if (rootInfos.Count == 1)
                 {
-                    return m_rootSet.DescribeRoot(rootInfos[0].rootIndex, fullyQualified);
+                    return m_rootSet.DescribeRoot(rootInfos[0].rootIndex, output, fullyQualified);
                 }
                 else
                 {
                     var sb = new StringBuilder();
-                    m_tracedHeap.DescribeRootIndices(nodeIndex, sb);
+                    m_tracedHeap.DescribeRootIndices(nodeIndex, sb, output);
                     return sb.ToString();
                 }
             }
 
+            output.AddProperty("nodeKind", "object");
+            output.AddProperty("objectIndex", postorderIndex);
+            m_traceableHeap.TypeSystem.OutputType(output, "objectType", typeIndex);
             string typeName = fullyQualified ?
                 $"{m_traceableHeap.TypeSystem.Assembly(typeIndex)}:{m_traceableHeap.TypeSystem.QualifiedName(typeIndex)}" :
                 m_traceableHeap.TypeSystem.UnqualifiedName(typeIndex);
 
             string? objectName = m_traceableHeap.GetObjectName(m_tracedHeap.PostorderAddress(postorderIndex));
             if (objectName != null)
             {
+                output.AddProperty("objectName", objectName);
                 return string.Format("{0}('{1}')#{2}",
                     typeName,
                     objectName,

diff --git a/Analysis/GroupingBacktracer.cs b/Analysis/GroupingBacktracer.cs
@@ -8,6 +8,7 @@
 using MemorySnapshotAnalyzer.AbstractMemorySnapshot;
 using System;
 using System.Collections.Generic;
+using System.Reflection;
 
 namespace MemorySnapshotAnalyzer.Analysis
 {
@@ -148,33 +149,45 @@ int IBacktracer.PostorderIndexToNodeIndex(int postorderIndex)
             return m_parentBacktracer.PostorderIndexToNodeIndex(postorderIndex);
         }
 
-        string IBacktracer.DescribeNodeIndex(int nodeIndex, bool fullyQualified)
+        string IBacktracer.DescribeNodeIndex(int nodeIndex, IStructuredOutput output, bool fullyQualified)
         {
             if (nodeIndex == m_rootNodeIndex)
             {
-                return m_parentBacktracer.DescribeNodeIndex(m_parentBacktracer.RootNodeIndex, fullyQualified);
+                return m_parentBacktracer.DescribeNodeIndex(m_parentBacktracer.RootNodeIndex, output, fullyQualified);
             }
             else if (nodeIndex == m_unreachableNodeIndex)
             {
-                return m_parentBacktracer.DescribeNodeIndex(m_parentBacktracer.UnreachableNodeIndex, fullyQualified);
+                return m_parentBacktracer.DescribeNodeIndex(m_parentBacktracer.UnreachableNodeIndex, output, fullyQualified);
             }
             else if (nodeIndex >= m_firstAssemblyIndex)
             {
-                return $"{m_assemblyNames[nodeIndex - m_firstAssemblyIndex]}#{nodeIndex}";
+                string assemblyName = m_assemblyNames[nodeIndex - m_firstAssemblyIndex];
+                output.AddProperty("nodeKind", "assembly");
+                output.AddProperty("nodeIndex", nodeIndex);
+                output.AddProperty("assemblyName", assemblyName);
+                return $"{assemblyName}#{nodeIndex}";
             }
             else if (nodeIndex >= m_firstNamespaceIndex)
             {
                 int namespaceIndex = nodeIndex - m_firstNamespaceIndex;
-                return $"{m_namespaceNames[namespaceIndex]}#{nodeIndex}";
+                string namespaceName = m_namespaceNames[namespaceIndex];
+                output.AddProperty("nodeKind", "namespace");
+                output.AddProperty("nodeIndex", nodeIndex);
+                output.AddProperty("namespaceName", namespaceName);
+                return $"{namespaceName}#{nodeIndex}";
             }
             else if (nodeIndex >= m_firstClassIndex)
             {
                 int classIndex = nodeIndex - m_firstClassIndex;
-                return $"{m_classNames[classIndex]}#{nodeIndex}";
+                string className = m_classNames[classIndex];
+                output.AddProperty("nodeKind", "class");
+                output.AddProperty("nodeIndex", nodeIndex);
+                output.AddProperty("className", className);
+                return $"{className}#{nodeIndex}";
             }
             else
             {
-                return m_parentBacktracer.DescribeNodeIndex(nodeIndex, fullyQualified);
+                return m_parentBacktracer.DescribeNodeIndex(nodeIndex, output, fullyQualified);
             }
         }
 

diff --git a/Analysis/IBacktracer.cs b/Analysis/IBacktracer.cs
@@ -6,6 +6,7 @@
  */
 
 using System.Collections.Generic;
+using MemorySnapshotAnalyzer.AbstractMemorySnapshot;
 
 namespace MemorySnapshotAnalyzer.Analysis
 {
@@ -27,7 +28,7 @@ public interface IBacktracer
 
         int PostorderIndexToNodeIndex(int postorderIndex);
 
-        string DescribeNodeIndex(int nodeIndex, bool fullyQualified);
+        string DescribeNodeIndex(int nodeIndex, IStructuredOutput output, bool fullyQualified);
 
         string NodeType(int nodeIndex);
 

diff --git a/Analysis/IRootSet.cs b/Analysis/IRootSet.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * This source code is licensed under the MIT license found in the
@@ -30,7 +30,7 @@ public struct StaticRootInfo
 
         bool IsGCHandle(int rootIndex);
 
-        string DescribeRoot(int rootIndex, bool fullyQualified);
+        string DescribeRoot(int rootIndex, IStructuredOutput output, bool fullyQualified);
 
         StaticRootInfo GetStaticRootInfo(int rootIndex);
     }

diff --git a/Analysis/RootSet.cs b/Analysis/RootSet.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * This source code is licensed under the MIT license found in the
@@ -109,22 +109,29 @@ bool IRootSet.IsGCHandle(int rootIndex)
             return m_roots[rootIndex].PointerInfo.TypeIndex == -1;
         }
 
-        string IRootSet.DescribeRoot(int rootIndex, bool fullyQualified)
+        string IRootSet.DescribeRoot(int rootIndex, IStructuredOutput output, bool fullyQualified)
         {
             RootEntry entry = m_roots[rootIndex];
             int typeIndex = entry.PointerInfo.TypeIndex;
             if (typeIndex == -1)
             {
+                output.AddProperty("rootKind", "gcHandle");
+                output.AddProperty("rootNumber", entry.PointerInfo.FieldNumber);
                 return $"GCHandle#{entry.PointerInfo.FieldNumber}";
             }
             else
             {
+                output.AddProperty("rootKind", "staticVariable");
+                m_traceableHeap.TypeSystem.OutputType(output, "containingType", typeIndex);
                 string typeName = fullyQualified ?
                     $"{m_traceableHeap.TypeSystem.Assembly(typeIndex)}:{m_traceableHeap.TypeSystem.QualifiedName(typeIndex)}" :
                     m_traceableHeap.TypeSystem.UnqualifiedName(typeIndex);
+                string fieldName = m_traceableHeap.TypeSystem.FieldName(typeIndex, entry.PointerInfo.FieldNumber);
+                output.AddProperty("fieldName", fieldName);
+                output.AddProperty("offset", entry.Offset);
                 return string.Format("{0}.{1}+0x{2:X}",
                     typeName,
-                    m_traceableHeap.TypeSystem.FieldName(typeIndex, entry.PointerInfo.FieldNumber),
+                    fieldName,
                     entry.Offset);
             }
         }

diff --git a/Analysis/SingletonRootSet.cs b/Analysis/SingletonRootSet.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * Copyright (c) Meta Platforms, Inc. and affiliates.
  *
  * This source code is licensed under the MIT license found in the
@@ -44,8 +44,10 @@ bool IRootSet.IsGCHandle(int rootIndex)
             return false;
         }
 
-        string IRootSet.DescribeRoot(int rootIndex, bool fullyQualified)
+        string IRootSet.DescribeRoot(int rootIndex, IStructuredOutput output, bool fullyQualified)
         {
+            output.AddProperty("rootTargetKind", "object");
+            output.AddProperty("rootTargetAddress", m_address.ToString());
             return $"Object@{m_address}";
         }
 

diff --git a/Analysis/StitchedTraceableHeap.cs b/Analysis/StitchedTraceableHeap.cs
@@ -181,10 +181,10 @@ public override bool ContainsAddress(NativeWord address)
             return m_secondary.ContainsAddress(address) || base.ContainsAddress(address);
         }
 
-        public override string? DescribeAddress(NativeWord address)
+        public override string? DescribeAddress(NativeWord address, IStructuredOutput output)
         {
-            string? secondaryDescription = m_secondary.DescribeAddress(address);
-            string? primaryDescription = m_primary.DescribeAddress(address);
+            string? secondaryDescription = m_secondary.DescribeAddress(address, output);
+            string? primaryDescription = m_primary.DescribeAddress(address, output);
             if (secondaryDescription != null && primaryDescription != null)
             {
                 return $"{primaryDescription}/{secondaryDescription}";