[WIP] Adding OOM Diagnostics to GCStats and the Events View

src/PerfView/GcStats.cs

@@ -2,6 +2,7 @@
 using Microsoft.Diagnostics.Tracing.Analysis;
 using Microsoft.Diagnostics.Tracing.Analysis.GC;
 using Microsoft.Diagnostics.Tracing.Parsers.Clr;
+using Microsoft.Diagnostics.Tracing.Parsers.GCDynamic;
 using Microsoft.Diagnostics.Utilities;
 using System;
 using System.Collections.Generic;
@@ -221,6 +222,85 @@ public static void ToHtml(TextWriter writer, TraceProcess stats, TraceLoadedDotN
                 writer.WriteLine("<P><A HREF=\"command:excelFinalization/{0}\">View the full list</A> in Excel.<P>", stats.ProcessID);
             }
 
+
+            if (runtime.GC.Stats().OOMDetails.Count > 0)
+            {
+                string InterpretOOMReason(OOMReason reason)
+                {
+                    switch (reason)
+                    {
+                        case OOMReason.Budget:
+                        case OOMReason.CantReserve: 
+                            return "OOM was due to an internal .Net error, likely a bug in the GC";
+                        case OOMReason.CantCommit:
+                            return "Didn't have enough memory to commit";
+                        case OOMReason.LOH:
+                            return "Didn't have enough memory to allocate an LOH segment";
+                        case OOMReason.LowMemory:
+                            return "Low on memory during GC";
+                        case OOMReason.UnproductiveFullGC:
+                            return "Could not do a full GC";
+                        default:
+                            return reason.ToString(); // shouldn't happen, we handle all cases above
+                    }
+                }
+
+                string InterpretFailureGetMemory(FailureGetMemory fgm)
+                {
+                    switch (fgm)
+                    {
+                        case FailureGetMemory.ReserveSegment:
+                            return "Failed to reserve memory";
+                        case FailureGetMemory.CommitSegmentBeg:
+                            return "Didn't have enough memory to commit beginning of the segment";
+                        case FailureGetMemory.CommitEphSegment:
+                            return "Didn't have enough memory to commit the new ephemeral segment";
+                        case FailureGetMemory.GrowTable:
+                            return "Didn't have enough memory to grow the internal GC data structures";
+                        case FailureGetMemory.CommitTable:
+                            return "Didn't have enough memory to commit the internal GC data structures";
+                        case FailureGetMemory.CommitHeap:
+                            return "Didn't have enough memory to commit the heap.";
+                        default:
+                            return fgm.ToString();
+                    }
+                }
+
+                void WriteOOMHeader()
+                {
+                    writer.WriteLine(@"<TR>
+                                        <TH>GC Index</TH>
+                                        <TH>Reason</TH>
+                                        <TH>Failure Get Memory Reason</TH>
+                                        <TH>Is LOH</TH>
+                                        <TH>Available Page Memory (MB)</TH>
+                                        <TH>Memory Load (%)</TH>
+                                       </TR>");
+                }
+
+                void WriteOOMRow(OOMDetails oomDetails)
+                {
+                    writer.WriteLine("<TR>" +
+                        $"<TD Align=\"Center\">{oomDetails.GCIndex}</TD>" +
+                        $"<TD Align=\"Center\">{InterpretOOMReason(oomDetails.Reason)}</TD>" +
+                        $"<TD Align=\"Center\">{InterpretFailureGetMemory(oomDetails.FailureToGetMemory)}</TD>" +
+                        $"<TD Align=\"Center\">{oomDetails.IsLOH}</TD>" +
+                        $"<TD Align=\"Center\">{oomDetails.AvailablePageFileMB.ToString("N0")}</TD>" +
+                        $"<TD Align=\"Center\">{oomDetails.MemoryLoad}</TD>" +
+                        "</TR>");
+                }
+
+                writer.WriteLine("<HR/>");
+                writer.WriteLine($"<H4><A Name=\"OOM_{stats.ProcessID}\">OOM Details for {stats.ProcessID,5}: {stats.Name}<A></H4>");
+                writer.WriteLine("<Center><Table Border=\"1\">");
+                WriteOOMHeader();
+                foreach (var oomDetails in runtime.GC.Stats().OOMDetails)
+                {
+                    WriteOOMRow(oomDetails);
+                }
+                writer.WriteLine("</Table></Center>");
+            }
+
             writer.WriteLine("<HR/><HR/><BR/><BR/>");
         }
 

src/TraceEvent/Computers/TraceManagedProcess.cs

@@ -879,6 +879,12 @@ internal static void SetupCallbacks(TraceEventDispatcher source)
                     GCStats.ProcessCommittedUsage(stats, committedUsage);
                 };
 
+                source.Clr.GCDynamicEvent.GCOOMDetails += delegate (OOMDetailsTraceEvent oomDetails)
+                {
+                    var stats = currentManagedProcess(oomDetails.UnderlyingEvent);
+                    GCStats.ProcessOOMEvent(stats, oomDetails);
+                };
+
                 source.Clr.GCDynamicEvent.GCDynamicTraceEvent += delegate (GCDynamicTraceEvent gcDynamic)
                 {
                     var stats = currentManagedProcess(gcDynamic.UnderlyingEvent);
@@ -1738,6 +1744,11 @@ private void Calculate()
                     m_stats.MaxSizePeakMB = Math.Max(m_stats.MaxSizePeakMB, _gc.HeapSizePeakMB);
                     m_stats.MaxAllocRateMBSec = Math.Max(m_stats.MaxAllocRateMBSec, _gc.AllocRateMBSec);
                     m_stats.MaxSuspendDurationMSec = Math.Max(m_stats.MaxSuspendDurationMSec, _gc.SuspendDurationMSec);
+
+                    foreach (var oom in _gc.OOMDetails)
+                    {
+                        m_stats.OOMDetails.Add(oom);
+                    }
                 }
 
                 m_prvcount = m_gcs.Count;
@@ -2146,6 +2157,7 @@ public double PromotedMB
 
         public CommittedUsage CommittedUsageBefore { get; internal set; }
         public CommittedUsage CommittedUsageAfter { get; internal set; }
+        public List<OOMDetails> OOMDetails { get; internal set; } = new List<OOMDetails>();
 
         /// <summary>
         /// Memory survival percentage by generation
@@ -4683,6 +4695,10 @@ public double GetGCPauseTimePercentage()
         /// Indicator if PerHeapHistories is present
         /// </summary>
         public bool HasDetailedGCInfo;
+        /// <summary>
+        /// OOMs detected in the trace.
+        /// </summary>
+        public List<OOMDetails> OOMDetails = new List<OOMDetails>();
 
         #region private
 
@@ -4994,6 +5010,25 @@ internal static void ProcessCommittedUsage(TraceLoadedDotNetRuntime proc, Commit
             }
         }
 
+        internal static void ProcessOOMEvent(TraceLoadedDotNetRuntime proc, OOMDetailsTraceEvent oomDetailsTrace)
+        {
+            TraceGC _event = GetLastGC(proc);
+            if (_event != null)
+            {
+                _event.OOMDetails.Add(new OOMDetails
+                {
+                    GCIndex = oomDetailsTrace.GCIndex,
+                    Reason = (OOMReason)oomDetailsTrace.Reason,
+                    AllocSize = oomDetailsTrace.AllocSize,
+                    FailureToGetMemory = (FailureGetMemory)oomDetailsTrace.FailureGetMemory,
+                    Size = oomDetailsTrace.Size,
+                    IsLOH = oomDetailsTrace.IsLOH,
+                    MemoryLoad = oomDetailsTrace.MemoryLoad,
+                    AvailablePageFileMB = oomDetailsTrace.AvailablePageFileMB,
+                });
+            }
+        }
+
         internal static void ProcessGCDynamicEvent(TraceLoadedDotNetRuntime proc, GCDynamicTraceEvent gcDynamic)
         {
             TraceGC _event = GetLastGC(proc);

src/TraceEvent/Parsers/GCDynamicTraceEventParser.cs

@@ -32,6 +32,7 @@ public GCDynamicTraceEventParser(TraceEventSource source) : base(source)
             // They ensure that Dispatch is called so that the specific event handlers are called for each event.
             ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.GCDynamicTemplate));
             ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.CommittedUsageTemplate));
+            ((ITraceParserServices)source).RegisterEventTemplate(GCDynamicTemplate(Dispatch, GCDynamicEventBase.OOMDetailsTemplate));
         }
 
         protected override string GetProviderName()
@@ -43,14 +44,15 @@ protected internal override void EnumerateTemplates(Func<string, string, EventFi
         {
             if (s_templates == null)
             {
-                var templates = new TraceEvent[2];
+                var templates = new TraceEvent[3];
 
                 // This template ensures that all GC dynamic events are parsed properly.
                 templates[0] = GCDynamicTemplate(null, GCDynamicEventBase.GCDynamicTemplate);
 
                 // A template must be registered for each dynamic event type.  This ensures that after the event is converted
                 // to its final form and saved in a TraceLog, that it can still be properly parsed and dispatched.
                 templates[1] = GCDynamicTemplate(null, GCDynamicEventBase.CommittedUsageTemplate);
+                templates[2] = GCDynamicTemplate(null, GCDynamicEventBase.OOMDetailsTemplate);
 
                 s_templates = templates;
             }
@@ -86,6 +88,19 @@ public event Action<CommittedUsageTraceEvent> GCCommittedUsage
             }
         }
 
+        private event Action<OOMDetailsTraceEvent> _OOMDetails;
+        public event Action<OOMDetailsTraceEvent> GCOOMDetails
+        {
+            add
+            {
+                _OOMDetails += value;
+            }
+            remove
+            {
+                _OOMDetails -= value;
+            }
+        }
+
         /// <summary>
         /// Responsible for dispatching the event after we determine its type
         /// and parse it.
@@ -98,6 +113,12 @@ private void Dispatch(GCDynamicTraceEventImpl data)
                 _gcCommittedUsage(data.EventPayload as CommittedUsageTraceEvent);
             }
 
+            else if (_OOMDetails != null &&
+                data.eventID == GCDynamicEventBase.OOMDetailsTemplate.ID)
+            {
+                _OOMDetails(data.EventPayload as OOMDetailsTraceEvent);
+            }
+
             else if (_gcDynamicTraceEvent != null &&
                 data.EventPayload is GCDynamicTraceEvent)
             {
@@ -160,6 +181,7 @@ protected internal override Delegate Target
 
         private readonly CommittedUsageTraceEvent _committedUsageTemplate = new CommittedUsageTraceEvent();
         private readonly GCDynamicTraceEvent _gcDynamicTemplate = new GCDynamicTraceEvent();
+        private readonly OOMDetailsTraceEvent _oomDetailsTemplate = new OOMDetailsTraceEvent();
 
         /// <summary>
         /// Contains the fully parsed payload of the dynamic event.
@@ -173,6 +195,11 @@ public GCDynamicEventBase EventPayload
                     return _committedUsageTemplate.Bind(this);
                 }
 
+                else if (eventID == GCDynamicEventBase.OOMDetailsTemplate.ID)
+                {
+                    return _oomDetailsTemplate.Bind(this);
+                }
+
                 return _gcDynamicTemplate.Bind(this);
             }
         }
@@ -214,6 +241,11 @@ private void SelectEventMetadata()
                 eventTemplate = GCDynamicEventBase.CommittedUsageTemplate;
             }
 
+            else if (Name.Equals(GCDynamicEventBase.OOMDetailsTemplate.OpcodeName, StringComparison.InvariantCultureIgnoreCase))
+            {
+                eventTemplate = GCDynamicEventBase.OOMDetailsTemplate;
+            }
+
             SetMetadataFromTemplate(eventTemplate);
         }
 
@@ -237,6 +269,7 @@ public abstract class GCDynamicEventBase
         /// </summary>
         internal static readonly GCDynamicTraceEvent GCDynamicTemplate = new GCDynamicTraceEvent();
         internal static readonly CommittedUsageTraceEvent CommittedUsageTemplate = new CommittedUsageTraceEvent();
+        internal static readonly OOMDetailsTraceEvent OOMDetailsTemplate = new OOMDetailsTraceEvent();
 
         /// <summary>
         /// Metadata that must be specified for each specific type of dynamic event.
@@ -329,6 +362,119 @@ internal override IEnumerable<KeyValuePair<string, object>> PayloadValues
         }
     }
 
+    public sealed class OOMDetailsTraceEvent : GCDynamicEventBase
+    {
+        public short Version { get { return BitConverter.ToInt16(DataField, 0); } }
+        public long GCIndex { get { return BitConverter.ToInt64(DataField, 2); } }
+        public long AllocSize { get { return BitConverter.ToInt64(DataField, 10); } }
+        public short Reason { get { return BitConverter.ToInt16(DataField, 18); } }
+        public short FailureGetMemory { get { return BitConverter.ToInt16(DataField, 20); } }
+        public long Size { get { return BitConverter.ToInt64(DataField, 22); } }
+        public bool IsLOH { get { return BitConverter.ToBoolean(DataField, 30); } }
+        public int MemoryLoad { get { return BitConverter.ToInt32(DataField, 31); } }
+        public long AvailablePageFileMB { get { return BitConverter.ToInt64(DataField, 35); } }
+
+        internal override TraceEventID ID => TraceEventID.Illegal - 12;
+
+        internal override string TaskName => "GC";
+
+        internal override string OpcodeName => "OOMDetails";
+
+        internal override string EventName => "GC/OOMDetails";
+
+        private string[] _payloadNames;
+        internal override string[] PayloadNames
+        {
+            get
+            {
+                if (_payloadNames == null)
+                {
+                    _payloadNames = new string[] { "Version", "GCIndex", "AllocSize", "Reason", "FailureGetMemory", "Size", "IsLOH", "MemoryLoad", "AvailablePageFileMB" };
+                }
+
+                return _payloadNames;
+            }
+        }
+
+        internal override IEnumerable<KeyValuePair<string, object>> PayloadValues
+        {
+            get
+            {
+                yield return new KeyValuePair<string, object>("Version", Version);
+                yield return new KeyValuePair<string, object>("GCIndex", GCIndex);
+                yield return new KeyValuePair<string, object>("AllocSize", AllocSize);
+                yield return new KeyValuePair<string, object>("Reason", Reason);
+                yield return new KeyValuePair<string, object>("FailureGetMemory", FailureGetMemory);
+                yield return new KeyValuePair<string, object>("Size", Size);
+                yield return new KeyValuePair<string, object>("IsLOH", IsLOH);
+                yield return new KeyValuePair<string, object>("MemoryLoad", MemoryLoad);
+                yield return new KeyValuePair<string, object>("AvailablePageFileMB", AvailablePageFileMB);
+            }
+        }
+
+        internal override object PayloadValue(int index)
+        {
+            switch (index)
+            {
+                case 0:
+                    return Version;
+                case 1:
+                    return GCIndex;
+                case 2:
+                    return AllocSize;
+                case 3:
+                    return Reason;
+                case 4:
+                    return FailureGetMemory;
+                case 5:
+                    return Size;
+                case 6:
+                    return IsLOH;
+                case 7:
+                    return MemoryLoad;
+                case 8:
+                    return AvailablePageFileMB;
+                default:
+                    Debug.Assert(false, "Bad field index");
+                    return null;
+            }
+        }
+    }
+
+    public enum FailureGetMemory
+    {
+        NoFailure = 0,
+        ReserveSegment = 1,
+        CommitSegmentBeg = 2,
+        CommitEphSegment = 3,
+        GrowTable = 4,
+        CommitTable = 5,
+        CommitHeap = 6
+    };
+
+    public enum OOMReason
+    {
+        NoFailure = 0,
+        Budget = 1,
+        CantCommit = 2,
+        CantReserve = 3,
+        LOH = 4,
+        LowMemory = 5,
+        UnproductiveFullGC = 6
+    }
+
+    public sealed class OOMDetails
+    {
+        public long GCIndex { get; internal set; }
+        public long AllocSize { get; internal set; }
+        public OOMReason Reason { get; internal set; }
+        public FailureGetMemory FailureToGetMemory { get; internal set; }
+        public long Size { get; internal set; }
+        public bool IsLOH { get; internal set; }
+        public int MemoryLoad { get; internal set; }
+        public long AvailablePageFileMB { get; internal set; }
+    }
+
     public sealed class CommittedUsageTraceEvent : GCDynamicEventBase
     {
         public short Version { get { return BitConverter.ToInt16(DataField, 0); } }

src/TraceEvent/TraceEvent.cs

@@ -2972,6 +2972,7 @@ private void ConfirmAllEventsAreInEnumeration()
             if (string.Equals(GetType().Name, nameof(GCDynamicTraceEventParser), StringComparison.OrdinalIgnoreCase))
             {
                 declaredSet.Remove("CommittedUsage");
+                declaredSet.Remove("OOMDetails");
             }
 
             var enumSet = new SortedDictionary<string, string>();
@@ -3682,7 +3683,8 @@ internal TraceEvent Lookup(TraceEventNativeMethods.EVENT_RECORD* eventRecord)
                             // Make sure that the assert below doesn't fail by checking if _any_ of the event header ids match.
                             bool gcDynamicTemplateEventHeaderMatch =
                                  eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.GCDynamicTemplate.ID ||
-                                 eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.CommittedUsageTemplate.ID;
+                                 eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.CommittedUsageTemplate.ID ||
+                                 eventRecord->EventHeader.Id == (ushort)GCDynamicEventBase.OOMDetailsTemplate.ID;
 
                             // Ignore the failure for GC dynamic events because they are all
                             // dispatched through the same template and we vary the event ID.