diff --git a/src/Directory.Build.props b/src/Directory.Build.props
index 263b235fa..762a2e18c 100644
--- a/src/Directory.Build.props
+++ b/src/Directory.Build.props
@@ -6,7 +6,7 @@
- 6
+ 7
strict
diff --git a/src/FastSerialization/FastSerialization.cs b/src/FastSerialization/FastSerialization.cs
index 15bc9868e..5d150e4a0 100644
--- a/src/FastSerialization/FastSerialization.cs
+++ b/src/FastSerialization/FastSerialization.cs
@@ -2359,7 +2359,7 @@ internal enum Tags : byte
Int64,
SkipRegion,
String, // Size of string (in bytes) followed by UTF8 bytes.
- Blob, // Size of bytes followed by bytes.
+ Blob,
Limit, // Just past the last valid tag, used for asserts.
}
#endregion
diff --git a/src/TraceEvent/EventPipe/EventPipeEventSource.cs b/src/TraceEvent/EventPipe/EventPipeEventSource.cs
index 9afb007c5..cee2252a4 100644
--- a/src/TraceEvent/EventPipe/EventPipeEventSource.cs
+++ b/src/TraceEvent/EventPipe/EventPipeEventSource.cs
@@ -1,4 +1,6 @@
-using FastSerialization;
+#define SUPPORT_V1_V2
+
+using FastSerialization;
using Microsoft.Diagnostics.Tracing.EventPipe;
using Microsoft.Diagnostics.Tracing.Parsers;
using Microsoft.Diagnostics.Tracing.Parsers.Clr;
@@ -11,8 +13,8 @@
namespace Microsoft.Diagnostics.Tracing
{
///
- /// EventPipeEventSource knows how to decode EventPipe (generated by the .NET
- /// core runtime).
+ /// EventPipeEventSource knows how to decode EventPipe (generated by the .NET core runtime).
+ /// Please see for details on the file format.
///
/// By conventions files of such a format are given the .netperf suffix and are logically
/// very much like a ETL file in that they have a header that indicete things about
@@ -22,10 +24,8 @@ namespace Microsoft.Diagnostics.Tracing
/// Ordinary events then point at these meta-data event so that logically all
/// events have a name some basic information (process, thread, timestamp, activity
/// ID) and user defined field names and values of various types.
- ///
- /// See the E
///
- unsafe public class EventPipeEventSource : TraceEventDispatcher, IFastSerializable
+ unsafe public class EventPipeEventSource : TraceEventDispatcher, IFastSerializable, IFastSerializableVersion
{
public EventPipeEventSource(string fileName)
{
@@ -34,8 +34,14 @@ public EventPipeEventSource(string fileName)
cpuSpeedMHz = 10;
_deserializer = new Deserializer(new PinnedStreamReader(fileName, 0x20000), fileName);
+
+#if SUPPORT_V1_V2
+ // This is only here for V2 and V1. V3+ should use the name EventTrace, it can be removed when we drop support.
_deserializer.RegisterFactory("Microsoft.DotNet.Runtime.EventPipeFile", delegate { return this; });
-
+#endif
+ _deserializer.RegisterFactory("Trace", delegate { return this; });
+ _deserializer.RegisterFactory("EventBlock", delegate { return new EventPipeEventBlock(this); });
+
var entryObj = _deserializer.GetEntryObject(); // this call invokes FromStream and reads header data
// Because we told the deserialize to use 'this' when creating a EventPipeFile, we
@@ -45,10 +51,34 @@ public EventPipeEventSource(string fileName)
_eventParser = new EventPipeTraceEventParser(this);
}
- #region private
+#region private
// I put these in the private section because they are overrides, and thus don't ADD to the API.
public override int EventsLost => 0;
+ ///
+ /// This is the version number reader and writer (although we don't don't have a writer at the moment)
+ /// It MUST be updated (as well as MinimumReaderVersion), if breaking changes have been made.
+ /// If your changes are forward compatible (old readers can still read the new format) you
+ /// don't have to update the version number but it is useful to do so (while keeping MinimumReaderVersion unchanged)
+ /// so that readers can quickly determine what new content is available.
+ ///
+ public int Version => 3;
+
+ ///
+ /// This field is only used for writers, and this code does not have writers so it is not used.
+ /// It should be set to Version unless changes since the last version are forward compatible
+ /// (old readers can still read this format), in which case this shoudl be unchanged.
+ ///
+ public int MinimumReaderVersion => Version;
+
+ ///
+ /// This is the smallest version that the deserializer here can read. Currently
+ /// we are careful about backward compat so our deserializer can read anything that
+ /// has ever been produced. We may change this when we believe old writers basically
+ /// no longer exist (and we can remove that support code).
+ ///
+ public int MinimumVersionCanRead => 0;
+
protected override void Dispose(bool disposing)
{
_deserializer.Dispose();
@@ -58,80 +88,84 @@ protected override void Dispose(bool disposing)
public override bool Process()
{
- PinnedStreamReader deserializerReader = (PinnedStreamReader)_deserializer.Reader;
-
- deserializerReader.Goto(_startEventOfStream);
- while (deserializerReader.Current < _endOfEventStream)
+ if (_fileFormatVersionNumber >= 3)
{
- TraceEventNativeMethods.EVENT_RECORD* eventRecord = ReadEvent(deserializerReader);
- if (eventRecord != null)
+ // loop through the stream until we hit a null object. Deserialization of
+ // EventPipeEventBlocks will cause dispatch to happen.
+ // ReadObject uses registered factories and recognizes types by names, then derserializes them with FromStream
+ while (_deserializer.ReadObject() != null)
+ { }
+ }
+#if SUPPORT_V1_V2
+ else
+ {
+ PinnedStreamReader deserializerReader = (PinnedStreamReader)_deserializer.Reader;
+ while (deserializerReader.Current < _endOfEventStream)
{
- // in the code below we set sessionEndTimeQPC to be the timestamp of the last event.
- // Thus the new timestamp should be later, and not more than 1 day later.
- Debug.Assert(sessionEndTimeQPC <= eventRecord->EventHeader.TimeStamp);
- Debug.Assert(sessionEndTimeQPC == 0 || eventRecord->EventHeader.TimeStamp - sessionEndTimeQPC < _QPCFreq * 24 * 3600);
-
- TraceEvent event_ = Lookup(eventRecord);
- Dispatch(event_);
- sessionEndTimeQPC = eventRecord->EventHeader.TimeStamp;
+ TraceEventNativeMethods.EVENT_RECORD* eventRecord = ReadEvent(deserializerReader);
+ if (eventRecord != null)
+ {
+ // in the code below we set sessionEndTimeQPC to be the timestamp of the last event.
+ // Thus the new timestamp should be later, and not more than 1 day later.
+ Debug.Assert(sessionEndTimeQPC <= eventRecord->EventHeader.TimeStamp);
+ Debug.Assert(sessionEndTimeQPC == 0 || eventRecord->EventHeader.TimeStamp - sessionEndTimeQPC < _QPCFreq * 24 * 3600);
+
+ var traceEvent = Lookup(eventRecord);
+ Dispatch(traceEvent);
+ sessionEndTimeQPC = eventRecord->EventHeader.TimeStamp;
+ }
}
}
-
+#endif
return true;
}
- internal override string ProcessName(int processID, long timeQPC)
- {
- return _processName;
- }
+ internal override string ProcessName(int processID, long timeQPC) => _processName;
- private TraceEventNativeMethods.EVENT_RECORD* ReadEvent(PinnedStreamReader reader)
+ internal TraceEventNativeMethods.EVENT_RECORD* ReadEvent(PinnedStreamReader reader)
{
- // Guess that the event is < 1000 bytes or whatever is left in the stream.
- int eventSizeGuess = Math.Min(1000, _endOfEventStream.Sub(reader.Current));
- EventPipeEventHeader* eventData = (EventPipeEventHeader*)reader.GetPointer(eventSizeGuess);
+ EventPipeEventHeader* eventData = (EventPipeEventHeader*)reader.GetPointer(EventPipeEventHeader.HeaderSize);
+ eventData = (EventPipeEventHeader*)reader.GetPointer(eventData->TotalEventSize); // now we now the real size and get read entire event
+
// Basic sanity checks. Are the timestamps and sizes sane.
Debug.Assert(sessionEndTimeQPC <= eventData->TimeStamp);
Debug.Assert(sessionEndTimeQPC == 0 || eventData->TimeStamp - sessionEndTimeQPC < _QPCFreq * 24 * 3600);
Debug.Assert(0 <= eventData->PayloadSize && eventData->PayloadSize <= eventData->TotalEventSize);
- Debug.Assert(eventData->MetaDataId <= reader.Current); // IDs are the location in the of of the data, so it comes before
Debug.Assert(0 < eventData->TotalEventSize && eventData->TotalEventSize < 0x20000); // TODO really should be 64K but BulkSurvivingObjectRanges needs fixing.
+ Debug.Assert(_fileFormatVersionNumber < 3 ||
+ ((int)EventPipeEventHeader.PayloadBytes(eventData) % 4 == 0 && eventData->TotalEventSize % 4 == 0)); // ensure 4 byte alignment
- if (eventSizeGuess < eventData->TotalEventSize)
- eventData = (EventPipeEventHeader*)reader.GetPointer(eventData->TotalEventSize);
+ StreamLabel eventDataEnd = reader.Current.Add(eventData->TotalEventSize);
Debug.Assert(0 <= EventPipeEventHeader.StackBytesSize(eventData) && EventPipeEventHeader.StackBytesSize(eventData) <= eventData->TotalEventSize);
- // This asserts that the header size + payload + stackSize field + StackSize == TotalEventSize;
- Debug.Assert(eventData->PayloadSize + EventPipeEventHeader.HeaderSize + sizeof(int) + EventPipeEventHeader.StackBytesSize(eventData) == eventData->TotalEventSize);
- TraceEventNativeMethods.EVENT_RECORD* ret = null;
- EventPipeEventMetaData metaData;
- if (eventData->MetaDataId == 0) // Is this a Meta-data event?
- {
+ TraceEventNativeMethods.EVENT_RECORD* ret = null;;
+ if (eventData->IsMetadata())
+ {
int totalEventSize = eventData->TotalEventSize;
int payloadSize = eventData->PayloadSize;
- StreamLabel metaDataStreamOffset = reader.Current; // Used as the 'id' for the meta-data
+
// Note that this skip invalidates the eventData pointer, so it is important to pull any fields out we need first.
reader.Skip(EventPipeEventHeader.HeaderSize);
- metaData = new EventPipeEventMetaData(reader, payloadSize, _fileFormatVersionNumber, PointerSize, _processId);
- _eventMetadataDictionary.Add(metaDataStreamOffset, metaData);
- _eventParser.AddTemplate(metaData);
- int stackBytes = reader.ReadInt32(); // Meta-data events should always have a empty stack.
- Debug.Assert(stackBytes == 0);
-
- // We have read all the bytes in the event
- Debug.Assert(reader.Current == metaDataStreamOffset.Add(totalEventSize));
+
+ var metaData = new EventPipeEventMetaData(reader, payloadSize, _fileFormatVersionNumber, PointerSize, _processId);
+ _eventMetadataDictionary.Add(metaData.MetaDataId, metaData);
+
+ _eventParser.AddTemplate(metaData); // if we don't add the templates to this parse, we are going to have unhadled events (see https://github.com/Microsoft/perfview/issues/461)
+
+ int stackBytes = reader.ReadInt32();
+ Debug.Assert(stackBytes == 0, "Meta-data events should always have a empty stack");
}
else
{
-
- if (_eventMetadataDictionary.TryGetValue(eventData->MetaDataId, out metaData))
+ if (_eventMetadataDictionary.TryGetValue(eventData->MetaDataId, out var metaData))
ret = metaData.GetEventRecordForEventData(eventData);
else
Debug.Assert(false, "Warning can't find metaData for ID " + eventData->MetaDataId.ToString("x"));
- reader.Skip(eventData->TotalEventSize);
}
+ reader.Goto(eventDataEnd);
+
return ret;
}
@@ -142,24 +176,19 @@ internal override unsafe Guid GetRelatedActivityID(TraceEventNativeMethods.EVENT
return event_->RelatedActivityID;
}
- // We dont ever serialize one of these in managed code so we don't need to implement ToSTream
- public void ToStream(Serializer serializer)
- {
- throw new NotImplementedException();
- }
+ public void ToStream(Serializer serializer) => throw new InvalidOperationException("We dont ever serialize one of these in managed code so we don't need to implement ToSTream");
public void FromStream(Deserializer deserializer)
{
_fileFormatVersionNumber = deserializer.VersionBeingRead;
- if (deserializer.VersionBeingRead >= 3)
+#if SUPPORT_V1_V2
+ if (deserializer.VersionBeingRead < 3)
{
- var startEventStreamReference = deserializer.ReadForwardReference();
- _startEventOfStream = deserializer.ResolveForwardReference(startEventStreamReference, preserveCurrent: true);
+ ForwardReference reference = deserializer.ReadForwardReference();
+ _endOfEventStream = deserializer.ResolveForwardReference(reference, preserveCurrent: true);
}
- ForwardReference reference = deserializer.ReadForwardReference();
- _endOfEventStream = deserializer.ResolveForwardReference(reference, preserveCurrent: true);
-
+#endif
// The start time is stored as a SystemTime which is a bunch of shorts, convert to DateTime.
short year = deserializer.ReadInt16();
short month = deserializer.ReadInt16();
@@ -175,38 +204,92 @@ public void FromStream(Deserializer deserializer)
sessionStartTimeQPC = _syncTimeQPC;
- if (deserializer.VersionBeingRead >= 3)
+ if (3 <= deserializer.VersionBeingRead)
{
deserializer.Read(out pointerSize);
deserializer.Read(out _processId);
deserializer.Read(out numberOfProcessors);
deserializer.Read(out _expectedCPUSamplingRate);
}
+#if SUPPORT_V1_V2
else
{
_processId = 0; // V1 && V2 tests expect 0 for process Id
pointerSize = 8; // V1 EventPipe only supports Linux which is x64 only.
numberOfProcessors = 1;
-
- _startEventOfStream = deserializer.Current; // Events immediately after the header.
}
+#endif
}
- int _fileFormatVersionNumber;
- StreamLabel _startEventOfStream;
+#if SUPPORT_V1_V2
StreamLabel _endOfEventStream;
-
- Dictionary _eventMetadataDictionary = new Dictionary();
+#endif
+ int _fileFormatVersionNumber;
+ Dictionary _eventMetadataDictionary = new Dictionary();
Deserializer _deserializer;
EventPipeTraceEventParser _eventParser; // TODO does this belong here?
string _processName;
internal int _processId;
internal int _expectedCPUSamplingRate;
-
- #endregion
+#endregion
}
- #region private classes
+#region private classes
+
+ ///
+ /// An EVentPipeEventBlock represents a block of events. It basicaly only has
+ /// one field, which is the size in bytes of the block. But when its FromStream
+ /// is called, it will perform the callbacks for the events (thus deserializing
+ /// it performs dispatch).
+ ///
+ internal class EventPipeEventBlock : IFastSerializable
+ {
+ public EventPipeEventBlock(EventPipeEventSource source) => _source = source;
+
+ unsafe public void FromStream(Deserializer deserializer)
+ {
+ // blockSizeInBytes INCLUDES any padding bytes to ensure alignment.
+ var blockSizeInBytes = deserializer.ReadInt();
+
+ // after the block size comes eventual padding, we just need to skip it by jumping to the nearest aligned address
+ if((int)deserializer.Current % 4 != 0)
+ {
+ var nearestAlignedAddress = deserializer.Current.Add(4 - ((int)deserializer.Current % 4));
+ deserializer.Goto(nearestAlignedAddress);
+ }
+
+ _startEventData = deserializer.Current;
+ _endEventData = _startEventData.Add(blockSizeInBytes);
+ Debug.Assert((int)_startEventData % 4 == 0 && (int)_endEventData % 4 == 0); // make sure that the data is aligned
+
+ // Dispatch through all the events.
+ PinnedStreamReader deserializerReader = (PinnedStreamReader)deserializer.Reader;
+
+ while (deserializerReader.Current < _endEventData)
+ {
+ TraceEventNativeMethods.EVENT_RECORD* eventRecord = _source.ReadEvent(deserializerReader);
+ if (eventRecord != null)
+ {
+ // in the code below we set sessionEndTimeQPC to be the timestamp of the last event.
+ // Thus the new timestamp should be later, and not more than 1 day later.
+ Debug.Assert(_source.sessionEndTimeQPC <= eventRecord->EventHeader.TimeStamp);
+ Debug.Assert(_source.sessionEndTimeQPC == 0 || eventRecord->EventHeader.TimeStamp - _source.sessionEndTimeQPC < _source._QPCFreq * 24 * 3600);
+
+ var traceEvent = _source.Lookup(eventRecord);
+ _source.Dispatch(traceEvent);
+ _source.sessionEndTimeQPC = eventRecord->EventHeader.TimeStamp;
+ }
+ }
+
+ deserializerReader.Goto(_endEventData); // go to the end of block, in case some padding was not skipped yet
+ }
+
+ public void ToStream(Serializer serializer) => throw new InvalidOperationException();
+
+ StreamLabel _startEventData;
+ StreamLabel _endEventData;
+ EventPipeEventSource _source;
+ }
///
/// Private utility class.
@@ -238,8 +321,7 @@ unsafe class EventPipeEventMetaData
///
public EventPipeEventMetaData(PinnedStreamReader reader, int length, int fileFormatVersionNumber, int pointerSize, int processId)
{
- StreamLabel eventDataEnd = reader.Current.Add(length);
-
+ // Get the event record and fill in fields that we can without deserializing anything.
_eventRecord = (TraceEventNativeMethods.EVENT_RECORD*)Marshal.AllocHGlobal(sizeof(TraceEventNativeMethods.EVENT_RECORD));
ClearMemory(_eventRecord, sizeof(TraceEventNativeMethods.EVENT_RECORD));
@@ -250,71 +332,33 @@ public EventPipeEventMetaData(PinnedStreamReader reader, int length, int fileFor
_eventRecord->EventHeader.ProcessId = processId;
- StreamLabel metaDataStart = reader.Current;
- if (fileFormatVersionNumber == 1)
- _eventRecord->EventHeader.ProviderId = reader.ReadGuid();
- else
+ // Read the metaData
+ StreamLabel eventDataEnd = reader.Current.Add(length);
+ if (3 <= fileFormatVersionNumber)
{
+ MetaDataId = reader.ReadInt32();
ProviderName = reader.ReadNullTerminatedUnicodeString();
_eventRecord->EventHeader.ProviderId = GetProviderGuidFromProviderName(ProviderName);
- }
-
- var eventId = (ushort)reader.ReadInt32();
- _eventRecord->EventHeader.Id = eventId;
- Debug.Assert(_eventRecord->EventHeader.Id == eventId); // No truncation
-
- var version = reader.ReadInt32();
- _eventRecord->EventHeader.Version = (byte)version;
- Debug.Assert(_eventRecord->EventHeader.Version == version); // No truncation
- if (fileFormatVersionNumber >= 3)
- {
- long keywords = reader.ReadInt64();
- _eventRecord->EventHeader.Keyword = (ulong)keywords;
+ ReadEventMetaData(reader, fileFormatVersionNumber);
}
+#if SUPPORT_V1_V2
+ else
+ ReadObsoleteEventMetaData(reader, fileFormatVersionNumber);
+#endif
- int metadataLength = reader.ReadInt32();
- Debug.Assert(0 <= metadataLength && metadataLength < length);
- if (0 < metadataLength)
+ Debug.Assert(reader.Current == eventDataEnd);
+ }
+
+ ~EventPipeEventMetaData()
+ {
+ if (_eventRecord != null)
{
- // TODO why do we repeat the event number it is redundant.
- eventId = (ushort)reader.ReadInt32();
- Debug.Assert(_eventRecord->EventHeader.Id == eventId); // No truncation
- EventName = reader.ReadNullTerminatedUnicodeString();
- Debug.Assert(EventName.Length < length / 2);
-
- // Deduce the opcode from the name.
- if (EventName.EndsWith("Start", StringComparison.OrdinalIgnoreCase))
- _eventRecord->EventHeader.Opcode = (byte)TraceEventOpcode.Start;
- else if (EventName.EndsWith("Stop", StringComparison.OrdinalIgnoreCase))
- _eventRecord->EventHeader.Opcode = (byte)TraceEventOpcode.Stop;
-
- _eventRecord->EventHeader.Keyword = (ulong)reader.ReadInt64();
-
- // TODO why do we repeat the event number it is redundant.
- version = reader.ReadInt32();
- Debug.Assert(_eventRecord->EventHeader.Version == version); // No truncation
-
- _eventRecord->EventHeader.Level = (byte)reader.ReadInt32();
- Debug.Assert(_eventRecord->EventHeader.Level <= 5);
-
- // Fetch the parameter information
- int parameterCount = reader.ReadInt32();
- Debug.Assert(0 <= parameterCount && parameterCount < length / 8); // Each parameter takes at least 8 bytes.
- if (parameterCount > 0)
- {
- ParameterDefinitions = new Tuple[parameterCount];
- for (int i = 0; i < parameterCount; i++)
- {
- var type = (TypeCode)reader.ReadInt32();
- Debug.Assert((uint)type < 24); // There only a handful of type codes.
- var name = reader.ReadNullTerminatedUnicodeString();
- ParameterDefinitions[i] = new Tuple(type, name);
- Debug.Assert(reader.Current <= eventDataEnd);
- }
- }
+ if (_eventRecord->ExtendedData != null)
+ Marshal.FreeHGlobal((IntPtr)_eventRecord->ExtendedData);
+ Marshal.FreeHGlobal((IntPtr)_eventRecord);
+ _eventRecord = null;
}
- Debug.Assert(reader.Current == eventDataEnd);
}
///
@@ -325,7 +369,6 @@ public EventPipeEventMetaData(PinnedStreamReader reader, int length, int fileFor
///
internal TraceEventNativeMethods.EVENT_RECORD* GetEventRecordForEventData(EventPipeEventHeader* eventData)
{
-
// We have already initialize all the fields of _eventRecord that do no vary from event to event.
// Now we only have to copy over the fields that are specific to particular event.
_eventRecord->EventHeader.ThreadId = eventData->ThreadId;
@@ -373,6 +416,12 @@ public EventPipeEventMetaData(PinnedStreamReader reader, int length, int fileFor
return _eventRecord;
}
+ ///
+ /// This is a number that is unique to this meta-data blob. It is expected to be a small integer
+ /// that starts at 1 (since 0 is reserved) and increases from there (thus an array can be used).
+ /// It is what is matched up with EventPipeEventHeader.MetaDataId
+ ///
+ public int MetaDataId { get; private set; }
public string ProviderName { get; private set; }
public string EventName { get; private set; }
public Tuple[] ParameterDefinitions { get; private set; }
@@ -382,18 +431,77 @@ public EventPipeEventMetaData(PinnedStreamReader reader, int length, int fileFor
public ulong Keywords { get { return _eventRecord->EventHeader.Keyword; } }
public int Level { get { return _eventRecord->EventHeader.Level; } }
- #region private
- ~EventPipeEventMetaData()
+ ///
+ /// Reads the meta data for information specific to one event.
+ ///
+ private void ReadEventMetaData(PinnedStreamReader reader, int fileFormatVersionNumber)
{
- if (_eventRecord != null)
+ int eventId = (ushort)reader.ReadInt32();
+ _eventRecord->EventHeader.Id = (ushort)eventId;
+ Debug.Assert(_eventRecord->EventHeader.Id == eventId); // No truncation
+
+ EventName = reader.ReadNullTerminatedUnicodeString();
+
+ // Deduce the opcode from the name.
+ if (EventName.EndsWith("Start", StringComparison.OrdinalIgnoreCase))
+ _eventRecord->EventHeader.Opcode = (byte)TraceEventOpcode.Start;
+ else if (EventName.EndsWith("Stop", StringComparison.OrdinalIgnoreCase))
+ _eventRecord->EventHeader.Opcode = (byte)TraceEventOpcode.Stop;
+
+ _eventRecord->EventHeader.Keyword = (ulong)reader.ReadInt64();
+
+ int version = reader.ReadInt32();
+ _eventRecord->EventHeader.Version = (byte)version;
+ Debug.Assert(_eventRecord->EventHeader.Version == version); // No truncation
+
+ _eventRecord->EventHeader.Level = (byte)reader.ReadInt32();
+ Debug.Assert(_eventRecord->EventHeader.Level <= 5);
+
+ // Fetch the parameter information
+ int parameterCount = reader.ReadInt32();
+ Debug.Assert(0 <= parameterCount && parameterCount < 0x4000);
+ if (0 < parameterCount)
{
- if (_eventRecord->ExtendedData != null)
- Marshal.FreeHGlobal((IntPtr)_eventRecord->ExtendedData);
- Marshal.FreeHGlobal((IntPtr)_eventRecord);
- _eventRecord = null;
+ ParameterDefinitions = new Tuple[parameterCount];
+ for (int i = 0; i < parameterCount; i++)
+ {
+ var type = (TypeCode)reader.ReadInt32();
+ Debug.Assert((uint)type < 24); // There only a handful of type codes.
+ var name = reader.ReadNullTerminatedUnicodeString();
+ ParameterDefinitions[i] = new Tuple(type, name);
+ }
}
+ }
+
+#if SUPPORT_V1_V2
+ private void ReadObsoleteEventMetaData(PinnedStreamReader reader, int fileFormatVersionNumber)
+ {
+ Debug.Assert(fileFormatVersionNumber < 3);
+
+ // Old versions use the stream offset as the MetaData ID, but the reader has advanced to the payload so undo it.
+ MetaDataId = ((int)reader.Current) - EventPipeEventHeader.HeaderSize;
+ if (fileFormatVersionNumber == 1)
+ _eventRecord->EventHeader.ProviderId = reader.ReadGuid();
+ else
+ {
+ ProviderName = reader.ReadNullTerminatedUnicodeString();
+ _eventRecord->EventHeader.ProviderId = GetProviderGuidFromProviderName(ProviderName);
+ }
+
+ var eventId = (ushort)reader.ReadInt32();
+ _eventRecord->EventHeader.Id = eventId;
+ Debug.Assert(_eventRecord->EventHeader.Id == eventId); // No truncation
+
+ var version = reader.ReadInt32();
+ _eventRecord->EventHeader.Version = (byte)version;
+ Debug.Assert(_eventRecord->EventHeader.Version == version); // No truncation
+
+ int metadataLength = reader.ReadInt32();
+ if (0 < metadataLength)
+ ReadEventMetaData(reader, fileFormatVersionNumber);
}
+#endif
private void ClearMemory(void* buffer, int length)
{
@@ -403,56 +511,36 @@ private void ClearMemory(void* buffer, int length)
*ptr++ = 0;
--length;
}
-
}
+
public static Guid GetProviderGuidFromProviderName(string name)
{
- if (String.IsNullOrEmpty(name))
- {
+ if (string.IsNullOrEmpty(name))
return Guid.Empty;
- }
// Legacy GUID lookups (events which existed before the current Guid generation conventions)
if (name == TplEtwProviderTraceEventParser.ProviderName)
- {
return TplEtwProviderTraceEventParser.ProviderGuid;
- }
else if (name == ClrTraceEventParser.ProviderName)
- {
return ClrTraceEventParser.ProviderGuid;
- }
else if (name == ClrPrivateTraceEventParser.ProviderName)
- {
return ClrPrivateTraceEventParser.ProviderGuid;
- }
else if (name == ClrRundownTraceEventParser.ProviderName)
- {
return ClrRundownTraceEventParser.ProviderGuid;
- }
else if (name == ClrStressTraceEventParser.ProviderName)
- {
return ClrStressTraceEventParser.ProviderGuid;
- }
else if (name == FrameworkEventSourceTraceEventParser.ProviderName)
- {
return FrameworkEventSourceTraceEventParser.ProviderGuid;
- }
- // Needed as long as eventpipeinstance v1 objects are supported
+#if SUPPORT_V1_V2
else if (name == SampleProfilerTraceEventParser.ProviderName)
- {
return SampleProfilerTraceEventParser.ProviderGuid;
- }
-
+#endif
// Hash the name according to current event source naming conventions
else
- {
return TraceEventProviders.GetEventSourceGuidFromName(name);
- }
}
-
TraceEventNativeMethods.EVENT_RECORD* _eventRecord;
- #endregion
}
///
@@ -470,7 +558,7 @@ public static Guid GetProviderGuidFromProviderName(string name)
unsafe struct EventPipeEventHeader
{
private int EventSize; // Size bytes of this header and the payload and stacks if any. does NOT incode the size of the EventSize field itself.
- public StreamLabel MetaDataId; // a number identifying the description of this event. It is a stream location.
+ public int MetaDataId; // a number identifying the description of this event.
public int ThreadId;
public long TimeStamp;
public Guid ActivityID;
@@ -478,22 +566,26 @@ unsafe struct EventPipeEventHeader
public int PayloadSize; // size in bytes of the user defined payload data.
public fixed byte Payload[4]; // Actually of variable size. 4 is used to avoid potential alignment issues. This 4 also appears in HeaderSize below.
- public int TotalEventSize { get { return EventSize + sizeof(int); } } // Includes the size of the EventSize field itself
+ public int TotalEventSize => EventSize + sizeof(int); // Includes the size of the EventSize field itself
+
+ public bool IsMetadata() => MetaDataId == 0; // 0 means that it's a metadata Id
///
/// Header Size is defined to be the number of bytes before the Payload bytes.
///
- static public int HeaderSize { get { return sizeof(EventPipeEventHeader) - 4; } }
- static public EventPipeEventHeader* HeaderFromPayloadPointer(byte* payloadPtr) { return (EventPipeEventHeader*)(payloadPtr - HeaderSize); }
+ static public int HeaderSize => sizeof(EventPipeEventHeader) - 4;
- static public int StackBytesSize(EventPipeEventHeader* header)
- {
- return *((int*)(&header->Payload[header->PayloadSize]));
- }
- static public byte* StackBytes(EventPipeEventHeader* header)
- {
- return (byte*)(&header->Payload[header->PayloadSize + 4]);
- }
+ static public EventPipeEventHeader* HeaderFromPayloadPointer(byte* payloadPtr)
+ => (EventPipeEventHeader*)(payloadPtr - HeaderSize);
+
+ static public int StackBytesSize(EventPipeEventHeader* header)
+ => *((int*)(&header->Payload[header->PayloadSize]));
+
+ static public byte* StackBytes(EventPipeEventHeader* header)
+ => &header->Payload[header->PayloadSize + 4];
+
+ static public byte* PayloadBytes(EventPipeEventHeader* header)
+ => &header->Payload[0];
}
- #endregion
+#endregion
}
diff --git a/src/TraceEvent/EventPipe/EventPipeFormat.md b/src/TraceEvent/EventPipe/EventPipeFormat.md
new file mode 100644
index 000000000..e431d5cbb
--- /dev/null
+++ b/src/TraceEvent/EventPipe/EventPipeFormat.md
@@ -0,0 +1,241 @@
+# EventPipe (File) Format
+
+EventPipe is the name of the logging mechanism given to system used by the .NET Core
+runtime to log events in a OS independent way. It is meant to serve roughly the same
+niche as ETW does on Windows, but works equally well on Linux.
+
+By convention files in this format are call *.netperf files and this can be thought
+of as the NetPerf File format. However the format is more flexible than that.
+
+The format was designed to take advantage of the facilities of the FastSerialization
+library used by TraceEvent, however the format can be understood on its own, and here
+we describe everything you need to know to use the format.
+
+Fundamentally, the data can be thought of as a serialization of objects. we want the
+format to be Simple, Extensible (it can tolerate multiple versions) and
+make it as easy as possible to be both backward (new readers can read old data version)
+and forward (old readers can read new data versions). We also want to be efficient
+and STREAMABLE (no need for seek, you can do most operations with just 'read').
+
+Assumptions of the Format:
+
+We assume the following:
+
+* Primitive Types: The format assumes you can emit the primitive data types
+ (byte, short, int, long). It is in little endian (least significant byte first)
+* Strings: Strings can be emitted by emitting a int BYTE count followed by the
+ UTF8 encoding
+* StreamLabels: The format assumes you know the start of the stream (0) and
+ you keep track of your position. The format currently assumes this is
+ a 32 bit number (thus limiting references using StreamLabels to 4GB)
+ This may change but it is a format change if you do).
+* Compression: The format does not try to be particularly smart about compression
+ The idea is that compression is VERY likely to be best done by compressing
+ the stream as a whole so it is not that important that we do 'smart' things
+ like make variable length integers etc. Instead the format is tuned for
+ making it easy for the memory to be used 'in place' and assumes that compression
+ will be done on the stream outside of the serialization/deserialization.
+ * Alignment: by default the stream is only assumed to be byte aligned. However
+ as you will see particular objects have a lot of flexibility in their encoding
+ and they may choose to align their data. The is valuable because it allows
+ efficient 'in place' use of the data stream, however it is more the exception
+ than the rule.
+
+## First Bytes: The Stream Header:
+
+The beginning of the format is always the stream header. This header's only purpose
+is to quickly identify the format of this stream (file) as a whole, and to indicate
+exactly which version of the basic Stream library should be used. It is exactly
+one (length prefixed UTF string with the value "!FastSerialization.1" This declares
+the the rest of file uses the FastSerialization version 1 conventions.
+
+Thus the first 24 bytes of the file will be
+ 4 bytes little endian number 20 (number of bytes in "!FastSerialization.1"
+ 20 bytes of the UTF8 encoding of "!FastSerialization.1"
+
+After the format is a list of objects.
+
+## Objects:
+
+The format has the concept of an object. Indeed the stream can be thought of as
+simply the serialization of a list of objects.
+
+Tags: The format uses a number of byte-sized tags that are used in the serialization
+and use of objects. In particular there are BeginObject and EndObject which
+are used to define a new object, as well as a few other (discussed below) which
+allow you to refer to objects.
+There are only a handful of them, see the Tags Enum for a complete list.
+
+Object Types: every object has a type. A type at a minimum represents
+ 1. The name of the type (which allows the serializer and deserializer to agree what
+ is being transmitted
+ 2. The version number for the data being sent.
+ 3. A minumum version number. new format MAY be compatible with old readers
+ this version indicates the oldest reader that can read this format.
+
+An object's structure is
+
+* BeginObject Tag
+* SERIALIZED TYPE
+* SERIALIZED DATA
+* EndObject Tag
+
+As mentioned a type is just another object, but the if that is true it needs a type
+which leads to infinite recursion. Thus the type of a type is alwasy simply
+a special tag call the NullReference that represent null.
+
+## The First Object: The EventTrace Object
+
+After the Trace Header comes the EventTrace object, which represents all the data
+about the Trace as a whole.
+
+* BeginObject Tag (begins the EventTrace Object)
+* BeginObject Tag (begins the Type Object for EventTrace)
+* NullReference Tag (represents the type of type, which is by convention null)
+* 4 byte integer Version field for type
+* 4 byte integer MinimumReaderVersion field for type
+* SERIALIZED STRING for FullName Field for type (4 byte length + UTF8 bytes)
+* EndObject Tag (ends Type Object)
+* DATA FIELDS FOR EVENTTRACE OBJECT
+* End Object Tag (for EventTrace object)
+
+The data field for object depend are deserialized in the 'FromStream' for
+the class that deserialize the object. EventPipeEventSource is the class
+that deserializes the EventTrace object, so you can see its fields there.
+These fields are the things like the time the trace was collected, the
+units of the event timestamps, and other things that apply to all events.
+
+## Next Objects : The EventBlock Object
+
+After the EventTrace object there are zero or more EventBlock objects.
+they look very much like the EventTrace object's layout ultimate fields
+are different
+
+* BeginObject Tag (begins the EventBlock Object)
+* BeginObject Tag (begins the Type Object for EventBlock)
+* NullReference Tag (represents the type of type, which is by convention null)
+* 4 byte integer Version field for type
+* 4 byte integer MinimumReaderVersion field for type
+* SERIALIZED STRING for FullName Field for type (4 byte length + UTF8 bytes)
+* EndObject Tag (ends Type Object)
+* DATA FIELDS FOR EVENTBLOCK OBJECT (size of blob + event bytes blob)
+* End Object Tag (for EventBlock object)
+
+The data in an EventBlock is simply an integer representing the size (in
+bytes not including the size int itself) of the data blob and the event
+data blob itself.
+
+The event blob itself is simply a list of 'event' blobs. each blob has
+a header (defined by EventPipeEventHeader), following by some number of
+bytes of payload data, followed by the byteSize and bytes for the stack
+associated with the event. See EventPipeEventHeader for details.
+
+Some events are actually not true data events but represent meta-data
+about an event. This data includes the name of the event, the name
+of the provider of the event and the names and types of all the fields
+of the event. This meta-data is given an small integer numeric ID
+(starts at 1 and grows incrementally),
+
+One of the fields for an event is this Meta-data ID. An event with
+a Meta-data ID of 0 is expected to be a Meta-data event itself.
+See the constructor of EventPipeEventMetaData for details of the
+format of this event.
+
+## Ending the stream: The NullReference Tag
+
+After the last EventBlock is emitted, the stream is ended by
+emitting a NullReference Tag which indicates that there are no
+more objects in the stream to read.
+
+## Versioning the Format While Maintaining Compatibility
+
+### Backward compatibility
+
+It is a relatively straightforward excercise to update the file format
+to add more information while maintaining backward compatibility (that is
+new readers can read old writers). What is necessary is to
+
+1. For the EventTrace Type, Increment the Version number
+and set the MinimumReaderVersion number to this same value.
+2. Update the reader for the changed type to look at the Version
+number of the type and if it is less than the new version do
+what you did before, and if it is the new version read the new format
+for that object.
+
+By doing (1) we make it so that every OLD reader does not simply
+crash misinterpreting data, but will learly notice that it does
+not support this new version (because the readers Version is less
+than the MinimumReaderVersion value), and can issue a clean error
+that is useful to the user.
+
+Doing (2) is also straightforward, but it does mean keeping the old
+reading code. This is the price of compatibility.
+
+### Forward compatibility
+
+Making changes so that we preserve FORWARD compatibility (old readers
+can read new writers) is more constaining, because old readers have
+to at least know how to 'skip' things they don't understand.
+
+There are however several ways to do this. The simplest way is to
+
+* Add Tagged values to an object.
+
+Every object has a begin tag, a type, data objects, and an end tag.
+One feature of the FastSerialiable library is that it has a tag
+for all the different data types (bool, byte, short, int, long, string blob).
+It also has logic that after parsing the data area it 'looks' for
+the end tag (so we know the data is partially sane at least). However
+during this search if it finds other tags, it knows how to skip them.
+Thus if after the 'Know Version 0' data objects, you place tagged
+data, ANY reader will know how to skip it (it skips all tagged things
+until it finds an endObject tag).
+
+This allows you to add new fields to an object in a way that OLD
+readers can still parse (at least enough to skip them).
+
+Another way to add new data to the file is to
+
+* Add new object (and object types) to the list of objects.
+
+The format is basically a list of objects, but there is no requirement
+that there are only very loose requirements on the order or number of these
+Thus you can create a new object type and insert that object in the
+stream (that object must have only tagged fields however but a tagged
+blob can do almost anything). This allows whole new objects to be
+added to the file format without breaking existing readers.
+
+#### Version Numbers and forward compatibility.
+
+There is no STRONG reason to update the version number when you make
+changes to the format that are both forward (and backward compatible).
+However it can be useful to update the file version because it allows
+readers to quickly determine the set of things it can 'count on' and
+therefore what user interface can be supported. Thus it can be useful
+to update the version number when a non-trival amount of new functionality
+is added.
+
+You can update the Version number but KEEP the MinimumReaderVersion
+unchanged to do this. THus readers quickly know what they can count on
+but old readers can still read the new format.
+
+## Suport for Random Access Streams
+
+So far the features used in the file format are the simplest. In particular
+on object never directly 'points' at another and the stream can be
+processed usefully without needing information later in the file.
+
+But we pay a price for this: namely you have to read all the data in the
+file even if you only care about a small fraction of it. If however
+you have random access (seeking) for your stream (that is it is a file),
+you can overcome this.
+
+The serialization library allows this by supporting a table of pointers
+to objects and placing this table at the end of the stream (when you
+know the stream locations of all objects). This would allow you to
+seek to any particular object and only read what you need.
+
+The FastSerialization library supports this, but the need for this kind
+of 'random access' is not clear at this time (mostly the data needs
+to be processed again and thus you need to read it all anyway). For
+now it is is enough to know that this capability exists if we need it.
\ No newline at end of file
diff --git a/src/TraceEvent/TraceEvent.Tests/EventPipeParsing.cs b/src/TraceEvent/TraceEvent.Tests/EventPipeParsing.cs
index 16001e7da..4f40b7e6b 100644
--- a/src/TraceEvent/TraceEvent.Tests/EventPipeParsing.cs
+++ b/src/TraceEvent/TraceEvent.Tests/EventPipeParsing.cs
@@ -100,13 +100,13 @@ public void CanParseHeaderOfV3EventPipeFile()
using (var eventPipeSource = new EventPipeEventSource(eventPipeFilePath))
{
Assert.Equal(4, eventPipeSource.PointerSize);
- Assert.Equal(11376, eventPipeSource._processId);
+ Assert.Equal(3312, eventPipeSource._processId);
Assert.Equal(4, eventPipeSource.NumberOfProcessors);
Assert.Equal(1000000, eventPipeSource._expectedCPUSamplingRate);
- Assert.Equal(636522350205880000, eventPipeSource._syncTimeUTC.Ticks);
- Assert.Equal(44518740604, eventPipeSource._syncTimeQPC);
- Assert.Equal(2533308, eventPipeSource._QPCFreq);
+ Assert.Equal(636531024984420000, eventPipeSource._syncTimeUTC.Ticks);
+ Assert.Equal(20461004832, eventPipeSource._syncTimeQPC);
+ Assert.Equal(2533315, eventPipeSource._QPCFreq);
Assert.Equal(10, eventPipeSource.CpuSpeedMHz);
}
diff --git a/src/TraceEvent/TraceEvent.Tests/inputs/eventpipe-dotnetcore2.1-win-x86-objver3.netperf.baseline.txt b/src/TraceEvent/TraceEvent.Tests/inputs/eventpipe-dotnetcore2.1-win-x86-objver3.netperf.baseline.txt
index 4f2cf14ba..50b65d78a 100644
--- a/src/TraceEvent/TraceEvent.Tests/inputs/eventpipe-dotnetcore2.1-win-x86-objver3.netperf.baseline.txt
+++ b/src/TraceEvent/TraceEvent.Tests/inputs/eventpipe-dotnetcore2.1-win-x86-objver3.netperf.baseline.txt
@@ -1,21 +1,27 @@
-Microsoft-DotNETCore-SampleProfiler/Thread/Sample, 1, \r\n
-Microsoft-Windows-DotNETRuntime/AppDomainResourceManagement/ThreadCreated, 1,
-Microsoft-Windows-DotNETRuntime/GC/AllocationTick, 1,
-Microsoft-Windows-DotNETRuntime/GC/GlobalHeapHistory, 10,
-Microsoft-Windows-DotNETRuntime/GC/HeapStats, 10,
-Microsoft-Windows-DotNETRuntime/GC/MarkWithType, 30,
-Microsoft-Windows-DotNETRuntime/GC/PerHeapHistory, 10, \r\n
-Microsoft-Windows-DotNETRuntime/GC/RestartEEStart, 11,
-Microsoft-Windows-DotNETRuntime/GC/RestartEEStop, 11,
-Microsoft-Windows-DotNETRuntime/GC/SuspendEEStart, 11,
-Microsoft-Windows-DotNETRuntime/GC/SuspendEEStop, 11,
-Microsoft-Windows-DotNETRuntime/GC/Triggered, 10,
-Microsoft-Windows-DotNETRuntime/Method/JittingStarted, 2,
-Microsoft-Windows-DotNETRuntime/Method/LoadVerbose, 2,
-Microsoft-Windows-DotNETRuntimeRundown/Loader/AppDomainDCStop, 2,
-Microsoft-Windows-DotNETRuntimeRundown/Loader/AssemblyDCStop, 13,
-Microsoft-Windows-DotNETRuntimeRundown/Loader/DomainModuleDCStop, 12,
-Microsoft-Windows-DotNETRuntimeRundown/Loader/ModuleDCStop, 13,
-Microsoft-Windows-DotNETRuntimeRundown/Method/DCStopComplete, 1,
-Microsoft-Windows-DotNETRuntimeRundown/Method/DCStopInit, 1,
+Microsoft-DotNETCore-SampleProfiler/Thread/Sample, 6, \r\n
+Microsoft-Windows-DotNETRuntime/AppDomainResourceManagement/ThreadCreated, 1,
+Microsoft-Windows-DotNETRuntime/GC/AllocationTick, 1,
+Microsoft-Windows-DotNETRuntime/GC/BulkMovedObjectRanges, 1, \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n
+Microsoft-Windows-DotNETRuntime/GC/GenerationRange, 80,
+Microsoft-Windows-DotNETRuntime/GC/GlobalHeapHistory, 10,
+Microsoft-Windows-DotNETRuntime/GC/HeapStats, 10,
+Microsoft-Windows-DotNETRuntime/GC/MarkWithType, 30,
+Microsoft-Windows-DotNETRuntime/GC/PerHeapHistory, 10, \r\n
+Microsoft-Windows-DotNETRuntime/GC/RestartEEStart, 16,
+Microsoft-Windows-DotNETRuntime/GC/RestartEEStop, 16,
+Microsoft-Windows-DotNETRuntime/GC/Start, 10,
+Microsoft-Windows-DotNETRuntime/GC/Stop, 10,
+Microsoft-Windows-DotNETRuntime/GC/SuspendEEStart, 16,
+Microsoft-Windows-DotNETRuntime/GC/SuspendEEStop, 16,
+Microsoft-Windows-DotNETRuntime/GC/Triggered, 10,
+Microsoft-Windows-DotNETRuntime/Method/JittingStarted, 2,
+Microsoft-Windows-DotNETRuntime/Method/LoadVerbose, 2,
+Microsoft-Windows-DotNETRuntimePrivate/GC/PinPlugAtGCTime, 19,
+Microsoft-Windows-DotNETRuntimeRundown/Loader/AppDomainDCStop, 2,
+Microsoft-Windows-DotNETRuntimeRundown/Loader/AssemblyDCStop, 13,
+Microsoft-Windows-DotNETRuntimeRundown/Loader/DomainModuleDCStop, 12,
+Microsoft-Windows-DotNETRuntimeRundown/Loader/ModuleDCStop, 13,
+Microsoft-Windows-DotNETRuntimeRundown/Method/DCStopComplete, 1,
+Microsoft-Windows-DotNETRuntimeRundown/Method/DCStopInit, 1,
diff --git a/src/TraceEvent/TraceEvent.Tests/inputs/eventpipe-dotnetcore2.1-win-x86-objver3.netperf.zip b/src/TraceEvent/TraceEvent.Tests/inputs/eventpipe-dotnetcore2.1-win-x86-objver3.netperf.zip
index 6b98576b5..97d131f64 100644
Binary files a/src/TraceEvent/TraceEvent.Tests/inputs/eventpipe-dotnetcore2.1-win-x86-objver3.netperf.zip and b/src/TraceEvent/TraceEvent.Tests/inputs/eventpipe-dotnetcore2.1-win-x86-objver3.netperf.zip differ
diff --git a/src/TraceEvent/TraceEvent.csproj b/src/TraceEvent/TraceEvent.csproj
index 58e416315..4f0744b94 100644
--- a/src/TraceEvent/TraceEvent.csproj
+++ b/src/TraceEvent/TraceEvent.csproj
@@ -146,6 +146,7 @@
PreserveNewest
False
+