Event Pipe File V3 (#16107)

* write missing information to the event pipe file (pointer size to make it work fo x86)

* define where the events start, not only where they end

* include process Id in the event pipe file, bump the version so old consumers get clear error message

* write the missing EndObject tag to close the header

* include expected CPU sampling rate in the event pipe header file

* include keywords in V3 of EventPipe metadata, fixes #11934

* remove forward references

* entry object comes after the header and ends after it's data, before the event block objects

* introduce event block

* fix the GC contracts

* generate metadata ids

* end the file with null reference tag

* getting it work

* 4 byte alignment of serialized event data

* Revert "include keywords in V3 of EventPipe metadata, fixes #11934"

This reverts commit 98ef2f5.

* remove event Id and event version from metadata buffer (it was duplicated with native code)

* increase the block size to be the same as buffer size

* Write the last event block to the file after disabling the event pipe, right after last events

* include the sife in itself

* the native part was supposed to not duplicate the event id and version, not manged

* ensure 4 byte alignment

* build metadata when it's not provided, so payload is never empty (no need to serialize length)

* this todo is no longer valid

* don't align everything, just the content of event block as suggested by @vancem

* improvements after code review

* update TraceEvent dependency, make the test verify new feature

* InterlockedIncrement(Int32) is not available for non-Windows OSes

* code improvements after Skype code review from @jorive

dependencies.props

@@ -38,7 +38,7 @@
     <XunitPackageVersion>2.2.0-beta2-build3300</XunitPackageVersion>
     <XunitConsoleNetcorePackageVersion>1.0.2-prerelease-00177</XunitConsoleNetcorePackageVersion>
     <XunitPerformanceApiPackageVersion>1.0.0-beta-build0015</XunitPerformanceApiPackageVersion>
-    <MicrosoftDiagnosticsTracingTraceEventPackageVersion>2.0.2</MicrosoftDiagnosticsTracingTraceEventPackageVersion>
+    <MicrosoftDiagnosticsTracingTraceEventPackageVersion>2.0.4</MicrosoftDiagnosticsTracingTraceEventPackageVersion>
     <VCRuntimeVersion>1.2.0</VCRuntimeVersion>
   </PropertyGroup>
 

src/vm/CMakeLists.txt

@@ -184,6 +184,7 @@ set(VM_SOURCES_WKS
     eventpipeconfiguration.cpp
     eventpipeevent.cpp
     eventpipeeventinstance.cpp
+    eventpipeblock.cpp
     eventpipefile.cpp
     eventpipejsonfile.cpp
     eventpipeprovider.cpp

src/vm/eventpipeblock.cpp

@@ -0,0 +1,146 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#include "common.h"
+#include "eventpipeblock.h"
+#include "eventpipeeventinstance.h"
+#include "fastserializableobject.h"
+#include "fastserializer.h"
+
+#ifdef FEATURE_PERFTRACING
+
+EventPipeBlock::EventPipeBlock(unsigned int maxBlockSize)
+{
+    CONTRACTL
+    {
+        NOTHROW;
+        GC_NOTRIGGER;
+        MODE_ANY;
+    }
+    CONTRACTL_END;
+
+    m_pBlock = new (nothrow) BYTE[maxBlockSize];
+    if (m_pBlock == NULL)
+    {
+        return;
+    }
+
+    memset(m_pBlock, 0, maxBlockSize);
+    m_pWritePointer = m_pBlock;
+    m_pEndOfTheBuffer = m_pBlock + maxBlockSize;
+}
+
+EventPipeBlock::~EventPipeBlock()
+{
+    CONTRACTL
+    {
+        NOTHROW;
+        GC_NOTRIGGER;
+        MODE_ANY;
+    }
+    CONTRACTL_END;
+
+    if(m_pBlock != NULL)
+    {
+        m_pEndOfTheBuffer = NULL;
+        m_pWritePointer = NULL;
+        delete[] m_pBlock;
+        m_pBlock = NULL;
+    }
+}
+
+bool EventPipeBlock::WriteEvent(EventPipeEventInstance &instance)
+{
+    CONTRACTL
+    {
+        NOTHROW;
+        GC_NOTRIGGER;
+        MODE_ANY;
+    }
+    CONTRACTL_END;
+
+    if (m_pBlock == NULL)
+    {
+        return false;
+    }
+
+    unsigned int totalSize = instance.GetAlignedTotalSize();
+    if (m_pWritePointer + totalSize >= m_pEndOfTheBuffer)
+    {
+        return false;
+    }
+
+    BYTE* alignedEnd = m_pWritePointer + totalSize + sizeof(totalSize); 
+
+    memcpy(m_pWritePointer, &totalSize, sizeof(totalSize));
+    m_pWritePointer += sizeof(totalSize);
+
+    unsigned int metadataId = instance.GetMetadataId();
+    memcpy(m_pWritePointer, &metadataId, sizeof(metadataId));
+    m_pWritePointer += sizeof(metadataId);
+
+    DWORD threadId = instance.GetThreadId();
+    memcpy(m_pWritePointer, &threadId, sizeof(threadId));
+    m_pWritePointer += sizeof(threadId);
+
+    const LARGE_INTEGER* timeStamp = instance.GetTimeStamp();
+    memcpy(m_pWritePointer, timeStamp, sizeof(*timeStamp));
+    m_pWritePointer += sizeof(*timeStamp);
+
+    const GUID* activityId = instance.GetActivityId();
+    memcpy(m_pWritePointer, activityId, sizeof(*activityId));
+    m_pWritePointer += sizeof(*activityId);
+
+    const GUID* relatedActivityId = instance.GetRelatedActivityId();
+    memcpy(m_pWritePointer, relatedActivityId, sizeof(*relatedActivityId));
+    m_pWritePointer += sizeof(*relatedActivityId);
+
+    unsigned int dataLength = instance.GetDataLength();
+    memcpy(m_pWritePointer, &dataLength, sizeof(dataLength));
+    m_pWritePointer += sizeof(dataLength);
+
+    if (dataLength > 0)
+    {
+        memcpy(m_pWritePointer, instance.GetData(), dataLength);
+        m_pWritePointer += dataLength;
+    }
+
+    unsigned int stackSize = instance.GetStackSize();
+    memcpy(m_pWritePointer, &stackSize, sizeof(stackSize));
+    m_pWritePointer += sizeof(stackSize);
+
+    if (stackSize > 0)
+    {
+        memcpy(m_pWritePointer, instance.GetStack(), stackSize);
+        m_pWritePointer += stackSize;
+    }
+
+    while (m_pWritePointer < alignedEnd)
+    {
+        *m_pWritePointer++ = (BYTE)0; // put padding at the end to get 4 bytes alignment of the payload
+    }
+
+    return true;
+}
+
+void EventPipeBlock::Clear()
+{
+    CONTRACTL
+    {
+        NOTHROW;
+        GC_NOTRIGGER;
+        MODE_ANY;
+    }
+    CONTRACTL_END;
+
+    if (m_pBlock == NULL)
+    {
+        return;
+    }
+
+    memset(m_pBlock, 0, GetSize());
+    m_pWritePointer = m_pBlock;
+}
+
+#endif // FEATURE_PERFTRACING

src/vm/eventpipeblock.h

@@ -0,0 +1,92 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#ifndef __EVENTPIPE_BLOCK_H__
+#define __EVENTPIPE_BLOCK_H__
+
+#ifdef FEATURE_PERFTRACING
+
+#include "eventpipeeventinstance.h"
+#include "fastserializableobject.h"
+#include "fastserializer.h"
+
+class EventPipeBlock : public FastSerializableObject
+{
+    public:
+        EventPipeBlock(unsigned int maxBlockSize);
+
+        ~EventPipeBlock();
+
+        // Write an event to the block.
+        // Returns:
+        //  - true: The write succeeded.
+        //  - false: The write failed.  In this case, the block should be considered full.
+        bool WriteEvent(EventPipeEventInstance &instance);
+
+        void Clear();
+
+        const char* GetTypeName()
+        {
+            LIMITED_METHOD_CONTRACT;
+            return "EventBlock";
+        }
+
+        void FastSerialize(FastSerializer *pSerializer)
+        {
+            CONTRACTL
+            {
+                NOTHROW;
+                GC_NOTRIGGER;
+                MODE_PREEMPTIVE;
+                PRECONDITION(pSerializer != NULL);
+            }
+            CONTRACTL_END;
+
+            if (m_pBlock == NULL)
+            {
+                return;
+            }
+
+            unsigned int eventsSize = (unsigned int)(m_pWritePointer - m_pBlock);
+            pSerializer->WriteBuffer((BYTE*)&eventsSize, sizeof(eventsSize));
+
+            if (eventsSize == 0)
+            {
+                return;
+            }
+
+            size_t currentPosition = pSerializer->GetCurrentPosition();
+            if (currentPosition % ALIGNMENT_SIZE != 0)
+            {
+                BYTE maxPadding[ALIGNMENT_SIZE - 1] = {}; // it's longest possible padding, we are going to use only part of it
+                unsigned int paddingLength = ALIGNMENT_SIZE - (currentPosition % ALIGNMENT_SIZE);
+                pSerializer->WriteBuffer(maxPadding, paddingLength); // we write zeros here, the reader is going to always read from the first aligned address of the serialized content 
+
+                _ASSERTE(pSerializer->GetCurrentPosition() % ALIGNMENT_SIZE == 0);
+            }
+
+            pSerializer->WriteBuffer(m_pBlock, eventsSize);
+        }
+
+    private:
+        BYTE *m_pBlock;
+        BYTE *m_pWritePointer;
+        BYTE *m_pEndOfTheBuffer;
+
+        unsigned int GetSize() const
+        {
+            LIMITED_METHOD_CONTRACT;
+
+            if (m_pBlock == NULL)
+            {
+                return 0;
+            }
+
+            return (unsigned int)(m_pEndOfTheBuffer - m_pBlock);
+        }
+};
+
+#endif // FEATURE_PERFTRACING
+
+#endif // __EVENTPIPE_BLOCK_H__

src/vm/eventpipebuffer.cpp

@@ -97,7 +97,7 @@ bool EventPipeBuffer::WriteEvent(Thread *pThread, EventPipeSession &session, Eve
         }
 
         // Save the most recent event timestamp.
-        m_mostRecentTimeStamp = pInstance->GetTimeStamp();
+        m_mostRecentTimeStamp = *pInstance->GetTimeStamp();
 
     }
     EX_CATCH
@@ -174,7 +174,7 @@ EventPipeEventInstance* EventPipeBuffer::GetNext(EventPipeEventInstance *pEvent,
 
         // We have a pointer within the bounds of the buffer.
         // Find the next event by skipping the current event with it's data payload immediately after the instance.
-        pNextInstance = (EventPipeEventInstance *)(pEvent->GetData() + pEvent->GetLength());
+        pNextInstance = (EventPipeEventInstance *)(pEvent->GetData() + pEvent->GetDataLength());
 
         // Check to see if we've reached the end of the written portion of the buffer.
         if((BYTE*)pNextInstance >= m_pCurrent)
@@ -184,7 +184,7 @@ EventPipeEventInstance* EventPipeBuffer::GetNext(EventPipeEventInstance *pEvent,
     }
 
     // Ensure that the timestamp is valid.  The buffer is zero'd before use, so a zero timestamp is invalid.
-    LARGE_INTEGER nextTimeStamp = pNextInstance->GetTimeStamp();
+    LARGE_INTEGER nextTimeStamp = *pNextInstance->GetTimeStamp();
     if(nextTimeStamp.QuadPart == 0)
     {
         return NULL;
@@ -260,10 +260,10 @@ bool EventPipeBuffer::EnsureConsistency()
         _ASSERTE(pInstance->EnsureConsistency());
 
         // Validate that payload and length match.
-        _ASSERTE((pInstance->GetData() != NULL && pInstance->GetLength() > 0) || (pInstance->GetData() != NULL && pInstance->GetLength() == 0));
+        _ASSERTE((pInstance->GetData() != NULL && pInstance->GetDataLength() > 0) || (pInstance->GetData() != NULL && pInstance->GetDataLength() == 0));
 
         // Skip the event.
-        ptr += sizeof(*pInstance) + pInstance->GetLength();
+        ptr += sizeof(*pInstance) + pInstance->GetDataLength();
     }
 
     // When we're done walking the filled portion of the buffer,

src/vm/eventpipebuffermanager.cpp

@@ -430,7 +430,7 @@ void EventPipeBufferManager::WriteAllBuffersToFile(EventPipeFile *pFile, LARGE_I
             {
                 // If it's the oldest event we've seen, then save it.
                 if((pOldestInstance == NULL) ||
-                   (pOldestInstance->GetTimeStamp().QuadPart > pNext->GetTimeStamp().QuadPart)) 
+                   (pOldestInstance->GetTimeStamp()->QuadPart > pNext->GetTimeStamp()->QuadPart)) 
                 {
                     pOldestInstance = pNext;
                     pOldestContainingBuffer = pContainingBuffer;

src/vm/eventpipeconfiguration.cpp

@@ -448,7 +448,7 @@ void EventPipeConfiguration::EnableRundown(EventPipeSession *pSession)
     Enable(pSession);
 }
 
-EventPipeEventInstance* EventPipeConfiguration::BuildEventMetadataEvent(EventPipeEventInstance &sourceInstance)
+EventPipeEventInstance* EventPipeConfiguration::BuildEventMetadataEvent(EventPipeEventInstance &sourceInstance, unsigned int metadataId)
 {
     CONTRACTL
     {
@@ -459,43 +459,30 @@ EventPipeEventInstance* EventPipeConfiguration::BuildEventMetadataEvent(EventPip
     CONTRACTL_END;
 
     // The payload of the event should contain:
+    // - Metadata ID
     // - GUID ProviderID.
-    // - unsigned int EventID.
-    // - unsigned int EventVersion.
     // - Optional event description payload.
 
     // Calculate the size of the event.
     EventPipeEvent &sourceEvent = *sourceInstance.GetEvent();
     const SString &providerName = sourceEvent.GetProvider()->GetProviderName();
-    unsigned int eventID = sourceEvent.GetEventID();
-    unsigned int eventVersion = sourceEvent.GetEventVersion();
     BYTE *pPayloadData = sourceEvent.GetMetadata();
     unsigned int payloadLength = sourceEvent.GetMetadataLength();
     unsigned int providerNameLength = (providerName.GetCount() + 1) * sizeof(WCHAR);
-    unsigned int instancePayloadSize = providerNameLength + sizeof(eventID) + sizeof(eventVersion) + sizeof(payloadLength) + payloadLength;
+    unsigned int instancePayloadSize = sizeof(metadataId) + providerNameLength + payloadLength;
 
     // Allocate the payload.
     BYTE *pInstancePayload = new BYTE[instancePayloadSize];
 
     // Fill the buffer with the payload.
     BYTE *currentPtr = pInstancePayload;
 
-    // Write the provider ID.
+    memcpy(currentPtr, &metadataId, sizeof(metadataId));
+    currentPtr += sizeof(metadataId);
+
     memcpy(currentPtr, (BYTE*)providerName.GetUnicode(), providerNameLength);
     currentPtr += providerNameLength;
 
-    // Write the event name as null-terminated unicode.
-    memcpy(currentPtr, &eventID, sizeof(eventID));
-    currentPtr += sizeof(eventID);
-
-    // Write the event version.
-    memcpy(currentPtr, &eventVersion, sizeof(eventVersion));
-    currentPtr += sizeof(eventVersion);
-
-    // Write the size of the metadata.
-    memcpy(currentPtr, &payloadLength, sizeof(payloadLength));
-    currentPtr += sizeof(payloadLength);
-
     // Write the incoming payload data.
     memcpy(currentPtr, pPayloadData, payloadLength);
 
@@ -511,7 +498,7 @@ EventPipeEventInstance* EventPipeConfiguration::BuildEventMetadataEvent(EventPip
 
     // Set the timestamp to match the source event, because the metadata event
     // will be emitted right before the source event.
-    pInstance->SetTimeStamp(sourceInstance.GetTimeStamp());
+    pInstance->SetTimeStamp(*sourceInstance.GetTimeStamp());
 
     return pInstance;
 }

src/vm/eventpipeconfiguration.h

@@ -77,7 +77,7 @@ class EventPipeConfiguration
     void EnableRundown(EventPipeSession *pSession);
 
     // Get the event used to write metadata to the event stream.
-    EventPipeEventInstance* BuildEventMetadataEvent(EventPipeEventInstance &sourceInstance);
+    EventPipeEventInstance* BuildEventMetadataEvent(EventPipeEventInstance &sourceInstance, unsigned int metdataId);
 
     // Delete deferred providers.
     void DeleteDeferredProviders();