[UE5.6] Feature: Dump latency stats to CSV | Re-ship: 4.27 browser send offer support (#647)

.backportrc.json

@@ -1,7 +1,7 @@
 {
     "repoOwner": "EpicGamesExt",
     "repoName": "PixelStreamingInfrastructure",
-    "targetBranchChoices": ["master", "UE5.2", "UE5.3", "UE5.4", "UE5.5", "UE5.6", "LatencyTest"],
+    "targetBranchChoices": ["master", "UE5.2", "UE5.3", "UE5.4", "UE5.5", "UE5.6"],
     "autoMerge": true,
     "autoMergeMethod": "squash"
 }

.changeset/weak-files-wait.md

@@ -0,0 +1,31 @@
+---
+'@epicgames-ps/lib-pixelstreamingfrontend-ui-ue5.6': minor
+'@epicgames-ps/lib-pixelstreamingfrontend-ue5.6': minor
+---
+
+## Latency Session Test and dump to csv
+
+Added a new feature to run a variable length latency test session (e.g. a 60s window)
+and dump that stats from the session to two .csv files:
+
+1. latency.csv - Which contains the video timing stats
+2. stats.csv - Which contains all WebRTC stats the library currently tracks
+
+To enable the latency session test use the flag/url parameter ?LatencyCSV
+to enable this feature (by default it is disabled and not UI-configurable).
+
+To use this latency session test feature:
+
+1. Navigate to http://localhost/?LatencyCSV
+2. Open the stats panel and click the "Run Test" button under the "Session Test" heading.
+
+## 4.27 support restored
+
+Re-shipped UE 4.27 support by restoring the ?BrowserSendOffer flag.
+It was found useful to support running this latency session test against UE 4.27
+for internal historical testing so support for connecting to this version has been restored.
+
+To connect to a 4.27 project:
+
+1. Navigate to http://localhost/?BrowserSendOffer
+2. Connect (warning: this option is not compatible with all newer UE versions)

Frontend/library/src/Config/Config.ts

@@ -36,6 +36,8 @@ export class Flags {
     static WaitForStreamer = 'WaitForStreamer' as const;
     static HideUI = 'HideUI' as const;
     static EnableCaptureTimeExt = 'EnableCaptureTimeExt' as const;
+    static BrowserSendOffer = 'BrowserSendOffer' as const;
+    static LatencyCSV = 'LatencyCSV' as const;
 }
 
 export type FlagsKeys = Exclude<keyof typeof Flags, 'prototype'>;
@@ -580,6 +582,32 @@ export class Config {
             )
         );
 
+        this.flags.set(
+            Flags.BrowserSendOffer,
+            new SettingFlag(
+                Flags.BrowserSendOffer,
+                'Browser send offer (4.27 ONLY)',
+                'Browser will initiate the WebRTC handshake by sending the offer to the streamer (4.27 ONLY)',
+                settings && Object.prototype.hasOwnProperty.call(settings, Flags.BrowserSendOffer)
+                    ? settings[Flags.BrowserSendOffer]
+                    : false,
+                useUrlParams
+            )
+        );
+
+        this.flags.set(
+            Flags.LatencyCSV,
+            new SettingFlag(
+                Flags.LatencyCSV,
+                'Export Latency CSV',
+                'Shows a button in the stats panel that allows to run a latency test and export the results to a CSV file.',
+                settings && Object.prototype.hasOwnProperty.call(settings, Flags.LatencyCSV)
+                    ? settings[Flags.LatencyCSV]
+                    : false,
+                useUrlParams
+            )
+        );
+
         /**
          * Numeric parameters
          */

Frontend/library/src/PeerConnectionController/AggregatedStats.ts

@@ -34,7 +34,10 @@ export class AggregatedStats {
     constructor() {
         this.inboundVideoStats = new InboundVideoStats();
         this.inboundAudioStats = new InboundAudioStats();
+        this.candidatePairs = new Array<CandidatePairStats>();
         this.datachannelStats = new DataChannelStats();
+        this.localCandidates = new Array<CandidateStat>();
+        this.remoteCandidates = new Array<CandidateStat>();
         this.outboundVideoStats = new OutboundRTPStats();
         this.outboundAudioStats = new OutboundRTPStats();
         this.remoteOutboundAudioStats = new RemoteOutboundRTPStats();

Frontend/library/src/WebRtcPlayer/WebRtcPlayerController.ts

@@ -206,8 +206,11 @@ export class WebRtcPlayerController {
             this.handleIceCandidate(iceCandidateMessage.candidate);
         });
         this.protocol.transport.addListener('open', () => {
-            const message = MessageHelpers.createMessage(Messages.listStreamers);
-            this.protocol.sendMessage(message);
+            const BrowserSendOffer = this.config.isFlagEnabled(Flags.BrowserSendOffer);
+            if (!BrowserSendOffer) {
+                const message = MessageHelpers.createMessage(Messages.listStreamers);
+                this.protocol.sendMessage(message);
+            }
             this.reconnectAttempt = 0;
             this.isReconnecting = false;
         });
@@ -1151,6 +1154,19 @@ export class WebRtcPlayerController {
         /* RTC Peer Connection on Track event -> handle on track */
         this.peerConnectionController.onTrack = (trackEvent: RTCTrackEvent) =>
             this.streamController.handleOnTrack(trackEvent);
+
+        const BrowserSendOffer = this.config.isFlagEnabled(Flags.BrowserSendOffer);
+        if (BrowserSendOffer) {
+            // If browser is sending the offer, create an offer and send it to the streamer
+            this.sendrecvDataChannelController.createDataChannel(
+                this.peerConnectionController.peerConnection,
+                'cirrus',
+                this.datachannelOptions
+            );
+            this.sendrecvDataChannelController.handleOnMessage = (ev: MessageEvent<ArrayBuffer>) =>
+                this.handleOnMessage(ev);
+            this.peerConnectionController.createOffer(this.sdpConstraints, this.config);
+        }
     }
 
     /**

Frontend/ui-library/src/Application/Application.ts

@@ -130,7 +130,7 @@ export class Application {
 
         if (isPanelEnabled(options.statsPanelConfig)) {
             // Add stats panel
-            this.statsPanel = new StatsPanel(options.statsPanelConfig);
+            this.statsPanel = new StatsPanel(options.statsPanelConfig, this.stream.config);
             this.uiFeaturesElement.appendChild(this.statsPanel.rootElement);
         }
 

Frontend/ui-library/src/Config/ConfigUI.ts

@@ -141,6 +141,8 @@ export class ConfigUI {
                     psSettingsSection,
                     this.textParametersUi.get(TextParameters.SignallingServerUrl)
                 );
+            if (isSettingEnabled(settingsConfig, Flags.BrowserSendOffer))
+                this.addSettingFlag(psSettingsSection, this.flagsUi.get(Flags.BrowserSendOffer));
             if (isSettingEnabled(settingsConfig, OptionParameters.StreamerId))
                 this.addSettingOption(
                     psSettingsSection,

Frontend/ui-library/src/UI/SessionTest.ts

@@ -0,0 +1,227 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+import {
+    AggregatedStats,
+    LatencyInfo,
+    Logger,
+    SettingNumber
+} from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.6';
+import { SettingUINumber } from '../Config/SettingUINumber';
+
+/**
+ * Session test UI elements and results handling.
+ * Creates a button to start the test and collects stats and latency info during the test.
+ * After the test is finished, it generates CSV files for stats and latency info.
+ * The test runs for a specified time frame, which can be set in the UI.
+ */
+export class SessionTest {
+    _rootElement: HTMLElement;
+    _latencyTestButton: HTMLInputElement;
+    _testTimeFrameSetting: SettingNumber<'TestTimeFrame'>;
+
+    isCollectingStats: boolean;
+
+    records: AggregatedStats[];
+    latencyRecords: LatencyInfo[];
+
+    constructor() {
+        this.isCollectingStats = false;
+    }
+
+    /**
+     * Make the elements for the session test: e.g. button and test time input.
+     */
+    public get rootElement(): HTMLElement {
+        if (!this._rootElement) {
+            this._rootElement = document.createElement('section');
+            this._rootElement.classList.add('settingsContainer');
+
+            // make heading
+            const heading = document.createElement('div');
+            heading.id = 'latencyTestHeader';
+            heading.classList.add('settings-text');
+            heading.classList.add('settingsHeader');
+            this._rootElement.appendChild(heading);
+
+            const headingText = document.createElement('div');
+            headingText.innerHTML = 'Session Test';
+            heading.appendChild(headingText);
+
+            // make test results element
+            const resultsParentElem = document.createElement('div');
+            resultsParentElem.id = 'latencyTestContainer';
+            resultsParentElem.classList.add('d-none');
+            this._rootElement.appendChild(resultsParentElem);
+
+            this._testTimeFrameSetting = new SettingNumber(
+                'TestTimeFrame',
+                'Test Time Frame',
+                'How long the test runs for (seconds)',
+                0 /*min*/,
+                3600 /*max*/,
+                60 /*default*/,
+                false
+            );
+            const testTimeFrameSetting = new SettingUINumber(this._testTimeFrameSetting);
+            resultsParentElem.appendChild(testTimeFrameSetting.rootElement);
+            resultsParentElem.appendChild(this.latencyTestButton);
+        }
+        return this._rootElement;
+    }
+
+    public get latencyTestButton(): HTMLInputElement {
+        if (!this._latencyTestButton) {
+            this._latencyTestButton = document.createElement('input');
+            this._latencyTestButton.type = 'button';
+            this._latencyTestButton.value = 'Run Test';
+            this._latencyTestButton.id = 'btn-start-latency-test';
+            this._latencyTestButton.classList.add('streamTools-button');
+            this._latencyTestButton.classList.add('btn-flat');
+
+            this._latencyTestButton.onclick = () => {
+                this.records = [];
+                this.latencyRecords = [];
+                this.isCollectingStats = true;
+                this._latencyTestButton.disabled = true;
+                this._latencyTestButton.value = 'Running...';
+                Logger.Info(`Starting session test. Duration: [${this._testTimeFrameSetting.number}]`);
+                setTimeout(() => {
+                    this.onCollectingFinished();
+                    this._latencyTestButton.disabled = false;
+                    this._latencyTestButton.value = 'Run Test';
+                }, this._testTimeFrameSetting.number * 1000);
+            };
+        }
+        return this._latencyTestButton;
+    }
+
+    public handleStats(stats: AggregatedStats) {
+        if (!this.isCollectingStats) {
+            return;
+        }
+
+        const statsCopy = structuredClone(stats);
+        this.records.push(statsCopy);
+    }
+
+    public handleLatencyInfo(latencyInfo: LatencyInfo) {
+        if (!this.isCollectingStats) {
+            return;
+        }
+
+        const latencyInfoCopy = structuredClone(latencyInfo);
+        this.latencyRecords.push(latencyInfoCopy);
+    }
+
+    private onCollectingFinished() {
+        this.isCollectingStats = false;
+        Logger.Info(`Finished session test`);
+
+        this.generateStatsCsv();
+        this.generateLatencyCsv();
+    }
+
+    private generateStatsCsv() {
+        const csvHeader: string[] = [];
+
+        this.records.forEach((record: AggregatedStats) => {
+            for (const i in record) {
+                const obj: {} = record[i as never];
+
+                if (Array.isArray(obj)) {
+                    for (let j = 0; j < obj.length; j++) {
+                        const arrayVal = obj[j];
+                        for (const k in arrayVal) {
+                            if (csvHeader.indexOf(`${i}.${j}.${k}`) === -1) {
+                                csvHeader.push(`${i}.${j}.${k}`);
+                            }
+                        }
+                    }
+                } else if (obj instanceof Map) {
+                    for (const j in obj.keys()) {
+                        const mapVal = obj.get(j);
+                        for (const k in mapVal) {
+                            if (csvHeader.indexOf(`${i}.${j}.${k}`) === -1) {
+                                csvHeader.push(`${i}.${j}.${k}`);
+                            }
+                        }
+                    }
+                } else {
+                    for (const j in obj) {
+                        if (csvHeader.indexOf(`${i}.${j}`) === -1) {
+                            csvHeader.push(`${i}.${j}`);
+                        }
+                    }
+                }
+            }
+        });
+
+        let csvBody = '';
+        this.records.forEach((record) => {
+            csvHeader.forEach((field) => {
+                try {
+                    csvBody += `"${field.split('.').reduce((o, k) => o[k as never], record)}",`;
+                } catch (_) {
+                    csvBody += `"",`;
+                }
+            });
+            csvBody += `\n`;
+        });
+
+        const file = new Blob([`${csvHeader.join(',')}\n${csvBody}`], { type: 'text/plain' });
+        const a = document.createElement('a');
+        const url = URL.createObjectURL(file);
+        a.href = url;
+        a.download = 'stats.csv';
+        document.body.appendChild(a);
+        a.click();
+        setTimeout(function () {
+            document.body.removeChild(a);
+            window.URL.revokeObjectURL(url);
+        }, 0);
+    }
+
+    private generateLatencyCsv() {
+        const csvHeader: string[] = [];
+
+        this.latencyRecords.forEach((record) => {
+            for (const i in record) {
+                const obj = record[i as never];
+
+                if (typeof obj === 'object') {
+                    for (const j in obj as object) {
+                        if (csvHeader.indexOf(`${i}.${j}`) === -1) {
+                            csvHeader.push(`${i}.${j}`);
+                        }
+                    }
+                } else if (csvHeader.indexOf(`${i}`) === -1) {
+                    csvHeader.push(`${i}`);
+                }
+            }
+        });
+
+        let csvBody = '';
+        this.latencyRecords.forEach((record) => {
+            csvHeader.forEach((field) => {
+                try {
+                    csvBody += `"${field.split('.').reduce((o, k) => o[k as never], record)}",`;
+                } catch (_) {
+                    csvBody += `"",`;
+                }
+            });
+            csvBody += `\n`;
+        });
+
+        const file = new Blob([`${csvHeader.join(',')}\n${csvBody}`], { type: 'text/plain' });
+        const a = document.createElement('a');
+        const url = URL.createObjectURL(file);
+        a.href = url;
+        a.download = 'latency.csv';
+        document.body.appendChild(a);
+        a.click();
+        setTimeout(function () {
+            document.body.removeChild(a);
+            window.URL.revokeObjectURL(url);
+        }, 0);
+    }
+}