Reduce WebVR idle times (#2970)

* Reduce WebVR idle times

* Fix discarded frames during the transition to one frame ahead prediction

app/src/main/cpp/BrowserWorld.cpp

@@ -144,7 +144,6 @@ PerformanceObserver::PerformanceRestored(const double& aTargetFrameRate, const d
 } // namespace
 
 namespace crow {
-
 struct BrowserWorld::State {
   BrowserWorldWeakPtr self;
   std::vector<WidgetPtr> widgets;
@@ -934,7 +933,7 @@ BrowserWorld::EndFrame() {
     m.frameEndHandler();
     m.frameEndHandler = nullptr;
   } else {
-    m.device->EndFrame(false);
+    m.device->EndFrame();
   }
   m.drawHandler = nullptr;
 
@@ -1422,14 +1421,33 @@ BrowserWorld::TickImmersive() {
   m.externalVR->SetCompositorEnabled(false);
   m.device->SetRenderMode(device::RenderMode::Immersive);
 
-  m.device->StartFrame();
+  const bool supportsFrameAhead = m.device->SupportsFramePrediction(DeviceDelegate::FramePrediction::ONE_FRAME_AHEAD);
   VRB_GL_CHECK(glDepthMask(GL_FALSE));
-  m.externalVR->PushFramePoses(m.device->GetHeadTransform(), m.controllers->GetControllers(), m.context->GetTimestamp());
+  if (!supportsFrameAhead || (m.externalVR->GetVRState() != ExternalVR::VRState::Rendering)) {
+      // Do not use one frame ahead prediction if not supported or we are rendering the spinner.
+      m.device->StartFrame(DeviceDelegate::FramePrediction::NO_FRAME_AHEAD);
+      m.externalVR->PushFramePoses(m.device->GetHeadTransform(), m.controllers->GetControllers(),
+                                   m.context->GetTimestamp());
+  }
   int32_t surfaceHandle, textureWidth, textureHeight = 0;
   device::EyeRect leftEye, rightEye;
   bool aDiscardFrame = !m.externalVR->WaitFrameResult();
   m.externalVR->GetFrameResult(surfaceHandle, textureWidth, textureHeight, leftEye, rightEye);
   ExternalVR::VRState state = m.externalVR->GetVRState();
+  if (supportsFrameAhead) {
+      if (m.externalVR->WasFirstPresentingFrame()) {
+          // StartFrame() has been already called to render the spinner, do not call it again.
+          // Instead, repeat the XR frame and render the spinner while we transition
+          // to one frame ahead prediction.
+          state = ExternalVR::VRState::Loading;
+      } else {
+          // Predict poses for one frame ahead and push the data to shmem so Gecko
+          // can start the next XR RAF ASAP.
+          m.device->StartFrame(DeviceDelegate::FramePrediction::ONE_FRAME_AHEAD);
+      }
+      m.externalVR->PushFramePoses(m.device->GetHeadTransform(), m.controllers->GetControllers(),
+              m.context->GetTimestamp());
+  }
   if (state == ExternalVR::VRState::Rendering) {
     if (!aDiscardFrame) {
       if (textureWidth > 0 && textureHeight > 0) {
@@ -1441,7 +1459,7 @@ BrowserWorld::TickImmersive() {
       };
     }
     m.frameEndHandler = [=]() {
-      m.device->EndFrame(aDiscardFrame);
+      m.device->EndFrame(aDiscardFrame ? DeviceDelegate::FrameEndMode::DISCARD : DeviceDelegate::FrameEndMode::APPLY);
       m.blitter->EndFrame();
     };
   } else {
@@ -1494,7 +1512,7 @@ BrowserWorld::TickSplashAnimation() {
       if (m.fadeAnimation) {
         m.fadeAnimation->FadeIn();
       }
-      m.device->EndFrame(false);
+      m.device->EndFrame();
     };
   }
 }

app/src/main/cpp/DeviceDelegate.h

@@ -45,6 +45,14 @@ class ImmersiveDisplay {
 
 class DeviceDelegate {
 public:
+  enum class FramePrediction {
+      NO_FRAME_AHEAD,
+      ONE_FRAME_AHEAD,
+  };
+  enum class FrameEndMode {
+      APPLY,
+      DISCARD
+  };
   virtual device::DeviceType GetDeviceType() { return device::UnknownType; }
   virtual void SetRenderMode(const device::RenderMode aMode) = 0;
   virtual device::RenderMode GetRenderMode() = 0;
@@ -63,9 +71,12 @@ class DeviceDelegate {
   virtual const std::string GetControllerModelName(const int32_t aModelIndex) const = 0;
   virtual void SetCPULevel(const device::CPULevel aLevel) {};
   virtual void ProcessEvents() = 0;
-  virtual void StartFrame() = 0;
+  virtual bool SupportsFramePrediction(FramePrediction aPrediction) const {
+    return aPrediction == FramePrediction::NO_FRAME_AHEAD;
+  }
+  virtual void StartFrame(const FramePrediction aPrediction = FramePrediction::NO_FRAME_AHEAD) = 0;
   virtual void BindEye(const device::Eye aWhich) = 0;
-  virtual void EndFrame(bool aDiscard = false) = 0;
+  virtual void EndFrame(const FrameEndMode aMode = FrameEndMode::APPLY) = 0;
   virtual bool IsInGazeMode() const { return false; };
   virtual int32_t GazeModeIndex() const { return -1; };
   virtual VRLayerQuadPtr CreateLayerQuad(int32_t aWidth, int32_t aHeight,

app/src/main/cpp/ExternalVR.cpp

@@ -127,6 +127,7 @@ struct ExternalVR::State {
   vrb::Vector eyeOffsets[device::EyeCount];
   uint64_t lastFrameId = 0;
   bool firstPresentingFrame = false;
+  bool wasFirstPresentingFrame = false;
   bool compositorEnabled = false;
   bool waitingForExit = false;
 
@@ -455,6 +456,7 @@ ExternalVR::WaitFrameResult() {
   m.PullBrowserStateWhileLocked();
   while (true) {
     if (!IsPresenting() || m.browser.layerState[0].layer_stereo_immersive.frameId != m.lastFrameId) {
+      m.wasFirstPresentingFrame = m.firstPresentingFrame;
       m.firstPresentingFrame = false;
       m.system.displayState.lastSubmittedFrameSuccessful = true;
       m.system.displayState.lastSubmittedFrameId = m.browser.layerState[0].layer_stereo_immersive.frameId;
@@ -480,6 +482,10 @@ ExternalVR::WaitFrameResult() {
   return true;
 }
 
+bool ExternalVR::WasFirstPresentingFrame() const {
+  return m.wasFirstPresentingFrame;
+}
+
 void
 ExternalVR::CompleteEnumeration()
 {

app/src/main/cpp/ExternalVR.h

@@ -55,6 +55,7 @@ class ExternalVR : public ImmersiveDisplay {
   VRState GetVRState() const;
   void PushFramePoses(const vrb::Matrix& aHeadTransform, const std::vector<Controller>& aControllers, const double aTimestamp);
   bool WaitFrameResult();
+  bool WasFirstPresentingFrame() const;
   void GetFrameResult(int32_t& aSurfaceHandle,
                       int32_t& aTextureWidth,
                       int32_t& aTextureHeight,

app/src/noapi/cpp/DeviceDelegateNoAPI.cpp

@@ -211,7 +211,7 @@ DeviceDelegateNoAPI::ProcessEvents() {
 }
 
 void
-DeviceDelegateNoAPI::StartFrame() {
+DeviceDelegateNoAPI::StartFrame(const FramePrediction aPrediction) {
   VRB_GL_CHECK(glClearColor(m.clearColor.Red(), m.clearColor.Green(), m.clearColor.Blue(), m.clearColor.Alpha()));
   VRB_GL_CHECK(glEnable(GL_DEPTH_TEST));
   VRB_GL_CHECK(glEnable(GL_CULL_FACE));
@@ -240,7 +240,7 @@ DeviceDelegateNoAPI::BindEye(const device::Eye aEye) {
 }
 
 void
-DeviceDelegateNoAPI::EndFrame(const bool aDiscard) {
+DeviceDelegateNoAPI::EndFrame(const FrameEndMode aMode) {
   // noop
 }
 

app/src/noapi/cpp/DeviceDelegateNoAPI.h

@@ -37,9 +37,9 @@ class DeviceDelegateNoAPI : public DeviceDelegate {
   int32_t GetControllerModelCount() const override;
   const std::string GetControllerModelName(const int32_t aModelIndex) const override;
   void ProcessEvents() override;
-  void StartFrame() override;
+  void StartFrame(const FramePrediction aPrediction) override;
   void BindEye(const device::Eye) override;
-  void EndFrame(const bool aDiscard) override;
+  void EndFrame(const FrameEndMode aMode) override;
   // DeviceDelegateNoAPI interface
   void InitializeJava(JNIEnv* aEnv, jobject aActivity);
   void ShutdownJava();

app/src/oculusvr/cpp/DeviceDelegateOculusVR.cpp

@@ -88,7 +88,10 @@ struct DeviceDelegateOculusVR::State {
   vrb::FBOPtr previousFBO;
   vrb::CameraEyePtr cameras[2];
   uint32_t frameIndex = 0;
+  FramePrediction framePrediction = FramePrediction::NO_FRAME_AHEAD;
+  double prevPredictedDisplayTime = 0;
   double predictedDisplayTime = 0;
+  ovrTracking2 prevPredictedTracking = {};
   ovrTracking2 predictedTracking = {};
   ovrTracking2 discardPredictedTracking = {};
   uint32_t discardedFrameIndex = 0;
@@ -822,15 +825,28 @@ DeviceDelegateOculusVR::ProcessEvents() {
   }
 }
 
+bool
+DeviceDelegateOculusVR::SupportsFramePrediction(FramePrediction aPrediction) const {
+  return true;
+}
+
 void
-DeviceDelegateOculusVR::StartFrame() {
+DeviceDelegateOculusVR::StartFrame(const FramePrediction aPrediction) {
   if (!m.ovr) {
     VRB_LOG("StartFrame called while not in VR mode");
     return;
   }
 
+  m.framePrediction = aPrediction;
   m.frameIndex++;
-  m.predictedDisplayTime = vrapi_GetPredictedDisplayTime(m.ovr, m.frameIndex);
+  if (aPrediction == FramePrediction::ONE_FRAME_AHEAD) {
+    m.prevPredictedDisplayTime = m.predictedDisplayTime;
+    m.prevPredictedTracking = m.predictedTracking;
+    m.predictedDisplayTime = vrapi_GetPredictedDisplayTime(m.ovr, m.frameIndex + 1);
+  } else {
+    m.predictedDisplayTime = vrapi_GetPredictedDisplayTime(m.ovr, m.frameIndex);
+  }
+
   m.predictedTracking = vrapi_GetPredictedTracking2(m.ovr, m.predictedDisplayTime);
 
   float ipd = vrapi_GetInterpupillaryDistance(&m.predictedTracking);
@@ -912,7 +928,7 @@ DeviceDelegateOculusVR::BindEye(const device::Eye aWhich) {
 }
 
 void
-DeviceDelegateOculusVR::EndFrame(const bool aDiscard) {
+DeviceDelegateOculusVR::EndFrame(const FrameEndMode aEndMode) {
   if (!m.ovr) {
     VRB_LOG("EndFrame called while not in VR mode");
     return;
@@ -922,11 +938,15 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) {
     m.currentFBO.reset();
   }
 
-  if (aDiscard) {
+  const bool frameAhead = m.framePrediction == FramePrediction::ONE_FRAME_AHEAD;
+  const ovrTracking2& tracking = frameAhead ? m.prevPredictedTracking : m.predictedTracking;
+  const double displayTime = frameAhead ? m.prevPredictedDisplayTime : m.predictedDisplayTime;
+
+  if (aEndMode == FrameEndMode::DISCARD) {
     // Reuse the last frame when a frame is discarded.
     // The last frame is timewarped by the VR compositor.
     if (m.discardCount == 0) {
-      m.discardPredictedTracking = m.predictedTracking;
+      m.discardPredictedTracking = tracking;
       m.discardedFrameIndex = m.frameIndex;
     }
     m.discardCount++;
@@ -940,13 +960,13 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) {
   const ovrLayerHeader2* layers[ovrMaxLayerCount] = {};
 
   if (m.cubeLayer && m.cubeLayer->IsLoaded() && m.cubeLayer->IsDrawRequested()) {
-    m.cubeLayer->Update(m.predictedTracking, m.clearColorSwapChain);
+    m.cubeLayer->Update(tracking, m.clearColorSwapChain);
     layers[layerCount++] = m.cubeLayer->Header();
     m.cubeLayer->ClearRequestDraw();
   }
 
   if (m.equirectLayer && m.equirectLayer->IsDrawRequested()) {
-    m.equirectLayer->Update(m.predictedTracking, m.clearColorSwapChain);
+    m.equirectLayer->Update(tracking, m.clearColorSwapChain);
     layers[layerCount++] = m.equirectLayer->Header();
     m.equirectLayer->ClearRequestDraw();
   }
@@ -959,7 +979,7 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) {
   // Draw back layers
   for (const OculusLayerPtr& layer: m.uiLayers) {
     if (!layer->GetDrawInFront() && layer->IsDrawRequested() && (layerCount < ovrMaxLayerCount - 1)) {
-      layer->Update(m.predictedTracking, m.clearColorSwapChain);
+      layer->Update(tracking, m.clearColorSwapChain);
       layers[layerCount++] = layer->Header();
       layer->ClearRequestDraw();
     }
@@ -971,7 +991,7 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) {
   const ovrMatrix4f projectionMatrix = ovrMatrix4f_CreateProjectionFov(fovX, fovY, 0.0f, 0.0f, VRAPI_ZNEAR, 0.0f);
 
   ovrLayerProjection2 projection = vrapi_DefaultLayerProjection2();
-  projection.HeadPose = m.predictedTracking.HeadPose;
+  projection.HeadPose = tracking.HeadPose;
   projection.Header.SrcBlend = VRAPI_FRAME_LAYER_BLEND_ONE;
   projection.Header.DstBlend = VRAPI_FRAME_LAYER_BLEND_ONE_MINUS_SRC_ALPHA;
   for (int i = 0; i < VRAPI_FRAME_LAYER_EYE_MAX; ++i) {
@@ -987,7 +1007,7 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) {
   // Draw front layers
   for (const OculusLayerPtr& layer: m.uiLayers) {
     if (layer->GetDrawInFront() && layer->IsDrawRequested() && layerCount < ovrMaxLayerCount) {
-      layer->Update(m.predictedTracking, m.clearColorSwapChain);
+      layer->Update(tracking, m.clearColorSwapChain);
       layers[layerCount++] = layer->Header();
       layer->ClearRequestDraw();
     }
@@ -1002,7 +1022,7 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) {
   }
   frameDesc.SwapInterval = 1;
   frameDesc.FrameIndex = m.frameIndex;
-  frameDesc.DisplayTime = m.predictedDisplayTime;
+  frameDesc.DisplayTime = displayTime;
 
   frameDesc.LayerCount = layerCount;
   frameDesc.Layers = layers;

app/src/oculusvr/cpp/DeviceDelegateOculusVR.h

@@ -41,9 +41,10 @@ class DeviceDelegateOculusVR : public DeviceDelegate {
   const std::string GetControllerModelName(const int32_t aModelIndex) const override;
   void SetCPULevel(const device::CPULevel aLevel) override;
   void ProcessEvents() override;
-  void StartFrame() override;
+  bool SupportsFramePrediction(FramePrediction aPrediction) const override;
+  void StartFrame(const FramePrediction aPrediction) override;
   void BindEye(const device::Eye aWhich) override;
-  void EndFrame(const bool aDiscard) override;
+  void EndFrame(const FrameEndMode aMode) override;
   VRLayerQuadPtr CreateLayerQuad(int32_t aWidth, int32_t aHeight,
                                  VRLayerSurface::SurfaceType aSurfaceType) override;
   VRLayerQuadPtr CreateLayerQuad(const VRLayerSurfacePtr& aMoveLayer) override;

app/src/picovr/cpp/DeviceDelegatePicoVR.cpp

@@ -385,7 +385,7 @@ DeviceDelegatePicoVR::ProcessEvents() {
 }
 
 void
-DeviceDelegatePicoVR::StartFrame() {
+DeviceDelegatePicoVR::StartFrame(const FramePrediction aPrediction) {
   vrb::Matrix head = vrb::Matrix::Rotation(m.orientation);
   head.TranslateInPlace(m.position);
 
@@ -431,7 +431,7 @@ DeviceDelegatePicoVR::BindEye(const device::Eye aWhich) {
 }
 
 void
-DeviceDelegatePicoVR::EndFrame(const bool aDiscard) {
+DeviceDelegatePicoVR::EndFrame(const FrameEndMode aMode) {
 
 }
 

app/src/picovr/cpp/DeviceDelegatePicoVR.h

@@ -37,9 +37,9 @@ class DeviceDelegatePicoVR : public DeviceDelegate {
   int32_t GetControllerModelCount() const override;
   const std::string GetControllerModelName(const int32_t aModelIndex) const override;
   void ProcessEvents() override;
-  void StartFrame() override;
+  void StartFrame(const FramePrediction aPrediction) override;
   void BindEye(const device::Eye aWhich) override;
-  void EndFrame(const bool aDiscard) override;
+  void EndFrame(const FrameEndMode aMode) override;
   bool IsInGazeMode() const override;
   int32_t GazeModeIndex() const override;
   // Custom methods

app/src/wavevr/cpp/DeviceDelegateWaveVR.cpp

@@ -16,6 +16,7 @@
 #include "vrb/Matrix.h"
 #include "vrb/RenderContext.h"
 #include "vrb/Vector.h"
+#include "../../main/cpp/DeviceDelegate.h"
 
 #include <array>
 #include <vector>
@@ -693,7 +694,7 @@ HandToString(ElbowModel::HandEnum hand) {
 }
 
 void
-DeviceDelegateWaveVR::StartFrame() {
+DeviceDelegateWaveVR::StartFrame(const FramePrediction aPrediction) {
   VRB_GL_CHECK(glClearColor(m.clearColor.Red(), m.clearColor.Green(), m.clearColor.Blue(), m.clearColor.Alpha()));
   if (!m.lastSubmitDiscarded) {
     m.leftFBOIndex = WVR_GetAvailableTextureIndex(m.leftTextureQueue);
@@ -801,14 +802,14 @@ DeviceDelegateWaveVR::BindEye(const device::Eye aWhich) {
 }
 
 void
-DeviceDelegateWaveVR::EndFrame(const bool aDiscard) {
+DeviceDelegateWaveVR::EndFrame(const FrameEndMode aMode) {
   if (m.currentFBO) {
     m.currentFBO->Unbind();
     m.currentFBO = nullptr;
   }
 
-  m.lastSubmitDiscarded = aDiscard;
-  if (aDiscard) {
+  m.lastSubmitDiscarded = aMode == DeviceDelegate::FrameEndMode::DISCARD;
+  if (m.lastSubmitDiscarded) {
     return;
   }
   // Left eye

app/src/wavevr/cpp/DeviceDelegateWaveVR.h

@@ -31,9 +31,9 @@ class DeviceDelegateWaveVR : public DeviceDelegate {
   int32_t GetControllerModelCount() const override;
   const std::string GetControllerModelName(const int32_t aModelIndex) const override;
   void ProcessEvents() override;
-  void StartFrame() override;
+  void StartFrame(const FramePrediction aPrediction) override;
   void BindEye(const device::Eye aWhich) override;
-  void EndFrame(const bool aDiscard) override;
+  void EndFrame(const FrameEndMode aMode) override;
   // DeviceDelegateWaveVR interface
   bool IsRunning();
 protected: