Don't pause videos when element is in pictureInPicture mode (#622)

* Don't pause videos in adaptiveStream when in pictureInPicture mode

* update tests

* address comments

* update state first

* Create .changeset/eight-years-boil.md

* remove debug logs

.changeset/eight-years-boil.md

@@ -0,0 +1,5 @@
+---
+"livekit-client": patch
+---
+
+Don't auto-pause videos when element is in pictureInPicture mode (only applies when adaptiveStream is enabled)

src/room/track/RemoteVideoTrack.test.ts

@@ -130,6 +130,8 @@ class MockElementInfo implements ElementInfo {
 
   visible = false;
 
+  pictureInPicture = false;
+
   setVisible = (visible: boolean) => {
     if (this.visible !== visible) {
       this.visible = visible;

src/room/track/RemoteVideoTrack.ts

@@ -3,7 +3,12 @@ import log from '../../logger';
 import { TrackEvent } from '../events';
 import { computeBitrate, VideoReceiverStats } from '../stats';
 import CriticalTimers from '../timers';
-import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
+import {
+  getIntersectionObserver,
+  getResizeObserver,
+  isWeb,
+  ObservableMediaElement,
+} from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { attachToElement, detachTrack, Track } from './Track';
 import type { AdaptiveStreamSettings } from './types';
@@ -223,7 +228,9 @@ export default class RemoteVideoTrack extends RemoteTrack {
       this.adaptiveStreamSettings?.pauseVideoInBackground ?? true // default to true
         ? this.isInBackground
         : false;
-    const isVisible = this.elementInfos.some((info) => info.visible) && !backgroundPause;
+    const isPiPMode = this.elementInfos.some((info) => info.pictureInPicture);
+    const isVisible =
+      (this.elementInfos.some((info) => info.visible) && !backgroundPause) || isPiPMode;
 
     if (this.lastVisible === isVisible) {
       return;
@@ -273,6 +280,7 @@ export interface ElementInfo {
   width(): number;
   height(): number;
   visible: boolean;
+  pictureInPicture: boolean;
   visibilityChangedAt: number | undefined;
 
   handleResize?: () => void;
@@ -284,17 +292,28 @@ export interface ElementInfo {
 class HTMLElementInfo implements ElementInfo {
   element: HTMLMediaElement;
 
-  visible: boolean;
+  get visible(): boolean {
+    return this.isPiP || this.isIntersecting;
+  }
+
+  get pictureInPicture(): boolean {
+    return this.isPiP;
+  }
 
   visibilityChangedAt: number | undefined;
 
   handleResize?: () => void;
 
   handleVisibilityChanged?: () => void;
 
+  private isPiP: boolean;
+
+  private isIntersecting: boolean;
+
   constructor(element: HTMLMediaElement, visible?: boolean) {
     this.element = element;
-    this.visible = visible ?? isElementInViewport(element);
+    this.isIntersecting = visible ?? isElementInViewport(element);
+    this.isPiP = isWeb() && document.pictureInPictureElement === element;
     this.visibilityChangedAt = 0;
   }
 
@@ -307,27 +326,51 @@ class HTMLElementInfo implements ElementInfo {
   }
 
   observe() {
+    // make sure we update the current visible state once we start to observe
+    this.isIntersecting = isElementInViewport(this.element);
+    this.isPiP = document.pictureInPictureElement === this.element;
+
     (this.element as ObservableMediaElement).handleResize = () => {
       this.handleResize?.();
     };
     (this.element as ObservableMediaElement).handleVisibilityChanged = this.onVisibilityChanged;
 
     getIntersectionObserver().observe(this.element);
     getResizeObserver().observe(this.element);
+    (this.element as HTMLVideoElement).addEventListener('enterpictureinpicture', this.onEnterPiP);
+    (this.element as HTMLVideoElement).addEventListener('leavepictureinpicture', this.onLeavePiP);
   }
 
   private onVisibilityChanged = (entry: IntersectionObserverEntry) => {
     const { target, isIntersecting } = entry;
     if (target === this.element) {
-      this.visible = isIntersecting;
+      this.isIntersecting = isIntersecting;
       this.visibilityChangedAt = Date.now();
       this.handleVisibilityChanged?.();
     }
   };
 
+  private onEnterPiP = () => {
+    this.isPiP = true;
+    this.handleVisibilityChanged?.();
+  };
+
+  private onLeavePiP = () => {
+    this.isPiP = false;
+    this.handleVisibilityChanged?.();
+  };
+
   stopObserving() {
     getIntersectionObserver()?.unobserve(this.element);
     getResizeObserver()?.unobserve(this.element);
+    (this.element as HTMLVideoElement).removeEventListener(
+      'enterpictureinpicture',
+      this.onEnterPiP,
+    );
+    (this.element as HTMLVideoElement).removeEventListener(
+      'leavepictureinpicture',
+      this.onLeavePiP,
+    );
   }
 }