diff --git a/examples/local_video/src/publisher.rs b/examples/local_video/src/publisher.rs index bc43745d4..dc56f5969 100644 --- a/examples/local_video/src/publisher.rs +++ b/examples/local_video/src/publisher.rs @@ -185,7 +185,7 @@ async fn run(args: Args, ctrl_c_received: Arc) -> Result<()> { let pace_fps = args.fps as f64; // Create LiveKit video source and track - let rtc_source = NativeVideoSource::new(VideoResolution { width, height }); + let rtc_source = NativeVideoSource::new(VideoResolution { width, height }, false); let track = LocalVideoTrack::create_video_track("camera", RtcVideoSource::Native(rtc_source.clone())); diff --git a/examples/screensharing/src/lib.rs b/examples/screensharing/src/lib.rs index 044d260e0..7902199c3 100644 --- a/examples/screensharing/src/lib.rs +++ b/examples/screensharing/src/lib.rs @@ -101,7 +101,7 @@ mod test { let resolution = wait_for_resolution(&resolution_signal); log::info!("Detected capture resolution: {}x{}", resolution.width, resolution.height); - let buffer_source = NativeVideoSource::new(resolution.clone()); + let buffer_source = NativeVideoSource::new(resolution.clone(), true); { let mut slot = video_source_slot.lock().unwrap(); *slot = Some(buffer_source.clone()); diff --git a/examples/wgpu_room/src/logo_track.rs b/examples/wgpu_room/src/logo_track.rs index 03b1a0ec4..a7d46dee0 100644 --- a/examples/wgpu_room/src/logo_track.rs +++ b/examples/wgpu_room/src/logo_track.rs @@ -47,10 +47,10 @@ pub struct LogoTrack { impl LogoTrack { pub fn new(room: Arc) -> Self { Self { - rtc_source: NativeVideoSource::new(VideoResolution { - width: FB_WIDTH as u32, - height: FB_HEIGHT as u32, - }), + rtc_source: NativeVideoSource::new( + VideoResolution { width: FB_WIDTH as u32, height: FB_HEIGHT as u32 }, + false, + ), room, handle: None, } diff --git a/libwebrtc/src/native/video_source.rs b/libwebrtc/src/native/video_source.rs index 91a59df99..a6e78e014 100644 --- a/libwebrtc/src/native/video_source.rs +++ b/libwebrtc/src/native/video_source.rs @@ -50,11 +50,12 @@ struct VideoSourceInner { } impl NativeVideoSource { - pub fn new(resolution: VideoResolution) -> NativeVideoSource { + pub fn new(resolution: VideoResolution, is_screencast: bool) -> NativeVideoSource { let source = Self { - sys_handle: vt_sys::ffi::new_video_track_source(&vt_sys::ffi::VideoResolution::from( - resolution.clone(), - )), + sys_handle: vt_sys::ffi::new_video_track_source( + &vt_sys::ffi::VideoResolution::from(resolution.clone()), + is_screencast, + ), inner: Arc::new(Mutex::new(VideoSourceInner { captured_frames: 0 })), }; diff --git a/libwebrtc/src/video_source.rs b/libwebrtc/src/video_source.rs index d73491748..f8dc9964a 100644 --- a/libwebrtc/src/video_source.rs +++ b/libwebrtc/src/video_source.rs @@ -65,13 +65,13 @@ pub mod native { impl Default for NativeVideoSource { fn default() -> Self { - Self::new(VideoResolution::default()) + Self::new(VideoResolution::default(), false) } } impl NativeVideoSource { - pub fn new(resolution: VideoResolution) -> Self { - Self { handle: vs_imp::NativeVideoSource::new(resolution) } + pub fn new(resolution: VideoResolution, is_screencast: bool) -> Self { + Self { handle: vs_imp::NativeVideoSource::new(resolution, is_screencast) } } pub fn capture_frame>(&self, frame: &VideoFrame) { diff --git a/livekit-ffi-node-bindings/src/proto/video_frame_pb.ts b/livekit-ffi-node-bindings/src/proto/video_frame_pb.ts index 87f433b39..1a051773c 100644 --- a/livekit-ffi-node-bindings/src/proto/video_frame_pb.ts +++ b/livekit-ffi-node-bindings/src/proto/video_frame_pb.ts @@ -422,12 +422,17 @@ export class NewVideoSourceRequest extends Message { /** * Used to determine which encodings to use + simulcast layers - * Most of the time it corresponds to the source resolution + * Most of the time it corresponds to the source resolution * * @generated from field: required livekit.proto.VideoSourceResolution resolution = 2; */ resolution?: VideoSourceResolution; + /** + * @generated from field: optional bool is_screencast = 3; + */ + isScreencast?: boolean; + constructor(data?: PartialMessage) { super(); proto2.util.initPartial(data, this); @@ -438,6 +443,7 @@ export class NewVideoSourceRequest extends Message { static readonly fields: FieldList = proto2.util.newFieldList(() => [ { no: 1, name: "type", kind: "enum", T: proto2.getEnumType(VideoSourceType), req: true }, { no: 2, name: "resolution", kind: "message", T: VideoSourceResolution, req: true }, + { no: 3, name: "is_screencast", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): NewVideoSourceRequest { diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto index d85c5ca98..71a7cea6f 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto @@ -44,11 +44,12 @@ message VideoStreamFromParticipantResponse { required OwnedVideoStream stream = // Create a new VideoSource // VideoSource is used to send video frame to a track -message NewVideoSourceRequest { - required VideoSourceType type = 1; +message NewVideoSourceRequest { + required VideoSourceType type = 1; // Used to determine which encodings to use + simulcast layers - // Most of the time it corresponds to the source resolution + // Most of the time it corresponds to the source resolution required VideoSourceResolution resolution = 2; + optional bool is_screencast = 3; } message NewVideoSourceResponse { required OwnedVideoSource source = 1; } diff --git a/livekit-ffi/src/server/video_source.rs b/livekit-ffi/src/server/video_source.rs index 5af7d9a38..7f33b67c3 100644 --- a/livekit-ffi/src/server/video_source.rs +++ b/livekit-ffi/src/server/video_source.rs @@ -36,7 +36,9 @@ impl FfiVideoSource { proto::VideoSourceType::VideoSourceNative => { use livekit::webrtc::video_source::native::NativeVideoSource; - let video_source = NativeVideoSource::new(new_source.resolution.into()); + let is_screencast = new_source.is_screencast.unwrap_or(false); + let video_source = + NativeVideoSource::new(new_source.resolution.into(), is_screencast); RtcVideoSource::Native(video_source) } _ => return Err(FfiError::InvalidRequest("unsupported video source type".into())), diff --git a/webrtc-sys/include/livekit/video_track.h b/webrtc-sys/include/livekit/video_track.h index 222458c75..d27eb4daa 100644 --- a/webrtc-sys/include/livekit/video_track.h +++ b/webrtc-sys/include/livekit/video_track.h @@ -87,10 +87,10 @@ std::shared_ptr new_native_video_sink( class VideoTrackSource { class InternalSource : public webrtc::AdaptedVideoTrackSource { public: - InternalSource(const VideoResolution& - resolution); // (0, 0) means no resolution/optional, the - // source will guess the resolution at the - // first captured frame + InternalSource(const VideoResolution& resolution, + bool is_screencast); // (0, 0) means no resolution/optional, the + // source will guess the resolution at the + // first captured frame ~InternalSource() override; bool is_screencast() const override; @@ -104,10 +104,11 @@ class VideoTrackSource { mutable webrtc::Mutex mutex_; webrtc::TimestampAligner timestamp_aligner_; VideoResolution resolution_; + bool is_screencast_; }; public: - VideoTrackSource(const VideoResolution& resolution); + VideoTrackSource(const VideoResolution& resolution, bool is_screencast); VideoResolution video_resolution() const; @@ -121,7 +122,7 @@ class VideoTrackSource { }; std::shared_ptr new_video_track_source( - const VideoResolution& resolution); + const VideoResolution& resolution, bool is_screencast); static std::shared_ptr video_to_media( std::shared_ptr track) { diff --git a/webrtc-sys/src/video_track.cpp b/webrtc-sys/src/video_track.cpp index b9a9661d9..111575786 100644 --- a/webrtc-sys/src/video_track.cpp +++ b/webrtc-sys/src/video_track.cpp @@ -105,13 +105,13 @@ std::shared_ptr new_native_video_sink( } VideoTrackSource::InternalSource::InternalSource( - const VideoResolution& resolution) - : webrtc::AdaptedVideoTrackSource(4), resolution_(resolution) {} + const VideoResolution& resolution, bool is_screencast) + : webrtc::AdaptedVideoTrackSource(4), resolution_(resolution), is_screencast_(is_screencast) {} VideoTrackSource::InternalSource::~InternalSource() {} bool VideoTrackSource::InternalSource::is_screencast() const { - return false; + return is_screencast_; } std::optional VideoTrackSource::InternalSource::needs_denoising() const { @@ -175,8 +175,8 @@ bool VideoTrackSource::InternalSource::on_captured_frame( return true; } -VideoTrackSource::VideoTrackSource(const VideoResolution& resolution) { - source_ = webrtc::make_ref_counted(resolution); +VideoTrackSource::VideoTrackSource(const VideoResolution& resolution, bool is_screencast) { + source_ = webrtc::make_ref_counted(resolution, is_screencast); } VideoResolution VideoTrackSource::video_resolution() const { @@ -195,8 +195,8 @@ webrtc::scoped_refptr VideoTrackSource::get() } std::shared_ptr new_video_track_source( - const VideoResolution& resolution) { - return std::make_shared(resolution); + const VideoResolution& resolution, bool is_screencast) { + return std::make_shared(resolution, is_screencast); } } // namespace livekit_ffi diff --git a/webrtc-sys/src/video_track.rs b/webrtc-sys/src/video_track.rs index 2e2681d8b..d18712281 100644 --- a/webrtc-sys/src/video_track.rs +++ b/webrtc-sys/src/video_track.rs @@ -67,7 +67,10 @@ pub mod ffi { fn video_resolution(self: &VideoTrackSource) -> VideoResolution; fn on_captured_frame(self: &VideoTrackSource, frame: &UniquePtr) -> bool; - fn new_video_track_source(resolution: &VideoResolution) -> SharedPtr; + fn new_video_track_source( + resolution: &VideoResolution, + is_screencast: bool, + ) -> SharedPtr; fn video_to_media(track: SharedPtr) -> SharedPtr; unsafe fn media_to_video(track: SharedPtr) -> SharedPtr; fn _shared_video_track() -> SharedPtr;