feat: add --preset, for setting quality/bandwidth usage dynamically

In particular, this makes it a first-class concept, and allows clients to set
the quality level.

mm-client/src/bin/mmclient.rs

@@ -106,8 +106,11 @@ struct Cli {
     /// Framerate to render at on the server side.
     #[arg(long, default_value = "30")]
     framerate: u32,
-    /// Open in fullscreen mode.
+    #[arg(short, long, default_value = "6")]
+    /// The quality preset to use, from 0-9.
+    preset: u32,
     #[arg(long)]
+    /// Open in fullscreen mode.
     fullscreen: bool,
     /// Enable the overlay, which shows various stats.
     #[arg(long)]
@@ -122,6 +125,7 @@ struct App {
     configured_resolution: Resolution,
     configured_codec: protocol::VideoCodec,
     configured_framerate: u32,
+    configured_preset: u32,
 
     window: Arc<winit::window::Window>,
     _proxy: EventLoopProxy<AppEvent>,
@@ -663,6 +667,7 @@ impl App {
                                 session_id: self.session_id,
                                 streaming_resolution: self.remote_display_params.resolution.clone(),
                                 video_codec: self.configured_codec.into(),
+                                quality_preset: self.configured_preset,
                                 ..Default::default()
                             },
                             None,
@@ -1002,6 +1007,7 @@ fn main() -> Result<()> {
             session_id: session.session_id,
             streaming_resolution: Some(streaming_resolution),
             video_codec: configured_codec.into(),
+            quality_preset: args.preset + 1,
             ..Default::default()
         },
         None,
@@ -1029,6 +1035,7 @@ fn main() -> Result<()> {
         configured_codec,
         configured_framerate: args.framerate,
         configured_resolution: args.resolution,
+        configured_preset: args.preset + 1,
 
         window,
         _proxy: proxy.clone(),

mm-protocol/Cargo.toml

@@ -4,7 +4,7 @@
 
 [package]
 name = "mm-protocol"
-version = "0.2.0"
+version = "0.2.1"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

mm-protocol/src/messages.proto

@@ -286,6 +286,16 @@ message Error {
 // scale UI elements or make other user-experience improvements subject to UI
 // scale.
 //
+// ### Quality preset
+//
+// Clients can use the `quality_preset` field of the `030 - Attach` message to
+// tune the quality of the stream, which is inversely related to the bandwidth
+// usage. The value ranges from 1 to 10, with 1 indicating that the client
+// wishes the server to optimize for the the lowest possible bandwidth usage,
+// and 10 indicating that the client wishes the server to optimize for the
+// highest possible quality. How these values are interpreted is determined by
+// the server.
+//
 // ### Concurrent attachments
 //
 // Servers may support multiple concurrent attachments from different clients,
@@ -448,6 +458,7 @@ message Attach {
 
   VideoCodec video_codec = 10;
   Size streaming_resolution = 11;
+  uint32 quality_preset = 13; // Must be in the range 1-10.
 
   AudioCodec audio_codec = 15;
   AudioChannels channels = 16;
@@ -468,6 +479,7 @@ message Attached {
 
   VideoCodec video_codec = 10;    // Required.
   Size streaming_resolution = 11; // Required.
+  uint32 quality_preset = 13;     // Required.
 
   AudioCodec audio_codec = 15; // Required.
   AudioChannels channels = 16; // Required.

mm-server/src/compositor/control.rs

@@ -22,6 +22,7 @@ pub struct VideoStreamParams {
     pub width: u32,
     pub height: u32,
     pub codec: VideoCodec,
+    pub preset: u32,
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]

mm-server/src/compositor/video/cpu_encode/ffmpeg.rs

@@ -106,7 +106,7 @@ pub fn new_encoder(
     encoder.set_time_base(timebase);
     encoder.set_frame_rate(Some((framerate as i32, 1)));
     encoder.set_gop(120);
-    encoder.set_quality(25);
+    encoder.set_quality(40 - (2 * params.preset) as usize);
 
     // This just tags the output - it doesn't actually perform any
     // conversion.

mm-server/src/compositor/video/cpu_encode/svt.rs

@@ -86,7 +86,7 @@ pub fn new_hevc(
         .enable_fps_in_vps(true)
         .pred_structure(svt::hevc::PredictionStructure::LowDelayP)
         .rate_control_mode(svt::hevc::RateControlMode::ConstantQp)
-        .qp(17)
+        .qp(40 - (2 * params.preset))
         .intra_refresh_type(svt::hevc::IntraRefreshType::Closed(300))
         .thread_count(1)
         .create_encoder(params.width, params.height, svt::SubsamplingFormat::Yuv420)?;
@@ -109,7 +109,9 @@ pub fn new_av1(
         .enable_screen_content_mode(true)
         .enable_fast_decode(true)
         .pred_structure(svt::av1::PredictionStructure::LowDelay)
-        .rate_control_mode(svt::av1::RateControlMode::ConstantRateFactor(17))
+        .rate_control_mode(svt::av1::RateControlMode::ConstantRateFactor(
+            40 - (2 * params.preset),
+        ))
         .intra_refresh_type(svt::av1::IntraRefreshType::Closed)
         .create_encoder(params.width, params.height, svt::SubsamplingFormat::Yuv420)?;
 

mm-server/src/compositor/video/vulkan_encode/h264.rs

@@ -120,8 +120,12 @@ impl H264Encoder {
         //     quality_props.h264_props
         // );
 
-        let rc_mode =
-            super::rate_control::select_rc_mode(params.width, params.height, &caps.encode_caps);
+        let rc_mode = super::rate_control::select_rc_mode(
+            params.width,
+            params.height,
+            params.preset,
+            &caps.encode_caps,
+        );
         debug!(?rc_mode, "selected rate control mode");
 
         let structure = super::default_structure(

mm-server/src/compositor/video/vulkan_encode/h265.rs

@@ -121,8 +121,12 @@ impl H265Encoder {
         //     quality_props.h265_props
         // );
 
-        let rc_mode =
-            super::rate_control::select_rc_mode(params.width, params.height, &caps.encode_caps);
+        let rc_mode = super::rate_control::select_rc_mode(
+            params.width,
+            params.height,
+            params.preset,
+            &caps.encode_caps,
+        );
         debug!(?rc_mode, "selected rate control mode");
 
         // let mut layers = std::cmp::min(4, caps.h265_caps.max_sub_layer_count);

mm-server/src/compositor/video/vulkan_encode/rate_control.rs

@@ -3,11 +3,14 @@
 // SPDX-License-Identifier: BUSL-1.1
 
 use ash::vk;
+use tracing::warn;
 
 // Bitrate is defined here in terms of 1080p, and scaled nonlinearly to the
-// target resolution.
-const BASELINE_AVG_BITRATE: f32 = 5_000_000.0;
-const BASELINE_PEAK_BITRATE: f32 = 12_000_000.0;
+// target resolution. Values are indexed by quality preset. Values 7/8/9 are
+// only used if CRF is unsupported by the driver.
+const BASELINE_AVG_BITRATE_MBPS: [f32; 10] = [3.0, 3.0, 3.0, 4.0, 6.0, 8.0, 10.0, 12.0, 25.0, 50.0];
+const BASELINE_PEAK_BITRATE_MBPS: [f32; 10] =
+    [3.0, 4.0, 6.0, 8.0, 12.0, 16.0, 20.0, 24.0, 50.0, 100.0];
 const BASELINE_DIMS: f32 = 1920.0 * 1080.0;
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
@@ -39,26 +42,49 @@ pub struct VbrSettings {
 pub fn select_rc_mode(
     width: u32,
     height: u32,
+    preset: u32,
     caps: &vk::VideoEncodeCapabilitiesKHR,
 ) -> RateControlMode {
-    if caps
+    assert!(preset <= 9);
+
+    let target_qp = 40 - (2 * preset); // 22 - 40;
+
+    let supports_crf = caps
         .rate_control_modes
-        .contains(vk::VideoEncodeRateControlModeFlagsKHR::VBR)
-    {
+        .contains(vk::VideoEncodeRateControlModeFlagsKHR::DISABLED);
+    let supports_vbr = caps
+        .rate_control_modes
+        .contains(vk::VideoEncodeRateControlModeFlagsKHR::VBR);
+
+    if preset >= 7 && supports_crf {
+        // Presets 7/8/9 use a very low constant QP.
+        RateControlMode::ConstantQp(target_qp)
+    } else if supports_vbr {
+        // 6 and lower use VBR, starting with a high peak and reducing as the
+        // presets get lower.
         let scale = ((width * height) as f32 / BASELINE_DIMS).sqrt();
+
+        const MBPS: f32 = 1_000_000.0;
+        let average_bitrate =
+            (BASELINE_AVG_BITRATE_MBPS[preset as usize] * MBPS * scale).round() as u64;
+        let peak_bitrate =
+            (BASELINE_PEAK_BITRATE_MBPS[preset as usize] * MBPS * scale).round() as u64;
+
         RateControlMode::Vbr(VbrSettings {
             vbv_size_ms: 5000,
-            average_bitrate: (BASELINE_AVG_BITRATE * scale).round() as u64,
-            peak_bitrate: (BASELINE_PEAK_BITRATE * scale).round() as u64,
-            min_qp: 25,
-            max_qp: 35,
+            average_bitrate,
+            peak_bitrate,
+            min_qp: 17,
+            max_qp: 32,
         })
     } else if caps
         .rate_control_modes
         .contains(vk::VideoEncodeRateControlModeFlagsKHR::DISABLED)
     {
-        RateControlMode::ConstantQp(27)
+        // Fall back to CRF with a high bitrate.
+        RateControlMode::ConstantQp(target_qp)
     } else {
+        warn!("no rate control modes available, using driver defaults!");
         RateControlMode::Defaults
     }
 }

mm-server/src/server/handlers.rs

@@ -303,6 +303,8 @@ fn attach(
         attachment_id = handle.attachment_id
     );
 
+    debug!(?video_params, ?audio_params, "attaching with params");
+
     let _guard = span.enter();
 
     let handle = scopeguard::guard(handle, |h| {
@@ -324,6 +326,8 @@ fn attach(
             height: video_params.height,
         }),
 
+        quality_preset: video_params.preset,
+
         audio_codec: audio_codec.into(),
         sample_rate_hz: audio_params.sample_rate,
         channels: Some(protocol::AudioChannels {

mm-server/src/server/handlers/validation.rs

@@ -45,6 +45,7 @@ pub fn validate_attachment(
 ) -> Result<(VideoStreamParams, AudioStreamParams)> {
     let (width, height) = validate_resolution(params.streaming_resolution)?;
     let video_codec = validate_video_codec(params.video_codec)?;
+    let preset = validate_preset(params.quality_preset)?;
 
     let sample_rate = validate_sample_rate(params.sample_rate_hz)?;
     let channels = validate_channels(params.channels)?;
@@ -55,6 +56,7 @@ pub fn validate_attachment(
             width,
             height,
             codec: video_codec,
+            preset,
         },
         AudioStreamParams {
             sample_rate,
@@ -104,6 +106,13 @@ pub fn validate_video_codec(codec: i32) -> Result<VideoCodec> {
     }
 }
 
+pub fn validate_preset(preset: u32) -> Result<u32> {
+    match preset {
+        0 => Ok(6), // Default to 6
+        v if v <= 10 => Ok(v - 1),
+        _ => Err(ValidationError::Invalid("invalid preset".into())),
+    }
+}
 pub fn validate_framerate(framerate: u32) -> Result<u32> {
     match framerate {
         60 | 30 => Ok(framerate),