From 17e0f1a81699813bc4c341cdb41066ca81dedc29 Mon Sep 17 00:00:00 2001 From: Mariotaku Date: Sat, 1 Jun 2024 11:50:34 +0900 Subject: [PATCH] feat(audio): custom surround-params (#2424) --- src/audio.cpp | 50 ++++++++++++++++-------- src/audio.h | 10 +++++ src/nvhttp.cpp | 1 + src/process.cpp | 1 + src/rtsp.cpp | 29 ++++++++++++++ src/rtsp.h | 1 + tests/conftest.cpp | 6 ++- tests/unit/test_audio.cpp | 81 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 tests/unit/test_audio.cpp diff --git a/src/audio.cpp b/src/audio.cpp index f55046c078c..b608466d5c1 100644 --- a/src/audio.cpp +++ b/src/audio.cpp @@ -34,6 +34,8 @@ namespace audio { start_audio_control(audio_ctx_t &ctx); static void stop_audio_control(audio_ctx_t &); + static void + apply_surround_params(opus_stream_config_t &stream, const stream_params_t ¶ms); int map_stream(int channels, bool quality); @@ -98,28 +100,31 @@ namespace audio { void encodeThread(sample_queue_t samples, config_t config, void *channel_data) { auto packets = mail::man->queue(mail::audio_packets); - auto stream = &stream_configs[map_stream(config.channels, config.flags[config_t::HIGH_QUALITY])]; + auto stream = stream_configs[map_stream(config.channels, config.flags[config_t::HIGH_QUALITY])]; + if (config.flags[config_t::CUSTOM_SURROUND_PARAMS]) { + apply_surround_params(stream, config.customStreamParams); + } // Encoding takes place on this thread platf::adjust_thread_priority(platf::thread_priority_e::high); opus_t opus { opus_multistream_encoder_create( - stream->sampleRate, - stream->channelCount, - stream->streams, - stream->coupledStreams, - stream->mapping, + stream.sampleRate, + stream.channelCount, + stream.streams, + stream.coupledStreams, + stream.mapping, OPUS_APPLICATION_RESTRICTED_LOWDELAY, nullptr) }; - opus_multistream_encoder_ctl(opus.get(), OPUS_SET_BITRATE(stream->bitrate)); + opus_multistream_encoder_ctl(opus.get(), OPUS_SET_BITRATE(stream.bitrate)); opus_multistream_encoder_ctl(opus.get(), OPUS_SET_VBR(0)); - BOOST_LOG(info) << "Opus initialized: "sv << stream->sampleRate / 1000 << " kHz, "sv - << stream->channelCount << " channels, "sv - << stream->bitrate / 1000 << " kbps (total), LOWDELAY"sv; + BOOST_LOG(info) << "Opus initialized: "sv << stream.sampleRate / 1000 << " kHz, "sv + << stream.channelCount << " channels, "sv + << stream.bitrate / 1000 << " kbps (total), LOWDELAY"sv; - auto frame_size = config.packetDuration * stream->sampleRate / 1000; + auto frame_size = config.packetDuration * stream.sampleRate / 1000; while (auto sample = samples->pop()) { buffer_t packet { 1400 }; @@ -139,7 +144,10 @@ namespace audio { void capture(safe::mail_t mail, config_t config, void *channel_data) { auto shutdown_event = mail->event(mail::shutdown); - auto stream = &stream_configs[map_stream(config.channels, config.flags[config_t::HIGH_QUALITY])]; + auto stream = stream_configs[map_stream(config.channels, config.flags[config_t::HIGH_QUALITY])]; + if (config.flags[config_t::CUSTOM_SURROUND_PARAMS]) { + apply_surround_params(stream, config.customStreamParams); + } auto ref = control_shared.ref(); if (!ref) { @@ -171,7 +179,7 @@ namespace audio { // Prefer the virtual sink if host playback is disabled or there's no other sink if (ref->sink.null && (!config.flags[config_t::HOST_AUDIO] || sink->empty())) { auto &null = *ref->sink.null; - switch (stream->channelCount) { + switch (stream.channelCount) { case 2: sink = &null.stereo; break; @@ -195,8 +203,8 @@ namespace audio { } } - auto frame_size = config.packetDuration * stream->sampleRate / 1000; - auto mic = control->microphone(stream->mapping, stream->channelCount, stream->sampleRate, frame_size); + auto frame_size = config.packetDuration * stream.sampleRate / 1000; + auto mic = control->microphone(stream.mapping, stream.channelCount, stream.sampleRate, frame_size); if (!mic) { return; } @@ -217,7 +225,7 @@ namespace audio { shutdown_event->view(); }); - int samples_per_frame = frame_size * stream->channelCount; + int samples_per_frame = frame_size * stream.channelCount; while (!shutdown_event->peek()) { std::vector sample_buffer; @@ -233,7 +241,7 @@ namespace audio { BOOST_LOG(info) << "Reinitializing audio capture"sv; mic.reset(); do { - mic = control->microphone(stream->mapping, stream->channelCount, stream->sampleRate, frame_size); + mic = control->microphone(stream.mapping, stream.channelCount, stream.sampleRate, frame_size); if (!mic) { BOOST_LOG(warning) << "Couldn't re-initialize audio input"sv; } @@ -303,4 +311,12 @@ namespace audio { ctx.control->set_sink(sink); } } + + void + apply_surround_params(opus_stream_config_t &stream, const stream_params_t ¶ms) { + stream.channelCount = params.channelCount; + stream.streams = params.streams; + stream.coupledStreams = params.coupledStreams; + stream.mapping = params.mapping; + } } // namespace audio diff --git a/src/audio.h b/src/audio.h index fe22c94611d..cbe4ae98b1c 100644 --- a/src/audio.h +++ b/src/audio.h @@ -26,12 +26,20 @@ namespace audio { int bitrate; }; + struct stream_params_t { + int channelCount; + int streams; + int coupledStreams; + std::uint8_t mapping[8]; + }; + extern opus_stream_config_t stream_configs[MAX_STREAM_CONFIG]; struct config_t { enum flags_e : int { HIGH_QUALITY, HOST_AUDIO, + CUSTOM_SURROUND_PARAMS, MAX_FLAGS }; @@ -39,6 +47,8 @@ namespace audio { int channels; int mask; + stream_params_t customStreamParams; + std::bitset flags; }; diff --git a/src/nvhttp.cpp b/src/nvhttp.cpp index f03be3c76b9..bd8434e5534 100644 --- a/src/nvhttp.cpp +++ b/src/nvhttp.cpp @@ -330,6 +330,7 @@ namespace nvhttp { launch_session->appid = util::from_view(get_arg(args, "appid", "unknown")); launch_session->enable_sops = util::from_view(get_arg(args, "sops", "0")); launch_session->surround_info = util::from_view(get_arg(args, "surroundAudioInfo", "196610")); + launch_session->surround_params = (get_arg(args, "surroundParams", "")); launch_session->gcmap = util::from_view(get_arg(args, "gcmap", "0")); launch_session->enable_hdr = util::from_view(get_arg(args, "hdrMode", "0")); diff --git a/src/process.cpp b/src/process.cpp index 89dc4dc5ae5..32af14ebd9d 100644 --- a/src/process.cpp +++ b/src/process.cpp @@ -185,6 +185,7 @@ namespace proc { _env["SUNSHINE_CLIENT_AUDIO_CONFIGURATION"] = "7.1"; break; } + _env["SUNSHINE_CLIENT_AUDIO_SURROUND_PARAMS"] = launch_session->surround_params; if (!_app.output.empty() && _app.output != "null"sv) { #ifdef _WIN32 diff --git a/src/rtsp.cpp b/src/rtsp.cpp index 99b9f0de8f8..d690c0c7c3f 100644 --- a/src/rtsp.cpp +++ b/src/rtsp.cpp @@ -819,6 +819,12 @@ namespace rtsp_stream { ss << "a=rtpmap:98 AV1/90000"sv << std::endl; } + if (!session.surround_params.empty()) { + // If we have our own surround parameters, advertise them twice first + ss << "a=fmtp:97 surround-params="sv << session.surround_params << std::endl; + ss << "a=fmtp:97 surround-params="sv << session.surround_params << std::endl; + } + for (int x = 0; x < audio::MAX_STREAM_CONFIG; ++x) { auto &stream_config = audio::stream_configs[x]; std::uint8_t mapping[platf::speaker::MAX_SPEAKERS]; @@ -1033,6 +1039,29 @@ namespace rtsp_stream { } } } + else if (session.surround_params.length() > 3) { + // Channels + std::uint8_t c = session.surround_params[0] - '0'; + // Streams + std::uint8_t n = session.surround_params[1] - '0'; + // Coupled streams + std::uint8_t m = session.surround_params[2] - '0'; + auto valid = false; + if ((c == 6 || c == 8) && c == config.audio.channels && n + m == c && session.surround_params.length() == c + 3) { + config.audio.customStreamParams.channelCount = c; + config.audio.customStreamParams.streams = n; + config.audio.customStreamParams.coupledStreams = m; + valid = true; + for (std::uint8_t i = 0; i < c; i++) { + config.audio.customStreamParams.mapping[i] = session.surround_params[i + 3] - '0'; + if (config.audio.customStreamParams.mapping[i] >= c) { + valid = false; + break; + } + } + } + config.audio.flags[audio::config_t::CUSTOM_SURROUND_PARAMS] = valid; + } // If the client sent a configured bitrate, we will choose the actual bitrate ourselves // by using FEC percentage and audio quality settings. If the calculated bitrate ends up diff --git a/src/rtsp.h b/src/rtsp.h index 20bb8453998..16dba1e0592 100644 --- a/src/rtsp.h +++ b/src/rtsp.h @@ -29,6 +29,7 @@ namespace rtsp_stream { int gcmap; int appid; int surround_info; + std::string surround_params; bool enable_hdr; bool enable_sops; diff --git a/tests/conftest.cpp b/tests/conftest.cpp index 4fcafb95547..38721b67ea8 100644 --- a/tests/conftest.cpp +++ b/tests/conftest.cpp @@ -176,7 +176,7 @@ class PlatformInitBase: public virtual BaseTest { std::cout << "PlatformInitTest:: starting Fixture SetUp" << std::endl; // initialize the platform - auto deinit_guard = platf::init(); + deinit_guard = platf::init(); if (!deinit_guard) { FAIL() << "Platform failed to initialize"; } @@ -187,8 +187,12 @@ class PlatformInitBase: public virtual BaseTest { void TearDown() override { std::cout << "PlatformInitTest:: starting Fixture TearDown" << std::endl; + deinit_guard.reset(nullptr); std::cout << "PlatformInitTest:: finished Fixture TearDown" << std::endl; } + +private: + std::unique_ptr deinit_guard; }; class DocsPythonVenvBase: public virtual BaseTest { diff --git a/tests/unit/test_audio.cpp b/tests/unit/test_audio.cpp new file mode 100644 index 00000000000..7f555048fa8 --- /dev/null +++ b/tests/unit/test_audio.cpp @@ -0,0 +1,81 @@ +/** + * @file tests/test_audio.cpp + * @brief Test src/audio.*. + */ +#include +#include + +#include + +using namespace audio; + +class AudioTest: public virtual BaseTest, public PlatformInitBase, public ::testing::WithParamInterface, config_t>> { +protected: + void + SetUp() override { + BaseTest::SetUp(); + PlatformInitBase::SetUp(); + + std::string_view p_name = std::get<0>(GetParam()); + std::cout << "AudioTest(" << p_name << "):: starting Fixture SetUp" << std::endl; + + m_config = std::get<1>(GetParam()); + m_mail = std::make_shared(); + } + + void + TearDown() override { + PlatformInitBase::TearDown(); + BaseTest::TearDown(); + } + +protected: + config_t m_config; + safe::mail_t m_mail; +}; + +static std::bitset +config_flags(int flag = -1) { + std::bitset<3> result = std::bitset(); + if (flag >= 0) { + result.set(flag); + } + return result; +} + +INSTANTIATE_TEST_SUITE_P( + Configurations, + AudioTest, + ::testing::Values( + std::make_tuple("HIGH_STEREO", config_t { 5, 2, 0x3, { 0 }, config_flags(config_t::HIGH_QUALITY) }), + std::make_tuple("SURROUND51", config_t { 5, 6, 0x3F, { 0 }, config_flags() }), + std::make_tuple("SURROUND71", config_t { 5, 8, 0x63F, { 0 }, config_flags() }), + std::make_tuple("SURROUND51_CUSTOM", config_t { 5, 6, 0x3F, { 6, 4, 2, { 0, 1, 4, 5, 2, 3 } }, config_flags(config_t::CUSTOM_SURROUND_PARAMS) }))); + +TEST_P(AudioTest, TestEncode) { + std::thread timer([&] { + // Terminate the audio capture after 5 seconds. + std::this_thread::sleep_for(5s); + auto shutdown_event = m_mail->event(mail::shutdown); + auto audio_packets = m_mail->queue(mail::audio_packets); + shutdown_event->raise(true); + audio_packets->stop(); + }); + std::thread capture([&] { + auto packets = m_mail->queue(mail::audio_packets); + auto shutdown_event = m_mail->event(mail::shutdown); + while (auto packet = packets->pop()) { + if (shutdown_event->peek()) { + break; + } + auto packet_data = packet->second; + if (packet_data.size() == 0) { + FAIL() << "Empty packet data"; + } + } + }); + audio::capture(m_mail, m_config, nullptr); + + timer.join(); + capture.join(); +}