diff --git a/osu.Framework.Tests/Audio/AudioManagerWithDeviceLoss.cs b/osu.Framework.Tests/Audio/AudioManagerWithDeviceLoss.cs index 311468c0b6..7d065f47bf 100644 --- a/osu.Framework.Tests/Audio/AudioManagerWithDeviceLoss.cs +++ b/osu.Framework.Tests/Audio/AudioManagerWithDeviceLoss.cs @@ -14,7 +14,7 @@ namespace osu.Framework.Tests.Audio /// that can simulate the loss of a device. /// This will NOT work without a physical audio device! /// - internal class AudioManagerWithDeviceLoss : AudioManager + internal class AudioManagerWithDeviceLoss : BassAudioManager { public AudioManagerWithDeviceLoss(AudioThread audioThread, ResourceStore trackStore, ResourceStore sampleStore) : base(audioThread, trackStore, sampleStore) diff --git a/osu.Framework.Tests/Audio/AudioTestComponents.cs b/osu.Framework.Tests/Audio/AudioTestComponents.cs new file mode 100644 index 0000000000..39796d2ed1 --- /dev/null +++ b/osu.Framework.Tests/Audio/AudioTestComponents.cs @@ -0,0 +1,91 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using osu.Framework.Audio.Mixing; +using osu.Framework.Audio.Sample; +using osu.Framework.Audio.Track; +using osu.Framework.Audio; +using System.IO; +using osu.Framework.IO.Stores; + +namespace osu.Framework.Tests.Audio +{ + public abstract class AudioTestComponents : IDisposable + { + public enum Type + { + BASS, + SDL3 + } + + internal readonly AudioMixer Mixer; + public readonly DllResourceStore Resources; + internal readonly TrackStore TrackStore; + internal readonly SampleStore SampleStore; + + protected readonly AudioCollectionManager AllComponents = new AudioCollectionManager(); + protected readonly AudioCollectionManager MixerComponents = new AudioCollectionManager(); + + protected AudioTestComponents(bool init) + { + Prepare(); + + if (init) + Init(); + + AllComponents.AddItem(MixerComponents); + + Mixer = CreateMixer(); + Resources = new DllResourceStore(typeof(TrackBassTest).Assembly); + TrackStore = new TrackStore(Resources, Mixer, CreateTrack); + SampleStore = new SampleStore(Resources, Mixer, CreateSampleFactory); + + Add(TrackStore, SampleStore); + } + + protected virtual void Prepare() + { + } + + internal abstract Track CreateTrack(Stream data, string name); + + internal abstract SampleFactory CreateSampleFactory(Stream stream, string name, AudioMixer mixer, int playbackConcurrency); + + public abstract void Init(); + + public virtual void Add(params AudioComponent[] component) + { + foreach (var c in component) + AllComponents.AddItem(c); + } + + public abstract AudioMixer CreateMixer(); + + public virtual void Update() + { + RunOnAudioThread(AllComponents.Update); + } + + /// + /// Runs an on a newly created audio thread, and blocks until it has been run to completion. + /// + /// The action to run on the audio thread. + public virtual void RunOnAudioThread(Action action) => AudioTestHelper.RunOnAudioThread(action); + + internal Track GetTrack() => TrackStore.Get("Resources.Tracks.sample-track.mp3"); + internal Sample GetSample() => SampleStore.Get("Resources.Tracks.sample-track.mp3"); + + public void Dispose() => RunOnAudioThread(() => + { + AllComponents.Dispose(); + AllComponents.Update(); // Actually runs the disposal. + + DisposeInternal(); + }); + + public virtual void DisposeInternal() + { + } + } +} diff --git a/osu.Framework.Tests/Audio/BassAudioMixerTest.cs b/osu.Framework.Tests/Audio/BassAudioMixerTest.cs index 23b5a10a75..775d4e5bdd 100644 --- a/osu.Framework.Tests/Audio/BassAudioMixerTest.cs +++ b/osu.Framework.Tests/Audio/BassAudioMixerTest.cs @@ -8,6 +8,7 @@ using ManagedBass; using ManagedBass.Mix; using NUnit.Framework; +using osu.Framework.Audio.Mixing; using osu.Framework.Audio.Mixing.Bass; using osu.Framework.Audio.Sample; using osu.Framework.Audio.Track; @@ -18,170 +19,233 @@ namespace osu.Framework.Tests.Audio [TestFixture] public class BassAudioMixerTest { - private BassTestComponents bass; - private TrackBass track; - private SampleBass sample; + private AudioTestComponents.Type type; + private AudioTestComponents audio; + private AudioMixer mixer; + private Track track; + private Sample sample; - [SetUp] - public void Setup() + [TearDown] + public void Teardown() + { + audio?.Dispose(); + } + + private void setupBackend(AudioTestComponents.Type id, bool loadTrack = false) { - bass = new BassTestComponents(); - track = bass.GetTrack(); - sample = bass.GetSample(); + type = id; + + if (id == AudioTestComponents.Type.BASS) + { + audio = new BassTestComponents(); + track = audio.GetTrack(); + sample = audio.GetSample(); + } + else if (id == AudioTestComponents.Type.SDL3) + { + audio = new SDL3AudioTestComponents(); + track = audio.GetTrack(); + sample = audio.GetSample(); - bass.Update(); + if (loadTrack) + ((SDL3AudioTestComponents)audio).WaitUntilTrackIsLoaded((TrackSDL3)track); + } + else + { + throw new InvalidOperationException("not a supported id"); + } + + audio.Update(); + mixer = audio.Mixer; } - [TearDown] - public void Teardown() + private void assertThatMixerContainsChannel(AudioMixer mixer, IAudioChannel channel) { - bass?.Dispose(); + if (type == AudioTestComponents.Type.BASS) + Assert.That(BassMix.ChannelGetMixer(((IBassAudioChannel)channel).Handle), Is.EqualTo(((BassAudioMixer)mixer).Handle)); + else + Assert.That(channel.Mixer == mixer, Is.True); } [Test] public void TestMixerInitialised() { - Assert.That(bass.Mixer.Handle, Is.Not.Zero); + setupBackend(AudioTestComponents.Type.BASS); + + Assert.That(((BassAudioMixer)mixer).Handle, Is.Not.Zero); } - [Test] - public void TestAddedToGlobalMixerByDefault() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestAddedToGlobalMixerByDefault(AudioTestComponents.Type id) { - Assert.That(BassMix.ChannelGetMixer(getHandle()), Is.EqualTo(bass.Mixer.Handle)); + setupBackend(id); + + assertThatMixerContainsChannel(mixer, track); } - [Test] - public void TestCannotBeRemovedFromGlobalMixer() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestCannotBeRemovedFromGlobalMixerBass(AudioTestComponents.Type id) { - bass.Mixer.Remove(track); - bass.Update(); + setupBackend(id); + + mixer.Remove(track); + audio.Update(); - Assert.That(BassMix.ChannelGetMixer(getHandle()), Is.EqualTo(bass.Mixer.Handle)); + assertThatMixerContainsChannel(mixer, track); } - [Test] - public void TestTrackIsMovedBetweenMixers() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestTrackIsMovedBetweenMixers(AudioTestComponents.Type id) { - var secondMixer = bass.CreateMixer(); + setupBackend(id); + + var secondMixer = audio.CreateMixer(); secondMixer.Add(track); - bass.Update(); + audio.Update(); - Assert.That(BassMix.ChannelGetMixer(getHandle()), Is.EqualTo(secondMixer.Handle)); + assertThatMixerContainsChannel(secondMixer, track); - bass.Mixer.Add(track); - bass.Update(); + mixer.Add(track); + audio.Update(); - Assert.That(BassMix.ChannelGetMixer(getHandle()), Is.EqualTo(bass.Mixer.Handle)); + assertThatMixerContainsChannel(mixer, track); } - [Test] - public void TestMovedToGlobalMixerWhenRemovedFromMixer() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestMovedToGlobalMixerWhenRemovedFromMixer(AudioTestComponents.Type id) { - var secondMixer = bass.CreateMixer(); + setupBackend(id); + + var secondMixer = audio.CreateMixer(); secondMixer.Add(track); secondMixer.Remove(track); - bass.Update(); + audio.Update(); - Assert.That(BassMix.ChannelGetMixer(getHandle()), Is.EqualTo(bass.Mixer.Handle)); + assertThatMixerContainsChannel(mixer, track); } - [Test] - public void TestVirtualTrackCanBeAddedAndRemoved() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestVirtualTrackCanBeAddedAndRemoved(AudioTestComponents.Type id) { - var secondMixer = bass.CreateMixer(); - var virtualTrack = bass.TrackStore.GetVirtual(); + setupBackend(id); + + var secondMixer = audio.CreateMixer(); + var virtualTrack = audio.TrackStore.GetVirtual(); secondMixer.Add(virtualTrack); - bass.Update(); + audio.Update(); secondMixer.Remove(virtualTrack); - bass.Update(); + audio.Update(); } - [Test] - public void TestFreedChannelRemovedFromDefault() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestFreedChannelRemovedFromDefault(AudioTestComponents.Type id) { + setupBackend(id); + track.Dispose(); - bass.Update(); + audio.Update(); - Assert.That(BassMix.ChannelGetMixer(getHandle()), Is.Zero); + if (id == AudioTestComponents.Type.BASS) + Assert.That(BassMix.ChannelGetMixer(((IBassAudioChannel)track).Handle), Is.Zero); + else + Assert.That(((IAudioChannel)track).Mixer, Is.Null); } - [Test] - public void TestChannelMovedToGlobalMixerAfterDispose() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestChannelMovedToGlobalMixerAfterDispose(AudioTestComponents.Type id) { - var secondMixer = bass.CreateMixer(); + setupBackend(id); + + var secondMixer = audio.CreateMixer(); secondMixer.Add(track); - bass.Update(); + audio.Update(); secondMixer.Dispose(); - bass.Update(); + audio.Update(); - Assert.That(BassMix.ChannelGetMixer(getHandle()), Is.EqualTo(bass.Mixer.Handle)); + assertThatMixerContainsChannel(mixer, track); } - [Test] - public void TestPlayPauseStop() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestPlayPauseStop(AudioTestComponents.Type id) { + setupBackend(id, true); + Assert.That(!track.IsRunning); - bass.RunOnAudioThread(() => track.Start()); - bass.Update(); + audio.RunOnAudioThread(() => track.Start()); + audio.Update(); Assert.That(track.IsRunning); - bass.RunOnAudioThread(() => track.Stop()); - bass.Update(); + audio.RunOnAudioThread(() => track.Stop()); + audio.Update(); Assert.That(!track.IsRunning); - bass.RunOnAudioThread(() => + audio.RunOnAudioThread(() => { track.Seek(track.Length - 1000); track.Start(); }); - bass.Update(); + audio.Update(); Assert.That(() => { - bass.Update(); + audio.Update(); return !track.IsRunning; }, Is.True.After(3000)); } - [Test] - public void TestChannelRetainsPlayingStateWhenMovedBetweenMixers() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestChannelRetainsPlayingStateWhenMovedBetweenMixers(AudioTestComponents.Type id) { - var secondMixer = bass.CreateMixer(); + setupBackend(id); + + var secondMixer = audio.CreateMixer(); secondMixer.Add(track); - bass.Update(); + audio.Update(); Assert.That(!track.IsRunning); - bass.RunOnAudioThread(() => track.Start()); - bass.Update(); + audio.RunOnAudioThread(() => track.Start()); + audio.Update(); Assert.That(track.IsRunning); - bass.Mixer.Add(track); - bass.Update(); + mixer.Add(track); + audio.Update(); Assert.That(track.IsRunning); } - [Test] - public void TestTrackReferenceLostWhenTrackIsDisposed() + [TestCase(AudioTestComponents.Type.SDL3)] + [TestCase(AudioTestComponents.Type.BASS)] + public void TestTrackReferenceLostWhenTrackIsDisposed(AudioTestComponents.Type id) { + setupBackend(id); + var trackReference = testDisposeTrackWithoutReference(); // The first update disposes the track, the second one removes the track from the TrackStore. - bass.Update(); - bass.Update(); + audio.Update(); + audio.Update(); GC.Collect(); GC.WaitForPendingFinalizers(); @@ -189,9 +253,9 @@ public void TestTrackReferenceLostWhenTrackIsDisposed() Assert.That(!trackReference.TryGetTarget(out _)); } - private WeakReference testDisposeTrackWithoutReference() + private WeakReference testDisposeTrackWithoutReference() { - var weakRef = new WeakReference(track); + var weakRef = new WeakReference(track); track.Dispose(); track = null; @@ -199,21 +263,24 @@ private WeakReference testDisposeTrackWithoutReference() return weakRef; } - [Test] - public void TestSampleChannelReferenceLostWhenSampleChannelIsDisposed() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestSampleChannelReferenceLostWhenSampleChannelIsDisposed(AudioTestComponents.Type id) { + setupBackend(id); + var channelReference = runTest(sample); // The first update disposes the track, the second one removes the track from the TrackStore. - bass.Update(); - bass.Update(); + audio.Update(); + audio.Update(); GC.Collect(); GC.WaitForPendingFinalizers(); Assert.That(!channelReference.TryGetTarget(out _)); - static WeakReference runTest(SampleBass sample) + static WeakReference runTest(Sample sample) { var channel = sample.GetChannel(); @@ -225,47 +292,59 @@ static WeakReference runTest(SampleBass sample) } } - [Test] - public void TestChannelDoesNotPlayIfReachedEndAndSeekedBackwards() + private void assertThatTrackIsPlaying() { - bass.RunOnAudioThread(() => + if (type == AudioTestComponents.Type.BASS) + Assert.That(((BassAudioMixer)mixer).ChannelIsActive((TrackBass)track), Is.Not.EqualTo(PlaybackState.Playing)); + else + Assert.That(track.IsRunning, Is.Not.True); + } + + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestChannelDoesNotPlayIfReachedEndAndSeekedBackwards(AudioTestComponents.Type id) + { + setupBackend(id, true); + + audio.RunOnAudioThread(() => { track.Seek(track.Length - 1); track.Start(); }); Thread.Sleep(50); - bass.Update(); + audio.Update(); - Assert.That(bass.Mixer.ChannelIsActive(track), Is.Not.EqualTo(PlaybackState.Playing)); + assertThatTrackIsPlaying(); - bass.RunOnAudioThread(() => track.SeekAsync(0).WaitSafely()); - bass.Update(); + audio.RunOnAudioThread(() => track.SeekAsync(0).WaitSafely()); + audio.Update(); - Assert.That(bass.Mixer.ChannelIsActive(track), Is.Not.EqualTo(PlaybackState.Playing)); + assertThatTrackIsPlaying(); } - [Test] - public void TestChannelDoesNotPlayIfReachedEndAndMovedMixers() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestChannelDoesNotPlayIfReachedEndAndMovedMixers(AudioTestComponents.Type id) { - bass.RunOnAudioThread(() => + setupBackend(id, true); + + audio.RunOnAudioThread(() => { track.Seek(track.Length - 1); track.Start(); }); Thread.Sleep(50); - bass.Update(); + audio.Update(); - Assert.That(bass.Mixer.ChannelIsActive(track), Is.Not.EqualTo(PlaybackState.Playing)); + assertThatTrackIsPlaying(); - var secondMixer = bass.CreateMixer(); + var secondMixer = audio.CreateMixer(); secondMixer.Add(track); - bass.Update(); + audio.Update(); - Assert.That(secondMixer.ChannelIsActive(track), Is.Not.EqualTo(PlaybackState.Playing)); + assertThatTrackIsPlaying(); } - - private int getHandle() => ((IBassAudioChannel)track).Handle; } } diff --git a/osu.Framework.Tests/Audio/BassTestComponents.cs b/osu.Framework.Tests/Audio/BassTestComponents.cs index 416b0ccbf9..2bf02e0148 100644 --- a/osu.Framework.Tests/Audio/BassTestComponents.cs +++ b/osu.Framework.Tests/Audio/BassTestComponents.cs @@ -1,13 +1,13 @@ // Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. // See the LICENCE file in the repository root for full licence text. -using System; +using System.IO; using ManagedBass; -using osu.Framework.Audio; +using osu.Framework.Audio.Mixing; using osu.Framework.Audio.Mixing.Bass; using osu.Framework.Audio.Sample; using osu.Framework.Audio.Track; -using osu.Framework.IO.Stores; +using osu.Framework.Extensions; using osu.Framework.Threading; namespace osu.Framework.Tests.Audio @@ -15,32 +15,14 @@ namespace osu.Framework.Tests.Audio /// /// Provides a BASS audio pipeline to be used for testing audio components. /// - public class BassTestComponents : IDisposable + public class BassTestComponents : AudioTestComponents { - internal readonly BassAudioMixer Mixer; - public readonly DllResourceStore Resources; - internal readonly TrackStore TrackStore; - internal readonly SampleStore SampleStore; - - private readonly AudioCollectionManager allComponents = new AudioCollectionManager(); - private readonly AudioCollectionManager mixerComponents = new AudioCollectionManager(); - public BassTestComponents(bool init = true) + : base(init) { - if (init) - Init(); - - allComponents.AddItem(mixerComponents); - - Mixer = CreateMixer(); - Resources = new DllResourceStore(typeof(TrackBassTest).Assembly); - TrackStore = new TrackStore(Resources, Mixer); - SampleStore = new SampleStore(Resources, Mixer); - - Add(TrackStore, SampleStore); } - public void Init() + public override void Init() { AudioThread.PreloadBass(); @@ -49,38 +31,29 @@ public void Init() Bass.Init(0); } - public void Add(params AudioComponent[] component) - { - foreach (var c in component) - allComponents.AddItem(c); - } - - internal BassAudioMixer CreateMixer() + public override AudioMixer CreateMixer() { var mixer = new BassAudioMixer(null, Mixer, "Test mixer"); - mixerComponents.AddItem(mixer); + MixerComponents.AddItem(mixer); return mixer; } - public void Update() + public override void DisposeInternal() { - RunOnAudioThread(() => allComponents.Update()); + base.DisposeInternal(); + Bass.Free(); } - /// - /// Runs an on a newly created audio thread, and blocks until it has been run to completion. - /// - /// The action to run on the audio thread. - public void RunOnAudioThread(Action action) => AudioTestHelper.RunOnAudioThread(action); + internal override Track CreateTrack(Stream data, string name) => new TrackBass(data, name); - internal TrackBass GetTrack() => (TrackBass)TrackStore.Get("Resources.Tracks.sample-track.mp3"); - internal SampleBass GetSample() => (SampleBass)SampleStore.Get("Resources.Tracks.sample-track.mp3"); - - public void Dispose() => RunOnAudioThread(() => + internal override SampleFactory CreateSampleFactory(Stream stream, string name, AudioMixer mixer, int playbackConcurrency) { - allComponents.Dispose(); - allComponents.Update(); // Actually runs the disposal. - Bass.Free(); - }); + byte[] data; + + using (stream) + data = stream.ReadAllBytesToArray(); + + return new SampleBassFactory(data, name, (BassAudioMixer)mixer, playbackConcurrency); + } } } diff --git a/osu.Framework.Tests/Audio/SDL3AudioTestComponents.cs b/osu.Framework.Tests/Audio/SDL3AudioTestComponents.cs new file mode 100644 index 0000000000..041ee1f9f7 --- /dev/null +++ b/osu.Framework.Tests/Audio/SDL3AudioTestComponents.cs @@ -0,0 +1,72 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using osu.Framework.Audio; +using osu.Framework.Audio.Mixing; +using osu.Framework.Audio.Mixing.SDL3; +using osu.Framework.Audio.Sample; +using osu.Framework.Audio.Track; +using static SDL.SDL3; + +namespace osu.Framework.Tests.Audio +{ + /// + /// Provides a SDL3 audio pipeline to be used for testing audio components. + /// + public class SDL3AudioTestComponents : AudioTestComponents + { + private SDL3AudioManager.SDL3BaseAudioManager baseManager = null!; + + public SDL3AudioTestComponents(bool init = true) + : base(init) + { + } + + protected override void Prepare() + { + base.Prepare(); + + SDL_SetHint(SDL_HINT_AUDIO_DRIVER, "dummy"u8); + baseManager = new SDL3AudioManager.SDL3BaseAudioManager(MixerComponents.Items.OfType); + } + + public override void Init() + { + if (!baseManager.SetAudioDevice(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK)) + throw new InvalidOperationException($"Failed to open SDL3 audio device: {SDL_GetError()}"); + } + + public override AudioMixer CreateMixer() + { + var mixer = new SDL3AudioMixer(Mixer, "Test mixer"); + baseManager.RunWhileLockingAudioStream(() => MixerComponents.AddItem(mixer)); + + return mixer; + } + + public void WaitUntilTrackIsLoaded(TrackSDL3 track) + { + // TrackSDL3 doesn't have data readily available right away after constructed. + while (!track.IsCompletelyLoaded) + { + Update(); + Thread.Sleep(10); + } + } + + public override void DisposeInternal() + { + base.DisposeInternal(); + baseManager.Dispose(); + } + + internal override Track CreateTrack(Stream data, string name) => baseManager.GetNewTrack(data, name); + + internal override SampleFactory CreateSampleFactory(Stream stream, string name, AudioMixer mixer, int playbackConcurrency) + => baseManager.GetSampleFactory(stream, name, mixer, playbackConcurrency); + } +} diff --git a/osu.Framework.Tests/Audio/SampleBassTest.cs b/osu.Framework.Tests/Audio/SampleBassTest.cs index ed0155b18e..c11abdff4b 100644 --- a/osu.Framework.Tests/Audio/SampleBassTest.cs +++ b/osu.Framework.Tests/Audio/SampleBassTest.cs @@ -13,28 +13,43 @@ namespace osu.Framework.Tests.Audio [TestFixture] public class SampleBassTest { - private BassTestComponents bass; + private AudioTestComponents audio; private Sample sample; + private SampleChannel channel; - [SetUp] - public void Setup() + [TearDown] + public void Teardown() { - bass = new BassTestComponents(); - sample = bass.GetSample(); - - bass.Update(); + audio?.Dispose(); } - [TearDown] - public void Teardown() + private void setupBackend(AudioTestComponents.Type id) { - bass?.Dispose(); + if (id == AudioTestComponents.Type.BASS) + { + audio = new BassTestComponents(); + sample = audio.GetSample(); + } + else if (id == AudioTestComponents.Type.SDL3) + { + audio = new SDL3AudioTestComponents(); + sample = audio.GetSample(); + } + else + { + throw new InvalidOperationException("not a supported id"); + } + + audio.Update(); } - [Test] - public void TestGetChannelOnDisposed() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestGetChannelOnDisposed(AudioTestComponents.Type id) { + setupBackend(id); + sample.Dispose(); sample.Update(); @@ -43,50 +58,63 @@ public void TestGetChannelOnDisposed() Assert.Throws(() => sample.Play()); } - [Test] - public void TestStart() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestStart(AudioTestComponents.Type id) { + setupBackend(id); + channel = sample.Play(); - bass.Update(); + + audio.Update(); Thread.Sleep(50); - bass.Update(); + audio.Update(); Assert.IsTrue(channel.Playing); } - [Test] - public void TestStop() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestStop(AudioTestComponents.Type id) { + setupBackend(id); + channel = sample.Play(); - bass.Update(); + audio.Update(); channel.Stop(); - bass.Update(); + audio.Update(); Assert.IsFalse(channel.Playing); } - [Test] - public void TestStopBeforeLoadFinished() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestStopBeforeLoadFinished(AudioTestComponents.Type id) { + setupBackend(id); + channel = sample.Play(); channel.Stop(); - bass.Update(); + audio.Update(); Assert.IsFalse(channel.Playing); } - [Test] - public void TestStopsWhenFactoryDisposed() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestStopsWhenFactoryDisposed(AudioTestComponents.Type id) { + setupBackend(id); + channel = sample.Play(); - bass.Update(); + audio.Update(); - bass.SampleStore.Dispose(); - bass.Update(); + audio.SampleStore.Dispose(); + audio.Update(); Assert.IsFalse(channel.Playing); } @@ -95,10 +123,13 @@ public void TestStopsWhenFactoryDisposed() /// Tests the case where a play call can be run inline due to already being on the audio thread. /// Because it's immediately executed, a `Bass.Update()` call is not required before the channel's state is updated. /// - [Test] - public void TestPlayingUpdatedAfterInlinePlay() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestPlayingUpdatedAfterInlinePlay(AudioTestComponents.Type id) { - bass.RunOnAudioThread(() => channel = sample.Play()); + setupBackend(id); + + audio.RunOnAudioThread(() => channel = sample.Play()); Assert.That(channel.Playing, Is.True); } @@ -106,13 +137,16 @@ public void TestPlayingUpdatedAfterInlinePlay() /// Tests the case where a stop call can be run inline due to already being on the audio thread. /// Because it's immediately executed, a `Bass.Update()` call is not required before the channel's state is updated. /// - [Test] - public void TestPlayingUpdatedAfterInlineStop() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestPlayingUpdatedAfterInlineStop(AudioTestComponents.Type id) { + setupBackend(id); + channel = sample.Play(); - bass.Update(); + audio.Update(); - bass.RunOnAudioThread(() => channel.Stop()); + audio.RunOnAudioThread(() => channel.Stop()); Assert.That(channel.Playing, Is.False); } } diff --git a/osu.Framework.Tests/Audio/TrackBassTest.cs b/osu.Framework.Tests/Audio/TrackBassTest.cs index 49b711428e..63794294fd 100644 --- a/osu.Framework.Tests/Audio/TrackBassTest.cs +++ b/osu.Framework.Tests/Audio/TrackBassTest.cs @@ -17,46 +17,66 @@ namespace osu.Framework.Tests.Audio [TestFixture] public class TrackBassTest { - private BassTestComponents bass; - private TrackBass track; + private AudioTestComponents audio; + private Track track; - [SetUp] - public void Setup() + [TearDown] + public void Teardown() { - bass = new BassTestComponents(); - track = bass.GetTrack(); - - bass.Update(); + audio?.Dispose(); } - [TearDown] - public void Teardown() + private void setupBackend(AudioTestComponents.Type id, bool loadTrack = false) { - bass?.Dispose(); + if (id == AudioTestComponents.Type.BASS) + { + audio = new BassTestComponents(); + track = audio.GetTrack(); + } + else if (id == AudioTestComponents.Type.SDL3) + { + audio = new SDL3AudioTestComponents(); + track = audio.GetTrack(); + + if (loadTrack) + ((SDL3AudioTestComponents)audio).WaitUntilTrackIsLoaded((TrackSDL3)track); + } + else + { + throw new InvalidOperationException("not a supported id"); + } + + audio.Update(); } - [Test] - public void TestStart() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestStart(AudioTestComponents.Type id) { + setupBackend(id, true); + track.StartAsync(); - bass.Update(); + audio.Update(); Thread.Sleep(50); - bass.Update(); + audio.Update(); Assert.IsTrue(track.IsRunning); Assert.Greater(track.CurrentTime, 0); } - [Test] - public void TestStop() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestStop(AudioTestComponents.Type id) { + setupBackend(id); + track.StartAsync(); - bass.Update(); + audio.Update(); track.StopAsync(); - bass.Update(); + audio.Update(); Assert.IsFalse(track.IsRunning); @@ -66,20 +86,23 @@ public void TestStop() Assert.AreEqual(expectedTime, track.CurrentTime); } - [Test] - public void TestStopWhenDisposed() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestStopWhenDisposed(AudioTestComponents.Type id) { + setupBackend(id); + track.StartAsync(); - bass.Update(); + audio.Update(); Thread.Sleep(50); - bass.Update(); + audio.Update(); Assert.IsTrue(track.IsAlive); Assert.IsTrue(track.IsRunning); track.Dispose(); - bass.Update(); + audio.Update(); Assert.IsFalse(track.IsAlive); Assert.IsFalse(track.IsRunning); @@ -90,42 +113,51 @@ public void TestStopWhenDisposed() Assert.AreEqual(expectedTime, track.CurrentTime); } - [Test] - public void TestStopAtEnd() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestStopAtEnd(AudioTestComponents.Type id) { + setupBackend(id, true); + startPlaybackAt(track.Length - 1); Thread.Sleep(50); - bass.Update(); + audio.Update(); track.StopAsync(); - bass.Update(); + audio.Update(); Assert.IsFalse(track.IsRunning); Assert.AreEqual(track.Length, track.CurrentTime); } - [Test] - public void TestSeek() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestSeek(AudioTestComponents.Type id) { + setupBackend(id, true); + track.SeekAsync(1000); - bass.Update(); + audio.Update(); Assert.IsFalse(track.IsRunning); Assert.AreEqual(1000, track.CurrentTime); } - [Test] - public void TestSeekWhileRunning() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestSeekWhileRunning(AudioTestComponents.Type id) { + setupBackend(id); + track.StartAsync(); - bass.Update(); + audio.Update(); track.SeekAsync(1000); - bass.Update(); + audio.Update(); Thread.Sleep(50); - bass.Update(); + audio.Update(); Assert.IsTrue(track.IsRunning); Assert.GreaterOrEqual(track.CurrentTime, 1000); @@ -134,41 +166,49 @@ public void TestSeekWhileRunning() /// /// Bass does not allow seeking to the end of the track. It should fail and the current time should not change. /// - [Test] - public void TestSeekToEndFails() + [TestCase(AudioTestComponents.Type.BASS)] + public void TestSeekToEndFails(AudioTestComponents.Type id) { + setupBackend(id); + bool? success = null; - bass.RunOnAudioThread(() => { success = track.Seek(track.Length); }); - bass.Update(); + audio.RunOnAudioThread(() => { success = track.Seek(track.Length); }); + audio.Update(); Assert.AreEqual(0, track.CurrentTime); Assert.IsFalse(success); } - [Test] - public void TestSeekBackToSamePosition() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestSeekBackToSamePosition(AudioTestComponents.Type id) { + setupBackend(id, true); + track.SeekAsync(1000); track.SeekAsync(0); - bass.Update(); + audio.Update(); Thread.Sleep(50); - bass.Update(); + audio.Update(); Assert.GreaterOrEqual(track.CurrentTime, 0); Assert.Less(track.CurrentTime, 1000); } - [Test] - public void TestPlaybackToEnd() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestPlaybackToEnd(AudioTestComponents.Type id) { + setupBackend(id, true); + startPlaybackAt(track.Length - 1); Thread.Sleep(50); - bass.Update(); + audio.Update(); Assert.IsFalse(track.IsRunning); Assert.AreEqual(track.Length, track.CurrentTime); @@ -178,51 +218,63 @@ public void TestPlaybackToEnd() /// Bass restarts the track from the beginning if Start is called when the track has been completed. /// This is blocked locally in , so this test expects the track to not restart. /// - [Test] - public void TestStartFromEndDoesNotRestart() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestStartFromEndDoesNotRestart(AudioTestComponents.Type id) { + setupBackend(id, true); + startPlaybackAt(track.Length - 1); Thread.Sleep(50); - bass.Update(); + audio.Update(); track.StartAsync(); - bass.Update(); + audio.Update(); Assert.AreEqual(track.Length, track.CurrentTime); } - [Test] - public void TestRestart() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestRestart(AudioTestComponents.Type id) { + setupBackend(id); + startPlaybackAt(1000); Thread.Sleep(50); - bass.Update(); + audio.Update(); restartTrack(); Assert.IsTrue(track.IsRunning); Assert.Less(track.CurrentTime, 1000); } - [Test] - public void TestRestartAtEnd() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestRestartAtEnd(AudioTestComponents.Type id) { + setupBackend(id); + startPlaybackAt(track.Length - 1); Thread.Sleep(50); - bass.Update(); + audio.Update(); restartTrack(); Assert.IsTrue(track.IsRunning); Assert.LessOrEqual(track.CurrentTime, 1000); } - [Test] - public void TestRestartFromRestartPoint() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestRestartFromRestartPoint(AudioTestComponents.Type id) { + setupBackend(id); + track.RestartPoint = 1000; startPlaybackAt(3000); @@ -233,10 +285,14 @@ public void TestRestartFromRestartPoint() Assert.Less(track.CurrentTime, 3000); } - [TestCase(0)] - [TestCase(1000)] - public void TestLoopingRestart(double restartPoint) + [TestCase(AudioTestComponents.Type.BASS, 0)] + [TestCase(AudioTestComponents.Type.SDL3, 0)] + [TestCase(AudioTestComponents.Type.BASS, 1000)] + [TestCase(AudioTestComponents.Type.SDL3, 1000)] + public void TestLoopingRestart(AudioTestComponents.Type id, double restartPoint) { + setupBackend(id, true); + track.Looping = true; track.RestartPoint = restartPoint; @@ -246,12 +302,12 @@ public void TestLoopingRestart(double restartPoint) // In a perfect world the track will be running after the update above, but during testing it's possible that the track is in // a stalled state due to updates running on Bass' own thread, so we'll loop until the track starts running again - // Todo: This should be fixed in the future if/when we invoke Bass.Update() ourselves + // Todo: This should be fixed in the future if/when we invoke audio.Update() ourselves int loopCount = 0; while (++loopCount < 50 && !track.IsRunning) { - bass.Update(); + audio.Update(); Thread.Sleep(10); } @@ -262,9 +318,12 @@ public void TestLoopingRestart(double restartPoint) Assert.LessOrEqual(track.CurrentTime, restartPoint + 1000); } - [Test] - public void TestSetTempoNegative() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestSetTempoNegative(AudioTestComponents.Type id) { + setupBackend(id); + Assert.Throws(() => track.Tempo.Value = -1); Assert.Throws(() => track.Tempo.Value = 0.04f); @@ -276,16 +335,22 @@ public void TestSetTempoNegative() Assert.AreEqual(0.05f, track.Tempo.Value); } - [Test] - public void TestRateWithAggregateAdjustments() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestRateWithAggregateAdjustments(AudioTestComponents.Type id) { + setupBackend(id); + track.AddAdjustment(AdjustableProperty.Frequency, new BindableDouble(1.5f)); Assert.AreEqual(1.5, track.Rate); } - [Test] - public void TestLoopingTrackDoesntSetCompleted() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestLoopingTrackDoesntSetCompleted(AudioTestComponents.Type id) { + setupBackend(id); + bool completedEvent = false; track.Completed += () => completedEvent = true; @@ -296,14 +361,17 @@ public void TestLoopingTrackDoesntSetCompleted() Assert.IsFalse(track.HasCompleted); Assert.IsFalse(completedEvent); - bass.Update(); + audio.Update(); Assert.IsTrue(track.IsRunning); } - [Test] - public void TestHasCompletedResetsOnSeekBack() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestHasCompletedResetsOnSeekBack(AudioTestComponents.Type id) { + setupBackend(id, true); + // start playback and wait for completion. startPlaybackAt(track.Length - 1); takeEffectsAndUpdateAfter(50); @@ -312,20 +380,23 @@ public void TestHasCompletedResetsOnSeekBack() // ensure seeking to end doesn't reset completed state. track.SeekAsync(track.Length); - bass.Update(); + audio.Update(); Assert.IsTrue(track.HasCompleted); // seeking back reset completed state. track.SeekAsync(track.Length - 1); - bass.Update(); + audio.Update(); Assert.IsFalse(track.HasCompleted); } - [Test] - public void TestZeroFrequencyHandling() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestZeroFrequencyHandling(AudioTestComponents.Type id) { + setupBackend(id, true); + // start track. track.StartAsync(); takeEffectsAndUpdateAfter(50); @@ -336,13 +407,13 @@ public void TestZeroFrequencyHandling() // now set to zero frequency and update track to take effects. track.Frequency.Value = 0; - bass.Update(); + audio.Update(); double currentTime = track.CurrentTime; // assert time is frozen after 50ms sleep and didn't change with full precision, but "IsRunning" is still true. Thread.Sleep(50); - bass.Update(); + audio.Update(); Assert.IsTrue(track.IsRunning); Assert.AreEqual(currentTime, track.CurrentTime); @@ -360,9 +431,12 @@ public void TestZeroFrequencyHandling() /// /// Ensure setting a paused (or not yet played) track's frequency from zero to one doesn't resume / play it. /// - [Test] - public void TestZeroFrequencyDoesntResumeTrack() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestZeroFrequencyDoesntResumeTrack(AudioTestComponents.Type id) { + setupBackend(id); + // start at zero frequency and wait a bit. track.Frequency.Value = 0; track.StartAsync(); @@ -374,7 +448,7 @@ public void TestZeroFrequencyDoesntResumeTrack() // stop track and update. track.StopAsync(); - bass.Update(); + audio.Update(); Assert.IsFalse(track.IsRunning); @@ -387,71 +461,83 @@ public void TestZeroFrequencyDoesntResumeTrack() Assert.AreEqual(0, track.CurrentTime); } - [Test] - public void TestBitrate() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestBitrate(AudioTestComponents.Type id) { + setupBackend(id, true); + Assert.Greater(track.Bitrate, 0); } /// /// Tests the case where a start call can be run inline due to already being on the audio thread. - /// Because it's immediately executed, a `Bass.Update()` call is not required before the channel's state is updated. + /// Because it's immediately executed, a `audio.Update()` call is not required before the channel's state is updated. /// - [Test] - public void TestIsRunningUpdatedAfterInlineStart() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestIsRunningUpdatedAfterInlineStart(AudioTestComponents.Type id) { - bass.RunOnAudioThread(() => track.Start()); + setupBackend(id); + + audio.RunOnAudioThread(() => track.Start()); Assert.That(track.IsRunning, Is.True); } /// /// Tests the case where a stop call can be run inline due to already being on the audio thread. - /// Because it's immediately executed, a `Bass.Update()` call is not required before the channel's state is updated. + /// Because it's immediately executed, a `audio.Update()` call is not required before the channel's state is updated. /// - [Test] - public void TestIsRunningUpdatedAfterInlineStop() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestIsRunningUpdatedAfterInlineStop(AudioTestComponents.Type id) { + setupBackend(id); + track.StartAsync(); - bass.Update(); + audio.Update(); - bass.RunOnAudioThread(() => track.Stop()); + audio.RunOnAudioThread(() => track.Stop()); Assert.That(track.IsRunning, Is.False); } /// /// Tests the case where a seek call can be run inline due to already being on the audio thread. - /// Because it's immediately executed, a `Bass.Update()` call is not required before the channel's state is updated. + /// Because it's immediately executed, a `audio.Update()` call is not required before the channel's state is updated. /// - [Test] - public void TestCurrentTimeUpdatedAfterInlineSeek() + [TestCase(AudioTestComponents.Type.BASS)] + [TestCase(AudioTestComponents.Type.SDL3)] + public void TestCurrentTimeUpdatedAfterInlineSeek(AudioTestComponents.Type id) { + setupBackend(id); + track.StartAsync(); - bass.Update(); + audio.Update(); - bass.RunOnAudioThread(() => track.Seek(20000)); + audio.RunOnAudioThread(() => track.Seek(20000)); Assert.That(track.CurrentTime, Is.EqualTo(20000).Within(100)); } private void takeEffectsAndUpdateAfter(int after) { - bass.Update(); + audio.Update(); Thread.Sleep(after); - bass.Update(); + audio.Update(); } private void startPlaybackAt(double time) { track.SeekAsync(time); track.StartAsync(); - bass.Update(); + audio.Update(); } private void restartTrack() { - bass.RunOnAudioThread(() => + audio.RunOnAudioThread(() => { track.Restart(); - bass.Update(); + audio.Update(); }); } } diff --git a/osu.Framework.Tests/Visual/Audio/TestSceneAudioManager.cs b/osu.Framework.Tests/Visual/Audio/TestSceneAudioManager.cs index da0939c8dc..2877b5c352 100644 --- a/osu.Framework.Tests/Visual/Audio/TestSceneAudioManager.cs +++ b/osu.Framework.Tests/Visual/Audio/TestSceneAudioManager.cs @@ -2,6 +2,7 @@ // See the LICENCE file in the repository root for full licence text. using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -65,6 +66,12 @@ public override byte[] Get(string name) return base.Get(name); } + public override Stream GetStream(string name) + { + attemptedLookups.Add(name); + return base.GetStream(name); + } + public override Task GetAsync(string name, CancellationToken cancellationToken = default) { attemptedLookups.Add(name); diff --git a/osu.Framework.Tests/Visual/Audio/TestSceneTrackAmplitudes.cs b/osu.Framework.Tests/Visual/Audio/TestSceneTrackAmplitudes.cs index 2741c1cbca..d98e1b5f9d 100644 --- a/osu.Framework.Tests/Visual/Audio/TestSceneTrackAmplitudes.cs +++ b/osu.Framework.Tests/Visual/Audio/TestSceneTrackAmplitudes.cs @@ -15,24 +15,24 @@ namespace osu.Framework.Tests.Visual.Audio { public partial class TestSceneTrackAmplitudes : FrameworkTestScene { - private DrawableTrack track; + private DrawableTrack drawableTrack; private Box leftChannel; private Box rightChannel; - private TrackBass bassTrack; + private Track track; private Container amplitudeBoxes; [BackgroundDependencyLoader] private void load(ITrackStore tracks) { - bassTrack = (TrackBass)tracks.Get("sample-track.mp3"); - int length = bassTrack.CurrentAmplitudes.FrequencyAmplitudes.Length; + track = tracks.Get("sample-track.mp3"); + int length = track.CurrentAmplitudes.FrequencyAmplitudes.Length; Children = new Drawable[] { - track = new DrawableTrack(bassTrack), + drawableTrack = new DrawableTrack(track), new GridContainer { RelativeSizeAxes = Axes.Both, @@ -87,16 +87,16 @@ protected override void LoadComplete() { base.LoadComplete(); - track.Looping = true; - AddStep("start track", () => track.Start()); - AddStep("stop track", () => track.Stop()); + drawableTrack.Looping = true; + AddStep("start track", () => drawableTrack.Start()); + AddStep("stop track", () => drawableTrack.Stop()); } protected override void Update() { base.Update(); - var amplitudes = bassTrack.CurrentAmplitudes; + var amplitudes = track.CurrentAmplitudes; rightChannel.Width = amplitudes.RightChannel * 0.5f; leftChannel.Width = amplitudes.LeftChannel * 0.5f; diff --git a/osu.Framework/Audio/AggregateAdjustmentExtensions.cs b/osu.Framework/Audio/AggregateAdjustmentExtensions.cs index 3dda8ef660..bbbeebd531 100644 --- a/osu.Framework/Audio/AggregateAdjustmentExtensions.cs +++ b/osu.Framework/Audio/AggregateAdjustmentExtensions.cs @@ -34,5 +34,24 @@ public static IBindable GetAggregate(this IAggregateAudioAdjustment adju throw new ArgumentOutOfRangeException(nameof(type), "Invalid adjustable property type."); } } + + /// + /// Get aggregated stereo volume by decreasing the opponent channel. + /// + /// The audio adjustments to return from. + /// Aggregated stereo volume. + internal static (double, double) GetAggregatedStereoVolume(this IAggregateAudioAdjustment adjustment) + { + double volume = adjustment.AggregateVolume.Value; + double balance = adjustment.AggregateBalance.Value; + double balanceAbs = 1.0 - Math.Abs(balance); + + if (balance < 0) + return (volume, volume * balanceAbs); + else if (balance > 0) + return (volume * balanceAbs, volume); + + return (volume, volume); + } } } diff --git a/osu.Framework/Audio/AudioManager.cs b/osu.Framework/Audio/AudioManager.cs index f6c366271d..cb5e895aad 100644 --- a/osu.Framework/Audio/AudioManager.cs +++ b/osu.Framework/Audio/AudioManager.cs @@ -6,55 +6,33 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.Diagnostics; -using System.Linq; +using System.IO; using System.Threading; -using ManagedBass; -using ManagedBass.Fx; -using ManagedBass.Mix; using osu.Framework.Audio.Mixing; -using osu.Framework.Audio.Mixing.Bass; using osu.Framework.Audio.Sample; using osu.Framework.Audio.Track; using osu.Framework.Bindables; -using osu.Framework.Development; -using osu.Framework.Extensions.TypeExtensions; using osu.Framework.IO.Stores; -using osu.Framework.Logging; using osu.Framework.Threading; namespace osu.Framework.Audio { - public class AudioManager : AudioCollectionManager + public abstract class AudioManager : AudioCollectionManager { /// - /// The number of BASS audio devices preceding the first real audio device. - /// Consisting of and . + /// The thread audio operations (mainly Bass calls) are ran on. /// - protected const int BASS_INTERNAL_DEVICE_COUNT = 2; - - /// - /// The index of the BASS audio device denoting the OS default. - /// - /// - /// See http://www.un4seen.com/doc/#bass/BASS_CONFIG_DEV_DEFAULT.html for more information on the included device. - /// - private const int bass_default_device = 1; + protected readonly AudioThread CurrentAudioThread; /// /// The manager component responsible for audio tracks (e.g. songs). /// - public ITrackStore Tracks => globalTrackStore.Value; + public ITrackStore Tracks => AudioGlobalTrackStore.Value; /// /// The manager component responsible for audio samples (e.g. sound effects). /// - public ISampleStore Samples => globalSampleStore.Value; - - /// - /// The thread audio operations (mainly Bass calls) are ran on. - /// - private readonly AudioThread thread; + public ISampleStore Samples => AudioGlobalSampleStore.Value; /// /// The global mixer which all tracks are routed into by default. @@ -72,18 +50,24 @@ public class AudioManager : AudioCollectionManager /// /// This property does not contain the names of disabled audio devices. /// - public IEnumerable AudioDeviceNames => audioDeviceNames; + public IEnumerable AudioDeviceNames => DeviceNames; /// /// Is fired whenever a new audio device is discovered and provides its name. /// public event Action OnNewDevice; + // workaround as c# doesn't allow actions to get invoked outside of this class + protected virtual void InvokeOnNewDevice(string deviceName) => OnNewDevice?.Invoke(deviceName); + /// /// Is fired whenever an audio device is lost and provides its name. /// public event Action OnLostDevice; + // same as above + protected virtual void InvokeOnLostDevice(string deviceName) => OnLostDevice?.Invoke(deviceName); + /// /// The preferred audio device we should use. A value of /// denotes the OS default. @@ -111,51 +95,27 @@ public class AudioManager : AudioCollectionManager /// /// Whether a global mixer is being used for audio routing. /// For now, this is only the case on Windows when using shared mode WASAPI initialisation. + /// Need to be moved from here as it's a BASS only thing but cannot happen due to possible code change from osu! (osu#26154) /// - public IBindable UsingGlobalMixer => usingGlobalMixer; - - private readonly Bindable usingGlobalMixer = new BindableBool(); - - /// - /// If a global mixer is being used, this will be the BASS handle for it. - /// If non-null, all game mixers should be added to this mixer. - /// - /// - /// When this is non-null, all mixers created via - /// will themselves be added to the global mixer, which will handle playback itself. - /// - /// In this mode of operation, nested mixers will be created with the - /// flag, meaning they no longer handle playback directly. - /// - /// An eventual goal would be to use a global mixer across all platforms as it can result - /// in more control and better playback performance. - /// - internal readonly IBindable GlobalMixerHandle = new Bindable(); - - public override bool IsLoaded => base.IsLoaded && - // bass default device is a null device (-1), not the actual system default. - Bass.CurrentDevice != Bass.DefaultDevice; + public readonly Bindable UsingGlobalMixer = new BindableBool(); // Mutated by multiple threads, must be thread safe. - private ImmutableArray audioDevices = ImmutableArray.Empty; - private ImmutableList audioDeviceNames = ImmutableList.Empty; + protected ImmutableList DeviceNames = ImmutableList.Empty; - private Scheduler scheduler => thread.Scheduler; + protected Scheduler AudioScheduler => CurrentAudioThread.Scheduler; - private Scheduler eventScheduler => EventScheduler ?? scheduler; - - private readonly CancellationTokenSource cancelSource = new CancellationTokenSource(); + protected readonly CancellationTokenSource CancelSource = new CancellationTokenSource(); /// /// The scheduler used for invoking publicly exposed delegate events. /// public Scheduler EventScheduler; - internal IBindableList ActiveMixers => activeMixers; - private readonly BindableList activeMixers = new BindableList(); + internal IBindableList ActiveMixers => AudioActiveMixers; + protected readonly BindableList AudioActiveMixers = new BindableList(); - private readonly Lazy globalTrackStore; - private readonly Lazy globalSampleStore; + private protected readonly Lazy AudioGlobalTrackStore; + private protected readonly Lazy AudioGlobalSampleStore; /// /// Constructs an AudioStore given a track resource store, and a sample resource store. @@ -163,70 +123,49 @@ public class AudioManager : AudioCollectionManager /// The host's audio thread. /// The resource store containing all audio tracks to be used in the future. /// The sample store containing all audio samples to be used in the future. - public AudioManager(AudioThread audioThread, ResourceStore trackStore, ResourceStore sampleStore) + protected AudioManager(AudioThread audioThread, ResourceStore trackStore, ResourceStore sampleStore) { - thread = audioThread; + Prepare(); - thread.RegisterManager(this); + CurrentAudioThread = audioThread; - AudioDevice.ValueChanged += _ => onDeviceChanged(); - GlobalMixerHandle.ValueChanged += handle => - { - onDeviceChanged(); - usingGlobalMixer.Value = handle.NewValue.HasValue; - }; + CurrentAudioThread.RegisterManager(this); + + AudioDevice.ValueChanged += _ => OnDeviceChanged(); - AddItem(TrackMixer = createAudioMixer(null, nameof(TrackMixer))); - AddItem(SampleMixer = createAudioMixer(null, nameof(SampleMixer))); + AddItem(TrackMixer = AudioCreateAudioMixer(null, nameof(TrackMixer))); + AddItem(SampleMixer = AudioCreateAudioMixer(null, nameof(SampleMixer))); - globalTrackStore = new Lazy(() => + AudioGlobalTrackStore = new Lazy(() => { - var store = new TrackStore(trackStore, TrackMixer); + var store = new TrackStore(trackStore, TrackMixer, GetNewTrack); AddItem(store); store.AddAdjustment(AdjustableProperty.Volume, VolumeTrack); return store; }); - globalSampleStore = new Lazy(() => + AudioGlobalSampleStore = new Lazy(() => { - var store = new SampleStore(sampleStore, SampleMixer); + var store = new SampleStore(sampleStore, SampleMixer, GetSampleFactory); AddItem(store); store.AddAdjustment(AdjustableProperty.Volume, VolumeSample); return store; }); + } - CancellationToken token = cancelSource.Token; - - syncAudioDevices(); - scheduler.AddDelayed(() => - { - // sync audioDevices every 1000ms - new Thread(() => - { - while (!token.IsCancellationRequested) - { - try - { - if (CheckForDeviceChanges(audioDevices)) - syncAudioDevices(); - Thread.Sleep(1000); - } - catch - { - } - } - }) - { - IsBackground = true - }.Start(); - }, 1000); + protected virtual void Prepare() + { } + internal abstract Track.Track GetNewTrack(Stream data, string name); + + internal abstract SampleFactory GetSampleFactory(Stream data, string name, AudioMixer mixer, int playbackConcurrency); + protected override void Dispose(bool disposing) { - cancelSource.Cancel(); + CancelSource.Cancel(); - thread.UnregisterManager(this); + CurrentAudioThread.UnregisterManager(this); OnNewDevice = null; OnLostDevice = null; @@ -234,21 +173,9 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } - private void onDeviceChanged() + protected void OnDeviceChanged() { - scheduler.Add(() => setAudioDevice(AudioDevice.Value)); - } - - private void onDevicesChanged() - { - scheduler.Add(() => - { - if (cancelSource.IsCancellationRequested) - return; - - if (!IsCurrentDeviceValid()) - setAudioDevice(); - }); + AudioScheduler.Add(() => SetAudioDevice(AudioDevice.Value)); } private static int userMixerID; @@ -261,27 +188,22 @@ private void onDevicesChanged() /// /// An identifier displayed on the audio mixer visualiser. public AudioMixer CreateAudioMixer(string identifier = default) => - createAudioMixer(SampleMixer, !string.IsNullOrEmpty(identifier) ? identifier : $"user #{Interlocked.Increment(ref userMixerID)}"); + AudioCreateAudioMixer(SampleMixer, !string.IsNullOrEmpty(identifier) ? identifier : $"user #{Interlocked.Increment(ref userMixerID)}"); - private AudioMixer createAudioMixer(AudioMixer fallbackMixer, string identifier) - { - var mixer = new BassAudioMixer(this, fallbackMixer, identifier); - AddItem(mixer); - return mixer; - } + protected abstract AudioMixer AudioCreateAudioMixer(AudioMixer fallbackMixer, string identifier); protected override void ItemAdded(AudioComponent item) { base.ItemAdded(item); if (item is AudioMixer mixer) - activeMixers.Add(mixer); + AudioActiveMixers.Add(mixer); } protected override void ItemRemoved(AudioComponent item) { base.ItemRemoved(item); if (item is AudioMixer mixer) - activeMixers.Remove(mixer); + AudioActiveMixers.Remove(mixer); } /// @@ -292,10 +214,10 @@ protected override void ItemRemoved(AudioComponent item) /// The to use for tracks created by this store. Defaults to the global . public ITrackStore GetTrackStore(IResourceStore store = null, AudioMixer mixer = null) { - if (store == null) return globalTrackStore.Value; + if (store == null) return AudioGlobalTrackStore.Value; - TrackStore tm = new TrackStore(store, mixer ?? TrackMixer); - globalTrackStore.Value.AddItem(tm); + TrackStore tm = new TrackStore(store, mixer ?? TrackMixer, GetNewTrack); + AudioGlobalTrackStore.Value.AddItem(tm); return tm; } @@ -312,204 +234,19 @@ public ITrackStore GetTrackStore(IResourceStore store = null, AudioMixer /// The to use for samples created by this store. Defaults to the global . public ISampleStore GetSampleStore(IResourceStore store = null, AudioMixer mixer = null) { - if (store == null) return globalSampleStore.Value; + if (store == null) return AudioGlobalSampleStore.Value; - SampleStore sm = new SampleStore(store, mixer ?? SampleMixer); - globalSampleStore.Value.AddItem(sm); + SampleStore sm = new SampleStore(store, mixer ?? SampleMixer, GetSampleFactory); + AudioGlobalSampleStore.Value.AddItem(sm); return sm; } - /// - /// Sets the output audio device by its name. - /// This will automatically fall back to the system default device on failure. - /// - /// Name of the audio device, or null to use the configured device preference . - private bool setAudioDevice(string deviceName = null) - { - deviceName ??= AudioDevice.Value; - - // try using the specified device - int deviceIndex = audioDeviceNames.FindIndex(d => d == deviceName); - if (deviceIndex >= 0 && setAudioDevice(BASS_INTERNAL_DEVICE_COUNT + deviceIndex)) - return true; - - // try using the system default if there is any device present. - if (audioDeviceNames.Count > 0 && setAudioDevice(bass_default_device)) - return true; - - // no audio devices can be used, so try using Bass-provided "No sound" device as last resort. - if (setAudioDevice(Bass.NoSoundDevice)) - return true; - - // we're boned. even "No sound" device won't initialise. - return false; - } - - private bool setAudioDevice(int deviceIndex) - { - var device = audioDevices.ElementAtOrDefault(deviceIndex); - - // device is invalid - if (!device.IsEnabled) - return false; - - // we don't want bass initializing with real audio device on headless test runs. - if (deviceIndex != Bass.NoSoundDevice && DebugUtils.IsNUnitRunning) - return false; - - // initialize new device - bool initSuccess = InitBass(deviceIndex); - if (Bass.LastError != Errors.Already && BassUtils.CheckFaulted(false)) - return false; - - if (!initSuccess) - { - Logger.Log("BASS failed to initialize but did not provide an error code", level: LogLevel.Error); - return false; - } - - Logger.Log($@"🔈 BASS initialised - BASS version: {Bass.Version} - BASS FX version: {BassFx.Version} - BASS MIX version: {BassMix.Version} - Device: {device.Name} - Driver: {device.Driver} - Update period: {Bass.UpdatePeriod} ms - Device buffer length: {Bass.DeviceBufferLength} ms - Playback buffer length: {Bass.PlaybackBufferLength} ms"); - - //we have successfully initialised a new device. - UpdateDevice(deviceIndex); - - return true; - } - - /// - /// This method calls . - /// It can be overridden for unit testing. - /// - protected virtual bool InitBass(int device) - { - if (Bass.CurrentDevice == device) - return true; - - // this likely doesn't help us but also doesn't seem to cause any issues or any cpu increase. - Bass.UpdatePeriod = 5; - - // reduce latency to a known sane minimum. - Bass.DeviceBufferLength = 10; - Bass.PlaybackBufferLength = 100; - - // ensure there are no brief delays on audio operations (causing stream stalls etc.) after periods of silence. - Bass.DeviceNonStop = true; - - // without this, if bass falls back to directsound legacy mode the audio playback offset will be way off. - Bass.Configure(ManagedBass.Configuration.TruePlayPosition, 0); - - // For iOS devices, set the default audio policy to one that obeys the mute switch. - Bass.Configure(ManagedBass.Configuration.IOSMixAudio, 5); - - // Always provide a default device. This should be a no-op, but we have asserts for this behaviour. - Bass.Configure(ManagedBass.Configuration.IncludeDefaultDevice, true); - - // Enable custom BASS_CONFIG_MP3_OLDGAPS flag for backwards compatibility. - Bass.Configure((ManagedBass.Configuration)68, 1); - - // Disable BASS_CONFIG_DEV_TIMEOUT flag to keep BASS audio output from pausing on device processing timeout. - // See https://www.un4seen.com/forum/?topic=19601 for more information. - Bass.Configure((ManagedBass.Configuration)70, false); - - if (!thread.InitDevice(device)) - return false; - - return true; - } - - private void syncAudioDevices() - { - audioDevices = GetAllDevices(); - - // Bass should always be providing "No sound" and "Default" device. - Trace.Assert(audioDevices.Length >= BASS_INTERNAL_DEVICE_COUNT, "Bass did not provide any audio devices."); - - var oldDeviceNames = audioDeviceNames; - var newDeviceNames = audioDeviceNames = audioDevices.Skip(BASS_INTERNAL_DEVICE_COUNT).Where(d => d.IsEnabled).Select(d => d.Name).ToImmutableList(); - - onDevicesChanged(); - - var newDevices = newDeviceNames.Except(oldDeviceNames).ToList(); - var lostDevices = oldDeviceNames.Except(newDeviceNames).ToList(); - - if (newDevices.Count > 0 || lostDevices.Count > 0) - { - eventScheduler.Add(delegate - { - foreach (string d in newDevices) - OnNewDevice?.Invoke(d); - foreach (string d in lostDevices) - OnLostDevice?.Invoke(d); - }); - } - } - - /// - /// Check whether any audio device changes have occurred. - /// - /// Changes supported are: - /// - A new device is added - /// - An existing device is Enabled/Disabled or set as Default - /// - /// - /// This method is optimised to incur the lowest overhead possible. - /// - /// The previous audio devices array. - /// Whether a change was detected. - protected virtual bool CheckForDeviceChanges(ImmutableArray previousDevices) - { - int deviceCount = Bass.DeviceCount; - - if (previousDevices.Length != deviceCount) - return true; - - for (int i = 0; i < deviceCount; i++) - { - var prevInfo = previousDevices[i]; - - Bass.GetDeviceInfo(i, out var info); - - if (info.IsEnabled != prevInfo.IsEnabled) - return true; - - if (info.IsDefault != prevInfo.IsDefault) - return true; - } - - return false; - } - - protected virtual ImmutableArray GetAllDevices() - { - int deviceCount = Bass.DeviceCount; - - var devices = ImmutableArray.CreateBuilder(deviceCount); - for (int i = 0; i < deviceCount; i++) - devices.Add(Bass.GetDeviceInfo(i)); - - return devices.MoveToImmutable(); - } + protected abstract bool SetAudioDevice(string deviceName = null); + protected abstract bool SetAudioDevice(int deviceIndex); // The current device is considered valid if it is enabled, initialized, and not a fallback device. - protected virtual bool IsCurrentDeviceValid() - { - var device = audioDevices.ElementAtOrDefault(Bass.CurrentDevice); - bool isFallback = string.IsNullOrEmpty(AudioDevice.Value) ? !device.IsDefault : device.Name != AudioDevice.Value; - return device.IsEnabled && device.IsInitialized && !isFallback; - } + protected abstract bool IsCurrentDeviceValid(); - public override string ToString() - { - string deviceName = audioDevices.ElementAtOrDefault(Bass.CurrentDevice).Name; - return $@"{GetType().ReadableName()} ({deviceName ?? "Unknown"})"; - } + public abstract override string ToString(); } } diff --git a/osu.Framework/Audio/BassAudioManager.cs b/osu.Framework/Audio/BassAudioManager.cs new file mode 100644 index 0000000000..5e08ef2124 --- /dev/null +++ b/osu.Framework/Audio/BassAudioManager.cs @@ -0,0 +1,336 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +#nullable disable + +using System; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using ManagedBass; +using ManagedBass.Fx; +using ManagedBass.Mix; +using osu.Framework.Audio.Mixing; +using osu.Framework.Audio.Mixing.Bass; +using osu.Framework.Audio.Sample; +using osu.Framework.Audio.Track; +using osu.Framework.Bindables; +using osu.Framework.Development; +using osu.Framework.Extensions; +using osu.Framework.Extensions.TypeExtensions; +using osu.Framework.IO.Stores; +using osu.Framework.Logging; +using osu.Framework.Threading; + +namespace osu.Framework.Audio +{ + public class BassAudioManager : AudioManager + { + /// + /// The number of BASS audio devices preceding the first real audio device. + /// Consisting of and . + /// + protected const int BASS_INTERNAL_DEVICE_COUNT = 2; + + /// + /// The index of the BASS audio device denoting the OS default. + /// + /// + /// See http://www.un4seen.com/doc/#bass/BASS_CONFIG_DEV_DEFAULT.html for more information on the included device. + /// + private const int bass_default_device = 1; + + /// + /// If a global mixer is being used, this will be the BASS handle for it. + /// If non-null, all game mixers should be added to this mixer. + /// + /// + /// When this is non-null, all mixers created via + /// will themselves be added to the global mixer, which will handle playback itself. + /// + /// In this mode of operation, nested mixers will be created with the + /// flag, meaning they no longer handle playback directly. + /// + /// An eventual goal would be to use a global mixer across all platforms as it can result + /// in more control and better playback performance. + /// + internal readonly IBindable GlobalMixerHandle = new Bindable(); + + public override bool IsLoaded => base.IsLoaded && + // bass default device is a null device (-1), not the actual system default. + Bass.CurrentDevice != Bass.DefaultDevice; + + // Mutated by multiple threads, must be thread safe. + private ImmutableArray audioDevices = ImmutableArray.Empty; + + private Scheduler eventScheduler => EventScheduler ?? CurrentAudioThread.Scheduler; + + /// + /// Constructs an AudioStore given a track resource store, and a sample resource store. + /// + /// The host's audio thread. + /// The resource store containing all audio tracks to be used in the future. + /// The sample store containing all audio samples to be used in the future. + public BassAudioManager(AudioThread audioThread, ResourceStore trackStore, ResourceStore sampleStore) + : base(audioThread, trackStore, sampleStore) + { + GlobalMixerHandle.ValueChanged += handle => + { + OnDeviceChanged(); + UsingGlobalMixer.Value = handle.NewValue.HasValue; + }; + + CancellationToken token = CancelSource.Token; + + syncAudioDevices(); + AudioScheduler.AddDelayed(() => + { + // sync audioDevices every 1000ms + new Thread(() => + { + while (!token.IsCancellationRequested) + { + try + { + if (CheckForDeviceChanges(audioDevices)) + syncAudioDevices(); + Thread.Sleep(1000); + } + catch + { + } + } + }) + { + IsBackground = true + }.Start(); + }, 1000); + } + + protected void OnDevicesChanged() + { + AudioScheduler.Add(() => + { + if (CancelSource.IsCancellationRequested) + return; + + if (!IsCurrentDeviceValid()) + SetAudioDevice(); + }); + } + + internal override Track.Track GetNewTrack(Stream data, string name) => new TrackBass(data, name); + + internal override SampleFactory GetSampleFactory(Stream stream, string name, AudioMixer mixer, int playbackConcurrency) + { + byte[] data; + + using (stream) + data = stream.ReadAllBytesToArray(); + + return new SampleBassFactory(data, name, (BassAudioMixer)mixer, playbackConcurrency); + } + + protected override AudioMixer AudioCreateAudioMixer(AudioMixer fallbackMixer, string identifier) + { + var mixer = new BassAudioMixer(this, fallbackMixer, identifier); + AddItem(mixer); + return mixer; + } + + /// + /// Sets the output audio device by its name. + /// This will automatically fall back to the system default device on failure. + /// + /// Name of the audio device, or null to use the configured device preference. + protected override bool SetAudioDevice(string deviceName = null) + { + deviceName ??= AudioDevice.Value; + + // try using the specified device + int deviceIndex = DeviceNames.FindIndex(d => d == deviceName); + if (deviceIndex >= 0 && SetAudioDevice(BASS_INTERNAL_DEVICE_COUNT + deviceIndex)) + return true; + + // try using the system default if there is any device present. + if (DeviceNames.Count > 0 && SetAudioDevice(bass_default_device)) + return true; + + // no audio devices can be used, so try using Bass-provided "No sound" device as last resort. + if (SetAudioDevice(Bass.NoSoundDevice)) + return true; + + // we're boned. even "No sound" device won't initialise. + return false; + } + + protected override bool SetAudioDevice(int deviceIndex) + { + var device = audioDevices.ElementAtOrDefault(deviceIndex); + + // device is invalid + if (!device.IsEnabled) + return false; + + // we don't want bass initializing with real audio device on headless test runs. + if (deviceIndex != Bass.NoSoundDevice && DebugUtils.IsNUnitRunning) + return false; + + // initialize new device + bool initSuccess = InitBass(deviceIndex); + if (Bass.LastError != Errors.Already && BassUtils.CheckFaulted(false)) + return false; + + if (!initSuccess) + { + Logger.Log("BASS failed to initialize but did not provide an error code", level: LogLevel.Error); + return false; + } + + Logger.Log($@"🔈 BASS initialised + BASS version: {Bass.Version} + BASS FX version: {BassFx.Version} + BASS MIX version: {BassMix.Version} + Device: {device.Name} + Driver: {device.Driver} + Update period: {Bass.UpdatePeriod} ms + Device buffer length: {Bass.DeviceBufferLength} ms + Playback buffer length: {Bass.PlaybackBufferLength} ms"); + + //we have successfully initialised a new device. + UpdateDevice(deviceIndex); + + return true; + } + + /// + /// This method calls . + /// It can be overridden for unit testing. + /// + protected virtual bool InitBass(int device) + { + if (Bass.CurrentDevice == device) + return true; + + // this likely doesn't help us but also doesn't seem to cause any issues or any cpu increase. + Bass.UpdatePeriod = 5; + + // reduce latency to a known sane minimum. + Bass.DeviceBufferLength = 10; + Bass.PlaybackBufferLength = 100; + + // ensure there are no brief delays on audio operations (causing stream stalls etc.) after periods of silence. + Bass.DeviceNonStop = true; + + // without this, if bass falls back to directsound legacy mode the audio playback offset will be way off. + Bass.Configure(ManagedBass.Configuration.TruePlayPosition, 0); + + // For iOS devices, set the default audio policy to one that obeys the mute switch. + Bass.Configure(ManagedBass.Configuration.IOSMixAudio, 5); + + // Always provide a default device. This should be a no-op, but we have asserts for this behaviour. + Bass.Configure(ManagedBass.Configuration.IncludeDefaultDevice, true); + + // Enable custom BASS_CONFIG_MP3_OLDGAPS flag for backwards compatibility. + Bass.Configure((ManagedBass.Configuration)68, 1); + + // Disable BASS_CONFIG_DEV_TIMEOUT flag to keep BASS audio output from pausing on device processing timeout. + // See https://www.un4seen.com/forum/?topic=19601 for more information. + Bass.Configure((ManagedBass.Configuration)70, false); + + if (!CurrentAudioThread.InitDevice(device)) + return false; + + return true; + } + + private void syncAudioDevices() + { + audioDevices = GetAllDevices(); + + // Bass should always be providing "No sound" and "Default" device. + Trace.Assert(audioDevices.Length >= BASS_INTERNAL_DEVICE_COUNT, "Bass did not provide any audio devices."); + + var oldDeviceNames = DeviceNames; + var newDeviceNames = DeviceNames = audioDevices.Skip(BASS_INTERNAL_DEVICE_COUNT).Where(d => d.IsEnabled).Select(d => d.Name).ToImmutableList(); + + OnDevicesChanged(); + + var newDevices = newDeviceNames.Except(oldDeviceNames).ToList(); + var lostDevices = oldDeviceNames.Except(newDeviceNames).ToList(); + + if (newDevices.Count > 0 || lostDevices.Count > 0) + { + eventScheduler.Add(delegate + { + foreach (string d in newDevices) + InvokeOnNewDevice(d); + foreach (string d in lostDevices) + InvokeOnLostDevice(d); + }); + } + } + + /// + /// Check whether any audio device changes have occurred. + /// + /// Changes supported are: + /// - A new device is added + /// - An existing device is Enabled/Disabled or set as Default + /// + /// + /// This method is optimised to incur the lowest overhead possible. + /// + /// The previous audio devices array. + /// Whether a change was detected. + protected virtual bool CheckForDeviceChanges(ImmutableArray previousDevices) + { + int deviceCount = Bass.DeviceCount; + + if (previousDevices.Length != deviceCount) + return true; + + for (int i = 0; i < deviceCount; i++) + { + var prevInfo = previousDevices[i]; + + Bass.GetDeviceInfo(i, out var info); + + if (info.IsEnabled != prevInfo.IsEnabled) + return true; + + if (info.IsDefault != prevInfo.IsDefault) + return true; + } + + return false; + } + + protected virtual ImmutableArray GetAllDevices() + { + int deviceCount = Bass.DeviceCount; + + var devices = ImmutableArray.CreateBuilder(deviceCount); + for (int i = 0; i < deviceCount; i++) + devices.Add(Bass.GetDeviceInfo(i)); + + return devices.MoveToImmutable(); + } + + // The current device is considered valid if it is enabled, initialized, and not a fallback device. + protected override bool IsCurrentDeviceValid() + { + var device = audioDevices.ElementAtOrDefault(Bass.CurrentDevice); + bool isFallback = string.IsNullOrEmpty(AudioDevice.Value) ? !device.IsDefault : device.Name != AudioDevice.Value; + return device.IsEnabled && device.IsInitialized && !isFallback; + } + + public override string ToString() + { + string deviceName = audioDevices.ElementAtOrDefault(Bass.CurrentDevice).Name; + return $@"{GetType().ReadableName()} ({deviceName ?? "Unknown"})"; + } + } +} diff --git a/osu.Framework/Audio/Mixing/Bass/BassAudioMixer.cs b/osu.Framework/Audio/Mixing/Bass/BassAudioMixer.cs index 07147af650..daaff8b528 100644 --- a/osu.Framework/Audio/Mixing/Bass/BassAudioMixer.cs +++ b/osu.Framework/Audio/Mixing/Bass/BassAudioMixer.cs @@ -17,7 +17,7 @@ namespace osu.Framework.Audio.Mixing.Bass /// internal class BassAudioMixer : AudioMixer, IBassAudio { - private readonly AudioManager? manager; + private readonly BassAudioManager? manager; /// /// The handle for this mixer. @@ -42,7 +42,7 @@ internal class BassAudioMixer : AudioMixer, IBassAudio public BassAudioMixer(AudioManager? manager, AudioMixer? fallbackMixer, string identifier) : base(fallbackMixer, identifier) { - this.manager = manager; + this.manager = (BassAudioManager?)manager; EnqueueAction(createMixer); } diff --git a/osu.Framework/Audio/Mixing/SDL3/ISDL3AudioChannel.cs b/osu.Framework/Audio/Mixing/SDL3/ISDL3AudioChannel.cs new file mode 100644 index 0000000000..f8c7ea4476 --- /dev/null +++ b/osu.Framework/Audio/Mixing/SDL3/ISDL3AudioChannel.cs @@ -0,0 +1,28 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +namespace osu.Framework.Audio.Mixing.SDL3 +{ + /// + /// Interface for audio channels that feed audio to . + /// + internal interface ISDL3AudioChannel : IAudioChannel + { + /// + /// Returns remaining audio samples. + /// + /// Channel puts audio in this array. Length of this determines how much data needs to be filled. + /// Sample count + int GetRemainingSamples(float[] data); + + /// + /// Mixer won't call if this returns false. + /// + bool Playing { get; } + + /// + /// Mixer uses this as volume, Value should be within 0 and 1. + /// + (float left, float right) Volume { get; } + } +} diff --git a/osu.Framework/Audio/Mixing/SDL3/SDL3AudioMixer.cs b/osu.Framework/Audio/Mixing/SDL3/SDL3AudioMixer.cs new file mode 100644 index 0000000000..9034b73f11 --- /dev/null +++ b/osu.Framework/Audio/Mixing/SDL3/SDL3AudioMixer.cs @@ -0,0 +1,247 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Collections.Generic; +using System.Linq; +using ManagedBass; +using ManagedBass.Fx; +using osu.Framework.Statistics; +using NAudio.Dsp; +using System; + +namespace osu.Framework.Audio.Mixing.SDL3 +{ + /// + /// Mixes instances and applies effects on top of them. + /// + internal class SDL3AudioMixer : AudioMixer + { + private readonly object syncRoot = new object(); + + /// + /// List of instances that are active. + /// + private readonly LinkedList activeChannels = new LinkedList(); + + /// + /// Creates a new + /// + /// + /// An identifier displayed on the audio mixer visualiser. + public SDL3AudioMixer(AudioMixer? globalMixer, string identifier) + : base(globalMixer, identifier) + { + } + + protected override void AddInternal(IAudioChannel channel) + { + if (channel is not ISDL3AudioChannel sdlChannel) + return; + + lock (syncRoot) + activeChannels.AddLast(sdlChannel); + } + + protected override void RemoveInternal(IAudioChannel channel) + { + if (channel is not ISDL3AudioChannel sdlChannel) + return; + + lock (syncRoot) + activeChannels.Remove(sdlChannel); + } + + protected override void UpdateState() + { + FrameStatistics.Add(StatisticsCounterType.MixChannels, channelCount); + base.UpdateState(); + } + + private void mixAudio(float[] dst, float[] src, int samples, float left, float right) + { + if (left <= 0 && right <= 0) + return; + + for (int i = 0; i < samples; i++) + { + dst[i] += src[i] * (i % 2 == 0 ? left : right); + + if (dst[i] > 1.0f) + dst[i] = 1.0f; + else if (dst[i] < -1.0f) + dst[i] = -1.0f; + } + } + + private float[]? ret; + + private float[]? filterArray; + + private volatile int channelCount; + + /// + /// Mix into a float array given as an argument. + /// + /// A float array that audio will be mixed into. + /// Size of data + public void MixChannelsInto(float[] data, int sampleCount) + { + lock (syncRoot) + { + if (ret == null || sampleCount != ret.Length) + ret = new float[sampleCount]; + + bool useFilters = activeEffects.Count > 0; + + if (useFilters) + { + if (filterArray == null || filterArray.Length != sampleCount) + filterArray = new float[sampleCount]; + + Array.Fill(filterArray, 0); + } + + var node = activeChannels.First; + + while (node != null) + { + var next = node.Next; + var channel = node.Value; + + if (!(channel is AudioComponent ac && ac.IsAlive)) + { + activeChannels.Remove(node); + } + else if (channel.Playing) + { + int size = channel.GetRemainingSamples(ret); + + if (size > 0) + { + var (left, right) = channel.Volume; + mixAudio(useFilters ? filterArray! : data, ret, size, left, right); + } + } + + node = next; + } + + channelCount = activeChannels.Count; + + if (useFilters) + { + foreach (var filter in activeEffects.Values) + { + for (int i = 0; i < sampleCount; i++) + filterArray![i] = filter.Transform(filterArray[i]); + } + + mixAudio(data, filterArray!, sampleCount, 1, 1); + } + } + } + + private static BiQuadFilter updateFilter(BiQuadFilter? filter, float freq, BQFParameters bqfp) + { + switch (bqfp.lFilter) + { + case BQFType.LowPass: + if (filter == null) + return BiQuadFilter.LowPassFilter(freq, bqfp.fCenter, bqfp.fQ); + else + filter.SetLowPassFilter(freq, bqfp.fCenter, bqfp.fQ); + + return filter; + + case BQFType.HighPass: + if (filter == null) + return BiQuadFilter.HighPassFilter(freq, bqfp.fCenter, bqfp.fQ); + else + filter.SetHighPassFilter(freq, bqfp.fCenter, bqfp.fQ); + + return filter; + + case BQFType.PeakingEQ: + if (filter == null) + return BiQuadFilter.PeakingEQ(freq, bqfp.fCenter, bqfp.fQ, bqfp.fGain); + else + filter.SetPeakingEq(freq, bqfp.fCenter, bqfp.fQ, bqfp.fGain); + + return filter; + + case BQFType.BandPass: + return BiQuadFilter.BandPassFilterConstantPeakGain(freq, bqfp.fCenter, bqfp.fQ); + + case BQFType.BandPassQ: + return BiQuadFilter.BandPassFilterConstantSkirtGain(freq, bqfp.fCenter, bqfp.fQ); + + case BQFType.Notch: + return BiQuadFilter.NotchFilter(freq, bqfp.fCenter, bqfp.fQ); + + case BQFType.LowShelf: + return BiQuadFilter.LowShelf(freq, bqfp.fCenter, bqfp.fS, bqfp.fGain); + + case BQFType.HighShelf: + return BiQuadFilter.HighShelf(freq, bqfp.fCenter, bqfp.fS, bqfp.fGain); + + case BQFType.AllPass: + default: + return BiQuadFilter.AllPassFilter(freq, bqfp.fCenter, bqfp.fQ); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + // Move all contained channels back to the default mixer. + foreach (var channel in activeChannels.ToArray()) + Remove(channel); + } + + public void StreamFree(IAudioChannel channel) + { + Remove(channel, false); + } + + // Would like something like BiMap in Java, but I cannot write the whole collection here. + private readonly SortedDictionary activeEffects = new SortedDictionary(); + private readonly Dictionary parameterDict = new Dictionary(); + + public override void AddEffect(IEffectParameter effect, int priority = 0) => EnqueueAction(() => + { + if (parameterDict.ContainsKey(effect) || effect is not BQFParameters bqfp) + return; + + while (activeEffects.ContainsKey(priority)) + priority++; + + BiQuadFilter filter = updateFilter(null, SDL3AudioManager.AUDIO_FREQ, bqfp); + + lock (syncRoot) + activeEffects[priority] = filter; + + parameterDict[effect] = priority; + }); + + public override void RemoveEffect(IEffectParameter effect) => EnqueueAction(() => + { + if (!parameterDict.TryGetValue(effect, out int index)) + return; + + lock (syncRoot) + activeEffects.Remove(index); + + parameterDict.Remove(effect); + }); + + public override void UpdateEffect(IEffectParameter effect) => EnqueueAction(() => + { + if (!parameterDict.TryGetValue(effect, out int index) || effect is not BQFParameters bqfp) + return; + + lock (syncRoot) + activeEffects[index] = updateFilter(activeEffects[index], SDL3AudioManager.AUDIO_FREQ, bqfp); + }); + } +} diff --git a/osu.Framework/Audio/ResamplingPlayer.cs b/osu.Framework/Audio/ResamplingPlayer.cs new file mode 100644 index 0000000000..64a80062db --- /dev/null +++ b/osu.Framework/Audio/ResamplingPlayer.cs @@ -0,0 +1,109 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using NAudio.Dsp; + +namespace osu.Framework.Audio +{ + /// + /// Abstract class that's meant to be used with a real player implementation. + /// This class provides resampling on the fly for players. + /// + internal abstract class ResamplingPlayer + { + private double relativeRate = 1; + + /// + /// Represents current relative rate. + /// + public double RelativeRate + { + get => relativeRate; + set => setRate(value); + } + + private WdlResampler? resampler; + + internal readonly int SrcRate; + internal readonly int SrcChannels; + + /// + /// Creates a new . + /// + /// Sampling rate of audio that's given from + /// Channels of audio that's given from + protected ResamplingPlayer(int srcRate, int srcChannels) + { + SrcRate = srcRate; + SrcChannels = srcChannels; + } + + /// + /// Sets relative rate of audio. + /// + /// Rate that is relative to the original frequency. 1.0 is normal rate. + private void setRate(double relativeRate) + { + if (relativeRate == 0) + { + this.relativeRate = relativeRate; + return; + } + + if (relativeRate < 0 || this.relativeRate == relativeRate) + return; + + if (resampler == null) + { + resampler = new WdlResampler(); + resampler.SetMode(true, 1, false); + resampler.SetFilterParms(); + resampler.SetFeedMode(false); + } + + resampler.SetRates(SrcRate, SrcRate / relativeRate); + this.relativeRate = relativeRate; + } + + protected virtual double GetProcessingLatency() + { + if (resampler == null || RelativeRate == 1) + return 0; + + return resampler.GetCurrentLatency() * 1000.0d; + } + + public virtual void Clear() + { + resampler?.Reset(); + } + + /// + /// Returns rate adjusted audio samples. It calls a parent method if is 1. + /// + /// An array to put samples in + /// The number of samples put into the array + public virtual int GetRemainingSamples(float[] data) + { + if (RelativeRate == 0) + return 0; + + if (resampler == null || RelativeRate == 1) + return GetRemainingRawFloats(data, 0, data.Length); + + int requested = data.Length / SrcChannels; + int needed = resampler.ResamplePrepare(requested, SrcChannels, out float[] inBuffer, out int inBufferOffset); + int rawGot = GetRemainingRawFloats(inBuffer, inBufferOffset, needed * SrcChannels); + + if (rawGot > 0) + { + int got = resampler.ResampleOut(data, 0, rawGot / SrcChannels, requested, SrcChannels); + return got * SrcChannels; + } + + return 0; + } + + protected abstract int GetRemainingRawFloats(float[] data, int offset, int needed); + } +} diff --git a/osu.Framework/Audio/SDL3AmplitudeProcessor.cs b/osu.Framework/Audio/SDL3AmplitudeProcessor.cs new file mode 100644 index 0000000000..a828f9050a --- /dev/null +++ b/osu.Framework/Audio/SDL3AmplitudeProcessor.cs @@ -0,0 +1,52 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using NAudio.Dsp; +using osu.Framework.Audio.Track; +using osu.Framework.Extensions; + +namespace osu.Framework.Audio +{ + internal class SDL3AmplitudeProcessor + { + /// + /// The most recent amplitude data. Note that this is updated on an ongoing basis and there is no guarantee it is in a consistent (single sample) state. + /// If you need consistent data, make a copy of FrequencyAmplitudes while on the audio thread. + /// + public ChannelAmplitudes CurrentAmplitudes { get; private set; } = ChannelAmplitudes.Empty; + + private readonly Complex[] fftSamples = new Complex[ChannelAmplitudes.AMPLITUDES_SIZE * 2]; + private readonly float[] fftResult = new float[ChannelAmplitudes.AMPLITUDES_SIZE]; + + public void Update(float[] samples, int channels) + { + if (samples.Length / channels < ChannelAmplitudes.AMPLITUDES_SIZE) + return; // not enough data + + float leftAmplitude = 0; + float rightAmplitude = 0; + int secondCh = channels < 2 ? 0 : 1; + int fftIndex = 0; + + for (int i = 0; i < samples.Length; i += channels) + { + leftAmplitude = Math.Max(leftAmplitude, Math.Abs(samples[i])); + rightAmplitude = Math.Max(rightAmplitude, Math.Abs(samples[i + secondCh])); + + if (fftIndex < fftSamples.Length) + { + fftSamples[fftIndex].Y = 0; + fftSamples[fftIndex++].X = samples[i] + samples[i + secondCh]; + } + } + + FastFourierTransform.FFT(true, (int)Math.Log2(fftSamples.Length), fftSamples); + + for (int i = 0; i < fftResult.Length; i++) + fftResult[i] = fftSamples[i].ComputeMagnitude(); + + CurrentAmplitudes = new ChannelAmplitudes(Math.Min(1f, leftAmplitude), Math.Min(1f, rightAmplitude), fftResult); + } + } +} diff --git a/osu.Framework/Audio/SDL3AudioDecoderManager.cs b/osu.Framework/Audio/SDL3AudioDecoderManager.cs new file mode 100644 index 0000000000..4a9563e30c --- /dev/null +++ b/osu.Framework/Audio/SDL3AudioDecoderManager.cs @@ -0,0 +1,543 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.IO; +using System.Threading; +using osu.Framework.Logging; +using System.Collections.Generic; +using SDL; +using ManagedBass.Mix; +using ManagedBass; +using osu.Framework.Audio.Callbacks; +using osu.Framework.Graphics.Video; + +namespace osu.Framework.Audio +{ + /// + /// Decodes audio from , and convert it to appropriate format. + /// It needs a lot of polishing... + /// + public class SDL3AudioDecoderManager : IDisposable + { + public interface ISDL3AudioDataReceiver + { + /// + /// Interface to get decoded audio data from the decoder. + /// + /// Decoded audio. The format depends on you specified, + /// so you may need to actual data format. + /// This may be used by decoder later to reduce allocation, so you need to copy the data before exiting from this delegate, otherwise you may end up with wrong data. + /// Length in byte of decoded audio. Use this instead of data.Length + /// Whether if this is the last data or not. + void GetData(byte[] data, int length, bool done); + + void GetMetaData(int bitrate, double length, long byteLength); + } + + private readonly LinkedList jobs = new LinkedList(); + + private readonly Thread decoderThread; + private readonly AutoResetEvent decoderWaitHandle; + private readonly CancellationTokenSource tokenSource; + + /// + /// Creates a new decoder that is not managed by the decoder thread. + /// + /// Refer to + /// Refer to + /// Refer to + /// Refer to + /// Refer to + /// A new instance. + internal static SDL3AudioDecoder CreateDecoder(Stream stream, SDL_AudioSpec audioSpec, bool isTrack, bool autoDisposeStream = true, ISDL3AudioDataReceiver? pass = null) + { + SDL3AudioDecoder decoder = Bass.CurrentDevice >= 0 + ? new SDL3AudioDecoder.BassAudioDecoder(stream, audioSpec, isTrack, autoDisposeStream, pass) + : new SDL3AudioDecoder.FFmpegAudioDecoder(stream, audioSpec, isTrack, autoDisposeStream, pass); + + return decoder; + } + + private readonly bool bassInit; + + /// + /// Starts a decoder thread. + /// + public SDL3AudioDecoderManager() + { + tokenSource = new CancellationTokenSource(); + decoderWaitHandle = new AutoResetEvent(false); + + decoderThread = new Thread(() => loop(tokenSource.Token)) + { + IsBackground = true + }; + + Bass.Configure((ManagedBass.Configuration)68, 1); + + if (Bass.CurrentDevice < 0) + bassInit = Bass.Init(Bass.NoSoundDevice); + + decoderThread.Start(); + } + + /// + /// Creates a new decoder, and adds it to the job list of a decoder thread. + /// + /// Refer to + /// Refer to + /// Refer to + /// Refer to + /// A new instance. + public SDL3AudioDecoder StartDecodingAsync(Stream stream, SDL_AudioSpec audioSpec, bool isTrack, ISDL3AudioDataReceiver pass) + { + if (disposedValue) + throw new InvalidOperationException($"Cannot start decoding on disposed {nameof(SDL3AudioDecoderManager)}"); + + SDL3AudioDecoder decoder = CreateDecoder(stream, audioSpec, isTrack, true, pass); + + lock (jobs) + jobs.AddFirst(decoder); + + decoderWaitHandle.Set(); + + return decoder; + } + + private void loop(CancellationToken token) + { + while (!token.IsCancellationRequested) + { + int jobCount; + + lock (jobs) + { + jobCount = jobs.Count; + + if (jobCount > 0) + { + var node = jobs.First; + + while (node != null) + { + var next = node.Next; + SDL3AudioDecoder decoder = node.Value; + + if (!decoder.StopJob) + { + try + { + int read = decodeAudio(decoder, out byte[] decoded); + + if (!decoder.MetadataSended) + { + decoder.MetadataSended = true; + decoder.Pass?.GetMetaData(decoder.Bitrate, decoder.Length, decoder.ByteLength); + } + + decoder.Pass?.GetData(decoded, read, !decoder.Loading); + } + catch (ObjectDisposedException) + { + decoder.StopJob = true; + } + + if (!decoder.Loading) + jobs.Remove(node); + } + else + { + decoder.Dispose(); + jobs.Remove(node); + } + + node = next; + } + } + } + + if (jobCount <= 0) + decoderWaitHandle.WaitOne(); + } + } + + private bool disposedValue; + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + tokenSource.Cancel(); + decoderWaitHandle.Set(); + + decoderThread.Join(); + tokenSource.Dispose(); + decoderWaitHandle.Dispose(); + + lock (jobs) + { + foreach (var job in jobs) + { + job.Dispose(); + } + + jobs.Clear(); + } + + if (bassInit) + { + Bass.CurrentDevice = Bass.NoSoundDevice; + Bass.Free(); + } + + disposedValue = true; + } + } + + ~SDL3AudioDecoderManager() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private static int decodeAudio(SDL3AudioDecoder decoder, out byte[] decoded) + { + int read = decoder.LoadFromStream(out byte[] temp); + + if (!decoder.Loading || decoder.IsTrack) + { + decoded = temp; + return read; + } + + // fallback if it couldn't decode at once + using (MemoryStream memoryStream = new MemoryStream()) + { + memoryStream.Write(temp, 0, read); + + while (decoder.Loading) + { + read = decoder.LoadFromStream(out temp); + memoryStream.Write(temp, 0, read); + } + + decoded = memoryStream.ToArray(); + return (int)memoryStream.Length; + } + } + + /// + /// Contains decoder information, and perform the actual decoding. + /// + public abstract class SDL3AudioDecoder + { + /// + /// Decoder will decode audio data from this. + /// It accepts most formats. (e.g. MP3, OGG, WAV and so on...) + /// + internal readonly Stream Stream; + + /// + /// Decoder will convert audio data according to this spec if needed. + /// + internal readonly SDL_AudioSpec AudioSpec; + + /// + /// Decoder will call multiple times with partial data if true. + /// It's a receiver's job to combine the data in this case. Otherwise, It will call only once with the entirely decoded data if false. + /// + internal readonly bool IsTrack; + + /// + /// It will automatically dispose once decoding is done/failed. + /// + internal readonly bool AutoDisposeStream; + + /// + /// Decoder will call once or more to pass the decoded audio data. + /// + internal readonly ISDL3AudioDataReceiver? Pass; + + private int bitrate; + + /// + /// Audio bitrate. Decoder may fill this in after the first call of . + /// + public int Bitrate + { + get => bitrate; + set => Interlocked.Exchange(ref bitrate, value); + } + + private double length; + + /// + /// Audio length in milliseconds. Decoder may fill this in after the first call of . + /// + public double Length + { + get => length; + set => Interlocked.Exchange(ref length, value); + } + + private long byteLength; + + /// + /// Audio length in byte. Note that this may not be accurate. You cannot depend on this value entirely. + /// You can find out the actual byte length by summing up byte counts you received once decoding is done. + /// Decoder may fill this in after the first call of . + /// + public long ByteLength + { + get => byteLength; + set => Interlocked.Exchange(ref byteLength, value); + } + + internal bool MetadataSended; + + internal volatile bool StopJob; + + private volatile bool loading; + + /// + /// Whether it is decoding or not. + /// + public bool Loading { get => loading; protected set => loading = value; } + + protected SDL3AudioDecoder(Stream stream, SDL_AudioSpec audioSpec, bool isTrack, bool autoDisposeStream, ISDL3AudioDataReceiver? pass) + { + Stream = stream; + AudioSpec = audioSpec; + IsTrack = isTrack; + AutoDisposeStream = autoDisposeStream; + Pass = pass; + } + + /// + /// Add a flag to stop decoding in the next loop of decoder thread. + /// + public void Stop() + { + StopJob = true; + } + + // Not using IDisposable since things must be handled in a decoder thread + internal virtual void Dispose() + { + if (AutoDisposeStream) + Stream.Dispose(); + } + + protected abstract int LoadFromStreamInternal(out byte[] decoded); + + /// + /// Decodes and resamples audio from job.Stream, and pass it to decoded. + /// You may need to run this multiple times. + /// Don't call this yourself if this decoder is in the decoder thread job list. + /// + /// Decoded audio + public int LoadFromStream(out byte[] decoded) + { + int read = 0; + + try + { + read = LoadFromStreamInternal(out decoded); + } + catch (Exception e) + { + Logger.Log(e.Message, level: LogLevel.Important); + Loading = false; + decoded = Array.Empty(); + } + finally + { + if (!Loading) + Dispose(); + } + + return read; + } + + /// + /// This is only for using BASS as a decoder for SDL3 backend! + /// + internal class BassAudioDecoder : SDL3AudioDecoder + { + private int decodeStream; + private FileCallbacks? fileCallbacks; + + private int resampler; + + private byte[]? decodeData; + + private Resolution resolution + { + get + { + if (AudioSpec.format == SDL_AudioFormat.SDL_AUDIO_S8) + return Resolution.Byte; + else if (AudioSpec.format == SDL3.SDL_AUDIO_S16) // uses constant due to endian + return Resolution.Short; + else + return Resolution.Float; + } + } + + private ushort bits => (ushort)SDL3.SDL_AUDIO_BITSIZE(AudioSpec.format); + + public BassAudioDecoder(Stream stream, SDL_AudioSpec audioSpec, bool isTrack, bool autoDisposeStream, ISDL3AudioDataReceiver? pass) + : base(stream, audioSpec, isTrack, autoDisposeStream, pass) + { + } + + internal override void Dispose() + { + fileCallbacks?.Dispose(); + fileCallbacks = null; + + decodeData = null; + + if (resampler != 0) + { + Bass.StreamFree(resampler); + resampler = 0; + } + + if (decodeStream != 0) + { + Bass.StreamFree(decodeStream); + decodeStream = 0; + } + + base.Dispose(); + } + + protected override int LoadFromStreamInternal(out byte[] decoded) + { + if (Bass.CurrentDevice < 0) + throw new InvalidOperationException($"Initialize a BASS device to decode audio: {Bass.LastError}"); + + if (!Loading) + { + fileCallbacks = new FileCallbacks(new DataStreamFileProcedures(Stream)); + + BassFlags bassFlags = BassFlags.Decode | resolution.ToBassFlag(); + if (IsTrack) bassFlags |= BassFlags.Prescan; + + decodeStream = Bass.CreateStream(StreamSystem.NoBuffer, bassFlags, fileCallbacks.Callbacks); + + if (decodeStream == 0) + throw new FormatException($"Couldn't create stream: {Bass.LastError}"); + + if (Bass.ChannelGetInfo(decodeStream, out var info)) + { + ByteLength = Bass.ChannelGetLength(decodeStream); + Length = Bass.ChannelBytes2Seconds(decodeStream, ByteLength) * 1000.0d; + Bitrate = (int)Math.Round(Bass.ChannelGetAttribute(decodeStream, ChannelAttribute.Bitrate)); + + if (info.Channels != AudioSpec.channels || info.Frequency != AudioSpec.freq) + { + resampler = BassMix.CreateMixerStream(AudioSpec.freq, AudioSpec.channels, BassFlags.Decode | resolution.ToBassFlag()); + + if (resampler == 0) + throw new FormatException($"Failed to create BASS Mixer: {Bass.LastError}"); + + if (!BassMix.MixerAddChannel(resampler, decodeStream, BassFlags.MixerChanNoRampin | BassFlags.MixerChanLimit)) + throw new FormatException($"Failed to add a channel to BASS Mixer: {Bass.LastError}"); + + ByteLength /= info.Channels * (bits / 8); + ByteLength = (long)Math.Ceiling((decimal)ByteLength / info.Frequency * AudioSpec.freq); + ByteLength *= AudioSpec.channels * (bits / 8); + } + } + else + { + if (IsTrack) + throw new FormatException($"Couldn't get channel info: {Bass.LastError}"); + } + + Loading = true; + } + + int handle = resampler == 0 ? decodeStream : resampler; + + int bufferLen = (int)Bass.ChannelSeconds2Bytes(handle, 1); + + if (bufferLen <= 0) + bufferLen = 44100 * 2 * 4 * 1; + + if (decodeData == null || decodeData.Length < bufferLen) + decodeData = new byte[bufferLen]; + + int got = Bass.ChannelGetData(handle, decodeData, bufferLen); + + if (got == -1) + { + Loading = false; + + if (Bass.LastError != Errors.Ended) + throw new FormatException($"Couldn't decode: {Bass.LastError}"); + } + else if (got < bufferLen) + { + // originally used synchandle to detect end, but it somehow created strong handle + Loading = false; + } + + decoded = decodeData; + return Math.Max(0, got); + } + } + + internal class FFmpegAudioDecoder : SDL3AudioDecoder + { + private VideoDecoder? ffmpeg; + + public FFmpegAudioDecoder(Stream stream, SDL_AudioSpec audioSpec, bool isTrack, bool autoDisposeStream, ISDL3AudioDataReceiver? pass) + : base(stream, audioSpec, isTrack, autoDisposeStream, pass) + { + } + + internal override void Dispose() + { + ffmpeg?.Dispose(); + ffmpeg = null; + + base.Dispose(); + } + + protected override int LoadFromStreamInternal(out byte[] decoded) + { + if (ffmpeg == null) + { + ffmpeg = new VideoDecoder(Stream, AudioSpec.freq, AudioSpec.channels, + SDL3.SDL_AUDIO_ISFLOAT(AudioSpec.format), SDL3.SDL_AUDIO_BITSIZE(AudioSpec.format), SDL3.SDL_AUDIO_ISSIGNED(AudioSpec.format)); + + ffmpeg.PrepareDecoding(); + ffmpeg.RecreateCodecContext(); + + Bitrate = (int)ffmpeg.AudioBitrate; + Length = ffmpeg.Duration; + ByteLength = (long)Math.Ceiling(ffmpeg.Duration / 1000.0d * AudioSpec.freq) * AudioSpec.channels * (SDL3.SDL_AUDIO_BITSIZE(AudioSpec.format) / 8); // FIXME + + Loading = true; + } + + int got = ffmpeg.DecodeNextAudioFrame(out decoded, !IsTrack); + + if (ffmpeg.State != VideoDecoder.DecoderState.Running) + Loading = false; + + return got; + } + } + } + } +} diff --git a/osu.Framework/Audio/SDL3AudioManager.cs b/osu.Framework/Audio/SDL3AudioManager.cs new file mode 100644 index 0000000000..1df1a6802d --- /dev/null +++ b/osu.Framework/Audio/SDL3AudioManager.cs @@ -0,0 +1,391 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +#nullable disable + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using osu.Framework.Allocation; +using osu.Framework.Audio.Mixing; +using osu.Framework.Audio.Mixing.SDL3; +using osu.Framework.Audio.Sample; +using osu.Framework.Audio.Track; +using osu.Framework.Extensions.TypeExtensions; +using osu.Framework.IO.Stores; +using osu.Framework.Logging; +using osu.Framework.Threading; +using SDL; +using static SDL.SDL3; + +namespace osu.Framework.Audio +{ + public class SDL3AudioManager : AudioManager + { + public static readonly int AUDIO_FREQ = 44100; + public static readonly int AUDIO_CHANNELS = 2; + public static readonly SDL_AudioFormat AUDIO_FORMAT = SDL_AUDIO_F32; + + private readonly List sdlMixerList = new List(); + + private ImmutableArray deviceIdArray = ImmutableArray.Empty; + + private Scheduler eventScheduler => EventScheduler ?? CurrentAudioThread.Scheduler; + + protected override void InvokeOnNewDevice(string deviceName) => eventScheduler.Add(() => base.InvokeOnNewDevice(deviceName)); + + protected override void InvokeOnLostDevice(string deviceName) => eventScheduler.Add(() => base.InvokeOnLostDevice(deviceName)); + + private SDL3BaseAudioManager baseManager; + + /// + /// Creates a new . + /// + /// The host's audio thread. + /// The resource store containing all audio tracks to be used in the future. + /// The sample store containing all audio samples to be used in the future. + public SDL3AudioManager(AudioThread audioThread, ResourceStore trackStore, ResourceStore sampleStore) + : base(audioThread, trackStore, sampleStore) + { + AudioScheduler.Add(syncAudioDevices); + } + + protected override void Prepare() + { + baseManager = new SDL3BaseAudioManager(() => sdlMixerList); + } + + public override string ToString() + { + return $@"{GetType().ReadableName()} ({baseManager.DeviceName})"; + } + + protected override AudioMixer AudioCreateAudioMixer(AudioMixer fallbackMixer, string identifier) + { + var mixer = new SDL3AudioMixer(fallbackMixer, identifier); + AddItem(mixer); + return mixer; + } + + protected override void ItemAdded(AudioComponent item) + { + base.ItemAdded(item); + + if (item is SDL3AudioMixer mixer) + baseManager.RunWhileLockingAudioStream(() => sdlMixerList.Add(mixer)); + } + + protected override void ItemRemoved(AudioComponent item) + { + base.ItemRemoved(item); + + if (item is SDL3AudioMixer mixer) + baseManager.RunWhileLockingAudioStream(() => sdlMixerList.Remove(mixer)); + } + + internal void OnNewDeviceEvent(SDL_AudioDeviceID addedDeviceIndex) + { + AudioScheduler.Add(() => + { + // the index is only valid until next SDL_GetNumAudioDevices call, so get the name first. + string name = SDL_GetAudioDeviceName(addedDeviceIndex); + + syncAudioDevices(); + InvokeOnNewDevice(name); + }); + } + + internal void OnLostDeviceEvent(SDL_AudioDeviceID removedDeviceId) + { + AudioScheduler.Add(() => + { + // SDL doesn't retain information about removed device. + syncAudioDevices(); + + if (!IsCurrentDeviceValid()) // current device lost + { + InvokeOnLostDevice(baseManager.DeviceName); + SetAudioDevice(); + } + else + { + // we can probably guess the name by comparing the old list and the new one, but it won't be reliable + InvokeOnLostDevice(string.Empty); + } + }); + } + + private unsafe void syncAudioDevices() + { + int count = 0; + SDL_AudioDeviceID* idArrayPtr = SDL_GetAudioPlaybackDevices(&count); + + var idArray = ImmutableArray.CreateBuilder(count); + var nameArray = ImmutableArray.CreateBuilder(count); + + for (int i = 0; i < count; i++) + { + SDL_AudioDeviceID id = *(idArrayPtr + i); + string name = SDL_GetAudioDeviceName(id); + + if (string.IsNullOrEmpty(name)) + continue; + + idArray.Add(id); + nameArray.Add(name); + } + + deviceIdArray = idArray.ToImmutable(); + DeviceNames = nameArray.ToImmutableList(); + } + + private bool setAudioDevice(SDL_AudioDeviceID targetId) + { + if (baseManager.SetAudioDevice(targetId)) + { + Logger.Log($@"🔈 SDL Audio initialised + Driver: {SDL_GetCurrentAudioDriver()} + Device Name: {baseManager.DeviceName} + Format: {baseManager.AudioSpec.freq}hz {baseManager.AudioSpec.channels}ch + Sample size: {baseManager.BufferSize}"); + + return true; + } + + return false; + } + + protected override bool SetAudioDevice(string deviceName = null) + { + deviceName ??= AudioDevice.Value; + + int deviceIndex = DeviceNames.FindIndex(d => d == deviceName); + if (deviceIndex >= 0) + return setAudioDevice(deviceIdArray[deviceIndex]); + + return setAudioDevice(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK); + } + + protected override bool SetAudioDevice(int deviceIndex) + { + if (deviceIndex < deviceIdArray.Length && deviceIndex >= 0) + return setAudioDevice(deviceIdArray[deviceIndex]); + + return SetAudioDevice(); + } + + protected override bool IsCurrentDeviceValid() => baseManager.DeviceId > 0 && !SDL_AudioDevicePaused(baseManager.DeviceId); + + internal override Track.Track GetNewTrack(Stream data, string name) => baseManager.GetNewTrack(data, name); + + internal override SampleFactory GetSampleFactory(Stream data, string name, AudioMixer mixer, int playbackConcurrency) + => baseManager.GetSampleFactory(data, name, mixer, playbackConcurrency); + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + baseManager.Dispose(); + } + + /// + /// To share basic playback logic with audio tests. + /// + internal unsafe class SDL3BaseAudioManager : IDisposable + { + internal SDL_AudioSpec AudioSpec { get; private set; } + + internal SDL_AudioDeviceID DeviceId { get; private set; } + internal SDL_AudioStream* DeviceStream { get; private set; } + + internal int BufferSize { get; private set; } = (int)(AUDIO_FREQ * 0.01); + + internal string DeviceName { get; private set; } = "Not loaded"; + + private readonly Func> mixerIterator; + + private ObjectHandle objectHandle; + + private readonly SDL3AudioDecoderManager decoderManager = new SDL3AudioDecoderManager(); + + internal SDL3BaseAudioManager(Func> mixerIterator) + { + if (!SDL_InitSubSystem(SDL_InitFlags.SDL_INIT_AUDIO)) + { + throw new InvalidOperationException($"Failed to initialise SDL Audio: {SDL_GetError()}"); + } + + this.mixerIterator = mixerIterator; + + objectHandle = new ObjectHandle(this, GCHandleType.Normal); + AudioSpec = new SDL_AudioSpec + { + freq = AUDIO_FREQ, + channels = AUDIO_CHANNELS, + format = AUDIO_FORMAT + }; + } + + internal void RunWhileLockingAudioStream(Action action) + { + SDL_AudioStream* stream = DeviceStream; + + if (stream != null) + SDL_LockAudioStream(stream); + + try + { + action(); + } + finally + { + if (stream != null) + SDL_UnlockAudioStream(stream); + } + } + + internal bool SetAudioDevice(SDL_AudioDeviceID targetId) + { + if (DeviceStream != null) + { + SDL_DestroyAudioStream(DeviceStream); + DeviceStream = null; + } + + SDL_AudioSpec spec = AudioSpec; + + SDL_AudioStream* deviceStream = SDL_OpenAudioDeviceStream(targetId, &spec, &audioCallback, objectHandle.Handle); + + if (deviceStream != null) + { + SDL_DestroyAudioStream(DeviceStream); + DeviceStream = deviceStream; + AudioSpec = spec; + + DeviceId = SDL_GetAudioStreamDevice(deviceStream); + + int sampleFrameSize = 0; + SDL_AudioSpec temp; // this has 'real' device info which is useless since SDL converts audio according to the spec we provided + if (SDL_GetAudioDeviceFormat(DeviceId, &temp, &sampleFrameSize)) + BufferSize = sampleFrameSize * (int)Math.Ceiling((double)spec.freq / temp.freq); + } + + if (deviceStream == null) + { + if (targetId == SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK) + return false; + + return SetAudioDevice(SDL_AUDIO_DEVICE_DEFAULT_PLAYBACK); + } + + SDL_ResumeAudioDevice(DeviceId); + + DeviceName = SDL_GetAudioDeviceName(targetId); + + return true; + } + + [UnmanagedCallersOnly(CallConvs = new[] { typeof(CallConvCdecl) })] + private static void audioCallback(IntPtr userdata, SDL_AudioStream* stream, int additionalAmount, int totalAmount) + { + var handle = new ObjectHandle(userdata); + if (handle.GetTarget(out SDL3BaseAudioManager audioManager)) + audioManager.internalAudioCallback(stream, additionalAmount); + } + + private float[] audioBuffer; + + private void internalAudioCallback(SDL_AudioStream* stream, int additionalAmount) + { + additionalAmount /= 4; + + if (audioBuffer == null || audioBuffer.Length < additionalAmount) + audioBuffer = new float[additionalAmount]; + + Array.Fill(audioBuffer, 0); + + try + { + foreach (var mixer in mixerIterator()) + { + if (mixer.IsAlive) + mixer.MixChannelsInto(audioBuffer, additionalAmount); + } + + fixed (float* ptr = audioBuffer) + SDL_PutAudioStreamData(stream, (IntPtr)ptr, additionalAmount * 4); + } + catch (Exception e) + { + Logger.Error(e, "Error while pushing audio to SDL"); + } + } + + /// + /// With how decoders work, we need this to get test passed + /// I don't want this either... otherwise we have to dispose decoder in tests + /// + private class ReceiverGCWrapper : SDL3AudioDecoderManager.ISDL3AudioDataReceiver + { + private readonly WeakReference channelWeakReference; + + internal ReceiverGCWrapper(WeakReference channel) + { + channelWeakReference = channel; + } + + void SDL3AudioDecoderManager.ISDL3AudioDataReceiver.GetData(byte[] data, int length, bool done) + { + if (channelWeakReference.TryGetTarget(out SDL3AudioDecoderManager.ISDL3AudioDataReceiver r)) + r.GetData(data, length, done); + else + throw new ObjectDisposedException("channel is already disposed"); + } + + void SDL3AudioDecoderManager.ISDL3AudioDataReceiver.GetMetaData(int bitrate, double length, long byteLength) + { + if (channelWeakReference.TryGetTarget(out SDL3AudioDecoderManager.ISDL3AudioDataReceiver r)) + r.GetMetaData(bitrate, length, byteLength); + else + throw new ObjectDisposedException("channel is already disposed"); + } + } + + internal Track.Track GetNewTrack(Stream data, string name) + { + TrackSDL3 track = new TrackSDL3(name, AudioSpec, BufferSize); + ReceiverGCWrapper receiverGC = new ReceiverGCWrapper(new WeakReference(track)); + decoderManager.StartDecodingAsync(data, AudioSpec, true, receiverGC); + return track; + } + + internal SampleFactory GetSampleFactory(Stream data, string name, AudioMixer mixer, int playbackConcurrency) + { + SampleSDL3Factory sampleFactory = new SampleSDL3Factory(name, (SDL3AudioMixer)mixer, playbackConcurrency, AudioSpec); + ReceiverGCWrapper receiverGC = new ReceiverGCWrapper(new WeakReference(sampleFactory)); + decoderManager.StartDecodingAsync(data, AudioSpec, false, receiverGC); + return sampleFactory; + } + + public void Dispose() + { + if (DeviceStream != null) + { + SDL_DestroyAudioStream(DeviceStream); + DeviceStream = null; + DeviceId = 0; + // Destroying audio stream will close audio device because we use SDL3 OpenAudioDeviceStream + // won't use multiple AudioStream for now since it's barely useful + } + + objectHandle.Dispose(); + decoderManager.Dispose(); + + SDL_QuitSubSystem(SDL_InitFlags.SDL_INIT_AUDIO); + } + } + } +} diff --git a/osu.Framework/Audio/Sample/Sample.cs b/osu.Framework/Audio/Sample/Sample.cs index 54bd3f1174..dcfcccf228 100644 --- a/osu.Framework/Audio/Sample/Sample.cs +++ b/osu.Framework/Audio/Sample/Sample.cs @@ -14,6 +14,12 @@ public abstract class Sample : AudioCollectionManager, ISample public string Name { get; } + internal Sample(SampleFactory factory) + : this(factory.Name) + { + PlaybackConcurrency.BindTo(factory.PlaybackConcurrency); + } + protected Sample(string name) { Name = name; diff --git a/osu.Framework/Audio/Sample/SampleBass.cs b/osu.Framework/Audio/Sample/SampleBass.cs index 5052a2cd5e..0f80ceabec 100644 --- a/osu.Framework/Audio/Sample/SampleBass.cs +++ b/osu.Framework/Audio/Sample/SampleBass.cs @@ -15,12 +15,10 @@ internal sealed class SampleBass : Sample private readonly BassAudioMixer mixer; internal SampleBass(SampleBassFactory factory, BassAudioMixer mixer) - : base(factory.Name) + : base(factory) { this.factory = factory; this.mixer = mixer; - - PlaybackConcurrency.BindTo(factory.PlaybackConcurrency); } protected override SampleChannel CreateChannel() diff --git a/osu.Framework/Audio/Sample/SampleBassFactory.cs b/osu.Framework/Audio/Sample/SampleBassFactory.cs index 3b863a68b3..12f0fc8531 100644 --- a/osu.Framework/Audio/Sample/SampleBassFactory.cs +++ b/osu.Framework/Audio/Sample/SampleBassFactory.cs @@ -14,42 +14,28 @@ namespace osu.Framework.Audio.Sample /// /// A factory for objects sharing a common sample ID (and thus playback concurrency). /// - internal class SampleBassFactory : AudioCollectionManager + internal class SampleBassFactory : SampleFactory { - /// - /// A name identifying the sample to be created by this factory. - /// - public string Name { get; } - public int SampleId { get; private set; } public override bool IsLoaded => SampleId != 0; - public double Length { get; private set; } - - /// - /// Todo: Expose this to support per-sample playback concurrency once ManagedBass has been updated (https://github.com/ManagedBass/ManagedBass/pull/85). - /// - internal readonly Bindable PlaybackConcurrency = new Bindable(Sample.DEFAULT_CONCURRENCY); - private readonly BassAudioMixer mixer; private NativeMemoryTracker.NativeMemoryLease? memoryLease; + private byte[]? data; - public SampleBassFactory(byte[] data, string name, BassAudioMixer mixer) + public SampleBassFactory(byte[] data, string name, BassAudioMixer mixer, int playbackConcurrency) + : base(name, playbackConcurrency) { this.data = data; this.mixer = mixer; - Name = name; - EnqueueAction(loadSample); - - PlaybackConcurrency.BindValueChanged(updatePlaybackConcurrency); } - private void updatePlaybackConcurrency(ValueChangedEvent concurrency) + protected override void UpdatePlaybackConcurrency(ValueChangedEvent concurrency) { EnqueueAction(() => { @@ -87,7 +73,6 @@ private void loadSample() if (Bass.LastError == Errors.Init) return; - // We've done as best as we could to init the sample. It may still have failed by some other cause (such as malformed data), but allow the GC to now clean up the locally-stored data. data = null; if (!IsLoaded) @@ -97,12 +82,7 @@ private void loadSample() memoryLease = NativeMemoryTracker.AddMemory(this, dataLength); } - public Sample CreateSample() => new SampleBass(this, mixer) { OnPlay = onPlay }; - - private void onPlay(Sample sample) - { - AddItem(sample); - } + public override Sample CreateSample() => new SampleBass(this, mixer) { OnPlay = SampleFactoryOnPlay }; ~SampleBassFactory() { diff --git a/osu.Framework/Audio/Sample/SampleChannelSDL3.cs b/osu.Framework/Audio/Sample/SampleChannelSDL3.cs new file mode 100644 index 0000000000..76ebaefa62 --- /dev/null +++ b/osu.Framework/Audio/Sample/SampleChannelSDL3.cs @@ -0,0 +1,97 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System.Threading; +using osu.Framework.Audio.Mixing.SDL3; + +namespace osu.Framework.Audio.Sample +{ + internal sealed class SampleChannelSDL3 : SampleChannel, ISDL3AudioChannel + { + private readonly SampleSDL3AudioPlayer player; + + private volatile bool playing; + public override bool Playing => playing; + + private volatile bool looping; + public override bool Looping { get => looping; set => looping = value; } + + public SampleChannelSDL3(SampleSDL3 sample, SampleSDL3AudioPlayer player) + : base(sample.Name) + { + this.player = player; + } + + public override void Play() + { + if (started) + return; + + // ensure state is correct before starting. + InvalidateState(); + + playing = true; + base.Play(); + } + + public override void Stop() + { + playing = false; + started = false; + base.Stop(); + } + + private volatile bool started; + + int ISDL3AudioChannel.GetRemainingSamples(float[] data) + { + if (player.RelativeRate != rate) + player.RelativeRate = rate; + + if (player.Loop != looping) + player.Loop = looping; + + if (!started) + { + player.Reset(); + started = true; + } + + int ret = player.GetRemainingSamples(data); + + if (player.Done) + { + playing = false; + } + + return ret; + } + + private (float, float) volume = (1.0f, 1.0f); + + private double rate = 1.0f; + + internal override void OnStateChanged() + { + base.OnStateChanged(); + + volume = ((float, float))Adjustments.GetAggregatedStereoVolume(); + + Interlocked.Exchange(ref rate, AggregateFrequency.Value); + } + + (float, float) ISDL3AudioChannel.Volume => volume; + + bool ISDL3AudioChannel.Playing => playing; + + protected override void Dispose(bool disposing) + { + if (IsDisposed) + return; + + (Mixer as SDL3AudioMixer)?.StreamFree(this); + + base.Dispose(disposing); + } + } +} diff --git a/osu.Framework/Audio/Sample/SampleFactory.cs b/osu.Framework/Audio/Sample/SampleFactory.cs new file mode 100644 index 0000000000..a868e71df8 --- /dev/null +++ b/osu.Framework/Audio/Sample/SampleFactory.cs @@ -0,0 +1,42 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using osu.Framework.Bindables; + +namespace osu.Framework.Audio.Sample +{ + /// + /// A factory for objects sharing a common sample ID (and thus playback concurrency). + /// + internal abstract class SampleFactory : AudioCollectionManager + { + /// + /// A name identifying the sample to be created by this factory. + /// + public string Name { get; } + + public double Length { get; private protected set; } + + /// + /// Todo: Expose this to support per-sample playback concurrency once ManagedBass has been updated (https://github.com/ManagedBass/ManagedBass/pull/85). + /// + internal readonly Bindable PlaybackConcurrency = new Bindable(Sample.DEFAULT_CONCURRENCY); + + protected SampleFactory(string name, int playbackConcurrency) + { + Name = name; + PlaybackConcurrency.Value = playbackConcurrency; + + PlaybackConcurrency.BindValueChanged(UpdatePlaybackConcurrency); + } + + protected abstract void UpdatePlaybackConcurrency(ValueChangedEvent concurrency); + + public abstract Sample CreateSample(); + + protected void SampleFactoryOnPlay(Sample sample) + { + AddItem(sample); + } + } +} diff --git a/osu.Framework/Audio/Sample/SampleSDL3.cs b/osu.Framework/Audio/Sample/SampleSDL3.cs new file mode 100644 index 0000000000..a8edac0c05 --- /dev/null +++ b/osu.Framework/Audio/Sample/SampleSDL3.cs @@ -0,0 +1,29 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using osu.Framework.Audio.Mixing.SDL3; + +namespace osu.Framework.Audio.Sample +{ + internal sealed class SampleSDL3 : Sample + { + public override bool IsLoaded => factory.IsLoaded; + + private readonly SampleSDL3Factory factory; + private readonly SDL3AudioMixer mixer; + + public SampleSDL3(SampleSDL3Factory factory, SDL3AudioMixer mixer) + : base(factory) + { + this.factory = factory; + this.mixer = mixer; + } + + protected override SampleChannel CreateChannel() + { + var channel = new SampleChannelSDL3(this, factory.CreatePlayer()); + mixer.Add(channel); + return channel; + } + } +} diff --git a/osu.Framework/Audio/Sample/SampleSDL3AudioPlayer.cs b/osu.Framework/Audio/Sample/SampleSDL3AudioPlayer.cs new file mode 100644 index 0000000000..c75f1c59dd --- /dev/null +++ b/osu.Framework/Audio/Sample/SampleSDL3AudioPlayer.cs @@ -0,0 +1,68 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; + +namespace osu.Framework.Audio.Sample +{ + internal class SampleSDL3AudioPlayer : ResamplingPlayer + { + private int position; + + private volatile bool done; + public bool Done => done; + + private readonly float[] audioData; + + public bool Loop { get; set; } + + public SampleSDL3AudioPlayer(float[] audioData, int rate, int channels) + : base(rate, channels) + { + this.audioData = audioData; + } + + protected override int GetRemainingRawFloats(float[] data, int offset, int needed) + { + if (audioData.Length <= 0) + { + done = true; + return 0; + } + + int i = 0; + + while (i < needed) + { + int put = Math.Min(needed - i, audioData.Length - position); + + if (put > 0) + Array.Copy(audioData, position, data, offset + i, put); + + i += put; + position += put; + + // done playing + if (position >= audioData.Length) + { + if (Loop) // back to start if looping + position = 0; + else + { + done = true; + break; + } + } + } + + return i; + } + + public void Reset(bool resetIndex = true) + { + done = false; + if (resetIndex) + position = 0; + } + } +} diff --git a/osu.Framework/Audio/Sample/SampleSDL3Factory.cs b/osu.Framework/Audio/Sample/SampleSDL3Factory.cs new file mode 100644 index 0000000000..0d1d59341b --- /dev/null +++ b/osu.Framework/Audio/Sample/SampleSDL3Factory.cs @@ -0,0 +1,82 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Threading; +using osu.Framework.Audio.Mixing.SDL3; +using osu.Framework.Bindables; +using SDL; + +namespace osu.Framework.Audio.Sample +{ + internal class SampleSDL3Factory : SampleFactory, SDL3AudioDecoderManager.ISDL3AudioDataReceiver + { + private volatile bool isLoaded; + public override bool IsLoaded => isLoaded; + + private readonly SDL3AudioMixer mixer; + private readonly SDL_AudioSpec spec; + + private float[] decodedAudio = Array.Empty(); + + private readonly AutoResetEvent completion = new AutoResetEvent(false); + + public SampleSDL3Factory(string name, SDL3AudioMixer mixer, int playbackConcurrency, SDL_AudioSpec spec) + : base(name, playbackConcurrency) + { + this.mixer = mixer; + this.spec = spec; + } + + void SDL3AudioDecoderManager.ISDL3AudioDataReceiver.GetData(byte[] audio, int byteLen, bool done) + { + if (IsDisposed) + return; + + if (byteLen > 0) + { + decodedAudio = new float[byteLen / 4]; + Buffer.BlockCopy(audio, 0, decodedAudio, 0, byteLen); + } + + Length = byteLen / 4d / spec.freq / spec.channels * 1000d; + isLoaded = true; + + completion.Set(); + } + + public SampleSDL3AudioPlayer CreatePlayer() + { + if (!isLoaded) + completion.WaitOne(); // may cause deadlock in bad situation, but needed to get tests passed + + return new SampleSDL3AudioPlayer(decodedAudio, spec.freq, spec.channels); + } + + public override Sample CreateSample() => new SampleSDL3(this, mixer) { OnPlay = SampleFactoryOnPlay }; + + protected override void UpdatePlaybackConcurrency(ValueChangedEvent concurrency) + { + } + + ~SampleSDL3Factory() + { + Dispose(false); + } + + protected override void Dispose(bool disposing) + { + if (IsDisposed) + return; + + decodedAudio = Array.Empty(); + + completion.Dispose(); + base.Dispose(disposing); + } + + void SDL3AudioDecoderManager.ISDL3AudioDataReceiver.GetMetaData(int bitrate, double length, long byteLength) + { + } // not needed + } +} diff --git a/osu.Framework/Audio/Sample/SampleStore.cs b/osu.Framework/Audio/Sample/SampleStore.cs index a47a9f15af..2cd9b2daaa 100644 --- a/osu.Framework/Audio/Sample/SampleStore.cs +++ b/osu.Framework/Audio/Sample/SampleStore.cs @@ -10,7 +10,6 @@ using System.Threading.Tasks; using JetBrains.Annotations; using osu.Framework.Audio.Mixing; -using osu.Framework.Audio.Mixing.Bass; using osu.Framework.IO.Stores; using osu.Framework.Statistics; @@ -20,15 +19,19 @@ internal class SampleStore : AudioCollectionManager, I { private readonly ResourceStore store; private readonly AudioMixer mixer; + private readonly GetSampleFactoryDelegate getSampleFactoryDelegate; - private readonly Dictionary factories = new Dictionary(); + private readonly Dictionary factories = new Dictionary(); + + public delegate SampleFactory GetSampleFactoryDelegate(Stream stream, string name, AudioMixer mixer, int playbackConcurrency); public int PlaybackConcurrency { get; set; } = Sample.DEFAULT_CONCURRENCY; - internal SampleStore([NotNull] IResourceStore store, [NotNull] AudioMixer mixer) + internal SampleStore([NotNull] IResourceStore store, [NotNull] AudioMixer mixer, [NotNull] GetSampleFactoryDelegate getSampleFactoryDelegate) { this.store = new ResourceStore(store); this.mixer = mixer; + this.getSampleFactoryDelegate = getSampleFactoryDelegate; AddExtension(@"wav"); AddExtension(@"mp3"); @@ -44,12 +47,12 @@ public Sample Get(string name) lock (factories) { - if (!factories.TryGetValue(name, out SampleBassFactory factory)) + if (!factories.TryGetValue(name, out SampleFactory factory)) { this.LogIfNonBackgroundThread(name); - byte[] data = store.Get(name); - factory = factories[name] = data == null ? null : new SampleBassFactory(data, name, (BassAudioMixer)mixer) { PlaybackConcurrency = { Value = PlaybackConcurrency } }; + Stream data = store.GetStream(name); + factory = factories[name] = data == null ? null : getSampleFactoryDelegate(data, name, mixer, PlaybackConcurrency); if (factory != null) AddItem(factory); diff --git a/osu.Framework/Audio/Track/TempoSDL3AudioPlayer.cs b/osu.Framework/Audio/Track/TempoSDL3AudioPlayer.cs new file mode 100644 index 0000000000..6472cac0f0 --- /dev/null +++ b/osu.Framework/Audio/Track/TempoSDL3AudioPlayer.cs @@ -0,0 +1,176 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using SoundTouch; + +namespace osu.Framework.Audio.Track +{ + internal class TempoSDL3AudioPlayer : TrackSDL3AudioPlayer + { + private SoundTouchProcessor? soundTouch; + + private double tempo = 1; + + /// + /// Represents current speed. + /// + public double Tempo + { + get => tempo; + set => setTempo(value); + } + + private readonly int samplesize; + + private bool doneFilling; + private bool donePlaying; + + public override bool Done => base.Done && (soundTouch == null || donePlaying); + + /// + /// Creates a new . + /// + /// + /// + /// will prepare this amount of samples (or more) on every update. + public TempoSDL3AudioPlayer(int rate, int channels, int samples) + : base(rate, channels) + { + samplesize = samples; + } + + public void FillRequiredSamples() => fillSamples(samplesize); + + /// + /// Fills SoundTouch buffer until it has a specific amount of samples. + /// + /// Needed sample count + private void fillSamples(int samples) + { + if (soundTouch == null || tempo == 1.0f) + return; + + while (!base.Done && soundTouch.AvailableSamples < samples) + { + // process 10ms more to reduce overhead + int getSamples = (int)Math.Ceiling((samples + (int)(SrcRate * 0.01) - soundTouch.AvailableSamples) * Tempo) * SrcChannels; + float[] src = new float[getSamples]; + getSamples = base.GetRemainingSamples(src); + if (getSamples <= 0) + break; + + soundTouch.PutSamples(src, getSamples / SrcChannels); + } + + if (!doneFilling && base.Done) + { + soundTouch.Flush(); + doneFilling = true; + } + } + + /// + /// Sets tempo. This initializes if it's set to some value else than 1.0, and once it's set again to 1.0, it disposes . + /// + /// New tempo value + private void setTempo(double tempo) + { + if (soundTouch == null && tempo == 1.0f) + return; + + if (soundTouch == null) + { + soundTouch = new SoundTouchProcessor + { + SampleRate = SrcRate, + Channels = SrcChannels + }; + soundTouch.SetSetting(SettingId.UseQuickSeek, 1); + soundTouch.SetSetting(SettingId.OverlapDurationMs, 4); + soundTouch.SetSetting(SettingId.SequenceDurationMs, 30); + } + + if (this.tempo != tempo) + { + this.tempo = tempo; + + if (Tempo == 1.0f) + { + if (AudioData != null) + { + int latency = GetTempoLatencyInSamples() * SrcChannels; + long temp = !ReversePlayback ? AudioDataPosition - latency : AudioDataPosition + latency; + + if (temp >= 0) + AudioDataPosition = temp; + } + } + else + { + double tempochange = Math.Clamp((Math.Abs(tempo) - 1.0d) * 100.0d, -95, 5000); + soundTouch.TempoChange = tempochange; + } + + Clear(); + } + } + + /// + /// Returns tempo and rate adjusted audio samples. It calls a parent method if is 1. + /// + /// An array to put samples in + /// The number of samples put + public override int GetRemainingSamples(float[] ret) + { + if (soundTouch == null || tempo == 1.0f) + return base.GetRemainingSamples(ret); + + if (RelativeRate == 0) + return 0; + + int expected = ret.Length / SrcChannels; + + if (!doneFilling && soundTouch.AvailableSamples < expected) + fillSamples(expected); + + int got = soundTouch.ReceiveSamples(ret, expected); + + if (got == 0 && doneFilling) + donePlaying = true; + + return got * SrcChannels; + } + + public override void Reset(bool resetPosition = true) + { + base.Reset(resetPosition); + + doneFilling = false; + donePlaying = false; + } + + protected int GetTempoLatencyInSamples() + { + if (soundTouch == null) + return 0; + + return (int)(soundTouch.UnprocessedSampleCount + (soundTouch.AvailableSamples * Tempo)); + } + + protected override double GetProcessingLatency() => base.GetProcessingLatency() + (GetTempoLatencyInSamples() * 1000.0 / SrcRate); + + public override void Clear() + { + base.Clear(); + soundTouch?.Clear(); + } + + public override void Seek(double seek) + { + base.Seek(seek); + if (soundTouch != null) + FillRequiredSamples(); + } + } +} diff --git a/osu.Framework/Audio/Track/TrackSDL3.cs b/osu.Framework/Audio/Track/TrackSDL3.cs new file mode 100644 index 0000000000..3f5b233a3a --- /dev/null +++ b/osu.Framework/Audio/Track/TrackSDL3.cs @@ -0,0 +1,269 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Threading; +using System.Threading.Tasks; +using osu.Framework.Audio.Mixing.SDL3; +using osu.Framework.Extensions; +using osu.Framework.Logging; +using SDL; + +namespace osu.Framework.Audio.Track +{ + public sealed class TrackSDL3 : Track, ISDL3AudioChannel, SDL3AudioDecoderManager.ISDL3AudioDataReceiver + { + private readonly TempoSDL3AudioPlayer player; + + public override bool IsDummyDevice => false; + + private volatile bool isLoaded; + public override bool IsLoaded => isLoaded; + + private volatile bool isCompletelyLoaded; + + /// + /// Audio can be played without interrupt once it's set to true. means that it at least has 'some' data to play. + /// + public bool IsCompletelyLoaded => isCompletelyLoaded; + + private double currentTime; + public override double CurrentTime => currentTime; + + private volatile bool isRunning; + public override bool IsRunning => isRunning; + + private volatile bool hasCompleted; + public override bool HasCompleted => hasCompleted; + + private volatile int bitrate; + public override int? Bitrate => bitrate; + + public TrackSDL3(string name, SDL_AudioSpec spec, int samples) + : base(name) + { + // SoundTouch limitation + const float tempo_minimum_supported = 0.05f; + AggregateTempo.ValueChanged += t => + { + if (t.NewValue < tempo_minimum_supported) + throw new ArgumentException($"{nameof(TrackSDL3)} does not support {nameof(Tempo)} specifications below {tempo_minimum_supported}. Use {nameof(Frequency)} instead."); + }; + + player = new TempoSDL3AudioPlayer(spec.freq, spec.channels, samples); + } + + private readonly object syncRoot = new object(); + + void SDL3AudioDecoderManager.ISDL3AudioDataReceiver.GetData(byte[] audio, int length, bool done) + { + if (IsDisposed) + return; + + lock (syncRoot) + { + if (!player.IsLoaded) + { + if (!player.IsLoading) + { + Logger.Log("GetMetaData should be called first, falling back to default buffer size", level: LogLevel.Important); + player.PrepareStream(); + } + + player.PutSamplesInStream(audio, length); + + if (done) + { + player.DonePutting(); + Length = player.AudioLength; + isCompletelyLoaded = true; + } + } + } + } + + void SDL3AudioDecoderManager.ISDL3AudioDataReceiver.GetMetaData(int bitrate, double length, long byteLength) + { + if (!isLoaded) + { + Length = length; + this.bitrate = bitrate; + + lock (syncRoot) + { + if (!player.IsLoading) + player.PrepareStream(byteLength); + } + + isLoaded = true; + } + } + + private double lastTime; + private float[]? samples; + + private SDL3AmplitudeProcessor? amplitudeProcessor; + + public override ChannelAmplitudes CurrentAmplitudes => (amplitudeProcessor ??= new SDL3AmplitudeProcessor()).CurrentAmplitudes; + + protected override void UpdateState() + { + base.UpdateState(); + + if (player.Done && isRunning) + { + if (Looping) + { + seekInternal(RestartPoint); + } + else + { + isRunning = false; + hasCompleted = true; + RaiseCompleted(); + } + } + + if (AggregateTempo.Value != 1 && isRunning) + { + lock (syncRoot) + player.FillRequiredSamples(); + } + + if (amplitudeProcessor != null && isRunning && Math.Abs(currentTime - lastTime) > 1000.0 / 60.0) + { + lastTime = currentTime; + samples ??= new float[(int)(player.SrcRate * (1f / 60)) * player.SrcChannels]; + player.Peek(samples, lastTime); + + amplitudeProcessor.Update(samples, player.SrcChannels); + } + } + + public override bool Seek(double seek) => SeekAsync(seek).GetResultSafely(); + + public override async Task SeekAsync(double seek) + { + double conservativeLength = Length == 0 ? double.MaxValue : Length; + double conservativeClamped = Math.Clamp(seek, 0, conservativeLength); + + await EnqueueAction(() => seekInternal(seek)).ConfigureAwait(false); + + return conservativeClamped == seek; + } + + private void seekInternal(double seek) + { + double time; + + lock (syncRoot) + { + player.Seek(seek); + + if (seek < Length) + { + player.Reset(false); + hasCompleted = false; + } + + time = player.GetCurrentTime(); + } + + Interlocked.Exchange(ref currentTime, time); + } + + public override void Start() + { + if (IsDisposed) + throw new ObjectDisposedException(ToString(), "Can not start disposed tracks."); + + StartAsync().WaitSafely(); + } + + public override Task StartAsync() => EnqueueAction(() => + { + // ensure state is correct before starting. + InvalidateState(); + + lock (syncRoot) + player.Reset(false); + + isRunning = true; + hasCompleted = false; + }); + + public override void Stop() => StopAsync().WaitSafely(); + + public override Task StopAsync() => EnqueueAction(() => + { + isRunning = false; + }); + + int ISDL3AudioChannel.GetRemainingSamples(float[] data) + { + if (!IsLoaded) return 0; + + int ret; + double time; + + lock (syncRoot) + { + ret = player.GetRemainingSamples(data); + time = player.GetCurrentTime(); + } + + Interlocked.Exchange(ref currentTime, time); + + if (ret < 0) + { + EnqueueAction(RaiseFailed); + return 0; + } + + return ret; + } + + private (float, float) volume = (1.0f, 1.0f); + + internal override void OnStateChanged() + { + base.OnStateChanged(); + + lock (syncRoot) + { + if (!player.ReversePlayback && AggregateFrequency.Value < 0) + player.ReversePlayback = true; + else if (player.ReversePlayback && AggregateFrequency.Value >= 0) + player.ReversePlayback = false; + + player.RelativeRate = Math.Abs(AggregateFrequency.Value); + player.Tempo = AggregateTempo.Value; + } + + volume = ((float, float))Adjustments.GetAggregatedStereoVolume(); + } + + bool ISDL3AudioChannel.Playing => isRunning && !player.Done; + + (float, float) ISDL3AudioChannel.Volume => volume; + + ~TrackSDL3() + { + Dispose(false); + } + + protected override void Dispose(bool disposing) + { + if (IsDisposed) + return; + + isRunning = false; + (Mixer as SDL3AudioMixer)?.StreamFree(this); + + lock (syncRoot) + player.Dispose(); + + base.Dispose(disposing); + } + } +} diff --git a/osu.Framework/Audio/Track/TrackSDL3AudioPlayer.cs b/osu.Framework/Audio/Track/TrackSDL3AudioPlayer.cs new file mode 100644 index 0000000000..03b73a4036 --- /dev/null +++ b/osu.Framework/Audio/Track/TrackSDL3AudioPlayer.cs @@ -0,0 +1,286 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +using System; +using System.Threading; + +namespace osu.Framework.Audio.Track +{ + /// + /// Mainly returns audio data to . + /// + internal class TrackSDL3AudioPlayer : ResamplingPlayer, IDisposable + { + private volatile bool isLoaded; + public bool IsLoaded => isLoaded; + + private volatile bool isLoading; + public bool IsLoading => isLoading; + + private volatile bool done; + public virtual bool Done => done; + + /// + /// Returns a data position converted into milliseconds with configuration set for this player. + /// + /// Position to convert + /// + public double GetMsFromIndex(long pos) => pos * 1000.0d / SrcRate / SrcChannels; + + /// + /// Returns a position in milliseconds converted from a byte position with configuration set for this player. + /// + /// A position in milliseconds to convert + /// + public long GetIndexFromMs(double seconds) => (long)Math.Ceiling(seconds / 1000.0d * SrcRate) * SrcChannels; + + /// + /// Stores raw audio data. + /// + protected float[]? AudioData; + + protected long AudioDataPosition; + + private long audioDataLength; + + public double AudioLength => GetMsFromIndex(audioDataLength); + + /// + /// Play backwards if set to true. + /// + public bool ReversePlayback { get; set; } + + /// + /// Creates a new . Use if you want to adjust tempo. + /// + /// Sampling rate of audio + /// Channels of audio + public TrackSDL3AudioPlayer(int rate, int channels) + : base(rate, channels) + { + isLoading = false; + isLoaded = false; + } + + private void prepareArray(long wanted) + { + if (wanted <= AudioData?.LongLength) + return; + + float[] temp = new float[wanted]; + + if (AudioData != null) + Array.Copy(AudioData, 0, temp, 0, audioDataLength); + + AudioData = temp; + } + + internal void PrepareStream(long byteLength = 3 * 60 * 44100 * 2 * 4) + { + if (isDisposed) + return; + + if (AudioData == null) + prepareArray(byteLength / 4); + + isLoading = true; + } + + internal void PutSamplesInStream(byte[] next, int length) + { + if (isDisposed) + return; + + if (AudioData == null) + throw new InvalidOperationException($"Use {nameof(PrepareStream)} before calling this"); + + int floatLen = length / sizeof(float); + long currentLen = audioDataLength; + + if (currentLen + floatLen > AudioData.LongLength) + prepareArray(currentLen + floatLen); + + for (int i = 0; i < floatLen; i++) + { + float src = BitConverter.ToSingle(next, i * sizeof(float)); + AudioData[currentLen++] = src; + } + + Interlocked.Exchange(ref audioDataLength, currentLen); + } + + internal void DonePutting() + { + if (isDisposed) + return; + + // Saved seek was over data length + if (SaveSeek > audioDataLength) + SaveSeek = 0; + + isLoading = false; + isLoaded = true; + } + + protected override int GetRemainingRawFloats(float[] data, int offset, int needed) + { + if (AudioData == null) + return 0; + + if (audioDataLength <= 0) + { + done = true; + return 0; + } + + if (SaveSeek > 0) + { + // set to 0 if position is over saved seek + if (AudioDataPosition > SaveSeek) + SaveSeek = 0; + + // player now has audio data to play + if (audioDataLength > SaveSeek) + { + AudioDataPosition = SaveSeek; + SaveSeek = 0; + } + + // if player didn't reach the position, don't play + if (SaveSeek > 0) + return 0; + } + + int read; + + if (ReversePlayback) + { + for (read = 0; read < needed; read += 2) + { + if (AudioDataPosition < 0) + { + AudioDataPosition = 0; + break; + } + + // swap stereo channel + data[read + 1 + offset] = AudioData[AudioDataPosition--]; + data[read + offset] = AudioData[AudioDataPosition--]; + } + } + else + { + long remain = audioDataLength - AudioDataPosition; + read = (int)Math.Min(needed, remain); + + Array.Copy(AudioData, AudioDataPosition, data, offset, read); + AudioDataPosition += read; + } + + if (ReversePlayback ? AudioDataPosition <= 0 : AudioDataPosition >= audioDataLength && !isLoading) + done = true; + + return read; + } + + /// + /// Puts recently played audio samples into data. Mostly used to calculate amplitude of a track. + /// + /// A float array to put data in + /// + /// True if succeeded + public bool Peek(float[] data, double posMs) + { + if (AudioData == null) + return false; + + long pos = GetIndexFromMs(posMs); + long len = Interlocked.Read(ref audioDataLength); + + long start = Math.Clamp(pos, 0, len); + long remain = len - start; + + Array.Copy(AudioData, start, data, 0, Math.Min(data.Length, remain)); + return true; + } + + /// + /// Clears 'done' status. + /// + /// Goes back to the start if set to true. + public virtual void Reset(bool resetPosition = true) + { + done = false; + + if (resetPosition) + { + SaveSeek = 0; + Seek(0); + } + + Clear(); + } + + /// + /// Returns current position converted into milliseconds. + /// + public double GetCurrentTime() + { + if (SaveSeek > 0) + return GetMsFromIndex(SaveSeek); + + if (AudioData == null) + return 0; + + return !ReversePlayback + ? GetMsFromIndex(AudioDataPosition) - GetProcessingLatency() + : GetMsFromIndex(AudioDataPosition) + GetProcessingLatency(); + } + + protected long SaveSeek; + + /// + /// Sets the position of this player. + /// If the given value is over current , it will be saved and pause playback until decoding reaches the position. + /// However, if the value is still over after the decoding is over, it will be discarded. + /// + /// Position in milliseconds + public virtual void Seek(double seek) + { + long tmp = GetIndexFromMs(seek); + + if (!isLoaded && tmp > audioDataLength) + { + SaveSeek = tmp; + } + else if (AudioData != null) + { + SaveSeek = 0; + AudioDataPosition = Math.Clamp(tmp, 0, Math.Max(0, audioDataLength - 1)); + Clear(); + } + } + + private volatile bool isDisposed; + + protected virtual void Dispose(bool disposing) + { + if (!isDisposed) + { + AudioData = null; + isDisposed = true; + } + } + + ~TrackSDL3AudioPlayer() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + } +} diff --git a/osu.Framework/Audio/Track/TrackStore.cs b/osu.Framework/Audio/Track/TrackStore.cs index 495e3f0713..8d6a19276f 100644 --- a/osu.Framework/Audio/Track/TrackStore.cs +++ b/osu.Framework/Audio/Track/TrackStore.cs @@ -18,11 +18,15 @@ internal class TrackStore : AudioCollectionManager, IT { private readonly IResourceStore store; private readonly AudioMixer mixer; + private readonly GetNewTrackDelegate getNewTrackDelegate; - internal TrackStore([NotNull] IResourceStore store, [NotNull] AudioMixer mixer) + public delegate Track GetNewTrackDelegate(Stream dataStream, string name); + + internal TrackStore([NotNull] IResourceStore store, [NotNull] AudioMixer mixer, [NotNull] GetNewTrackDelegate getNewTrackDelegate) { this.store = store; this.mixer = mixer; + this.getNewTrackDelegate = getNewTrackDelegate; (store as ResourceStore)?.AddExtension(@"mp3"); } @@ -47,12 +51,12 @@ public Track Get(string name) if (dataStream == null) return null; - TrackBass trackBass = new TrackBass(dataStream, name); + Track track = getNewTrackDelegate(dataStream, name); - mixer.Add(trackBass); - AddItem(trackBass); + mixer.Add(track); + AddItem(track); - return trackBass; + return track; } public Task GetAsync(string name, CancellationToken cancellationToken = default) => diff --git a/osu.Framework/Audio/Track/Waveform.cs b/osu.Framework/Audio/Track/Waveform.cs index e38da672bc..fc51063306 100644 --- a/osu.Framework/Audio/Track/Waveform.cs +++ b/osu.Framework/Audio/Track/Waveform.cs @@ -3,15 +3,14 @@ using System; using System.Buffers; -using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; -using ManagedBass; using osu.Framework.Utils; -using osu.Framework.Audio.Callbacks; using osu.Framework.Extensions; -using osu.Framework.Logging; +using NAudio.Dsp; +using System.Collections.Generic; +using SDL; namespace osu.Framework.Audio.Track { @@ -25,15 +24,10 @@ public class Waveform : IDisposable /// private const float resolution = 0.001f; - /// - /// The data stream is iteratively decoded to provide this many points per iteration so as to not exceed BASS's internal buffer size. - /// - private const int points_per_iteration = 1000; - /// /// FFT1024 gives ~40hz accuracy. /// - private const DataFlags fft_samples = DataFlags.FFT1024; + private const int fft_samples = 1024; /// /// Number of bins generated by the FFT. Must correspond to . @@ -83,182 +77,127 @@ public Waveform(Stream? data) readTask = Task.Run(() => { - if (data == null) - return; + this.data = null; - // for the time being, this code cannot run if there is no bass device available. - if (Bass.CurrentDevice < 0) - { - Logger.Log("Failed to read waveform as no bass device is available."); + if (data == null) return; - } - - FileCallbacks fileCallbacks = new FileCallbacks(new DataStreamFileProcedures(data)); const int bytes_per_sample = 4; + const int sample_rate = 44100; - int decodeStream = Bass.CreateStream(StreamSystem.NoBuffer, BassFlags.Decode | BassFlags.Float, fileCallbacks.Callbacks, fileCallbacks.Handle); + // Code below assumes stereo + channels = 2; - if (decodeStream == 0) + SDL_AudioSpec spec = new SDL_AudioSpec { - logBassError("could not create stream"); - return; - } + freq = sample_rate, + channels = channels, + format = SDL3.SDL_AUDIO_F32 + }; - float[]? sampleBuffer = null; + // AudioDecoder will resample data into specified sample rate and channels (44100hz 2ch float) + SDL3AudioDecoderManager.SDL3AudioDecoder decoder = SDL3AudioDecoderManager.CreateDecoder(data, spec, true, false); + + Complex[] complexBuffer = ArrayPool.Shared.Rent(fft_samples); try { - if (!Bass.ChannelGetInfo(decodeStream, out ChannelInfo info)) - { - logBassError("could not retrieve channel information"); - return; - } - - long length = Bass.ChannelGetLength(decodeStream); - - if (length < 0) - { - logBassError("could not retrieve channel length"); - return; - } - // Each "point" is generated from a number of samples, each sample contains a number of channels - int samplesPerPoint = (int)(info.Frequency * resolution * info.Channels); + int samplesPerPoint = (int)(sample_rate * resolution * channels); - int bytesPerPoint = samplesPerPoint * bytes_per_sample; + // Use List as entire length may be inaccurate + List pointList = new List(); - int pointCount = (int)(length / bytesPerPoint); + int fftPointIndex = 0; - points = new Point[pointCount]; + int complexBufferIndex = 0; - // Each iteration pulls in several samples - int bytesPerIteration = bytesPerPoint * points_per_iteration; + Point point = new Point(); - sampleBuffer = ArrayPool.Shared.Rent(bytesPerIteration / bytes_per_sample); + int pointSamples = 0; - int pointIndex = 0; + int m = (int)Math.Log2(fft_samples); - // Read sample data - while (length > 0) + do { - length = Bass.ChannelGetData(decodeStream, sampleBuffer, bytesPerIteration); + int read = decoder.LoadFromStream(out byte[] currentBytes) / bytes_per_sample; + int sampleIndex = 0; - if (length < 0 && Bass.LastError != Errors.Ended) + while (sampleIndex < read) { - logBassError("could not retrieve sample data"); - return; - } - - int samplesRead = (int)(length / bytes_per_sample); - - // Each point is composed of multiple samples - for (int i = 0; i < samplesRead && pointIndex < pointCount; i += samplesPerPoint) - { - // We assume one or more channels. - // For non-stereo tracks, we'll use the single track for both amplitudes. - // For anything above two tracks we'll use the first and second track. - Debug.Assert(info.Channels >= 1); - int secondChannelIndex = info.Channels > 1 ? 1 : 0; - - // Channels are interleaved in the sample data (data[0] -> channel0, data[1] -> channel1, data[2] -> channel0, etc) - // samplesPerPoint assumes this interleaving behaviour - var point = new Point(); - - for (int j = i; j < i + samplesPerPoint; j += info.Channels) + // Each point is composed of multiple samples + for (; pointSamples < samplesPerPoint && sampleIndex < read; pointSamples += channels, sampleIndex += channels) { // Find the maximum amplitude for each channel in the point - point.AmplitudeLeft = Math.Max(point.AmplitudeLeft, Math.Abs(sampleBuffer[j])); - point.AmplitudeRight = Math.Max(point.AmplitudeRight, Math.Abs(sampleBuffer[j + secondChannelIndex])); - } + float left = BitConverter.ToSingle(currentBytes, sampleIndex * bytes_per_sample); + float right = BitConverter.ToSingle(currentBytes, (sampleIndex + 1) * bytes_per_sample); - // BASS may provide unclipped samples, so clip them ourselves - point.AmplitudeLeft = Math.Min(1, point.AmplitudeLeft); - point.AmplitudeRight = Math.Min(1, point.AmplitudeRight); + point.AmplitudeLeft = Math.Max(point.AmplitudeLeft, Math.Abs(left)); + point.AmplitudeRight = Math.Max(point.AmplitudeRight, Math.Abs(right)); - points[pointIndex++] = point; - } - } + complexBuffer[complexBufferIndex].X = left + right; + complexBuffer[complexBufferIndex].Y = 0; - if (!Bass.ChannelSetPosition(decodeStream, 0)) - { - logBassError("could not reset channel position"); - return; - } - - length = Bass.ChannelGetLength(decodeStream); + if (++complexBufferIndex >= fft_samples) + { + complexBufferIndex = 0; - if (length < 0) - { - logBassError("could not retrieve channel length"); - return; - } + FastFourierTransform.FFT(true, m, complexBuffer); - // Read FFT data - float[] bins = new float[fft_bins]; - int currentPoint = 0; - long currentByte = 0; + point.LowIntensity = computeIntensity(sample_rate, complexBuffer, low_min, mid_min); + point.MidIntensity = computeIntensity(sample_rate, complexBuffer, mid_min, high_min); + point.HighIntensity = computeIntensity(sample_rate, complexBuffer, high_min, high_max); - while (length > 0) - { - length = Bass.ChannelGetData(decodeStream, bins, (int)fft_samples); + for (; fftPointIndex < pointList.Count; fftPointIndex++) + { + var prevPoint = pointList[fftPointIndex]; + prevPoint.LowIntensity = point.LowIntensity; + prevPoint.MidIntensity = point.MidIntensity; + prevPoint.HighIntensity = point.HighIntensity; + pointList[fftPointIndex] = prevPoint; + } - if (length < 0 && Bass.LastError != Errors.Ended) - { - logBassError("could not retrieve FFT data"); - return; - } + fftPointIndex++; // current Point is going to be added + } + } - currentByte += length; + if (pointSamples >= samplesPerPoint) + { + // There may be unclipped samples, so clip them ourselves + point.AmplitudeLeft = Math.Min(1, point.AmplitudeLeft); + point.AmplitudeRight = Math.Min(1, point.AmplitudeRight); - float lowIntensity = computeIntensity(info, bins, low_min, mid_min); - float midIntensity = computeIntensity(info, bins, mid_min, high_min); - float highIntensity = computeIntensity(info, bins, high_min, high_max); + pointList.Add(point); - // In general, the FFT function will read more data than the amount of data we have in one point - // so we'll be setting intensities for all points whose data fits into the amount read by the FFT - // We know that each data point required sampleDataPerPoint amount of data - for (; currentPoint < points.Length && currentPoint * bytesPerPoint < currentByte; currentPoint++) - { - var point = points[currentPoint]; - point.LowIntensity = lowIntensity; - point.MidIntensity = midIntensity; - point.HighIntensity = highIntensity; - points[currentPoint] = point; + point = new Point(); + pointSamples = 0; + } } - } + } while (decoder.Loading); - channels = info.Channels; + points = pointList.ToArray(); } finally { - if (!Bass.StreamFree(decodeStream)) - logBassError("failed to free decode stream"); - - fileCallbacks.Dispose(); + ArrayPool.Shared.Return(complexBuffer); data.Dispose(); - this.data = data = null; - - if (sampleBuffer != null) - ArrayPool.Shared.Return(sampleBuffer); + data = null; } }, cancelSource.Token); - - void logBassError(string reason) => Logger.Log($"BASS failure while reading waveform: {reason} ({Bass.LastError})"); } - private float computeIntensity(ChannelInfo info, float[] bins, float startFrequency, float endFrequency) + private float computeIntensity(int frequency, Complex[] bins, float startFrequency, float endFrequency) { - int startBin = (int)(fft_bins * 2 * startFrequency / info.Frequency); - int endBin = (int)(fft_bins * 2 * endFrequency / info.Frequency); + int startBin = (int)(fft_samples * startFrequency / frequency); + int endBin = (int)(fft_samples * endFrequency / frequency); - startBin = Math.Clamp(startBin, 0, bins.Length); - endBin = Math.Clamp(endBin, 0, bins.Length); + startBin = Math.Clamp(startBin, 0, fft_bins); + endBin = Math.Clamp(endBin, 0, fft_bins); float value = 0; for (int i = startBin; i < endBin; i++) - value += bins[i]; + value += bins[i].ComputeMagnitude(); return value; } diff --git a/osu.Framework/Configuration/AudioDriver.cs b/osu.Framework/Configuration/AudioDriver.cs new file mode 100644 index 0000000000..41a619f08d --- /dev/null +++ b/osu.Framework/Configuration/AudioDriver.cs @@ -0,0 +1,11 @@ +// Copyright (c) ppy Pty Ltd . Licensed under the MIT Licence. +// See the LICENCE file in the repository root for full licence text. + +namespace osu.Framework.Configuration +{ + public enum AudioDriver + { + BASS, + SDL3 + } +} diff --git a/osu.Framework/Configuration/FrameworkConfigManager.cs b/osu.Framework/Configuration/FrameworkConfigManager.cs index c38bd77d69..6d36d19438 100644 --- a/osu.Framework/Configuration/FrameworkConfigManager.cs +++ b/osu.Framework/Configuration/FrameworkConfigManager.cs @@ -31,6 +31,7 @@ protected override void InitialiseDefaults() SetDefault(FrameworkSetting.WindowedPositionX, 0.5, -0.5, 1.5); SetDefault(FrameworkSetting.WindowedPositionY, 0.5, -0.5, 1.5); SetDefault(FrameworkSetting.LastDisplayDevice, DisplayIndex.Default); + SetDefault(FrameworkSetting.AudioDriver, AudioDriver.BASS); SetDefault(FrameworkSetting.AudioDevice, string.Empty); SetDefault(FrameworkSetting.VolumeUniversal, 1.0, 0.0, 1.0, 0.01); SetDefault(FrameworkSetting.VolumeMusic, 1.0, 0.0, 1.0, 0.01); @@ -78,6 +79,7 @@ public enum FrameworkSetting { ShowLogOverlay, + AudioDriver, AudioDevice, VolumeUniversal, VolumeEffect, diff --git a/osu.Framework/Extensions/ExtensionMethods.cs b/osu.Framework/Extensions/ExtensionMethods.cs index 3728936c20..4add4d1e61 100644 --- a/osu.Framework/Extensions/ExtensionMethods.cs +++ b/osu.Framework/Extensions/ExtensionMethods.cs @@ -13,6 +13,7 @@ using System.Reflection; using System.Security.Cryptography; using System.Text; +using NAudio.Dsp; using osu.Framework.Extensions.ObjectExtensions; using osu.Framework.Localisation; @@ -336,5 +337,12 @@ public static bool CheckIsValidUrl(this string url) || url.StartsWith("http://", StringComparison.Ordinal) || url.StartsWith("mailto:", StringComparison.Ordinal); } + + /// + /// Computes magnitude of a given . + /// + /// NAudio Complex number + /// Magnitude (Absolute number) of a given complex. + public static float ComputeMagnitude(this Complex complex) => (float)Math.Sqrt((complex.X * complex.X) + (complex.Y * complex.Y)); } } diff --git a/osu.Framework/Game.cs b/osu.Framework/Game.cs index 01042b525c..36b9c0b798 100644 --- a/osu.Framework/Game.cs +++ b/osu.Framework/Game.cs @@ -24,6 +24,7 @@ using osu.Framework.IO.Stores; using osu.Framework.Localisation; using osu.Framework.Platform; +using osu.Framework.Platform.SDL3; using osuTK; namespace osu.Framework @@ -165,8 +166,23 @@ private void load(FrameworkConfigManager config) samples.AddStore(new NamespacedResourceStore(Resources, @"Samples")); samples.AddStore(new OnlineStore()); - Audio = new AudioManager(Host.AudioThread, tracks, samples) { EventScheduler = Scheduler }; - dependencies.Cache(Audio); + switch (config.Get(FrameworkSetting.AudioDriver)) + { + case AudioDriver.SDL3 when Host.Window is SDL3Window sdl3Window: + { + SDL3AudioManager sdl3Audio = new SDL3AudioManager(Host.AudioThread, tracks, samples) { EventScheduler = Scheduler }; + sdl3Window.AudioDeviceAdded += sdl3Audio.OnNewDeviceEvent; + sdl3Window.AudioDeviceRemoved += sdl3Audio.OnLostDeviceEvent; + Audio = sdl3Audio; + break; + } + + default: + Audio = new BassAudioManager(Host.AudioThread, tracks, samples) { EventScheduler = Scheduler }; + break; + } + + dependencies.CacheAs(typeof(AudioManager), Audio); dependencies.CacheAs(Audio.Tracks); dependencies.CacheAs(Audio.Samples); diff --git a/osu.Framework/Graphics/Video/FFmpegFuncs.cs b/osu.Framework/Graphics/Video/FFmpegFuncs.cs index ccdc8de6ad..9bcf50aeb1 100644 --- a/osu.Framework/Graphics/Video/FFmpegFuncs.cs +++ b/osu.Framework/Graphics/Video/FFmpegFuncs.cs @@ -92,6 +92,24 @@ public unsafe class FFmpegFuncs public delegate int SwsScaleDelegate(SwsContext* c, byte*[] srcSlice, int[] srcStride, int srcSliceY, int srcSliceH, byte*[] dst, int[] dstStride); + public delegate SwrContext* SwrAllocSetOptsDelegate(SwrContext* s, long outChLayout, AVSampleFormat outSampleFmt, int outSampleRate, long inChLayout, AVSampleFormat inSampleFmt, int inSampleRate, int logOffset, void* logCtx); + + public delegate int SwrInitDelegate(SwrContext* s); + + public delegate int SwrIsInitializedDelegate(SwrContext* s); + + public delegate void SwrFreeDelegate(SwrContext** s); + + public delegate void SwrCloseDelegate(SwrContext* s); + + public delegate int SwrConvertDelegate(SwrContext* s, byte** dst, int outCount, byte** src, int inCount); + + public delegate long SwrGetDelayDelegate(SwrContext* s, long value); + + public delegate long AvGetDefaultChannelLayoutDelegate(int nbChannels); + + public delegate int SwrGetOutSamplesDelegate(SwrContext* s, int inSamples); + #endregion [CanBeNull] @@ -136,6 +154,15 @@ public unsafe class FFmpegFuncs public SwsFreeContextDelegate sws_freeContext; public SwsGetCachedContextDelegate sws_getCachedContext; public SwsScaleDelegate sws_scale; + public SwrAllocSetOptsDelegate swr_alloc_set_opts; + public SwrInitDelegate swr_init; + public SwrIsInitializedDelegate swr_is_initialized; + public SwrFreeDelegate swr_free; + public SwrCloseDelegate swr_close; + public SwrConvertDelegate swr_convert; + public SwrGetDelayDelegate swr_get_delay; + public AvGetDefaultChannelLayoutDelegate av_get_default_channel_layout; + public SwrGetOutSamplesDelegate swr_get_out_samples; // Touching AutoGen.ffmpeg or its LibraryLoader in any way on non-Desktop platforms // will cause it to throw in static constructor, which can't be bypassed. diff --git a/osu.Framework/Graphics/Video/VideoDecoder.cs b/osu.Framework/Graphics/Video/VideoDecoder.cs index 62c87389ff..11a967e1ab 100644 --- a/osu.Framework/Graphics/Video/VideoDecoder.cs +++ b/osu.Framework/Graphics/Video/VideoDecoder.cs @@ -23,6 +23,7 @@ using osu.Framework.Logging; using osu.Framework.Platform; using osu.Framework.Platform.Linux.Native; +using System.Buffers; namespace osu.Framework.Graphics.Video { @@ -54,12 +55,12 @@ public unsafe class VideoDecoder : IDisposable /// /// The frame rate of the video stream this decoder is decoding. /// - public double FrameRate => stream == null ? 0 : stream->avg_frame_rate.GetValue(); + public double FrameRate => videoStream == null ? 0 : videoStream->avg_frame_rate.GetValue(); /// /// True if the decoder can seek, false otherwise. Determined by the stream this decoder was created with. /// - public bool CanSeek => videoStream?.CanSeek == true; + public bool CanSeek => dataStream?.CanSeek == true; /// /// The current state of the decoding process. @@ -74,19 +75,25 @@ public unsafe class VideoDecoder : IDisposable // libav-context-related private AVFormatContext* formatContext; private AVIOContext* ioContext; - private AVStream* stream; - private AVCodecContext* codecContext; + + private AVStream* videoStream; + private AVCodecContext* videoCodecContext; private SwsContext* swsContext; + private AVStream* audioStream; + private AVCodecContext* audioCodecContext; + private SwrContext* swrContext; + private avio_alloc_context_read_packet readPacketCallback; private avio_alloc_context_seek seekCallback; private bool inputOpened; private bool isDisposed; private bool hwDecodingAllowed = true; - private Stream videoStream; + private Stream dataStream; - private double timeBaseInSeconds; + private double videoTimeBaseInSeconds; + private double audioTimeBaseInSeconds; // active decoder state private volatile float lastDecodedFrameTime; @@ -125,6 +132,7 @@ void loadVersionedLibraryGlobally(string name) loadVersionedLibraryGlobally("avcodec"); loadVersionedLibraryGlobally("avformat"); loadVersionedLibraryGlobally("swscale"); + loadVersionedLibraryGlobally("swresample"); } } @@ -138,25 +146,32 @@ public VideoDecoder(IRenderer renderer, string filename) { } + private VideoDecoder(Stream stream) + { + ffmpeg = CreateFuncs(); + dataStream = stream; + if (!dataStream.CanRead) + throw new InvalidOperationException($"The given stream does not support reading. A stream used for a {nameof(VideoDecoder)} must support reading."); + + State = DecoderState.Ready; + decoderCommands = new ConcurrentQueue(); + handle = new ObjectHandle(this, GCHandleType.Normal); + } + /// /// Creates a new video decoder that decodes the given video stream. /// /// The renderer to display the video. /// The stream that should be decoded. public VideoDecoder(IRenderer renderer, Stream videoStream) + : this(videoStream) { - ffmpeg = CreateFuncs(); - this.renderer = renderer; - this.videoStream = videoStream; - if (!videoStream.CanRead) - throw new InvalidOperationException($"The given stream does not support reading. A stream used for a {nameof(VideoDecoder)} must support reading."); - State = DecoderState.Ready; decodedFrames = new ConcurrentQueue(); - decoderCommands = new ConcurrentQueue(); availableTextures = new ConcurrentQueue(); // TODO: use "real" object pool when there's some public pool supporting disposables - handle = new ObjectHandle(this, GCHandleType.Normal); + scalerFrames = new ConcurrentQueue(); + hwTransferFrames = new ConcurrentQueue(); TargetHardwareVideoDecoders.BindValueChanged(_ => { @@ -164,10 +179,54 @@ public VideoDecoder(IRenderer renderer, Stream videoStream) if (formatContext == null) return; - decoderCommands.Enqueue(recreateCodecContext); + decoderCommands.Enqueue(RecreateCodecContext); }); } + private bool isAudioEnabled; + private readonly bool audioOnly; + + private int audioRate; + private int audioChannels; + private int audioBits; + private long audioChannelLayout; + private AVSampleFormat audioFmt; + + public long AudioBitrate => audioCodecContext->bit_rate; + public long AudioFrameCount => audioStream->nb_frames; + + // Audio mode + public VideoDecoder(Stream audioStream, int rate, int channels, bool isFloat, int bits, bool signed) + : this(audioStream) + { + audioOnly = true; + hwDecodingAllowed = false; + EnableAudioDecoding(rate, channels, isFloat, bits, signed); + } + + public void EnableAudioDecoding(int rate, int channels, bool isFloat, int bits, bool signed) + { + audioRate = rate; + audioChannels = channels; + audioBits = bits; + + isAudioEnabled = true; + audioChannelLayout = ffmpeg.av_get_default_channel_layout(channels); + + memoryStream = new MemoryStream(); + + if (isFloat) + audioFmt = AVSampleFormat.AV_SAMPLE_FMT_FLT; + else if (!signed && bits == 8) + audioFmt = AVSampleFormat.AV_SAMPLE_FMT_U8; + else if (signed && bits == 16) + audioFmt = AVSampleFormat.AV_SAMPLE_FMT_S16; + else if (signed && bits == 32) + audioFmt = AVSampleFormat.AV_SAMPLE_FMT_S32; + else + throw new InvalidOperationException("swresample doesn't support provided format!"); + } + /// /// Seek the decoder to the given timestamp. This will fail if is false. /// @@ -179,8 +238,18 @@ public void Seek(double targetTimestamp) decoderCommands.Enqueue(() => { - ffmpeg.avcodec_flush_buffers(codecContext); - ffmpeg.av_seek_frame(formatContext, stream->index, (long)(targetTimestamp / timeBaseInSeconds / 1000.0), FFmpegFuncs.AVSEEK_FLAG_BACKWARD); + if (!audioOnly) + { + ffmpeg.avcodec_flush_buffers(videoCodecContext); + ffmpeg.av_seek_frame(formatContext, videoStream->index, (long)(targetTimestamp / videoTimeBaseInSeconds / 1000.0), FFmpegFuncs.AVSEEK_FLAG_BACKWARD); + } + + if (audioStream != null) + { + ffmpeg.avcodec_flush_buffers(audioCodecContext); + ffmpeg.av_seek_frame(formatContext, audioStream->index, (long)(targetTimestamp / audioTimeBaseInSeconds / 1000.0), FFmpegFuncs.AVSEEK_FLAG_BACKWARD); + } + skipOutputUntilTime = targetTimestamp; State = DecoderState.Ready; }); @@ -212,8 +281,8 @@ public void StartDecoding() { try { - prepareDecoding(); - recreateCodecContext(); + PrepareDecoding(); + RecreateCodecContext(); } catch (Exception e) { @@ -265,10 +334,10 @@ public IEnumerable GetDecodedFrames() // https://en.wikipedia.org/wiki/YCbCr public Matrix3 GetConversionMatrix() { - if (codecContext == null) + if (videoCodecContext == null) return Matrix3.Zero; - switch (codecContext->colorspace) + switch (videoCodecContext->colorspace) { case AVColorSpace.AVCOL_SPC_BT709: return new Matrix3(1.164f, 1.164f, 1.164f, @@ -293,7 +362,7 @@ private static int readPacket(void* opaque, byte* bufferPtr, int bufferSize) return 0; var span = new Span(bufferPtr, bufferSize); - int bytesRead = decoder.videoStream.Read(span); + int bytesRead = decoder.dataStream.Read(span); return bytesRead != 0 ? bytesRead : FFmpegFuncs.AVERROR_EOF; } @@ -305,36 +374,38 @@ private static long streamSeekCallbacks(void* opaque, long offset, int whence) if (!handle.GetTarget(out VideoDecoder decoder)) return -1; - if (!decoder.videoStream.CanSeek) + if (!decoder.dataStream.CanSeek) throw new InvalidOperationException("Tried seeking on a video sourced by a non-seekable stream."); switch (whence) { case StdIo.SEEK_CUR: - decoder.videoStream.Seek(offset, SeekOrigin.Current); + decoder.dataStream.Seek(offset, SeekOrigin.Current); break; case StdIo.SEEK_END: - decoder.videoStream.Seek(offset, SeekOrigin.End); + decoder.dataStream.Seek(offset, SeekOrigin.End); break; case StdIo.SEEK_SET: - decoder.videoStream.Seek(offset, SeekOrigin.Begin); + decoder.dataStream.Seek(offset, SeekOrigin.Begin); break; case FFmpegFuncs.AVSEEK_SIZE: - return decoder.videoStream.Length; + return decoder.dataStream.Length; default: return -1; } - return decoder.videoStream.Position; + return decoder.dataStream.Position; } // sets up libavformat state: creates the AVFormatContext, the frames, etc. to start decoding, but does not actually start the decodingLoop - private void prepareDecoding() + internal void PrepareDecoding() { + dataStream.Position = 0; + const int context_buffer_size = 4096; readPacketCallback = readPacket; seekCallback = streamSeekCallbacks; @@ -363,26 +434,61 @@ private void prepareDecoding() if (findStreamInfoResult < 0) throw new InvalidOperationException($"Error finding stream info: {getErrorMessage(findStreamInfoResult)}"); - int streamIndex = ffmpeg.av_find_best_stream(formatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, null, 0); - if (streamIndex < 0) - throw new InvalidOperationException($"Couldn't find video stream: {getErrorMessage(streamIndex)}"); + packet = ffmpeg.av_packet_alloc(); + receiveFrame = ffmpeg.av_frame_alloc(); - stream = formatContext->streams[streamIndex]; - timeBaseInSeconds = stream->time_base.GetValue(); + int streamIndex = -1; - if (stream->duration > 0) - Duration = stream->duration * timeBaseInSeconds * 1000.0; - else - Duration = formatContext->duration / (double)FFmpegFuncs.AV_TIME_BASE * 1000.0; + if (!audioOnly) + { + streamIndex = ffmpeg.av_find_best_stream(formatContext, AVMediaType.AVMEDIA_TYPE_VIDEO, -1, -1, null, 0); + if (streamIndex < 0) + throw new InvalidOperationException($"Couldn't find stream: {getErrorMessage(streamIndex)}"); + + videoStream = formatContext->streams[streamIndex]; + videoTimeBaseInSeconds = videoStream->time_base.GetValue(); + + if (videoStream->duration > 0) + Duration = videoStream->duration * videoTimeBaseInSeconds * 1000.0; + else + Duration = formatContext->duration / (double)FFmpegFuncs.AV_TIME_BASE * 1000.0; + } + + if (isAudioEnabled) + { + streamIndex = ffmpeg.av_find_best_stream(formatContext, AVMediaType.AVMEDIA_TYPE_AUDIO, -1, streamIndex, null, 0); + if (streamIndex < 0 && audioOnly) + throw new InvalidOperationException($"Couldn't find stream: {getErrorMessage(streamIndex)}"); + + audioStream = formatContext->streams[streamIndex]; + audioTimeBaseInSeconds = audioStream->time_base.GetValue(); + + if (audioOnly) + { + if (audioStream->duration > 0) + Duration = audioStream->duration * audioTimeBaseInSeconds * 1000.0; + else + Duration = formatContext->duration / (double)FFmpegFuncs.AV_TIME_BASE * 1000.0; + } + } } - private void recreateCodecContext() + internal void RecreateCodecContext() + { + RecreateCodecContext(ref videoStream, ref videoCodecContext, hwDecodingAllowed); + RecreateCodecContext(ref audioStream, ref audioCodecContext, false); + + if (audioCodecContext != null && !prepareResampler()) + throw new InvalidDataException("Error trying to prepare audio resampler"); + } + + internal void RecreateCodecContext(ref AVStream* stream, ref AVCodecContext* codecContext, bool allowHwDecoding) { if (stream == null) return; var codecParams = *stream->codecpar; - var targetHwDecoders = hwDecodingAllowed ? TargetHardwareVideoDecoders.Value : HardwareVideoDecoder.None; + var targetHwDecoders = allowHwDecoding ? TargetHardwareVideoDecoders.Value : HardwareVideoDecoder.None; bool openSuccessful = false; foreach (var (decoder, hwDeviceType) in GetAvailableDecoders(formatContext->iformat, codecParams.codec_id, targetHwDecoders)) @@ -443,11 +549,37 @@ private void recreateCodecContext() throw new InvalidOperationException($"No usable decoder found for codec ID {codecParams.codec_id}"); } - private void decodingLoop(CancellationToken cancellationToken) + private bool prepareResampler() { - var packet = ffmpeg.av_packet_alloc(); - var receiveFrame = ffmpeg.av_frame_alloc(); + long srcChLayout = ffmpeg.av_get_default_channel_layout(audioCodecContext->channels); + AVSampleFormat srcAudioFmt = audioCodecContext->sample_fmt; + int srcRate = audioCodecContext->sample_rate; + + if (audioChannelLayout == srcChLayout && audioFmt == srcAudioFmt && audioRate == srcRate) + { + swrContext = null; + return true; + } + + swrContext = ffmpeg.swr_alloc_set_opts(null, audioChannelLayout, audioFmt, audioRate, + srcChLayout, srcAudioFmt, srcRate, 0, null); + + if (swrContext == null) + { + Logger.Log("Failed allocating memory for swresampler", level: LogLevel.Error); + return false; + } + + ffmpeg.swr_init(swrContext); + + return ffmpeg.swr_is_initialized(swrContext) > 0; + } + private AVPacket* packet; + private AVFrame* receiveFrame; + + private void decodingLoop(CancellationToken cancellationToken) + { const int max_pending_frames = 3; try @@ -499,14 +631,46 @@ private void decodingLoop(CancellationToken cancellationToken) } finally { - ffmpeg.av_packet_free(&packet); - ffmpeg.av_frame_free(&receiveFrame); - if (State != DecoderState.Faulted) State = DecoderState.Stopped; } } + private MemoryStream memoryStream; + + internal int DecodeNextAudioFrame(out byte[] decodedAudio, bool decodeUntilEnd = false) + { + if (audioStream == null) + { + decodedAudio = Array.Empty(); + return 0; + } + + memoryStream.Position = 0; + + try + { + do + { + decodeNextFrame(packet, receiveFrame); + + if (State != DecoderState.Running) + decodeUntilEnd = false; + } while (decodeUntilEnd); + } + catch (Exception e) + { + Logger.Error(e, "VideoDecoder faulted while decoding audio"); + State = DecoderState.Faulted; + decodedAudio = Array.Empty(); + return 0; + } + + decodedAudio = memoryStream.GetBuffer(); + + return (int)memoryStream.Position; + } + private void decodeNextFrame(AVPacket* packet, AVFrame* receiveFrame) { // read data from input into AVPacket. @@ -521,9 +685,13 @@ private void decodeNextFrame(AVPacket* packet, AVFrame* receiveFrame) bool unrefPacket = true; - if (packet->stream_index == stream->index) + AVCodecContext* codecContext = + !audioOnly && packet->stream_index == videoStream->index ? videoCodecContext + : audioStream != null && packet->stream_index == audioStream->index ? audioCodecContext : null; + + if (codecContext != null) { - int sendPacketResult = sendPacket(receiveFrame, packet); + int sendPacketResult = sendPacket(codecContext, receiveFrame, packet); // keep the packet data for next frame if we didn't send it successfully. if (sendPacketResult == -FFmpegFuncs.EAGAIN) @@ -538,7 +706,14 @@ private void decodeNextFrame(AVPacket* packet, AVFrame* receiveFrame) else if (readFrameResult == FFmpegFuncs.AVERROR_EOF) { // Flush decoder. - sendPacket(receiveFrame, null); + if (!audioOnly) + sendPacket(videoCodecContext, receiveFrame, null); + + if (audioStream != null) + { + sendPacket(audioCodecContext, receiveFrame, null); + resampleAndAppendToAudioStream(null); // flush audio resampler + } if (Looping) { @@ -562,7 +737,7 @@ private void decodeNextFrame(AVPacket* packet, AVFrame* receiveFrame) } } - private int sendPacket(AVFrame* receiveFrame, AVPacket* packet) + private int sendPacket(AVCodecContext* codecContext, AVFrame* receiveFrame, AVPacket* packet) { // send the packet for decoding. int sendPacketResult = ffmpeg.avcodec_send_packet(codecContext, packet); @@ -571,7 +746,7 @@ private int sendPacket(AVFrame* receiveFrame, AVPacket* packet) // otherwise we would get stuck in an infinite loop. if (sendPacketResult == 0 || sendPacketResult == -FFmpegFuncs.EAGAIN) { - readDecodedFrames(receiveFrame); + readDecodedFrames(codecContext, receiveFrame); } else { @@ -582,10 +757,10 @@ private int sendPacket(AVFrame* receiveFrame, AVPacket* packet) return sendPacketResult; } - private readonly ConcurrentQueue hwTransferFrames = new ConcurrentQueue(); + private readonly ConcurrentQueue hwTransferFrames; private void returnHwTransferFrame(FFmpegFrame frame) => hwTransferFrames.Enqueue(frame); - private void readDecodedFrames(AVFrame* receiveFrame) + private void readDecodedFrames(AVCodecContext* codecContext, AVFrame* receiveFrame) { while (true) { @@ -606,61 +781,140 @@ private void readDecodedFrames(AVFrame* receiveFrame) // but some HW codecs don't set it in which case fallback to `pts` long frameTimestamp = receiveFrame->best_effort_timestamp != FFmpegFuncs.AV_NOPTS_VALUE ? receiveFrame->best_effort_timestamp : receiveFrame->pts; - double frameTime = (frameTimestamp - stream->start_time) * timeBaseInSeconds * 1000; + double frameTime = 0.0; - if (skipOutputUntilTime > frameTime) - continue; + if (audioStream != null && codecContext->codec_type == AVMediaType.AVMEDIA_TYPE_AUDIO) + { + frameTime = (frameTimestamp - audioStream->start_time) * audioTimeBaseInSeconds * 1000; - // get final frame. - FFmpegFrame frame; + if (skipOutputUntilTime > frameTime) + continue; - if (((AVPixelFormat)receiveFrame->format).IsHardwarePixelFormat()) + resampleAndAppendToAudioStream(receiveFrame); + } + else if (!audioOnly && codecContext->codec_type == AVMediaType.AVMEDIA_TYPE_VIDEO) { - // transfer data from HW decoder to RAM. - if (!hwTransferFrames.TryDequeue(out var hwTransferFrame)) - hwTransferFrame = new FFmpegFrame(ffmpeg, returnHwTransferFrame); + frameTime = (frameTimestamp - videoStream->start_time) * videoTimeBaseInSeconds * 1000; + + if (skipOutputUntilTime > frameTime) + continue; - // WARNING: frames from `av_hwframe_transfer_data` have their timestamps set to AV_NOPTS_VALUE instead of real values. - // if you need to use them later, take them from `receiveFrame`. - int transferResult = ffmpeg.av_hwframe_transfer_data(hwTransferFrame.Pointer, receiveFrame, 0); + // get final frame. + FFmpegFrame frame; - if (transferResult < 0) + if (((AVPixelFormat)receiveFrame->format).IsHardwarePixelFormat()) { - Logger.Log($"Failed to transfer frame from HW decoder: {getErrorMessage(transferResult)}"); - tryDisableHwDecoding(transferResult); + // transfer data from HW decoder to RAM. + if (!hwTransferFrames.TryDequeue(out var hwTransferFrame)) + hwTransferFrame = new FFmpegFrame(ffmpeg, returnHwTransferFrame); - hwTransferFrame.Dispose(); - continue; + // WARNING: frames from `av_hwframe_transfer_data` have their timestamps set to AV_NOPTS_VALUE instead of real values. + // if you need to use them later, take them from `receiveFrame`. + int transferResult = ffmpeg.av_hwframe_transfer_data(hwTransferFrame.Pointer, receiveFrame, 0); + + if (transferResult < 0) + { + Logger.Log($"Failed to transfer frame from HW decoder: {getErrorMessage(transferResult)}"); + tryDisableHwDecoding(transferResult); + + hwTransferFrame.Dispose(); + continue; + } + + frame = hwTransferFrame; } + else + { + // copy data to a new AVFrame so that `receiveFrame` can be reused. + frame = new FFmpegFrame(ffmpeg); + ffmpeg.av_frame_move_ref(frame.Pointer, receiveFrame); + } + + // Note: this is the pixel format that `VideoTexture` expects internally + frame = ensureFramePixelFormat(frame, AVPixelFormat.AV_PIX_FMT_YUV420P); + if (frame == null) + continue; + + if (!availableTextures.TryDequeue(out var tex)) + tex = renderer.CreateVideoTexture(frame.Pointer->width, frame.Pointer->height); + + var upload = new VideoTextureUpload(frame); - frame = hwTransferFrame; + // We do not support videos with transparency at this point, so the upload's opacity as well as the texture's opacity is always opaque. + tex.SetData(upload, Opacity.Opaque); + decodedFrames.Enqueue(new DecodedFrame { Time = frameTime, Texture = tex }); } - else + + lastDecodedFrameTime = (float)frameTime; + } + } + + private void resampleAndAppendToAudioStream(AVFrame* frame) + { + if (memoryStream == null || audioStream == null) + return; + + int sampleCount; + byte*[] source; + + if (swrContext != null) + { + sampleCount = (int)ffmpeg.swr_get_delay(swrContext, audioRate); + source = null; + + if (frame != null) { - // copy data to a new AVFrame so that `receiveFrame` can be reused. - frame = new FFmpegFrame(ffmpeg); - ffmpeg.av_frame_move_ref(frame.Pointer, receiveFrame); + sampleCount = ffmpeg.swr_get_out_samples(swrContext, frame->nb_samples); + source = frame->data.ToArray(); } - lastDecodedFrameTime = (float)frameTime; + // no frame, no remaining samples in resampler + if (sampleCount <= 0) + return; + } + else if (frame != null) + { + sampleCount = frame->nb_samples; + source = frame->data.ToArray(); + } + else // no frame, no resampler + { + return; + } - // Note: this is the pixel format that `VideoTexture` expects internally - frame = ensureFramePixelFormat(frame, AVPixelFormat.AV_PIX_FMT_YUV420P); - if (frame == null) - continue; + int audioSize = sampleCount * audioChannels * (audioBits / 8); + byte[] audioDest = ArrayPool.Shared.Rent(audioSize); + int nbSamples = 0; - if (!availableTextures.TryDequeue(out var tex)) - tex = renderer.CreateVideoTexture(frame.Pointer->width, frame.Pointer->height); + try + { + if (swrContext != null) + { + fixed (byte** data = source) + fixed (byte* dest = audioDest) + nbSamples = ffmpeg.swr_convert(swrContext, &dest, sampleCount, data, frame != null ? frame->nb_samples : 0); + } + else if (source != null) + { + // assuming that the destination and source are not planar as we never define planar in ctor + nbSamples = sampleCount; - var upload = new VideoTextureUpload(frame); + for (int i = 0; i < audioSize; i++) + { + audioDest[i] = *(source[0] + i); + } + } - // We do not support videos with transparency at this point, so the upload's opacity as well as the texture's opacity is always opaque. - tex.SetData(upload, Opacity.Opaque); - decodedFrames.Enqueue(new DecodedFrame { Time = frameTime, Texture = tex }); + if (nbSamples > 0) + memoryStream.Write(audioDest, 0, nbSamples * (audioBits / 8) * audioChannels); + } + finally + { + ArrayPool.Shared.Return(audioDest); } } - private readonly ConcurrentQueue scalerFrames = new ConcurrentQueue(); + private readonly ConcurrentQueue scalerFrames; private void returnScalerFrame(FFmpegFrame frame) => scalerFrames.Enqueue(frame); [CanBeNull] @@ -724,7 +978,7 @@ private FFmpegFrame ensureFramePixelFormat(FFmpegFrame frame, AVPixelFormat targ private void tryDisableHwDecoding(int errorCode) { - if (!hwDecodingAllowed || TargetHardwareVideoDecoders.Value == HardwareVideoDecoder.None || codecContext == null || codecContext->hw_device_ctx == null) + if (!hwDecodingAllowed || TargetHardwareVideoDecoders.Value == HardwareVideoDecoder.None || videoCodecContext == null || videoCodecContext->hw_device_ctx == null) return; hwDecodingAllowed = false; @@ -740,7 +994,7 @@ private void tryDisableHwDecoding(int errorCode) { Logger.Log("Disabling hardware decoding of the current video due to an unexpected error"); - decoderCommands.Enqueue(recreateCodecContext); + decoderCommands.Enqueue(RecreateCodecContext); } } @@ -884,7 +1138,16 @@ protected virtual FFmpegFuncs CreateFuncs() avio_context_free = FFmpeg.AutoGen.ffmpeg.avio_context_free, sws_freeContext = FFmpeg.AutoGen.ffmpeg.sws_freeContext, sws_getCachedContext = FFmpeg.AutoGen.ffmpeg.sws_getCachedContext, - sws_scale = FFmpeg.AutoGen.ffmpeg.sws_scale + sws_scale = FFmpeg.AutoGen.ffmpeg.sws_scale, + swr_alloc_set_opts = FFmpeg.AutoGen.ffmpeg.swr_alloc_set_opts, + swr_init = FFmpeg.AutoGen.ffmpeg.swr_init, + swr_is_initialized = FFmpeg.AutoGen.ffmpeg.swr_is_initialized, + swr_free = FFmpeg.AutoGen.ffmpeg.swr_free, + swr_close = FFmpeg.AutoGen.ffmpeg.swr_close, + swr_convert = FFmpeg.AutoGen.ffmpeg.swr_convert, + swr_get_delay = FFmpeg.AutoGen.ffmpeg.swr_get_delay, + av_get_default_channel_layout = FFmpeg.AutoGen.ffmpeg.av_get_default_channel_layout, + swr_get_out_samples = FFmpeg.AutoGen.ffmpeg.swr_get_out_samples, }; } @@ -910,8 +1173,20 @@ protected virtual void Dispose(bool disposing) decoderCommands.Clear(); - StopDecodingAsync().ContinueWith(_ => + void freeFFmpeg() { + if (packet != null) + { + fixed (AVPacket** ptr = &packet) + ffmpeg.av_packet_free(ptr); + } + + if (receiveFrame != null) + { + fixed (AVFrame** ptr = &receiveFrame) + ffmpeg.av_frame_free(ptr); + } + if (formatContext != null && inputOpened) { fixed (AVFormatContext** ptr = &formatContext) @@ -928,38 +1203,64 @@ protected virtual void Dispose(bool disposing) ffmpeg.avio_context_free(ptr); } - if (codecContext != null) + if (videoCodecContext != null) { - fixed (AVCodecContext** ptr = &codecContext) + fixed (AVCodecContext** ptr = &videoCodecContext) + ffmpeg.avcodec_free_context(ptr); + } + + if (audioCodecContext != null) + { + fixed (AVCodecContext** ptr = &audioCodecContext) ffmpeg.avcodec_free_context(ptr); } seekCallback = null; readPacketCallback = null; - videoStream.Dispose(); - videoStream = null; + if (!audioOnly) + dataStream.Dispose(); + + dataStream = null; if (swsContext != null) ffmpeg.sws_freeContext(swsContext); - while (decodedFrames.TryDequeue(out var f)) + if (swrContext != null) { - f.Texture.FlushUploads(); - f.Texture.Dispose(); + fixed (SwrContext** ptr = &swrContext) + ffmpeg.swr_free(ptr); } - while (availableTextures.TryDequeue(out var t)) - t.Dispose(); + memoryStream?.Dispose(); + + memoryStream = null; - while (hwTransferFrames.TryDequeue(out var hwF)) - hwF.Dispose(); + if (!audioOnly) + { + while (decodedFrames.TryDequeue(out var f)) + { + f.Texture.FlushUploads(); + f.Texture.Dispose(); + } + + while (availableTextures.TryDequeue(out var t)) + t.Dispose(); + + while (hwTransferFrames.TryDequeue(out var hwF)) + hwF.Dispose(); - while (scalerFrames.TryDequeue(out var sf)) - sf.Dispose(); + while (scalerFrames.TryDequeue(out var sf)) + sf.Dispose(); + } handle.Dispose(); - }); + } + + if (audioOnly) + freeFFmpeg(); + else + StopDecodingAsync().ContinueWith(_ => freeFFmpeg()); } #endregion diff --git a/osu.Framework/IO/Stores/ResourceStore.cs b/osu.Framework/IO/Stores/ResourceStore.cs index 27309d9de9..e46c539071 100644 --- a/osu.Framework/IO/Stores/ResourceStore.cs +++ b/osu.Framework/IO/Stores/ResourceStore.cs @@ -125,7 +125,7 @@ public virtual T Get(string name) return default; } - public Stream GetStream(string name) + public virtual Stream GetStream(string name) { if (name == null) return null; diff --git a/osu.Framework/Platform/SDL3/SDL3Window.cs b/osu.Framework/Platform/SDL3/SDL3Window.cs index ad30b20ad9..813bc7477a 100644 --- a/osu.Framework/Platform/SDL3/SDL3Window.cs +++ b/osu.Framework/Platform/SDL3/SDL3Window.cs @@ -588,6 +588,28 @@ protected virtual void HandleEvent(SDL_Event e) case SDL_EventType.SDL_EVENT_PEN_MOTION: handlePenMotionEvent(e.pmotion); break; + + case SDL_EventType.SDL_EVENT_AUDIO_DEVICE_ADDED: + case SDL_EventType.SDL_EVENT_AUDIO_DEVICE_REMOVED: + handleAudioDeviceEvent(e.adevice); + break; + } + } + + private void handleAudioDeviceEvent(SDL_AudioDeviceEvent evtAudioDevice) + { + if (evtAudioDevice.recording) // recording device + return; + + switch (evtAudioDevice.type) + { + case SDL_EventType.SDL_EVENT_AUDIO_DEVICE_ADDED: + AudioDeviceAdded?.Invoke(evtAudioDevice.which); + break; + + case SDL_EventType.SDL_EVENT_AUDIO_DEVICE_REMOVED: + AudioDeviceRemoved?.Invoke(evtAudioDevice.which); // it is only uint if a device is removed + break; } } @@ -664,6 +686,16 @@ internal virtual void SetIconFromGroup(IconGroup iconGroup) /// public event Action? DragDrop; + /// + /// Invoked when a new audio device is added, only when using SDL3 audio + /// + public event Action? AudioDeviceAdded; + + /// + /// Invoked when a new audio device is removed, only when using SDL3 audio + /// + public event Action? AudioDeviceRemoved; + #endregion public void Dispose() diff --git a/osu.Framework/Threading/AudioThread.cs b/osu.Framework/Threading/AudioThread.cs index 171f43280a..955a433ab8 100644 --- a/osu.Framework/Threading/AudioThread.cs +++ b/osu.Framework/Threading/AudioThread.cs @@ -77,7 +77,8 @@ internal void RegisterManager(AudioManager manager) managers.Add(manager); } - manager.GlobalMixerHandle.BindTo(globalMixerHandle); + if (manager is BassAudioManager bassManager) + bassManager.GlobalMixerHandle.BindTo(globalMixerHandle); } internal void UnregisterManager(AudioManager manager) @@ -85,7 +86,8 @@ internal void UnregisterManager(AudioManager manager) lock (managers) managers.Remove(manager); - manager.GlobalMixerHandle.UnbindFrom(globalMixerHandle); + if (manager is BassAudioManager bassManager) + bassManager.GlobalMixerHandle.UnbindFrom(globalMixerHandle); } protected override void OnExit() diff --git a/osu.Framework/osu.Framework.csproj b/osu.Framework/osu.Framework.csproj index 0133948419..db8cfe1088 100644 --- a/osu.Framework/osu.Framework.csproj +++ b/osu.Framework/osu.Framework.csproj @@ -23,6 +23,7 @@ + @@ -38,6 +39,7 @@ +