diff --git a/Cargo.toml b/Cargo.toml index 785243b2..038159df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,6 @@ opt-level = 3 [profile.dev.package.image] opt-level = 3 + +[profile.dev.package.symphonia] +opt-level = 3 diff --git a/examples/beat-saber-clone/Cargo.toml b/examples/beat-saber-clone/Cargo.toml index 6db7c736..5efd0357 100644 --- a/examples/beat-saber-clone/Cargo.toml +++ b/examples/beat-saber-clone/Cargo.toml @@ -24,7 +24,7 @@ serde_json = "1.0" approx = "0.5" [target.'cfg(target_os = "android")'.dependencies] -ndk-glue = "0.5.0" +ndk-glue = "=0.6.0" [package.metadata.android] apk_label = "Hotham Beat Saber Example" diff --git a/examples/beat-saber-clone/assets/Tell Me That I Can't - NEFFEX.mp3 b/examples/beat-saber-clone/assets/Tell Me That I Can't - NEFFEX.mp3 new file mode 100644 index 00000000..ef3fb6f0 Binary files /dev/null and b/examples/beat-saber-clone/assets/Tell Me That I Can't - NEFFEX.mp3 differ diff --git a/examples/beat-saber-clone/src/lib.rs b/examples/beat-saber-clone/src/lib.rs index 546ad938..fa235fe3 100644 --- a/examples/beat-saber-clone/src/lib.rs +++ b/examples/beat-saber-clone/src/lib.rs @@ -25,8 +25,8 @@ use hotham::{ gltf_loader::{self, add_model_to_world}, legion::{Resources, Schedule, World}, resources::{ - physics_context::PANEL_COLLISION_GROUP, GuiContext, HapticContext, PhysicsContext, - RenderContext, XrContext, + physics_context::PANEL_COLLISION_GROUP, AudioContext, GuiContext, HapticContext, + PhysicsContext, RenderContext, XrContext, }, schedule_functions::{ apply_haptic_feedback, begin_frame, begin_pbr_renderpass, end_frame, end_pbr_renderpass, @@ -75,6 +75,7 @@ pub fn real_main() -> HothamResult<()> { &render_context.descriptor_set_layouts, )?; let haptic_context = HapticContext::default(); + let mut audio_context = AudioContext::default(); // Add Environment add_environment_models(&models, &mut world, &vulkan_context, &render_context); @@ -99,6 +100,11 @@ pub fn real_main() -> HothamResult<()> { &mut physics_context, ); + // Add Sound + let mp3_bytes = include_bytes!("../../../test_assets/Quartet 14 - Clip.mp3").to_vec(); + let audio_source = audio_context.create_audio_source(mp3_bytes); + world.push((audio_source,)); + // // Add Blue Saber // add_saber( // Colour::Blue, @@ -163,6 +169,7 @@ pub fn real_main() -> HothamResult<()> { resources.insert(0 as usize); resources.insert(GameState::default()); resources.insert(haptic_context); + resources.insert(audio_context); let schedule = Schedule::builder() .add_thread_local_fn(begin_frame) diff --git a/examples/simple-scene/Cargo.toml b/examples/simple-scene/Cargo.toml index d75c57e2..91bff8f5 100644 --- a/examples/simple-scene/Cargo.toml +++ b/examples/simple-scene/Cargo.toml @@ -15,7 +15,7 @@ path = "src/main.rs" hotham = {path = "../../hotham"} [target.'cfg(target_os = "android")'.dependencies] -ndk-glue = "0.5.0" +ndk-glue = "0.6.0" [package.metadata.android] apk_label = "Hotham Simple Scene Example" diff --git a/hotham/Cargo.toml b/hotham/Cargo.toml index 1f46c3a9..75709307 100644 --- a/hotham/Cargo.toml +++ b/hotham/Cargo.toml @@ -12,9 +12,11 @@ shaderc = "0.7" anyhow = "1.0" ash = "0.33.2" console = "0.14" +cpal = {git = "https://github.com/RustAudio/cpal", rev = "f8b1ab53b46ef7a635c89c8758674cb36caea190"} crossbeam = "0.8.1" ctrlc = {version = "3", features = ["termination"]} egui = "0.15" +generational-arena = "0.2.8" gltf = {version = "0.16", features = ["KHR_materials_pbrSpecularGlossiness"]} hotham-debug-server = {path = "../hotham-debug-server"} image = "0.23" @@ -25,13 +27,13 @@ memoffset = "0.5.1" mint = "0.5.6" nalgebra = {features = ["convert-mint", "serde-serialize"], version = "0.29.0"} oddio = "0.5" -openxr = {features = ["loaded", "mint"], version = "0.15.5"} +openxr = {features = ["loaded", "mint"], git = "https://github.com/Ralith/openxrs", rev = "ca06a64557abc94559bdb30ceafb51b031e5ac9a"} rand = "0.8" rapier3d = "0.11.1" renderdoc = "0.10" -schemars = "0.8" serde = {version = "1.0", features = ["derive"]} serde_json = "1.0" +symphonia = {version = "0.4", features = ["mp3"]} thiserror = "1.0" uuid = {version = "0.8", features = ["serde", "v4"]} @@ -40,8 +42,8 @@ approx = "0.5" [target.'cfg(target_os = "android")'.dependencies] jni = "0.18.0" -ndk = "0.4.0" -ndk-glue = "0.5" +ndk = "0.6.0" +ndk-glue = "=0.6.0" [dev-dependencies.criterion] features = ["html_reports"] diff --git a/hotham/src/components/mod.rs b/hotham/src/components/mod.rs index cd3e8503..c09dfaf1 100644 --- a/hotham/src/components/mod.rs +++ b/hotham/src/components/mod.rs @@ -13,6 +13,7 @@ pub mod primitive; pub mod rigid_body; pub mod root; pub mod skin; +pub mod sound_emitter; pub mod transform; pub mod transform_matrix; pub mod visible; @@ -32,6 +33,7 @@ pub use primitive::Primitive; pub use rigid_body::RigidBody; pub use root::Root; pub use skin::Skin; +pub use sound_emitter::SoundEmitter; pub use transform::Transform; pub use transform_matrix::TransformMatrix; pub use visible::Visible; diff --git a/hotham/src/components/sound_emitter.rs b/hotham/src/components/sound_emitter.rs new file mode 100644 index 00000000..c0bc0073 --- /dev/null +++ b/hotham/src/components/sound_emitter.rs @@ -0,0 +1,59 @@ +use std::sync::Arc; + +use oddio::{Frames, Stop}; + +type AudioHandle = oddio::Handle>>>; + +pub struct SoundEmitter { + pub frames: Arc>, + pub handle: Option, + pub next_state: SoundState, +} + +#[derive(Debug, Clone, Copy)] +pub enum SoundState { + Stopped, + Playing, + Paused, +} + +impl SoundEmitter { + pub fn new(frames: Arc>) -> Self { + Self { + frames, + handle: None, + next_state: SoundState::Stopped, + } + } + + pub fn current_state(&mut self) -> SoundState { + if let Some(handle) = self.handle.as_mut() { + let control = handle.control::, _>(); + if control.is_paused() { + return SoundState::Paused; + } + if control.is_stopped() { + return SoundState::Stopped; + } + return SoundState::Playing; + } else { + return SoundState::Stopped; + } + } + + pub fn play(&mut self) { + self.next_state = SoundState::Playing; + } + + pub fn pause(&mut self) { + self.next_state = SoundState::Paused; + } + + pub fn stop(&mut self) { + self.next_state = SoundState::Stopped; + } + + pub fn resume(&mut self) { + self.next_state = SoundState::Playing; + } +} diff --git a/hotham/src/resources/audio_context.rs b/hotham/src/resources/audio_context.rs new file mode 100644 index 00000000..31b7df4f --- /dev/null +++ b/hotham/src/resources/audio_context.rs @@ -0,0 +1,240 @@ +use std::sync::{Arc, Mutex}; + +use crate::components::SoundEmitter; +use cpal::{ + traits::{DeviceTrait, HostTrait, StreamTrait}, + Stream, +}; +use oddio::{Frames, FramesSignal, Handle, Mixer, SpatialBuffered, SpatialScene, Stop}; +use symphonia::core::{audio::SampleBuffer, io::MediaSourceStream, probe::Hint}; + +type MusicTrackHandle = Handle>>; +use generational_arena::{Arena, Index}; + +pub struct AudioContext { + pub scene_handle: oddio::Handle, + pub mixer_handle: oddio::Handle>, + pub stream: Arc>, + pub current_music_track: Option, + music_tracks: Arena>>, + music_track_handle: Option, +} + +impl Default for AudioContext { + fn default() -> Self { + // Configure cpal + let host = cpal::default_host(); + let device = host + .default_output_device() + .expect("no output device available"); + println!( + "[HOTHAM_AUDIO_CONTEXT] Using default audio device: {}", + device.name().unwrap() + ); + let sample_rate = device.default_output_config().unwrap().sample_rate(); + let config = cpal::StreamConfig { + channels: 2, + sample_rate, + buffer_size: cpal::BufferSize::Default, + }; + println!("[HOTHAM_AUDIO_CONTEXT] cpal AudioConfig: {:?}", config); + + // Create a spatialised audio scene + let (scene_handle, scene) = oddio::split(oddio::SpatialScene::new(sample_rate.0, 0.1)); + + // Create a mixer + let (mut mixer_handle, mixer) = oddio::split(oddio::Mixer::new()); + + // Pipe the spatialised scene to the mixer + let _ = mixer_handle.control().play(scene); + + // Pipe the mixer to the audio hardware. + let stream = device + .build_output_stream( + &config, + move |out_flat: &mut [f32], _: &cpal::OutputCallbackInfo| { + let out_stereo: &mut [[f32; 2]] = oddio::frame_stereo(out_flat); + oddio::run(&mixer, sample_rate.0, out_stereo); + }, + |err| { + eprintln!( + "[HOTHAM_AUDIO_CONTEXT] An error occurred playing the audio stream: {}", + err + ) + }, + ) + .unwrap(); + stream + .play() + .expect("[HOTHAM_AUDIO_CONTEXT] Unable to play to audio hardware!"); + + Self { + scene_handle, + mixer_handle, + stream: Arc::new(Mutex::new(stream)), + music_tracks: Arena::new(), + music_track_handle: None, + current_music_track: None, + } + } +} + +// SAFETY: I solemly promise to be good. +// We have no intention to mutate `Stream`, we just have to hold +// a reference onto it so that sound keeps playing. +unsafe impl Send for AudioContext {} + +impl AudioContext { + pub fn create_audio_source(&mut self, mp3_bytes: Vec) -> SoundEmitter { + let frames = get_frames_from_mp3(mp3_bytes); + + SoundEmitter::new(frames) + } + + pub fn play_audio( + &mut self, + audio_source: &mut SoundEmitter, + position: mint::Point3, + velocity: mint::Vector3, + ) { + let signal: oddio::FramesSignal<_> = oddio::FramesSignal::from(audio_source.frames.clone()); + let handle = self.scene_handle.control().play_buffered( + signal, + oddio::SpatialOptions { + position, + velocity, + radius: 1.0, // + }, + 1000.0, + ); + audio_source.handle = Some(handle); + } + + pub fn resume_audio(&mut self, audio_source: &mut SoundEmitter) { + audio_source + .handle + .as_mut() + .map(|h| h.control::, _>().resume()); + } + + pub fn pause_audio(&mut self, audio_source: &mut SoundEmitter) { + audio_source + .handle + .as_mut() + .map(|h| h.control::, _>().pause()); + } + + pub fn stop_audio(&mut self, audio_source: &mut SoundEmitter) { + audio_source + .handle + .take() + .map(|mut h| h.control::, _>().stop()); + } + + pub fn update_motion( + &mut self, + audio_source: &mut SoundEmitter, + position: mint::Point3, + velocity: mint::Vector3, + ) { + audio_source.handle.as_mut().map(|h| { + h.control::, _>() + .set_motion(position, velocity, false) + }); + } + + pub fn add_music_track(&mut self, mp3_bytes: Vec) -> Index { + let frames = get_stereo_frames_from_mp3(mp3_bytes); + self.music_tracks.insert(frames) + } + + pub fn play_music_track(&mut self, index: Index) { + if let Some(mut handle) = self.music_track_handle.take() { + handle.control::, _>().stop(); + } + + let frames = self.music_tracks[index].clone(); + let signal = oddio::FramesSignal::from(frames); + self.music_track_handle = Some(self.mixer_handle.control().play(signal)); + self.current_music_track = Some(index.clone()); + } + + pub fn pause_music_track(&mut self) { + self.music_track_handle + .as_mut() + .map(|h| h.control::, _>().pause()); + } + + pub fn resume_music_track(&mut self) { + self.music_track_handle + .as_mut() + .map(|h| h.control::, _>().resume()); + } +} + +fn get_frames_from_mp3(mp3_bytes: Vec) -> Arc> { + let (samples, sample_rate) = decode_mp3(mp3_bytes); + oddio::Frames::from_slice(sample_rate, &samples) +} + +fn get_stereo_frames_from_mp3(mp3_bytes: Vec) -> Arc> { + let (mut samples, sample_rate) = decode_mp3(mp3_bytes); + let stereo = oddio::frame_stereo(&mut samples); + oddio::Frames::from_slice(sample_rate, &stereo) +} + +fn decode_mp3(mp3_bytes: Vec) -> (Vec, u32) { + let cursor = Box::new(std::io::Cursor::new(mp3_bytes)); + let mss = MediaSourceStream::new(cursor, Default::default()); + let hint = Hint::new(); + let format_opts = Default::default(); + let metadata_opts = Default::default(); + let decode_opts = Default::default(); + let probed = symphonia::default::get_probe() + .format(&hint, mss, &format_opts, &metadata_opts) + .expect("Failed to parse MP3 file"); + + let mut reader = probed.format; + let track = reader.default_track().unwrap(); + let track_id = track.id; + let mut decoder = symphonia::default::get_codecs() + .make(&track.codec_params, &decode_opts) + .expect("Unable to get decoder"); + let sample_rate = decoder.codec_params().sample_rate.unwrap(); + + let mut samples: Vec = Vec::new(); + + // Decode all packets, ignoring all decode errors. + loop { + let packet = match reader.next_packet() { + Ok(packet) => packet, + Err(err) => { + eprintln!("Error reading packet: {:?}", err); + break; + } + }; + + // If the packet does not belong to the selected track, skip over it. + if packet.track_id() != track_id { + continue; + } + + // Decode the packet into audio samples. + match decoder.decode(&packet) { + Ok(decoded) => { + let mut sample_buf = + SampleBuffer::::new(decoded.capacity() as u64, *decoded.spec()); + sample_buf.copy_interleaved_ref(decoded); + for sample in sample_buf.samples() { + samples.push(*sample); + } + } + Err(err) => { + eprintln!("Error while decoding: {:?}", err); + break; + } + } + } + + (samples, sample_rate) +} diff --git a/hotham/src/resources/mod.rs b/hotham/src/resources/mod.rs index 7d9f2911..b25b88ee 100644 --- a/hotham/src/resources/mod.rs +++ b/hotham/src/resources/mod.rs @@ -1,3 +1,4 @@ +pub mod audio_context; pub mod gui_context; pub mod haptic_context; pub mod physics_context; @@ -5,6 +6,7 @@ pub mod render_context; pub mod vulkan_context; pub mod xr_context; +pub use audio_context::AudioContext; pub use gui_context::GuiContext; pub use haptic_context::HapticContext; pub use physics_context::PhysicsContext; diff --git a/hotham/src/systems/audio.rs b/hotham/src/systems/audio.rs new file mode 100644 index 00000000..690a55ce --- /dev/null +++ b/hotham/src/systems/audio.rs @@ -0,0 +1,163 @@ +use legion::system; +use nalgebra::Point3; + +use crate::{ + components::{sound_emitter::SoundState, RigidBody, SoundEmitter}, + resources::{AudioContext, PhysicsContext, XrContext}, + util::posef_to_isometry, +}; + +#[system(for_each)] +pub fn audio( + sound_emitter: &mut SoundEmitter, + #[resource] audio_context: &mut AudioContext, + #[resource] physics_context: &PhysicsContext, + #[resource] xr_context: &XrContext, + rigid_body: &RigidBody, +) { + // First, where is the listener? + let listener_location = posef_to_isometry(xr_context.views[1].pose) + .lerp_slerp(&posef_to_isometry(xr_context.views[0].pose), 0.5); + + // Get the position and velocity of the entity. + let rigid_body = physics_context + .rigid_bodies + .get(rigid_body.handle) + .expect("Unable to get RigidBody"); + + let velocity = (*rigid_body.linvel()).into(); + + // Now transform the position of the entity w.r.t. the listener + let position = listener_location + .transform_point(&Point3::from(*rigid_body.translation())) + .into(); + + // Determine what we should do with the audio source + match (sound_emitter.current_state(), &sound_emitter.next_state) { + (SoundState::Stopped, SoundState::Playing) => { + audio_context.play_audio(sound_emitter, position, velocity); + } + (SoundState::Paused, SoundState::Playing) => { + audio_context.resume_audio(sound_emitter); + } + (SoundState::Playing | SoundState::Paused, SoundState::Paused) => { + audio_context.pause_audio(sound_emitter); + } + (_, SoundState::Stopped) => { + audio_context.stop_audio(sound_emitter); + } + _ => {} + } + + // Update its position and velocity + audio_context.update_motion(sound_emitter, position, velocity); +} + +#[cfg(test)] +mod tests { + use std::{ + thread, + time::{Duration, Instant}, + }; + + use legion::{IntoQuery, Resources, Schedule, World}; + use rapier3d::prelude::RigidBodyBuilder; + const DURATION_SECS: u32 = 30; + + use crate::{resources::XrContext, VIEW_TYPE}; + + use super::*; + #[test] + pub fn test_audio_system() { + // Create resources + let (xr_context, _) = XrContext::new().unwrap(); + let mut audio_context = AudioContext::default(); + let mut physics_context = PhysicsContext::default(); + + // Load MP3s from disk + let beethoven = include_bytes!("../../../test_assets/Quartet 14 - Clip.mp3").to_vec(); + let right_here = + include_bytes!("../../../test_assets/right_here_beside_you_clip.mp3").to_vec(); + let tell_me_that_i_cant = + include_bytes!("../../../test_assets/tell_me_that_i_cant_clip.mp3").to_vec(); + + let beethoven = audio_context.add_music_track(beethoven); + let right_here = audio_context.add_music_track(right_here); + let tell_me_that_i_cant = audio_context.add_music_track(tell_me_that_i_cant); + audio_context.play_music_track(beethoven); + + // Create rigid body for the test entity + let sound_effect = include_bytes!("../../../test_assets/ice_crash.mp3").to_vec(); + let rigid_body = RigidBodyBuilder::new_dynamic() + .linvel([0.5, 0., 0.].into()) + .translation([-2., 0., 0.].into()) + .build(); + let handle = physics_context.rigid_bodies.insert(rigid_body); + let rigid_body = RigidBody { handle }; + let sound_emitter = audio_context.create_audio_source(sound_effect); + + // Create world + let mut world = World::default(); + let audio_entity = world.push((sound_emitter, rigid_body)); + + // Create resources + let mut resources = Resources::default(); + resources.insert(audio_context); + resources.insert(physics_context); + resources.insert(xr_context); + let start = Instant::now(); + + let mut schedule = Schedule::builder() + .add_thread_local_fn(move |world, resources| { + let mut query = <(&mut SoundEmitter, &mut RigidBody)>::query(); + let mut xr_context = resources.get_mut::().unwrap(); + let mut physics_context = resources.get_mut::().unwrap(); + let mut audio_context = resources.get_mut::().unwrap(); + + let (source, _) = query.get_mut(world, audio_entity).unwrap(); + + let (frame_state, _) = xr_context.begin_frame().unwrap(); + let (view_state_flags, views) = xr_context + .session + .locate_views( + VIEW_TYPE, + frame_state.predicted_display_time, + &xr_context.reference_space, + ) + .unwrap(); + xr_context.views = views; + xr_context.view_state_flags = view_state_flags; + + match source.current_state() { + SoundState::Stopped => source.play(), + _ => {} + } + + if start.elapsed().as_secs() >= 8 + && audio_context.current_music_track.unwrap() != right_here + { + audio_context.play_music_track(right_here); + } else if start.elapsed().as_secs() >= 4 + && start.elapsed().as_secs() < 8 + && audio_context.current_music_track.unwrap() != tell_me_that_i_cant + { + audio_context.play_music_track(tell_me_that_i_cant); + } + + physics_context.update(); + xr_context.end_frame().unwrap(); + }) + .add_system(audio_system()) + .build(); + + loop { + thread::sleep(Duration::from_millis(50)); + let dt = start.elapsed(); + if dt >= Duration::from_secs(DURATION_SECS as u64) { + break; + } + + schedule.execute(&mut world, &mut resources); + } + } +} diff --git a/hotham/src/systems/mod.rs b/hotham/src/systems/mod.rs index df7c9754..7973efb2 100644 --- a/hotham/src/systems/mod.rs +++ b/hotham/src/systems/mod.rs @@ -1,4 +1,5 @@ pub mod animation; +pub mod audio; pub mod collision; pub mod draw_gui; pub mod grabbing; @@ -11,6 +12,7 @@ pub mod update_rigid_body_transforms; pub mod update_transform_matrix; pub use animation::animation_system; +pub use audio::audio_system; pub use collision::collision_system; pub use draw_gui::draw_gui_system; pub use grabbing::grabbing_system; diff --git a/test_assets/Quartet 14 - Clip.mp3 b/test_assets/Quartet 14 - Clip.mp3 new file mode 100644 index 00000000..c826f421 Binary files /dev/null and b/test_assets/Quartet 14 - Clip.mp3 differ diff --git a/test_assets/Quartet 14.mp3 b/test_assets/Quartet 14.mp3 new file mode 100644 index 00000000..1931486a Binary files /dev/null and b/test_assets/Quartet 14.mp3 differ diff --git a/test_assets/gymnopedie.mp3 b/test_assets/gymnopedie.mp3 new file mode 100644 index 00000000..2cce09f0 Binary files /dev/null and b/test_assets/gymnopedie.mp3 differ diff --git a/test_assets/ice_crash.mp3 b/test_assets/ice_crash.mp3 new file mode 100644 index 00000000..ef975967 Binary files /dev/null and b/test_assets/ice_crash.mp3 differ diff --git a/test_assets/right_here_beside_you_clip.mp3 b/test_assets/right_here_beside_you_clip.mp3 new file mode 100644 index 00000000..09dce936 Binary files /dev/null and b/test_assets/right_here_beside_you_clip.mp3 differ diff --git a/test_assets/tell_me_that_i_cant_clip.mp3 b/test_assets/tell_me_that_i_cant_clip.mp3 new file mode 100644 index 00000000..86e7f047 Binary files /dev/null and b/test_assets/tell_me_that_i_cant_clip.mp3 differ