From 6d6a07dfc787931d6abc8bcb07f3caab62ebdb86 Mon Sep 17 00:00:00 2001 From: Mouwrice Date: Fri, 3 May 2024 11:13:06 +0200 Subject: [PATCH] feat: improve tracking and drum --- drumpy/app/main.py | 1 + drumpy/drum/drum.py | 6 ++++-- drumpy/drum/sound.py | 18 ++++++++++++++++-- drumpy/mediapipe_pose/mediapipe_pose.py | 2 +- drumpy/mediapipe_pose/process_result.py | 1 - drumpy/tracking/drum_trackers.py | 9 +++++---- drumpy/tracking/marker_tracker.py | 11 +++++++++-- drumpy/tracking/marker_tracker_wrapper.py | 4 ++-- 8 files changed, 38 insertions(+), 14 deletions(-) diff --git a/drumpy/app/main.py b/drumpy/app/main.py index 4ef7ef1..6258cc0 100644 --- a/drumpy/app/main.py +++ b/drumpy/app/main.py @@ -41,6 +41,7 @@ def __init__( self.model = model pygame.init() + # pygame.mixer.init(channels=8) pygame.display.set_caption("DrumPy") initial_window_size = (900, 900) diff --git a/drumpy/drum/drum.py b/drumpy/drum/drum.py index 6dd8596..6fc1059 100644 --- a/drumpy/drum/drum.py +++ b/drumpy/drum/drum.py @@ -63,6 +63,7 @@ def find_and_play_sound( self: Self, position: Position, marker: MarkerEnum, + velocity: float, sounds: Optional[list[Sound]] = None, ) -> None: """ @@ -71,6 +72,7 @@ def find_and_play_sound( :param marker: The marker that hit the sound :param sounds: List of sounds to consider, if None, all sounds will be considered :param position: A 3D position as a numpy array + :param velocity: The velocity of the marker, used to determine the hit strength and sound volume :return: """ @@ -81,7 +83,7 @@ def find_and_play_sound( if (distance := sound.is_hit(position)) is not None: # Check if the sound is calibrating if sound.state == SoundState.CALIBRATING: - sound.hit(position) + sound.hit(position, velocity) print( f"\t{marker}: {sound.name} with distance {distance:.3f} at {position_str(position)}" ) @@ -92,7 +94,7 @@ def find_and_play_sound( closest_distance = distance if closest_sound is not None: - closest_sound.hit(position) + closest_sound.hit(position, velocity) print( f"{marker}: {closest_sound.name} with distance {closest_distance:.3f} " f"at {position_str(position)}" diff --git a/drumpy/drum/sound.py b/drumpy/drum/sound.py index 70ea9e5..f8bc3c1 100644 --- a/drumpy/drum/sound.py +++ b/drumpy/drum/sound.py @@ -34,6 +34,7 @@ def __init__( path: str, margin: float, position: Optional[Position] = None, + velocity_multiplier: float = 20, ) -> None: """ Initialize the sound @@ -56,6 +57,8 @@ def __init__( self.hit_count = 0 self.hits: list[Position] = [] + self.velocity_multiplier = velocity_multiplier + # the margin that the sound can be hit with, aka the distance to the sound, or the size of the sound area self.margin: float = margin @@ -106,11 +109,15 @@ def is_hit(self: Self, position: Position) -> Optional[float]: return distance return None - def hit(self: Self, position: Position) -> None: + def hit(self: Self, position: Position, velocity: float) -> None: """ Update the position of the sound slowly to the given position and play it - :param position: + :param position: The position of the hit + :param velocity: The velocity of the hit, used to determine the volume of the sound """ + # print(f"\n{self.name} hit at {position_str(position)} with velocity {velocity}") + volume = min(1.0, max(0.0, abs(velocity) * self.velocity_multiplier)) + self.sound.set_volume(volume) self.sound.play() self.hit_count += 1 self.position = 0.99 * self.position + 0.01 * position @@ -122,6 +129,7 @@ def __init__(self) -> None: "Snare Drum", "./resources/sounds/CKV1_Snare Loud.wav", margin=MARGIN, + velocity_multiplier=20, ) @@ -131,6 +139,7 @@ def __init__(self) -> None: "High Hat", "./resources/sounds/CKV1_HH Closed Loud.wav", margin=MARGIN, + velocity_multiplier=20, ) @@ -140,6 +149,7 @@ def __init__(self) -> None: "Kick Drum", "./resources/sounds/CKV1_Kick Loud.wav", margin=MARGIN, + velocity_multiplier=40, ) @@ -149,6 +159,7 @@ def __init__(self) -> None: "High Hat Foot", "./resources/sounds/CKV1_HH Foot.wav", margin=MARGIN, + velocity_multiplier=40, ) @@ -158,6 +169,7 @@ def __init__(self) -> None: "Tom 1", "./DrumSamples/Tom1.wav", margin=MARGIN, + velocity_multiplier=20, ) @@ -167,6 +179,7 @@ def __init__(self) -> None: "Tom 2", "./resources/sounds/Tom2.wav", margin=MARGIN, + velocity_multiplier=20, ) @@ -176,4 +189,5 @@ def __init__(self) -> None: "Cymbal", "./resources/sounds/Hop_Crs.wav", margin=MARGIN, + velocity_multiplier=20, ) diff --git a/drumpy/mediapipe_pose/mediapipe_pose.py b/drumpy/mediapipe_pose/mediapipe_pose.py index 2ff7fda..1f010d2 100644 --- a/drumpy/mediapipe_pose/mediapipe_pose.py +++ b/drumpy/mediapipe_pose/mediapipe_pose.py @@ -117,7 +117,7 @@ def result_callback( :param timestamp_ms: The timestamp of the frame :return: """ - result = self.result_processor.process_result(result, timestamp_ms) + # result = self.result_processor.process_result(result, timestamp_ms) self.detection_result = result self.latency = timestamp_ms - self.latest_timestamp self.latest_timestamp = timestamp_ms diff --git a/drumpy/mediapipe_pose/process_result.py b/drumpy/mediapipe_pose/process_result.py index e6a556f..ee0309f 100644 --- a/drumpy/mediapipe_pose/process_result.py +++ b/drumpy/mediapipe_pose/process_result.py @@ -51,7 +51,6 @@ def process_result( Process the result of the pose estimation """ if result.pose_landmarks is None or len(result.pose_landmarks) == 0: - print("No pose landmarks found") return result for i, landmark in enumerate(result.pose_landmarks[0]): diff --git a/drumpy/tracking/drum_trackers.py b/drumpy/tracking/drum_trackers.py index 0c9687c..7f9a289 100644 --- a/drumpy/tracking/drum_trackers.py +++ b/drumpy/tracking/drum_trackers.py @@ -3,7 +3,7 @@ from mediapipe.tasks.python.components.containers.landmark import Landmark # pyright: ignore from drumpy.drum.drum import Drum -from drumpy.drum.sound import SnareDrum, HiHat, KickDrum, HiHatFoot +from drumpy.drum.sound import SnareDrum, HiHat, KickDrum, HiHatFoot, Cymbal from drumpy.tracking.marker_tracker_wrapper import MarkerTrackerWrapper, Foot, Hand @@ -18,13 +18,14 @@ def __init__(self) -> None: hi_hat = HiHat() kick_drum = KickDrum() hi_hat_foot = HiHatFoot() + cymbal = Cymbal() - self.drum = Drum([snare_drum, hi_hat, kick_drum]) + self.drum = Drum([snare_drum, hi_hat, kick_drum, cymbal]) self.drum.auto_calibrate() self.trackers: list[MarkerTrackerWrapper] = [ - Hand.left_hand(self.drum, [snare_drum, hi_hat]), - Hand.right_hand(self.drum, [snare_drum, hi_hat]), + Hand.left_hand(self.drum, [snare_drum, hi_hat, cymbal]), + Hand.right_hand(self.drum, [snare_drum, hi_hat, cymbal]), Foot.left_foot(self.drum, [hi_hat_foot]), Foot.right_foot(self.drum, [kick_drum]), ] diff --git a/drumpy/tracking/marker_tracker.py b/drumpy/tracking/marker_tracker.py index 214c051..aed1a7b 100644 --- a/drumpy/tracking/marker_tracker.py +++ b/drumpy/tracking/marker_tracker.py @@ -53,6 +53,9 @@ def __init__( self.downward_trend = downward_trend self.upward_trend = upward_trend + # The current z-axis velocity + self.velocity = 0 + self.drum = drum def update(self: Self, position: Position) -> None: @@ -70,9 +73,12 @@ def update(self: Self, position: Position) -> None: self.velocities.pop(0) if self.is_hit(): - self.time_until_next_hit = self.memory + self.time_until_next_hit = self.memory / 2 self.drum.find_and_play_sound( - self.positions[-self.look_ahead], self.marker, self.sounds + self.positions[-self.look_ahead], + self.marker, + self.velocity, + self.sounds, ) def get_velocity(self: Self) -> float: @@ -93,6 +99,7 @@ def is_hit(self: Self) -> bool: return False avg_z_vel = mean(self.velocities[: -self.look_ahead]) + self.velocity = avg_z_vel avg_z_look_ahead = mean(self.velocities[-self.look_ahead :]) return ( diff --git a/drumpy/tracking/marker_tracker_wrapper.py b/drumpy/tracking/marker_tracker_wrapper.py index 6d79edd..4f3b199 100644 --- a/drumpy/tracking/marker_tracker_wrapper.py +++ b/drumpy/tracking/marker_tracker_wrapper.py @@ -87,8 +87,8 @@ def __init__(self, toe_tip: MarkerEnum, drum: Drum, sounds: list[Sound]) -> None MarkerEnum.LEFT_FOOT_INDEX, drum=drum, sounds=sounds, - downward_trend=-0.004, - upward_trend=-0.001, + downward_trend=-0.003, + upward_trend=0.0, ) def update(self: Self, markers: list[Landmark]) -> None: