feat: improve tracking and drum

drumpy/app/main.py

@@ -41,6 +41,7 @@ def __init__(
         self.model = model
 
         pygame.init()
+        # pygame.mixer.init(channels=8)
 
         pygame.display.set_caption("DrumPy")
         initial_window_size = (900, 900)

drumpy/drum/drum.py

@@ -63,6 +63,7 @@ def find_and_play_sound(
         self: Self,
         position: Position,
         marker: MarkerEnum,
+        velocity: float,
         sounds: Optional[list[Sound]] = None,
     ) -> None:
         """
@@ -71,6 +72,7 @@ def find_and_play_sound(
         :param marker: The marker that hit the sound
         :param sounds: List of sounds to consider, if None, all sounds will be considered
         :param position: A 3D position as a numpy array
+        :param velocity: The velocity of the marker, used to determine the hit strength and sound volume
         :return:
         """
 
@@ -81,7 +83,7 @@ def find_and_play_sound(
             if (distance := sound.is_hit(position)) is not None:
                 # Check if the sound is calibrating
                 if sound.state == SoundState.CALIBRATING:
-                    sound.hit(position)
+                    sound.hit(position, velocity)
                     print(
                         f"\t{marker}: {sound.name} with distance {distance:.3f} at {position_str(position)}"
                     )
@@ -92,7 +94,7 @@ def find_and_play_sound(
                     closest_distance = distance
 
         if closest_sound is not None:
-            closest_sound.hit(position)
+            closest_sound.hit(position, velocity)
             print(
                 f"{marker}: {closest_sound.name} with distance {closest_distance:.3f} "
                 f"at {position_str(position)}"

drumpy/drum/sound.py

@@ -34,6 +34,7 @@ def __init__(
         path: str,
         margin: float,
         position: Optional[Position] = None,
+        velocity_multiplier: float = 20,
     ) -> None:
         """
         Initialize the sound
@@ -56,6 +57,8 @@ def __init__(
         self.hit_count = 0
         self.hits: list[Position] = []
 
+        self.velocity_multiplier = velocity_multiplier
+
         # the margin that the sound can be hit with, aka the distance to the sound, or the size of the sound area
         self.margin: float = margin
 
@@ -106,11 +109,15 @@ def is_hit(self: Self, position: Position) -> Optional[float]:
                     return distance
                 return None
 
-    def hit(self: Self, position: Position) -> None:
+    def hit(self: Self, position: Position, velocity: float) -> None:
         """
         Update the position of the sound slowly to the given position and play it
-        :param position:
+        :param position: The position of the hit
+        :param velocity: The velocity of the hit, used to determine the volume of the sound
         """
+        # print(f"\n{self.name} hit at {position_str(position)} with velocity {velocity}")
+        volume = min(1.0, max(0.0, abs(velocity) * self.velocity_multiplier))
+        self.sound.set_volume(volume)
         self.sound.play()
         self.hit_count += 1
         self.position = 0.99 * self.position + 0.01 * position
@@ -122,6 +129,7 @@ def __init__(self) -> None:
             "Snare Drum",
             "./resources/sounds/CKV1_Snare Loud.wav",
             margin=MARGIN,
+            velocity_multiplier=20,
         )
 
 
@@ -131,6 +139,7 @@ def __init__(self) -> None:
             "High Hat",
             "./resources/sounds/CKV1_HH Closed Loud.wav",
             margin=MARGIN,
+            velocity_multiplier=20,
         )
 
 
@@ -140,6 +149,7 @@ def __init__(self) -> None:
             "Kick Drum",
             "./resources/sounds/CKV1_Kick Loud.wav",
             margin=MARGIN,
+            velocity_multiplier=40,
         )
 
 
@@ -149,6 +159,7 @@ def __init__(self) -> None:
             "High Hat Foot",
             "./resources/sounds/CKV1_HH Foot.wav",
             margin=MARGIN,
+            velocity_multiplier=40,
         )
 
 
@@ -158,6 +169,7 @@ def __init__(self) -> None:
             "Tom 1",
             "./DrumSamples/Tom1.wav",
             margin=MARGIN,
+            velocity_multiplier=20,
         )
 
 
@@ -167,6 +179,7 @@ def __init__(self) -> None:
             "Tom 2",
             "./resources/sounds/Tom2.wav",
             margin=MARGIN,
+            velocity_multiplier=20,
         )
 
 
@@ -176,4 +189,5 @@ def __init__(self) -> None:
             "Cymbal",
             "./resources/sounds/Hop_Crs.wav",
             margin=MARGIN,
+            velocity_multiplier=20,
         )

drumpy/mediapipe_pose/mediapipe_pose.py

@@ -117,7 +117,7 @@ def result_callback(
         :param timestamp_ms: The timestamp of the frame
         :return:
         """
-        result = self.result_processor.process_result(result, timestamp_ms)
+        # result = self.result_processor.process_result(result, timestamp_ms)
         self.detection_result = result
         self.latency = timestamp_ms - self.latest_timestamp
         self.latest_timestamp = timestamp_ms

drumpy/mediapipe_pose/process_result.py

@@ -51,7 +51,6 @@ def process_result(
         Process the result of the pose estimation
         """
         if result.pose_landmarks is None or len(result.pose_landmarks) == 0:
-            print("No pose landmarks found")
             return result
 
         for i, landmark in enumerate(result.pose_landmarks[0]):

drumpy/tracking/drum_trackers.py

@@ -3,7 +3,7 @@
 from mediapipe.tasks.python.components.containers.landmark import Landmark  # pyright: ignore
 
 from drumpy.drum.drum import Drum
-from drumpy.drum.sound import SnareDrum, HiHat, KickDrum, HiHatFoot
+from drumpy.drum.sound import SnareDrum, HiHat, KickDrum, HiHatFoot, Cymbal
 from drumpy.tracking.marker_tracker_wrapper import MarkerTrackerWrapper, Foot, Hand
 
 
@@ -18,13 +18,14 @@ def __init__(self) -> None:
         hi_hat = HiHat()
         kick_drum = KickDrum()
         hi_hat_foot = HiHatFoot()
+        cymbal = Cymbal()
 
-        self.drum = Drum([snare_drum, hi_hat, kick_drum])
+        self.drum = Drum([snare_drum, hi_hat, kick_drum, cymbal])
         self.drum.auto_calibrate()
 
         self.trackers: list[MarkerTrackerWrapper] = [
-            Hand.left_hand(self.drum, [snare_drum, hi_hat]),
-            Hand.right_hand(self.drum, [snare_drum, hi_hat]),
+            Hand.left_hand(self.drum, [snare_drum, hi_hat, cymbal]),
+            Hand.right_hand(self.drum, [snare_drum, hi_hat, cymbal]),
             Foot.left_foot(self.drum, [hi_hat_foot]),
             Foot.right_foot(self.drum, [kick_drum]),
         ]

drumpy/tracking/marker_tracker.py

@@ -53,6 +53,9 @@ def __init__(
         self.downward_trend = downward_trend
         self.upward_trend = upward_trend
 
+        # The current z-axis velocity
+        self.velocity = 0
+
         self.drum = drum
 
     def update(self: Self, position: Position) -> None:
@@ -70,9 +73,12 @@ def update(self: Self, position: Position) -> None:
             self.velocities.pop(0)
 
         if self.is_hit():
-            self.time_until_next_hit = self.memory
+            self.time_until_next_hit = self.memory / 2
             self.drum.find_and_play_sound(
-                self.positions[-self.look_ahead], self.marker, self.sounds
+                self.positions[-self.look_ahead],
+                self.marker,
+                self.velocity,
+                self.sounds,
             )
 
     def get_velocity(self: Self) -> float:
@@ -93,6 +99,7 @@ def is_hit(self: Self) -> bool:
             return False
 
         avg_z_vel = mean(self.velocities[: -self.look_ahead])
+        self.velocity = avg_z_vel
         avg_z_look_ahead = mean(self.velocities[-self.look_ahead :])
 
         return (

drumpy/tracking/marker_tracker_wrapper.py

@@ -87,8 +87,8 @@ def __init__(self, toe_tip: MarkerEnum, drum: Drum, sounds: list[Sound]) -> None
             MarkerEnum.LEFT_FOOT_INDEX,
             drum=drum,
             sounds=sounds,
-            downward_trend=-0.004,
-            upward_trend=-0.001,
+            downward_trend=-0.003,
+            upward_trend=0.0,
         )
 
     def update(self: Self, markers: list[Landmark]) -> None: