fix: fix camera input axis transformation and improve hit detection

drumpy/app/main.py

@@ -105,7 +105,7 @@ def start(self: Self) -> None:
 
 def main() -> None:
     app = App(
-        source=Source.FILE,
+        source=Source.CAMERA,
         running_mode=RunningMode.VIDEO,  # type: ignore
         model=LandmarkerModel.LITE,
         delegate=BaseOptions.Delegate.CPU,  # type: ignore

drumpy/app/video_source.py

@@ -155,6 +155,7 @@ def __init__(self, camera_index: int) -> None:
         self.camera_id = camera_index
         self.camera = camera.Camera(cameras[camera_index])
         self.camera.start()
+        self.camera.set_controls(hflip=True)
         original_size = self.camera.get_size()
         # Crop the image to a square aspect ratio
         min_size = min(original_size)

drumpy/mediapipe_pose/mediapipe_pose.py

@@ -117,7 +117,7 @@ def result_callback(
         :param timestamp_ms: The timestamp of the frame
         :return:
         """
-        result = self.result_processor.process_result(result, timestamp_ms)
+        # result = self.result_processor.process_result(result, timestamp_ms)
         self.detection_result = result
         self.latency = timestamp_ms - self.latest_timestamp
         self.latest_timestamp = timestamp_ms

drumpy/tracking/drum_trackers.py

@@ -3,7 +3,7 @@
 from mediapipe.tasks.python.components.containers.landmark import Landmark  # pyright: ignore
 
 from drumpy.drum.drum import Drum
-from drumpy.drum.sound import SnareDrum, HiHat, KickDrum
+from drumpy.drum.sound import SnareDrum, HiHat, KickDrum, HiHatFoot
 from drumpy.tracking.marker_tracker_wrapper import MarkerTrackerWrapper, Foot, Hand
 
 
@@ -17,14 +17,15 @@ def __init__(self) -> None:
         snare_drum = SnareDrum()
         hi_hat = HiHat()
         kick_drum = KickDrum()
+        hi_hat_foot = HiHatFoot()
 
         self.drum = Drum([snare_drum, hi_hat, kick_drum])
         self.drum.auto_calibrate()
 
         self.trackers: list[MarkerTrackerWrapper] = [
             Hand.left_hand(self.drum, [snare_drum, hi_hat]),
             Hand.right_hand(self.drum, [snare_drum, hi_hat]),
-            Foot.left_foot(self.drum, [kick_drum]),
+            Foot.left_foot(self.drum, [hi_hat_foot]),
             Foot.right_foot(self.drum, [kick_drum]),
         ]
 

drumpy/tracking/marker_tracker.py

@@ -19,9 +19,9 @@ def __init__(
         marker: MarkerEnum,
         drum: Drum,
         sounds: list[Sound],
-        memory: int = 15,
-        downward_trend: float = -0.01,
-        upward_trend: float = 0.01,
+        memory: int = 10,
+        downward_trend: float = -0.005,
+        upward_trend: float = 0.001,
     ) -> None:
         """
         Initialize the marker tracker
@@ -47,7 +47,7 @@ def __init__(
 
         # how many positions to look ahead to determine if a hit is registered
         # should be smaller than memory
-        self.look_ahead = 5
+        self.look_ahead = 3
         assert self.look_ahead < self.memory
 
         self.downward_trend = downward_trend

drumpy/tracking/marker_tracker_wrapper.py

@@ -80,10 +80,16 @@ def right_hand(drum: Drum, sounds: list[Sound]) -> MarkerTrackerWrapper:
 
 
 class Foot(MarkerTrackerWrapper):
-    def __init__(self, toe_tip: MarkerEnum, tracker: MarkerTracker) -> None:
+    def __init__(self, toe_tip: MarkerEnum, drum: Drum, sounds: list[Sound]) -> None:
         self.toe_tip = toe_tip
         self.position: Position = np.array([0, 0, 0])
-        self.tracker = tracker
+        self.tracker = MarkerTracker(
+            MarkerEnum.LEFT_FOOT_INDEX,
+            drum=drum,
+            sounds=sounds,
+            downward_trend=-0.004,
+            upward_trend=-0.001,
+        )
 
     def update(self: Self, markers: list[Landmark]) -> None:
         self.position = landmark_to_position(markers[self.toe_tip])
@@ -92,24 +98,18 @@ def update(self: Self, markers: list[Landmark]) -> None:
 
     @staticmethod
     def left_foot(drum: Drum, sounds: list[Sound]) -> MarkerTrackerWrapper:
-        toe_tip = MarkerEnum.LEFT_FOOT_INDEX
-        return Foot(
-            toe_tip, MarkerTracker(MarkerEnum.LEFT_FOOT, drum=drum, sounds=sounds)
-        )
+        return Foot(MarkerEnum.LEFT_FOOT_INDEX, drum, sounds)
 
     @staticmethod
     def right_foot(drum: Drum, sounds: list[Sound]) -> MarkerTrackerWrapper:
-        toe_tip = MarkerEnum.RIGHT_FOOT_INDEX
-        return Foot(
-            toe_tip, MarkerTracker(MarkerEnum.RIGHT_FOOT, drum=drum, sounds=sounds)
-        )
+        return Foot(MarkerEnum.RIGHT_FOOT_INDEX, drum, sounds)
 
 
 class Hand(MarkerTrackerWrapper):
-    def __init__(self, wrist: MarkerEnum, tracker: MarkerTracker) -> None:
+    def __init__(self, wrist: MarkerEnum, drum: Drum, sounds: list[Sound]) -> None:
         self.wrist = wrist
         self.position: Position = np.array([0, 0, 0])
-        self.tracker = tracker
+        self.tracker = MarkerTracker(wrist, drum=drum, sounds=sounds)
 
     def update(self: Self, markers: list[Landmark]) -> None:
         self.position = landmark_to_position(markers[self.wrist])
@@ -118,10 +118,8 @@ def update(self: Self, markers: list[Landmark]) -> None:
 
     @staticmethod
     def left_hand(drum: Drum, sounds: list[Sound]) -> MarkerTrackerWrapper:
-        wrist = MarkerEnum.LEFT_WRIST
-        return Hand(wrist, MarkerTracker(wrist, drum=drum, sounds=sounds))
+        return Hand(MarkerEnum.LEFT_WRIST, drum, sounds)
 
     @staticmethod
     def right_hand(drum: Drum, sounds: list[Sound]) -> MarkerTrackerWrapper:
-        wrist = MarkerEnum.RIGHT_WRIST
-        return Hand(wrist, MarkerTracker(wrist, drum=drum, sounds=sounds))
+        return Hand(MarkerEnum.RIGHT_WRIST, drum, sounds)

drumpy/util.py

@@ -19,16 +19,16 @@ def landmark_to_position(landmark: Landmark) -> Position:
     """
     Convert a mediapipe landmark to a numpy array
     Also switches some axes around:
-    x -> y, the horizontal axis
-    y -> z, the vertical axis
-    z -> x, the depth axis
+    x = z
+    y = y
+    z = -x
     """
     assert landmark.x is not None
     assert landmark.y is not None
     assert landmark.z is not None
-    x = float(landmark.y)
-    y = float(landmark.z)
-    z = float(landmark.x)
+    x = landmark.z
+    y = landmark.y
+    z = -landmark.x
     return np.array([x, y, z])