From afbef60e58474f2caeb750489645325b65725c36 Mon Sep 17 00:00:00 2001 From: Amanda Lin Dietz Date: Fri, 25 Oct 2024 22:59:00 +0000 Subject: [PATCH] [FaceGaze] Exclude blink when detecting eye squint left/right * Add logic to GestureDetector to ignore certain instances of gestures if certain MediapipeFacialGestures are detected at the same time at a confidence of > 0.5. * For EYE_SQUINT_LEFT and EYE_SQUINT_RIGHT, if an eye squint or blink on the other side is also detected and at > 0.5, do not register the gesture as the user is likely blinking involuntarily. * Add test to check that blinking will not trigger EYE_SQUINT_LEFT or EYE_SQUINT_RIGHT. AX-Relnotes: N/A Fixed: 374774499 Change-Id: I937588c4ac1246e428b86b430a77fa58aa4c6d79 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/5962467 Reviewed-by: Akihiro Ota Commit-Queue: Amanda Lin Dietz Cr-Commit-Position: refs/heads/main@{#1374180} --- .../facegaze/facegaze_test.js | 89 +++++++++++++++++++ .../facegaze/gesture_detector.ts | 55 ++++++++++++ 2 files changed, 144 insertions(+) diff --git a/chrome/browser/resources/chromeos/accessibility/accessibility_common/facegaze/facegaze_test.js b/chrome/browser/resources/chromeos/accessibility/accessibility_common/facegaze/facegaze_test.js index ad5c9b66948d37..2ae03ac0599008 100644 --- a/chrome/browser/resources/chromeos/accessibility/accessibility_common/facegaze/facegaze_test.js +++ b/chrome/browser/resources/chromeos/accessibility/accessibility_common/facegaze/facegaze_test.js @@ -2221,3 +2221,92 @@ AX_TEST_F('FaceGazeTest', 'GesturesDisabledDuringDictation', async function() { .addGestureWithConfidence(MediapipeFacialGesture.JAW_OPEN, 0.9)); assertEquals(result.macros.length, 2); }); + +AX_TEST_F('FaceGazeTest', 'BlinkDoesNotTriggerEyeSquint', async function() { + const gestureToMacroName = + new Map() + .set(FacialGesture.EYES_BLINK, MacroName.MOUSE_CLICK_LEFT) + .set(FacialGesture.EYE_SQUINT_LEFT, MacroName.TOGGLE_SCROLL_MODE) + .set(FacialGesture.EYE_SQUINT_RIGHT, MacroName.TOGGLE_FACEGAZE); + const gestureToConfidence = new Map() + .set(FacialGesture.EYES_BLINK, 0.6) + .set(FacialGesture.EYE_SQUINT_LEFT, 0.6) + .set(FacialGesture.EYE_SQUINT_RIGHT, 0.6); + const config = new Config() + .withMouseLocation({x: 600, y: 400}) + .withGestureToMacroName(gestureToMacroName) + .withGestureToConfidence(gestureToConfidence) + .withRepeatDelayMs(0); + await this.configureFaceGaze(config); + + const gestureHandler = this.getFaceGaze().gestureHandler_; + + // If eye squint on one side occurs at same time as a blink or squint on the + // wrong side, then the gesture should not register as an eye squint on the + // intended side. + let result = gestureHandler.detectMacros( + new MockFaceLandmarkerResult() + .addGestureWithConfidence(MediapipeFacialGesture.EYE_SQUINT_LEFT, 0.9) + .addGestureWithConfidence(MediapipeFacialGesture.EYE_BLINK_LEFT, 0.6) + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_BLINK_RIGHT, 0.6)); + assertEquals(result.macros.length, 1); + + result = gestureHandler.detectMacros( + new MockFaceLandmarkerResult() + .addGestureWithConfidence(MediapipeFacialGesture.EYE_SQUINT_LEFT, 0.9) + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_SQUINT_RIGHT, 0.6)); + assertEquals(result.macros.length, 0); + + result = gestureHandler.detectMacros( + new MockFaceLandmarkerResult() + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_SQUINT_RIGHT, 0.9) + .addGestureWithConfidence(MediapipeFacialGesture.EYE_BLINK_LEFT, 0.6) + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_BLINK_RIGHT, 0.6)); + assertEquals(result.macros.length, 1); + + result = gestureHandler.detectMacros( + new MockFaceLandmarkerResult() + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_SQUINT_RIGHT, 0.9) + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_SQUINT_LEFT, 0.6)); + assertEquals(result.macros.length, 0); + + // Low confidence levels of blink or squint on the wrong side should register + // as a squint on the intended side. + result = gestureHandler.detectMacros( + new MockFaceLandmarkerResult() + .addGestureWithConfidence(MediapipeFacialGesture.EYE_SQUINT_LEFT, 0.9) + .addGestureWithConfidence(MediapipeFacialGesture.EYE_BLINK_LEFT, 0.3) + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_BLINK_RIGHT, 0.3)); + assertEquals(result.macros.length, 1); + + result = gestureHandler.detectMacros( + new MockFaceLandmarkerResult() + .addGestureWithConfidence(MediapipeFacialGesture.EYE_SQUINT_LEFT, 0.9) + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_SQUINT_RIGHT, 0.3)); + assertEquals(result.macros.length, 1); + + result = gestureHandler.detectMacros( + new MockFaceLandmarkerResult() + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_SQUINT_RIGHT, 0.9) + .addGestureWithConfidence(MediapipeFacialGesture.EYE_BLINK_LEFT, 0.3) + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_BLINK_RIGHT, 0.3)); + assertEquals(result.macros.length, 1); + + result = gestureHandler.detectMacros( + new MockFaceLandmarkerResult() + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_SQUINT_RIGHT, 0.9) + .addGestureWithConfidence( + MediapipeFacialGesture.EYE_SQUINT_LEFT, 0.3)); + assertEquals(result.macros.length, 1); +}); diff --git a/chrome/browser/resources/chromeos/accessibility/accessibility_common/facegaze/gesture_detector.ts b/chrome/browser/resources/chromeos/accessibility/accessibility_common/facegaze/gesture_detector.ts index e1083a74d5bf03..bba6f0b79a3a99 100644 --- a/chrome/browser/resources/chromeos/accessibility/accessibility_common/facegaze/gesture_detector.ts +++ b/chrome/browser/resources/chromeos/accessibility/accessibility_common/facegaze/gesture_detector.ts @@ -110,6 +110,28 @@ export const FacialGesturesToMediapipeGestures = new Map([ ], ]); +/** + * Mapping of gestures supported by FaceGaze to mediapipe gestures that should + * not be present in order to confidently trigger the supported gesture, so as + * to reduce triggering gestures with involuntary facial movement like blinking. + */ +export const FacialGesturesToExcludedMediapipeGestures = new Map([ + [ + FacialGesture.EYE_SQUINT_LEFT, + [ + MediapipeFacialGesture.EYE_BLINK_RIGHT, + MediapipeFacialGesture.EYE_SQUINT_RIGHT, + ] + ], + [ + FacialGesture.EYE_SQUINT_RIGHT, + [ + MediapipeFacialGesture.EYE_BLINK_LEFT, + MediapipeFacialGesture.EYE_SQUINT_LEFT, + ] + ], +]); + export class GestureDetector { private static mediapipeFacialGestureSet_ = new Set(Object.values(MediapipeFacialGesture)); @@ -174,6 +196,26 @@ export class GestureDetector { continue; } + let hasExcludedGesture = false; + const excludedGestures = + FacialGesturesToExcludedMediapipeGestures.get(faceGazeGesture); + + // Check for gestures that should not be present in order to confidently + // register this gesture. + if (excludedGestures) { + for (const excludedGesture of excludedGestures) { + if (recognizedGestures.has(excludedGesture) && + recognizedGestures.get(excludedGesture) > + GestureDetector.EXCLUDED_GESTURE_THRESHOLD) { + hasExcludedGesture = true; + } + } + } + + if (hasExcludedGesture) { + continue; + } + // For gestures detected with a confidence value over a threshold value of // 1, add the gesture and confidence value to the array of information // that will be sent to settings. @@ -199,6 +241,19 @@ export class GestureDetector { } } +export namespace GestureDetector { + /** + * The confidence level at which a detected gesture B should disqualify a + * different gesture A from being triggered if A relies on B not being + * present. For example, if EYE_SQUINT_LEFT is detected but EYE_SQUINT_RIGHT + * is also detected at confidence > EXCLUDED_GESTURE_THRESHOLD, then the + * gesture was likely involuntary blinking and EYE_SQUINT_LEFT should not be + * triggered. A confidence level of 0.5 seems to be the approximate confidence + * that occurs for most involuntary blinking. + */ + export const EXCLUDED_GESTURE_THRESHOLD = 0.5; +} + TestImportManager.exportForTesting( GestureDetector, ['FacialGesture', FacialGesture], ['MediapipeFacialGesture', MediapipeFacialGesture],