diff --git a/ports/nrf/common-hal/rotaryio/IncrementalEncoder.c b/ports/nrf/common-hal/rotaryio/IncrementalEncoder.c
index 28f563da6fb37..92295a7d90acd 100644
--- a/ports/nrf/common-hal/rotaryio/IncrementalEncoder.c
+++ b/ports/nrf/common-hal/rotaryio/IncrementalEncoder.c
@@ -51,10 +51,10 @@ static void _intr_handler(nrfx_gpiote_pin_t pin, nrf_gpiote_polarity_t action) {
 
     // logic from the atmel-samd port: provides some damping and scales movement
     // down by 4:1.
-    if (self->quarter >= 4) {
+    if (self->quarter > 0 && new_state == 2) {
         self->position++;
         self->quarter = 0;
-    } else if (self->quarter <= -4) {
+    } else if (self->quarter < 0 && new_state == 2) {
         self->position--;
         self->quarter = 0;
     }