diff --git a/src/ui/handler/drag_pan.js b/src/ui/handler/drag_pan.js
index 197e9ddb17b..877bb3953ae 100644
--- a/src/ui/handler/drag_pan.js
+++ b/src/ui/handler/drag_pan.js
@@ -27,7 +27,6 @@ class DragPanHandler {
     _active: boolean;
     _pos: Point;
     _previousPos: Point;
-    _startPos: Point;
     _inertia: Array<[number, Point]>;
     _lastMoveEvent: MouseEvent | TouchEvent | void;
 
@@ -107,8 +106,8 @@ class DragPanHandler {
         window.addEventListener('blur', this._onMouseUp);
 
         this._active = false;
-        this._startPos = this._previousPos = DOM.mousePos(this._el, e);
-        this._inertia = [[browser.now(), this._startPos]];
+        this._previousPos = DOM.mousePos(this._el, e);
+        this._inertia = [[browser.now(), this._previousPos]];
     }
 
     _onMove(e: MouseEvent | TouchEvent) {
@@ -116,9 +115,18 @@ class DragPanHandler {
         this._lastMoveEvent = e;
         e.preventDefault();
 
-        this._pos = DOM.mousePos(this._el, e);
+        const pos = DOM.mousePos(this._el, e);
         this._drainInertiaBuffer();
-        this._inertia.push([browser.now(), this._pos]);
+        this._inertia.push([browser.now(), pos]);
+
+        // if the dragging animation was interrupted (e.g. by another handler),
+        // we need to reestablish a _previousPos before we can resume dragging
+        if (!this._previousPos) {
+            this._previousPos = pos;
+            return;
+        }
+
+        this._pos = pos;
 
         if (!this.isActive()) {
             // we treat the first move event (rather than the mousedown event)
@@ -160,7 +168,6 @@ class DragPanHandler {
 
         this._active = false;
         delete this._lastMoveEvent;
-        delete this._startPos;
         delete this._previousPos;
         delete this._pos;
 
diff --git a/src/ui/handler/drag_rotate.js b/src/ui/handler/drag_rotate.js
index 5b3f46315f5..7251cd31760 100644
--- a/src/ui/handler/drag_rotate.js
+++ b/src/ui/handler/drag_rotate.js
@@ -36,7 +36,6 @@ class DragRotateHandler {
     _lastMoveEvent: MouseEvent;
     _pos: Point;
     _previousPos: Point;
-    _startPos: Point;
     _inertia: Array<[number, number]>;
     _center: Point;
 
@@ -132,7 +131,7 @@ class DragRotateHandler {
 
         this._active = false;
         this._inertia = [[browser.now(), this._map.getBearing()]];
-        this._startPos = this._previousPos = DOM.mousePos(this._el, e);
+        this._previousPos = DOM.mousePos(this._el, e);
         this._center = this._map.transform.centerPoint;  // Center of rotation
 
         e.preventDefault();
@@ -140,7 +139,15 @@ class DragRotateHandler {
 
     _onMove(e: MouseEvent) {
         this._lastMoveEvent = e;
-        this._pos = DOM.mousePos(this._el, e);
+        const pos = DOM.mousePos(this._el, e);
+        // if the dragging animation was interrupted (e.g. by another handler),
+        // we need to reestablish a _previousPos before we can resume dragging
+        if (!this._previousPos) {
+            this._previousPos = pos;
+            return;
+        }
+
+        this._pos = pos;
 
         if (!this.isActive()) {
             this._active = true;
diff --git a/test/unit/ui/handler/drag_pan.test.js b/test/unit/ui/handler/drag_pan.test.js
index 579b809ee35..03a7aa3d6a9 100644
--- a/test/unit/ui/handler/drag_pan.test.js
+++ b/test/unit/ui/handler/drag_pan.test.js
@@ -32,3 +32,22 @@ test('DragPanHandler requests a new render frame after each mousemove event', (t
     t.equal(update.callCount, 1);
     t.end();
 });
+
+test('DragPanHandler recovers after interruption by another handler', (t) => {
+    // https://github.com/mapbox/mapbox-gl-js/issues/6106 
+    const map = createMap();
+    const initialCenter = map.getCenter();
+    simulate.mousedown(map.getCanvas(), {bubbles: true, buttons: 2, clientX: 10, clientY: 10});
+    simulate.mousemove(map.getCanvas(), {bubbles: true, buttons: 2, clientX: 12, clientY: 10});
+    map._updateCamera();
+
+    // simluates another handler taking over
+    map.stop();
+
+    simulate.mousemove(map.getCanvas(), {bubbles: true, buttons: 2, clientX: 14, clientY: 10});
+    simulate.mousemove(map.getCanvas(), {bubbles: true, buttons: 2, clientX: 16, clientY: 10});
+    map._updateCamera();
+    t.equalWithPrecision(map.getCenter().lng, initialCenter.lng - 2.8125, 1e-4);
+    t.end();
+});
+
diff --git a/test/unit/ui/handler/drag_rotate.test.js b/test/unit/ui/handler/drag_rotate.test.js
index 9f84c417930..876329e8fdf 100644
--- a/test/unit/ui/handler/drag_rotate.test.js
+++ b/test/unit/ui/handler/drag_rotate.test.js
@@ -361,3 +361,20 @@ test('DragRotateHandler ends rotation if the window blurs (#3389)', (t) => {
 
     t.end();
 });
+
+test('DragRotateHandler recovers after interruptino by another handler', (t) => {
+    // https://github.com/mapbox/mapbox-gl-js/issues/6106 
+    const map = createMap();
+    const initialBearing = map.getBearing();
+    simulate.mousedown(map.getCanvas(), {bubbles: true, buttons: 2, button: 2, clientX: 10, clientY: 10});
+    simulate.mousemove(map.getCanvas(), {bubbles: true, buttons: 2, clientX: 12, clientY: 10});
+    map._updateCamera();
+
+    // simluates another handler taking over
+    map.stop();
+    simulate.mousemove(map.getCanvas(), {bubbles: true, buttons: 2, clientX: 12, clientY: 10});
+    simulate.mousemove(map.getCanvas(), {bubbles: true, buttons: 2, clientX: 14, clientY: 10});
+    map._updateCamera();
+    t.equalWithPrecision(map.getBearing(), initialBearing + 3.2, 1e-6);
+    t.end();
+});