diff --git a/lib/src/service/selection_service.dart b/lib/src/service/selection_service.dart
index b522aa9ce..a2e716c9e 100644
--- a/lib/src/service/selection_service.dart
+++ b/lib/src/service/selection_service.dart
@@ -347,8 +347,9 @@ class _AppFlowySelectionState extends State<AppFlowySelection>
 
   void _onPanStart(DragStartDetails details) {
     clearSelection();
+    _clearToolbar();
 
-    _panStartOffset = details.globalPosition;
+    _panStartOffset = details.globalPosition.translate(-3.0, 0);
     _panStartScrollDy = editorState.service.scrollService?.dy;
 
     _enableInteraction();