diff --git a/packages/block-editor/src/components/block-list/index.js b/packages/block-editor/src/components/block-list/index.js
index e2e019d4a9bf69..8a20b1ac0e0db6 100644
--- a/packages/block-editor/src/components/block-list/index.js
+++ b/packages/block-editor/src/components/block-list/index.js
@@ -233,7 +233,6 @@ function Items( {
 						<ZoomOutSeparator
 							clientId={ clientId }
 							rootClientId={ rootClientId }
-							position="top"
 						/>
 					) }
 					<BlockListBlock
@@ -244,7 +243,6 @@ function Items( {
 						<ZoomOutSeparator
 							clientId={ clientId }
 							rootClientId={ rootClientId }
-							position="bottom"
 						/>
 					) }
 				</AsyncModeProvider>
diff --git a/packages/block-editor/src/components/block-list/zoom-out-separator.js b/packages/block-editor/src/components/block-list/zoom-out-separator.js
index be5af549630607..5bdc83f5d27743 100644
--- a/packages/block-editor/src/components/block-list/zoom-out-separator.js
+++ b/packages/block-editor/src/components/block-list/zoom-out-separator.js
@@ -20,15 +20,12 @@ import { useState } from '@wordpress/element';
 import { store as blockEditorStore } from '../../store';
 import { unlock } from '../../lock-unlock';
 
-export function ZoomOutSeparator( {
-	clientId,
-	rootClientId = '',
-	position = 'top',
-} ) {
+export function ZoomOutSeparator( { clientId, rootClientId = '' } ) {
 	const [ isDraggedOver, setIsDraggedOver ] = useState( false );
 	const {
 		sectionRootClientId,
 		sectionClientIds,
+		isBlockSelected,
 		blockInsertionPoint,
 		blockInsertionPointVisible,
 	} = useSelect( ( select ) => {
@@ -37,11 +34,13 @@ export function ZoomOutSeparator( {
 			getBlockOrder,
 			isBlockInsertionPointVisible,
 			getSectionRootClientId,
+			isBlockSelected: _isBlockSelected,
 		} = unlock( select( blockEditorStore ) );
 
 		const root = getSectionRootClientId();
 		const sectionRootClientIds = getBlockOrder( root );
 		return {
+			isBlockSelected: _isBlockSelected,
 			sectionRootClientId: root,
 			sectionClientIds: sectionRootClientIds,
 			blockOrder: getBlockOrder( root ),
@@ -67,18 +66,10 @@ export function ZoomOutSeparator( {
 		return null;
 	}
 
-	if ( position === 'top' ) {
-		isVisible =
-			blockInsertionPointVisible &&
-			blockInsertionPoint.index === 0 &&
-			clientId === sectionClientIds[ blockInsertionPoint.index ];
-	}
-
-	if ( position === 'bottom' ) {
-		isVisible =
-			blockInsertionPointVisible &&
-			clientId === sectionClientIds[ blockInsertionPoint.index - 1 ];
-	}
+	isVisible =
+		isBlockSelected( clientId ) ||
+		( blockInsertionPointVisible &&
+			clientId === sectionClientIds[ blockInsertionPoint.index ] );
 
 	return (
 		<AnimatePresence>
diff --git a/packages/block-editor/src/components/block-tools/index.js b/packages/block-editor/src/components/block-tools/index.js
index 099323925384b8..a29f3f60152fde 100644
--- a/packages/block-editor/src/components/block-tools/index.js
+++ b/packages/block-editor/src/components/block-tools/index.js
@@ -22,7 +22,6 @@ import BlockToolbarPopover from './block-toolbar-popover';
 import ZoomOutPopover from './zoom-out-popover';
 import { store as blockEditorStore } from '../../store';
 import usePopoverScroll from '../block-popover/use-popover-scroll';
-import ZoomOutModeInserters from './zoom-out-mode-inserters';
 import { useShowBlockTools } from './use-show-block-tools';
 import { unlock } from '../../lock-unlock';
 import getEditorRegion from '../../utils/get-editor-region';
@@ -241,11 +240,6 @@ export default function BlockTools( {
 					name="__unstable-block-tools-after"
 					ref={ blockToolbarAfterRef }
 				/>
-				{ isZoomOutMode && (
-					<ZoomOutModeInserters
-						__unstableContentRef={ __unstableContentRef }
-					/>
-				) }
 			</InsertionPointOpenRef.Provider>
 		</div>
 	);
diff --git a/packages/block-editor/src/components/block-tools/zoom-out-mode-inserter-button.js b/packages/block-editor/src/components/block-tools/zoom-out-mode-inserter-button.js
deleted file mode 100644
index 8ea80a53830135..00000000000000
--- a/packages/block-editor/src/components/block-tools/zoom-out-mode-inserter-button.js
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * External dependencies
- */
-import clsx from 'clsx';
-
-/**
- * WordPress dependencies
- */
-import { useState } from '@wordpress/element';
-import { Button } from '@wordpress/components';
-import { plus } from '@wordpress/icons';
-import { _x } from '@wordpress/i18n';
-
-function ZoomOutModeInserterButton( { isVisible, onClick } ) {
-	const [
-		zoomOutModeInserterButtonHovered,
-		setZoomOutModeInserterButtonHovered,
-	] = useState( false );
-
-	return (
-		<Button
-			variant="primary"
-			icon={ plus }
-			size="compact"
-			className={ clsx(
-				'block-editor-button-pattern-inserter__button',
-				'block-editor-block-tools__zoom-out-mode-inserter-button',
-				{
-					'is-visible': isVisible || zoomOutModeInserterButtonHovered,
-				}
-			) }
-			onClick={ onClick }
-			onMouseOver={ () => {
-				setZoomOutModeInserterButtonHovered( true );
-			} }
-			onMouseOut={ () => {
-				setZoomOutModeInserterButtonHovered( false );
-			} }
-			label={ _x(
-				'Add pattern',
-				'Generic label for pattern inserter button'
-			) }
-		/>
-	);
-}
-
-export default ZoomOutModeInserterButton;
diff --git a/packages/block-editor/src/components/block-tools/zoom-out-mode-inserters.js b/packages/block-editor/src/components/block-tools/zoom-out-mode-inserters.js
deleted file mode 100644
index 79f8be3f9cfe97..00000000000000
--- a/packages/block-editor/src/components/block-tools/zoom-out-mode-inserters.js
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * WordPress dependencies
- */
-import { useSelect, useDispatch } from '@wordpress/data';
-import { useEffect, useState } from '@wordpress/element';
-
-/**
- * Internal dependencies
- */
-import BlockPopoverInbetween from '../block-popover/inbetween';
-import ZoomOutModeInserterButton from './zoom-out-mode-inserter-button';
-import { store as blockEditorStore } from '../../store';
-import { unlock } from '../../lock-unlock';
-
-function ZoomOutModeInserters() {
-	const [ isReady, setIsReady ] = useState( false );
-	const {
-		hasSelection,
-		blockInsertionPoint,
-		blockOrder,
-		blockInsertionPointVisible,
-		setInserterIsOpened,
-		sectionRootClientId,
-		selectedBlockClientId,
-		hoveredBlockClientId,
-	} = useSelect( ( select ) => {
-		const {
-			getSettings,
-			getBlockInsertionPoint,
-			getBlockOrder,
-			getSelectionStart,
-			getSelectedBlockClientId,
-			getHoveredBlockClientId,
-			isBlockInsertionPointVisible,
-			getSectionRootClientId,
-		} = unlock( select( blockEditorStore ) );
-
-		const root = getSectionRootClientId();
-
-		return {
-			hasSelection: !! getSelectionStart().clientId,
-			blockInsertionPoint: getBlockInsertionPoint(),
-			blockOrder: getBlockOrder( root ),
-			blockInsertionPointVisible: isBlockInsertionPointVisible(),
-			sectionRootClientId: root,
-			setInserterIsOpened:
-				getSettings().__experimentalSetIsInserterOpened,
-			selectedBlockClientId: getSelectedBlockClientId(),
-			hoveredBlockClientId: getHoveredBlockClientId(),
-		};
-	}, [] );
-
-	const { showInsertionPoint } = useDispatch( blockEditorStore );
-
-	// Defer the initial rendering to avoid the jumps due to the animation.
-	useEffect( () => {
-		const timeout = setTimeout( () => {
-			setIsReady( true );
-		}, 500 );
-		return () => {
-			clearTimeout( timeout );
-		};
-	}, [] );
-
-	if ( ! isReady ) {
-		return null;
-	}
-
-	return [ undefined, ...blockOrder ].map( ( clientId, index ) => {
-		const shouldRenderInsertionPoint =
-			blockInsertionPointVisible && blockInsertionPoint.index === index;
-
-		const previousClientId = clientId;
-		const nextClientId = blockOrder[ index ];
-
-		const isSelected =
-			hasSelection &&
-			( selectedBlockClientId === previousClientId ||
-				selectedBlockClientId === nextClientId );
-
-		const isHovered =
-			hoveredBlockClientId === previousClientId ||
-			hoveredBlockClientId === nextClientId;
-
-		return (
-			<BlockPopoverInbetween
-				key={ index }
-				previousClientId={ previousClientId }
-				nextClientId={ nextClientId }
-			>
-				{ ! shouldRenderInsertionPoint && (
-					<ZoomOutModeInserterButton
-						isVisible={ isSelected || isHovered }
-						onClick={ () => {
-							setInserterIsOpened( {
-								rootClientId: sectionRootClientId,
-								insertionIndex: index,
-								tab: 'patterns',
-								category: 'all',
-							} );
-							showInsertionPoint( sectionRootClientId, index, {
-								operation: 'insert',
-							} );
-						} }
-					/>
-				) }
-			</BlockPopoverInbetween>
-		);
-	} );
-}
-
-export default ZoomOutModeInserters;
diff --git a/packages/editor/src/components/zoom-out-toggle/index.js b/packages/editor/src/components/zoom-out-toggle/index.js
index dd4669d04bec18..765972f732c27f 100644
--- a/packages/editor/src/components/zoom-out-toggle/index.js
+++ b/packages/editor/src/components/zoom-out-toggle/index.js
@@ -12,6 +12,7 @@ import { store as preferencesStore } from '@wordpress/preferences';
 /**
  * Internal dependencies
  */
+import { store as editorStore } from '../../store';
 import { unlock } from '../../lock-unlock';
 
 const ZoomOutToggle = () => {
@@ -27,11 +28,18 @@ const ZoomOutToggle = () => {
 		useDispatch( blockEditorStore )
 	);
 
+	const { setIsInserterOpened } = unlock( useDispatch( editorStore ) );
+
 	const handleZoomOut = () => {
 		if ( isZoomOut ) {
+			setIsInserterOpened( false );
 			resetZoomLevel();
 		} else {
 			setZoomLevel( 50 );
+			setIsInserterOpened( {
+				tab: 'patterns',
+				category: 'all',
+			} );
 		}
 		__unstableSetEditorMode( isZoomOut ? 'edit' : 'zoom-out' );
 	};