diff --git a/extensions/image-preview/media/main.js b/extensions/image-preview/media/main.js
index 94364c28503b7..a657ca04e4025 100644
--- a/extensions/image-preview/media/main.js
+++ b/extensions/image-preview/media/main.js
@@ -70,7 +70,8 @@
 	let ctrlPressed = false;
 	let altPressed = false;
 	let hasLoadedImage = false;
-	let consumeClick = false;
+	let consumeClick = true;
+	let isActive = false;
 
 	// Elements
 	const container = document.body;
@@ -117,10 +118,16 @@
 		});
 	}
 
-	function changeActive(value) {
+	function setActive(value) {
+		isActive = value;
 		if (value) {
-			container.classList.add('zoom-in');
-			consumeClick = true;
+			if (isMac ? altPressed : ctrlPressed) {
+				container.classList.remove('zoom-in');
+				container.classList.add('zoom-out');
+			} else {
+				container.classList.remove('zoom-out');
+				container.classList.add('zoom-in');
+			}
 		} else {
 			ctrlPressed = false;
 			altPressed = false;
@@ -202,7 +209,10 @@
 			return;
 		}
 
-		consumeClick = false;
+		ctrlPressed = e.ctrlKey;
+		altPressed = e.altKey;
+
+		consumeClick = !isActive;
 	});
 
 	container.addEventListener('click', (/** @type {MouseEvent} */ e) => {
@@ -214,14 +224,6 @@
 			return;
 		}
 
-		ctrlPressed = e.ctrlKey;
-		altPressed = e.altKey;
-
-		if (isMac ? altPressed : ctrlPressed) {
-			container.classList.remove('zoom-in');
-			container.classList.add('zoom-out');
-		}
-
 		if (consumeClick) {
 			consumeClick = false;
 			return;
@@ -308,7 +310,7 @@
 				break;
 
 			case 'setActive':
-				changeActive(e.data.value);
+				setActive(e.data.value);
 				break;
 
 			case 'zoomIn':
diff --git a/extensions/image-preview/src/preview.ts b/extensions/image-preview/src/preview.ts
index a5eb47e308f60..7326c0721bd5b 100644
--- a/extensions/image-preview/src/preview.ts
+++ b/extensions/image-preview/src/preview.ts
@@ -115,6 +115,7 @@ class Preview extends Disposable {
 
 		this._register(webviewEditor.onDidChangeViewState(() => {
 			this.update();
+			this.webviewEditor.webview.postMessage({ type: 'setActive', value: this.webviewEditor.active });
 		}));
 
 		this._register(webviewEditor.onDidDispose(() => {
@@ -139,6 +140,7 @@ class Preview extends Disposable {
 
 		this.render();
 		this.update();
+		this.webviewEditor.webview.postMessage({ type: 'setActive', value: this.webviewEditor.active });
 	}
 
 	public zoomIn() {
@@ -175,7 +177,6 @@ class Preview extends Disposable {
 			}
 			this._previewState = PreviewState.Visible;
 		}
-		this.webviewEditor.webview.postMessage({ type: 'setActive', value: this.webviewEditor.active });
 	}
 
 	private getWebiewContents(): string {