diff --git a/packages/rich-text/src/remove-format.js b/packages/rich-text/src/remove-format.js
index 4a4c9c820f9008..5ce2fcd7ce8333 100644
--- a/packages/rich-text/src/remove-format.js
+++ b/packages/rich-text/src/remove-format.js
@@ -48,11 +48,6 @@ export function removeFormat(
 				filterFormats( newFormats, endIndex, formatType );
 				endIndex++;
 			}
-		} else {
-			return {
-				...value,
-				activeFormats: reject( activeFormats, { type: formatType } ),
-			};
 		}
 	} else {
 		for ( let i = startIndex; i < endIndex; i++ ) {
@@ -62,7 +57,11 @@ export function removeFormat(
 		}
 	}
 
-	return normaliseFormats( { ...value, formats: newFormats } );
+	return normaliseFormats( {
+		...value,
+		formats: newFormats,
+		activeFormats: reject( activeFormats, { type: formatType } ),
+	} );
 }
 
 function filterFormats( formats, index, formatType ) {
diff --git a/packages/rich-text/src/test/remove-format.js b/packages/rich-text/src/test/remove-format.js
index 4d6b10f0d27fbe..343eb8c47e011f 100644
--- a/packages/rich-text/src/test/remove-format.js
+++ b/packages/rich-text/src/test/remove-format.js
@@ -21,6 +21,7 @@ describe( 'removeFormat', () => {
 		};
 		const expected = {
 			formats: [ , , , , [ em ], [ em ], [ em ], , , , , , , ],
+			activeFormats: [],
 			text: 'one two three',
 		};
 		const result = removeFormat( deepFreeze( record ), 'strong', 3, 6 );
@@ -37,6 +38,7 @@ describe( 'removeFormat', () => {
 		};
 		const expected = {
 			formats: [ , , , , [ em ], [ em ], [ em ], , , , , , , ],
+			activeFormats: [],
 			text: 'one two three',
 		};
 		const result = removeFormat( deepFreeze( record ), 'strong', 4, 4 );