diff --git a/RELEASENOTES.md b/RELEASENOTES.md
index 9f83a40e0fc..02876af90c0 100644
--- a/RELEASENOTES.md
+++ b/RELEASENOTES.md
@@ -2,6 +2,8 @@
 
 ### dev-v2 (not yet released) ###
 
+* Fix issue where audio focus handling could not be disabled after enabling
+  it ([#5055](https://github.com/google/ExoPlayer/issues/5055)).
 * Support for playing spherical videos on Daydream.
 * Improve decoder re-use between playbacks. TODO: Write and link a blog post
   here ([#2826](https://github.com/google/ExoPlayer/issues/2826)).
diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioFocusManager.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioFocusManager.java
index 39422c28734..09fd1f3f02c 100644
--- a/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioFocusManager.java
+++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/AudioFocusManager.java
@@ -141,10 +141,6 @@ public float getVolumeMultiplier() {
    */
   public @PlayerCommand int setAudioAttributes(
       @Nullable AudioAttributes audioAttributes, boolean playWhenReady, int playerState) {
-    if (audioAttributes == null) {
-      return PLAYER_COMMAND_PLAY_WHEN_READY;
-    }
-
     if (!Util.areEqual(this.audioAttributes, audioAttributes)) {
       this.audioAttributes = audioAttributes;
       focusGain = convertAudioAttributesToFocusGain(audioAttributes);
diff --git a/library/core/src/test/java/com/google/android/exoplayer2/audio/AudioFocusManagerTest.java b/library/core/src/test/java/com/google/android/exoplayer2/audio/AudioFocusManagerTest.java
index 1cf812559c0..b175828d442 100644
--- a/library/core/src/test/java/com/google/android/exoplayer2/audio/AudioFocusManagerTest.java
+++ b/library/core/src/test/java/com/google/android/exoplayer2/audio/AudioFocusManagerTest.java
@@ -58,7 +58,38 @@ public void setAudioAttributes_withNullUsage_doesNotManageAudioFocus() {
     assertThat(
             audioFocusManager.setAudioAttributes(
                 /* audioAttributes= */ null, /* playWhenReady= */ false, Player.STATE_IDLE))
+        .isEqualTo(PLAYER_COMMAND_WAIT_FOR_CALLBACK);
+    assertThat(
+            audioFocusManager.setAudioAttributes(
+                /* audioAttributes= */ null, /* playWhenReady= */ true, Player.STATE_READY))
         .isEqualTo(PLAYER_COMMAND_PLAY_WHEN_READY);
+    ShadowAudioManager.AudioFocusRequest request =
+        Shadows.shadowOf(audioManager).getLastAudioFocusRequest();
+    assertThat(request).isNull();
+  }
+
+  @Test
+  public void setAudioAttributes_withNullUsage_releasesAudioFocus() {
+    // Create attributes and request audio focus.
+    AudioAttributes media = new AudioAttributes.Builder().setUsage(C.USAGE_MEDIA).build();
+    Shadows.shadowOf(audioManager)
+        .setNextFocusRequestResponse(AudioManager.AUDIOFOCUS_REQUEST_GRANTED);
+    assertThat(
+            audioFocusManager.setAudioAttributes(
+                media, /* playWhenReady= */ true, Player.STATE_READY))
+        .isEqualTo(PLAYER_COMMAND_PLAY_WHEN_READY);
+    ShadowAudioManager.AudioFocusRequest request =
+        Shadows.shadowOf(audioManager).getLastAudioFocusRequest();
+    assertThat(request.durationHint).isEqualTo(AudioManager.AUDIOFOCUS_GAIN);
+
+    // Ensure that setting null audio attributes with audio focus releases audio focus.
+    assertThat(
+            audioFocusManager.setAudioAttributes(
+                /* audioAttributes= */ null, /* playWhenReady= */ true, Player.STATE_READY))
+        .isEqualTo(PLAYER_COMMAND_PLAY_WHEN_READY);
+    AudioManager.OnAudioFocusChangeListener lastRequest =
+        Shadows.shadowOf(audioManager).getLastAbandonedAudioFocusListener();
+    assertThat(lastRequest).isNotNull();
   }
 
   @Test
@@ -296,7 +327,7 @@ public void handleStop_withoutHandlingAudioFocus_isNoOp() {
     assertThat(
             audioFocusManager.setAudioAttributes(
                 /* audioAttributes= */ null, /* playWhenReady= */ false, Player.STATE_READY))
-        .isEqualTo(PLAYER_COMMAND_PLAY_WHEN_READY);
+        .isEqualTo(PLAYER_COMMAND_DO_NOT_PLAY);
     assertThat(Shadows.shadowOf(audioManager).getLastAbandonedAudioFocusListener()).isNull();
     ShadowAudioManager.AudioFocusRequest request =
         Shadows.shadowOf(audioManager).getLastAudioFocusRequest();