diff --git a/src/controller/level-controller.ts b/src/controller/level-controller.ts
index 7f36de467d3..181c3fc17ea 100644
--- a/src/controller/level-controller.ts
+++ b/src/controller/level-controller.ts
@@ -369,7 +369,8 @@ export default class LevelController extends BasePlaylistController {
           data.frag?.type === PlaylistLevelType.MAIN
             ? data.frag.level
             : this.currentLevelIndex;
-        levelError = true;
+        // Do not retry level. Escalate to fatal if switching levels fails.
+        data.levelRetry = false;
         break;
       case ErrorDetails.LEVEL_LOAD_ERROR:
       case ErrorDetails.LEVEL_LOAD_TIMEOUT:
@@ -444,6 +445,9 @@ export default class LevelController extends BasePlaylistController {
           this.warn(`${errorDetails}: switch to ${nextLevel}`);
           errorEvent.levelRetry = true;
           this.hls.nextAutoLevel = nextLevel;
+        } else if (errorEvent.levelRetry === false) {
+          // No levels to switch to and no more retries
+          errorEvent.fatal = true;
         }
       }
     }