diff --git a/src/master-playlist-controller.js b/src/master-playlist-controller.js
index 96ea798e2..abb1c5a24 100644
--- a/src/master-playlist-controller.js
+++ b/src/master-playlist-controller.js
@@ -1346,6 +1346,9 @@ export class MasterPlaylistController extends videojs.EventTarget {
     if (usingAudioLoader && unsupportedAudio && this.media().attributes.AUDIO) {
       const audioGroup = this.media().attributes.AUDIO;
 
+      this.mediaTypes_.AUDIO.activePlaylistLoader.pause();
+      this.audioSegmentLoader_.pause();
+      this.audioSegmentLoader_.abort();
       this.master().playlists.forEach(variant => {
         const variantAudioGroup = variant.attributes && variant.attributes.AUDIO;