diff --git a/src/sync-controller.js b/src/sync-controller.js
index 46bce919a..fdab91c4e 100644
--- a/src/sync-controller.js
+++ b/src/sync-controller.js
@@ -5,7 +5,7 @@
 import {sumDurations, getPartsAndSegments} from './playlist';
 import videojs from 'video.js';
 import logger from './util/logger';
-import MediaSequenceSync from './util/media-sequence-sync';
+import {MediaSequenceSync, DependantMediaSequenceSync} from './util/media-sequence-sync';
 
 // The maximum gap allowed between two media sequence tags when trying to
 // synchronize expired playlist segments.
@@ -233,11 +233,11 @@ export default class SyncController extends videojs.EventTarget {
     //  For some reason this map helps with syncing between quality switch for MPEG-DASH as well.
     //  Moreover if we disable this map for MPEG-DASH - quality switch will be broken.
     //  MPEG-DASH should have its own separate sync strategy
-    this.mediaSequenceStorage_ = {
-      main: new MediaSequenceSync(),
-      audio: new MediaSequenceSync(),
-      vtt: new MediaSequenceSync()
-    };
+    const main = new MediaSequenceSync();
+    const audio = new DependantMediaSequenceSync(main);
+    const vtt = new DependantMediaSequenceSync(main);
+
+    this.mediaSequenceStorage_ = {main, audio, vtt};
     this.logger_ = logger('SyncController');
   }
 
diff --git a/src/util/media-sequence-sync.js b/src/util/media-sequence-sync.js
index 41797cea6..21aeb6eb9 100644
--- a/src/util/media-sequence-sync.js
+++ b/src/util/media-sequence-sync.js
@@ -79,11 +79,11 @@ class SyncInfoData {
   }
 }
 
-export default class MediaSequenceSync {
+export class MediaSequenceSync {
   constructor() {
     /**
      * @type {Map<number, SyncInfoData>}
-     * @private
+     * @protected
      */
     this.storage_ = new Map();
     this.diagnostics_ = '';
@@ -160,6 +160,10 @@ export default class MediaSequenceSync {
     return null;
   }
 
+  getSyncInfoForMediaSequence(mediaSequence) {
+    return this.storage_.get(mediaSequence);
+  }
+
   updateStorage_(segments, startingMediaSequence, startingTime) {
     const newStorage = new Map();
     let newDiagnostics = '\n';
@@ -244,3 +248,25 @@ export default class MediaSequenceSync {
     return mediaSequence !== undefined && mediaSequence !== null && Array.isArray(segments) && segments.length;
   }
 }
+
+export class DependantMediaSequenceSync extends MediaSequenceSync {
+  constructor(parent) {
+    super();
+
+    this.parent_ = parent;
+  }
+
+  calculateBaseTime_(mediaSequence, fallback) {
+    if (!this.storage_.size) {
+      const info = this.parent_.getSyncInfoForMediaSequence(mediaSequence);
+
+      if (info) {
+        return info.segmentSyncInfo.start;
+      }
+
+      return 0;
+    }
+
+    return super.calculateBaseTime_(mediaSequence, fallback);
+  }
+}