[MSE] [webcodecs] API for playing webcodecs media with MSE

Stubs new MSE methods and overloads that, when fully implemented in
later changes, would allow:

1. use of WebCodecs decoder configs as addSourceBuffer() and
   changeType() arguments (in lieu of parsing initialization segments
   from a container bytestream), and
2. buffering of WebCodecs encoded chunks via appendEncodedChunks() (in
   lieu of parsing media segments from a container bytestream).

Much of the complexity of this initial change is in the coordination of
the IDL bindings generator to achieve disambiguated overload resolution,
primarily to keep the exposed API simple (only 1 actual new method name
is added, corresponding to bullet 2, above), using two approaches:

* Dictionary of Dictionaries: SourceBufferConfig wraps either a
  WebCodecs audio or video decoder configuration. Without such a
  distinct new type wrapping them, unioning or overloading would fail to
  resolve.

* Unions, with caveats: the new appendEncodedChunks method takes either
  sequences of audio or video chunks, or single audio or video chunks,
  all in a single argument of IDL union type.

  Caveat: "sequence<A> or sequence<V>" cannot be disambiguated by the
  bindings, so sequence<A or V> is used in this change. Regardless, the
  eventual implementation would need to validate that all in the
  sequence are either A or all are V (along with the usual validation
  that appended chunks or frames also appear to use the most recent
  SourceBufferConfig).

  Caveat: The bindings generator requires help when generated union type
  identifiers are too long for some platforms. This change adds a
  seventh case to the existing hard-coded lists of names that need
  shortening with the generator.

I2P: https://groups.google.com/a/chromium.org/g/blink-dev/c/bejy1nmoWmU/m/CQ90X3j5BQAJ
TAG early-design review request: w3ctag/design-reviews#576
Explainer: https://github.com/wolenetz/mse-for-webcodecs/blob/main/explainer.md
MSE spec bug: w3c/media-source#184

BUG=1144908

Change-Id: Ibc8bd806fe1790ae74fe5ce86865cdfebcdc3096
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2515199
Commit-Queue: Matthew Wolenetz <wolenetz@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Kentaro Hara <haraken@chromium.org>
Reviewed-by: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Chrome Cunningham <chcunningham@chromium.org>
Cr-Commit-Position: refs/heads/master@{#830837}

third_party/blink/public/mojom/web_feature/web_feature.mojom

@@ -3059,6 +3059,9 @@ enum WebFeature {
   kAddEventListenerWithAbortSignal = 3730,
   kXRSessionRequestLightProbe = 3731,
   kBeforematchRevealedHiddenMatchable = 3732,
+  kAddSourceBufferUsingConfig = 3733,
+  kChangeTypeUsingConfig = 3734,
+  kV8SourceBuffer_AppendEncodedChunks_Method = 3735,
 
   // Add new features immediately above this line. Don't change assigned
   // numbers of any item, and don't reuse removed slots.

third_party/blink/renderer/bindings/generated_in_modules.gni

@@ -749,6 +749,8 @@ generated_dictionary_sources_in_modules = [
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_sensor_options.h",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_share_data.cc",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_share_data.h",
+  "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_source_buffer_config.cc",
+  "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_source_buffer_config.h",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_spatial_sensor_options.cc",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_spatial_sensor_options.h",
   "$root_gen_dir/third_party/blink/renderer/bindings/modules/v8/v8_speech_recognition_error_event_init.cc",

third_party/blink/renderer/bindings/idl_in_modules.gni

@@ -363,6 +363,7 @@ static_idl_files_in_modules = get_path_info(
           "//third_party/blink/renderer/modules/mediasource/html_video_element_media_source.idl",
           "//third_party/blink/renderer/modules/mediasource/media_source.idl",
           "//third_party/blink/renderer/modules/mediasource/source_buffer.idl",
+          "//third_party/blink/renderer/modules/mediasource/source_buffer_config.idl",
           "//third_party/blink/renderer/modules/mediasource/source_buffer_list.idl",
           "//third_party/blink/renderer/modules/mediasource/track_default.idl",
           "//third_party/blink/renderer/modules/mediasource/track_default_list.idl",

third_party/blink/renderer/bindings/modules/v8/generated.gni

@@ -49,6 +49,10 @@ bindings_modules_generated_union_type_files = [
   "$bindings_modules_v8_output_dir/double_or_dom_point.cc",
   "$bindings_modules_v8_output_dir/double_sequence_or_gpu_color_dict.cc",
   "$bindings_modules_v8_output_dir/double_sequence_or_gpu_color_dict.h",
+  "$bindings_modules_v8_output_dir/encoded_audio_chunk_or_encoded_video_chunk.cc",
+  "$bindings_modules_v8_output_dir/encoded_audio_chunk_or_encoded_video_chunk.h",
+  "$bindings_modules_v8_output_dir/encoded_av_chunk_sequence_or_encoded_av_chunk.cc",
+  "$bindings_modules_v8_output_dir/encoded_av_chunk_sequence_or_encoded_av_chunk.h",
   "$bindings_modules_v8_output_dir/float32_array_or_float64_array_or_dom_matrix.cc",
   "$bindings_modules_v8_output_dir/float32_array_or_float64_array_or_dom_matrix.h",
   "$bindings_modules_v8_output_dir/gpu_buffer_or_array_buffer.cc",

third_party/blink/renderer/bindings/scripts/bind_gen/path_manager.py

@@ -230,4 +230,7 @@ def _join(dirpath, filename, ext=None):
     # modules/beacon/navigator_beacon.idl
     'ReadableStreamOrBlobOrArrayBufferOrArrayBufferViewOrFormDataOrURLSearchParamsOrUSVString':
     'ReadableStreamOrXMLHttpRequestBodyInit',
+    # modules/mediasource/source_buffer.idl
+    'EncodedAudioChunkOrEncodedVideoChunkSequenceOrEncodedAudioChunkOrEncodedVideoChunk':
+    'EncodedAVChunkSequenceOrEncodedAVChunk',
 }

third_party/blink/renderer/bindings/scripts/utilities.py

@@ -497,6 +497,9 @@ def shorten_union_name(union_type):
         # modules/beacon/navigator_beacon.idl
         'ReadableStreamOrBlobOrArrayBufferOrArrayBufferViewOrFormDataOrURLSearchParamsOrUSVString':
         'ReadableStreamOrXMLHttpRequestBodyInit',
+        # modules/mediasource/source_buffer.idl
+        'EncodedAudioChunkOrEncodedVideoChunkSequenceOrEncodedAudioChunkOrEncodedVideoChunk':
+        'EncodedAVChunkSequenceOrEncodedAVChunk',
     }
 
     idl_type = union_type

third_party/blink/renderer/modules/mediasource/BUILD.gn

@@ -35,4 +35,8 @@ blink_modules_sources("mediasource") {
     "video_playback_quality.cc",
     "video_playback_quality.h",
   ]
+  deps = [
+    # Ensure the generated webcodecs v8 config and chunk bindings are available.
+    "//third_party/blink/renderer/modules/webcodecs:webcodecs",
+  ]
 }

third_party/blink/renderer/modules/mediasource/DEPS

@@ -4,4 +4,5 @@ include_rules = [
     "+third_party/blink/renderer/modules/event_modules.h",
     "+third_party/blink/renderer/modules/event_target_modules.h",
     "+third_party/blink/renderer/modules/mediasource",
+    "+third_party/blink/renderer/modules/webcodecs",
 ]

third_party/blink/renderer/modules/mediasource/idls.gni

@@ -11,6 +11,8 @@ modules_idl_files = [
   "video_playback_quality.idl",
 ]
 
+modules_dictionary_idl_files = [ "source_buffer_config.idl" ]
+
 modules_dependency_idl_files = [
   "audio_track_source_buffer.idl",
   "html_video_element_media_source.idl",

third_party/blink/renderer/modules/mediasource/media_source.cc

@@ -14,6 +14,9 @@
 #include "third_party/blink/public/common/privacy_budget/identifiable_surface.h"
 #include "third_party/blink/public/platform/web_media_source.h"
 #include "third_party/blink/public/platform/web_source_buffer.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_audio_decoder_config.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_source_buffer_config.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_video_decoder_config.h"
 #include "third_party/blink/renderer/core/dom/events/event.h"
 #include "third_party/blink/renderer/core/dom/events/event_queue.h"
 #include "third_party/blink/renderer/core/frame/deprecation.h"
@@ -218,6 +221,18 @@ SourceBuffer* MediaSource::addSourceBuffer(const String& type,
   return source_buffer;
 }
 
+SourceBuffer* MediaSource::AddSourceBufferUsingConfig(
+    const SourceBufferConfig* config,
+    ExceptionState& exception_state) {
+  DVLOG(2) << __func__ << " this=" << this;
+  // TODO(crbug.com/1144908): Validate allowed in current state (and take lock
+  // at appropriate point), unwrap the config, validate it, create sourcebuffer,
+  // etc.
+  exception_state.ThrowTypeError(
+      "unimplemented - see https://crbug.com/1144908");
+  return nullptr;
+}
+
 void MediaSource::AddSourceBuffer_Locked(
     const String& type,
     ExceptionState* exception_state,

third_party/blink/renderer/modules/mediasource/media_source.h

@@ -31,6 +31,7 @@ class ExceptionState;
 class HTMLMediaElement;
 class CrossThreadMediaSourceAttachment;
 class SameThreadMediaSourceAttachment;
+class SourceBufferConfig;
 class TrackBase;
 class WebSourceBuffer;
 
@@ -67,6 +68,9 @@ class MediaSource final : public EventTargetWithInlineData,
   }
   SourceBuffer* addSourceBuffer(const String& type, ExceptionState&)
       LOCKS_EXCLUDED(attachment_link_lock_);
+  SourceBuffer* AddSourceBufferUsingConfig(const SourceBufferConfig*,
+                                           ExceptionState&)
+      LOCKS_EXCLUDED(attachment_link_lock_);
   void removeSourceBuffer(SourceBuffer*, ExceptionState&)
       LOCKS_EXCLUDED(attachment_link_lock_);
   void setDuration(double, ExceptionState&)

third_party/blink/renderer/modules/mediasource/media_source.idl

@@ -50,15 +50,28 @@ enum EndOfStreamError {
     attribute EventHandler onsourceopen;
     attribute EventHandler onsourceended;
     attribute EventHandler onsourceclose;
+
     [RaisesException] SourceBuffer addSourceBuffer(DOMString type);
+
+    // For MSE-for-WebCodecs. Explainer:
+    // https://github.com/wolenetz/mse-for-webcodecs/blob/main/explainer.md
+    [
+        RaisesException,
+        RuntimeEnabled=MediaSourceExtensionsForWebCodecs,
+        ImplementedAs=AddSourceBufferUsingConfig,
+        MeasureAs=AddSourceBufferUsingConfig
+    ] SourceBuffer addSourceBuffer(SourceBufferConfig config);
+
     [RaisesException] void removeSourceBuffer(SourceBuffer buffer);
     readonly attribute DOMString readyState;
     [RaisesException] void endOfStream(optional EndOfStreamError error);
 
     [RaisesException] void setLiveSeekableRange(double start, double end);
     [RaisesException] void clearLiveSeekableRange();
 
-    [CallWith=ExecutionContext] static boolean isTypeSupported (DOMString type);
+    // TODO(crbug.com/1144908): Consider adding an overload which is given a
+    // SourceBufferConfig.
+    [CallWith=ExecutionContext] static boolean isTypeSupported(DOMString type);
 
     // Enables proactive feature-detection of MSE-in-Workers support from the
     // main thread (or anywhere this interface is exposed.) If the attribute is

third_party/blink/renderer/modules/mediasource/source_buffer.cc

@@ -38,6 +38,10 @@
 #include "media/base/logging_override_if_enabled.h"
 #include "third_party/blink/public/platform/task_type.h"
 #include "third_party/blink/public/platform/web_source_buffer.h"
+#include "third_party/blink/renderer/bindings/modules/v8/encoded_av_chunk_sequence_or_encoded_av_chunk.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_audio_decoder_config.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_source_buffer_config.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_video_decoder_config.h"
 #include "third_party/blink/renderer/core/dom/document.h"
 #include "third_party/blink/renderer/core/dom/events/event.h"
 #include "third_party/blink/renderer/core/dom/events/event_queue.h"
@@ -53,6 +57,8 @@
 #include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer_view.h"
 #include "third_party/blink/renderer/modules/mediasource/media_source.h"
 #include "third_party/blink/renderer/modules/mediasource/source_buffer_track_base_supplement.h"
+#include "third_party/blink/renderer/modules/webcodecs/encoded_audio_chunk.h"
+#include "third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h"
 #include "third_party/blink/renderer/platform/bindings/exception_messages.h"
 #include "third_party/blink/renderer/platform/bindings/exception_state.h"
 #include "third_party/blink/renderer/platform/heap/heap.h"
@@ -316,8 +322,9 @@ void SourceBuffer::setTimestampOffset(double offset,
   // 3. If the updating attribute equals true, then throw an InvalidStateError
   //    exception and abort these steps.
   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
-                                        exception_state))
+                                        exception_state)) {
     return;
+  }
 
   // Do the remainder of steps only if attachment is usable and underlying
   // demuxer is protected from destruction (applicable especially for
@@ -404,8 +411,9 @@ void SourceBuffer::setAppendWindowStart(double start,
   // 2. If the updating attribute equals true, then throw an InvalidStateError
   //    exception and abort these steps.
   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
-                                        exception_state))
+                                        exception_state)) {
     return;
+  }
 
   // 3. If the new value is less than 0 or greater than or equal to
   //    appendWindowEnd then throw a TypeError exception and abort these steps.
@@ -460,8 +468,9 @@ void SourceBuffer::setAppendWindowEnd(double end,
   // 2. If the updating attribute equals true, then throw an InvalidStateError
   //    exception and abort these steps.
   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
-                                        exception_state))
+                                        exception_state)) {
     return;
+  }
 
   // 3. If the new value equals NaN, then throw a TypeError and abort these
   //    steps.
@@ -526,6 +535,31 @@ void SourceBuffer::appendBuffer(NotShared<DOMArrayBufferView> data,
       data.View()->byteLength(), exception_state);
 }
 
+ScriptPromise SourceBuffer::appendEncodedChunks(
+    ScriptState* script_state,
+    const EncodedChunks& chunks,
+    ExceptionState& exception_state) {
+  // Note that |chunks| may be a sequence of mixed audio and video encoded
+  // chunks (which should cause underlying buffering validation to emit error
+  // akin to appending video to an audio track or vice-versa). It was impossible
+  // to get the bindings generator to disambiguate sequence<audio> vs
+  // sequence<video>, hence we could not use simple overloading in the IDL for
+  // these two. Neither could the IDL union attempt similar. We must enforce
+  // that semantic in implementation. Further note, |chunks| may instead be a
+  // single audio or a single video chunk as a helpful additional overload for
+  // one-chunk-at-a-time append use-cases.
+
+  DVLOG(2) << __func__ << " this=" << this;
+
+  // TODO(crbug.com/1144908): Validate allowed in current state (and take lock
+  // at appropriate point), unwrap the chunks, get a promise and its resolver,
+  // give the resolver to the async validation and buffering of the chunks,
+  // return the promise.
+  exception_state.ThrowTypeError(
+      "unimplemented - see https://crbug.com/1144908");
+  return ScriptPromise();
+}
+
 void SourceBuffer::abort(ExceptionState& exception_state) {
   DVLOG(2) << __func__ << " this=" << this;
   // http://w3c.github.io/media-source/#widl-SourceBuffer-abort-void
@@ -620,8 +654,9 @@ void SourceBuffer::remove(double start,
   // 2. If the updating attribute equals true, then throw an InvalidStateError
   //    exception and abort these steps.
   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
-                                        exception_state))
+                                        exception_state)) {
     return;
+  }
 
   // Do remainder of steps only if attachment is usable and underlying demuxer
   // is protected from destruction (applicable especially for MSE-in-Worker
@@ -722,8 +757,9 @@ void SourceBuffer::changeType(const String& type,
   // 3. If the updating attribute equals true, then throw an InvalidStateError
   //    exception and abort these steps.
   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
-                                        exception_state))
+                                        exception_state)) {
     return;
+  }
 
   // Do remainder of steps only if attachment is usable and underlying demuxer
   // is protected from destruction (applicable especially for MSE-in-Worker
@@ -740,6 +776,39 @@ void SourceBuffer::changeType(const String& type,
   }
 }
 
+void SourceBuffer::ChangeTypeUsingConfig(const SourceBufferConfig* config,
+                                         ExceptionState& exception_state) {
+  DVLOG(2) << __func__ << " this=" << this;
+
+  // If this object has been removed from the sourceBuffers attribute of the
+  //    parent media source, then throw an InvalidStateError exception and abort
+  //    these steps.
+  // If the updating attribute equals true, then throw an InvalidStateError
+  //    exception and abort these steps.
+  if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
+                                        exception_state)) {
+    return;
+  }
+
+  // Before this IDL overload was added, changeType(null) yielded a
+  // kNotSupportedError, so preserve that behavior if the bindings resolve us
+  // instead of the original changeType(DOMString) when given a null parameter.
+  // Fortunately, a null or empty SourceBufferConfig here similarly should yield
+  // a kNotSupportedError.
+  if (!config || (!config->hasAudioConfig() && !config->hasVideoConfig())) {
+    MediaSource::LogAndThrowDOMException(
+        exception_state, DOMExceptionCode::kNotSupportedError,
+        "Changing to the type provided ('null' config) is not supported.");
+    return;
+  }
+
+  // TODO(crbug.com/1144908): Further validate allowed in current state (and
+  // take lock at appropriate point), unwrap the config, validate it, update
+  // internals to new config, etc.
+  exception_state.ThrowTypeError(
+      "unimplemented - see https://crbug.com/1144908");
+}
+
 void SourceBuffer::ChangeType_Locked(
     const String& type,
     ExceptionState* exception_state,
@@ -806,8 +875,9 @@ void SourceBuffer::setTrackDefaults(TrackDefaultList* track_defaults,
   // 2. If the updating attribute equals true, then throw an InvalidStateError
   //    exception and abort these steps.
   if (ThrowExceptionIfRemovedOrUpdating(IsRemoved(), updating_,
-                                        exception_state))
+                                        exception_state)) {
     return;
+  }
 
   // 3. Update the attribute to the new value.
   track_defaults_ = track_defaults;

third_party/blink/renderer/modules/mediasource/source_buffer.h

@@ -36,6 +36,7 @@
 #include "base/memory/scoped_refptr.h"
 #include "third_party/blink/public/platform/web_source_buffer_client.h"
 #include "third_party/blink/renderer/bindings/core/v8/active_script_wrappable.h"
+#include "third_party/blink/renderer/bindings/core/v8/script_promise.h"
 #include "third_party/blink/renderer/core/execution_context/execution_context_lifecycle_observer.h"
 #include "third_party/blink/renderer/core/typed_arrays/array_buffer_view_helpers.h"
 #include "third_party/blink/renderer/modules/event_target_modules.h"
@@ -50,11 +51,15 @@ namespace blink {
 class AudioTrackList;
 class DOMArrayBuffer;
 class DOMArrayBufferView;
+class
+    EncodedAudioChunkOrEncodedVideoChunkSequenceOrEncodedAudioChunkOrEncodedVideoChunk;
 class EventQueue;
 class ExceptionState;
 class MediaSource;
 class MediaSourceTracer;
 class MediaSourceAttachmentSupplement;
+class ScriptState;
+class SourceBufferConfig;
 class TimeRanges;
 class VideoTrackList;
 class WebSourceBuffer;
@@ -70,6 +75,10 @@ class SourceBuffer final : public EventTargetWithInlineData,
   static AtomicString SegmentsKeyword();
   static AtomicString SequenceKeyword();
 
+  // Mirror the IDL's typedef for EncodedChunks.
+  using EncodedChunks =
+      EncodedAudioChunkOrEncodedVideoChunkSequenceOrEncodedAudioChunkOrEncodedVideoChunk;
+
   SourceBuffer(std::unique_ptr<WebSourceBuffer>, MediaSource*, EventQueue*);
   ~SourceBuffer() override;
 
@@ -82,9 +91,13 @@ class SourceBuffer final : public EventTargetWithInlineData,
   void setTimestampOffset(double, ExceptionState&);
   void appendBuffer(DOMArrayBuffer* data, ExceptionState&);
   void appendBuffer(NotShared<DOMArrayBufferView> data, ExceptionState&);
+  ScriptPromise appendEncodedChunks(ScriptState*,
+                                    const EncodedChunks&,
+                                    ExceptionState&);
   void abort(ExceptionState&);
   void remove(double start, double end, ExceptionState&);
   void changeType(const String& type, ExceptionState&);
+  void ChangeTypeUsingConfig(const SourceBufferConfig*, ExceptionState&);
   double appendWindowStart() const;
   void setAppendWindowStart(double, ExceptionState&);
   double appendWindowEnd() const;