Merge pull request #4494 from remotion-dev/webcodecs-handler

packages/convert/app/components/ConvertForm.tsx

@@ -1,6 +1,6 @@
 import {
 	ConvertMediaAudioCodec,
-	ConvertMediaTo,
+	ConvertMediaContainer,
 	ConvertMediaVideoCodec,
 } from '@remotion/webcodecs';
 import React from 'react';
@@ -17,8 +17,10 @@ import {
 } from './ui/select';
 
 export const ConvertForm: React.FC<{
-	readonly container: ConvertMediaTo;
-	readonly setContainer: React.Dispatch<React.SetStateAction<ConvertMediaTo>>;
+	readonly container: ConvertMediaContainer;
+	readonly setContainer: React.Dispatch<
+		React.SetStateAction<ConvertMediaContainer>
+	>;
 	readonly videoCodec: ConvertMediaVideoCodec;
 	readonly setVideoCodec: React.Dispatch<
 		React.SetStateAction<ConvertMediaVideoCodec>

packages/convert/app/components/ConvertUi.tsx

@@ -5,7 +5,7 @@ import {webFileReader} from '@remotion/media-parser/web-file';
 import {
 	convertMedia,
 	ConvertMediaAudioCodec,
-	ConvertMediaTo,
+	ConvertMediaContainer,
 	ConvertMediaVideoCodec,
 } from '@remotion/webcodecs';
 import {useCallback, useEffect, useRef, useState} from 'react';
@@ -18,7 +18,7 @@ import {flipVideoFrame} from './flip-video';
 import {Badge} from './ui/badge';
 
 export default function ConvertUI({src}: {readonly src: Source}) {
-	const [container, setContainer] = useState<ConvertMediaTo>('webm');
+	const [container, setContainer] = useState<ConvertMediaContainer>('webm');
 	const [videoCodec, setVideoCodec] = useState<ConvertMediaVideoCodec>('vp8');
 	const [audioCodec, setAudioCodec] = useState<ConvertMediaAudioCodec>('opus');
 	const [state, setState] = useState<ConvertState>({type: 'idle'});
@@ -64,7 +64,7 @@ export default function ConvertUI({src}: {readonly src: Source}) {
 			},
 			videoCodec: videoCodec as 'vp8',
 			audioCodec: audioCodec as 'opus',
-			to: container as 'webm',
+			container: container as 'webm',
 			signal: abortController.signal,
 			fields: {
 				name: true,

packages/docs/docs/webcodecs/TableOfContents.tsx

@@ -8,7 +8,15 @@ export const TableOfContents: React.FC = () => {
 			<Grid>
 				<TOCItem link="/docs/webcodecs/convert-media">
 					<strong>{'convertMedia()'}</strong>
-					<div>Re-encodes a video using WebCodecs and Media Parser</div>
+					<div>Converts a video using WebCodecs and Media Parser</div>
+				</TOCItem>
+				<TOCItem link="/docs/webcodecs/can-reencode-video-track">
+					<strong>{'canReencodeVideoTrack()'}</strong>
+					<div>Determine if a video track can be re-encoded</div>
+				</TOCItem>
+				<TOCItem link="/docs/webcodecs/can-reencode-audio-track">
+					<strong>{'canReencodeAudioTrack()'}</strong>
+					<div>Determine if a audio track can be re-encoded</div>
 				</TOCItem>
 			</Grid>
 		</div>

packages/docs/docs/webcodecs/can-reencode-audio-track.mdx

@@ -0,0 +1,95 @@
+---
+image: /generated/articles-docs-webcodecs-can-reencode-audio-track.png
+id: can-reencode-audio-track
+title: canReencodeAudioTrack()
+slug: /webcodecs/can-reencode-audio-track
+crumb: '@remotion/webcodecs'
+---
+
+_Part of the [`@remotion/webcodecs`](/docs/webcodecs) package._
+
+:::warning
+**Unstable API**: This package is experimental. We might change the API at any time, until we remove this notice.
+:::
+
+Given an `AudioTrack`, determine if it can be re-encoded to another track.
+
+You can obtain an `AudioTrack` using [`parseMedia()`](/docs/media-parser/parse-media) or during the conversion process using the [`onAudioTrack`](/docs/webcodecs/convert-media#onaudiotrack) callback of [`convertMedia()`](/docs/webcodecs/convert-media).
+
+## Examples
+
+```tsx twoslash title="Check if an audio track can be re-encoded to Opus"
+// @module: es2022
+// @target: es2017
+
+import {parseMedia} from '@remotion/media-parser';
+import {canReencodeAudioTrack} from '@remotion/webcodecs';
+
+const {audioTracks} = await parseMedia({
+  src: 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4',
+  fields: {
+    tracks: true,
+  },
+});
+
+for (const track of audioTracks) {
+  await canReencodeAudioTrack({
+    track,
+    audioCodec: 'opus',
+    bitrate: 128000,
+  });
+}
+```
+
+```tsx twoslash title="Convert an audio track to Opus, otherwise drop it"
+// @module: es2022
+// @target: es2017
+import {convertMedia, canReencodeAudioTrack} from '@remotion/webcodecs';
+
+await convertMedia({
+  src: 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4',
+  container: 'webm',
+  videoCodec: 'vp8',
+  audioCodec: 'opus',
+  onAudioTrack: async ({track}) => {
+    const canReencode = await canReencodeAudioTrack({
+      track,
+      audioCodec: 'opus',
+      bitrate: 128000,
+    });
+
+    if (canReencode) {
+      return {type: 'reencode', audioCodec: 'opus', bitrate: 128000};
+    }
+
+    return {type: 'drop'};
+  },
+});
+```
+
+## API
+
+### `track`
+
+A `AudioTrack` object.
+
+### `audioCodec`
+
+_string_ <TsType type="ConvertMediaAudioCodec" source="@remotion/webcodecs" />
+
+### `bitrate`
+
+_number_
+
+The bitrate with which you'd like to re-encode the audio track.
+
+## Return value
+
+Returns a `Promise<boolean>`.
+
+## See also
+
+- [Source code for this function on GitHub](https://github.com/remotion-dev/remotion/blob/main/packages/webcodecs/src/can-reencode-audio-track.ts)
+- [`canReencodeVideoTrack()`](/docs/webcodecs/can-reencode-video-track)
+- [`convertMedia()`](/docs/webcodecs/convert-media)
+- [`parseMedia()`](/docs/media-parser/parse-media)

packages/docs/docs/webcodecs/can-reencode-video-track.mdx

@@ -0,0 +1,88 @@
+---
+image: /generated/articles-docs-webcodecs-can-reencode-video-track.png
+id: can-reencode-video-track
+title: canReencodeVideoTrack()
+slug: /webcodecs/can-reencode-video-track
+crumb: '@remotion/webcodecs'
+---
+
+_Part of the [`@remotion/webcodecs`](/docs/webcodecs) package._
+
+:::warning
+**Unstable API**: This package is experimental. We might change the API at any time, until we remove this notice.
+:::
+
+Given a `VideoTrack`, determine if it can be re-encoded to another track.
+
+You can obtain a `VideoTrack` using [`parseMedia()`](/docs/media-parser/parse-media) or during the conversion process using the [`onVideoTrack`](/docs/webcodecs/convert-media#onvideotrack) callback of [`convertMedia()`](/docs/webcodecs/convert-media).
+
+## Examples
+
+```tsx twoslash title="Check if a video tracks can be re-encoded to VP8"
+// @module: es2022
+// @target: es2017
+
+import {parseMedia} from '@remotion/media-parser';
+import {canReencodeVideoTrack} from '@remotion/webcodecs';
+
+const {videoTracks} = await parseMedia({
+  src: 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4',
+  fields: {
+    tracks: true,
+  },
+});
+
+for (const track of videoTracks) {
+  await canReencodeVideoTrack({
+    track,
+    videoCodec: 'vp8',
+  });
+}
+```
+
+```tsx twoslash title="Convert a video track to VP8, otherwise drop it"
+// @module: es2022
+// @target: es2017
+import {convertMedia, canReencodeVideoTrack} from '@remotion/webcodecs';
+
+await convertMedia({
+  src: 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4',
+  container: 'webm',
+  videoCodec: 'vp8',
+  audioCodec: 'opus',
+  onVideoTrack: async ({track}) => {
+    const canReencode = await canReencodeVideoTrack({
+      track,
+      videoCodec: 'vp8',
+    });
+
+    if (canReencode) {
+      return {type: 'reencode', videoCodec: 'vp8'};
+    }
+
+    return {type: 'drop'};
+  },
+});
+```
+
+## API
+
+### `track`
+
+A `VideoTrack` object.
+
+### `videoCodec`
+
+_string_ <TsType type="ConvertMediaVideoCodec" source="@remotion/webcodecs" />
+
+One of the supported video codecs: `"vp8"`, `"vp9"`.
+
+## Return value
+
+Returns a `Promise<boolean>`.
+
+## See also
+
+- [Source code for this function on GitHub](https://github.com/remotion-dev/remotion/blob/main/packages/webcodecs/src/can-reencode-video-track.ts)
+- [`convertMedia()`](/docs/webcodecs/convert-media)
+- [`parseMedia()`](/docs/media-parser/parse-media)

packages/docs/docs/webcodecs/convert-media.mdx

@@ -1,4 +1,5 @@
 ---
+image: /generated/articles-docs-webcodecs-convert-media.png
 id: convert-media
 title: convertMedia()
 slug: /webcodecs/convert-media
@@ -42,9 +43,9 @@ A reader interface.
 Default value: [`fetchReader`](/docs/media-parser/fetch-reader), which uses `fetch()` to read the video.  
 If you pass [`webFileReader`](/docs/media-parser/web-file-reader), you must also pass a `File` as the `src` argument.
 
-### `to`
+### `container`
 
-_string_ <TsType type="ConvertMediaTo" source="@remotion/webcodecs"/>
+_string_ <TsType type="ConvertMediaContainer" source="@remotion/webcodecs"/>
 
 The container format to convert to. Currently, only `"webm"` is supported.
 

packages/docs/docs/webcodecs/index.mdx

@@ -1,4 +1,5 @@
 ---
+image: /generated/articles-docs-webcodecs-index.png
 sidebar_label: Overview
 title: '@remotion/webcodecs'
 ---

packages/docs/sidebars.js

@@ -572,7 +572,12 @@ module.exports = {
 				type: 'doc',
 				id: 'webcodecs/index',
 			},
-			items: ['webcodecs/index', 'webcodecs/convert-media'],
+			items: [
+				'webcodecs/index',
+				'webcodecs/convert-media',
+				'webcodecs/can-reencode-video-track',
+				'webcodecs/can-reencode-audio-track',
+			],
 		},
 		{
 			type: 'category',

packages/docs/src/data/articles.ts

@@ -3284,6 +3284,34 @@ export const articles = [
 		compId: 'articles-docs-audio',
 		crumb: 'How to',
 	},
+	{
+		id: 'convert-media',
+		title: 'convertMedia()',
+		relativePath: 'docs/webcodecs/convert-media.mdx',
+		compId: 'articles-docs-webcodecs-convert-media',
+		crumb: '@remotion/webcodecs',
+	},
+	{
+		id: 'can-reencode-audio-track',
+		title: 'canReencodeAudioTrack()',
+		relativePath: 'docs/webcodecs/can-reencode-audio-track.mdx',
+		compId: 'articles-docs-webcodecs-can-reencode-audio-track',
+		crumb: '@remotion/webcodecs',
+	},
+	{
+		id: 'webcodecs/index',
+		title: '@remotion/webcodecs',
+		relativePath: 'docs/webcodecs/index.mdx',
+		compId: 'articles-docs-webcodecs-index',
+		crumb: null,
+	},
+	{
+		id: 'can-reencode-video-track',
+		title: 'canReencodeVideoTrack()',
+		relativePath: 'docs/webcodecs/can-reencode-video-track.mdx',
+		compId: 'articles-docs-webcodecs-can-reencode-video-track',
+		crumb: '@remotion/webcodecs',
+	},
 	{
 		id: 'visualize-audio',
 		title: 'visualizeAudio()',

packages/example/src/Encoder/SrcEncoder.tsx

@@ -144,7 +144,7 @@ export const SrcEncoder: React.FC<{
 				},
 				videoCodec: 'vp9',
 				audioCodec: 'opus',
-				to: 'webm',
+				container: 'webm',
 				signal: abortController.signal,
 			});
 			setDownloadFn(() => {

packages/webcodecs/src/can-reencode-audio-track.ts

@@ -0,0 +1,30 @@
+import type {AudioTrack} from '@remotion/media-parser';
+import {getAudioDecoderConfig} from './audio-decoder-config';
+import {getAudioEncoderConfig} from './audio-encoder-config';
+import type {ConvertMediaAudioCodec} from './codec-id';
+
+export const canReencodeAudioTrack = async ({
+	track,
+	audioCodec,
+	bitrate,
+}: {
+	track: AudioTrack;
+	audioCodec: ConvertMediaAudioCodec;
+	bitrate: number;
+}): Promise<boolean> => {
+	const audioDecoderConfig = await getAudioDecoderConfig({
+		codec: track.codec,
+		numberOfChannels: track.numberOfChannels,
+		sampleRate: track.sampleRate,
+		description: track.description,
+	});
+
+	const audioEncoderConfig = await getAudioEncoderConfig({
+		codec: audioCodec,
+		numberOfChannels: track.numberOfChannels,
+		sampleRate: track.sampleRate,
+		bitrate,
+	});
+
+	return Boolean(audioDecoderConfig && audioEncoderConfig);
+};

packages/webcodecs/src/can-reencode-video-track.ts

@@ -0,0 +1,22 @@
+import type {VideoTrack} from '@remotion/media-parser';
+import type {ConvertMediaVideoCodec} from './codec-id';
+import {getVideoDecoderConfigWithHardwareAcceleration} from './video-decoder-config';
+import {getVideoEncoderConfig} from './video-encoder-config';
+
+export const canReencodeVideoTrack = async ({
+	videoCodec,
+	track,
+}: {
+	videoCodec: ConvertMediaVideoCodec;
+	track: VideoTrack;
+}) => {
+	const videoEncoderConfig = await getVideoEncoderConfig({
+		codec: videoCodec,
+		height: track.displayAspectHeight,
+		width: track.displayAspectWidth,
+	});
+	const videoDecoderConfig =
+		await getVideoDecoderConfigWithHardwareAcceleration(track);
+
+	return Boolean(videoDecoderConfig && videoEncoderConfig);
+};