diff --git a/packages/convert/app/components/ConvertForm.tsx b/packages/convert/app/components/ConvertForm.tsx index 13066b820ab..b3045ab38c6 100644 --- a/packages/convert/app/components/ConvertForm.tsx +++ b/packages/convert/app/components/ConvertForm.tsx @@ -1,6 +1,6 @@ import { ConvertMediaAudioCodec, - ConvertMediaTo, + ConvertMediaContainer, ConvertMediaVideoCodec, } from '@remotion/webcodecs'; import React from 'react'; @@ -17,8 +17,10 @@ import { } from './ui/select'; export const ConvertForm: React.FC<{ - readonly container: ConvertMediaTo; - readonly setContainer: React.Dispatch>; + readonly container: ConvertMediaContainer; + readonly setContainer: React.Dispatch< + React.SetStateAction + >; readonly videoCodec: ConvertMediaVideoCodec; readonly setVideoCodec: React.Dispatch< React.SetStateAction diff --git a/packages/convert/app/components/ConvertUi.tsx b/packages/convert/app/components/ConvertUi.tsx index 85fe7cfcea1..9026a992ef7 100644 --- a/packages/convert/app/components/ConvertUi.tsx +++ b/packages/convert/app/components/ConvertUi.tsx @@ -5,7 +5,7 @@ import {webFileReader} from '@remotion/media-parser/web-file'; import { convertMedia, ConvertMediaAudioCodec, - ConvertMediaTo, + ConvertMediaContainer, ConvertMediaVideoCodec, } from '@remotion/webcodecs'; import {useCallback, useEffect, useRef, useState} from 'react'; @@ -18,7 +18,7 @@ import {flipVideoFrame} from './flip-video'; import {Badge} from './ui/badge'; export default function ConvertUI({src}: {readonly src: Source}) { - const [container, setContainer] = useState('webm'); + const [container, setContainer] = useState('webm'); const [videoCodec, setVideoCodec] = useState('vp8'); const [audioCodec, setAudioCodec] = useState('opus'); const [state, setState] = useState({type: 'idle'}); @@ -64,7 +64,7 @@ export default function ConvertUI({src}: {readonly src: Source}) { }, videoCodec: videoCodec as 'vp8', audioCodec: audioCodec as 'opus', - to: container as 'webm', + container: container as 'webm', signal: abortController.signal, fields: { name: true, diff --git a/packages/docs/docs/webcodecs/TableOfContents.tsx b/packages/docs/docs/webcodecs/TableOfContents.tsx index 9462b8660fa..b1648c33748 100644 --- a/packages/docs/docs/webcodecs/TableOfContents.tsx +++ b/packages/docs/docs/webcodecs/TableOfContents.tsx @@ -8,7 +8,15 @@ export const TableOfContents: React.FC = () => { {'convertMedia()'} -
Re-encodes a video using WebCodecs and Media Parser
+
Converts a video using WebCodecs and Media Parser
+
+ + {'canReencodeVideoTrack()'} +
Determine if a video track can be re-encoded
+
+ + {'canReencodeAudioTrack()'} +
Determine if a audio track can be re-encoded
diff --git a/packages/docs/docs/webcodecs/can-reencode-audio-track.mdx b/packages/docs/docs/webcodecs/can-reencode-audio-track.mdx new file mode 100644 index 00000000000..de89acd8407 --- /dev/null +++ b/packages/docs/docs/webcodecs/can-reencode-audio-track.mdx @@ -0,0 +1,95 @@ +--- +image: /generated/articles-docs-webcodecs-can-reencode-audio-track.png +id: can-reencode-audio-track +title: canReencodeAudioTrack() +slug: /webcodecs/can-reencode-audio-track +crumb: '@remotion/webcodecs' +--- + +_Part of the [`@remotion/webcodecs`](/docs/webcodecs) package._ + +:::warning +**Unstable API**: This package is experimental. We might change the API at any time, until we remove this notice. +::: + +Given an `AudioTrack`, determine if it can be re-encoded to another track. + +You can obtain an `AudioTrack` using [`parseMedia()`](/docs/media-parser/parse-media) or during the conversion process using the [`onAudioTrack`](/docs/webcodecs/convert-media#onaudiotrack) callback of [`convertMedia()`](/docs/webcodecs/convert-media). + +## Examples + +```tsx twoslash title="Check if an audio track can be re-encoded to Opus" +// @module: es2022 +// @target: es2017 + +import {parseMedia} from '@remotion/media-parser'; +import {canReencodeAudioTrack} from '@remotion/webcodecs'; + +const {audioTracks} = await parseMedia({ + src: 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4', + fields: { + tracks: true, + }, +}); + +for (const track of audioTracks) { + await canReencodeAudioTrack({ + track, + audioCodec: 'opus', + bitrate: 128000, + }); +} +``` + +```tsx twoslash title="Convert an audio track to Opus, otherwise drop it" +// @module: es2022 +// @target: es2017 +import {convertMedia, canReencodeAudioTrack} from '@remotion/webcodecs'; + +await convertMedia({ + src: 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4', + container: 'webm', + videoCodec: 'vp8', + audioCodec: 'opus', + onAudioTrack: async ({track}) => { + const canReencode = await canReencodeAudioTrack({ + track, + audioCodec: 'opus', + bitrate: 128000, + }); + + if (canReencode) { + return {type: 'reencode', audioCodec: 'opus', bitrate: 128000}; + } + + return {type: 'drop'}; + }, +}); +``` + +## API + +### `track` + +A `AudioTrack` object. + +### `audioCodec` + +_string_ + +### `bitrate` + +_number_ + +The bitrate with which you'd like to re-encode the audio track. + +## Return value + +Returns a `Promise`. + +## See also + +- [Source code for this function on GitHub](https://github.com/remotion-dev/remotion/blob/main/packages/webcodecs/src/can-reencode-audio-track.ts) +- [`canReencodeVideoTrack()`](/docs/webcodecs/can-reencode-video-track) +- [`convertMedia()`](/docs/webcodecs/convert-media) +- [`parseMedia()`](/docs/media-parser/parse-media) diff --git a/packages/docs/docs/webcodecs/can-reencode-video-track.mdx b/packages/docs/docs/webcodecs/can-reencode-video-track.mdx new file mode 100644 index 00000000000..77231a49307 --- /dev/null +++ b/packages/docs/docs/webcodecs/can-reencode-video-track.mdx @@ -0,0 +1,88 @@ +--- +image: /generated/articles-docs-webcodecs-can-reencode-video-track.png +id: can-reencode-video-track +title: canReencodeVideoTrack() +slug: /webcodecs/can-reencode-video-track +crumb: '@remotion/webcodecs' +--- + +_Part of the [`@remotion/webcodecs`](/docs/webcodecs) package._ + +:::warning +**Unstable API**: This package is experimental. We might change the API at any time, until we remove this notice. +::: + +Given a `VideoTrack`, determine if it can be re-encoded to another track. + +You can obtain a `VideoTrack` using [`parseMedia()`](/docs/media-parser/parse-media) or during the conversion process using the [`onVideoTrack`](/docs/webcodecs/convert-media#onvideotrack) callback of [`convertMedia()`](/docs/webcodecs/convert-media). + +## Examples + +```tsx twoslash title="Check if a video tracks can be re-encoded to VP8" +// @module: es2022 +// @target: es2017 + +import {parseMedia} from '@remotion/media-parser'; +import {canReencodeVideoTrack} from '@remotion/webcodecs'; + +const {videoTracks} = await parseMedia({ + src: 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4', + fields: { + tracks: true, + }, +}); + +for (const track of videoTracks) { + await canReencodeVideoTrack({ + track, + videoCodec: 'vp8', + }); +} +``` + +```tsx twoslash title="Convert a video track to VP8, otherwise drop it" +// @module: es2022 +// @target: es2017 +import {convertMedia, canReencodeVideoTrack} from '@remotion/webcodecs'; + +await convertMedia({ + src: 'https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4', + container: 'webm', + videoCodec: 'vp8', + audioCodec: 'opus', + onVideoTrack: async ({track}) => { + const canReencode = await canReencodeVideoTrack({ + track, + videoCodec: 'vp8', + }); + + if (canReencode) { + return {type: 'reencode', videoCodec: 'vp8'}; + } + + return {type: 'drop'}; + }, +}); +``` + +## API + +### `track` + +A `VideoTrack` object. + +### `videoCodec` + +_string_ + +One of the supported video codecs: `"vp8"`, `"vp9"`. + +## Return value + +Returns a `Promise`. + +## See also + +- [Source code for this function on GitHub](https://github.com/remotion-dev/remotion/blob/main/packages/webcodecs/src/can-reencode-video-track.ts) +- [`convertMedia()`](/docs/webcodecs/convert-media) +- [`parseMedia()`](/docs/media-parser/parse-media) diff --git a/packages/docs/docs/webcodecs/convert-media.mdx b/packages/docs/docs/webcodecs/convert-media.mdx index 332f8a65461..d983d283450 100644 --- a/packages/docs/docs/webcodecs/convert-media.mdx +++ b/packages/docs/docs/webcodecs/convert-media.mdx @@ -1,4 +1,5 @@ --- +image: /generated/articles-docs-webcodecs-convert-media.png id: convert-media title: convertMedia() slug: /webcodecs/convert-media @@ -42,9 +43,9 @@ A reader interface. Default value: [`fetchReader`](/docs/media-parser/fetch-reader), which uses `fetch()` to read the video. If you pass [`webFileReader`](/docs/media-parser/web-file-reader), you must also pass a `File` as the `src` argument. -### `to` +### `container` -_string_ +_string_ The container format to convert to. Currently, only `"webm"` is supported. diff --git a/packages/docs/docs/webcodecs/index.mdx b/packages/docs/docs/webcodecs/index.mdx index 0eeb8553568..ccb50e824dc 100644 --- a/packages/docs/docs/webcodecs/index.mdx +++ b/packages/docs/docs/webcodecs/index.mdx @@ -1,4 +1,5 @@ --- +image: /generated/articles-docs-webcodecs-index.png sidebar_label: Overview title: '@remotion/webcodecs' --- diff --git a/packages/docs/sidebars.js b/packages/docs/sidebars.js index 696668b2136..f15c700daf2 100644 --- a/packages/docs/sidebars.js +++ b/packages/docs/sidebars.js @@ -572,7 +572,12 @@ module.exports = { type: 'doc', id: 'webcodecs/index', }, - items: ['webcodecs/index', 'webcodecs/convert-media'], + items: [ + 'webcodecs/index', + 'webcodecs/convert-media', + 'webcodecs/can-reencode-video-track', + 'webcodecs/can-reencode-audio-track', + ], }, { type: 'category', diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.ts index ccd8e204e8d..bfee0f8f932 100644 --- a/packages/docs/src/data/articles.ts +++ b/packages/docs/src/data/articles.ts @@ -3284,6 +3284,34 @@ export const articles = [ compId: 'articles-docs-audio', crumb: 'How to', }, + { + id: 'convert-media', + title: 'convertMedia()', + relativePath: 'docs/webcodecs/convert-media.mdx', + compId: 'articles-docs-webcodecs-convert-media', + crumb: '@remotion/webcodecs', + }, + { + id: 'can-reencode-audio-track', + title: 'canReencodeAudioTrack()', + relativePath: 'docs/webcodecs/can-reencode-audio-track.mdx', + compId: 'articles-docs-webcodecs-can-reencode-audio-track', + crumb: '@remotion/webcodecs', + }, + { + id: 'webcodecs/index', + title: '@remotion/webcodecs', + relativePath: 'docs/webcodecs/index.mdx', + compId: 'articles-docs-webcodecs-index', + crumb: null, + }, + { + id: 'can-reencode-video-track', + title: 'canReencodeVideoTrack()', + relativePath: 'docs/webcodecs/can-reencode-video-track.mdx', + compId: 'articles-docs-webcodecs-can-reencode-video-track', + crumb: '@remotion/webcodecs', + }, { id: 'visualize-audio', title: 'visualizeAudio()', diff --git a/packages/docs/static/generated/articles-docs-webcodecs-can-reencode-audio-track.png b/packages/docs/static/generated/articles-docs-webcodecs-can-reencode-audio-track.png new file mode 100644 index 00000000000..f51d13d60ab Binary files /dev/null and b/packages/docs/static/generated/articles-docs-webcodecs-can-reencode-audio-track.png differ diff --git a/packages/docs/static/generated/articles-docs-webcodecs-can-reencode-video-track.png b/packages/docs/static/generated/articles-docs-webcodecs-can-reencode-video-track.png new file mode 100644 index 00000000000..4beaaf56344 Binary files /dev/null and b/packages/docs/static/generated/articles-docs-webcodecs-can-reencode-video-track.png differ diff --git a/packages/docs/static/generated/articles-docs-webcodecs-convert-media.png b/packages/docs/static/generated/articles-docs-webcodecs-convert-media.png new file mode 100644 index 00000000000..c86ec9d8a18 Binary files /dev/null and b/packages/docs/static/generated/articles-docs-webcodecs-convert-media.png differ diff --git a/packages/docs/static/generated/articles-docs-webcodecs-index.png b/packages/docs/static/generated/articles-docs-webcodecs-index.png new file mode 100644 index 00000000000..48f2beb9079 Binary files /dev/null and b/packages/docs/static/generated/articles-docs-webcodecs-index.png differ diff --git a/packages/example/src/Encoder/SrcEncoder.tsx b/packages/example/src/Encoder/SrcEncoder.tsx index 60c1b9bafb5..507db5cb8f5 100644 --- a/packages/example/src/Encoder/SrcEncoder.tsx +++ b/packages/example/src/Encoder/SrcEncoder.tsx @@ -144,7 +144,7 @@ export const SrcEncoder: React.FC<{ }, videoCodec: 'vp9', audioCodec: 'opus', - to: 'webm', + container: 'webm', signal: abortController.signal, }); setDownloadFn(() => { diff --git a/packages/webcodecs/src/can-reencode-audio-track.ts b/packages/webcodecs/src/can-reencode-audio-track.ts new file mode 100644 index 00000000000..badef12aaaf --- /dev/null +++ b/packages/webcodecs/src/can-reencode-audio-track.ts @@ -0,0 +1,30 @@ +import type {AudioTrack} from '@remotion/media-parser'; +import {getAudioDecoderConfig} from './audio-decoder-config'; +import {getAudioEncoderConfig} from './audio-encoder-config'; +import type {ConvertMediaAudioCodec} from './codec-id'; + +export const canReencodeAudioTrack = async ({ + track, + audioCodec, + bitrate, +}: { + track: AudioTrack; + audioCodec: ConvertMediaAudioCodec; + bitrate: number; +}): Promise => { + const audioDecoderConfig = await getAudioDecoderConfig({ + codec: track.codec, + numberOfChannels: track.numberOfChannels, + sampleRate: track.sampleRate, + description: track.description, + }); + + const audioEncoderConfig = await getAudioEncoderConfig({ + codec: audioCodec, + numberOfChannels: track.numberOfChannels, + sampleRate: track.sampleRate, + bitrate, + }); + + return Boolean(audioDecoderConfig && audioEncoderConfig); +}; diff --git a/packages/webcodecs/src/can-reencode-video-track.ts b/packages/webcodecs/src/can-reencode-video-track.ts new file mode 100644 index 00000000000..442259af7b0 --- /dev/null +++ b/packages/webcodecs/src/can-reencode-video-track.ts @@ -0,0 +1,22 @@ +import type {VideoTrack} from '@remotion/media-parser'; +import type {ConvertMediaVideoCodec} from './codec-id'; +import {getVideoDecoderConfigWithHardwareAcceleration} from './video-decoder-config'; +import {getVideoEncoderConfig} from './video-encoder-config'; + +export const canReencodeVideoTrack = async ({ + videoCodec, + track, +}: { + videoCodec: ConvertMediaVideoCodec; + track: VideoTrack; +}) => { + const videoEncoderConfig = await getVideoEncoderConfig({ + codec: videoCodec, + height: track.displayAspectHeight, + width: track.displayAspectWidth, + }); + const videoDecoderConfig = + await getVideoDecoderConfigWithHardwareAcceleration(track); + + return Boolean(videoDecoderConfig && videoEncoderConfig); +}; diff --git a/packages/webcodecs/src/convert-media.ts b/packages/webcodecs/src/convert-media.ts index 83bfb9a8c06..6e04d4e4c35 100644 --- a/packages/webcodecs/src/convert-media.ts +++ b/packages/webcodecs/src/convert-media.ts @@ -24,14 +24,8 @@ import type {ConvertMediaAudioCodec, ConvertMediaVideoCodec} from './codec-id'; import Error from './error-cause'; import {makeAudioTrackHandler} from './on-audio-track'; import {makeVideoTrackHandler} from './on-video-track'; -import { - defaultResolveAudioAction, - type ResolveAudioActionFn, -} from './resolve-audio-action'; -import { - defaultResolveVideoAction, - type ResolveVideoActionFn, -} from './resolve-video-action'; +import {type ResolveAudioActionFn} from './resolve-audio-action'; +import {type ResolveVideoActionFn} from './resolve-video-action'; import {withResolversAndWaitForReturn} from './with-resolvers'; export type ConvertMediaState = { @@ -45,7 +39,7 @@ export type ConvertMediaState = { overallProgress: number | null; }; -export type ConvertMediaTo = 'webm'; +export type ConvertMediaContainer = 'webm'; export type ConvertMediaResult = { save: () => Promise; @@ -65,7 +59,7 @@ export const convertMedia = async function < onVideoFrame, onMediaStateUpdate: onMediaStateDoNoCallDirectly, audioCodec, - to, + container, videoCodec, signal: userPassedAbortSignal, onAudioTrack: userAudioResolver, @@ -77,7 +71,7 @@ export const convertMedia = async function < ...more }: { src: ParseMediaOptions['src']; - to: ConvertMediaTo; + container: ConvertMediaContainer; onVideoFrame?: ConvertMediaOnVideoFrame; onMediaStateUpdate?: ConvertMediaOnMediaStateUpdate; videoCodec: ConvertMediaVideoCodec; @@ -93,7 +87,7 @@ export const convertMedia = async function < return Promise.reject(new Error('Aborted')); } - if (to !== 'webm') { + if (container !== 'webm') { return Promise.reject( new TypeError('Only `to: "webm"` is supported currently'), ); @@ -178,8 +172,9 @@ export const convertMedia = async function < convertMediaState, controller, videoCodec, - onVideoTrack: userVideoResolver ?? defaultResolveVideoAction, + onVideoTrack: userVideoResolver ?? null, logLevel, + container, }); const onAudioTrack: OnAudioTrack = makeAudioTrackHandler({ @@ -189,9 +184,9 @@ export const convertMedia = async function < convertMediaState, onMediaStateUpdate: onMediaStateUpdate ?? null, state, - onAudioTrack: userAudioResolver ?? defaultResolveAudioAction, - bitrate: 128000, + onAudioTrack: userAudioResolver ?? null, logLevel, + container, }); parseMedia({ diff --git a/packages/webcodecs/src/index.ts b/packages/webcodecs/src/index.ts index e65404da804..53c0eef2ac7 100644 --- a/packages/webcodecs/src/index.ts +++ b/packages/webcodecs/src/index.ts @@ -1,12 +1,14 @@ export {WebCodecsAudioDecoder, createAudioDecoder} from './audio-decoder'; export {WebCodecsAudioEncoder, createAudioEncoder} from './audio-encoder'; +export {canReencodeAudioTrack} from './can-reencode-audio-track'; +export {canReencodeVideoTrack} from './can-reencode-video-track'; export {ConvertMediaAudioCodec, ConvertMediaVideoCodec} from './codec-id'; export { + ConvertMediaContainer, ConvertMediaOnMediaStateUpdate, ConvertMediaOnVideoFrame, ConvertMediaResult, ConvertMediaState, - ConvertMediaTo, convertMedia, } from './convert-media'; export {ResolveAudioActionFn} from './resolve-audio-action'; diff --git a/packages/webcodecs/src/on-audio-track.ts b/packages/webcodecs/src/on-audio-track.ts index cb035975c5a..dbea813f428 100644 --- a/packages/webcodecs/src/on-audio-track.ts +++ b/packages/webcodecs/src/on-audio-track.ts @@ -5,10 +5,10 @@ import {createAudioEncoder} from './audio-encoder'; import {getAudioEncoderConfig} from './audio-encoder-config'; import type {ConvertMediaAudioCodec} from './codec-id'; import {convertEncodedChunk} from './convert-encoded-chunk'; -import type {ConvertMediaState} from './convert-media'; +import type {ConvertMediaContainer, ConvertMediaState} from './convert-media'; import Error from './error-cause'; import type {ResolveAudioActionFn} from './resolve-audio-action'; -import {resolveAudioAction} from './resolve-audio-action'; +import {defaultResolveAudioAction} from './resolve-audio-action'; export const makeAudioTrackHandler = ({ @@ -19,8 +19,8 @@ export const makeAudioTrackHandler = abortConversion, onMediaStateUpdate, onAudioTrack, - bitrate, logLevel, + container, }: { state: MediaFn; audioCodec: ConvertMediaAudioCodec; @@ -28,31 +28,16 @@ export const makeAudioTrackHandler = controller: AbortController; abortConversion: (errCause: Error) => void; onMediaStateUpdate: null | ((state: ConvertMediaState) => void); - onAudioTrack: ResolveAudioActionFn; - bitrate: number; + onAudioTrack: ResolveAudioActionFn | null; logLevel: LogLevel; + container: ConvertMediaContainer; }): OnAudioTrack => async (track) => { - const audioEncoderConfig = await getAudioEncoderConfig({ - codec: audioCodec, - numberOfChannels: track.numberOfChannels, - sampleRate: track.sampleRate, - bitrate, - }); - const audioDecoderConfig = await getAudioDecoderConfig({ - codec: track.codec, - numberOfChannels: track.numberOfChannels, - sampleRate: track.sampleRate, - description: track.description, - }); - - const audioOperation = await resolveAudioAction({ - audioDecoderConfig, - audioEncoderConfig, + const audioOperation = await (onAudioTrack ?? defaultResolveAudioAction)({ audioCodec, track, - resolverFunction: onAudioTrack, logLevel, + container, }); if (audioOperation.type === 'drop') { @@ -75,6 +60,19 @@ export const makeAudioTrackHandler = }; } + const audioEncoderConfig = await getAudioEncoderConfig({ + numberOfChannels: track.numberOfChannels, + sampleRate: track.sampleRate, + codec: audioOperation.audioCodec, + bitrate: audioOperation.bitrate, + }); + const audioDecoderConfig = await getAudioDecoderConfig({ + codec: track.codec, + numberOfChannels: track.numberOfChannels, + sampleRate: track.sampleRate, + description: track.description, + }); + if (!audioEncoderConfig) { abortConversion( new Error( diff --git a/packages/webcodecs/src/on-video-track.ts b/packages/webcodecs/src/on-video-track.ts index 3c942bff22d..131fe51a5c0 100644 --- a/packages/webcodecs/src/on-video-track.ts +++ b/packages/webcodecs/src/on-video-track.ts @@ -2,6 +2,7 @@ import type {LogLevel, MediaFn, OnVideoTrack} from '@remotion/media-parser'; import type {ConvertMediaVideoCodec} from './codec-id'; import {convertEncodedChunk} from './convert-encoded-chunk'; import type { + ConvertMediaContainer, ConvertMediaOnMediaStateUpdate, ConvertMediaOnVideoFrame, ConvertMediaState, @@ -9,7 +10,7 @@ import type { import Error from './error-cause'; import {onFrame} from './on-frame'; import type {ResolveVideoActionFn} from './resolve-video-action'; -import {resolveVideoAction} from './resolve-video-action'; +import {defaultResolveVideoAction} from './resolve-video-action'; import {createVideoDecoder} from './video-decoder'; import {getVideoDecoderConfigWithHardwareAcceleration} from './video-decoder-config'; import {createVideoEncoder} from './video-encoder'; @@ -26,6 +27,7 @@ export const makeVideoTrackHandler = videoCodec, onVideoTrack, logLevel, + container, }: { state: MediaFn; onVideoFrame: null | ConvertMediaOnVideoFrame; @@ -34,35 +36,27 @@ export const makeVideoTrackHandler = convertMediaState: ConvertMediaState; controller: AbortController; videoCodec: ConvertMediaVideoCodec; - onVideoTrack: ResolveVideoActionFn; + onVideoTrack: ResolveVideoActionFn | null; logLevel: LogLevel; + container: ConvertMediaContainer; }): OnVideoTrack => async (track) => { if (controller.signal.aborted) { throw new Error('Aborted'); } - const videoEncoderConfig = await getVideoEncoderConfig({ - codec: videoCodec === 'vp9' ? 'vp09.00.10.08' : videoCodec, - height: track.displayAspectHeight, - width: track.displayAspectWidth, - }); - const videoDecoderConfig = - await getVideoDecoderConfigWithHardwareAcceleration(track); - const videoOperation = await resolveVideoAction({ - videoDecoderConfig, - videoEncoderConfig, + const videoOperation = await (onVideoTrack ?? defaultResolveVideoAction)({ track, videoCodec, - resolverFunction: onVideoTrack, logLevel, + container, }); - if (videoOperation === 'drop') { + if (videoOperation.type === 'drop') { return null; } - if (videoOperation === 'copy') { + if (videoOperation.type === 'copy') { const videoTrack = await state.addTrack({ type: 'video', color: track.color, @@ -78,6 +72,14 @@ export const makeVideoTrackHandler = }; } + const videoEncoderConfig = await getVideoEncoderConfig({ + codec: videoOperation.videoCodec, + height: track.displayAspectHeight, + width: track.displayAspectWidth, + }); + const videoDecoderConfig = + await getVideoDecoderConfigWithHardwareAcceleration(track); + if (videoEncoderConfig === null) { abortConversion( new Error( diff --git a/packages/webcodecs/src/resolve-audio-action.ts b/packages/webcodecs/src/resolve-audio-action.ts index 607d5746b36..56c8cb8df3c 100644 --- a/packages/webcodecs/src/resolve-audio-action.ts +++ b/packages/webcodecs/src/resolve-audio-action.ts @@ -3,73 +3,83 @@ import type { LogLevel, MediaParserAudioCodec, } from '@remotion/media-parser'; +import {canReencodeAudioTrack} from './can-reencode-audio-track'; import type {ConvertMediaAudioCodec} from './codec-id'; +import type {ConvertMediaContainer} from './convert-media'; import {Log} from './log'; export type AudioOperation = - | {type: 'reencode'} + | {type: 'reencode'; bitrate: number; audioCodec: ConvertMediaAudioCodec} | {type: 'copy'} | {type: 'drop'}; -const canCopyAudioTrack = ( - inputCodec: MediaParserAudioCodec, - outputCodec: ConvertMediaAudioCodec, -) => { +const canCopyAudioTrack = ({ + inputCodec, + outputCodec, + container, +}: { + inputCodec: MediaParserAudioCodec; + outputCodec: ConvertMediaAudioCodec; + container: ConvertMediaContainer; +}) => { if (outputCodec === 'opus') { - return inputCodec === 'opus'; + return inputCodec === 'opus' && container === 'webm'; } throw new Error(`Unhandled codec: ${outputCodec satisfies never}`); }; export type ResolveAudioActionFn = (options: { - canReencode: boolean; - canCopy: boolean; + track: AudioTrack; + audioCodec: ConvertMediaAudioCodec; + logLevel: LogLevel; + container: ConvertMediaContainer; }) => AudioOperation | Promise; -export const defaultResolveAudioAction: ResolveAudioActionFn = ({ - canReencode, - canCopy, -}) => { - if (canCopy) { - return {type: 'copy'}; - } - - if (canReencode) { - return {type: 'reencode'}; - } - - // TODO: Make a fail option? - return {type: 'drop'}; -}; +const DEFAULT_BITRATE = 128_000; -export const resolveAudioAction = async ({ - audioDecoderConfig, - audioEncoderConfig, +export const defaultResolveAudioAction: ResolveAudioActionFn = async ({ track, audioCodec, - resolverFunction, logLevel, -}: { - audioDecoderConfig: AudioDecoderConfig | null; - audioEncoderConfig: AudioEncoderConfig | null; - track: AudioTrack; - audioCodec: ConvertMediaAudioCodec; - resolverFunction: ResolveAudioActionFn; - logLevel: LogLevel; + container, }): Promise => { - const canReencode = Boolean(audioDecoderConfig && audioEncoderConfig); - const canCopy = canCopyAudioTrack(track.codecWithoutConfig, audioCodec); + const bitrate = DEFAULT_BITRATE; + const canReencode = await canReencodeAudioTrack({ + audioCodec, + track, + bitrate, + }); - const resolved = await resolverFunction({ - canReencode, - canCopy, + const canCopy = canCopyAudioTrack({ + inputCodec: track.codecWithoutConfig, + outputCodec: audioCodec, + container, }); + if (canCopy) { + Log.verbose( + logLevel, + `Track ${track.trackId} (audio): Can re-encode = ${canReencode}, can copy = ${canCopy}, action = copy`, + ); + + return Promise.resolve({type: 'copy'}); + } + + if (canReencode) { + Log.verbose( + logLevel, + `Track ${track.trackId} (audio): Can re-encode = ${canReencode}, can copy = ${canCopy}, action = reencode`, + ); + + return Promise.resolve({type: 'reencode', bitrate, audioCodec}); + } + Log.verbose( logLevel, - `Track ${track.trackId} (audio): Can re-encode = ${canReencode}, can copy = ${canCopy}, action = ${resolved}`, + `Track ${track.trackId} (audio): Can re-encode = ${canReencode}, can copy = ${canCopy}, action = drop`, ); - return resolved; + // TODO: Make a fail option? + return Promise.resolve({type: 'drop'}); }; diff --git a/packages/webcodecs/src/resolve-video-action.ts b/packages/webcodecs/src/resolve-video-action.ts index 2c75eb34348..6dd4558f36b 100644 --- a/packages/webcodecs/src/resolve-video-action.ts +++ b/packages/webcodecs/src/resolve-video-action.ts @@ -3,73 +3,80 @@ import type { MediaParserVideoCodec, VideoTrack, } from '@remotion/media-parser'; +import {canReencodeVideoTrack} from './can-reencode-video-track'; import type {ConvertMediaVideoCodec} from './codec-id'; +import type {ConvertMediaContainer} from './convert-media'; import {Log} from './log'; -export type VideoOperation = 'reencode' | 'copy' | 'drop'; +export type VideoOperation = + | {type: 'reencode'; videoCodec: ConvertMediaVideoCodec} + | {type: 'copy'} + | {type: 'drop'}; -const canCopyVideoTrack = ( - inputCodec: MediaParserVideoCodec, - outputCodec: ConvertMediaVideoCodec, -) => { +const canCopyVideoTrack = ({ + inputCodec, + outputCodec, + container, +}: { + inputCodec: MediaParserVideoCodec; + outputCodec: ConvertMediaVideoCodec; + container: ConvertMediaContainer; +}) => { if (outputCodec === 'vp8') { - return inputCodec === 'vp8'; + return inputCodec === 'vp8' && container === 'webm'; } if (outputCodec === 'vp9') { - return inputCodec === 'vp9'; + return inputCodec === 'vp9' && container === 'webm'; } throw new Error(`Unhandled codec: ${outputCodec satisfies never}`); }; export type ResolveVideoActionFn = (options: { - canReencode: boolean; - canCopy: boolean; + videoCodec: ConvertMediaVideoCodec; + track: VideoTrack; + logLevel: LogLevel; + container: ConvertMediaContainer; }) => VideoOperation | Promise; -export const defaultResolveVideoAction: ResolveVideoActionFn = ({ - canReencode, - canCopy, -}) => { +export const defaultResolveVideoAction: ResolveVideoActionFn = async ({ + track, + videoCodec, + logLevel, + container, +}): Promise => { + const canCopy = canCopyVideoTrack({ + inputCodec: track.codecWithoutConfig, + outputCodec: videoCodec, + container, + }); + if (canCopy) { - return 'copy'; - } + Log.verbose( + logLevel, + `Track ${track.trackId} (video): Can copy, therefore copying`, + ); - if (canReencode) { - return 'reencode'; + return Promise.resolve({type: 'copy'}); } - // TODO: Make a fail option? - return 'drop'; -}; + const canReencode = await canReencodeVideoTrack({videoCodec, track}); -export const resolveVideoAction = async ({ - videoDecoderConfig, - videoEncoderConfig, - track, - videoCodec, - resolverFunction, - logLevel, -}: { - videoDecoderConfig: VideoDecoderConfig | null; - videoEncoderConfig: VideoEncoderConfig | null; - videoCodec: ConvertMediaVideoCodec; - track: VideoTrack; - resolverFunction: ResolveVideoActionFn; - logLevel: LogLevel; -}): Promise => { - const canReencode = Boolean(videoDecoderConfig && videoEncoderConfig); - const canCopy = canCopyVideoTrack(track.codecWithoutConfig, videoCodec); + if (canReencode) { + Log.verbose( + logLevel, + `Track ${track.trackId} (video): Cannot copy, but re-enconde, therefore re-encoding`, + ); + + return Promise.resolve({type: 'reencode', videoCodec}); + } - const resolved = await resolverFunction({ - canReencode, - canCopy, - }); Log.verbose( logLevel, - `Track ${track.trackId} (video): Can re-encode = ${canReencode}, can copy = ${canCopy}, action = ${resolved}`, + `Track ${track.trackId} (video): Can neither copy nor re-encode, therefore dropping`, ); - return resolved; + // TODO: Make a fail option? + return Promise.resolve({type: 'drop'}); }; diff --git a/packages/webcodecs/src/video-encoder-config.ts b/packages/webcodecs/src/video-encoder-config.ts index 534c0b635e1..3de803ca3f8 100644 --- a/packages/webcodecs/src/video-encoder-config.ts +++ b/packages/webcodecs/src/video-encoder-config.ts @@ -1,10 +1,24 @@ -export const getVideoEncoderConfig = async ( - config: VideoEncoderConfig, -): Promise => { +import type {ConvertMediaVideoCodec} from './codec-id'; + +export const getVideoEncoderConfig = async ({ + width, + height, + codec, +}: { + width: number; + height: number; + codec: ConvertMediaVideoCodec; +}): Promise => { if (typeof VideoEncoder === 'undefined') { return null; } + const config: VideoEncoderConfig = { + codec: codec === 'vp9' ? 'vp09.00.10.08' : codec, + height, + width, + }; + const hardware: VideoEncoderConfig = { ...config, hardwareAcceleration: 'prefer-hardware',