From ce0c4fdbcfa49384ad06a78a1f98c403317fa373 Mon Sep 17 00:00:00 2001 From: Przemek Galezki Date: Fri, 16 Feb 2024 22:56:51 +0100 Subject: [PATCH] implement: decode multiple videos at a time in parallel --- .../controllers/video-export/controller.ts | 101 +++++++++----- .../controllers/video-export/decode_worker.ts | 54 ++++++++ .../controllers/video-export/encode_worker.ts | 40 ++++++ s/context/controllers/video-export/worker.ts | 128 ------------------ s/tools/mp4boxjs/demuxer.ts | 6 +- 5 files changed, 167 insertions(+), 162 deletions(-) create mode 100644 s/context/controllers/video-export/decode_worker.ts create mode 100644 s/context/controllers/video-export/encode_worker.ts delete mode 100644 s/context/controllers/video-export/worker.ts diff --git a/s/context/controllers/video-export/controller.ts b/s/context/controllers/video-export/controller.ts index fd87667..f77d0ea 100644 --- a/s/context/controllers/video-export/controller.ts +++ b/s/context/controllers/video-export/controller.ts @@ -1,39 +1,36 @@ import {TimelineActions} from "../timeline/actions.js" import {Compositor} from "../compositor/controller.js" import {FFmpegHelper} from "./helpers/FFmpegHelper/helper.js" +import {generate_id} from "@benev/slate/x/tools/generate_id.js" import {FileSystemHelper} from "./helpers/FileSystemHelper/helper.js" import {AnyEffect, VideoEffect, XTimeline} from "../timeline/types.js" import {get_effects_at_timestamp} from "./utils/get_effects_at_timestamp.js" +interface DecodedFrame { + frame: VideoFrame + effect_id: string + timestamp: number + frames_count: number + frame_id: string +} + export class VideoExport { - #worker = new Worker(new URL("./worker.js", import.meta.url), {type: "module"}) + #encode_worker = new Worker(new URL("./encode_worker.js", import.meta.url), {type: "module"}) #file: Uint8Array | null = null #FileSystemHelper = new FileSystemHelper() + #timestamp = 0 #timestamp_end = 0 readonly canvas = document.createElement("canvas") ctx = this.canvas.getContext("2d")! + current_frame = 0 decoded_effects = new Map() + decoded_frames: Map = new Map() constructor(private actions: TimelineActions, private ffmpeg: FFmpegHelper, private compositor: Compositor) { this.canvas.width = 1280 this.canvas.height = 720 - this.#worker.addEventListener("message", async (msg: MessageEvent<{ - binary: Uint8Array, - progress: number, - action: string, - frame: ImageBitmap, - chunk: EncodedVideoChunk - }>) => { - if(msg.data.action === "binary") { - const binary_container_name = "raw.h264" - await ffmpeg.write_binary_into_container(msg.data.binary, binary_container_name) - await ffmpeg.mux(binary_container_name, "test.mp4") - const muxed_file = await ffmpeg.get_muxed_file() - this.#file = muxed_file - } - }) } async save_file() { @@ -44,16 +41,34 @@ export class VideoExport { export_start(timeline: XTimeline) { const sorted_effects = this.#sort_effects_by_track(timeline.effects) this.#timestamp_end = Math.max(...sorted_effects.map(effect => effect.start_at_position + effect.duration)) - this.export_process(sorted_effects) + this.#export_process(sorted_effects) this.actions.set_is_exporting(true) } - async export_process(effects: AnyEffect[]) { + #find_closest_effect_frame(effect: VideoEffect, timestamp: number) { + let closest: DecodedFrame | null = null + let current_difference = Infinity + this.decoded_frames.forEach(frame => { + if(frame.effect_id === effect.id) { + const difference = Math.abs(frame.timestamp - timestamp) + if(difference < current_difference) { + current_difference = difference + closest = frame + } + } + }) + return closest! + } + + async #export_process(effects: AnyEffect[]) { const effects_at_timestamp = get_effects_at_timestamp(effects, this.#timestamp) const draw_queue: (() => void)[] = [] + let frame_duration = null for(const effect of effects_at_timestamp) { if(effect.kind === "video") { - const frame = await this.#get_frame_from_video(effect, this.#timestamp) + const {frame, frames_count, frame_id} = await this.#get_frame_from_video(effect, this.#timestamp) + frame_duration = effect.duration / frames_count + this.decoded_frames.delete(frame_id) draw_queue.push(() => { this.ctx?.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height) frame.close() @@ -66,15 +81,24 @@ export class VideoExport { for(const draw of draw_queue) {draw()} this.#encode_composed_frame(this.canvas) - this.#timestamp += 1000/30 + this.#timestamp += frame_duration ?? Math.ceil(1000/60) const progress = this.#timestamp / this.#timestamp_end * 100 // for progress bar this.actions.set_export_progress(progress) + if(this.#timestamp >= this.#timestamp_end) { - this.#worker.postMessage({action: "get-binary"}) + this.#encode_worker.postMessage({action: "get-binary"}) + this.#encode_worker.onmessage = async (msg) => { + if(msg.data.action === "binary") { + const binary_container_name = "raw.h264" + await this.ffmpeg.write_binary_into_container(msg.data.binary, binary_container_name) + await this.ffmpeg.mux(binary_container_name, "test.mp4") + const muxed_file = await this.ffmpeg.get_muxed_file() + this.#file = muxed_file + } + } return } - - requestAnimationFrame(() => this.export_process(effects)) + requestAnimationFrame(() => this.#export_process(effects)) } get #frame_config(): VideoFrameInit { @@ -88,26 +112,39 @@ export class VideoExport { #encode_composed_frame(canvas: HTMLCanvasElement) { const frame = new VideoFrame(canvas, this.#frame_config) - this.#worker.postMessage({frame, action: "encode"}) + this.#encode_worker.postMessage({frame, action: "encode"}) frame.close() } - #get_frame_from_video(effect: VideoEffect, timestamp: number): Promise { + #get_frame_from_video(effect: VideoEffect, timestamp: number): Promise { if(!this.decoded_effects.has(effect.id)) { this.#extract_frames_from_video(effect, timestamp) } - return new Promise((resolve, reject) => { - this.#worker.postMessage({action: "get-frame", timestamp, effect_id: effect.id}) - this.#worker.onmessage = (e) => { - if(e.data.action === "frame") { - resolve(e.data.frame) - } + return new Promise((resolve) => { + const decoded = this.#find_closest_effect_frame(effect, timestamp) + if(decoded) { + resolve(decoded) + } else { + const interval = setInterval(() => { + const decoded = this.#find_closest_effect_frame(effect, timestamp) + if(decoded) { + resolve(decoded) + clearInterval(interval) + } + }, 100) } }) } async #extract_frames_from_video(effect: VideoEffect, timestamp: number) { - this.#worker.postMessage({action: "demux", effect: { + const worker = new Worker(new URL("./decode_worker.js", import.meta.url), {type: "module"}) + worker.addEventListener("message", (msg) => { + if(msg.data.action === "new-frame") { + const id = generate_id() + this.decoded_frames.set(id, {...msg.data.frame, frame_id: id}) + } + }) + worker.postMessage({action: "demux", effect: { ...effect, file: effect.file }, starting_timestamp: timestamp}) diff --git a/s/context/controllers/video-export/decode_worker.ts b/s/context/controllers/video-export/decode_worker.ts new file mode 100644 index 0000000..48eacc3 --- /dev/null +++ b/s/context/controllers/video-export/decode_worker.ts @@ -0,0 +1,54 @@ +import {VideoEffect} from "../timeline/types.js" +import {MP4Demuxer} from "../../../tools/mp4boxjs/demuxer.js" + +let timestamp = 0 +let end_timestamp = 0 +let wait_time = 0 +let interval_number = 0 +let decoded_effect: VideoEffect +let frames = 0 + +const decoder = new VideoDecoder({ + output(frame) { + self.postMessage({action: "new-frame", frame: {timestamp, frame, effect_id: decoded_effect.id, frames_count: frames}}) + frame.close() + timestamp += decoded_effect.duration / frames + wait_time = 0 + }, + error: (e) => console.log(e) +}) + +const interval = () => setInterval(() => { + wait_time += 100 + if(wait_time === 200) { + decoder.flush() + } + if(timestamp === end_timestamp) { + clearInterval(interval_number) + } +}, 100) + +const demux = (file: File) => new MP4Demuxer(file, { + async onConfig(config: VideoDecoderConfig) { + decoder.configure({...config}) + await decoder.flush() + }, + async onChunk(chunk: EncodedVideoChunk) { + decoder.decode(chunk) + end_timestamp += decoded_effect.duration / frames + }, + framesCount(frames_count) { + frames = frames_count + }, + setStatus() {} +}) + +self.addEventListener("message", async message => { + if(message.data.action === "demux") { + timestamp = message.data.starting_timestamp + end_timestamp = message.data.starting_timestamp + decoded_effect = message.data.effect + interval_number = interval() + demux(message.data.effect.file) + } +}) diff --git a/s/context/controllers/video-export/encode_worker.ts b/s/context/controllers/video-export/encode_worker.ts new file mode 100644 index 0000000..44dadbc --- /dev/null +++ b/s/context/controllers/video-export/encode_worker.ts @@ -0,0 +1,40 @@ +import {BinaryAccumulator} from "./tools/BinaryAccumulator/tool.js" + +const binary_accumulator = new BinaryAccumulator() + +async function handleChunk(chunk: EncodedVideoChunk) { + const chunkData = new Uint8Array(chunk.byteLength); + chunk.copyTo(chunkData) + binary_accumulator.addChunk(chunkData) +} + +// for later: https://github.com/gpac/mp4box.js/issues/243 +const config: VideoEncoderConfig = { + codec: "avc1.4d002a", // avc1.42001E / avc1.4d002a / avc1.640034 + avc: {format: "annexb"}, + width: 1280, + height: 720, + bitrate: 4_000_000, // 2 Mbps + framerate: 30, + bitrateMode: "constant" +} + +const encoder = new VideoEncoder({ + output: handleChunk, + error: (e: any) => { + console.log(e.message) + }, +}) + +encoder.configure(config) + +self.addEventListener("message", async message => { + if(message.data.action === "encode") { + const frame = message.data.frame as VideoFrame + encoder.encode(frame) + frame.close() + } + if(message.data.action === "get-binary") { + self.postMessage({action: "binary", binary: binary_accumulator.binary}) + } +}) diff --git a/s/context/controllers/video-export/worker.ts b/s/context/controllers/video-export/worker.ts deleted file mode 100644 index fc89dfe..0000000 --- a/s/context/controllers/video-export/worker.ts +++ /dev/null @@ -1,128 +0,0 @@ -import {MP4Demuxer} from "../../../tools/mp4boxjs/demuxer.js" -import {BinaryAccumulator} from "./tools/BinaryAccumulator/tool.js" - -const binary_accumulator = new BinaryAccumulator() -const frames: {timestamp: number, frame: VideoFrame, effect_id: string}[] = [] -let timestamp = 0 -let end_timestamp = 0 -let wait_time = 0 -let interval_number = 0 -let currently_decoded_effect = "" - -function draw_blank_canvas() { - const canvas = new OffscreenCanvas(720, 1280) - const ctx = canvas.getContext("2d") - ctx!.fillStyle = "blue"; - ctx!.fillRect(0, 0, canvas.width, canvas.height); -} - -async function handleChunk(chunk: EncodedVideoChunk) { - const chunkData = new Uint8Array(chunk.byteLength); - chunk.copyTo(chunkData) - binary_accumulator.addChunk(chunkData) -} - -const decoder = new VideoDecoder({ - async output(frame) { - frames.push({timestamp, frame, effect_id: currently_decoded_effect}) - timestamp += 1000/30 - wait_time = 0 - }, - error: (e) => console.log(e) -}) - -const interval = () => setInterval(() => { - wait_time += 10 - if(wait_time === 500) { - decoder.flush() - } - if(timestamp === end_timestamp) - clearInterval(interval_number) -}, 10) - -const demux = (file: File) => new MP4Demuxer(file, { - async onConfig(config: VideoDecoderConfig) { - decoder.configure(config) - await decoder.flush() - }, - async onChunk(chunk: EncodedVideoChunk) { - decoder.decode(chunk) - - end_timestamp += 1000/30 - }, - setStatus() {} -}) - - -// for later: https://github.com/gpac/mp4box.js/issues/243 -const config: VideoEncoderConfig = { - codec: "avc1.4d002a", // avc1.42001E / avc1.4d002a / avc1.640034 - avc: {format: "annexb"}, - width: 1280, - height: 720, - bitrate: 4_000_000, // 2 Mbps - framerate: 30, - bitrateMode: "constant" -} - -const encoder = new VideoEncoder({ - output: handleChunk, - error: (e: any) => { - console.log(e.message) - }, -}) - -encoder.configure(config) - -self.addEventListener("message", async message => { - if(message.data.action === "encode") { - const frame = message.data.frame as VideoFrame - encoder.encode(frame) - frame.close() - } - if(message.data.action === "get-binary") { - self.postMessage({action: "binary", binary: binary_accumulator.binary}) - } - if(message.data.action === "demux") { - timestamp = message.data.starting_timestamp - end_timestamp = message.data.starting_timestamp - currently_decoded_effect = message.data.effect.id - interval_number = interval() - demux(message.data.effect.file) - } - if(message.data.action === "get-frame") { - const frame = await getFrame(message.data.timestamp, message.data.effect_id) - self.postMessage({action: "frame", frame}) - frame.close() - } -}) - -function find_frame(timestamp: number, effect_id: string) { - let to_remove = 0 - const frame = frames.find((frame, i) => { - if(frame.timestamp === timestamp && frame.effect_id === effect_id) { - to_remove = i - return frame - } - }) - return {to_remove, frame} -} - -function getFrame(timestamp: number, effect_id: string): Promise { - return new Promise((resolve) => { - const {frame, to_remove} = find_frame(timestamp, effect_id) - if(frame) { - resolve(frame.frame) - frames.splice(to_remove, 1) - } else { - const interval = setInterval(() => { - const {frame, to_remove} = find_frame(timestamp, effect_id) - if(frame) { - resolve(frame.frame) - frames.splice(to_remove, 1) - clearInterval(interval) - } - }, 100) - } - }) -} diff --git a/s/tools/mp4boxjs/demuxer.ts b/s/tools/mp4boxjs/demuxer.ts index b465ef0..c2f911a 100644 --- a/s/tools/mp4boxjs/demuxer.ts +++ b/s/tools/mp4boxjs/demuxer.ts @@ -37,12 +37,14 @@ export class MP4Demuxer { #onConfig: OnConfig #onChunk: OnChunk #setStatus: SetStatus + #framesCount: (frames: number) => void #file: MP4File - constructor(file: File, {onConfig, onChunk, setStatus}: {onConfig: OnConfig, onChunk: OnChunk, setStatus: SetStatus}) { + constructor(file: File, {onConfig, onChunk, setStatus, framesCount}: {onConfig: OnConfig, onChunk: OnChunk, setStatus: SetStatus, framesCount: (frames: number) => void}) { this.#onConfig = onConfig this.#onChunk = onChunk this.#setStatus = setStatus + this.#framesCount = framesCount this.#file = MP4Box.createFile() this.#file.onError = error => setStatus("demux", error) @@ -76,7 +78,7 @@ export class MP4Demuxer { codedWidth: track.video.width, description: this.#description(track), }); - + this.#framesCount(track.nb_samples) this.#file.setExtractionOptions(track.id) this.#file.start() }