Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
wkozyra95 committed Dec 3, 2024
1 parent 7ccafb6 commit 993b69e
Show file tree
Hide file tree
Showing 29 changed files with 1,147 additions and 101 deletions.
4 changes: 2 additions & 2 deletions build_tools/nix/flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
# Fixes "ffplay" used in examples on Linux (not needed on NixOS)
env.LIBGL_DRIVERS_PATH = "${pkgs.mesa.drivers}/lib/dri";

env.LIBCLANG_PATH = "${pkgs.llvmPackages_16.libclang.lib}/lib";
env.LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib";
env.LD_LIBRARY_PATH = lib.makeLibraryPath (libcefDependencies ++ [ pkgs.mesa.drivers pkgs.libGL pkgs.blackmagic-desktop-video ]);

inputsFrom = [ packageWithoutChromium ];
Expand All @@ -89,7 +89,7 @@
nixos = pkgs.mkShell {
packages = devDependencies ++ [ pkgs.blackmagic-desktop-video];

env.LIBCLANG_PATH = "${pkgs.llvmPackages_16.libclang.lib}/lib";
env.LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib";
env.LD_LIBRARY_PATH = lib.makeLibraryPath (libcefDependencies ++ [ pkgs.blackmagic-desktop-video ]);

inputsFrom = [ packageWithoutChromium ];
Expand Down
7 changes: 4 additions & 3 deletions build_tools/nix/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
, ffmpeg_7-headless
, openssl
, pkg-config
, llvmPackages_16
, llvmPackages
, libGL
, cmake
, libopus
Expand All @@ -20,6 +20,7 @@ let
libopus
libGL
vulkan-loader
stdenv.cc.cc
] ++ lib.optionals stdenv.isDarwin [
darwin.apple_sdk.frameworks.Metal
darwin.apple_sdk.frameworks.Foundation
Expand All @@ -43,9 +44,9 @@ rustPlatform.buildRustPackage {
doCheck = false;

inherit buildInputs;
nativeBuildInputs = [ pkg-config llvmPackages_16.clang cmake makeWrapper ];
nativeBuildInputs = [ pkg-config llvmPackages.clang cmake makeWrapper ];

env.LIBCLANG_PATH = "${llvmPackages_16.libclang.lib}/lib";
env.LIBCLANG_PATH = "${llvmPackages.libclang.lib}/lib";

postFixup =
''
Expand Down
7 changes: 5 additions & 2 deletions ts/@live-compositor/core/src/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,14 @@ export class ApiClient {
});
}

public async unregisterOutput(outptuId: string): Promise<object> {
public async unregisterOutput(
outptuId: string,
body: { schedule_time_ms?: number }
): Promise<object> {
return this.serverManager.sendRequest({
method: 'POST',
route: `/api/output/${encodeURIComponent(outptuId)}/unregister`,
body: {},
body,
});
}

Expand Down
9 changes: 8 additions & 1 deletion ts/@live-compositor/core/src/compositorManager.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
import type { ApiRequest } from './api.js';

export interface SetupInstanceOptions {
/**
* sets LIVE_COMPOSITOR_AHEAD_OF_TIME_PROCESSING_ENABLE environment variable.
*/
aheadOfTimeProcessing: boolean;
}

export interface CompositorManager {
setupInstance(): Promise<void>;
setupInstance(opts: SetupInstanceOptions): Promise<void>;
sendRequest(request: ApiRequest): Promise<object>;
registerEventListener(cb: (event: unknown) => void): void;
}
4 changes: 2 additions & 2 deletions ts/@live-compositor/core/src/event.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ export function onCompositorEvent(store: InstanceContextStore, rawEvent: unknown
}
}

function parseEvent(event: any): CompositorEvent | null {
export function parseEvent(event: any): CompositorEvent | null {
if (!event.type) {
console.error(`Malformed event: ${event}`);
return null;
Expand All @@ -56,7 +56,7 @@ function parseEvent(event: any): CompositorEvent | null {
) {
return { type: event.type, inputId: event.input_id };
} else if (CompositorEventType.OUTPUT_DONE === event.type) {
return { type: event.type, outputId: event.outputId };
return { type: event.type, outputId: event.output_id };
} else {
console.error(`Unknown event type: ${event.type}`);
return null;
Expand Down
5 changes: 3 additions & 2 deletions ts/@live-compositor/core/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
export { ApiClient, ApiRequest } from './api.js';
export { LiveCompositor } from './compositor.js';
export { CompositorManager } from './compositorManager.js';
export { LiveCompositor } from './live/compositor.js';
export { OfflineCompositor } from './offline/compositor.js';
export { CompositorManager, SetupInstanceOptions } from './compositorManager.js';
export { RegisterInputRequest, RegisterInput } from './api/input.js';
export { RegisterOutputRequest, RegisterOutput } from './api/output.js';
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
import type { Renderers } from 'live-compositor';
import { _liveCompositorInternals } from 'live-compositor';
import { ApiClient } from './api.js';
import { ApiClient } from '../api.js';
import Output from './output.js';
import type { CompositorManager } from './compositorManager.js';
import type { RegisterOutput } from './api/output.js';
import { intoRegisterOutput } from './api/output.js';
import type { RegisterInput } from './api/input.js';
import { intoRegisterInput } from './api/input.js';
import { onCompositorEvent } from './event.js';
import { intoRegisterImage, intoRegisterWebRenderer } from './api/renderer.js';
import type { CompositorManager } from '../compositorManager.js';
import type { RegisterOutput } from '../api/output.js';
import { intoRegisterOutput } from '../api/output.js';
import type { RegisterInput } from '../api/input.js';
import { intoRegisterInput } from '../api/input.js';
import { onCompositorEvent } from '../event.js';
import { intoRegisterImage, intoRegisterWebRenderer } from '../api/renderer.js';

export class LiveCompositor {
private manager: CompositorManager;
private api: ApiClient;
private store: _liveCompositorInternals.InstanceContextStore;
private outputs: Record<string, Output> = {};
private startTime?: number;

public constructor(manager: CompositorManager) {
this.manager = manager;
Expand All @@ -24,11 +25,11 @@ export class LiveCompositor {

public async init(): Promise<void> {
this.manager.registerEventListener((event: unknown) => onCompositorEvent(this.store, event));
await this.manager.setupInstance();
await this.manager.setupInstance({ aheadOfTimeProcessing: false });
}

public async registerOutput(outputId: string, request: RegisterOutput): Promise<object> {
const output = new Output(outputId, request, this.api, this.store);
const output = new Output(outputId, request, this.api, this.store, this.startTime);

const apiRequest = intoRegisterOutput(request, output.scene());
const result = await this.api.registerOutput(outputId, apiRequest);
Expand All @@ -40,7 +41,8 @@ export class LiveCompositor {
public async unregisterOutput(outputId: string): Promise<object> {
this.outputs[outputId].close();
delete this.outputs[outputId];
return this.api.unregisterOutput(outputId);
// TODO: wait for event
return this.api.unregisterOutput(outputId, {});
}

public async registerInput(inputId: string, request: RegisterInput): Promise<object> {
Expand Down Expand Up @@ -90,6 +92,11 @@ export class LiveCompositor {
}

public async start(): Promise<void> {
return this.api.start();
const startTime = Date.now();
await this.api.start();
Object.values(this.outputs).forEach(output => {
output.initClock(startTime);
});
this.startTime = startTime;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,21 @@ import type { Outputs } from 'live-compositor';
import { _liveCompositorInternals, View } from 'live-compositor';
import type React from 'react';
import { createElement, useSyncExternalStore } from 'react';
import type { ApiClient, Api } from './api.js';
import Renderer from './renderer.js';
import type { RegisterOutput } from './api/output.js';
import { intoAudioInputsConfiguration } from './api/output.js';
import { throttle } from './utils.js';

type OutputContext = _liveCompositorInternals.OutputContext;
import type { ApiClient, Api } from '../api.js';
import Renderer from '../renderer.js';
import type { RegisterOutput } from '../api/output.js';
import { intoAudioInputsConfiguration } from '../api/output.js';
import { throttle } from '../utils.js';

type AudioContext = _liveCompositorInternals.AudioContext;
type LiveTimeContext = _liveCompositorInternals.LiveTimeContext;
type InstanceContextStore = _liveCompositorInternals.InstanceContextStore;

class Output {
api: ApiClient;
outputId: string;
outputCtx: OutputContext;
audioContext: AudioContext;
timeContext: LiveTimeContext;
outputShutdownStateStore: OutputShutdownStateStore;

shouldUpdateWhenReady: boolean = false;
Expand All @@ -26,7 +28,8 @@ class Output {
outputId: string,
registerRequest: RegisterOutput,
api: ApiClient,
store: InstanceContextStore
store: InstanceContextStore,
startTimestamp: number | undefined
) {
this.api = api;
this.outputId = outputId;
Expand All @@ -36,18 +39,23 @@ class Output {
this.shouldUpdateWhenReady = true;
};

const hasAudio = 'audio' in registerRequest && !!registerRequest.audio;
if (hasAudio) {
const supportsAudio = 'audio' in registerRequest && !!registerRequest.audio;
if (supportsAudio) {
this.initialAudioConfig = registerRequest.audio!.initial ?? { inputs: [] };
}

const onUpdate = () => this.throttledUpdate();
this.outputCtx = new _liveCompositorInternals.OutputContext(onUpdate, hasAudio);
this.audioContext = new _liveCompositorInternals.AudioContext(onUpdate, supportsAudio);
this.timeContext = new _liveCompositorInternals.LiveTimeContext();
if (startTimestamp !== undefined) {
this.timeContext.initClock(startTimestamp);
}

if (registerRequest.video) {
const rootElement = createElement(OutputRootComponent, {
instanceStore: store,
outputCtx: this.outputCtx,
audioContext: this.audioContext,
timeContext: this.timeContext,
outputRoot: registerRequest.video.root,
outputShutdownStateStore: this.outputShutdownStateStore,
});
Expand All @@ -61,7 +69,7 @@ class Output {
}

public scene(): { video?: Api.Video; audio?: Api.Audio } {
const audio = this.outputCtx.getAudioConfig() ?? this.initialAudioConfig;
const audio = this.audioContext.getAudioConfig() ?? this.initialAudioConfig;
return {
video: this.videoRenderer && { root: this.videoRenderer.scene() },
audio: audio && intoAudioInputsConfiguration(audio),
Expand All @@ -83,6 +91,10 @@ class Output {
this.throttledUpdate();
}
}

public initClock(timestamp: number) {
this.timeContext.initClock(timestamp);
}
}

// External store to share shutdown information between React tree
Expand Down Expand Up @@ -113,12 +125,14 @@ class OutputShutdownStateStore {
function OutputRootComponent({
outputRoot,
instanceStore,
outputCtx,
timeContext,
audioContext,
outputShutdownStateStore,
}: {
outputRoot: React.ReactElement;
instanceStore: InstanceContextStore;
outputCtx: OutputContext;
audioContext: AudioContext;
timeContext: LiveTimeContext;
outputShutdownStateStore: OutputShutdownStateStore;
}) {
const shouldShutdown = useSyncExternalStore(
Expand All @@ -133,7 +147,8 @@ function OutputRootComponent({

const reactCtx = {
instanceStore,
outputCtx,
timeContext,
audioContext,
};
return createElement(
_liveCompositorInternals.LiveCompositorContext.Provider,
Expand Down
103 changes: 103 additions & 0 deletions ts/@live-compositor/core/src/offline/compositor.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
import type { Renderers } from 'live-compositor';
import { _liveCompositorInternals, CompositorEventType } from 'live-compositor';
import { ApiClient } from '../api.js';
import type { CompositorManager } from '../compositorManager.js';
import type { RegisterOutput } from '../api/output.js';
import { intoRegisterOutput } from '../api/output.js';
import type { RegisterInput } from '../api/input.js';
import { intoRegisterInput } from '../api/input.js';
import { intoRegisterImage, intoRegisterWebRenderer } from '../api/renderer.js';
import OfflineOutput from './output.js';
import { parseEvent } from '../event.js';

/**
* Offline rendering only supports one output, so we can just pick any value to use
* as an output ID.
*/
const OFFLINE_OUTPUT_ID = 'offline_output';

export class OfflineCompositor {
private manager: CompositorManager;
private api: ApiClient;
private store: _liveCompositorInternals.InstanceContextStore;
private renderStarted: boolean = false;

public constructor(manager: CompositorManager) {
this.manager = manager;
this.api = new ApiClient(this.manager);
this.store = new _liveCompositorInternals.InstanceContextStore();
}

public async init(): Promise<void> {
this.checkNotStarted();
await this.manager.setupInstance({ aheadOfTimeProcessing: true });
}

public async render(request: RegisterOutput, durationMs: number) {
this.checkNotStarted();
this.renderStarted = true;

const output = new OfflineOutput(OFFLINE_OUTPUT_ID, request, this.api, this.store, durationMs);
const apiRequest = intoRegisterOutput(request, output.scene());
await this.api.registerOutput(OFFLINE_OUTPUT_ID, apiRequest);
await output.scheduleAllUpdates();

// at this point all scene update requests should already be delivered

await this.api.unregisterOutput(OFFLINE_OUTPUT_ID, { schedule_time_ms: durationMs });

const renderPromise = new Promise<void>((res, _rej) => {
this.manager.registerEventListener(rawEvent => {
const event = parseEvent(rawEvent);
if (
event &&
event.type === CompositorEventType.OUTPUT_DONE &&
event.outputId === OFFLINE_OUTPUT_ID
) {
res();
}
});
});

await this.api.start();

await renderPromise;
output.outputShutdownStateStore.close();
}

public async registerInput(inputId: string, request: RegisterInput): Promise<object> {
this.checkNotStarted();
return this.store.runBlocking(async updateStore => {
const result = await this.api.registerInput(inputId, intoRegisterInput(request));
updateStore({ type: 'add_input', input: { inputId } });
return result;
});
}

public async registerShader(
shaderId: string,
request: Renderers.RegisterShader
): Promise<object> {
this.checkNotStarted();
return this.api.registerShader(shaderId, request);
}

public async registerImage(imageId: string, request: Renderers.RegisterImage): Promise<object> {
this.checkNotStarted();
return this.api.registerImage(imageId, intoRegisterImage(request));
}

public async registerWebRenderer(
instanceId: string,
request: Renderers.RegisterWebRenderer
): Promise<object> {
this.checkNotStarted();
return this.api.registerWebRenderer(instanceId, intoRegisterWebRenderer(request));
}

private checkNotStarted() {
if (this.renderStarted) {
throw new Error('Render was already started.');
}
}
}
Loading

0 comments on commit 993b69e

Please sign in to comment.