📺 Readded remote output capture

- Small output fixes

src/electron/capture/CaptureHelper.ts

@@ -11,6 +11,7 @@ export class CaptureHelper {
     static Transmitter = CaptureTransmitter
 
     private static framerates: any = {
+        stage: 20,
         server: 30,
         unconnected: 1,
         connected: 30,
@@ -21,15 +22,16 @@ export class CaptureHelper {
         let screen: Display = this.getWindowScreen(window)
 
         let defaultFramerates = {
-            server: this.framerates.server,
             ndi: this.framerates.connected,
+            server: this.framerates.server,
+            stage: this.framerates.stage,
         }
 
         return {
             window,
             subscribed: false,
             displayFrequency: screen.displayFrequency || 60,
-            options: { server: false, ndi: false },
+            options: { ndi: false, server: false, stage: false },
             framerates: defaultFramerates,
             id,
         }
@@ -52,6 +54,9 @@ export class CaptureHelper {
                 CaptureTransmitter.startChannel(id, "ndi")
             }
         }
+
+        if (captureOptions.options.server) CaptureTransmitter.startChannel(id, "server")
+        if (captureOptions.options.stage) CaptureTransmitter.startChannel(id, "stage")
     }
 
     static getWindowScreen(window: BrowserWindow) {

src/electron/capture/helpers/CaptureLifecycle.ts

@@ -1,6 +1,7 @@
 import { NativeImage } from "electron"
 import { CaptureHelper } from "../CaptureHelper"
 import { OutputHelper } from "../../output/OutputHelper"
+import { CaptureTransmitter } from "./CaptureTransmitter"
 
 export class CaptureLifecycle {
     static startCapture(id: string, toggle: any = {}) {
@@ -17,6 +18,9 @@ export class CaptureLifecycle {
         if (output.captureOptions) {
             const capture = output.captureOptions
             Object.keys(toggle).map((key) => {
+                // turn off capture
+                if (capture.options[key] && !toggle[key]) CaptureTransmitter.stopChannel(id, key)
+                // set capture on/off
                 capture.options[key] = toggle[key]
             })
         }
@@ -30,10 +34,14 @@ export class CaptureLifecycle {
         cpuCapture()
         async function cpuCapture() {
             if (!output.captureOptions || output.captureOptions.window.isDestroyed()) return
+
             let image = await output.captureOptions.window.webContents.capturePage()
             processFrame(image)
+
+            if (!output.captureOptions) return
             let frameRate = output.captureOptions.framerates.ndi
             if (output.captureOptions.framerates.server > frameRate) frameRate = output.captureOptions.framerates.server
+
             const ms = Math.round(1000 / frameRate)
             setTimeout(cpuCapture, ms)
         }
@@ -74,7 +82,6 @@ export class CaptureLifecycle {
         function endSubscription() {
             if (!capture?.subscribed) return
 
-            capture.window.webContents.endFrameSubscription()
             capture.subscribed = false
         }
 

src/electron/capture/helpers/CaptureTransmitter.ts

@@ -1,12 +1,12 @@
 import type { NativeImage, Size } from "electron"
 import os from "os"
-import { OUTPUT_STREAM } from "../../../types/Channels"
-import { toServer } from "../../servers"
-import util from "../../ndi/vingester-util"
+import { OUTPUT, OUTPUT_STREAM } from "../../../types/Channels"
 import { NdiSender } from "../../ndi/NdiSender"
-import { CaptureHelper } from "../CaptureHelper"
-import { CaptureOptions } from "../CaptureOptions"
+import util from "../../ndi/vingester-util"
 import { OutputHelper } from "../../output/OutputHelper"
+import { toServer } from "../../servers"
+import { CaptureHelper } from "../CaptureHelper"
+import { toApp } from "../.."
 
 export type Channel = {
     key: string
@@ -27,6 +27,7 @@ export class CaptureTransmitter {
         //this.startChannel(captureId, "preview")
         if (capture.options.ndi) this.startChannel(captureId, "ndi")
         if (capture.options.server) this.startChannel(captureId, "server")
+        if (capture.options.stage) this.startChannel(captureId, "stage")
 
         if (capture.options.ndi) {
             //ENABLE TO TRACK NDI FRAME RATES
@@ -57,6 +58,11 @@ export class CaptureTransmitter {
         }
     }
 
+    static stopChannel(captureId: string, key: string) {
+        const combinedKey = `${captureId}-${key}`
+        clearInterval(this.channels[combinedKey].timer)
+    }
+
     static handleChannelInterval(captureId: string, key: string) {
         const combinedKey = `${captureId}-${key}`
         const channel = this.channels[combinedKey]
@@ -73,8 +79,11 @@ export class CaptureTransmitter {
                 this.sendBufferToNdi(channel.captureId, image, { size })
                 break
             case "server":
-                const options = OutputHelper.getOutput(captureId)?.captureOptions
-                if (options) this.sendBufferToServer(options, image)
+                // const options = OutputHelper.getOutput(captureId)?.captureOptions
+                this.sendBufferToServer(captureId, image)
+                break
+            case "stage":
+                this.sendBufferToMain(captureId, image)
                 break
         }
     }
@@ -116,10 +125,29 @@ export class CaptureTransmitter {
         this.requestList = newList
     }
 
+    // MAIN (STAGE OUTPUT)
+    static sendBufferToMain(captureId: string, image: NativeImage) {
+        if (!image) return
+        // image = this.resizeImage(image, options.size, previewSize)
+
+        const buffer = image.getBitmap()
+        const size = image.getSize()
+
+        /*  convert from ARGB/BGRA (Electron/Chromium capture output) to RGBA (Web canvas)  */
+        if (os.endianness() === "BE") util.ImageBufferAdjustment.ARGBtoRGBA(buffer)
+        else util.ImageBufferAdjustment.BGRAtoRGBA(buffer)
+
+        let msg = { channel: "BUFFER", data: { id: captureId, buffer, size } }
+        toApp(OUTPUT, msg)
+        this.sendToStageOutputs(msg)
+        this.sendToRequested(msg)
+    }
+
     // SERVER
 
     // const outputServerSize: Size = { width: 1280, height: 720 }
-    static sendBufferToServer(capture: CaptureOptions, image: NativeImage) {
+    static sendBufferToServer(outputId: string, image: NativeImage) {
+        // capture: CaptureOptions
         if (!image) return
 
         // send output image size
@@ -132,7 +160,7 @@ export class CaptureTransmitter {
         if (os.endianness() === "BE") util.ImageBufferAdjustment.ARGBtoRGBA(buffer)
         else util.ImageBufferAdjustment.BGRAtoRGBA(buffer)
 
-        toServer(OUTPUT_STREAM, { channel: "STREAM", data: { id: capture.id, buffer, size } })
+        toServer(OUTPUT_STREAM, { channel: "STREAM", data: { id: outputId, buffer, size } })
     }
 
     static requestPreview(data: any) {

src/electron/output/OutputHelper.ts

@@ -26,6 +26,7 @@ export class OutputHelper {
 
             //PREVIEW_RESOLUTION: (data: any) => () => {}, //TODO: Eliminate?   Was -> updatePreviewResolution(data)
             REQUEST_PREVIEW: (data: any) => CaptureHelper.Transmitter.requestPreview(data),
+            CAPTURE: (data: any) => CaptureHelper.Lifecycle.startCapture(data.id, data.captures),
 
             IDENTIFY_SCREENS: (data: any) => OutputHelper.Identify.identifyScreens(data),
             //PREVIEW_BOUNDS: (data: any) => OutputHelper.Bounds.setPreviewBounds(data),

src/electron/output/helpers/OutputSend.ts

@@ -6,7 +6,7 @@ export class OutputSend {
         OutputHelper.getAllOutputs().forEach(sendToWindow)
 
         function sendToWindow([id, output]: any) {
-            if ((msg.data?.id && msg.data.id !== id) || !output.window || output.window.isDestroyed()) return
+            if ((msg.data?.id && msg.data.id !== id) || !output?.window || output.window.isDestroyed()) return
 
             let tempMsg: any = JSON.parse(JSON.stringify(msg))
             if (msg.channel === "OUTPUTS") tempMsg = onlySendToMatchingId(tempMsg, id)
@@ -27,7 +27,7 @@ export class OutputSend {
 
     static sendToWindow(id: string, msg: any) {
         const output = OutputHelper.getOutput(id)
-        if (!output.window || output.window.isDestroyed()) return
+        if (!output?.window || output.window.isDestroyed()) return
         output.window.webContents.send(OUTPUT, msg)
         //if (!output.previewWindow || output.previewWindow.isDestroyed()) return
         //output.previewWindow.webContents.send(OUTPUT, msg)

src/electron/output/helpers/OutputValues.ts

@@ -32,7 +32,7 @@ export class OutputValues {
         const output = OutputHelper.getOutput(id)
         if (!this.setValues[key]) return
 
-        if (!output.window || output.window.isDestroyed()) return
+        if (!output?.window || output.window.isDestroyed()) return
         this.setValues[key](value, output.window, id)
     }
 }

src/electron/output/helpers/OutputVisibility.ts

@@ -93,6 +93,10 @@ export class OutputVisibility {
         window.hide()
 
         // WIP has to restart because window is unresponsive when hidden again (until showed again)...
+        // this is only needed if the output is being captured!! (has to reset for capture to work when window is hidden)
+        let captureEnabled = Object.values(OutputHelper.getOutput(data.id)?.captureOptions?.options || {}).find((a) => a === true)
+        if (!captureEnabled) return
+
         console.log("RESTARTING OUTPUT:", data.id)
         toApp(OUTPUT, { channel: "RESTART" })
     }

src/electron/preload.ts

@@ -11,7 +11,7 @@ import type { ValidChannels } from "../types/Channels"
 // wait to log messages until after intial load is done
 let appLoaded: boolean = false
 const LOG_MESSAGES: boolean = process.env.NODE_ENV !== "production"
-const filteredChannels: any[] = ["AUDIO_MAIN", "VIZUALISER_DATA", "STREAM", "PREVIEW", "REQUEST_STREAM", "MAIN_TIME", "GET_THUMBNAIL"]
+const filteredChannels: any[] = ["AUDIO_MAIN", "VIZUALISER_DATA", "STREAM", "BUFFER", "REQUEST_STREAM", "MAIN_TIME", "GET_THUMBNAIL"]
 
 let storedReceivers: any = {}
 

src/frontend/components/helpers/output.ts

@@ -4,7 +4,7 @@ import { OUTPUT } from "../../../types/Channels"
 import type { Output } from "../../../types/Output"
 import type { Resolution, Styles } from "../../../types/Settings"
 import type { Item, Layout, Media, OutSlide, Show, Slide, Template, TemplateSettings, Transition } from "../../../types/Show"
-import { currentOutputSettings, lockedOverlays, outputDisplay, outputs, overlays, playingVideos, showsCache, special, styles, templates, theme, themes, transitionData, videoExtensions } from "../../stores"
+import { currentOutputSettings, disabledServers, lockedOverlays, outputDisplay, outputs, overlays, playingVideos, serverData, showsCache, special, stageShows, styles, templates, theme, themes, transitionData, videoExtensions } from "../../stores"
 import { send } from "../../utils/request"
 import { sendBackgroundToStage } from "../../utils/stageTalk"
 import { getItemText, getSlideText } from "../edit/scripts/textStyle"
@@ -221,6 +221,31 @@ export function getResolution(initial: Resolution | undefined | null = null, _up
     return initial || style || slideRes || { width: 1920, height: 1080 }
 }
 
+export function checkWindowCapture() {
+    getActiveOutputs(get(outputs), false, true, true).forEach(shouldBeCaptured)
+}
+
+// NDI | OutputShow | Stage CurrentOutput
+export function shouldBeCaptured(outputId: string) {
+    let output = get(outputs)[outputId]
+    let captures: any = {
+        ndi: !!output.ndi,
+        server: !!(get(disabledServers).output_stream === false && (get(serverData)?.output_stream?.outputId || getActiveOutputs(get(outputs), false, true, true)[0]) === outputId),
+        stage: stageHasOutput(outputId),
+    }
+
+    send(OUTPUT, ["CAPTURE"], { id: outputId, captures })
+}
+function stageHasOutput(outputId: string) {
+    return !!Object.keys(get(stageShows)).find((stageId) => {
+        let stageLayout = get(stageShows)[stageId]
+        let outputItem = stageLayout.items?.["output#current_output"]
+
+        if (!outputItem?.enabled) return false
+        return (stageLayout.settings?.output || outputId) === outputId
+    })
+}
+
 // settings
 
 export const defaultOutput: Output = {

src/frontend/components/output/Output.svelte

@@ -23,6 +23,7 @@
     export let style = ""
     export let ratio: number = 0
     export let mirror: boolean = false
+    export let preview: boolean = false
 
     $: currentOutput = $outputs[outputId] || {}
 
@@ -97,7 +98,7 @@
 
     // slide styling
     $: resolution = getResolution(currentSlide?.settings?.resolution, { currentOutput, currentStyle })
-    $: transitions = getOutputTransitions(slideData, $transitionData, mirror)
+    $: transitions = getOutputTransitions(slideData, $transitionData, mirror && !preview)
     $: slideFilter = getSlideFilter(slideData)
 
     // custom template

src/frontend/components/output/preview/MultiOutputs.svelte

@@ -1,6 +1,5 @@
 <script lang="ts">
-    import { Resolution } from "../../../../types/Settings"
-    import { currentWindow, dictionary, outputs, previewBuffers, styles } from "../../../stores"
+    import { dictionary, outputs } from "../../../stores"
     import Icon from "../../helpers/Icon.svelte"
     import T from "../../helpers/T.svelte"
     import { clone } from "../../helpers/array"
@@ -23,6 +22,8 @@
 
         fullscreen = true
         fullscreenId = e.target.closest(".previewOutput")?.id
+
+        currentResolution()
     }
 
     let updatedList: any[] = []
@@ -38,8 +39,13 @@
         }, 500)
     }
 
-    let resolution: Resolution = getResolution()
-    $: if ($currentWindow === "output") resolution = getResolution(null, { $outputs, $styles }, true)
+    // let resolution: Resolution = getResolution()
+    // $: if ($currentWindow === "output") resolution = getResolution(null, { $outputs, $styles }, true)
+    let resolution: any = {}
+    function currentResolution() {
+        resolution = getResolution(null, null, true)
+        console.log(resolution)
+    }
 </script>
 
 <!-- aspect-ratio: {resolution?.width || 1920}/{resolution?.height || 1080}; -->
@@ -50,8 +56,8 @@
         </Button>
 
         <span class="resolution">
-            <p><b><T id="screen.width" />:</b> {$previewBuffers[fullscreenId]?.originalSize?.width || 0} <T id="screen.pixels" /></p>
-            <p><b><T id="screen.height" />:</b> {$previewBuffers[fullscreenId]?.originalSize?.height || 0} <T id="screen.pixels" /></p>
+            <p><b><T id="screen.width" />:</b> {resolution?.width || 0} <T id="screen.pixels" /></p>
+            <p><b><T id="screen.height" />:</b> {resolution?.height || 0} <T id="screen.pixels" /></p>
         </span>
     {/if}
 
@@ -78,6 +84,7 @@
 
     .fullscreen {
         position: fixed;
+        justify-content: center;
         background-color: var(--primary-darkest);
         top: 50%;
         left: 50%;

src/frontend/components/output/preview/PreviewOutput.svelte

@@ -13,14 +13,10 @@
     let resolution: Resolution = getResolution()
     let width = 160
     let height = 90
-
-    // $: if ($currentWindow === "output") resolution = getResolution(null, { $outputs, $styles }, true)
-    $: console.log("style", style + "; aspect-ratio: " + width / height)
-    $: console.log("getStyleResolution", getStyleResolution(resolution, width, height, "fit"))
 </script>
 
 <div class="center previewOutput" id={outputId} class:fullscreen class:disabled style={style + "; aspect-ratio: " + width / height} bind:offsetWidth={width} bind:offsetHeight={height}>
-    <Output {outputId} style={getStyleResolution(resolution, width, height, "fit")} mirror={true} />
+    <Output {outputId} style={getStyleResolution(resolution, width, height, "fit")} mirror preview />
 </div>
 
 <style>

src/frontend/components/settings/tabs/Connection.svelte

@@ -6,7 +6,7 @@
     import Icon from "../../helpers/Icon.svelte"
     import T from "../../helpers/T.svelte"
     import { clone, keysToID } from "../../helpers/array"
-    import { getActiveOutputs } from "../../helpers/output"
+    import { checkWindowCapture, getActiveOutputs } from "../../helpers/output"
     import Button from "../../inputs/Button.svelte"
     import Checkbox from "../../inputs/Checkbox.svelte"
     import CombinedInput from "../../inputs/CombinedInput.svelte"
@@ -65,6 +65,8 @@
             a[id] = value
             return a
         })
+
+        if (id === "output_stream") checkWindowCapture()
     }
 
     function toggleCompanion(e: any) {

src/frontend/components/stage/tools/Items.svelte

@@ -5,7 +5,7 @@
     import Icon from "../../helpers/Icon.svelte"
     import T from "../../helpers/T.svelte"
     import { keysToID } from "../../helpers/array"
-    import { getActiveOutputs, getResolution } from "../../helpers/output"
+    import { checkWindowCapture, getActiveOutputs, getResolution } from "../../helpers/output"
     import Button from "../../inputs/Button.svelte"
     import Center from "../../system/Center.svelte"
     import Panel from "../../system/Panel.svelte"
@@ -50,6 +50,8 @@
             return ss
         })
 
+        if (item === "output#current_output") checkWindowCapture()
+
         if (!timeout) {
             updateStageShow()
             timeout = setTimeout(() => {