Flickerless Depth Texture Read

packages/frontend-2/components/viewer/gendo/Panel.vue

@@ -55,6 +55,7 @@
 import { useApolloClient } from '@vue/apollo-composable'
 import { useTimeoutFn } from '@vueuse/core'
 import { getFirstErrorMessage } from '~/lib/common/helpers/graphql'
+import { PassReader } from '~/lib/viewer/extensions/PassReader'
 import { requestGendoAIRender } from '~~/lib/gendo/graphql/queriesAndMutations'
 import { useInjectedViewerState } from '~~/lib/viewer/composables/setup'
 
@@ -75,24 +76,10 @@ const prompt = ref<string>()
 const isLoading = ref(false)
 const timeOutWait = ref(false)
 
-const enqueMagic = () => {
+const enqueMagic = async () => {
   isLoading.value = true
-  viewerInstance.getRenderer().pipelineOptions = {
-    ...viewerInstance.getRenderer().pipelineOptions,
-    pipelineOutput: 1
-  }
-
-  viewerInstance.requestRender()
-  setTimeout(async () => {
-    const screenshot = await viewerInstance.screenshot()
-    void lodgeRequest(screenshot)
-    // Reset renderer back to normal
-    viewerInstance.getRenderer().pipelineOptions = {
-      ...viewerInstance.getRenderer().pipelineOptions,
-      pipelineOutput: 8
-    }
-    viewerInstance.requestRender()
-  }, 100)
+  const screenshot = await viewerInstance.getExtension(PassReader).read()
+  void lodgeRequest(screenshot)
 
   timeOutWait.value = true
 

packages/frontend-2/lib/viewer/composables/setup.ts

@@ -65,6 +65,7 @@ import {
 } from '~/lib/viewer/composables/setup/core'
 import { useSynchronizedCookie } from '~~/lib/common/composables/reactiveCookie'
 import { buildManualPromise } from '@speckle/ui-components'
+import { PassReader } from '../extensions/PassReader'
 
 export type LoadedModel = NonNullable<
   Get<ViewerLoadedResourcesQuery, 'project.models.items[0]'>
@@ -324,6 +325,7 @@ function createViewerDataBuilder(params: { viewerDebug: boolean }) {
       ...DefaultViewerParams,
       verbose: !!(process.client && params.viewerDebug)
     })
+    viewer.createExtension(PassReader)
     const initPromise = viewer.init()
 
     return {

packages/frontend-2/lib/viewer/extensions/PassReader.ts

@@ -0,0 +1,96 @@
+import { type SpecklePass } from '@speckle/viewer'
+import { Extension } from '@speckle/viewer'
+import type SpeckleRenderer from '@speckle/viewer/dist/modules/SpeckleRenderer'
+import { Vector3, Vector4, WebGLRenderTarget } from 'three'
+
+export class PassReader extends Extension {
+  private outputBuffer: Uint8Array = new Uint8Array()
+  private renderTarget: WebGLRenderTarget | undefined = undefined
+  private needsRead: boolean = false
+  private readbackExecutor: ((arg: string) => void) | null = null
+
+  public async read(): Promise<string> {
+    return new Promise<string>((resolve, reject) => {
+      const renderer: SpeckleRenderer = this.viewer.getRenderer()
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+      const dephPass: SpecklePass = renderer.pipeline.composer
+        .passes[0] as unknown as SpecklePass
+      // o_0
+      // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
+      this.renderTarget = dephPass.outputRenderTarget
+      if (!this.renderTarget) {
+        reject('Issue with depth pass render target')
+        return
+      }
+
+      const bufferSize = this.renderTarget.width * this.renderTarget.height * 4
+      if (this.outputBuffer.length !== bufferSize)
+        this.outputBuffer = new Uint8Array(bufferSize)
+      this.needsRead = true
+      this.readbackExecutor = resolve
+    })
+  }
+
+  public onRender(): void {
+    if (!this.needsRead || this.renderTarget === undefined) return
+
+    this.viewer
+      .getRenderer()
+      .renderer.readRenderTargetPixels(
+        this.renderTarget,
+        0,
+        0,
+        this.renderTarget.width,
+        this.renderTarget.height,
+        this.outputBuffer
+      )
+    const UnpackDownscale = 255 / 256
+    const PackFactors = new Vector3(256 * 256 * 256, 256 * 256, 256)
+    const UnpackFactors = new Vector4(
+      UnpackDownscale / PackFactors.x,
+      UnpackDownscale / PackFactors.y,
+      UnpackDownscale / PackFactors.z,
+      1
+    )
+
+    const v4 = new Vector4()
+    for (let k = 0; k < this.outputBuffer.length; k += 4) {
+      v4.set(
+        this.outputBuffer[k] / 255,
+        this.outputBuffer[k + 1] / 255,
+        this.outputBuffer[k + 2] / 255,
+        this.outputBuffer[k + 3] / 255
+      )
+      const res = v4.dot(UnpackFactors)
+      this.outputBuffer[k] = res * 255
+      this.outputBuffer[k + 1] = res * 255
+      this.outputBuffer[k + 2] = res * 255
+      this.outputBuffer[k + 3] = 255
+    }
+
+    const canvas = document.createElement('canvas')
+    const ctx = canvas.getContext('2d')
+    if (!ctx) return
+    canvas.width = this.renderTarget.width
+    canvas.height = this.renderTarget.height
+
+    // create imageData object
+    const idata = ctx.createImageData(this.renderTarget.width, this.renderTarget.height)
+
+    // set our buffer as source
+    idata.data.set(this.outputBuffer)
+
+    // update canvas with new data
+    ctx.putImageData(idata, 0, 0)
+    ctx.save()
+    /** Flipping the image by drawing it on itself
+     */
+    ctx.globalCompositeOperation = 'copy'
+    ctx.scale(1, -1)
+    ctx.drawImage(canvas, 0, 0, this.renderTarget.width, -this.renderTarget.height)
+    ctx.restore()
+
+    if (this.readbackExecutor) this.readbackExecutor(canvas.toDataURL())
+    this.needsRead = false
+  }
+}

packages/viewer/src/index.ts

@@ -72,6 +72,7 @@ import { NodeRenderView } from './modules/tree/NodeRenderView'
 import { type ExtendedIntersection } from './modules/objects/SpeckleRaycaster'
 import { SpeckleGeometryConverter } from './modules/loaders/Speckle/SpeckleGeometryConverter'
 import { Assets } from './modules/Assets'
+import { SpecklePass } from './modules/pipeline/SpecklePass'
 import { InstancedBatchObject } from './modules/batching/InstancedBatchObject'
 
 export {
@@ -148,7 +149,8 @@ export type {
   ViewerEventPayload,
   InputEventPayload,
   SectionToolEventPayload,
-  CameraEventPayload
+  CameraEventPayload,
+  SpecklePass
 }
 
 export * as UrlHelper from './modules/UrlHelper'

packages/viewer/src/modules/pipeline/DepthPass.ts

@@ -57,6 +57,10 @@ export class DepthPass extends BaseSpecklePass implements SpecklePass {
     return this.renderTargetHalf.texture
   }
 
+  get outputRenderTarget(): WebGLRenderTarget {
+    return this.renderTarget
+  }
+
   public set depthType(value: DepthType) {
     if (value === DepthType.LINEAR_DEPTH)
       if (this.depthMaterial.defines) {

packages/viewer/src/modules/pipeline/SpecklePass.ts

@@ -4,7 +4,8 @@ import {
   PerspectiveCamera,
   Plane,
   Scene,
-  Texture
+  Texture,
+  WebGLRenderTarget
 } from 'three'
 import { Pass } from 'three/examples/jsm/postprocessing/Pass.js'
 import { ObjectLayers } from '../../IViewer'
@@ -22,6 +23,7 @@ export type InputColorInterpolateTextureUniform = 'tDiffuseInterp'
 export interface SpecklePass {
   onBeforeRender?: () => void
   onAferRender?: () => void
+  outputRenderTarget?: WebGLRenderTarget
 
   get displayName(): string
   get outputTexture(): Texture | null