Parallel Compilation with no engine (#15898)

* Allow parallel shader compilation outside of the engine context

* resolve if not async/parallel

* come codedoc

* mark the function internal

* simplify interval scheduling

packages/dev/core/src/Engines/thinEngine.functions.ts

@@ -50,6 +50,8 @@ export function getStateObject(context: WebGLContext): IThinEngineStateObject {
             // use feature detection. instanceof returns false. This only exists on WebGL2 context
             _webGLVersion: (context as WebGL2RenderingContext).TEXTURE_BINDING_3D ? 2 : 1,
             _context: context,
+            // when using the function without an engine we need to set it to enable parallel compilation
+            parallelShaderCompile: context.getExtension("KHR_parallel_shader_compile") || undefined,
             cachedPipelines: {},
         };
         _stateObject.set(context, state);
@@ -189,6 +191,23 @@ export function _createShaderProgram(
     return shaderProgram;
 }
 
+/**
+ * @internal
+ */
+export function _isRenderingStateCompiled(pipelineContext: IPipelineContext, gl: WebGLContext, validateShaderPrograms?: boolean): boolean {
+    const webGLPipelineContext = pipelineContext as WebGLPipelineContext;
+    if (webGLPipelineContext._isDisposed) {
+        return false;
+    }
+    const stateObject = getStateObject(gl);
+    if (gl.getProgramParameter(webGLPipelineContext.program!, stateObject.parallelShaderCompile!.COMPLETION_STATUS_KHR)) {
+        _finalizePipelineContext(webGLPipelineContext, gl, validateShaderPrograms);
+        return true;
+    }
+
+    return false;
+}
+
 /**
  * @internal
  */

packages/dev/core/src/Engines/thinEngine.ts

@@ -32,6 +32,7 @@ import {
     getStateObject,
     _createShaderProgram,
     deleteStateObject,
+    _isRenderingStateCompiled,
 } from "./thinEngine.functions";
 
 import type { AbstractEngineOptions, ISceneLike, PrepareTextureFunction, PrepareTextureProcessFunction } from "./abstractEngine";
@@ -2152,16 +2153,10 @@ export class ThinEngine extends AbstractEngine {
      * @internal
      */
     public _isRenderingStateCompiled(pipelineContext: IPipelineContext): boolean {
-        const webGLPipelineContext = pipelineContext as WebGLPipelineContext;
-        if (this._isDisposed || webGLPipelineContext._isDisposed) {
+        if (this._isDisposed) {
             return false;
         }
-        if (this._gl.getProgramParameter(webGLPipelineContext.program!, this._caps.parallelShaderCompile!.COMPLETION_STATUS_KHR)) {
-            this._finalizePipelineContext(webGLPipelineContext);
-            return true;
-        }
-
-        return false;
+        return _isRenderingStateCompiled(pipelineContext, this._gl, this.validateShaderPrograms);
     }
 
     /**

packages/dev/core/src/Materials/effect.functions.ts

@@ -57,6 +57,11 @@ export interface IPipelineGenerationOptions {
      * extend the pipeline generation options
      */
     extendedCreatePipelineOptions?: Partial<ICreateAndPreparePipelineContextOptions>;
+
+    /**
+     * If true, generating a new pipeline will return when the pipeline is ready to be used
+     */
+    waitForIsReady?: boolean;
 }
 
 /**
@@ -316,3 +321,22 @@ export const createAndPreparePipelineContext = (
         throw e;
     }
 };
+
+export const _retryWithInterval = (condition: () => boolean, onSuccess: () => void, onError?: (e?: any) => void, step = 16, maxTimeout = 1000) => {
+    const int = setInterval(() => {
+        try {
+            if (condition()) {
+                clearInterval(int);
+                onSuccess();
+            }
+        } catch (e) {
+            clearInterval(int);
+            onError?.(e);
+        }
+        maxTimeout -= step;
+        if (maxTimeout < 0) {
+            clearInterval(int);
+            onError?.();
+        }
+    }, step);
+};

packages/dev/core/src/Materials/effect.ts

@@ -15,7 +15,7 @@ import { ShaderLanguage } from "./shaderLanguage";
 import type { InternalTexture } from "../Materials/Textures/internalTexture";
 import type { ThinTexture } from "../Materials/Textures/thinTexture";
 import type { IPipelineGenerationOptions } from "./effect.functions";
-import { _processShaderCode, getCachedPipeline, createAndPreparePipelineContext, resetCachedPipeline } from "./effect.functions";
+import { _processShaderCode, getCachedPipeline, createAndPreparePipelineContext, resetCachedPipeline, _retryWithInterval } from "./effect.functions";
 
 /**
  * Defines the route to the shader code. The priority is as follows:
@@ -587,29 +587,22 @@ export class Effect implements IDisposable {
         });
 
         if (!this._pipelineContext || this._pipelineContext.isAsync) {
-            setTimeout(() => {
-                this._checkIsReady(null);
-            }, 16);
+            this._checkIsReady(null);
         }
     }
 
     private _checkIsReady(previousPipelineContext: Nullable<IPipelineContext>) {
-        try {
-            if (this._isReadyInternal()) {
-                return;
+        _retryWithInterval(
+            () => {
+                return this._isReadyInternal() || this._isDisposed;
+            },
+            () => {
+                // no-op - done in the _isReadyInternal call
+            },
+            (e) => {
+                this._processCompilationErrors(e, previousPipelineContext);
             }
-        } catch (e) {
-            this._processCompilationErrors(e, previousPipelineContext);
-            return;
-        }
-
-        if (this._isDisposed) {
-            return;
-        }
-
-        setTimeout(() => {
-            this._checkIsReady(previousPipelineContext);
-        }, 16);
+        );
     }
 
     /**

packages/dev/core/src/Materials/effect.webgl.functions.ts

@@ -1,12 +1,13 @@
 import type { AbstractEngine } from "core/Engines/abstractEngine";
 import type { IPipelineGenerationOptions } from "./effect.functions";
-import { _processShaderCode, createAndPreparePipelineContext } from "./effect.functions";
+import { _processShaderCode, _retryWithInterval, createAndPreparePipelineContext } from "./effect.functions";
 import type { IPipelineContext } from "core/Engines/IPipelineContext";
-import { _executeWhenRenderingStateIsCompiled, _preparePipelineContext, createPipelineContext, getStateObject } from "core/Engines/thinEngine.functions";
+import { _executeWhenRenderingStateIsCompiled, _isRenderingStateCompiled, _preparePipelineContext, createPipelineContext, getStateObject } from "core/Engines/thinEngine.functions";
 import { ShaderLanguage } from "./shaderLanguage";
 import { _getGlobalDefines } from "core/Engines/abstractEngine.functions";
 import type { ProcessingOptions } from "core/Engines/Processors/shaderProcessingOptions";
 import { ShaderStore } from "core/Engines/shaderStore";
+import { WebGL2ShaderProcessor } from "core/Engines/WebGL/webGL2ShaderProcessors";
 
 /**
  * Generate a pipeline context from the provided options
@@ -37,7 +38,8 @@ export async function generatePipelineContext(
                 break;
             case "WEBGL2":
             default:
-                processor = new (await import("core/Engines/WebGL/webGL2ShaderProcessors")).WebGL2ShaderProcessor();
+                // default to WebGL2, which is included in the package. Avoid async-load the default.
+                processor = new WebGL2ShaderProcessor();
                 break;
         }
     }
@@ -87,7 +89,16 @@ export async function generatePipelineContext(
                             _preparePipelineContext,
                             _executeWhenRenderingStateIsCompiled
                         );
-                        resolve(pipeline);
+                        // the default behavior so far. If not async or no request to wait for isReady, resolve immediately
+                        if (!options.waitForIsReady || !pipeline.isAsync) {
+                            resolve(pipeline);
+                        } else {
+                            _retryWithInterval(
+                                () => _isRenderingStateCompiled(pipeline, context),
+                                () => resolve(pipeline),
+                                () => reject(new Error("Timeout while waiting for pipeline to be ready"))
+                            );
+                        }
                     } catch (e) {
                         reject(e);
                     }