From 1d937dccdfd87a4a6199d6b85b33e0d93984e40a Mon Sep 17 00:00:00 2001 From: Joao Victor Dell Agli Date: Thu, 13 Jul 2023 16:28:23 -0300 Subject: [PATCH] KDE demonstration refactoring --- docs/experimental/buffer_compose.py | 180 ------------ docs/experimental/framebuffer_example.py | 203 ------------- docs/experimental/viz_points_kde.py | 357 +++++++++++++++++++++++ 3 files changed, 357 insertions(+), 383 deletions(-) delete mode 100644 docs/experimental/buffer_compose.py delete mode 100644 docs/experimental/framebuffer_example.py create mode 100644 docs/experimental/viz_points_kde.py diff --git a/docs/experimental/buffer_compose.py b/docs/experimental/buffer_compose.py deleted file mode 100644 index d988ef26f..000000000 --- a/docs/experimental/buffer_compose.py +++ /dev/null @@ -1,180 +0,0 @@ -import numpy as np -from fury import window, actor, shaders -import vtk -import os -from fury.shaders import compose_shader, import_fury_shader -# Function to capture the framebuffer and bind it as a texture to the billboard -def capture_and_bind_texture(source_actor : actor.Actor, scene : window.Scene, target_actor : actor.Actor = None): - # Capture the framebuffer - windowToImageFilter = vtk.vtkWindowToImageFilter() - windowToImageFilter.SetInput(scene.GetRenderWindow()) - - windowToImageFilter.Update() - - # Convert the framebuffer to a texture - - texture = vtk.vtkTexture() - texture.SetInputConnection(windowToImageFilter.GetOutputPort()) - texture.SetEdgeClamp(texture.ClampToBorder) - texture.SetBorderColor(0.0, 0.0, 1.0, 1.0) - texture.SetWrap(3) - texture.MipmapOn() - texture.SetBlendingMode(0) # Additive Blend - - # Bind the framebuffer texture to the billboard - if target_actor == None: - source_actor.SetTexture(texture) - else: - target_actor.SetTexture(texture) - - - -def shader_custom_uniforms(actor : actor.Actor, shader_type : str): - """Eases the passing of uniform values to the shaders by returning ``actor.GetShaderProperty().GetVertexCustomUniforms()``, - that give access to the ``SetUniform`` methods. - Parameters - ---------- - actor : actor.Actor - Actor which the uniform values will be passed to. - shader_type : str - Shader type of the uniform values to be passed. It can be: - * "vertex" - * "fragment" - * "geometry" - """ - if shader_type == "vertex": - return actor.GetShaderProperty().GetVertexCustomUniforms() - elif shader_type == "fragment": - return actor.GetShaderProperty().GetFragmentCustomUniforms() - elif shader_type == "geometry": - return actor.GetShaderProperty().GetGeometryCustomUniforms() - else: raise ValueError("Shader type unknown.") - - -def normalize(img : np.ndarray, min, max): - """Function that converts an image to the given desired range.""" - return ((img - np.min(img))/(np.max(img) - np.min(img)))*(max - min) + min - - -frag_dec = """ -uniform sampler2D screenTexture; -""" - -kde_dec = """ -float kde(vec3 point, vec3 coord, float sigma){ - return exp(-1.0*pow(length(point - coord), 2.0)/(2.0*sigma*sigma) ); -} -""" - -kde_impl = """ -vec3 renorm_tex = normalizedVertexMCVSOutput*0.5 + 0.5; -vec3 last_kde = texture(screenTexture, renorm_tex.xy).rgb; - -if(i == 0) last_kde = vec3(0.0); - -float current_kde = kde(p, normalizedVertexMCVSOutput, sigma); -//color = vec3(current_kde) + last_kde/n_points; -color = 2.0*mix(vec3(current_kde), last_kde, 0.5); -//color = vec3(renorm_tex.xy, 0.0); -""" - - -tex_dec =""" -uniform sampler2D screenTexture; -""" - -tex_impl =""" -vec3 renorm_tex = normalizedVertexMCVSOutput*0.5 + 0.5; -color = texture(screenTexture, renorm_tex.xy).rgb; -""" - -fragoutput = """ -fragOutput0 = vec4(color, 1.0); -""" - - -fs_dec = shaders.compose_shader([frag_dec, kde_dec]) - -fs_impl = shaders.compose_shader([kde_impl, fragoutput]) - -tex_impl = shaders.compose_shader([tex_impl, fragoutput]) - -width, height = (600, 600) - -scene = window.Scene() - -scene.set_camera(position=(-6, 5, -10), - focal_point=(0.0, - 0.0, - 0.0), - view_up=(0.0, 0.0, 0.0)) - -scene.DebugOn() -scene.GlobalWarningDisplayOn() - -manager = window.ShowManager( - scene, - "demo", - (width, - height), - reset_camera=True, - order_transparent=True) - -# manager.window.SetOffScreenRendering(True) - -manager.scene.GetRenderWindow().DebugOn() -manager.scene.GetRenderWindow().GlobalWarningDisplayOn() - - -scale = np.array([[3.4, 3.4, 0.0]]) - - -billboard = actor.billboard(np.array([[0.0, 0.0, 0.0]]), (0.0, 0.0, 1.0), scales=scale, fs_dec=fs_dec, fs_impl=fs_impl) -textured_billboard = actor.billboard(np.array([[0.0, 0.0, 0.0]]), (0.0, 0.0, 1.0), scales=scale, fs_dec=tex_dec, fs_impl=tex_impl) -# shaders.shader_apply_effects(manager.window, billboard, window.gl_disable_depth) -# shaders.shader_apply_effects(manager.window, billboard, window.gl_set_additive_blending) - - -manager.scene.add(billboard) -manager.initialize() - - -n_points = 20 -sigma = 0.22 -shader_custom_uniforms(billboard, "fragment").SetUniformf("sigma", sigma) -shader_custom_uniforms(billboard, "fragment").SetUniformf("n_points", float(n_points)) -shader_custom_uniforms(billboard, "fragment").SetUniform2f("res", [width, height]) - - - -points = np.random.rand(n_points, 3) -points = normalize(points, -1, 1) -# print(points) -# points = np.array([[0.0, 0.0, 0.0], [-0.5, 0.0, 0.0], [0.7, 0.7, 0.0]]) - - -# Render the scene and capture the framebuffer after each mixing step -manager.window.SetOffScreenRendering(True) -for i in range(0, n_points - 1): - shader_custom_uniforms(billboard, "fragment").SetUniformi("i", i) - shader_custom_uniforms(billboard, "fragment").SetUniform3f("p", points[i, :].tolist()) - - manager.render() - - capture_and_bind_texture(billboard, manager.scene) - -shader_custom_uniforms(billboard, "fragment").SetUniformi("i", i) -shader_custom_uniforms(billboard, "fragment").SetUniform3f("p", points[n_points - 1, :].tolist()) - -manager.render() - -manager.window.SetOffScreenRendering(False) - -capture_and_bind_texture(billboard, manager.scene, textured_billboard) - - - -# manager.scene.RemoveActor(billboard) -# manager.scene.add(textured_billboard) - -manager.start() \ No newline at end of file diff --git a/docs/experimental/framebuffer_example.py b/docs/experimental/framebuffer_example.py deleted file mode 100644 index ad5d09b88..000000000 --- a/docs/experimental/framebuffer_example.py +++ /dev/null @@ -1,203 +0,0 @@ -import numpy as np -from fury import window, actor, lib, shaders -import vtk - - -def shader_custom_uniforms(actor : actor.Actor, shader_type : str): - """Eases the passing of uniform values to the shaders by returning ``actor.GetShaderProperty().GetVertexCustomUniforms()``, - that give access to the ``SetUniform`` methods. - Parameters - ---------- - actor : actor.Actor - Actor which the uniform values will be passed to. - shader_type : str - Shader type of the uniform values to be passed. It can be: - * "vertex" - * "fragment" - * "geometry" - """ - if shader_type == "vertex": - return actor.GetShaderProperty().GetVertexCustomUniforms() - elif shader_type == "fragment": - return actor.GetShaderProperty().GetFragmentCustomUniforms() - elif shader_type == "geometry": - return actor.GetShaderProperty().GetGeometryCustomUniforms() - else: raise ValueError("Shader type unknown.") - - - -# Don't mind these shaders, I just set them up to focus first on the framebuffer, then after on the shaders themselves -billboard_vert_decl = """/* Billboard vertex shader declaration */ -in vec3 center; -in vec2 in_tex; - -out vec3 centerVertexMCVSOutput; -out vec3 normalizedVertexMCVSOutput; -varying vec2 out_tex;""" - -billboard_vert_impl = """/* Billboard vertex shader implementation */ -centerVertexMCVSOutput = center; -normalizedVertexMCVSOutput = vertexMC.xyz - center; // 1st Norm. [-scale, scale] -float scalingFactor = 1. / abs(normalizedVertexMCVSOutput.x); -float size = abs(normalizedVertexMCVSOutput.x) * 2; -normalizedVertexMCVSOutput *= scalingFactor; // 2nd Norm. [-1, 1] -vec2 billboardSize = vec2(size, size); // Fixes the scaling issue -vec3 cameraRightMC = vec3(MCVCMatrix[0][0], MCVCMatrix[1][0], MCVCMatrix[2][0]); -vec3 cameraUpMC = vec3(MCVCMatrix[0][1], MCVCMatrix[1][1], MCVCMatrix[2][1]); -vec3 vertexPositionMC = center + - cameraRightMC * billboardSize.x * normalizedVertexMCVSOutput.x + - cameraUpMC * billboardSize.y * normalizedVertexMCVSOutput.y; -out_tex = in_tex; -gl_Position = MCDCMatrix * vec4(vertexPositionMC, 1.);""" - - -frag_decl = """in vec3 normalizedVertexMCVSOutput; -""" - -kde_func = """float kde(vec3 point, vec3 coord, float sigma){ - return exp(-pow(length(abs(point - coord)), 2.0)/(2.0*sigma*sigma)); -}; """ - -frag_impl = """ -vec3 k = vec3(kde(point0, vec3(gl_FragCoord.xy/res0, 0.0), 10.0)); -fragOutput0 = vec4(k, 0.5);""" - - -frag = """ - -varying vec2 out_tex; -void main(){ - gl_FragColor = vec4(1.0, 0.0, 1.0, 1.0); -}""" - - -frag_2 = """ - -varying vec2 out_tex; - -uniform sampler2D screenTexture; - -void main(){ - vec4 texture = texture(screenTexture, out_tex); - gl_FragColor = texture; - //gl_FragColor = vec4(0.0, 0.0, 1.0, 1.0); -}""" - - -width, height = (1920, 1080) - -scene = window.Scene() - -scene.set_camera(position=(-6, 5, -10), - focal_point=(0.0, - 0.0, - 0.0), - view_up=(0.0, 0.0, 0.0)) - - - -scene.DebugOn() -scene.GlobalWarningDisplayOn() - -manager = window.ShowManager( - scene, - "trees demo", - (1920, - 1080), - reset_camera=True, - order_transparent=True) - - -scale = np.array([[width, height, 0.0]]) - -# Actor setup -billboard = actor.billboard(np.array([[0.0, 0.0, 0.0]]), (1.0, 0.0, 0.0), scales=scale) - -# Adding first the kde rendering shaders -actor.shader_to_actor(billboard, "vertex", billboard_vert_impl, billboard_vert_decl) -actor.replace_shader_in_actor(billboard, "fragment", frag) -billboard_tex = np.array([[0.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 0.0]]) -actor.attribute_to_actor(billboard, billboard_tex, "in_tex") - - - -# Actor adding -manager.scene.add(billboard) - - - -manager.scene.GetRenderWindow().DebugOn() -manager.scene.GetRenderWindow().GlobalWarningDisplayOn() - - - -# FBO setup - -# Useful links -# https://vtk.org/doc/nightly/html/classvtkOpenGLFramebufferObject.html#details -# https://gitlab.kitware.com/vtk/vtk/-/blob/v9.2.0/Rendering/OpenGL2/Testing/Cxx/TestWindowBlits.cxx -# https://vtk.org/doc/nightly/html/classvtkTextureObject.html -# https://vtk.org/doc/nightly/html/classvtkOpenGLRenderWindow.html -# https://learnopengl.com/Advanced-OpenGL/Framebuffers -# https://github.com/JoaoDell/Comp-Grafica/blob/main/shadow3d.c -# https://github.com/Kitware/VTK/blob/8f88edb91d2efea2d9cef1a1399d7a856c47f3be/Rendering/OpenGL2/vtkOpenGLFramebufferObject.cxx -# https://fury.gl/latest/auto_examples/04_demos/viz_fine_tuning_gl_context.html#sphx-glr-auto-examples-04-demos-viz-fine-tuning-gl-context-py - - - -FBO = vtk.vtkOpenGLFramebufferObject() - -manager.window.SetOffScreenRendering(True) -manager.initialize() # sets everything for rendering - -FBO.DebugOn() -FBO.GlobalWarningDisplayOn() - -FBO.SetContext(manager.window) # Sets the context for the FBO. -FBO.SaveCurrentBindingsAndBuffers() -FBO.PopulateFramebuffer(width, height, True, 1, vtk.VTK_UNSIGNED_CHAR, False, 24, 0) -FBO.RestorePreviousBindingsAndBuffers() - - - - - -# Checking FBO status -print("FBO of index:", FBO.GetFBOIndex()) -print("Number of color attachments:", FBO.GetNumberOfColorAttachments()) -FBO.RestorePreviousBindingsAndBuffers() - -# RENDER TIME -shaders.shader_apply_effects(manager.window, billboard, window.gl_disable_depth) -shaders.shader_apply_effects(manager.window, billboard, window.gl_set_normal_blending) - -FBO.SaveCurrentBindingsAndBuffers() -FBO.Bind() -FBO.ActivateBuffer(0) - -manager.render() - -color_texture = FBO.GetColorAttachmentAsTextureObject(0) - -print("Color texture") -print(color_texture) - - -print("Color texture handle:", color_texture.GetHandle()) # corresponds to the value generated when we call glGenTextures() -print("Color texture unit:", color_texture.GetTextureUnit()) # corresponds to the texture unit the texture is currently bound (GL_TEXTURE0, etc) -print("Color texture target:", color_texture.GetTarget()) - -FBO.RestorePreviousBindingsAndBuffers() - - -# WIP below -manager.window.SetOffScreenRendering(False) # Then, after rendering everything in the offscreen FBO, time to render it to a simple billboard -actor.replace_shader_in_actor(billboard, "fragment", frag_2) -shaders.shader_apply_effects(manager.window, billboard, window.gl_disable_blend) - -color_texture.Activate() - -interactive = True - -if interactive: - manager.start() diff --git a/docs/experimental/viz_points_kde.py b/docs/experimental/viz_points_kde.py new file mode 100644 index 000000000..69e54d0ef --- /dev/null +++ b/docs/experimental/viz_points_kde.py @@ -0,0 +1,357 @@ +import numpy as np +from fury import window, actor +from vtk import vtkWindowToImageFilter +from fury.shaders import compose_shader, shader_apply_effects +from fury.lib import Texture +from fury.io import load_image +from fury.utils import rgb_to_vtk +from matplotlib import colormaps + + +def window_to_texture( + window : window.RenderWindow, + texture_name : str, + target_actor : actor.Actor, + blending_mode : str = "None", + wrap_mode : str = "ClampToBorder", + border_color : tuple = ( + 0.0, + 0.0, + 0.0, + 1.0), + interpolate : bool = True): + """Captures a rendered window and pass it as a texture to the given actor. + + Parameters + ---------- + window : window.RenderWindow + Window to be captured. + texture_name : str + Name of the texture to be passed to the actor. + target_actor : actor.Actor + Target actor to receive the texture. + blending_mode : str, optional + Texture blending mode. The options are: + 1. None + 2. Replace + 3. Modulate + 4. Add + 5. AddSigned + 6. Interpolate + 7. Subtract + + wrap_mode : str, optional + Texture wrapping mode. The options are: + 1. ClampToEdge + 2. Repeat + 3. MirroredRepeat + 4. ClampToBorder + + border_color : tuple (4, ), optional + Texture RGBA border color. + interpolate : bool, optional + Texture interpolation.""" + + wrap_mode_dic = {"ClampToEdge" : Texture.ClampToEdge, + "Repeat" : Texture.Repeat, + "MirroredRepeat" : Texture.MirroredRepeat, + "ClampToBorder" : Texture.ClampToBorder} + + blending_mode_dic = {"None" : 0, "Replace" : 1, + "Modulate" : 2, "Add" : 3, + "AddSigned" : 4, "Interpolate" : 5, + "Subtract" : 6} + + r, g, b, a = border_color + + windowToImageFilter = vtkWindowToImageFilter() + windowToImageFilter.SetInput(window) + + windowToImageFilter.Update() + + texture = Texture() + texture.SetInputConnection(windowToImageFilter.GetOutputPort()) + texture.SetBorderColor(r, g, b, a) + texture.SetWrap(wrap_mode_dic[wrap_mode]) + texture.SetInterpolate(interpolate) + texture.MipmapOn() + texture.SetBlendingMode(blending_mode_dic[blending_mode]) + + target_actor.GetProperty().SetTexture(texture_name, texture) + + +def texture_to_actor( + path_to_texture : str, + texture_name : str, + target_actor : actor.Actor, + blending_mode : str = "None", + wrap_mode : str = "ClampToBorder", + border_color : tuple = ( + 0.0, + 0.0, + 0.0, + 1.0), + interpolate : bool = True): + """Passes an imported texture to an actor. + + Parameters + ---------- + path_to_texture : str + Texture image path. + texture_name : str + Name of the texture to be passed to the actor. + target_actor : actor.Actor + Target actor to receive the texture. + blending_mode : str + Texture blending mode. The options are: + 1. None + 2. Replace + 3. Modulate + 4. Add + 5. AddSigned + 6. Interpolate + 7. Subtract + + wrap_mode : str + Texture wrapping mode. The options are: + 1. ClampToEdge + 2. Repeat + 3. MirroredRepeat + 4. ClampToBorder + + border_color : tuple (4, ) + Texture RGBA border color. + interpolate : bool + Texture interpolation.""" + + wrap_mode_dic = {"ClampToEdge" : Texture.ClampToEdge, + "Repeat" : Texture.Repeat, + "MirroredRepeat" : Texture.MirroredRepeat, + "ClampToBorder" : Texture.ClampToBorder} + + blending_mode_dic = {"None" : 0, "Replace" : 1, + "Modulate" : 2, "Add" : 3, + "AddSigned" : 4, "Interpolate" : 5, + "Subtract" : 6} + + r, g, b, a = border_color + + texture = Texture() + + colormapArray = load_image(path_to_texture) + colormapData = rgb_to_vtk(colormapArray) + + texture.SetInputDataObject(colormapData) + texture.SetBorderColor(r, g, b, a) + texture.SetWrap(wrap_mode_dic[wrap_mode]) + texture.SetInterpolate(interpolate) + texture.MipmapOn() + texture.SetBlendingMode(blending_mode_dic[blending_mode]) + + target_actor.GetProperty().SetTexture(texture_name, texture) + + +def colormap_to_texture( + colormap : np.array, + texture_name : str, + target_actor : actor.Actor, + interpolate : bool = True): + """Converts a colormap to a texture and pass it to an actor. + + Parameters + ---------- + colormap : np.array (N, 4) or (1, N, 4) + RGBA color map array. The array can be two dimensional, although a three dimensional one is preferred. + texture_name : str + Name of the color map texture to be passed to the actor. + target_actor : actor.Actor + Target actor to receive the color map texture. + interpolate : bool + Color map texture interpolation.""" + + if len(colormap.shape) == 2: + colormap = np.array([colormap]) + + texture = Texture() + + cmap = (255*colormap).astype(np.uint8) + cmap = rgb_to_vtk(cmap) + + texture.SetInputDataObject(cmap) + texture.SetWrap(Texture.ClampToEdge) + texture.SetInterpolate(interpolate) + texture.MipmapOn() + texture.SetBlendingMode(0) + + target_actor.GetProperty().SetTexture(texture_name, texture) + + +def shader_custom_uniforms(actor : actor.Actor, shader_type : str): + """Eases the passing of uniform values to the shaders by returning ``actor.GetShaderProperty().GetVertexCustomUniforms()``, + that give access to the ``SetUniform`` methods. + Parameters + ---------- + actor : actor.Actor + Actor which the uniform values will be passed to. + shader_type : str + Shader type of the uniform values to be passed. It can be: + * "vertex" + * "fragment" + * "geometry" + """ + if shader_type == "vertex": + return actor.GetShaderProperty().GetVertexCustomUniforms() + elif shader_type == "fragment": + return actor.GetShaderProperty().GetFragmentCustomUniforms() + elif shader_type == "geometry": + return actor.GetShaderProperty().GetGeometryCustomUniforms() + else: + raise ValueError("Shader type unknown.") + + +def normalize(array : np.array, min : float = 0.0, max : float = 1.0, axis : int = 0): + """Converts an array to a given desired range. + + Parameters + ---------- + array : np.ndarray + Array to be normalized. + min : float + Bottom value of the interval of normalization. If no value is given, it is passed as 0.0. + max : float + Upper value of the interval of normalization. If no value is given, it is passed as 1.0. + + Returns + ------- + + array : np.array + Array converted to the given desired range. + """ + if np.max(array) != np.min(array): + return ((array - np.min(array))/(np.max(array) - np.min(array)))*(max - min) + min + else: + raise ValueError( + "Can't normalize an array which maximum and minimum value are the same.") + + +kde_dec = """ +float kde(vec3 point, float sigma){ + return exp(-1.0*pow(length(point), 2.0)/(2.0*sigma*sigma) ); +} +""" + +kde_impl = """ +float current_kde = kde(normalizedVertexMCVSOutput, sigma); +color = vec3(current_kde); +fragOutput0 = vec4(color, 1.0); +""" + +tex_dec = """ +const float normalizingFactor = 1.0; // Parameter to be better defined later + +vec3 color_mapping(float intensity, float normalizingFactor){ + float normalizedIntensity = intensity/normalizingFactor; + return texture(colormapTexture, vec2(normalizedIntensity,0)).rgb; +} +""" + +tex_impl = """ +vec2 renorm_tex = normalizedVertexMCVSOutput.xy*0.5 + 0.5; +float intensity = texture(screenTexture, renorm_tex).r; + +if(intensity<=0.0){ + discard; +}else{ + color = color_mapping(intensity, normalizingFactor).rgb; + fragOutput0 = vec4(color, 1.0); +} +""" + + +fs_dec = compose_shader([kde_dec]) + +fs_impl = compose_shader([kde_impl]) + + +# Windows and scenes setup +width, height = (1920, 1080) +offWidth, offHeight = (1080, 1080) + +offScene = window.Scene() + +off_manager = window.ShowManager( + offScene, + "demo", + (offWidth, + offHeight), + reset_camera=True, + order_transparent=True) + +off_manager.window.SetOffScreenRendering(True) + +off_manager.initialize() + + +scene = window.Scene() +scene.set_camera(position=(-6, 5, -10), + focal_point=(0.0, + 0.0, + 0.0), + view_up=(0.0, 0.0, 0.0)) + +manager = window.ShowManager( + scene, + "demo", + (width, + height), + reset_camera=True, + order_transparent=True) + + +manager.initialize() + +n_points = 1000 +points = np.random.rand(n_points, 3) +points = normalize(points, -5, 5) +sigma = 0.25 +scale = 0.5 + +billboard = actor.billboard( + points, + (0.0, + 0.0, + 1.0), + scales=scale, + fs_dec=fs_dec, + fs_impl=fs_impl) + +# Blending and uniforms setup +shader_apply_effects(off_manager.window, billboard, window.gl_disable_depth) +shader_apply_effects(off_manager.window, billboard, window.gl_set_additive_blending) +shader_custom_uniforms(billboard, "fragment").SetUniformf("sigma", sigma) + +off_manager.scene.add(billboard) + +off_manager.render() + + +# Render to second billboard for color map post-processing. +textured_billboard = actor.billboard(np.array([[0.0, 0.0, 0.0]]), (1.0, 1.0, 1.0), + scales=3.0, fs_dec=tex_dec, fs_impl=tex_impl) + +cmap = colormaps["viridis"] +cmap = np.array([cmap(i) for i in np.arange(0.0, 1.0, 1/256)]) + + +colormap_to_texture(cmap, "colormapTexture", textured_billboard) + + +window_to_texture( + off_manager.window, + "screenTexture", + textured_billboard, + blending_mode="Interpolate") + +manager.scene.add(textured_billboard) + +manager.start()