From dfec942c00e447fa87bd070991d487b55fa0d789 Mon Sep 17 00:00:00 2001 From: haoyuhsu Date: Wed, 30 Oct 2024 00:31:27 -0500 Subject: [PATCH] [10/30] merge melting effect --- blender/all_rendering.py | 333 ++++++++++++++++++++++++++++++++- blender/blend_all.py | 73 +++----- edit_utils.py | 21 ++- gpt/prompts/planner_prompt.txt | 9 +- scene_representation.py | 94 ++++++++-- 5 files changed, 455 insertions(+), 75 deletions(-) diff --git a/blender/all_rendering.py b/blender/all_rendering.py index 92ff757..34f5f92 100644 --- a/blender/all_rendering.py +++ b/blender/all_rendering.py @@ -1460,6 +1460,7 @@ def fire_modifier_infinigen(node_tree): # def fire_modifier_youtube(node_tree): # """ # Add fire modifier to the node tree (references: https://www.youtube.com/watch?v=zyIJQHlFQs0) +# Note: the quality is not as good as the fire_modifier_infinigen # """ # # Create nodes # mat_volume_node = node_tree.nodes['Principled Volume'] @@ -1642,6 +1643,180 @@ def compute_area(obj): return area +######################################################### +# Fluid simulation (a.k.a. melting effect) +######################################################### +def create_melting_effect(obj_mesh, scene_mesh=None, melting_start_frame=1, melting_end_frame=100, resolution=128, cache_dir=None): + """ + Create a melting effect for the object (inspired by this YouTube tutorial: https://www.youtube.com/watch?v=Q42g1fsOHQY&list=LL&index=6) + """ + # prepare parameters + mean_color = compute_average_color_per_vertex(obj_mesh) + bbox_min, bbox_max = scene_bbox(obj_mesh) + max_scale = max(bbox_max - bbox_min) + obj_center = (bbox_max + bbox_min) / 2 + + ### Part 1: insert a cover object (a sphere) and setup its movement ### + bpy.ops.object.select_all(action="DESELECT") + bpy.ops.mesh.primitive_uv_sphere_add() + cover_obj = bpy.context.object + cover_obj.select_set(True) + cover_obj.location = obj_center + cover_obj.scale = [max_scale / 2 * 1.3] * 3 # sphere size to be slightly larger than the object + bpy.ops.object.shade_smooth() + bpy.ops.object.modifier_add(type='SUBSURF') + cover_obj.modifiers["Subdivision"].render_levels = 2 # adjust from 2 to 1 to avoid black spots + bpy.ops.object.modifier_add(type='DISPLACE') + cover_obj.modifiers["Displace"].texture = bpy.data.textures.new(name="Cloud", type='CLOUDS') + cover_obj.modifiers["Displace"].strength = 0.45 + bpy.ops.object.modifier_add(type='SUBSURF') + cover_obj.modifiers["Subdivision"].render_levels = 2 + bpy.ops.object.shade_smooth() + bpy.context.view_layer.update() + + cover_obj.keyframe_insert(data_path="location", frame=melting_start_frame) + cover_obj.location = obj_center + Vector([0, 0, max_scale * 1.3 + 0.1]) # moving upward until it totally above the object + cover_obj.keyframe_insert(data_path="location", frame=melting_end_frame) + bpy.context.view_layer.update() + + ### Part 2: duplicate the objects and apply boolean modifiers for transition effect ### + bpy.ops.object.select_all(action="DESELECT") + obj_mesh.select_set(True) + bpy.context.view_layer.objects.active = obj_mesh + # bpy.ops.object.modifier_add(type='SUBSURF') + # cover_obj.modifiers["Subdivision"].render_levels = 2 # apply subdivision modifier to avoid black spots (https://blenderartists.org/t/black-spots-on-fluid/600414) --> but mess up the 3D Gaussians rendering part + bpy.ops.object.modifier_add(type='BOOLEAN') + obj_mesh.modifiers["Boolean"].operation = 'INTERSECT' + obj_mesh.modifiers["Boolean"].object = cover_obj + + # duplicate both the original object and the cover object + bpy.ops.object.select_all(action="DESELECT") + obj_mesh.select_set(True) + bpy.context.view_layer.objects.active = obj_mesh + bpy.ops.object.duplicate() + obj_mesh_dup = bpy.context.object + obj_mesh_dup.name = obj_mesh.name + "_dup" + bpy.ops.object.select_all(action="DESELECT") + cover_obj.select_set(True) + bpy.context.view_layer.objects.active = cover_obj + bpy.ops.object.duplicate() + cover_obj_dup = bpy.context.object + cover_obj_dup.name = cover_obj.name + "_dup" + + # add additional boolean modifier to the obj_mesh_dup and set the cover_obj_dup as difference object + bpy.ops.object.select_all(action="DESELECT") + obj_mesh_dup.select_set(True) + bpy.context.view_layer.objects.active = obj_mesh_dup + bpy.ops.object.modifier_add(type='BOOLEAN') + obj_mesh_dup.modifiers["Boolean.001"].operation = 'DIFFERENCE' + obj_mesh_dup.modifiers["Boolean.001"].object = cover_obj_dup + + # add -0.05 to delta z-transform to the cover_obj_dup to make it slightly below the original object + cover_obj_dup.delta_location[2] = -0.05 + + # for the inserted_obj_dup, change first boolean to DIFFERENCE and second boolean to INTERSECT + obj_mesh_dup.modifiers["Boolean"].operation = 'DIFFERENCE' + obj_mesh_dup.modifiers["Boolean.001"].operation = 'INTERSECT' + bpy.context.view_layer.update() + + # ensure the sphere proxy is not rendered + cover_obj.hide_render = True + cover_obj_dup.hide_render = True + + ### Part 3: create fluid domain and setup the fluid simulation ### + bpy.ops.object.select_all(action="DESELECT") + bpy.ops.mesh.primitive_cube_add() + fluid_domain = bpy.context.object + fluid_domain.name = obj_mesh.name + "_fluid_domain" + fluid_domain.select_set(True) + + x_scale, y_scale, z_scale = 3, 3, 1.5 + fluid_domain.scale = [0.5, 0.5, (bbox_max[2] - bbox_min[2])/2 * z_scale] + obj_bottom_center = np.array([obj_center[0], obj_center[1], bbox_min[2]]) + if scene_mesh is not None: + z_drop_offset = -0.05 # ensure fluid interaction with the scene_mesh + else: + z_drop_offset = 0 + fluid_domain.location = obj_bottom_center + np.array([0, 0, (bbox_max[2] - bbox_min[2])/2 * z_scale + z_drop_offset]) + + # setup fluid domain settings + bpy.ops.object.modifier_add(type='FLUID') + fluid_domain.modifiers["Fluid"].fluid_type = 'DOMAIN' + fluid_domain.modifiers["Fluid"].domain_settings.domain_type = 'LIQUID' + fluid_domain.modifiers["Fluid"].domain_settings.resolution_max = resolution + fluid_domain.modifiers["Fluid"].domain_settings.time_scale = 0.4 # for slower liquid movement + fluid_domain.modifiers["Fluid"].domain_settings.timesteps_max = 10 # more timesteps for simulation stability (TODO: maybe the timesteps too high?) + fluid_domain.modifiers["Fluid"].domain_settings.timesteps_min = 5 + fluid_domain.modifiers["Fluid"].domain_settings.particle_randomness = 1 + fluid_domain.modifiers["Fluid"].domain_settings.use_diffusion = True # set viscosity to 2e-3 + fluid_domain.modifiers["Fluid"].domain_settings.viscosity_exponent = 3 + fluid_domain.modifiers["Fluid"].domain_settings.viscosity_base = 2 + fluid_domain.modifiers["Fluid"].domain_settings.use_mesh = True # enable fluid mesh + fluid_domain.modifiers["Fluid"].domain_settings.mesh_particle_radius = 1 + fluid_domain.modifiers["Fluid"].domain_settings.mesh_scale = 2 # TODO: consider tweaking this value + + # setup fluid domain material + bpy.ops.object.material_slot_add() + fluid_domain.material_slots[0].material = bpy.data.materials.new(name="Fluid") + fluid_domain.material_slots[0].material.use_nodes = True + fluid_domain.material_slots[0].material.node_tree.nodes.clear() + nodes = fluid_domain.material_slots[0].material.node_tree.nodes + links = fluid_domain.material_slots[0].material.node_tree.links + output_node = nodes.new('ShaderNodeOutputMaterial') + principled_node = nodes.new('ShaderNodeBsdfPrincipled') + links.new(principled_node.outputs['BSDF'], output_node.inputs['Surface']) + principled_node.inputs['Base Color'].default_value = (mean_color[0], mean_color[1], mean_color[2], 1) + principled_node.inputs[7].default_value = 0.7 # alias for specular value + principled_node.inputs['Metallic'].default_value = 0 + principled_node.inputs['Roughness'].default_value = 0.2 + bpy.context.view_layer.update() + + # setup fluid inflow settings + bpy.ops.object.select_all(action="DESELECT") + obj_mesh_dup.select_set(True) + bpy.context.view_layer.objects.active = obj_mesh_dup + bpy.ops.object.modifier_add(type='FLUID') + obj_mesh_dup.modifiers["Fluid"].fluid_type = 'FLOW' + obj_mesh_dup.modifiers["Fluid"].flow_settings.flow_type = 'LIQUID' + obj_mesh_dup.modifiers["Fluid"].flow_settings.flow_behavior = 'INFLOW' + obj_mesh_dup.modifiers["Fluid"].flow_settings.use_plane_init = True # set to True if the inflow object is non-closed + obj_mesh_dup.modifiers["Fluid"].flow_settings.subframes = 1 + obj_mesh_dup.modifiers["Fluid"].flow_settings.use_inflow = True + obj_mesh_dup.modifiers["Fluid"].flow_settings.keyframe_insert(data_path="use_inflow", frame=melting_end_frame) + obj_mesh_dup.modifiers["Fluid"].flow_settings.use_inflow = False + obj_mesh_dup.modifiers["Fluid"].flow_settings.keyframe_insert(data_path="use_inflow", frame=melting_end_frame+1) + bpy.context.view_layer.update() + + # # (optional) cache setting for baking + # fluid_domain.modifiers["Fluid"].domain_settings.cache_frame_start = scene.frame_start + # fluid_domain.modifiers["Fluid"].domain_settings.cache_frame_end = scene.frame_end + # fluid_domain.modifiers["Fluid"].domain_settings.cache_resumable = True + # # fluid_domain.modifiers["Fluid"].domain_settings.cache_type = 'ALL' # need 'bpy.ops.fluid.bake_all()' + # fluid_domain.modifiers["Fluid"].domain_settings.cache_type = 'REPLAY' # do not need baking (default) + + if scene_mesh is not None: + bpy.ops.object.select_all(action="DESELECT") + scene_mesh.select_set(True) + bpy.context.view_layer.objects.active = scene_mesh + bpy.ops.object.modifier_add(type='FLUID') + scene_mesh.modifiers["Fluid"].fluid_type = 'EFFECTOR' + scene_mesh.modifiers["Fluid"].effector_settings.effector_type = 'COLLISION' + scene_mesh.modifiers["Fluid"].effector_settings.subframes = 1 + scene_mesh.modifiers["Fluid"].effector_settings.surface_distance = 0.001 + scene_mesh.modifiers["Fluid"].effector_settings.use_effector = True + scene_mesh.modifiers["Fluid"].effector_settings.use_plane_init = True + + bpy.ops.object.select_all(action="DESELECT") + + # TODO: verbose + print("========================================================") + print("Name of obj_mesh: ", obj_mesh.name) + print("Name of obj_mesh_dup: ", obj_mesh_dup.name) + print("========================================================") + + return fluid_domain, [obj_mesh, obj_mesh_dup] + + ######################################################### # Others (Utility functions) ######################################################### @@ -1705,6 +1880,89 @@ def delete_object_recursive(obj): bpy.data.objects.remove(obj, do_unlink=True) +def get_num_active_vertices(obj): + """ + Useful for tracking active vertices in the object (especially after applying boolean modifier) + """ + depsgraph = bpy.context.evaluated_depsgraph_get() + evaluated_obj = obj.evaluated_get(depsgraph) + mesh = evaluated_obj.to_mesh() + n_active_vert = len(mesh.vertices) + evaluated_obj.to_mesh_clear() + return n_active_vert + + +def get_num_active_faces(obj): + """ + Useful for tracking active faces in the object (especially after applying boolean modifier) + """ + depsgraph = bpy.context.evaluated_depsgraph_get() + evaluated_obj = obj.evaluated_get(depsgraph) + mesh = evaluated_obj.to_mesh() + n_active_faces = len(mesh.polygons) + evaluated_obj.to_mesh_clear() + return n_active_faces + + +def export_object_mesh(obj, export_path): + """ + Export the object mesh to the given path + """ + bpy.ops.object.select_all(action="DESELECT") + obj.select_set(True) + if export_path.endswith(".ply"): + bpy.ops.export_mesh.ply( + filepath=export_path, + use_selection=True, + use_mesh_modifiers=True, + use_normals=True, + use_uv_coords=False, + use_colors=False + ) + if export_path.endswith(".obj"): + bpy.ops.export_scene.obj( + filepath=export_path, + use_selection=True, + use_mesh_modifiers=True, + use_normals=False, + use_uvs=False, + use_materials=False + ) + if export_path.endswith(".stl"): + bpy.ops.export_mesh.stl( + filepath=export_path, + use_selection=True, + use_mesh_modifiers=True, + ascii=False + ) + + +def merge_two_objects(obj1, obj2): + """ + Merge two objects into a single object using boolean modifier + """ + bpy.ops.object.select_all(action='DESELECT') + obj1.select_set(True) + bpy.context.view_layer.objects.active = obj1 + bpy.ops.object.duplicate() + new_obj = bpy.context.view_layer.objects.active + bpy.ops.object.modifier_add(type='BOOLEAN') + modifier = new_obj.modifiers[-1] + modifier.operation = 'UNION' + modifier.object = obj2 + bpy.ops.object.modifier_apply(modifier=modifier.name) + bpy.ops.object.select_all(action='DESELECT') + return new_obj + + +def check_obj_info_attribute(obj_info, attribute_name): + """ + Return value if the attribute name exists in the object_info dictionary + Otherwise, return False if attribute_name does not exist + """ + return obj_info[attribute_name] if attribute_name in obj_info.keys() else False + + ######################################################### # Event handler ######################################################### @@ -1728,7 +1986,7 @@ def event_parser(event): 'fire': ('start_fire', 'stop_fire', 'remove_fire'), 'smoke': ('start_smoke', 'stop_smoke', 'remove_smoke'), 'break': ('start_break', None), - 'incinerate': ('start_incinerate', None), + 'incinerate': ('start_incinerate', None), # TODO: not implemented yet } event_list = [] @@ -1885,6 +2143,11 @@ def execute_event(obj_id, action, **kwargs): smoke_proxy_obj_dict = {} # object_id -> smoke domain for proxy object all_3dgs_object_names = [] # list of names of all 3dgs objects (used for custom post-filtering after creating fractures) +fluid_domain_dict = {} # object_id -> fluid domain +melting_related_object_dict = {} # object_id -> fluid related object +melting_object_list = [] # list of object_id that has melting effect +fluid_toggle_on_list = [] # list of object_id that has fluid simulation toggle on + # COLLISION_MARGIN = 0.001 # DOMAIN_HEIGHT = 8.0 # CACHE_DIR = None @@ -2042,8 +2305,22 @@ def run_blender_render(config_path): object_list.append(object_mesh) if obj_info['from_3DGS']: all_3dgs_object_names.append(object_mesh.name) - if obj_info['fracture']: + if check_obj_info_attribute(obj_info, 'fracture'): fracture_object_list.append(object_mesh) + if check_obj_info_attribute(obj_info, 'melting'): + melting_object_list.append(object_mesh) + fluid_domain, melting_related_objects = create_melting_effect(object_mesh, scene_mesh=None, melting_start_frame=1, melting_end_frame=int(scene.frame_end * (2/3)), resolution=256) # TODO: change to 256 + fluid_domain_dict[obj_info['object_id']] = fluid_domain + melting_related_object_dict[obj_info['object_id']] = melting_related_objects + if obj_info['from_3DGS'] and obj_info['material'] is None: + object_3dgs_list.extend([obj for obj in melting_related_objects if obj not in object_3dgs_list]) # add duplicate one to the list + else: + object_list.extend([obj for obj in melting_related_objects if obj not in object_list]) # add duplicate one to the list + if check_obj_info_attribute(obj_info, 'break'): + obj_id = obj_info['object_id'] + if obj_id not in all_events_dict: + all_events_dict[obj_id] = [] + all_events_dict[obj_id].append((obj_id, 'start_break', scene.frame_end // 2)) all_object_dict[obj_info['object_id']] = object_mesh add_rigid_body(scene_mesh, 'PASSIVE', 'MESH', 1.0, 0.5, COLLISION_MARGIN) @@ -2094,10 +2371,6 @@ def run_blender_render(config_path): scene.rigidbody_world.point_cache.frame_start = scene.frame_start scene.rigidbody_world.point_cache.frame_end = scene.frame_end - # DO NOT bake if simulate with particles - # if len(smoke_object_info) == 0 and len(fire_object_info) == 0: - # bpy.ops.ptcache.bake_all(bake=True) - cam.initialize_depth_extractor() # initialize once rb_transform = {} @@ -2146,6 +2419,17 @@ def run_blender_render(config_path): delete_object_recursive(obj) # add the fractured objects to the list debris_object_list.extend(fracture_object) + + # check if inflow objects start emitting fluid particles + for obj_id, fluid_domain in fluid_domain_dict.items(): + melting_obj, melting_obj_dup = melting_related_object_dict[obj_id] + n_vertices = get_num_active_vertices(melting_obj_dup) + if n_vertices > 0: + fluid_domain.show_instancer_for_render = True + fluid_toggle_on_list.append(obj_id) + else: + if obj_id not in fluid_toggle_on_list: + fluid_domain.show_instancer_for_render = False # disable showing emitter with no particles bpy.context.view_layer.update() # Ensure the scene is fully updated @@ -2155,9 +2439,13 @@ def run_blender_render(config_path): for object_mesh in object_list + debris_object_list: set_visible_camera_recursive(object_mesh, True) for object_mesh in object_3dgs_list: + if object_mesh.name.endswith('_dup'): # prevent overlap vertices with fluid particles (z-fighting) + set_hide_render_recursive(object_mesh, True) set_visible_camera_recursive(object_mesh, False) for smoke_domain in smoke_domain_dict.values(): set_hide_render_recursive(smoke_domain, True) + for fluid_domain in fluid_domain_dict.values(): + set_hide_render_recursive(fluid_domain, False) scene_mesh.visible_camera = False if emitter_mesh is not None: emitter_mesh.visible_camera = False @@ -2172,9 +2460,13 @@ def run_blender_render(config_path): for object_mesh in object_list + debris_object_list: set_visible_camera_recursive(object_mesh, False) for object_mesh in object_3dgs_list: + if object_mesh.name.endswith('_dup'): + continue set_visible_camera_recursive(object_mesh, True) for smoke_domain in smoke_domain_dict.values(): set_hide_render_recursive(smoke_domain, True) + for fluid_domain in fluid_domain_dict.values(): + set_hide_render_recursive(fluid_domain, True) scene_mesh.visible_camera = False if emitter_mesh is not None: emitter_mesh.visible_camera = False @@ -2196,6 +2488,8 @@ def run_blender_render(config_path): set_visible_camera_recursive(object_mesh, False) for smoke_domain in smoke_domain_dict.values(): set_hide_render_recursive(smoke_domain, False) + for fluid_domain in fluid_domain_dict.values(): + set_hide_render_recursive(fluid_domain, True) scene_mesh.visible_camera = False if emitter_mesh is not None: emitter_mesh.visible_camera = False @@ -2222,6 +2516,8 @@ def run_blender_render(config_path): set_hide_render_recursive(object_mesh, True) for smoke_domain in smoke_domain_dict.values(): set_hide_render_recursive(smoke_domain, True) + for fluid_domain in fluid_domain_dict.values(): + set_hide_render_recursive(fluid_domain, True) scene_mesh.visible_camera = True if emitter_mesh is not None: emitter_mesh.visible_camera = True @@ -2237,10 +2533,15 @@ def run_blender_render(config_path): set_hide_render_recursive(object_mesh, False) set_visible_camera_recursive(object_mesh, True) for object_mesh in object_3dgs_list: - set_hide_render_recursive(object_mesh, False) + if object_mesh.name.endswith('_dup'): # prevent overlap vertices with fluid particles (z-fighting) + set_hide_render_recursive(object_mesh, True) + else: + set_hide_render_recursive(object_mesh, False) set_visible_camera_recursive(object_mesh, False) for smoke_domain in smoke_domain_dict.values(): set_hide_render_recursive(smoke_domain, False) + for fluid_domain in fluid_domain_dict.values(): + set_hide_render_recursive(fluid_domain, False) scene_mesh.visible_camera = True if emitter_mesh is not None: emitter_mesh.visible_camera = True @@ -2254,8 +2555,12 @@ def run_blender_render(config_path): else: cam.render_single_timestep_rgb_and_depth(cam_list[FRAME_INDEX-1], FRAME_INDEX, dir_name_rgb='rgb_all', dir_name_depth='depth_all') - # Step 6: save the rigid body transformation of 3dgs objects + # Step 6: save the rigid body transformation of 3dgs objects (Optional) for object_mesh in object_3dgs_list: + base_name = object_mesh.name.replace('_dup', '') + if base_name in fluid_domain_dict.keys(): + print("Skip storing rigid body transformation for fluid domain object: ", object_mesh.name) + continue if object_mesh.name not in rb_transform: rb_transform[object_mesh.name] = {} transform = {} @@ -2264,6 +2569,18 @@ def run_blender_render(config_path): transform['rot'] = rot.tolist() transform['scale'] = object_3dgs_scale_dict[object_mesh.name] rb_transform[object_mesh.name]['{0:03d}'.format(FRAME_INDEX)] = transform + + # Step 7: save the modified meshes during melting animation (Optional) + for object_mesh in object_3dgs_list: + if object_mesh.name in fluid_domain_dict.keys(): # no need to save again for duplicates + melting_meshes_output_dir = os.path.join(output_dir, 'melting_meshes') + meshes_output_dir = os.path.join(melting_meshes_output_dir, object_mesh.name) + os.makedirs(meshes_output_dir, exist_ok=True) + melting_obj, melting_obj_dup = melting_related_object_dict[obj_id] + if get_num_active_faces(melting_obj) > 0: + export_object_mesh(melting_obj, os.path.join(meshes_output_dir, '{0:03d}_obj.stl'.format(FRAME_INDEX))) + if get_num_active_faces(melting_obj_dup) > 0: + export_object_mesh(melting_obj_dup, os.path.join(meshes_output_dir, '{0:03d}_obj_dup.stl'.format(FRAME_INDEX))) # add rigid body transformation to the original config file if rb_transform: diff --git a/blender/blend_all.py b/blender/blend_all.py index 7a7765f..086a580 100644 --- a/blender/blend_all.py +++ b/blender/blend_all.py @@ -94,29 +94,36 @@ def blend_frames(blend_results_dir, input_config_path=None): root_dir = os.path.dirname(os.path.normpath(os.path.dirname(os.path.normpath(blend_results_dir)))) # get up two level # preload all frames path instead of loading all frames into memory + # input_config = None + # if input_config_path is not None: + # with open(input_config_path, 'r') as f: + # input_config = json.load(f) + # if 'render_type' in input_config and input_config['render_type'] == 'SINGLE_VIEW': + # anchor_frame_idx = input_config['anchor_frame_idx'] + # num_frames = input_config['num_frames'] + # anchor_frame_rgb_path = os.path.join(root_dir, 'images', f'{anchor_frame_idx:05}.png') + # anchor_frame_depth_path = os.path.join(root_dir, 'depth', f'{anchor_frame_idx:05}.npy') + # # copy both rgb & depth paths for num_frames times into bg_rgb and bg_depth + # bg_rgb = [anchor_frame_rgb_path] * num_frames + # bg_depth = [anchor_frame_depth_path] * num_frames + # else: + # # default: MULTI_VIEW option + # bg_rgb = sorted(glob.glob(os.path.join(root_dir, 'images', '*.png'))) + # bg_depth = sorted(glob.glob(os.path.join(root_dir, 'depth', '*.npy'))) + # else: + # bg_rgb = sorted(glob.glob(os.path.join(root_dir, 'images', '*.png'))) + # bg_depth = sorted(glob.glob(os.path.join(root_dir, 'depth', '*.npy'))) + input_config = None if input_config_path is not None: with open(input_config_path, 'r') as f: input_config = json.load(f) - if 'render_type' in input_config and input_config['render_type'] == 'SINGLE_VIEW': - anchor_frame_idx = input_config['anchor_frame_idx'] - num_frames = input_config['num_frames'] - anchor_frame_rgb_path = os.path.join(root_dir, 'images', f'{anchor_frame_idx:05}.png') - anchor_frame_depth_path = os.path.join(root_dir, 'depth', f'{anchor_frame_idx:05}.npy') - # copy both rgb & depth paths for num_frames times into bg_rgb and bg_depth - bg_rgb = [anchor_frame_rgb_path] * num_frames - bg_depth = [anchor_frame_depth_path] * num_frames - else: - # default: MULTI_VIEW option - bg_rgb = sorted(glob.glob(os.path.join(root_dir, 'images', '*.png'))) - bg_depth = sorted(glob.glob(os.path.join(root_dir, 'depth', '*.npy'))) - else: - bg_rgb = sorted(glob.glob(os.path.join(root_dir, 'images', '*.png'))) - bg_depth = sorted(glob.glob(os.path.join(root_dir, 'depth', '*.npy'))) - assert input_config is not None, 'input_config is required for blending frames' blender_cache_dir = os.path.join(input_config['blender_cache_dir'], input_config['output_dir_name']) + bg_rgb = sorted(glob.glob(os.path.join(root_dir, 'images', '*.png'))) + bg_depth = sorted(glob.glob(os.path.join(root_dir, 'depth', '*.npy'))) + rgb_all_img_path = glob.glob(os.path.join(blender_cache_dir, 'rgb_all', '*.png')) # use this to ensure an output video even if Blender crashes n_frame = len(rgb_all_img_path) @@ -242,22 +249,15 @@ def blend_frames(blend_results_dir, input_config_path=None): # New Implementation of blending frame = bg_c.copy() + ############################################################ ##### Step 1: blend shadow into background image ##### + ############################################################ if has_3dgs: depth_mask = depth_check(s_d, o_gs_d, option='naive', d_tol=0.1) obj_3dgs_alpha = o_gs_c[..., 3] / 255. non_obj_3dgs_alpha = 1. - obj_3dgs_alpha non_obj_3dgs_alpha[depth_mask] = 1.0 - - # if has_smoke or has_fire: - # obj_alpha = s_f_c[..., 3] / 255. - # depth_mask = depth_check(s_f_d, s_d, option='naive', d_tol=0.1) - # else: - # obj_alpha = o_c[..., 3] / 255. - # depth_mask = depth_check(o_d, s_d, option='naive', d_tol=0.1) - ############################################################ - # TODO: test fireball effects obj_alpha = o_c[..., 3] / 255. depth_mask = depth_check(o_d, s_d, option='naive', d_tol=0.1) @@ -266,13 +266,16 @@ def blend_frames(blend_results_dir, input_config_path=None): depth_mask_smoke = depth_check(s_f_d, s_d, option='naive', d_tol=0.1) obj_alpha = np.maximum(obj_alpha, obj_alpha_smoke) depth_mask = np.logical_or(depth_mask, depth_mask_smoke) - ############################################################ obj_mask = obj_alpha > 0.0 mask = np.logical_and(obj_mask, depth_mask) obj_alpha[~mask] = 0.0 non_object_alpha = 1. - obj_alpha + if has_3dgs: + obj_3dgs_front_mask = depth_check(o_gs_d, o_d, option='naive', d_tol=0.1) + obj_alpha[obj_3dgs_front_mask] *= non_obj_3dgs_alpha[obj_3dgs_front_mask] + fg_alpha = o_s_c[..., 3] / 255. if has_3dgs: shadow_catcher_alpha = non_object_alpha * fg_alpha * non_obj_3dgs_alpha @@ -288,23 +291,8 @@ def blend_frames(blend_results_dir, input_config_path=None): frame[mask] = frame[mask] * color_diff[mask] * shadow_catcher_alpha[mask, None] + frame[mask] * (1 - shadow_catcher_alpha[mask, None]) - ##### Step 2: blend object and 3DGS object into background image ##### - # obj_alpha = o_c[..., 3] / 255. - # obj_mask = obj_alpha > 0.0 - # depth_mask = depth_check(o_d, s_d, option='naive', d_tol=0.1) - - # mask = np.logical_and(obj_mask, depth_mask) - # if has_fire: - # mask = depth_mask - # frame[:, :, :3][mask] = s_f_c_pre[:, :, :3][mask] + frame[:, :, :3][mask] * (1 - obj_alpha[mask, None]) - # elif has_smoke: - # mask = depth_mask - # frame[:, :, :3][mask] = s_f_c[:, :, :3][mask] * obj_alpha[mask, None] + frame[:, :, :3][mask] * (1 - obj_alpha[mask, None]) - # else: - # frame[:, :, :3][mask] = o_c[:, :, :3][mask] * obj_alpha[mask, None] + frame[:, :, :3][mask] * (1 - obj_alpha[mask, None]) - ############################################################ - # TODO: test fireball effects + ##### Step 2: blend object and 3DGS object into background image ##### ############################################################ frame_tmp = frame.copy() mask = np.logical_and(obj_mask, depth_mask) @@ -312,7 +300,6 @@ def blend_frames(blend_results_dir, input_config_path=None): if has_fire: mask = depth_mask_smoke frame[:, :, :3][mask] = s_f_c_pre[:, :, :3][mask] + frame_tmp[:, :, :3][mask] * (1 - obj_alpha_smoke[mask, None]) - ############################################################ ############################################################ # temporary results (original frame, foreground object, foreground object mask, foreground object with shadow, shadow only) diff --git a/edit_utils.py b/edit_utils.py index 8d5477f..dc1ddd3 100644 --- a/edit_utils.py +++ b/edit_utils.py @@ -51,9 +51,10 @@ ##### Additional functions for time-varying scene editing ##### - (o) make_break -- (o) incinerate +- (NaN) incinerate - (o) add_event - (o) get_camera_position +- (o) make_melting ##### Additional functions for autonomous driving scene editing ##### - (o) get_vehicle_position @@ -86,6 +87,7 @@ def get_default_object_info(): 'material': None, 'fracture': False, 'break': False, + 'melting': False, 'incinerate': False, } @@ -487,15 +489,24 @@ def make_break(obj): return obj -def incinerate(obj): +def make_melting(obj): ''' - Turn object into ashes. + Melt object into liquid. ''' - obj['incinerate'] = True - print("Incinerating object: {} {}".format(obj['object_name'], obj['object_id'])) + obj['melting'] = True + print("Melting object: {} {}".format(obj['object_name'], obj['object_id'])) return obj +# def incinerate(obj): +# ''' +# Turn object into ashes. +# ''' +# obj['incinerate'] = True +# print("Incinerating object: {} {}".format(obj['object_name'], obj['object_id'])) +# return obj + + def get_camera_position(scene_representation): ''' Get camera position. diff --git a/gpt/prompts/planner_prompt.txt b/gpt/prompts/planner_prompt.txt index 28636f9..88d64ea 100644 --- a/gpt/prompts/planner_prompt.txt +++ b/gpt/prompts/planner_prompt.txt @@ -1,7 +1,7 @@ # Remember to import functions from edit_utils.py as needed. from edit_utils import detect_object, sample_point_on_object, sample_point_above_object, retrieve_asset, \ insert_object, remove_object, update_object, allow_physics, add_fire, add_smoke, set_static_animation, set_moving_animation \ - init_material, retrieve_material, apply_material, allow_fracture, make_break, incinerate \ + init_material, retrieve_material, apply_material, allow_fracture, make_break, make_melting \ get_object_center_position, get_object_bottom_position, translate_object, rotate_object, scale_object, get_random_2D_rotation, get_random_3D_rotation, make_copy \ add_event, get_camera_position @@ -10,7 +10,7 @@ from edit_utils import detect_object, sample_point_on_object, sample_point_above # Default position for detected objects from the scene may not be (0, 0, 0) and rotation is identity matrix. # translate_object takes in relative offset as input, not an absolute position. # rotate_object takes in 3x3 rotation matrix as input. -# allow_fracture breaks the object into pieces when it collides with other objects. make_break breaks the object into pieces immediately. incinerate turns the object into ashes immediately. +# allow_fracture breaks the object into pieces when it collides with other objects. make_break breaks the object into pieces immediately. make_melting turns the object into liquid immediately. # To control effects of the object at specific time, use add_event function. # add_event(scene, object, event_type, start_frame, end_frame) @@ -18,6 +18,11 @@ from edit_utils import detect_object, sample_point_on_object, sample_point_above # start_frame: default is 1 for 'static', 'animation', 'physics', 'fire', 'smoke', and half of total frames for 'break' and 'incinerate' # end_frame: default is the total frames + 1 for 'static', 'animation', 'physics', 'fire', 'smoke', and not applicable for 'break' and 'incinerate' +# Query: Melt down the plastic bottle into liquid. +plastic_bottle = detect_object(scene, 'plastic bottle') +plastic_bottle = make_melting(plastic_bottle) +update_object(scene, plastic_bottle) + # Query: Insert an apple and make it break into pieces in the middle of the video. apple = retrieve_asset(scene, 'apple') add_event(scene, apple, 'break', start_frame=scene.total_frames//2) diff --git a/scene_representation.py b/scene_representation.py index 68ee3fb..7f22b2d 100644 --- a/scene_representation.py +++ b/scene_representation.py @@ -46,6 +46,8 @@ from inpaint.retrain_utils import compute_lpips_loss, init_lpips_model, is_large_mask from sugar.gaussian_splatting.utils.loss_utils import ssim +import open3d as o3d +import trimesh CONSOLE = Console(width=120) @@ -88,6 +90,12 @@ def __init__(self, hparams): self.blender_cfg = {} self.rb_transform_info = None + self.blender_cache_dir = os.path.join( + self.cache_dir, + 'blender_rendering', + self.dataset_dir.rstrip('/').split('/')[-1], # scene name + self.custom_traj_name + ) bg_color = [1,1,1] if self.hparams.white_background else [0, 0, 0] self.background = torch.tensor(bg_color, dtype=torch.float32, device="cuda") @@ -220,10 +228,14 @@ def load_scene(self): self.gaussians = gaussians - def render_scene(self, skip_render_3DGS=True): + def render_scene(self, skip_render_3DGS=False): self.render_from_blender() - if not skip_render_3DGS or self.rb_transform_info is not None: - self.render_from_3DGS() + if ( + not skip_render_3DGS or + self.rb_transform_info is not None or + os.path.exists(os.path.join(self.blender_output_dir, 'melting_meshes')) + ): + self.render_from_3DGS(post_rendering=True) blend_all.blend_frames(self.blender_output_dir, self.cfg_path) @@ -235,7 +247,7 @@ def save_cfg(self, cfg, cfg_path): def set_basic_blender_cfg(self): new_cfg = {} new_cfg['edit_text'] = self.hparams.edit_text - new_cfg['blender_cache_dir'] = os.path.join(self.cache_dir, 'blender_rendering', self.dataset_dir.rstrip('/').split('/')[-1], self.custom_traj_name) + new_cfg['blender_cache_dir'] = self.blender_cache_dir new_cfg['im_width'], new_cfg['im_height'] = self.cameras['img_wh'] new_cfg['K'] = self.cameras['K'].tolist() new_cfg['c2w'] = self.cameras['c2w'].tolist() @@ -329,12 +341,17 @@ def get_sunlight_direction(self, img_path, c2w): return dir_vector - def render_from_3DGS(self, render_video=False): + def render_from_3DGS(self, render_video=False, post_rendering=False): self.load_scene() # reload the scene to get the latest gaussians camera_views = self.cameras['cameras'] # a list of Camera objects + if post_rendering and self.hparams.render_type == 'SINGLE_VIEW': + camera_views = [copy.deepcopy(self.cameras['cameras'][self.anchor_frame_idx]) for _ in range(self.total_frames)] + for cam_idx, view in enumerate(camera_views): + camera_views[cam_idx].image_name = '{0:05d}'.format(cam_idx) + render_path = os.path.join(self.traj_results_dir, "images") os.makedirs(render_path, exist_ok=True) depth_path = os.path.join(self.traj_results_dir, "depth") @@ -342,9 +359,6 @@ def render_from_3DGS(self, render_video=False): normal_path = os.path.join(self.traj_results_dir, "normal") os.makedirs(normal_path, exist_ok=True) - # pseudo_normal_path = os.path.join(self.traj_results_dir, "pseudo_normal") - # os.makedirs(pseudo_normal_path, exist_ok=True) - with torch.no_grad(): for idx, view in tqdm(enumerate(camera_views), desc="Rendering progress"): if self.rb_transform_info is not None: @@ -362,6 +376,56 @@ def render_from_3DGS(self, render_video=False): object_gaussians = load_gaussians(obj_gaussians_path, self.hparams.max_sh_degree - 1) transformed_gaussians = transform_gaussians(object_gaussians, center, rotation, scaling, initial_center) all_gaussians = merge_two_gaussians(all_gaussians, transformed_gaussians) + elif os.path.exists(os.path.join(self.blender_cache_dir, self.hparams.blender_output_dir_name, 'melting_meshes')): + all_gaussians = copy.deepcopy(self.gaussians) + mesh_output_dir = os.path.join(self.blender_cache_dir, self.hparams.blender_output_dir_name, 'melting_meshes') + for obj_id in sorted(os.listdir(mesh_output_dir)): + melting_mesh_dir = os.path.join(mesh_output_dir, obj_id) + obj_info = [ + obj for obj in self.blender_cfg['insert_object_info'] + if obj['object_id'] == obj_id + ][0] + orig_mesh_path = obj_info['object_path'] + orig_gaussians_path = os.path.join('/'.join(orig_mesh_path.split('/')[:-2]), 'object_gaussians.ply') + orig_mesh = trimesh.load_mesh(orig_mesh_path) + orig_gaussians = load_gaussians(orig_gaussians_path, self.hparams.max_sh_degree - 1) + # associate closest triangle in the original mesh to each Gaussian center + orig_mesh_o3d = o3d.t.geometry.RaycastingScene() + orig_mesh_o3d.add_triangles(o3d.t.geometry.TriangleMesh.from_legacy(orig_mesh.as_open3d)) + gaussians_xyz = orig_gaussians._xyz.detach().cpu().numpy() + ret_dict = orig_mesh_o3d.compute_closest_points( + o3d.core.Tensor.from_numpy(gaussians_xyz.astype(np.float32)) + ) + triangle_ids_from_gaussians = ret_dict['primitive_ids'].cpu().numpy() + # iterate over the melting meshes + melting_mesh_paths = [ + os.path.join(melting_mesh_dir, '{0:03d}_obj.stl'.format(idx + 1)), + os.path.join(melting_mesh_dir, '{0:03d}_obj_dup.stl'.format(idx + 1)) + ] + for melting_mesh_path in melting_mesh_paths: + if not os.path.exists(melting_mesh_path): + continue + melting_mesh = trimesh.load_mesh(melting_mesh_path) # meet ValueError: PLY is unexpected length! + # melting_mesh = o3d.io.read_triangle_mesh(melting_mesh_path) + # associate closest triangle in the original mesh to each vertex in the melting mesh + ret_dict = orig_mesh_o3d.compute_closest_points( + o3d.core.Tensor.from_numpy(np.array(melting_mesh.triangles_center).astype(np.float32)) + ) + # ret_dict = orig_mesh_o3d.compute_closest_points( + # o3d.core.Tensor.from_numpy(np.array(melting_mesh.vertices).astype(np.float32)) + # ) + triangle_ids_from_melting = ret_dict['primitive_ids'].cpu().numpy() + # keep the Gaussians sharing the same closest triangle with the melting mesh + matching_gaussians_mask = np.isin(triangle_ids_from_gaussians, triangle_ids_from_melting) + # create new Gaussians and merge the new Gaussians with the existing ones + new_gaussians = copy.deepcopy(orig_gaussians) + new_gaussians._xyz = orig_gaussians._xyz[matching_gaussians_mask] + new_gaussians._features_dc = orig_gaussians._features_dc[matching_gaussians_mask] + new_gaussians._features_rest = orig_gaussians._features_rest[matching_gaussians_mask] + new_gaussians._scaling = orig_gaussians._scaling[matching_gaussians_mask] + new_gaussians._rotation = orig_gaussians._rotation[matching_gaussians_mask] + new_gaussians._opacity = orig_gaussians._opacity[matching_gaussians_mask] + all_gaussians = merge_two_gaussians(all_gaussians, new_gaussians) else: all_gaussians = self.gaussians result = render(view, all_gaussians, self.pipe, self.background) @@ -380,12 +444,6 @@ def render_from_3DGS(self, render_video=False): normal = (normal * 255).astype(np.uint8) cv2.imwrite(os.path.join(normal_path, view.image_name + ".png"), cv2.cvtColor(normal, cv2.COLOR_RGB2BGR)) - # # pseudo normal map - # pseudo_normal = result["pseudo_normal"].cpu().numpy() - # pseudo_normal = (pseudo_normal + 1) / 2 - # pseudo_normal = (pseudo_normal * 255).astype(np.uint8) - # cv2.imwrite(os.path.join(pseudo_normal_path, view.image_name + ".png"), cv2.cvtColor(pseudo_normal, cv2.COLOR_RGB2BGR)) - # generate video from frames if render_video: rgb_frames_path = sorted(glob.glob(os.path.join(render_path, '*.png'))) @@ -395,9 +453,6 @@ def render_from_3DGS(self, render_video=False): normal_frames_path = sorted(glob.glob(os.path.join(normal_path, '*.png'))) generate_video_from_frames(normal_frames_path, os.path.join(self.traj_results_dir, 'render_normal.mp4'), fps=15) - # pseudo_normal_frames_path = sorted(glob.glob(os.path.join(pseudo_normal_path, '*.png'))) - # generate_video_from_frames(pseudo_normal_frames_path, os.path.join(self.traj_results_dir, 'render_pseudo_normal.mp4'), fps=15) - # def estimate_scene_scale(self): # """ @@ -600,6 +655,11 @@ def training_3DGS_for_inpainting(self, gaussians_path, image_dir, mask_dir, outp ##### Test rendering from Blender ##### # scene_representation.render_scene() + # with open(scene_representation.cfg_path, 'r') as f: + # scene_representation.blender_cfg = json.load(f) + # scene_representation.rb_transform_info = scene_representation.blender_cfg['rb_transform'] + # scene_representation.render_scene() + ##### Test rendering from 3DGS ##### scene_representation.load_scene() scene_representation.render_from_3DGS(render_video=True)