Merge pull request #1 from greenmagenta/development

Merge development to main

README.md

@@ -51,6 +51,18 @@ For latest features not yet implemented in stable version, you can download the
 | Model | Host | Average generation time |
 |---|---|---|
 | [Shap-E](https://github.com/openai/shap-e) | [hysts/Shap-E](https://huggingface.co/spaces/hysts/Shap-E) | ~13s |
+| [SDXL](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + [Shap-E](https://github.com/openai/shap-e) | [ByteDance/Hyper-SDXL-1Step-T2I](https://huggingface.co/spaces/ByteDance/Hyper-SDXL-1Step-T2I) + [hysts/Shap-E](https://huggingface.co/spaces/hysts/Shap-E) | ~30s |
+| [SDXL](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + [DreamGaussian](https://github.com/dreamgaussian/dreamgaussian) | [ByteDance/Hyper-SDXL-1Step-T2I](https://huggingface.co/spaces/ByteDance/Hyper-SDXL-1Step-T2I) + [jiawei011/dreamgaussian](https://huggingface.co/spaces/jiawei011/dreamgaussian) | ~600s |
+| [SDXL](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + [InstantMesh](https://github.com/TencentARC/InstantMesh) | [ByteDance/Hyper-SDXL-1Step-T2I](https://huggingface.co/spaces/ByteDance/Hyper-SDXL-1Step-T2I) + [TencentARC/InstantMesh](https://huggingface.co/spaces/TencentARC/InstantMesh) | ~60s |
+
+### Examples
+
+| Shap-E | SDXL + Shape-E | SDXL + DreamGaussian | SDXL + InstantMesh |
+|---|---|---|---|
+| <img src="assets/model_shape-e.jpg" height="200px" /> | <img src="assets/model_sdxl-shape-e.jpg" height="200px" /> | <img src="assets/model_sdxl-dreamgaussian.jpg" height="200px" /> | <img src="assets/model_sdxl-instantmesh.jpg" height="200px" /> |
+| A pinguin, 3d model ||||
+| <img src="assets/model_shape-e_2.jpg" height="200px" /> | <img src="assets/model_sdxl-shape-e_2.jpg" height="200px" /> | <img src="assets/model_sdxl-dreamgaussian_2.jpg" height="200px" /> | <img src="assets/model_sdxl-instantmesh_2.jpg" height="200px" /> |
+| A hamburger, 3d model ||||
 
 ## Implementations
 

autosculptor.py

@@ -1,7 +1,8 @@
 import bpy
 import random
-from gradio_client import Client
+from gradio_client import Client, file
 
+# Blender add-on information
 bl_info = {
     "name": "Autosculptor 3D Model Generator",
     "blender": (2, 80, 0),
@@ -16,104 +17,198 @@
     "isDraft": False
 }
 
+# Operator for generating 3D models
 class GeneratorOperator(bpy.types.Operator):
     bl_idname = "object.autosculptor_model_generator"
     bl_label = "Generate 3D Model"
     bl_options = {'REGISTER', 'UNDO'}
-    
+
     def execute(self, context):
         # Import properties from context
         autosculptor_props = context.scene.autosculptor_props
-
+
+        # Get properties from user input
         prompt = autosculptor_props.prompt
         seed = autosculptor_props.seed
         if autosculptor_props.random_seed:
             seed = random.randint(0, 2147483647)
         guidance_scale = autosculptor_props.guidance_scale
         num_inference_steps = autosculptor_props.num_inference_steps
         model_type = autosculptor_props.model_type
-
-        # Shape-E model
-        if model_type == "shape-e-text":
-            client = Client("hysts/Shap-E")
-            result = client.predict(
-                prompt=prompt,
-                seed=seed,
-                guidance_scale=guidance_scale,
-                num_inference_steps=num_inference_steps,
-                api_name="/text-to-3d"
-            )
-            model_path = result
 
-        else:
+        # Generate the 3D model
+        model_path = self.generate_model(prompt, seed, guidance_scale, num_inference_steps, model_type)
+
+        # Handle errors in model generation
+        if not model_path:
             self.report({'ERROR'}, "Invalid model type.")
             return {'CANCELLED'}
 
-        # Load the generated model
+        # Import the generated model into Blender
         bpy.ops.import_scene.gltf(filepath=model_path)
 
-        # Ensure an object has been imported
+        # Check if any object was imported
         if not bpy.context.selected_objects:
             self.report({'ERROR'}, "No object was imported.")
             return {'CANCELLED'}
-        
-        # Get the imported object (parent)
+
+        # Get the imported object
         parent_obj = bpy.context.selected_objects[0]
+        obj = next((child for child in parent_obj.children if child.type == 'MESH'), None)
 
-        # Find mesh object
-        obj = None
-        if parent_obj.children:
-            for child in parent_obj.children:
-                if child.type == 'MESH':
-                    obj = child
-                    break
-
+        # Handle errors in finding a mesh object
         if obj is None:
             self.report({'ERROR'}, "No mesh object found among imported children.")
             return {'CANCELLED'}
 
-        # Create a new material
+        # Assign material to the imported object
+        self.assign_material(obj)
+
+        return {'FINISHED'}
+
+    # Function to generate the model based on the type
+    def generate_model(self, prompt, seed, guidance_scale, num_inference_steps, model_type):
+        if model_type == "model-shape-e":
+            return self.generate_shape_e_model(prompt, seed, guidance_scale, num_inference_steps)
+        elif model_type == "model-sdxl-shape-e":
+            return self.generate_sdxl_shape_e_model(prompt, seed, guidance_scale, num_inference_steps)
+        elif model_type == "model-sdxl-dreamgaussian":
+            return self.generate_sdxl_dreamgaussian_model(prompt, seed, guidance_scale, num_inference_steps)
+        elif model_type == "model-sdxl-instantmesh":
+            return self.generate_sdxl_instantmesh_model(prompt, seed, guidance_scale, num_inference_steps)
+        return None
+
+    # Function to generate Shape-E model
+    def generate_shape_e_model(self, prompt, seed, guidance_scale, num_inference_steps):
+        client = Client("hysts/Shap-E")
+        result = client.predict(
+            prompt=prompt,
+            seed=seed,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+            api_name="/text-to-3d"
+        )
+        return result
+
+    # Function to generate SDXL + Shape-E model
+    def generate_sdxl_shape_e_model(self, prompt, seed, guidance_scale, num_inference_steps):
+        client1 = Client("ByteDance/Hyper-SDXL-1Step-T2I")
+        result = client1.predict(
+            num_images=1,
+            height=1024,
+            width=1024,
+            prompt=prompt,
+            seed=seed,
+            api_name="/process_image"
+        )
+        image_path = result[0]['image']
+
+        client2 = Client("https://one-2-3-45-one-2-3-45.hf.space/")
+        segmented_img_filepath = client2.predict(image_path, api_name="/preprocess")
+
+        client3 = Client("hysts/Shap-E")
+        result = client3.predict(
+            image=file(segmented_img_filepath),
+            seed=seed,
+            guidance_scale=guidance_scale,
+            num_inference_steps=num_inference_steps,
+            api_name="/image-to-3d"
+        )
+        return result
+
+    # Function to generate SDXL + DreamGaussian model
+    def generate_sdxl_dreamgaussian_model(self, prompt, seed, guidance_scale, num_inference_steps):
+        client1 = Client("ByteDance/Hyper-SDXL-1Step-T2I")
+        result = client1.predict(
+            num_images=1,
+            height=1024,
+            width=1024,
+            prompt=prompt,
+            seed=seed,
+            api_name="/process_image"
+        )
+        image_path = result[0]['image']
+
+        client2 = Client("https://one-2-3-45-one-2-3-45.hf.space/")
+        elevation_angle_deg = client2.predict(image_path, True, api_name="/estimate_elevation")
+
+        if elevation_angle_deg < -90 or elevation_angle_deg > 90:
+            elevation_angle_deg = 0
+
+        client3 = Client("https://jiawei011-dreamgaussian.hf.space/--replicas/e0l1g/")
+        result = client3.predict(image_path, True, elevation_angle_deg, fn_index=2)
+        return result
+
+    # Function to generate SDXL + InstantMesh model
+    def generate_sdxl_instantmesh_model(self, prompt, seed, guidance_scale, num_inference_steps):
+        client1 = Client("ByteDance/Hyper-SDXL-1Step-T2I")
+        result = client1.predict(
+            num_images=1,
+            height=1024,
+            width=1024,
+            prompt=prompt,
+            seed=seed,
+            api_name="/process_image"
+        )
+        image_path = result[0]['image']
+
+        client2 = Client("TencentARC/InstantMesh")
+        preprocessed_image = client2.predict(
+            input_image=file(image_path),
+            do_remove_background=True,
+            api_name="/preprocess"
+        )
+
+        mvs = client2.predict(
+            input_image=file(preprocessed_image),
+            sample_steps=num_inference_steps,
+            sample_seed=seed,
+            api_name="/generate_mvs"
+        )
+
+        result = client2.predict(api_name="/make3d")
+        return result[1]
+
+    # Function to assign material to the imported object
+    def assign_material(self, obj):
         material = bpy.data.materials.new(name="ImportedMaterial")
         material.use_nodes = True
-
-        # Initialize Principled BSDF node
-        bsdf = None
-        for node in material.node_tree.nodes:
-            if isinstance(node, bpy.types.ShaderNodeBsdfPrincipled):
-                bsdf = node
-                break
-        if bsdf is None:
+
+        # Find or create a Principled BSDF shader node
+        bsdf = next((node for node in material.node_tree.nodes if isinstance(node, bpy.types.ShaderNodeBsdfPrincipled)), None)
+        if not bsdf:
             bsdf = material.node_tree.nodes.new(type='ShaderNodeBsdfPrincipled')
-        
-        # Create an attribute node
+
+        # Create and configure a Vertex Color node
         attribute_node = material.node_tree.nodes.new('ShaderNodeVertexColor')
         if obj.data.vertex_colors:
             attribute_node.layer_name = obj.data.vertex_colors[0].name
         else:
             attribute_node.layer_name = "Color"
-        
-        # Assign the material to the object
+
+        # Link the Vertex Color node to the BSDF node
         material.node_tree.links.new(attribute_node.outputs['Color'], bsdf.inputs['Base Color'])
-
+
+        # Assign the material to the object
         if obj.data.materials:
             obj.data.materials[0] = material
         else:
             obj.data.materials.append(material)
-
-        return {'FINISHED'}
 
+# UI Panel for the add-on
 class GeneratorPanel(bpy.types.Panel):
     bl_label = "Autosculptor"
     bl_idname = "OBJECT_PT_autosculptor_model_generator"
     bl_space_type = 'VIEW_3D'
     bl_region_type = 'UI'
     bl_category = 'Autosculptor'
-    
+
     def draw(self, context):
         layout = self.layout
         scene = context.scene
         autosculptor_props = scene.autosculptor_props
-
+
+        # Add properties to the UI
         layout.prop(autosculptor_props, "prompt")
         layout.prop(autosculptor_props, "seed")
         layout.prop(autosculptor_props, "random_seed")
@@ -122,6 +217,7 @@ def draw(self, context):
         layout.prop(autosculptor_props, "model_type")
         layout.operator("object.autosculptor_model_generator")
 
+# Property group for user input
 class GeneratorProperties(bpy.types.PropertyGroup):
     prompt: bpy.props.StringProperty(name="Prompt")
     seed: bpy.props.IntProperty(name="Seed", default=0, min=0, max=2147483647)
@@ -131,9 +227,12 @@ class GeneratorProperties(bpy.types.PropertyGroup):
     model_type: bpy.props.EnumProperty(
         name="Model",
         items=[
-            ("shape-e-text", "Shap-E", "hysts/Shap-E")
+            ("model-shape-e", "Shap-E", "hysts/Shap-E (~13s)"),
+            ("model-sdxl-shape-e", "SDXL + Shap-E", "ByteDance/Hyper-SDXL-1Step-T2I + hysts/Shap-E (~30s)"),
+            ("model-sdxl-dreamgaussian", "SDXL + DreamGaussian", "ByteDance/Hyper-SDXL-1Step-T2I + jiawei011/dreamgaussian (~600s)"),
+            ("model-sdxl-instantmesh", "SDXL + InstantMesh", "ByteDance/Hyper-SDXL-1Step-T2I + TencentARC/InstantMesh (~60s)")
         ],
-        default="shape-e-text"
+        default="model-shape-e"
     )
 
 def register():