Add support for CLIP Skip

src/app.py

@@ -133,6 +133,12 @@
     action="store_true",
     help="Use offline model",
 )
+parser.add_argument(
+    "--clip_skip",
+    type=int,
+    help="CLIP Skip (1-12), default : 1 (disabled) ",
+    default=1,
+)
 parser.add_argument(
     "--use_safety_checker",
     action="store_true",
@@ -334,6 +340,7 @@
     else:
         config.lcm_diffusion_setting.use_seed = False
     config.lcm_diffusion_setting.use_offline_model = args.use_offline_model
+    config.lcm_diffusion_setting.clip_skip = args.clip_skip
     config.lcm_diffusion_setting.use_safety_checker = args.use_safety_checker
 
     # Read custom settings from JSON file

src/backend/lcm_text_to_image.py

@@ -309,6 +309,12 @@ def generate(
             if self.is_openvino_init:
                 self.is_openvino_init = False
 
+        pipeline_extra_args = {}
+        if lcm_diffusion_setting.clip_skip > 1:
+            # We follow the convention that "CLIP Skip == 2" means "skip
+            # the last layer", so "CLIP Skip == 1" means "no skipping"
+            pipeline_extra_args['clip_skip'] = lcm_diffusion_setting.clip_skip - 1
+
         if not lcm_diffusion_setting.use_safety_checker:
             self.pipeline.safety_checker = None
             if (
@@ -369,6 +375,7 @@ def generate(
                     height=lcm_diffusion_setting.image_height,
                     num_images_per_prompt=lcm_diffusion_setting.number_of_images,
                     timesteps=self._get_timesteps(),
+                    **pipeline_extra_args,
                     **controlnet_args,
                 ).images
 
@@ -386,6 +393,7 @@ def generate(
                     width=lcm_diffusion_setting.image_width,
                     height=lcm_diffusion_setting.image_height,
                     num_images_per_prompt=lcm_diffusion_setting.number_of_images,
+                    **pipeline_extra_args,
                     **controlnet_args,
                 ).images
         return result_images

src/backend/models/lcmdiffusion_setting.py

@@ -50,6 +50,7 @@ class LCMDiffusionSetting(BaseModel):
     image_width: Optional[int] = 512
     inference_steps: Optional[int] = 1
     guidance_scale: Optional[float] = 1
+    clip_skip: Optional[int] = 1
     number_of_images: Optional[int] = 1
     seed: Optional[int] = 123123
     use_seed: bool = False

src/frontend/gui/app_window.py

@@ -91,6 +91,9 @@ def init_ui_values(self):
         self.inference_steps.setValue(
             int(self.config.settings.lcm_diffusion_setting.inference_steps)
         )
+        self.clip_skip.setValue(
+            int(self.config.settings.lcm_diffusion_setting.clip_skip)
+        )
         self.seed_check.setChecked(self.config.settings.lcm_diffusion_setting.use_seed)
         self.seed_value.setText(str(self.config.settings.lcm_diffusion_setting.seed))
         self.use_local_model_folder.setChecked(
@@ -246,6 +249,13 @@ def create_settings_tab(self):
         self.guidance.setValue(10)
         self.guidance.valueChanged.connect(self.update_guidance_label)
 
+        self.clip_skip_value = QLabel("CLIP Skip: 1")
+        self.clip_skip = QSlider(orientation=Qt.Orientation.Horizontal)
+        self.clip_skip.setMaximum(12)
+        self.clip_skip.setMinimum(1)
+        self.clip_skip.setValue(1)
+        self.clip_skip.valueChanged.connect(self.update_clip_skip_label)
+
         self.width_value = QLabel("Width :")
         self.width = QComboBox(self)
         self.width.addItem("256")
@@ -340,6 +350,8 @@ def create_settings_tab(self):
         vlayout.addWidget(self.height)
         vlayout.addWidget(self.guidance_value)
         vlayout.addWidget(self.guidance)
+        vlayout.addWidget(self.clip_skip_value)
+        vlayout.addWidget(self.clip_skip)
         vlayout.addLayout(hlayout)
         vlayout.addWidget(self.safety_checker)
 
@@ -491,6 +503,10 @@ def use_lcm_lora_changed(self, state):
             self.neg_prompt.setEnabled(False)
             self.config.settings.lcm_diffusion_setting.use_lcm_lora = False
 
+    def update_clip_skip_label(self, value):
+        self.clip_skip_value.setText(f"CLIP Skip: {value}")
+        self.config.settings.lcm_diffusion_setting.clip_skip = value
+
     def use_safety_checker_changed(self, state):
         if state == 2:
             self.config.settings.lcm_diffusion_setting.use_safety_checker = True
@@ -604,6 +620,7 @@ def reset_all_settings(self):
         self.height.setCurrentText("512")
         self.inference_steps.setValue(4)
         self.guidance.setValue(10)
+        self.clip_skip.setValue(1)
         self.use_openvino_check.setChecked(False)
         self.seed_check.setChecked(False)
         self.safety_checker.setChecked(False)

src/frontend/webui/generation_settings_ui.py

@@ -25,6 +25,10 @@ def on_change_guidance_scale(guidance_scale):
     app_settings.settings.lcm_diffusion_setting.guidance_scale = guidance_scale
 
 
+def on_change_clip_skip(clip_skip):
+    app_settings.settings.lcm_diffusion_setting.clip_skip = clip_skip
+
+
 def on_change_seed_value(seed):
     app_settings.settings.lcm_diffusion_setting.seed = seed
 
@@ -103,6 +107,15 @@ def get_generation_settings_ui() -> None:
                     label="Guidance Scale",
                     interactive=True,
                 )
+                clip_skip = gr.Slider(
+                    1,
+                    12,
+                    value=app_settings.settings.lcm_diffusion_setting.clip_skip,
+                    step=1,
+                    label="CLIP Skip",
+                    interactive=True,
+                )
+
 
                 seed = gr.Slider(
                     value=app_settings.settings.lcm_diffusion_setting.seed,
@@ -145,6 +158,7 @@ def get_generation_settings_ui() -> None:
         image_width.change(on_change_image_width, image_width)
         num_images.change(on_change_num_images, num_images)
         guidance_scale.change(on_change_guidance_scale, guidance_scale)
+        clip_skip.change(on_change_clip_skip, clip_skip)
         seed.change(on_change_seed_value, seed)
         seed_checkbox.change(on_change_seed_checkbox, seed_checkbox)
         safety_checker_checkbox.change(