From c86b296ee27cc63ea92078cf8b4a6604f067e07a Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 04:26:15 -0600 Subject: [PATCH 01/11] fuxes --- .../community/pipeline_stg_hunyuan_video.py | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/examples/community/pipeline_stg_hunyuan_video.py b/examples/community/pipeline_stg_hunyuan_video.py index e41f99e13a22..6525a1466457 100644 --- a/examples/community/pipeline_stg_hunyuan_video.py +++ b/examples/community/pipeline_stg_hunyuan_video.py @@ -189,11 +189,14 @@ def retrieve_timesteps( elif sigmas is not None: accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) if not accept_sigmas: - raise ValueError( + print( f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" - f" sigmas schedules. Please check whether you are using the correct scheduler." + f" sigmas schedules. Please check whether you are using the correct scheduler. The pipeline" + f" will continue without setting sigma values" ) - scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + scheduler.set_timesteps(num_inference_steps, device=device) + else: + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) timesteps = scheduler.timesteps num_inference_steps = len(timesteps) else: @@ -651,6 +654,12 @@ def __call__( self._attention_kwargs = attention_kwargs self._current_timestep = None self._interrupt = False + + if self.do_spatio_temporal_guidance: + for i in stg_applied_layers_idx: + self.transformer.transformer_blocks[i].forward = types.MethodType( + forward_with_stg, self.transformer.transformer_blocks[i] + ) device = self._execution_device @@ -722,12 +731,6 @@ def __call__( # broadcast to batch dimension in a way that's compatible with ONNX/Core ML timestep = t.expand(latents.shape[0]).to(latents.dtype) - if self.do_spatio_temporal_guidance: - for i in stg_applied_layers_idx: - self.transformer.transformer_blocks[i].forward = types.MethodType( - forward_without_stg, self.transformer.transformer_blocks[i] - ) - noise_pred = self.transformer( hidden_states=latent_model_input, timestep=timestep, @@ -740,11 +743,6 @@ def __call__( )[0] if self.do_spatio_temporal_guidance: - for i in stg_applied_layers_idx: - self.transformer.transformer_blocks[i].forward = types.MethodType( - forward_with_stg, self.transformer.transformer_blocks[i] - ) - noise_pred_perturb = self.transformer( hidden_states=latent_model_input, timestep=timestep, From d5932aad4231ff09abf8af773021ac8cf98ed3ac Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 04:28:37 -0600 Subject: [PATCH 02/11] Make it a warning --- .../pipelines/hunyuan_video/pipeline_hunyuan_video.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py index 3cb91b3782f2..b0ca0b5e91a0 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py @@ -128,11 +128,14 @@ def retrieve_timesteps( elif sigmas is not None: accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) if not accept_sigmas: - raise ValueError( + print( f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" - f" sigmas schedules. Please check whether you are using the correct scheduler." + f" sigmas schedules. Please check whether you are using the correct scheduler. The pipeline" + f" will continue without setting sigma values" ) - scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + scheduler.set_timesteps(num_inference_steps, device=device) + else: + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) timesteps = scheduler.timesteps num_inference_steps = len(timesteps) else: From 1385e3ebc82ad042b42d9704575416f04e5dc903 Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 04:28:53 -0600 Subject: [PATCH 03/11] here too --- .../pipeline_hunyuan_skyreels_image2video.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py index 297d2a9c9396..1aef4fdac169 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py @@ -139,11 +139,14 @@ def retrieve_timesteps( elif sigmas is not None: accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) if not accept_sigmas: - raise ValueError( + print( f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" - f" sigmas schedules. Please check whether you are using the correct scheduler." + f" sigmas schedules. Please check whether you are using the correct scheduler. The pipeline" + f" will continue without setting sigma values" ) - scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + scheduler.set_timesteps(num_inference_steps, device=device) + else: + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) timesteps = scheduler.timesteps num_inference_steps = len(timesteps) else: From 0a2bf69862fc8602b772eef5b0c40460961427d5 Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 04:29:59 -0600 Subject: [PATCH 04/11] also here --- .../hunyuan_video/pipeline_hunyuan_video_image2video.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py index 5a600dda4326..8dc86f2f9c10 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py @@ -141,11 +141,14 @@ def retrieve_timesteps( elif sigmas is not None: accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) if not accept_sigmas: - raise ValueError( + print( f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" - f" sigmas schedules. Please check whether you are using the correct scheduler." + f" sigmas schedules. Please check whether you are using the correct scheduler. The pipeline" + f" will continue without setting sigma values" ) - scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + scheduler.set_timesteps(num_inference_steps, device=device) + else: + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) timesteps = scheduler.timesteps num_inference_steps = len(timesteps) else: From 72bb72d8d091284a3ed458544bc49556473156fe Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 16:30:37 -0600 Subject: [PATCH 05/11] Use logger Co-authored-by: hlky --- examples/community/pipeline_stg_hunyuan_video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/community/pipeline_stg_hunyuan_video.py b/examples/community/pipeline_stg_hunyuan_video.py index 6525a1466457..8e710d61fb72 100644 --- a/examples/community/pipeline_stg_hunyuan_video.py +++ b/examples/community/pipeline_stg_hunyuan_video.py @@ -189,7 +189,7 @@ def retrieve_timesteps( elif sigmas is not None: accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) if not accept_sigmas: - print( + logger.warning( f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" f" sigmas schedules. Please check whether you are using the correct scheduler. The pipeline" f" will continue without setting sigma values" From 1d00314ea06f77dbce5292242b4884b3e1ff135f Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 16:32:25 -0600 Subject: [PATCH 06/11] Use logger in skyreels --- .../hunyuan_video/pipeline_hunyuan_skyreels_image2video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py index 1aef4fdac169..7c2a958f8be6 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py @@ -139,7 +139,7 @@ def retrieve_timesteps( elif sigmas is not None: accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) if not accept_sigmas: - print( + logger.warning( f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" f" sigmas schedules. Please check whether you are using the correct scheduler. The pipeline" f" will continue without setting sigma values" From 2dbb4ce83ba518373dc73ea44d8e18c4ff7a93b4 Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 16:33:13 -0600 Subject: [PATCH 07/11] Use logger in img2vid --- .../hunyuan_video/pipeline_hunyuan_video_image2video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py index 8dc86f2f9c10..c3b1a8c25cd9 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py @@ -141,7 +141,7 @@ def retrieve_timesteps( elif sigmas is not None: accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) if not accept_sigmas: - print( + logger.warning( f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" f" sigmas schedules. Please check whether you are using the correct scheduler. The pipeline" f" will continue without setting sigma values" From 027167f0c2a55cb28c36052ce1e815948b991382 Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 16:33:33 -0600 Subject: [PATCH 08/11] Use logger in base --- src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py index b0ca0b5e91a0..68aa81ee8f3c 100644 --- a/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py +++ b/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py @@ -128,7 +128,7 @@ def retrieve_timesteps( elif sigmas is not None: accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) if not accept_sigmas: - print( + logger.warning( f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" f" sigmas schedules. Please check whether you are using the correct scheduler. The pipeline" f" will continue without setting sigma values" From bc16a5b70836ea032ba126bff98f4c0d68ff3d85 Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 16:42:15 -0600 Subject: [PATCH 09/11] Revert stg change but change index --- .../community/pipeline_stg_hunyuan_video.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/examples/community/pipeline_stg_hunyuan_video.py b/examples/community/pipeline_stg_hunyuan_video.py index 8e710d61fb72..368f4d6345bd 100644 --- a/examples/community/pipeline_stg_hunyuan_video.py +++ b/examples/community/pipeline_stg_hunyuan_video.py @@ -654,12 +654,6 @@ def __call__( self._attention_kwargs = attention_kwargs self._current_timestep = None self._interrupt = False - - if self.do_spatio_temporal_guidance: - for i in stg_applied_layers_idx: - self.transformer.transformer_blocks[i].forward = types.MethodType( - forward_with_stg, self.transformer.transformer_blocks[i] - ) device = self._execution_device @@ -730,6 +724,12 @@ def __call__( latent_model_input = latents.to(transformer_dtype) # broadcast to batch dimension in a way that's compatible with ONNX/Core ML timestep = t.expand(latents.shape[0]).to(latents.dtype) + + if self.do_spatio_temporal_guidance: + for stg_idx in stg_applied_layers_idx: + self.transformer.transformer_blocks[i].forward = types.MethodType( + forward_without_stg, self.transformer.transformer_blocks[stg_idx] + ) noise_pred = self.transformer( hidden_states=latent_model_input, @@ -743,6 +743,11 @@ def __call__( )[0] if self.do_spatio_temporal_guidance: + for stg_idx in stg_applied_layers_idx: + self.transformer.transformer_blocks[i].forward = types.MethodType( + forward_with_stg, self.transformer.transformer_blocks[stg_idx] + ) + noise_pred_perturb = self.transformer( hidden_states=latent_model_input, timestep=timestep, From 9bb2de83ed77773c9605e2dc1f5592a4406252f7 Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 16:43:32 -0600 Subject: [PATCH 10/11] Maintain whitespace --- examples/community/pipeline_stg_hunyuan_video.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/community/pipeline_stg_hunyuan_video.py b/examples/community/pipeline_stg_hunyuan_video.py index 368f4d6345bd..cbe0ddc3810d 100644 --- a/examples/community/pipeline_stg_hunyuan_video.py +++ b/examples/community/pipeline_stg_hunyuan_video.py @@ -724,7 +724,7 @@ def __call__( latent_model_input = latents.to(transformer_dtype) # broadcast to batch dimension in a way that's compatible with ONNX/Core ML timestep = t.expand(latents.shape[0]).to(latents.dtype) - + if self.do_spatio_temporal_guidance: for stg_idx in stg_applied_layers_idx: self.transformer.transformer_blocks[i].forward = types.MethodType( @@ -747,7 +747,7 @@ def __call__( self.transformer.transformer_blocks[i].forward = types.MethodType( forward_with_stg, self.transformer.transformer_blocks[stg_idx] ) - + noise_pred_perturb = self.transformer( hidden_states=latent_model_input, timestep=timestep, From 6f61d297e7d238de907c2837a5a18851106d8672 Mon Sep 17 00:00:00 2001 From: Edna <88869424+Ednaordinary@users.noreply.github.com> Date: Sun, 23 Mar 2025 17:56:41 -0600 Subject: [PATCH 11/11] more fixes --- examples/community/pipeline_stg_hunyuan_video.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/community/pipeline_stg_hunyuan_video.py b/examples/community/pipeline_stg_hunyuan_video.py index cbe0ddc3810d..06c3acd5d107 100644 --- a/examples/community/pipeline_stg_hunyuan_video.py +++ b/examples/community/pipeline_stg_hunyuan_video.py @@ -194,7 +194,7 @@ def retrieve_timesteps( f" sigmas schedules. Please check whether you are using the correct scheduler. The pipeline" f" will continue without setting sigma values" ) - scheduler.set_timesteps(num_inference_steps, device=device) + scheduler.set_timesteps(num_inference_steps=num_inference_steps, device=device, **kwargs) else: scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) timesteps = scheduler.timesteps @@ -727,7 +727,7 @@ def __call__( if self.do_spatio_temporal_guidance: for stg_idx in stg_applied_layers_idx: - self.transformer.transformer_blocks[i].forward = types.MethodType( + self.transformer.transformer_blocks[stg_idx].forward = types.MethodType( forward_without_stg, self.transformer.transformer_blocks[stg_idx] ) @@ -744,7 +744,7 @@ def __call__( if self.do_spatio_temporal_guidance: for stg_idx in stg_applied_layers_idx: - self.transformer.transformer_blocks[i].forward = types.MethodType( + self.transformer.transformer_blocks[stg_idx].forward = types.MethodType( forward_with_stg, self.transformer.transformer_blocks[stg_idx] )