Refine manual seed (#877)

This PR is done:

- [x] Refine 测试脚本，在 warmup 前固定 seed，以及在正式生成图像前固定 seed。确保 warmup
阶段结果一致，最终生成的图像也保持一致可重复。

---------

Co-authored-by: Xiaoyu Xu <xiaoyulink@gmail.com>

.github/workflows/examples.yml

@@ -239,6 +239,7 @@ jobs:
       #   run: docker exec -w /src/onediff/onediff_diffusers_extensions -e ONEFLOW_MLIR_ENABLE_INFERENCE_OPTIMIZATION=0 ${{ env.CONTAINER_NAME }} python3 examples/text_to_image_sdxl_reuse_pipe.py --base /share_nfs/hf_models/stable-diffusion-xl-base-1.0 --new_base /share_nfs/hf_models/dataautogpt3-OpenDalleV1.1
       - if: matrix.test-suite == 'diffusers_examples' && startsWith(matrix.image, 'onediff-pro')
         run: |
+          docker exec -w /src/onediff ${{ env.CONTAINER_NAME }} python3 onediff_diffusers_extensions/examples/text_to_image_sd_enterprise.py  --model /share_nfs/hf_models/stable-diffusion-v1-5-int8 --width 512 --height 512 --saved_image /src/onediff/output_enterprise_sd.png
           docker exec -w /src/onediff ${{ env.CONTAINER_NAME }} python3 tests/test_quantitative_quality.py
       
       - name: Shutdown docker for ComfyUI Test

onediff_diffusers_extensions/examples/text_to_image.py

@@ -46,12 +46,12 @@ def parse_args():
 prompt = args.prompt
 with flow.autocast("cuda"):
     for _ in range(args.warmup):
+        torch.manual_seed(args.seed)
         images = pipe(
             prompt, height=args.height, width=args.width, num_inference_steps=args.steps
         ).images
 
     torch.manual_seed(args.seed)
-
     images = pipe(
         prompt, height=args.height, width=args.width, num_inference_steps=args.steps
     ).images

onediff_diffusers_extensions/examples/text_to_image_deep_cache_sd.py

@@ -68,6 +68,7 @@
 # Warmup with chosen resolutions
 for resolution in resolutions:
     for i in range(args.warmup):
+        torch.manual_seed(args.seed)
         image = base(
             prompt=args.prompt,
             height=resolution[0],

onediff_diffusers_extensions/examples/text_to_image_deep_cache_sd_sdxl_enterprise.py

@@ -134,8 +134,6 @@ def parse_args():
         pipe.upcast_vae()
     pipe.vae.decoder = oneflow_compile(pipe.vae.decoder, options=compile_options)
 
-torch.manual_seed(args.seed)
-
 if args.load_graph:
     print("Loading graphs to avoid compilation...")
     start_t = time.time()
@@ -146,18 +144,21 @@ def parse_args():
     print(f"warmup with loading graph elapsed: {end_t - start_t} s")
     start_t = time.time()
     for _ in range(args.warmup):
+        torch.manual_seed(args.seed)
         image = pipe(**infer_args).images[0]
     end_t = time.time()
     print(f"warmup with run elapsed: {end_t - start_t} s")
 else:
     start_t = time.time()
     for _ in range(args.warmup):
+        torch.manual_seed(args.seed)
         image = pipe(**infer_args).images[0]
     end_t = time.time()
     print(f"warmup with run elapsed: {end_t - start_t} s")
 
 start_t = time.time()
 
+torch.manual_seed(args.seed)
 torch.cuda.cudart().cudaProfilerStart()
 image = pipe(**infer_args).images[0]
 torch.cuda.cudart().cudaProfilerStop()

onediff_diffusers_extensions/examples/text_to_image_lcm.py

@@ -65,6 +65,7 @@ def parse_args():
     pipe.unet = oneflow_compile(pipe.unet)
 
 for _ in range(args.warmup):
+    torch.manual_seed(args.seed)
     images = pipe(
         args.prompt,
         height=args.height,
@@ -73,7 +74,6 @@ def parse_args():
     ).images
 
 torch.manual_seed(args.seed)
-
 images = pipe(
     args.prompt, height=args.height, width=args.width, num_inference_steps=args.steps
 ).images

onediff_diffusers_extensions/examples/text_to_image_lcm_lora_sdxl.py

@@ -82,6 +82,7 @@ def parse_args():
     exit(1)
 
 for _ in range(args.warmup):
+    torch.manual_seed(args.seed)
     images = pipe(
         args.prompt,
         height=args.height,
@@ -90,7 +91,6 @@ def parse_args():
     ).images
 
 torch.manual_seed(args.seed)
-
 images = pipe(
     args.prompt, height=args.height, width=args.width, num_inference_steps=args.steps
 ).images

onediff_diffusers_extensions/examples/text_to_image_online_quant.py

@@ -108,7 +108,7 @@ def main():
                                     "linear_mae_threshold": args.linear_mae_threshold,
                                     "conv_compute_density_threshold": args.conv_compute_density_threshold,
                                     "linear_compute_density_threshold": args.linear_compute_density_threshold})
-
+    torch.manual_seed(args.seed)
     # Warm-up
     pipe(prompt=args.prompt, num_inference_steps=1)
 

onediff_diffusers_extensions/examples/text_to_image_sd_enterprise.py

@@ -102,7 +102,6 @@ def parse_args():
     pipe.unet = oneflow_compile(pipe.unet, options=compile_options)
     pipe.vae.decoder = oneflow_compile(pipe.vae.decoder, options=compile_options)
 
-torch.manual_seed(args.seed)
 
 if args.load_graph:
     print("Loading graphs to avoid compilation...")
@@ -113,18 +112,21 @@ def parse_args():
     print(f"warmup with loading graph elapsed: {end_t - start_t} s")
     start_t = time.time()
     for _ in range(args.warmup):
+        torch.manual_seed(args.seed)
         image = pipe(**infer_args).images[0]
     end_t = time.time()
     print(f"warmup with run elapsed: {end_t - start_t} s")
 else:
     start_t = time.time()
     for _ in range(args.warmup):
+        torch.manual_seed(args.seed)
         image = pipe(**infer_args).images[0]
     end_t = time.time()
     print(f"warmup with run elapsed: {end_t - start_t} s")
 
 start_t = time.time()
 
+torch.manual_seed(args.seed)
 torch.cuda.cudart().cudaProfilerStart()
 image = pipe(**infer_args).images[0]
 torch.cuda.cudart().cudaProfilerStop()

onediff_diffusers_extensions/examples/text_to_image_sdxl_enterprise.py

@@ -104,7 +104,6 @@ def parse_args():
     pipe.unet = oneflow_compile(pipe.unet, options=compile_options)
     pipe.vae.decoder = oneflow_compile(pipe.vae.decoder, options=compile_options)
 
-torch.manual_seed(args.seed)
 
 if args.load_graph:
     print("Loading graphs to avoid compilation...")
@@ -115,18 +114,21 @@ def parse_args():
     print(f"warmup with loading graph elapsed: {end_t - start_t} s")
     start_t = time.time()
     for _ in range(args.warmup):
+        torch.manual_seed(args.seed)
         image = pipe(**infer_args).images[0]
     end_t = time.time()
     print(f"warmup with run elapsed: {end_t - start_t} s")
 else:
     start_t = time.time()
     for _ in range(args.warmup):
+        torch.manual_seed(args.seed)
         image = pipe(**infer_args).images[0]
     end_t = time.time()
     print(f"warmup with run elapsed: {end_t - start_t} s")
 
 start_t = time.time()
 
+torch.manual_seed(args.seed)
 torch.cuda.cudart().cudaProfilerStart()
 image = pipe(**infer_args).images[0]
 torch.cuda.cudart().cudaProfilerStop()

onediff_diffusers_extensions/examples/text_to_image_sdxl_reuse_pipe.py

@@ -101,6 +101,7 @@
 new_base.to("cuda")
 
 print("New base running by torch backend")
+torch.manual_seed(args.seed)
 image = new_base(
     prompt=args.prompt,
     height=args.height,
@@ -128,6 +129,7 @@
 
 # Normal SDXL run
 print("Re-use the compiled graph")
+torch.manual_seed(args.seed)
 image = new_base(
     prompt=args.prompt,
     height=args.height,

onediff_diffusers_extensions/examples/text_to_image_sdxl_turbo.py

@@ -49,6 +49,7 @@
 
 # Warmup
 for i in range(args.warmup):
+    torch.manual_seed(args.seed)
     image = base(
         prompt=args.prompt,
         height=args.height,
@@ -60,7 +61,6 @@
 
 # Normal SDXL turbo run
 torch.manual_seed(args.seed)
-
 start_t = time.time()
 
 image = base(

tests/test_quantitative_quality.py

@@ -3,7 +3,7 @@
 import numpy as np
 import unittest
 
-class  QuantizeQuality(unittest.TestCase):
+class QuantizeQuality(unittest.TestCase):
 
     def test_validate(self):
         image1 = np.array(Image.open('/share_nfs/civitai/20240407-163408.jpg').convert('RGB'))