[ppdiffusers] support EulerAncestralDiscreteScheduler

[JunnYu]

PR types

New features

PR changes

APIs

Description

EulerAncestralDiscreteScheduler对齐

from ppdiffusers import EulerAncestralDiscreteScheduler as PPEulerAncestralDiscreteScheduler
from diffusers import EulerAncestralDiscreteScheduler as HFEulerAncestralDiscreteScheduler
import torch
import paddle
import random

init_kwargs = dict(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")
timestep = 1000
device = "cuda"
batch_size = 16

ppeuler = PPEulerAncestralDiscreteScheduler(**init_kwargs)
hfeuler = HFEulerAncestralDiscreteScheduler(**init_kwargs)
def compare(a, b):
    dif = a.cpu().numpy() - b.cpu().numpy()
    meandif, maxdif = abs(dif).mean(), abs(dif).max()
    print(meandif, maxdif)

# test set_timesteps
ppeuler.set_timesteps(timestep)
hfeuler.set_timesteps(timestep, device=device)
compare(ppeuler.sigmas, ppeuler.sigmas)
compare(ppeuler.timesteps, hfeuler.timesteps)

# test add_noise
hflatents = torch.randn((batch_size, 4, 64, 64)).to(device)
hfnoise = torch.randn((batch_size, 4, 64, 64)).to(device)
hftimesteps = torch.randint(0, timestep, (batch_size, )).long().to(device)
pplatents = paddle.to_tensor(hflatents.cpu().numpy())
ppnoise = paddle.to_tensor(hfnoise.cpu().numpy())
pptimesteps = paddle.to_tensor(hftimesteps.cpu().numpy())

hfnoisy_latents = hfeuler.add_noise(hflatents, hfnoise, hftimesteps)
ppnoisy_latents = ppeuler.add_noise(pplatents, ppnoise, pptimesteps)

compare(hfnoisy_latents, ppnoisy_latents)

t = random.randint(0, 1000)
hft = torch.tensor(t, device=device)
ppt = paddle.to_tensor(t)
# test scale_model_input
hfscale_model_input = hfeuler.scale_model_input(hflatents, hft)
ppscale_model_input = ppeuler.scale_model_input(pplatents, ppt)

compare(hfscale_model_input, ppscale_model_input)

# test step
seed = random.randint(0, 2**32)
generator = torch.Generator(device).manual_seed(seed)
paddle.seed(seed)
hfstep = hfeuler.step(hfnoise, hft, hflatents, generator=generator)
ppstep = ppeuler.step(ppnoise, ppt, pplatents)
compare(hfstep[0], ppstep[0])

# 0.0 0.0
# 0.0 0.0
# 4.1596826e-07 7.6293945e-06
# 0.0 0.0
# 8.9845884e-08 7.1525574e-07

runwayml/stable-diffusion-v1-5 及 runwayml/stable-diffusion-inpainting 权重转换及前向对齐 （Done）

import torch
torch.set_grad_enabled(False)
from diffusers import StableDiffusionPipeline as HFStableDiffusionPipeline
import paddle
paddle.set_grad_enabled(False)
from ppdiffusers import StableDiffusionPipeline as PPStableDiffusionPipeline
def compare(a, b):
    dif = a.cpu().numpy() - b.cpu().numpy()
    meandif, maxdif = abs(dif).mean(), abs(dif).max()
    print(meandif, maxdif)
device = "cuda"

hf_pipe = HFStableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5").to(device)
pp_pipe = PPStableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")

######## 1. vae
batch_size = 1
# encode
seed = 42
generator = torch.Generator(device).manual_seed(seed)
hfpixel_values = torch.randn((batch_size, 3, 224, 224), device=device, generator=generator)
hf_out = hf_pipe.vae.encode(hfpixel_values).latent_dist.sample(generator=generator)
paddle.seed(seed)
pppixel_values = paddle.randn((batch_size, 3, 224, 224))
pp_out = pp_pipe.vae.encode(pppixel_values).latent_dist.sample()
compare(hf_out, pp_out)

# decode
seed = 42
generator = torch.Generator(device).manual_seed(seed)
hfpixel_values = torch.randn((batch_size, 4, 64, 64), device=device, generator=generator)
hf_out = hf_pipe.vae.decode(hfpixel_values).sample
paddle.seed(seed)
pppixel_values = paddle.randn((batch_size, 4, 64, 64))
pp_out = pp_pipe.vae.decode(pppixel_values).sample
compare(hf_out, pp_out)

######## 2. unet
hflatents = torch.randn((batch_size, 4, 64, 64)).to(device)
hftimesteps = torch.randint(0, 1000, (batch_size, )).long().to(device)
hfembeddings = torch.randn((batch_size, 77, 768)).to(device)
pplatents = paddle.to_tensor(hflatents.cpu().numpy())
pptimesteps = paddle.to_tensor(hftimesteps.cpu().numpy())
ppembeddings = paddle.to_tensor(hfembeddings.cpu().numpy())
hf_out = hf_pipe.unet(hflatents, hftimesteps, encoder_hidden_states=hfembeddings)[0]
pp_out = pp_pipe.unet(pplatents, pptimesteps, encoder_hidden_states=ppembeddings)[0]
compare(hf_out, pp_out)


######## 3. text_encoder
hfinput_ids = torch.randint(4, 1000, (batch_size, 77), device=device).long()
ppinput_ids = paddle.to_tensor(hfinput_ids.cpu().numpy())
hf_out = hf_pipe.text_encoder(hfinput_ids)[0]
pp_out = pp_pipe.text_encoder(ppinput_ids)[0]
compare(hf_out, pp_out)
# runwayml/stable-diffusion-v1-5
# 2.0066313e-06 2.9563904e-05
# 1.4244446e-07 8.6426735e-06
# 1.533682e-07 1.0430813e-06
# 1.2732149e-06 1.1920929e-05

# runwayml/stable-diffusion-inpainting
# 2.0066313e-06 2.9563904e-05
# 1.4244446e-07 8.6426735e-06
# 4.6644558e-07 6.6310167e-06
# 1.3243238e-06 1.2397766e-05

runwayml/stable-diffusion-inpainting Inpaint 任务对齐 （Done）

from io import BytesIO
import torch
import requests
import PIL
from diffusers import StableDiffusionInpaintPipeline, EulerAncestralDiscreteScheduler

def download_image(url):
    response = requests.get(url)
    return PIL.Image.open(BytesIO(response.content)).convert("RGB")

device = "cuda"
img_url = "https://paddlenlp.bj.bcebos.com/models/community/CompVis/stable-diffusion-v1-4/overture-creations.png"
mask_url = "https://paddlenlp.bj.bcebos.com/models/community/CompVis/stable-diffusion-v1-4/overture-creations-mask.png"
init_image = download_image(img_url).resize((512, 512))
mask_image = download_image(mask_url).resize((512, 512))

scheduler = EulerAncestralDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")
pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting", scheduler=scheduler).to(device)

seed = 2022
generator = torch.Generator(device).manual_seed(seed)
prompt = "Face of a yellow cat, high resolution, sitting on a park bench"
hfimage = pipe(prompt=prompt, image=init_image, mask_image=mask_image, generator=generator).images[0]
hfimage.save("hf.jpg")

import paddle
import PIL
import requests
from io import BytesIO

from ppdiffusers import StableDiffusionInpaintPipeline, EulerAncestralDiscreteScheduler

def download_image(url):
    response = requests.get(url)
    return PIL.Image.open(BytesIO(response.content)).convert("RGB")

img_url = "https://paddlenlp.bj.bcebos.com/models/community/CompVis/stable-diffusion-v1-4/overture-creations.png"
mask_url = "https://paddlenlp.bj.bcebos.com/models/community/CompVis/stable-diffusion-v1-4/overture-creations-mask.png"
init_image = download_image(img_url).resize((512, 512))
mask_image = download_image(mask_url).resize((512, 512))

scheduler = EulerAncestralDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear")
pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting", scheduler=scheduler)

seed = 2022
prompt = "Face of a yellow cat, high resolution, sitting on a park bench"
ppimage = pipe(prompt=prompt, image=init_image, mask_image=mask_image, seed=seed).images[0]
ppimage.save("pp.jpg")

[guoshengCS]

LGTM