train_controlnet

[Mandarin1126]

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler
from diffusers.utils import load_image
import torch

base_model_path = "stable-diffusion-v1-5/stable-diffusion-v1-5"
controlnet_path = "./output/checkpoint-12500/controlnet/"

controlnet = ControlNetModel.from_pretrained(controlnet_path, torch_dtype=torch.float16)
pipe = StableDiffusionControlNetPipeline.from_pretrained(
base_model_path, controlnet=controlnet, torch_dtype=torch.float16
)

speed up diffusion process with faster scheduler and memory optimization

pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)

remove following line if xformers is not installed or when using Torch 2.0.

pipe.enable_xformers_memory_efficient_attention()

memory optimization.

pipe.enable_model_cpu_offload()

control_image = load_image("./conditioning_image_1.png")
prompt = "pale golden rod circle with old lace background"

generate image

generator = torch.manual_seed(0)
image = pipe(
prompt, num_inference_steps=20, generator=generator, image=control_image
).images[0]
image.save("./output.png")

Error when using the official inference script
RuntimeError: Data processing error: CAS service error : Reqwest Error: HTTP status client error (401 Unauthorized), domain: https://cas-server.xethub.hf.co/reconstructions/3e6084a7831fd989ab9e7a6a88745fce55823b69b6ea6832ff6c5cc08fee958f