Duplicate from GV05/stable-diffusion-mingle-prompts

Co-authored-by: Noam Shimshoviz <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+batman_venum.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Stable Diffusion Mingle Prompts
+emoji: 🦀
+colorFrom: red
+colorTo: pink
+sdk: gradio
+sdk_version: 3.7
+app_file: app.py
+pinned: false
+license: apache-2.0
+duplicated_from: GV05/stable-diffusion-mingle-prompts
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Utils.py

@@ -0,0 +1,88 @@
+from diffusers import AutoencoderKL, UNet2DConditionModel, LMSDiscreteScheduler
+from transformers import CLIPTextModel, CLIPTokenizer
+from tqdm.auto import tqdm
+from PIL import Image
+import torch
+
+class MingleModel:
+
+    def __init__(self):
+        # Set device
+        self.torch_device = "cuda" if torch.cuda.is_available() else "cpu"
+        # Load the autoencoder model which will be used to decode the latents into image space.
+        use_auth_token = "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"
+        self.vae = AutoencoderKL.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="vae",
+                                            use_auth_token=use_auth_token).to(self.torch_device)
+
+        # Load the tokenizer and text encoder to tokenize and encode the text.
+        self.tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14", use_auth_token=use_auth_token)
+        self.text_encoder = CLIPTextModel.from_pretrained("openai/clip-vit-large-patch14", use_auth_token=use_auth_token).to(self.torch_device)
+
+        # # The UNet model for generating the latents.
+        self.unet = UNet2DConditionModel.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="unet",use_auth_token=use_auth_token).to(self.torch_device)
+
+        # The noise scheduler
+        self.scheduler = LMSDiscreteScheduler(beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear",
+                                         num_train_timesteps=1000)
+
+    def do_tokenizer(self, prompt):
+        return self.tokenizer([prompt], padding="max_length", max_length=self.tokenizer.model_max_length, truncation=True,
+                  return_tensors="pt")
+
+    def get_text_encoder(self, text_input):
+        return self.text_encoder(text_input.input_ids.to(self.torch_device))[0]
+
+    def latents_to_pil(self, latents):
+        # bath of latents -> list of images
+        latents = (1 / 0.18215) * latents
+        with torch.no_grad():
+            image = self.vae.decode(latents).sample
+        image = (image / 2 + 0.5).clamp(0, 1)
+        image = image.detach().cpu().permute(0, 2, 3, 1).numpy()
+        images = (image * 255).round().astype("uint8")
+        pil_images = [Image.fromarray(image) for image in images]
+        return pil_images
+
+    def generate_with_embs(self, text_embeddings, generator_int=32, num_inference_steps=30, guidance_scale=7.5):
+        height = 512  # default height of Stable Diffusion
+        width = 512  # default width of Stable Diffusion
+        num_inference_steps = num_inference_steps  # Number of denoising steps
+        guidance_scale = guidance_scale  # Scale for classifier-free guidance
+        generator = torch.manual_seed(generator_int)  # Seed generator to create the inital latent noise
+        batch_size = 1
+
+        max_length = 77
+        uncond_input = self.tokenizer(
+            [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="pt"
+        )
+        with torch.no_grad():
+            uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.torch_device))[0]
+        text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
+
+        # Prep Scheduler
+        self.scheduler.set_timesteps(num_inference_steps)
+
+        # Prep latents
+        latents = torch.randn((batch_size, self.unet.in_channels, height // 8, width // 8), generator=generator)
+        latents = latents.to(self.torch_device)
+        latents = latents * self.scheduler.init_noise_sigma
+
+        # Loop
+        for i, t in tqdm(enumerate(self.scheduler.timesteps)):
+            # expand the latents if we are doing classifier-free guidance to avoid doing two forward passes.
+            latent_model_input = torch.cat([latents] * 2)
+            sigma = self.scheduler.sigmas[i]
+            latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
+
+            # predict the noise residual
+            with torch.no_grad():
+                noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings)["sample"]
+
+            # perform guidance
+            noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+            noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
+
+            # compute the previous noisy sample x_t -> x_t-1
+            latents = self.scheduler.step(noise_pred, t, latents).prev_sample
+
+        return self.latents_to_pil(latents)[0]

app.py

@@ -0,0 +1,74 @@
+import gradio as gr
+import torch
+from transformers import logging
+import random
+from PIL import Image
+from Utils import MingleModel
+
+logging.set_verbosity_error()
+
+
+def get_concat_h(images):
+    widths, heights = zip(*(i.size for i in images))
+
+    total_width = sum(widths)
+    max_height = max(heights)
+
+    dst = Image.new('RGB', (total_width, max_height))
+    x_offset = 0
+    for im in images:
+      dst.paste(im, (x_offset,0))
+      x_offset += im.size[0]
+    return dst
+
+
+mingle_model = MingleModel()
+
+
+def mingle_prompts(first_prompt, second_prompt):
+    imgs = []
+    text_input1 = mingle_model.do_tokenizer(first_prompt)
+    text_input2 = mingle_model.do_tokenizer(second_prompt)
+    with torch.no_grad():
+        text_embeddings1 = mingle_model.get_text_encoder(text_input1)
+        text_embeddings2 = mingle_model.get_text_encoder(text_input2)
+
+    rand_generator = random.randint(1, 2048)
+    # Mix them together
+    # mix_factors = [0.1, 0.3, 0.5, 0.7, 0.9]
+    mix_factors = [0.5]
+    for mix_factor in mix_factors:
+        mixed_embeddings = (text_embeddings1 * mix_factor + text_embeddings2 * (1 - mix_factor))
+
+        # Generate!
+        steps = 20
+        guidence_scale = 8.0
+        img = mingle_model.generate_with_embs(mixed_embeddings, rand_generator, num_inference_steps=steps,
+                                 guidance_scale=guidence_scale)
+        imgs.append(img)
+
+    return get_concat_h(imgs)
+
+
+with gr.Blocks() as demo:
+    gr.Markdown(
+        '''
+        <p style="text-align: center;">create a 'chimera' by averaging the embeddings of two different prompts!!</p>
+        ''')
+    gr.Image('batman_venum.png', shape=(1024, 205))
+
+    first_prompt = gr.Textbox(label="first_prompt")
+    second_prompt = gr.Textbox(label="second_prompt")
+    greet_btn = gr.Button("Submit")
+    gr.Markdown("## Text Examples")
+    gr.Examples([['batman, dynamic lighting, photorealistic fantasy concept art, trending on art station, stunning visuals, terrifying, creative, cinematic',
+                  'venom, dynamic lighting, photorealistic fantasy concept art, trending on art station, stunning visuals, terrifying, creative, cinematic'],
+                 ['A mouse', 'A leopard']], [first_prompt, second_prompt])
+
+    gr.Markdown("# Output Results")
+    output = gr.Image(shape=(512,512))
+
+    greet_btn.click(fn=mingle_prompts, inputs=[first_prompt, second_prompt], outputs=[output])
+
+demo.launch()
+

requirements.txt

@@ -0,0 +1,6 @@
+transformers
+diffusers
+ftfy
+torch
+gradio
+scipy