|
import subprocess |
|
subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True) |
|
|
|
from huggingface_hub import snapshot_download, hf_hub_download |
|
|
|
snapshot_download( |
|
repo_id="Wan-AI/Wan2.1-T2V-1.3B", |
|
local_dir="wan_models/Wan2.1-T2V-1.3B", |
|
local_dir_use_symlinks=False, |
|
resume_download=True, |
|
repo_type="model" |
|
) |
|
|
|
hf_hub_download( |
|
repo_id="gdhe17/Self-Forcing", |
|
filename="checkpoints/self_forcing_dmd.pt", |
|
local_dir=".", |
|
local_dir_use_symlinks=False |
|
) |
|
|
|
import os |
|
import re |
|
import random |
|
import argparse |
|
import hashlib |
|
import urllib.request |
|
import time |
|
from PIL import Image |
|
import spaces |
|
import numpy as np |
|
import torch |
|
import gradio as gr |
|
from omegaconf import OmegaConf |
|
from tqdm import tqdm |
|
import imageio |
|
|
|
|
|
from pipeline import CausalInferencePipeline |
|
from demo_utils.constant import ZERO_VAE_CACHE |
|
from demo_utils.vae_block3 import VAEDecoderWrapper |
|
from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder |
|
|
|
from transformers import pipeline, AutoTokenizer, AutoModelForCausalLM |
|
|
|
device = "cuda" if torch.cuda.is_available() else "cpu" |
|
|
|
model_checkpoint = "meta-llama/Meta-Llama-3-8B-Instruct" |
|
|
|
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint) |
|
|
|
quantization_config = BitsAndBytesConfig( |
|
load_in_4bit=True, |
|
bnb_4bit_compute_dtype=torch.bfloat16, |
|
bnb_4bit_quant_type="nf4", |
|
bnb_4bit_use_double_quant=True, |
|
) |
|
|
|
model = AutoModelForCausalLM.from_pretrained( |
|
model_checkpoint, |
|
torch_dtype=torch.bfloat16, |
|
attn_implementation="flash_attention_2", |
|
quantization_config=quantization_config, |
|
device_map="auto" |
|
) |
|
enhancer = pipeline( |
|
'text-generation', |
|
model=model, |
|
tokenizer=tokenizer, |
|
repetition_penalty=1.2, |
|
) |
|
|
|
T2V_CINEMATIC_PROMPT = """You are an expert cinematic director with many award winning movies, When writing prompts based on the user input, focus on detailed, chronological descriptions of actions and scenes. |
|
Include specific movements, appearances, camera angles, and environmental details - all in a single flowing paragraph. |
|
Start directly with the action, and keep descriptions literal and precise. |
|
Think like a cinematographer describing a shot list. |
|
Do not change the user input intent, just enhance it. |
|
Keep within 150 words. |
|
For best results, build your prompts using this structure: |
|
Start with main action in a single sentence |
|
Add specific details about movements and gestures |
|
Describe character/object appearances precisely |
|
Include background and environment details |
|
Specify camera angles and movements |
|
Describe lighting and colors |
|
Note any changes or sudden events |
|
Do not exceed the 150 word limit! |
|
Output the enhanced prompt only. |
|
""" |
|
|
|
@spaces.GPU |
|
def enhance_prompt(prompt): |
|
messages = [ |
|
{"role": "system", "content": T2V_CINEMATIC_PROMPT}, |
|
{"role": "user", "content": f"user_prompt: {prompt}"}, |
|
] |
|
answer = enhancer( |
|
messages, |
|
max_new_tokens=256, |
|
return_full_text=False, |
|
pad_token_id=tokenizer.eos_token_id |
|
) |
|
|
|
final_answer = answer[0]['generated_text'] |
|
return final_answer.strip() |
|
|
|
|
|
parser = argparse.ArgumentParser(description="Gradio Demo for Self-Forcing with Frame Streaming") |
|
parser.add_argument('--port', type=int, default=7860, help="Port to run the Gradio app on.") |
|
parser.add_argument('--host', type=str, default='0.0.0.0', help="Host to bind the Gradio app to.") |
|
parser.add_argument("--checkpoint_path", type=str, default='./checkpoints/self_forcing_dmd.pt', help="Path to the model checkpoint.") |
|
parser.add_argument("--config_path", type=str, default='./configs/self_forcing_dmd.yaml', help="Path to the model config.") |
|
parser.add_argument('--share', action='store_true', help="Create a public Gradio link.") |
|
parser.add_argument('--trt', action='store_true', help="Use TensorRT optimized VAE decoder.") |
|
parser.add_argument('--fps', type=float, default=15.0, help="Playback FPS for frame streaming.") |
|
args = parser.parse_args() |
|
|
|
gpu = "cuda" |
|
|
|
try: |
|
config = OmegaConf.load(args.config_path) |
|
default_config = OmegaConf.load("configs/default_config.yaml") |
|
config = OmegaConf.merge(default_config, config) |
|
except FileNotFoundError as e: |
|
print(f"Error loading config file: {e}\n. Please ensure config files are in the correct path.") |
|
exit(1) |
|
|
|
|
|
print("Initializing models...") |
|
text_encoder = WanTextEncoder() |
|
transformer = WanDiffusionWrapper(is_causal=True) |
|
|
|
try: |
|
state_dict = torch.load(args.checkpoint_path, map_location="cpu") |
|
transformer.load_state_dict(state_dict.get('generator_ema', state_dict.get('generator'))) |
|
except FileNotFoundError as e: |
|
print(f"Error loading checkpoint: {e}\nPlease ensure the checkpoint '{args.checkpoint_path}' exists.") |
|
exit(1) |
|
|
|
text_encoder.eval().to(dtype=torch.float16).requires_grad_(False) |
|
transformer.eval().to(dtype=torch.float16).requires_grad_(False) |
|
|
|
text_encoder.to(gpu) |
|
transformer.to(gpu) |
|
|
|
APP_STATE = { |
|
"torch_compile_applied": False, |
|
"fp8_applied": False, |
|
"current_use_taehv": False, |
|
"current_vae_decoder": None, |
|
} |
|
|
|
def initialize_vae_decoder(use_taehv=False, use_trt=False): |
|
if use_trt: |
|
from demo_utils.vae import VAETRTWrapper |
|
print("Initializing TensorRT VAE Decoder...") |
|
vae_decoder = VAETRTWrapper() |
|
APP_STATE["current_use_taehv"] = False |
|
elif use_taehv: |
|
print("Initializing TAEHV VAE Decoder...") |
|
from demo_utils.taehv import TAEHV |
|
taehv_checkpoint_path = "checkpoints/taew2_1.pth" |
|
if not os.path.exists(taehv_checkpoint_path): |
|
print(f"Downloading TAEHV checkpoint to {taehv_checkpoint_path}...") |
|
os.makedirs("checkpoints", exist_ok=True) |
|
download_url = "https://github.com/madebyollin/taehv/raw/main/taew2_1.pth" |
|
try: |
|
urllib.request.urlretrieve(download_url, taehv_checkpoint_path) |
|
except Exception as e: |
|
raise RuntimeError(f"Failed to download taew2_1.pth: {e}") |
|
|
|
class DotDict(dict): __getattr__ = dict.get |
|
|
|
class TAEHVDiffusersWrapper(torch.nn.Module): |
|
def __init__(self): |
|
super().__init__() |
|
self.dtype = torch.float16 |
|
self.taehv = TAEHV(checkpoint_path=taehv_checkpoint_path).to(self.dtype) |
|
self.config = DotDict(scaling_factor=1.0) |
|
def decode(self, latents, return_dict=None): |
|
return self.taehv.decode_video(latents, parallel=not LOW_MEMORY).mul_(2).sub_(1) |
|
|
|
vae_decoder = TAEHVDiffusersWrapper() |
|
APP_STATE["current_use_taehv"] = True |
|
else: |
|
print("Initializing Default VAE Decoder...") |
|
vae_decoder = VAEDecoderWrapper() |
|
try: |
|
vae_state_dict = torch.load('wan_models/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth', map_location="cpu") |
|
decoder_state_dict = {k: v for k, v in vae_state_dict.items() if 'decoder.' in k or 'conv2' in k} |
|
vae_decoder.load_state_dict(decoder_state_dict) |
|
except FileNotFoundError: |
|
print("Warning: Default VAE weights not found.") |
|
APP_STATE["current_use_taehv"] = False |
|
|
|
vae_decoder.eval().to(dtype=torch.float16).requires_grad_(False).to(gpu) |
|
APP_STATE["current_vae_decoder"] = vae_decoder |
|
print(f"โ
VAE decoder initialized: {'TAEHV' if use_taehv else 'Default VAE'}") |
|
|
|
|
|
initialize_vae_decoder(use_taehv=False, use_trt=args.trt) |
|
|
|
pipeline = CausalInferencePipeline( |
|
config, device=gpu, generator=transformer, text_encoder=text_encoder, |
|
vae=APP_STATE["current_vae_decoder"] |
|
) |
|
|
|
pipeline.to(dtype=torch.float16).to(gpu) |
|
|
|
|
|
@torch.no_grad() |
|
@spaces.GPU |
|
def video_generation_handler(prompt, seed=42, fps=15): |
|
""" |
|
Generator function that yields RGB frames for display in gr.Image. |
|
Includes timing delays for smooth playback. |
|
""" |
|
if seed == -1: |
|
seed = random.randint(0, 2**32 - 1) |
|
|
|
print(f"๐ฌ Starting video generation with prompt: '{prompt}' and seed: {seed}") |
|
|
|
|
|
frame_delay = 1.0 / fps if fps > 0 else 1.0 / 15.0 |
|
|
|
print("๐ค Encoding text prompt...") |
|
conditional_dict = text_encoder(text_prompts=[prompt]) |
|
for key, value in conditional_dict.items(): |
|
conditional_dict[key] = value.to(dtype=torch.float16) |
|
|
|
|
|
rnd = torch.Generator(gpu).manual_seed(int(seed)) |
|
pipeline._initialize_kv_cache(1, torch.float16, device=gpu) |
|
pipeline._initialize_crossattn_cache(1, torch.float16, device=gpu) |
|
noise = torch.randn([1, 21, 16, 60, 104], device=gpu, dtype=torch.float16, generator=rnd) |
|
|
|
vae_cache, latents_cache = None, None |
|
if not APP_STATE["current_use_taehv"] and not args.trt: |
|
vae_cache = [c.to(device=gpu, dtype=torch.float16) for c in ZERO_VAE_CACHE] |
|
|
|
num_blocks = 7 |
|
current_start_frame = 0 |
|
all_num_frames = [pipeline.num_frame_per_block] * num_blocks |
|
|
|
total_frames_yielded = 0 |
|
all_frames_for_video = [] |
|
|
|
for idx, current_num_frames in enumerate(all_num_frames): |
|
print(f"๐ฆ Processing block {idx+1}/{num_blocks} with {current_num_frames} frames") |
|
|
|
noisy_input = noise[:, current_start_frame : current_start_frame + current_num_frames] |
|
|
|
for step_idx, current_timestep in enumerate(pipeline.denoising_step_list): |
|
timestep = torch.ones([1, current_num_frames], device=noise.device, dtype=torch.int64) * current_timestep |
|
_, denoised_pred = pipeline.generator( |
|
noisy_image_or_video=noisy_input, conditional_dict=conditional_dict, |
|
timestep=timestep, kv_cache=pipeline.kv_cache1, |
|
crossattn_cache=pipeline.crossattn_cache, |
|
current_start=current_start_frame * pipeline.frame_seq_length |
|
) |
|
if step_idx < len(pipeline.denoising_step_list) - 1: |
|
next_timestep = pipeline.denoising_step_list[step_idx + 1] |
|
noisy_input = pipeline.scheduler.add_noise( |
|
denoised_pred.flatten(0, 1), torch.randn_like(denoised_pred.flatten(0, 1)), |
|
next_timestep * torch.ones([1 * current_num_frames], device=noise.device, dtype=torch.long) |
|
).unflatten(0, denoised_pred.shape[:2]) |
|
|
|
if idx < len(all_num_frames) - 1: |
|
pipeline.generator( |
|
noisy_image_or_video=denoised_pred, conditional_dict=conditional_dict, |
|
timestep=torch.zeros_like(timestep), kv_cache=pipeline.kv_cache1, |
|
crossattn_cache=pipeline.crossattn_cache, |
|
current_start=current_start_frame * pipeline.frame_seq_length, |
|
) |
|
|
|
|
|
if args.trt: |
|
pixels, vae_cache = pipeline.vae.forward(denoised_pred.half(), *vae_cache) |
|
elif APP_STATE["current_use_taehv"]: |
|
if latents_cache is None: |
|
latents_cache = denoised_pred |
|
else: |
|
denoised_pred = torch.cat([latents_cache, denoised_pred], dim=1) |
|
latents_cache = denoised_pred[:, -3:] |
|
pixels = pipeline.vae.decode(denoised_pred) |
|
else: |
|
pixels, vae_cache = pipeline.vae(denoised_pred.half(), *vae_cache) |
|
|
|
|
|
if idx == 0 and not args.trt: |
|
pixels = pixels[:, 3:] |
|
elif APP_STATE["current_use_taehv"] and idx > 0: |
|
pixels = pixels[:, 12:] |
|
|
|
print(f"๐น Decoded pixels shape: {pixels.shape}") |
|
|
|
|
|
actual_frames_this_block = pixels.shape[1] |
|
|
|
|
|
for frame_idx in range(actual_frames_this_block): |
|
frame_tensor = pixels[0, frame_idx] |
|
|
|
|
|
frame_np = torch.clamp(frame_tensor.float(), -1., 1.) * 127.5 + 127.5 |
|
frame_np = frame_np.to(torch.uint8).cpu().numpy() |
|
|
|
|
|
frame_np = np.transpose(frame_np, (1, 2, 0)) |
|
|
|
all_frames_for_video.append(frame_np) |
|
total_frames_yielded += 1 |
|
|
|
|
|
blocks_completed = idx |
|
current_block_progress = (frame_idx + 1) / actual_frames_this_block |
|
total_block_progress = (blocks_completed + current_block_progress) / num_blocks |
|
frame_progress_percent = total_block_progress * 100 |
|
|
|
|
|
frame_progress_percent = min(frame_progress_percent, 100.0) |
|
|
|
print(f"๐บ Yielding frame {total_frames_yielded}: shape {frame_np.shape}") |
|
|
|
|
|
if frame_idx == actual_frames_this_block - 1 and idx + 1 == num_blocks: |
|
status_html = ( |
|
f"<div style='padding: 16px; border: 1px solid #198754; background-color: #d1e7dd; border-radius: 8px; font-family: sans-serif; text-align: center;'>" |
|
f" <h4 style='margin: 0 0 8px 0; color: #0f5132; font-size: 18px;'>๐ Generation Complete!</h4>" |
|
f" <p style='margin: 0; color: #0f5132;'>" |
|
f" Total frames: {total_frames_yielded}. The final video is now available." |
|
f" </p>" |
|
f"</div>" |
|
) |
|
else: |
|
status_html = ( |
|
f"<div style='padding: 10px; border: 1px solid #ddd; border-radius: 8px; font-family: sans-serif;'>" |
|
f" <p style='margin: 0 0 8px 0; font-size: 16px; font-weight: bold;'>Generating Video...</p>" |
|
f" <div style='background: #e9ecef; border-radius: 4px; width: 100%; overflow: hidden;'>" |
|
f" <div style='width: {frame_progress_percent:.1f}%; height: 20px; background-color: #0d6efd; transition: width 0.2s;'></div>" |
|
f" </div>" |
|
f" <p style='margin: 8px 0 0 0; color: #555; font-size: 14px; text-align: right;'>" |
|
f" Block {idx+1}/{num_blocks} | Frame {total_frames_yielded} | {frame_progress_percent:.1f}%" |
|
f" </p>" |
|
f"</div>" |
|
) |
|
|
|
|
|
yield gr.update(visible=True, value=frame_np), gr.update(visible=False), status_html |
|
|
|
|
|
|
|
if not (frame_idx == actual_frames_this_block - 1 and idx + 1 == num_blocks): |
|
time.sleep(max(frame_delay, 0.1)) |
|
|
|
current_start_frame += current_num_frames |
|
|
|
print(f"โ
Video generation completed! Total frames yielded: {total_frames_yielded}") |
|
|
|
|
|
try: |
|
video_path = f"gradio_tmp/{seed}_{hashlib.md5(prompt.encode()).hexdigest()}.mp4" |
|
imageio.mimwrite(video_path, all_frames_for_video, fps=fps, quality=8) |
|
print(f"โ
Video saved to {video_path}") |
|
final_status_html = ( |
|
f"<div style='padding: 16px; border: 1px solid #198754; background-color: #d1e7dd; border-radius: 8px; font-family: sans-serif; text-align: center;'>" |
|
f" <h4 style='margin: 0 0 8px 0; color: #0f5132; font-size: 18px;'>๐ Generation Complete!</h4>" |
|
f" <p style='margin: 0; color: #0f5132;'>" |
|
f" Video saved successfully with {total_frames_yielded} frames at {fps} FPS." |
|
f" </p>" |
|
f"</div>" |
|
) |
|
yield gr.update(visible=False), gr.update(value=video_path, visible=True), final_status_html |
|
except Exception as e: |
|
print(f"โ ๏ธ Could not save final video: {e}") |
|
error_status_html = ( |
|
f"<div style='padding: 16px; border: 1px solid #dc3545; background-color: #f8d7da; border-radius: 8px; font-family: sans-serif; text-align: center;'>" |
|
f" <h4 style='margin: 0 0 8px 0; color: #721c24; font-size: 18px;'>โ ๏ธ Video Save Error</h4>" |
|
f" <p style='margin: 0; color: #721c24;'>" |
|
f" Could not save final video: {str(e)}" |
|
f" </p>" |
|
f"</div>" |
|
) |
|
yield None, None, error_status_html |
|
|
|
@torch.no_grad() |
|
@spaces.GPU |
|
def video_generation_handler_example(prompt, seed=42, fps=15): |
|
""" |
|
Simplified video generation function that returns the final video path. |
|
""" |
|
if seed == -1: |
|
seed = random.randint(0, 2**32 - 1) |
|
|
|
print(f"๐ฌ Starting video generation with prompt: '{prompt}' and seed: {seed}") |
|
|
|
|
|
print("๐ค Encoding text prompt...") |
|
conditional_dict = text_encoder(text_prompts=[prompt]) |
|
for key, value in conditional_dict.items(): |
|
conditional_dict[key] = value.to(dtype=torch.float16) |
|
|
|
|
|
rnd = torch.Generator(gpu).manual_seed(int(seed)) |
|
pipeline._initialize_kv_cache(1, torch.float16, device=gpu) |
|
pipeline._initialize_crossattn_cache(1, torch.float16, device=gpu) |
|
noise = torch.randn([1, 21, 16, 60, 104], device=gpu, dtype=torch.float16, generator=rnd) |
|
|
|
vae_cache, latents_cache = None, None |
|
if not APP_STATE["current_use_taehv"] and not args.trt: |
|
vae_cache = [c.to(device=gpu, dtype=torch.float16) for c in ZERO_VAE_CACHE] |
|
|
|
num_blocks = 7 |
|
current_start_frame = 0 |
|
all_num_frames = [pipeline.num_frame_per_block] * num_blocks |
|
all_frames_for_video = [] |
|
|
|
|
|
for idx, current_num_frames in enumerate(all_num_frames): |
|
print(f"๐ฆ Processing block {idx+1}/{num_blocks} with {current_num_frames} frames") |
|
|
|
noisy_input = noise[:, current_start_frame : current_start_frame + current_num_frames] |
|
|
|
|
|
for step_idx, current_timestep in enumerate(pipeline.denoising_step_list): |
|
timestep = torch.ones([1, current_num_frames], device=noise.device, dtype=torch.int64) * current_timestep |
|
_, denoised_pred = pipeline.generator( |
|
noisy_image_or_video=noisy_input, conditional_dict=conditional_dict, |
|
timestep=timestep, kv_cache=pipeline.kv_cache1, |
|
crossattn_cache=pipeline.crossattn_cache, |
|
current_start=current_start_frame * pipeline.frame_seq_length |
|
) |
|
if step_idx < len(pipeline.denoising_step_list) - 1: |
|
next_timestep = pipeline.denoising_step_list[step_idx + 1] |
|
noisy_input = pipeline.scheduler.add_noise( |
|
denoised_pred.flatten(0, 1), torch.randn_like(denoised_pred.flatten(0, 1)), |
|
next_timestep * torch.ones([1 * current_num_frames], device=noise.device, dtype=torch.long) |
|
).unflatten(0, denoised_pred.shape[:2]) |
|
|
|
if idx < len(all_num_frames) - 1: |
|
pipeline.generator( |
|
noisy_image_or_video=denoised_pred, conditional_dict=conditional_dict, |
|
timestep=torch.zeros_like(timestep), kv_cache=pipeline.kv_cache1, |
|
crossattn_cache=pipeline.crossattn_cache, |
|
current_start=current_start_frame * pipeline.frame_seq_length, |
|
) |
|
|
|
|
|
if args.trt: |
|
pixels, vae_cache = pipeline.vae.forward(denoised_pred.half(), *vae_cache) |
|
elif APP_STATE["current_use_taehv"]: |
|
if latents_cache is None: |
|
latents_cache = denoised_pred |
|
else: |
|
denoised_pred = torch.cat([latents_cache, denoised_pred], dim=1) |
|
latents_cache = denoised_pred[:, -3:] |
|
pixels = pipeline.vae.decode(denoised_pred) |
|
else: |
|
pixels, vae_cache = pipeline.vae(denoised_pred.half(), *vae_cache) |
|
|
|
|
|
if idx == 0 and not args.trt: |
|
pixels = pixels[:, 3:] |
|
elif APP_STATE["current_use_taehv"] and idx > 0: |
|
pixels = pixels[:, 12:] |
|
|
|
print(f"๐น Decoded pixels shape: {pixels.shape}") |
|
|
|
|
|
for frame_idx in range(pixels.shape[1]): |
|
frame_tensor = pixels[0, frame_idx] |
|
|
|
|
|
frame_np = torch.clamp(frame_tensor.float(), -1., 1.) * 127.5 + 127.5 |
|
frame_np = frame_np.to(torch.uint8).cpu().numpy() |
|
|
|
|
|
frame_np = np.transpose(frame_np, (1, 2, 0)) |
|
|
|
all_frames_for_video.append(frame_np) |
|
|
|
current_start_frame += current_num_frames |
|
|
|
print(f"โ
Video generation completed! Total frames: {len(all_frames_for_video)}") |
|
|
|
|
|
video_path = f"gradio_tmp/{seed}_{hashlib.md5(prompt.encode()).hexdigest()}.mp4" |
|
imageio.mimwrite(video_path, all_frames_for_video, fps=fps, quality=8) |
|
print(f"โ
Video saved to {video_path}") |
|
|
|
return video_path |
|
|
|
|
|
frame_display = gr.Image( |
|
label="Generated Frames", |
|
height=480, |
|
width=832, |
|
show_label=True, |
|
container=True, |
|
visible=False |
|
) |
|
final_video = gr.Video( |
|
label="Final Rendered Video", |
|
visible=True, |
|
interactive=False, |
|
height=400, |
|
autoplay=True |
|
) |
|
status_html = gr.HTML( |
|
value="<div style='text-align: center; padding: 20px; color: #666;'>Ready to start generation...</div>", |
|
label="Generation Status" |
|
) |
|
with gr.Blocks(title="Self-Forcing Frame Streaming Demo") as demo: |
|
gr.Markdown("# ๐ Self-Forcing Video Generation with Frame Streaming") |
|
gr.Markdown("Real-time video generation with frame-by-frame display. [[Model]](https://huggingface.co/gdhe17/Self-Forcing), [[Project page]](https://self-forcing.github.io), [[Paper]](https://huggingface.co/papers/2506.08009)") |
|
|
|
with gr.Row(): |
|
with gr.Column(scale=2): |
|
gr.Markdown("### ๐ Configure Generation") |
|
prompt = gr.Textbox( |
|
label="Prompt", |
|
placeholder="A stylish woman walks down a Tokyo street...", |
|
lines=4, |
|
) |
|
enhance_button = gr.Button("Enhance prompt") |
|
gr.Examples( |
|
examples=[ |
|
"A close-up shot of a ceramic teacup slowly pouring water into a glass mug. The water flows smoothly from the spout of the teacup into the mug, creating gentle ripples as it fills up. Both cups have detailed textures, with the teacup having a matte finish and the glass mug showcasing clear transparency. The background is a blurred kitchen countertop, adding context without distracting from the central action. The pouring motion is fluid and natural, emphasizing the interaction between the two cups.", |
|
"A playful cat is seen playing an electronic guitar, strumming the strings with its front paws. The cat has distinctive black facial markings and a bushy tail. It sits comfortably on a small stool, its body slightly tilted as it focuses intently on the instrument. The setting is a cozy, dimly lit room with vintage posters on the walls, adding a retro vibe. The cat's expressive eyes convey a sense of joy and concentration. Medium close-up shot, focusing on the cat's face and hands interacting with the guitar.", |
|
"A dynamic over-the-shoulder perspective of a chef meticulously plating a dish in a bustling kitchen. The chef, a middle-aged woman, deftly arranges ingredients on a pristine white plate. Her hands move with precision, each gesture deliberate and practiced. The background shows a crowded kitchen with steaming pots, whirring blenders, and the clatter of utensils. Bright lights highlight the scene, casting shadows across the busy workspace. The camera angle captures the chef's detailed work from behind, emphasizing his skill and dedication.", |
|
], |
|
inputs=[prompt], |
|
fn=video_generation_handler_example, |
|
outputs=[final_video], |
|
cache_examples="lazy" |
|
) |
|
|
|
with gr.Row(): |
|
seed = gr.Number(label="Seed", value=-1, info="Use -1 for a random seed.") |
|
fps = gr.Slider( |
|
label="Playback FPS", |
|
minimum=1, |
|
maximum=30, |
|
value=args.fps, |
|
step=1, |
|
visible=False, |
|
info="Frames per second for playback" |
|
) |
|
|
|
start_btn = gr.Button("๐ฌ Start Generation", variant="primary", size="lg") |
|
|
|
with gr.Column(scale=3): |
|
gr.Markdown("### ๐บ Live Frame Stream") |
|
gr.Markdown("*Click 'Start Generation' to begin frame streaming*") |
|
|
|
final_video.render() |
|
|
|
frame_display.render() |
|
|
|
status_html.render() |
|
|
|
|
|
start_btn.click( |
|
fn=video_generation_handler, |
|
inputs=[prompt, seed, fps], |
|
outputs=[frame_display, final_video, status_html] |
|
) |
|
enhance_button.click( |
|
fn=enhance_prompt, |
|
inputs=[prompt], |
|
outputs=[prompt] |
|
) |
|
|
|
|
|
if __name__ == "__main__": |
|
if os.path.exists("gradio_tmp"): |
|
import shutil |
|
shutil.rmtree("gradio_tmp") |
|
os.makedirs("gradio_tmp", exist_ok=True) |
|
|
|
demo.queue().launch( |
|
server_name=args.host, |
|
server_port=args.port, |
|
share=args.share, |
|
show_error=True |
|
) |