


"""simple docstring"""



from typing import Any, Callable, Dict, List, Optional, Union

import torch
from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DiffusionPipeline,
    LMSDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionPipeline,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker


UpperCAmelCase      		=			'''CompVis/stable-diffusion-v1-1'''
UpperCAmelCase      		=			'''CompVis/stable-diffusion-v1-2'''
UpperCAmelCase      		=			'''CompVis/stable-diffusion-v1-3'''
UpperCAmelCase      		=			'''CompVis/stable-diffusion-v1-4'''
class 					__magic_name__		(			__UpperCAmelCase     ):






       def __init__(   self  :    int  ,     snake_case__  :    AutoencoderKL  ,     snake_case__  :    CLIPTextModel  ,     snake_case__  :    CLIPTokenizer  ,     snake_case__  :    UNetaDConditionModel  ,     snake_case__  :    Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler]  ,     snake_case__  :    StableDiffusionSafetyChecker  ,     snake_case__  :    CLIPImageProcessor  ,     snake_case__  :    bool = True  ,     ):

              '''simple docstring'''






              super()._init_()

              lowercase		:List[str]       							=				StableDiffusionPipeline.from_pretrained(snake_case__				)
              lowercase		:str       							=				StableDiffusionPipeline.from_pretrained(snake_case__				)
              lowercase		:str       							=				StableDiffusionPipeline.from_pretrained(snake_case__				)
              lowercase		:Optional[Any]       							=				StableDiffusionPipeline(
                  vae=snake_case__  ,     text_encoder=snake_case__  ,     tokenizer=snake_case__  ,     unet=snake_case__  ,     scheduler=snake_case__  ,     safety_checker=snake_case__  ,     feature_extractor=snake_case__  ,     requires_safety_checker=snake_case__  ,     )

              self.register_modules(pipelinea=self.pipea  ,     pipelinea=self.pipea  ,     pipelinea=self.pipea  ,     pipelinea=self.pipea				)






       @property
       def  __snake_case						(   self  :    str				):

              '''simple docstring'''






              return {k: getattr(self  ,     snake_case__				) for k in self.config.keys() if not k.startswith('''_'''				)}






       def  __snake_case						(   self  :    int  ,     snake_case__  :    Optional[Union[str, int]] = "auto"				):

              '''simple docstring'''






              if slice_size == "auto":
                     # half the attention head size is usually a good trade-off between
                     # speed and memory
                     lowercase		:Dict       							=				self.unet.config.attention_head_dim // 2
              self.unet.set_attention_slice(snake_case__				)






       def  __snake_case						(   self  :    Any				):

              '''simple docstring'''






              self.enable_attention_slicing(snake_case__				)






       @torch.no_grad()
       def  __snake_case						(   self  :    int  ,     snake_case__  :    Union[str, List[str]]  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_0  ,     snake_case__  :    float = 7.5  ,     snake_case__  :    Optional[Union[str, List[str]]] = None  ,     snake_case__  :    Optional[int] = 1  ,     snake_case__  :    float = 0.0  ,     snake_case__  :    Optional[torch.Generator] = None  ,     snake_case__  :    Optional[torch.FloatTensor] = None  ,     snake_case__  :    Optional[str] = "pil"  ,     snake_case__  :    bool = True  ,     snake_case__  :    Optional[Callable[[int, int, torch.FloatTensor], None]] = None  ,     snake_case__  :    int = 1  ,     **snake_case__  :    Any  ,     ):

              '''simple docstring'''






              return self.pipea(
                  prompt=snake_case__  ,     height=snake_case__  ,     width=snake_case__  ,     num_inference_steps=snake_case__  ,     guidance_scale=snake_case__  ,     negative_prompt=snake_case__  ,     num_images_per_prompt=snake_case__  ,     eta=snake_case__  ,     generator=snake_case__  ,     latents=snake_case__  ,     output_type=snake_case__  ,     return_dict=snake_case__  ,     callback=snake_case__  ,     callback_steps=snake_case__  ,     **snake_case__  ,     )






       @torch.no_grad()
       def  __snake_case						(   self  :    Optional[Any]  ,     snake_case__  :    Union[str, List[str]]  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_0  ,     snake_case__  :    float = 7.5  ,     snake_case__  :    Optional[Union[str, List[str]]] = None  ,     snake_case__  :    Optional[int] = 1  ,     snake_case__  :    float = 0.0  ,     snake_case__  :    Optional[torch.Generator] = None  ,     snake_case__  :    Optional[torch.FloatTensor] = None  ,     snake_case__  :    Optional[str] = "pil"  ,     snake_case__  :    bool = True  ,     snake_case__  :    Optional[Callable[[int, int, torch.FloatTensor], None]] = None  ,     snake_case__  :    int = 1  ,     **snake_case__  :    Dict  ,     ):

              '''simple docstring'''






              return self.pipea(
                  prompt=snake_case__  ,     height=snake_case__  ,     width=snake_case__  ,     num_inference_steps=snake_case__  ,     guidance_scale=snake_case__  ,     negative_prompt=snake_case__  ,     num_images_per_prompt=snake_case__  ,     eta=snake_case__  ,     generator=snake_case__  ,     latents=snake_case__  ,     output_type=snake_case__  ,     return_dict=snake_case__  ,     callback=snake_case__  ,     callback_steps=snake_case__  ,     **snake_case__  ,     )






       @torch.no_grad()
       def  __snake_case						(   self  :    int  ,     snake_case__  :    Union[str, List[str]]  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_0  ,     snake_case__  :    float = 7.5  ,     snake_case__  :    Optional[Union[str, List[str]]] = None  ,     snake_case__  :    Optional[int] = 1  ,     snake_case__  :    float = 0.0  ,     snake_case__  :    Optional[torch.Generator] = None  ,     snake_case__  :    Optional[torch.FloatTensor] = None  ,     snake_case__  :    Optional[str] = "pil"  ,     snake_case__  :    bool = True  ,     snake_case__  :    Optional[Callable[[int, int, torch.FloatTensor], None]] = None  ,     snake_case__  :    int = 1  ,     **snake_case__  :    Optional[int]  ,     ):

              '''simple docstring'''






              return self.pipea(
                  prompt=snake_case__  ,     height=snake_case__  ,     width=snake_case__  ,     num_inference_steps=snake_case__  ,     guidance_scale=snake_case__  ,     negative_prompt=snake_case__  ,     num_images_per_prompt=snake_case__  ,     eta=snake_case__  ,     generator=snake_case__  ,     latents=snake_case__  ,     output_type=snake_case__  ,     return_dict=snake_case__  ,     callback=snake_case__  ,     callback_steps=snake_case__  ,     **snake_case__  ,     )






       @torch.no_grad()
       def  __snake_case						(   self  :    str  ,     snake_case__  :    Union[str, List[str]]  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_0  ,     snake_case__  :    float = 7.5  ,     snake_case__  :    Optional[Union[str, List[str]]] = None  ,     snake_case__  :    Optional[int] = 1  ,     snake_case__  :    float = 0.0  ,     snake_case__  :    Optional[torch.Generator] = None  ,     snake_case__  :    Optional[torch.FloatTensor] = None  ,     snake_case__  :    Optional[str] = "pil"  ,     snake_case__  :    bool = True  ,     snake_case__  :    Optional[Callable[[int, int, torch.FloatTensor], None]] = None  ,     snake_case__  :    int = 1  ,     **snake_case__  :    Tuple  ,     ):

              '''simple docstring'''






              return self.pipea(
                  prompt=snake_case__  ,     height=snake_case__  ,     width=snake_case__  ,     num_inference_steps=snake_case__  ,     guidance_scale=snake_case__  ,     negative_prompt=snake_case__  ,     num_images_per_prompt=snake_case__  ,     eta=snake_case__  ,     generator=snake_case__  ,     latents=snake_case__  ,     output_type=snake_case__  ,     return_dict=snake_case__  ,     callback=snake_case__  ,     callback_steps=snake_case__  ,     **snake_case__  ,     )







       @torch.no_grad()
       def  __snake_case						(   self  :    Union[str, Any]  ,     snake_case__  :    Union[str, List[str]]  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_1_2  ,     snake_case__  :    int = 5_0  ,     snake_case__  :    float = 7.5  ,     snake_case__  :    Optional[Union[str, List[str]]] = None  ,     snake_case__  :    Optional[int] = 1  ,     snake_case__  :    float = 0.0  ,     snake_case__  :    Optional[torch.Generator] = None  ,     snake_case__  :    Optional[torch.FloatTensor] = None  ,     snake_case__  :    Optional[str] = "pil"  ,     snake_case__  :    bool = True  ,     snake_case__  :    Optional[Callable[[int, int, torch.FloatTensor], None]] = None  ,     snake_case__  :    int = 1  ,     **snake_case__  :    Tuple  ,     ):

              '''simple docstring'''






              lowercase		:Any       							=				'''cuda''' if torch.cuda.is_available() else '''cpu'''
              self.to(snake_case__				)

              # Checks if the height and width are divisible by 8 or not
              if height % 8 != 0 or width % 8 != 0:
                     raise ValueError(f"""`height` and `width` must be divisible by 8 but are {height} and {width}."""				)

              # Get first result from Stable Diffusion Checkpoint v1.1
              lowercase		:Any       							=				self.textaimg_sda_a(
                  prompt=snake_case__  ,     height=snake_case__  ,     width=snake_case__  ,     num_inference_steps=snake_case__  ,     guidance_scale=snake_case__  ,     negative_prompt=snake_case__  ,     num_images_per_prompt=snake_case__  ,     eta=snake_case__  ,     generator=snake_case__  ,     latents=snake_case__  ,     output_type=snake_case__  ,     return_dict=snake_case__  ,     callback=snake_case__  ,     callback_steps=snake_case__  ,     **snake_case__  ,     )

              # Get first result from Stable Diffusion Checkpoint v1.2
              lowercase		:int       							=				self.textaimg_sda_a(
                  prompt=snake_case__  ,     height=snake_case__  ,     width=snake_case__  ,     num_inference_steps=snake_case__  ,     guidance_scale=snake_case__  ,     negative_prompt=snake_case__  ,     num_images_per_prompt=snake_case__  ,     eta=snake_case__  ,     generator=snake_case__  ,     latents=snake_case__  ,     output_type=snake_case__  ,     return_dict=snake_case__  ,     callback=snake_case__  ,     callback_steps=snake_case__  ,     **snake_case__  ,     )

              # Get first result from Stable Diffusion Checkpoint v1.3
              lowercase		:List[str]       							=				self.textaimg_sda_a(
                  prompt=snake_case__  ,     height=snake_case__  ,     width=snake_case__  ,     num_inference_steps=snake_case__  ,     guidance_scale=snake_case__  ,     negative_prompt=snake_case__  ,     num_images_per_prompt=snake_case__  ,     eta=snake_case__  ,     generator=snake_case__  ,     latents=snake_case__  ,     output_type=snake_case__  ,     return_dict=snake_case__  ,     callback=snake_case__  ,     callback_steps=snake_case__  ,     **snake_case__  ,     )

              # Get first result from Stable Diffusion Checkpoint v1.4
              lowercase		:Optional[int]       							=				self.textaimg_sda_a(
                  prompt=snake_case__  ,     height=snake_case__  ,     width=snake_case__  ,     num_inference_steps=snake_case__  ,     guidance_scale=snake_case__  ,     negative_prompt=snake_case__  ,     num_images_per_prompt=snake_case__  ,     eta=snake_case__  ,     generator=snake_case__  ,     latents=snake_case__  ,     output_type=snake_case__  ,     return_dict=snake_case__  ,     callback=snake_case__  ,     callback_steps=snake_case__  ,     **snake_case__  ,     )

              # Get all result images into a single list and pass it via StableDiffusionPipelineOutput for final result
              return StableDiffusionPipelineOutput([resa[0], resa[0], resa[0], resa[0]]				)





"""simple docstring"""



import argparse
import os
import sys
from unittest.mock import patch

import pytorch_lightning as pl
import timeout_decorator
import torch
from distillation import SummarizationDistiller, distill_main
from finetune import SummarizationModule, main

from transformers import MarianMTModel
from transformers.file_utils import cached_path
from transformers.testing_utils import TestCasePlus, require_torch_gpu, slow
from utils import load_json


UpperCAmelCase      		=			'''sshleifer/mar_enro_6_3_student'''
class 					__magic_name__		(			__UpperCAmelCase     ):






       def  __snake_case						(   self  :    int				):

              '''simple docstring'''






              super().setUp()

              lowercase		:Union[str, Any]       							=				cached_path(
                  '''https://cdn-datasets.huggingface.co/translation/wmt_en_ro-tr40k-va0.5k-te0.5k.tar.gz'''  ,     extract_compressed_file=snake_case__  ,     )
              lowercase		:int       							=				f"""{data_cached}/wmt_en_ro-tr40k-va0.5k-te0.5k"""






       @slow
       @require_torch_gpu
       def  __snake_case						(   self  :    Any				):

              '''simple docstring'''






              MarianMTModel.from_pretrained(snake_case__				)







       @slow
       @require_torch_gpu
       def  __snake_case						(   self  :    Tuple				):

              '''simple docstring'''






              lowercase		:str       							=				{
                  '''$MAX_LEN''': 6_4,
                  '''$BS''': 6_4,
                  '''$GAS''': 1,
                  '''$ENRO_DIR''': self.data_dir,
                  '''facebook/mbart-large-cc25''': MARIAN_MODEL,
                  # "val_check_interval=0.25": "val_check_interval=1.0",
                  '''--learning_rate=3e-5''': '''--learning_rate 3e-4''',
                  '''--num_train_epochs 6''': '''--num_train_epochs 1''',
              }

              # Clean up bash script
              lowercase		:str       							=				(self.test_file_dir / '''train_mbart_cc25_enro.sh''').open().read().split('''finetune.py'''				)[1].strip()
              lowercase		:Any       							=				bash_script.replace('''\\\n'''  ,     ''''''				).strip().replace('''"$@"'''  ,     ''''''				)
              for k, v in env_vars_to_replace.items():
                     lowercase		:Dict       							=				bash_script.replace(snake_case__  ,     str(snake_case__				)				)
              lowercase		:Any       							=				self.get_auto_remove_tmp_dir()

              # bash_script = bash_script.replace("--fp16 ", "")
              lowercase		:Optional[Any]       							=				f"""
            --output_dir {output_dir}
            --tokenizer_name Helsinki-NLP/opus-mt-en-ro
            --sortish_sampler
            --do_predict
            --gpus 1
            --freeze_encoder
            --n_train 40000
            --n_val 500
            --n_test 500
            --fp16_opt_level O1
            --num_sanity_val_steps 0
            --eval_beams 2
        """.split()
              # XXX: args.gpus > 1 : handle multi_gpu in the future

              lowercase		:str       							=				['''finetune.py'''] + bash_script.split() + args
              with patch.object(snake_case__  ,     '''argv'''  ,     snake_case__				):
                     lowercase		:Optional[int]       							=				argparse.ArgumentParser()
                     lowercase		:List[Any]       							=				pl.Trainer.add_argparse_args(snake_case__				)
                     lowercase		:Union[str, Any]       							=				SummarizationModule.add_model_specific_args(snake_case__  ,     os.getcwd()				)
                     lowercase		:int       							=				parser.parse_args()
                     lowercase		:int       							=				main(snake_case__				)

              # Check metrics
              lowercase		:int       							=				load_json(model.metrics_save_path				)
              lowercase		:Tuple       							=				metrics['''val'''][0]
              lowercase		:Any       							=				metrics['''val'''][-1]
              self.assertEqual(len(metrics['''val''']				)  ,     (args.max_epochs / args.val_check_interval)				)
              assert isinstance(last_step_stats[f"""val_avg_{model.val_metric}"""]  ,     snake_case__				)

              self.assertGreater(last_step_stats['''val_avg_gen_time''']  ,     0.01				)
              # model hanging on generate. Maybe bad config was saved. (XXX: old comment/assert?)
              self.assertLessEqual(last_step_stats['''val_avg_gen_time''']  ,     1.0				)

              # test learning requirements:

              # 1. BLEU improves over the course of training by more than 2 pts
              self.assertGreater(last_step_stats['''val_avg_bleu'''] - first_step_stats['''val_avg_bleu''']  ,     2				)

              # 2. BLEU finishes above 17
              self.assertGreater(last_step_stats['''val_avg_bleu''']  ,     1_7				)

              # 3. test BLEU and val BLEU within ~1.1 pt.
              self.assertLess(abs(metrics['''val'''][-1]['''val_avg_bleu'''] - metrics['''test'''][-1]['''test_avg_bleu''']				)  ,     1.1				)

              # check lightning ckpt can be loaded and has a reasonable statedict
              lowercase		:Any       							=				os.listdir(snake_case__				)
              lowercase		:int       							=				[x for x in contents if x.endswith('''.ckpt'''				)][0]
              lowercase		:Any       							=				os.path.join(args.output_dir  ,     snake_case__				)
              lowercase		:int       							=				torch.load(snake_case__  ,     map_location='''cpu'''				)
              lowercase		:Any       							=				'''model.model.decoder.layers.0.encoder_attn_layer_norm.weight'''
              assert expected_key in ckpt["state_dict"]
              assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.floataa

              # TODO: turn on args.do_predict when PL bug fixed.
              if args.do_predict:
                     lowercase		:Optional[Any]       							=				{os.path.basename(snake_case__				) for p in contents}
                     assert "test_generations.txt" in contents
                     assert "test_results.txt" in contents
                     # assert len(metrics["val"]) ==  desired_n_evals
                     assert len(metrics['''test''']				) == 1







class 					__magic_name__		(			__UpperCAmelCase     ):






       @timeout_decorator.timeout(6_0_0				)
       @slow
       @require_torch_gpu
       def  __snake_case						(   self  :    Optional[int]				):

              '''simple docstring'''






              lowercase		:Union[str, Any]       							=				f"""{self.test_file_dir_str}/test_data/wmt_en_ro"""
              lowercase		:List[Any]       							=				{
                  '''--fp16_opt_level=O1''': '''''',
                  '''$MAX_LEN''': 1_2_8,
                  '''$BS''': 1_6,
                  '''$GAS''': 1,
                  '''$ENRO_DIR''': data_dir,
                  '''$m''': '''sshleifer/student_marian_en_ro_6_1''',
                  '''val_check_interval=0.25''': '''val_check_interval=1.0''',
              }

              # Clean up bash script
              lowercase		:Dict       							=				(
                  (self.test_file_dir / '''distil_marian_no_teacher.sh''').open().read().split('''distillation.py'''				)[1].strip()
              )
              lowercase		:Union[str, Any]       							=				bash_script.replace('''\\\n'''  ,     ''''''				).strip().replace('''"$@"'''  ,     ''''''				)
              lowercase		:Dict       							=				bash_script.replace('''--fp16 '''  ,     ''' '''				)

              for k, v in env_vars_to_replace.items():
                     lowercase		:str       							=				bash_script.replace(snake_case__  ,     str(snake_case__				)				)
              lowercase		:Union[str, Any]       							=				self.get_auto_remove_tmp_dir()
              lowercase		:str       							=				bash_script.replace('''--fp16'''  ,     ''''''				)
              lowercase		:Union[str, Any]       							=				6
              lowercase		:str       							=				(
                  ['''distillation.py''']
                  + bash_script.split()
                  + [
                      f"""--output_dir={output_dir}""",
                      '''--gpus=1''',
                      '''--learning_rate=1e-3''',
                      f"""--num_train_epochs={epochs}""",
                      '''--warmup_steps=10''',
                      '''--val_check_interval=1.0''',
                      '''--do_predict''',
                  ]
              )
              with patch.object(snake_case__  ,     '''argv'''  ,     snake_case__				):
                     lowercase		:Optional[int]       							=				argparse.ArgumentParser()
                     lowercase		:Tuple       							=				pl.Trainer.add_argparse_args(snake_case__				)
                     lowercase		:Optional[Any]       							=				SummarizationDistiller.add_model_specific_args(snake_case__  ,     os.getcwd()				)
                     lowercase		:Optional[int]       							=				parser.parse_args()
                     # assert args.gpus == gpus THIS BREAKS for multi_gpu

                     lowercase		:Tuple       							=				distill_main(snake_case__				)

              # Check metrics
              lowercase		:Tuple       							=				load_json(model.metrics_save_path				)
              lowercase		:int       							=				metrics['''val'''][0]
              lowercase		:List[str]       							=				metrics['''val'''][-1]
              assert len(metrics['''val''']				) >= (args.max_epochs / args.val_check_interval)  # +1 accounts for val_sanity_check

              assert last_step_stats["val_avg_gen_time"] >= 0.01

              assert first_step_stats["val_avg_bleu"] < last_step_stats["val_avg_bleu"]  # model learned nothing
              assert 1.0 >= last_step_stats["val_avg_gen_time"]  # model hanging on generate. Maybe bad config was saved.
              assert isinstance(last_step_stats[f"""val_avg_{model.val_metric}"""]  ,     snake_case__				)

              # check lightning ckpt can be loaded and has a reasonable statedict
              lowercase		:Union[str, Any]       							=				os.listdir(snake_case__				)
              lowercase		:List[str]       							=				[x for x in contents if x.endswith('''.ckpt'''				)][0]
              lowercase		:Dict       							=				os.path.join(args.output_dir  ,     snake_case__				)
              lowercase		:Optional[int]       							=				torch.load(snake_case__  ,     map_location='''cpu'''				)
              lowercase		:str       							=				'''model.model.decoder.layers.0.encoder_attn_layer_norm.weight'''
              assert expected_key in ckpt["state_dict"]
              assert ckpt["state_dict"]["model.model.decoder.layers.0.encoder_attn_layer_norm.weight"].dtype == torch.floataa

              # TODO: turn on args.do_predict when PL bug fixed.
              if args.do_predict:
                     lowercase		:int       							=				{os.path.basename(snake_case__				) for p in contents}
                     assert "test_generations.txt" in contents
                     assert "test_results.txt" in contents
                     # assert len(metrics["val"]) ==  desired_n_evals
                     assert len(metrics['''test''']				) == 1

