





from typing import Any, Dict, List, Optional, Tuple, Union

import torch
from torch import nn
from torch.utils.data import DistributedSampler, RandomSampler

from transformers import PreTrainedModel, Trainer, logging
from transformers.integrations import is_fairscale_available
from transformers.models.fsmt.configuration_fsmt import FSMTConfig
from transformers.optimization import (
    Adafactor,
    AdamW,
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.trainer_pt_utils import get_tpu_sampler
from transformers.training_args import ParallelMode
from transformers.utils import is_torch_tpu_available


if is_fairscale_available():
    from fairscale.optim import OSS


__lowerCamelCase :  Any										=  logging.get_logger(__name__)

__lowerCamelCase :  Any										=  {
    """linear""": get_linear_schedule_with_warmup,
    """cosine""": get_cosine_schedule_with_warmup,
    """cosine_w_restarts""": get_cosine_with_hard_restarts_schedule_with_warmup,
    """polynomial""": get_polynomial_decay_schedule_with_warmup,
    """constant""": get_constant_schedule,
    """constant_w_warmup""": get_constant_schedule_with_warmup,
}



class       A__     ( __snake_case    ):





      def __init__(				self ,							A_=None ,							A_=None ,							*A_ ,							**A_     ):
       '''simple docstring'''




       super().__init__(*A_ ,							**A_     )

       if config is None:
        assert isinstance(self.model ,							A_     ), (
            "If no `config` is passed the model to be trained has to be of type `PreTrainedModel`, but is"
            F""" {self.model.__class__}"""
        )
        UpperCamelCase			:      Tuple 			=     self.model.config
       else:
        UpperCamelCase			:      List[Any] 			=     config

       UpperCamelCase			:      Any 			=     data_args
       UpperCamelCase			:      Union[str, Any] 			=     self.config.tgt_vocab_size if isinstance(self.config ,							A_     ) else self.config.vocab_size

       if self.args.label_smoothing != 0 or (self.data_args is not None and self.data_args.ignore_pad_token_for_loss):
        assert self.config.pad_token_id is not None, (
            "Make sure that `config.pad_token_id` is correcly defined when ignoring `pad_token` for loss"
            " calculation or doing label smoothing."
        )

       if self.config.pad_token_id is None and self.config.eos_token_id is not None:
        logger.warning(
            F"""The `config.pad_token_id` is `None`. Using `config.eos_token_id` = {self.config.eos_token_id} for"""
            " padding.."     )

       if self.args.label_smoothing == 0:
        UpperCamelCase			:      Any 			=     torch.nn.CrossEntropyLoss(ignore_index=self.config.pad_token_id     )
       else:
        # dynamically import label_smoothed_nll_loss
        from utils import label_smoothed_nll_loss

        UpperCamelCase			:      Union[str, Any] 			=     label_smoothed_nll_loss





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       if self.optimizer is None:
        UpperCamelCase			:      Tuple 			=     ["bias", "LayerNorm.weight"]
        UpperCamelCase			:      Optional[Any] 			=     [
            {
                "params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay     )],
                "weight_decay": self.args.weight_decay,
            },
            {
                "params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay     )],
                "weight_decay": 0.0,
            },
        ]
        UpperCamelCase			:      Any 			=     Adafactor if self.args.adafactor else AdamW
        if self.args.adafactor:
         UpperCamelCase			:      Any 			=     Adafactor
         UpperCamelCase			:      List[Any] 			=     {"scale_parameter": False, "relative_step": False}
        else:
         UpperCamelCase			:      str 			=     AdamW
         UpperCamelCase			:      Union[str, Any] 			=     {
             "betas": (self.args.adam_betaa, self.args.adam_betaa),
             "eps": self.args.adam_epsilon,
         }
        UpperCamelCase			:      Optional[int] 			=     self.args.learning_rate
        if self.sharded_ddp:
         UpperCamelCase			:      List[str] 			=     OSS(
             params=A_ ,							optim=A_ ,							**A_ ,							)
        else:
         UpperCamelCase			:      Dict 			=     optimizer_cls(A_ ,							**A_     )

       if self.lr_scheduler is None:
        UpperCamelCase			:      Any 			=     self._get_lr_scheduler(A_     )
       else:  # ignoring --lr_scheduler
        logger.warning("scheduler is passed to `Seq2SeqTrainer`, `--lr_scheduler` arg is ignored."     )





      def  __UpperCamelCase(				self ,							A_     ):
       '''simple docstring'''




       UpperCamelCase			:      Any 			=     arg_to_scheduler[self.args.lr_scheduler]
       if self.args.lr_scheduler == "constant":
        UpperCamelCase			:      List[Any] 			=     schedule_func(self.optimizer     )
       elif self.args.lr_scheduler == "constant_w_warmup":
        UpperCamelCase			:      Optional[int] 			=     schedule_func(self.optimizer ,							num_warmup_steps=self.args.warmup_steps     )
       else:
        UpperCamelCase			:      int 			=     schedule_func(
            self.optimizer ,							num_warmup_steps=self.args.warmup_steps ,							num_training_steps=A_     )
       return scheduler





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       if isinstance(self.train_dataset ,							torch.utils.data.IterableDataset     ):
        return None
       elif is_torch_tpu_available():
        return get_tpu_sampler(self.train_dataset     )
       else:
        if self.args.sortish_sampler:
         self.train_dataset.make_sortish_sampler(
             self.args.per_device_train_batch_size ,							distributed=(self.args.parallel_mode == ParallelMode.DISTRIBUTED) ,							)

        return (
            RandomSampler(self.train_dataset     )
            if self.args.local_rank == -1
            else DistributedSampler(self.train_dataset     )
        )





      def  __UpperCamelCase(				self ,							A_ ,							A_ ,							A_     ):
       '''simple docstring'''




       if self.args.label_smoothing == 0:
        if self.data_args is not None and self.data_args.ignore_pad_token_for_loss:
         # force training to ignore pad token
         UpperCamelCase			:      Any 			=     model(**A_ ,							use_cache=A_     )[0]
         UpperCamelCase			:      Optional[Any] 			=     self.loss_fn(logits.view(-1 ,							logits.shape[-1]     ) ,							labels.view(-1     )     )
        else:
         # compute usual loss via models
         UpperCamelCase			,					UpperCamelCase			:      Any 			=     model(**A_ ,							labels=A_ ,							use_cache=A_     )[:2]
       else:
        # compute label smoothed loss
        UpperCamelCase			:      Union[str, Any] 			=     model(**A_ ,							use_cache=A_     )[0]
        UpperCamelCase			:      List[str] 			=     torch.nn.functional.log_softmax(A_ ,							dim=-1     )
        UpperCamelCase			,					UpperCamelCase			:      Any 			=     self.loss_fn(A_ ,							A_ ,							self.args.label_smoothing ,							ignore_index=self.config.pad_token_id     )
       return loss, logits





      def  __UpperCamelCase(				self ,							A_ ,							A_     ):
       '''simple docstring'''




       UpperCamelCase			:      List[Any] 			=     inputs.pop("labels"     )
       UpperCamelCase			,					UpperCamelCase			:      Tuple 			=     self._compute_loss(A_ ,							A_ ,							A_     )
       return loss





      def  __UpperCamelCase(				self ,							A_ ,							A_ ,							A_ ,							A_ = None ,							):
       '''simple docstring'''




       UpperCamelCase			:      List[Any] 			=     self._prepare_inputs(A_     )

       UpperCamelCase			:      Union[str, Any] 			=     {
           "max_length": self.data_args.val_max_target_length
           if self.data_args is not None
           else self.config.max_length,
           "num_beams": self.data_args.eval_beams if self.data_args is not None else self.config.num_beams,
       }

       if self.args.predict_with_generate and not self.args.prediction_loss_only:
        UpperCamelCase			:      List[Any] 			=     self.model.generate(
            inputs["input_ids"] ,							attention_mask=inputs["attention_mask"] ,							**A_ ,							)
        # in case the batch is shorter than max length, the output should be padded
        if generated_tokens.shape[-1] < gen_kwargs["max_length"]:
         UpperCamelCase			:      Any 			=     self._pad_tensors_to_max_len(A_ ,							gen_kwargs["max_length"]     )

       UpperCamelCase			:      Union[str, Any] 			=     inputs.pop("labels"     )
       with torch.no_grad():
        # compute loss on predict data
        UpperCamelCase			,					UpperCamelCase			:      Optional[int] 			=     self._compute_loss(A_ ,							A_ ,							A_     )

       UpperCamelCase			:      Union[str, Any] 			=     loss.mean().detach()
       if self.args.prediction_loss_only:
        return (loss, None, None)

       UpperCamelCase			:      Union[str, Any] 			=     generated_tokens if self.args.predict_with_generate else logits

       if labels.shape[-1] < gen_kwargs["max_length"]:
        UpperCamelCase			:      Tuple 			=     self._pad_tensors_to_max_len(A_ ,							gen_kwargs["max_length"]     )

       return (loss, logits, labels)





      def  __UpperCamelCase(				self ,							A_ ,							A_     ):
       '''simple docstring'''




       UpperCamelCase			:      Dict 			=     self.config.pad_token_id if self.config.pad_token_id is not None else self.config.eos_token_id

       if pad_token_id is None:
        raise ValueError(
            "Make sure that either `config.pad_token_id` or `config.eos_token_id` is defined if tensor has to be"
            F""" padded to `max_length`={max_length}"""     )

       UpperCamelCase			:      Optional[int] 			=     pad_token_id * torch.ones(
           (tensor.shape[0], max_length) ,							dtype=tensor.dtype ,							device=tensor.device     )
       UpperCamelCase			:      Optional[int] 			=     tensor
       return padded_tensor







import gc
import random
import unittest

import numpy as np
import torch

from diffusers import DDIMScheduler, KandinskyVaaPipeline, KandinskyVaaPriorPipeline, UNetaDConditionModel, VQModel
from diffusers.utils import floats_tensor, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


enable_full_determinism()



class       A__     ( __snake_case ,     unittest.TestCase    ):
      _UpperCAmelCase			:str							=     KandinskyVaaPipeline
      _UpperCAmelCase			:str							=     [
          'image_embeds',
          'negative_image_embeds',
      ]
      _UpperCAmelCase			:str							=     ['image_embeds', 'negative_image_embeds']
      _UpperCAmelCase			:List[str]							=     [
          'generator',
          'height',
          'width',
          'latents',
          'guidance_scale',
          'num_inference_steps',
          'return_dict',
          'guidance_scale',
          'num_images_per_prompt',
          'output_type',
          'return_dict',
      ]
      _UpperCAmelCase			:List[str]							=     False





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return 32





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return 32





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return self.time_input_dim





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return self.time_input_dim * 4





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return 100





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       torch.manual_seed(0     )

       UpperCamelCase			:      List[str] 			=     {
           "in_channels": 4,
           # Out channels is double in channels because predicts mean and variance
           "out_channels": 8,
           "addition_embed_type": "image",
           "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"),
           "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"),
           "mid_block_type": "UNetMidBlock2DSimpleCrossAttn",
           "block_out_channels": (self.block_out_channels_a, self.block_out_channels_a * 2),
           "layers_per_block": 1,
           "encoder_hid_dim": self.text_embedder_hidden_size,
           "encoder_hid_dim_type": "image_proj",
           "cross_attention_dim": self.cross_attention_dim,
           "attention_head_dim": 4,
           "resnet_time_scale_shift": "scale_shift",
           "class_embed_type": None,
       }

       UpperCamelCase			:      Dict 			=     UNetaDConditionModel(**A_     )
       return model





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       return {
           "block_out_channels": [32, 64],
           "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"],
           "in_channels": 3,
           "latent_channels": 4,
           "layers_per_block": 1,
           "norm_num_groups": 8,
           "norm_type": "spatial",
           "num_vq_embeddings": 12,
           "out_channels": 3,
           "up_block_types": [
               "AttnUpDecoderBlock2D",
               "UpDecoderBlock2D",
           ],
           "vq_embed_dim": 4,
       }





      @property
      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       torch.manual_seed(0     )
       UpperCamelCase			:      Optional[Any] 			=     VQModel(**self.dummy_movq_kwargs     )
       return model





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Tuple 			=     self.dummy_unet
       UpperCamelCase			:      Optional[Any] 			=     self.dummy_movq

       UpperCamelCase			:      Dict 			=     DDIMScheduler(
           num_train_timesteps=1000 ,							beta_schedule="linear" ,							beta_start=0.0_00_85 ,							beta_end=0.0_12 ,							clip_sample=A_ ,							set_alpha_to_one=A_ ,							steps_offset=1 ,							prediction_type="epsilon" ,							thresholding=A_ ,							)

       UpperCamelCase			:      Tuple 			=     {
           "unet": unet,
           "scheduler": scheduler,
           "movq": movq,
       }
       return components





      def  __UpperCamelCase(				self ,							A_ ,							A_=0     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[int] 			=     floats_tensor((1, self.text_embedder_hidden_size) ,							rng=random.Random(A_     )     ).to(A_     )
       UpperCamelCase			:      Optional[Any] 			=     floats_tensor((1, self.text_embedder_hidden_size) ,							rng=random.Random(seed + 1     )     ).to(
           A_     )
       if str(A_     ).startswith("mps"     ):
        UpperCamelCase			:      Optional[Any] 			=     torch.manual_seed(A_     )
       else:
        UpperCamelCase			:      List[Any] 			=     torch.Generator(device=A_     ).manual_seed(A_     )
       UpperCamelCase			:      Optional[int] 			=     {
           "image_embeds": image_embeds,
           "negative_image_embeds": negative_image_embeds,
           "generator": generator,
           "height": 64,
           "width": 64,
           "guidance_scale": 4.0,
           "num_inference_steps": 2,
           "output_type": "np",
       }
       return inputs





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Optional[Any] 			=     "cpu"

       UpperCamelCase			:      List[str] 			=     self.get_dummy_components()

       UpperCamelCase			:      Tuple 			=     self.pipeline_class(**A_     )
       UpperCamelCase			:      List[str] 			=     pipe.to(A_     )

       pipe.set_progress_bar_config(disable=A_     )

       UpperCamelCase			:      Dict 			=     pipe(**self.get_dummy_inputs(A_     )     )
       UpperCamelCase			:      Optional[int] 			=     output.images

       UpperCamelCase			:      int 			=     pipe(
           **self.get_dummy_inputs(A_     ) ,							return_dict=A_ ,							)[0]

       UpperCamelCase			:      Tuple 			=     image[0, -3:, -3:, -1]
       UpperCamelCase			:      List[Any] 			=     image_from_tuple[0, -3:, -3:, -1]

       assert image.shape == (1, 64, 64, 3)

       UpperCamelCase			:      int 			=     np.array(
           [0.6_23_79_76, 1.0, 0.36_44_13_32, 1.0, 0.70_63_96_34, 0.29_87_71_86, 0.85_65_21_25, 0.5_21_68_43, 0.54_45_40_46]     )

       assert (
           np.abs(image_slice.flatten() - expected_slice     ).max() < 1e-2
       ), F""" expected_slice {expected_slice}, but got {image_slice.flatten()}"""

       assert (
           np.abs(image_from_tuple_slice.flatten() - expected_slice     ).max() < 1e-2
       ), F""" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}"""



@slow
@require_torch_gpu
class       A__     ( unittest.TestCase    ):





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       super().tearDown()
       gc.collect()
       torch.cuda.empty_cache()





      def  __UpperCamelCase(				self     ):
       '''simple docstring'''




       UpperCamelCase			:      Dict 			=     load_numpy(
           "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
           "/kandinskyv22/kandinskyv22_text2img_cat_fp16.npy"     )

       UpperCamelCase			:      Dict 			=     KandinskyVaaPriorPipeline.from_pretrained(
           "kandinsky-community/kandinsky-2-2-prior" ,							torch_dtype=torch.floataa     )
       pipe_prior.to(A_     )

       UpperCamelCase			:      Dict 			=     KandinskyVaaPipeline.from_pretrained(
           "kandinsky-community/kandinsky-2-2-decoder" ,							torch_dtype=torch.floataa     )
       UpperCamelCase			:      Tuple 			=     pipeline.to(A_     )
       pipeline.set_progress_bar_config(disable=A_     )

       UpperCamelCase			:      str 			=     "red cat, 4k photo"

       UpperCamelCase			:      str 			=     torch.Generator(device="cuda"     ).manual_seed(0     )
       UpperCamelCase			,					UpperCamelCase			:      Tuple 			=     pipe_prior(
           A_ ,							generator=A_ ,							num_inference_steps=5 ,							negative_prompt="" ,							).to_tuple()

       UpperCamelCase			:      int 			=     torch.Generator(device="cuda"     ).manual_seed(0     )
       UpperCamelCase			:      Tuple 			=     pipeline(
           image_embeds=A_ ,							negative_image_embeds=A_ ,							generator=A_ ,							num_inference_steps=100 ,							output_type="np" ,							)

       UpperCamelCase			:      Union[str, Any] 			=     output.images[0]

       assert image.shape == (512, 512, 3)

       assert_mean_pixel_difference(A_ ,							A_     )
