


from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings



@maybe_allow_in_graph
class 							_UpperCamelCase (						nn.Module						):

   '''simple docstring'''







   def __init__(  self  : List[str]       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : List[str]=0.0       ,  _lowerCamelCase  : Optional[int] = None       ,  _lowerCamelCase  : str = "geglu"       ,  _lowerCamelCase  : Optional[int] = None       ,  _lowerCamelCase  : bool = False       ,  _lowerCamelCase  : bool = False       ,  _lowerCamelCase  : bool = False       ,  _lowerCamelCase  : bool = False       ,  _lowerCamelCase  : bool = True       ,  _lowerCamelCase  : str = "layer_norm"       ,  _lowerCamelCase  : bool = False       ,  ):






         '''simple docstring'''



         super().__init__()
         __lowerCamelCase     :      List[Any]       						=						only_cross_attention

         __lowerCamelCase     :      Any       						=						(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
         __lowerCamelCase     :      Any       						=						(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

         if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
               raise ValueError(
                   F"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
                   F""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."""							)

         # Define 3 blocks. Each block has its own normalization layer.
         # 1. Self-Attn
         if self.use_ada_layer_norm:
               __lowerCamelCase     :      List[str]       						=						AdaLayerNorm(_lowerCamelCase       ,  _lowerCamelCase							)
         elif self.use_ada_layer_norm_zero:
               __lowerCamelCase     :      Optional[int]       						=						AdaLayerNormZero(_lowerCamelCase       ,  _lowerCamelCase							)
         else:
               __lowerCamelCase     :      Tuple       						=						nn.LayerNorm(_lowerCamelCase       ,  elementwise_affine=_lowerCamelCase							)
         __lowerCamelCase     :      Any       						=						Attention(
             query_dim=_lowerCamelCase       ,  heads=_lowerCamelCase       ,  dim_head=_lowerCamelCase       ,  dropout=_lowerCamelCase       ,  bias=_lowerCamelCase       ,  cross_attention_dim=cross_attention_dim if only_cross_attention else None       ,  upcast_attention=_lowerCamelCase       ,  )

         # 2. Cross-Attn
         if cross_attention_dim is not None or double_self_attention:
               # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
               # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
               # the second cross attention block.
               __lowerCamelCase     :      Dict       						=						(
                   AdaLayerNorm(_lowerCamelCase       ,  _lowerCamelCase							)
                   if self.use_ada_layer_norm
                   else nn.LayerNorm(_lowerCamelCase       ,  elementwise_affine=_lowerCamelCase							)
               )
               __lowerCamelCase     :      List[Any]       						=						Attention(
                   query_dim=_lowerCamelCase       ,  cross_attention_dim=cross_attention_dim if not double_self_attention else None       ,  heads=_lowerCamelCase       ,  dim_head=_lowerCamelCase       ,  dropout=_lowerCamelCase       ,  bias=_lowerCamelCase       ,  upcast_attention=_lowerCamelCase       ,  )  # is self-attn if encoder_hidden_states is none
         else:
               __lowerCamelCase     :      Any       						=						None
               __lowerCamelCase     :      Union[str, Any]       						=						None

         # 3. Feed-forward
         __lowerCamelCase     :      Union[str, Any]       						=						nn.LayerNorm(_lowerCamelCase       ,  elementwise_affine=_lowerCamelCase							)
         __lowerCamelCase     :      int       						=						FeedForward(_lowerCamelCase       ,  dropout=_lowerCamelCase       ,  activation_fn=_lowerCamelCase       ,  final_dropout=_lowerCamelCase							)

         # let chunk size default to None
         __lowerCamelCase     :      List[Any]       						=						None
         __lowerCamelCase     :      str       						=						0

   def   _snake_case    (  self  : str       ,  _lowerCamelCase  : Optional[int]       ,  _lowerCamelCase  : int							):






         '''simple docstring'''



         __lowerCamelCase     :      int       						=						chunk_size
         __lowerCamelCase     :      Optional[Any]       						=						dim


   def   _snake_case    (  self  : List[Any]       ,  _lowerCamelCase  : torch.FloatTensor       ,  _lowerCamelCase  : Optional[torch.FloatTensor] = None       ,  _lowerCamelCase  : Optional[torch.FloatTensor] = None       ,  _lowerCamelCase  : Optional[torch.FloatTensor] = None       ,  _lowerCamelCase  : Optional[torch.LongTensor] = None       ,  _lowerCamelCase  : Dict[str, Any] = None       ,  _lowerCamelCase  : Optional[torch.LongTensor] = None       ,  ):






         '''simple docstring'''



         if self.use_ada_layer_norm:
               __lowerCamelCase     :      Dict       						=						self.norma(_lowerCamelCase       ,  _lowerCamelCase							)
         elif self.use_ada_layer_norm_zero:
               __lowerCamelCase						, __lowerCamelCase						, __lowerCamelCase						, __lowerCamelCase						, __lowerCamelCase     :      List[Any]       						=						self.norma(
                   _lowerCamelCase       ,  _lowerCamelCase       ,  _lowerCamelCase       ,  hidden_dtype=hidden_states.dtype							)
         else:
               __lowerCamelCase     :      Any       						=						self.norma(_lowerCamelCase							)

         __lowerCamelCase     :      Tuple       						=						cross_attention_kwargs if cross_attention_kwargs is not None else {}

         __lowerCamelCase     :      Union[str, Any]       						=						self.attna(
             _lowerCamelCase       ,  encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None       ,  attention_mask=_lowerCamelCase       ,  **_lowerCamelCase       ,  )
         if self.use_ada_layer_norm_zero:
               __lowerCamelCase     :      Optional[int]       						=						gate_msa.unsqueeze(1							) * attn_output
         __lowerCamelCase     :      Union[str, Any]       						=						attn_output + hidden_states

         # 2. Cross-Attention
         if self.attna is not None:
               __lowerCamelCase     :      Union[str, Any]       						=						(
                   self.norma(_lowerCamelCase       ,  _lowerCamelCase							) if self.use_ada_layer_norm else self.norma(_lowerCamelCase							)
               )

               __lowerCamelCase     :      Union[str, Any]       						=						self.attna(
                   _lowerCamelCase       ,  encoder_hidden_states=_lowerCamelCase       ,  attention_mask=_lowerCamelCase       ,  **_lowerCamelCase       ,  )
               __lowerCamelCase     :      Optional[int]       						=						attn_output + hidden_states

         # 3. Feed-forward
         __lowerCamelCase     :      List[str]       						=						self.norma(_lowerCamelCase							)

         if self.use_ada_layer_norm_zero:
               __lowerCamelCase     :      str       						=						norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

         if self._chunk_size is not None:
               # "feed_forward_chunk_size" can be used to save memory
               if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                     raise ValueError(
                         F"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."""							)

               __lowerCamelCase     :      Optional[int]       						=						norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
               __lowerCamelCase     :      Tuple       						=						torch.cat(
                   [self.ff(_lowerCamelCase							) for hid_slice in norm_hidden_states.chunk(_lowerCamelCase       ,  dim=self._chunk_dim							)]       ,  dim=self._chunk_dim       ,  )
         else:
               __lowerCamelCase     :      Any       						=						self.ff(_lowerCamelCase							)

         if self.use_ada_layer_norm_zero:
               __lowerCamelCase     :      Union[str, Any]       						=						gate_mlp.unsqueeze(1							) * ff_output

         __lowerCamelCase     :      Any       						=						ff_output + hidden_states

         return hidden_states



class 							_UpperCamelCase (						nn.Module						):

   '''simple docstring'''







   def __init__(  self  : List[str]       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : Optional[int] = None       ,  _lowerCamelCase  : int = 4       ,  _lowerCamelCase  : float = 0.0       ,  _lowerCamelCase  : str = "geglu"       ,  _lowerCamelCase  : bool = False       ,  ):






         '''simple docstring'''



         super().__init__()
         __lowerCamelCase     :      Optional[Any]       						=						int(dim * mult							)
         __lowerCamelCase     :      Optional[int]       						=						dim_out if dim_out is not None else dim

         if activation_fn == "gelu":
               __lowerCamelCase     :      List[str]       						=						GELU(_lowerCamelCase       ,  _lowerCamelCase							)
         if activation_fn == "gelu-approximate":
               __lowerCamelCase     :      Optional[Any]       						=						GELU(_lowerCamelCase       ,  _lowerCamelCase       ,  approximate="""tanh"""							)
         elif activation_fn == "geglu":
               __lowerCamelCase     :      Any       						=						GEGLU(_lowerCamelCase       ,  _lowerCamelCase							)
         elif activation_fn == "geglu-approximate":
               __lowerCamelCase     :      List[Any]       						=						ApproximateGELU(_lowerCamelCase       ,  _lowerCamelCase							)

         __lowerCamelCase     :      List[Any]       						=						nn.ModuleList([]							)
         # project in
         self.net.append(_lowerCamelCase							)
         # project dropout
         self.net.append(nn.Dropout(_lowerCamelCase							)							)
         # project out
         self.net.append(nn.Linear(_lowerCamelCase       ,  _lowerCamelCase							)							)
         # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
         if final_dropout:
               self.net.append(nn.Dropout(_lowerCamelCase							)							)


   def   _snake_case    (  self  : Optional[Any]       ,  _lowerCamelCase  : List[Any]							):






         '''simple docstring'''



         for module in self.net:
               __lowerCamelCase     :      List[str]       						=						module(_lowerCamelCase							)
         return hidden_states



class 							_UpperCamelCase (						nn.Module						):

   '''simple docstring'''







   def __init__(  self  : Any       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : str = "none"							):






         '''simple docstring'''



         super().__init__()
         __lowerCamelCase     :      Tuple       						=						nn.Linear(_lowerCamelCase       ,  _lowerCamelCase							)
         __lowerCamelCase     :      Dict       						=						approximate

   def   _snake_case    (  self  : Union[str, Any]       ,  _lowerCamelCase  : str							):






         '''simple docstring'''



         if gate.device.type != "mps":
               return F.gelu(_lowerCamelCase       ,  approximate=self.approximate							)
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa							)       ,  approximate=self.approximate							).to(dtype=gate.dtype							)


   def   _snake_case    (  self  : Dict       ,  _lowerCamelCase  : List[str]							):






         '''simple docstring'''



         __lowerCamelCase     :      int       						=						self.proj(_lowerCamelCase							)
         __lowerCamelCase     :      Any       						=						self.gelu(_lowerCamelCase							)
         return hidden_states



class 							_UpperCamelCase (						nn.Module						):

   '''simple docstring'''







   def __init__(  self  : Optional[Any]       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : int							):






         '''simple docstring'''



         super().__init__()
         __lowerCamelCase     :      Optional[int]       						=						nn.Linear(_lowerCamelCase       ,  dim_out * 2							)

   def   _snake_case    (  self  : List[str]       ,  _lowerCamelCase  : List[str]							):






         '''simple docstring'''



         if gate.device.type != "mps":
               return F.gelu(_lowerCamelCase							)
         # mps: gelu is not implemented for float16
         return F.gelu(gate.to(dtype=torch.floataa							)							).to(dtype=gate.dtype							)


   def   _snake_case    (  self  : Optional[int]       ,  _lowerCamelCase  : Optional[int]							):






         '''simple docstring'''



         __lowerCamelCase						, __lowerCamelCase     :      Dict       						=						self.proj(_lowerCamelCase							).chunk(2       ,  dim=-1							)
         return hidden_states * self.gelu(_lowerCamelCase							)



class 							_UpperCamelCase (						nn.Module						):

   '''simple docstring'''







   def __init__(  self  : Union[str, Any]       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : int							):






         '''simple docstring'''



         super().__init__()
         __lowerCamelCase     :      str       						=						nn.Linear(_lowerCamelCase       ,  _lowerCamelCase							)


   def   _snake_case    (  self  : Dict       ,  _lowerCamelCase  : List[Any]							):






         '''simple docstring'''



         __lowerCamelCase     :      str       						=						self.proj(_lowerCamelCase							)
         return x * torch.sigmoid(1.702 * x							)



class 							_UpperCamelCase (						nn.Module						):

   '''simple docstring'''







   def __init__(  self  : Optional[Any]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : Dict							):






         '''simple docstring'''



         super().__init__()
         __lowerCamelCase     :      Tuple       						=						nn.Embedding(_lowerCamelCase       ,  _lowerCamelCase							)
         __lowerCamelCase     :      str       						=						nn.SiLU()
         __lowerCamelCase     :      str       						=						nn.Linear(_lowerCamelCase       ,  embedding_dim * 2							)
         __lowerCamelCase     :      Optional[Any]       						=						nn.LayerNorm(_lowerCamelCase       ,  elementwise_affine=_lowerCamelCase							)


   def   _snake_case    (  self  : Optional[Any]       ,  _lowerCamelCase  : Optional[int]       ,  _lowerCamelCase  : Dict							):






         '''simple docstring'''



         __lowerCamelCase     :      List[str]       						=						self.linear(self.silu(self.emb(_lowerCamelCase							)							)							)
         __lowerCamelCase						, __lowerCamelCase     :      int       						=						torch.chunk(_lowerCamelCase       ,  2							)
         __lowerCamelCase     :      Union[str, Any]       						=						self.norm(_lowerCamelCase							) * (1 + scale) + shift
         return x



class 							_UpperCamelCase (						nn.Module						):

   '''simple docstring'''







   def __init__(  self  : Union[str, Any]       ,  _lowerCamelCase  : str       ,  _lowerCamelCase  : List[str]							):






         '''simple docstring'''



         super().__init__()

         __lowerCamelCase     :      Optional[Any]       						=						CombinedTimestepLabelEmbeddings(_lowerCamelCase       ,  _lowerCamelCase							)

         __lowerCamelCase     :      Dict       						=						nn.SiLU()
         __lowerCamelCase     :      Any       						=						nn.Linear(_lowerCamelCase       ,  6 * embedding_dim       ,  bias=_lowerCamelCase							)
         __lowerCamelCase     :      Union[str, Any]       						=						nn.LayerNorm(_lowerCamelCase       ,  elementwise_affine=_lowerCamelCase       ,  eps=1E-6							)


   def   _snake_case    (  self  : Optional[Any]       ,  _lowerCamelCase  : Union[str, Any]       ,  _lowerCamelCase  : Optional[Any]       ,  _lowerCamelCase  : List[str]       ,  _lowerCamelCase  : Tuple=None							):






         '''simple docstring'''



         __lowerCamelCase     :      Dict       						=						self.linear(self.silu(self.emb(_lowerCamelCase       ,  _lowerCamelCase       ,  hidden_dtype=_lowerCamelCase							)							)							)
         __lowerCamelCase						, __lowerCamelCase						, __lowerCamelCase						, __lowerCamelCase						, __lowerCamelCase						, __lowerCamelCase     :      List[Any]       						=						emb.chunk(6       ,  dim=1							)
         __lowerCamelCase     :      Optional[int]       						=						self.norm(_lowerCamelCase							) * (1 + scale_msa[:, None]) + shift_msa[:, None]
         return x, gate_msa, shift_mlp, scale_mlp, gate_mlp



class 							_UpperCamelCase (						nn.Module						):

   '''simple docstring'''







   def __init__(  self  : Dict       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : int       ,  _lowerCamelCase  : Optional[str] = None       ,  _lowerCamelCase  : float = 1E-5							):






         '''simple docstring'''



         super().__init__()
         __lowerCamelCase     :      Tuple       						=						num_groups
         __lowerCamelCase     :      Tuple       						=						eps

         if act_fn is None:
               __lowerCamelCase     :      Any       						=						None
         else:
               __lowerCamelCase     :      Union[str, Any]       						=						get_activation(_lowerCamelCase							)

         __lowerCamelCase     :      Union[str, Any]       						=						nn.Linear(_lowerCamelCase       ,  out_dim * 2							)


   def   _snake_case    (  self  : List[str]       ,  _lowerCamelCase  : Tuple       ,  _lowerCamelCase  : Optional[Any]							):






         '''simple docstring'''



         if self.act:
               __lowerCamelCase     :      Any       						=						self.act(_lowerCamelCase							)
         __lowerCamelCase     :      Tuple       						=						self.linear(_lowerCamelCase							)
         __lowerCamelCase     :      int       						=						emb[:, :, None, None]
         __lowerCamelCase						, __lowerCamelCase     :      List[str]       						=						emb.chunk(2       ,  dim=1							)

         __lowerCamelCase     :      Optional[Any]       						=						F.group_norm(_lowerCamelCase       ,  self.num_groups       ,  eps=self.eps							)
         __lowerCamelCase     :      Dict       						=						x * (1 + scale) + shift
         return x







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase    :						Tuple     =							logging.get_logger(__name__)

__UpperCamelCase    :						Any     =							{
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}



class 							_UpperCamelCase (						A						):

   '''simple docstring'''







   a_				:							int    = "nllb-moe"
   a_				:							Optional[int]    = ["past_key_values"]
   a_				:							Optional[Any]    = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}

   def __init__(  self  : int       ,  _lowerCamelCase  : Union[str, Any]=1_2_8_1_1_2       ,  _lowerCamelCase  : List[str]=1_0_2_4       ,  _lowerCamelCase  : Optional[int]=1_2       ,  _lowerCamelCase  : int=4_0_9_6       ,  _lowerCamelCase  : Union[str, Any]=1_6       ,  _lowerCamelCase  : int=1_2       ,  _lowerCamelCase  : Optional[int]=4_0_9_6       ,  _lowerCamelCase  : Any=1_6       ,  _lowerCamelCase  : Any=0.05       ,  _lowerCamelCase  : List[Any]=0.05       ,  _lowerCamelCase  : Tuple=True       ,  _lowerCamelCase  : Any=True       ,  _lowerCamelCase  : Optional[int]="relu"       ,  _lowerCamelCase  : Dict=1_0_2_4       ,  _lowerCamelCase  : Optional[int]=0.1       ,  _lowerCamelCase  : Optional[int]=0.1       ,  _lowerCamelCase  : Optional[int]=0.0       ,  _lowerCamelCase  : List[Any]=0.02       ,  _lowerCamelCase  : List[str]=2       ,  _lowerCamelCase  : List[str]=True       ,  _lowerCamelCase  : Union[str, Any]=False       ,  _lowerCamelCase  : Optional[Any]="float32"       ,  _lowerCamelCase  : Union[str, Any]=False       ,  _lowerCamelCase  : Tuple=1_2_8       ,  _lowerCamelCase  : Dict=6_4       ,  _lowerCamelCase  : Any=4       ,  _lowerCamelCase  : List[Any]=4       ,  _lowerCamelCase  : str=0.001       ,  _lowerCamelCase  : Dict=0.001       ,  _lowerCamelCase  : Any="all"       ,  _lowerCamelCase  : Optional[Any]=False       ,  _lowerCamelCase  : List[Any]=False       ,  _lowerCamelCase  : Optional[Any]=1.0       ,  _lowerCamelCase  : List[str]=0.2       ,  _lowerCamelCase  : List[Any]=1       ,  _lowerCamelCase  : str=0       ,  _lowerCamelCase  : int=2       ,  _lowerCamelCase  : int=False       ,  **_lowerCamelCase  : List[str]       ,  ):






         '''simple docstring'''



         __lowerCamelCase     :      Optional[Any]       						=						vocab_size
         __lowerCamelCase     :      str       						=						max_position_embeddings
         __lowerCamelCase     :      int       						=						d_model
         __lowerCamelCase     :      Optional[Any]       						=						encoder_ffn_dim
         __lowerCamelCase     :      str       						=						encoder_layers
         __lowerCamelCase     :      Tuple       						=						encoder_attention_heads
         __lowerCamelCase     :      Union[str, Any]       						=						decoder_ffn_dim
         __lowerCamelCase     :      Dict       						=						decoder_layers
         __lowerCamelCase     :      Any       						=						decoder_attention_heads
         __lowerCamelCase     :      Union[str, Any]       						=						dropout
         __lowerCamelCase     :      Dict       						=						attention_dropout
         __lowerCamelCase     :      int       						=						activation_dropout
         __lowerCamelCase     :      Union[str, Any]       						=						activation_function
         __lowerCamelCase     :      Optional[int]       						=						init_std
         __lowerCamelCase     :      Union[str, Any]       						=						encoder_layerdrop
         __lowerCamelCase     :      Dict       						=						decoder_layerdrop
         __lowerCamelCase     :      List[str]       						=						use_cache
         __lowerCamelCase     :      Any       						=						encoder_layers
         __lowerCamelCase     :      List[Any]       						=						scale_embedding  # scale factor will be sqrt(d_model) if True
         __lowerCamelCase     :      Optional[Any]       						=						router_z_loss_coef
         __lowerCamelCase     :      Union[str, Any]       						=						router_aux_loss_coef
         __lowerCamelCase     :      Dict       						=						decoder_sparse_step
         __lowerCamelCase     :      Optional[int]       						=						encoder_sparse_step
         __lowerCamelCase     :      Any       						=						num_experts
         __lowerCamelCase     :      int       						=						expert_capacity
         __lowerCamelCase     :      Dict       						=						router_bias
         if router_dtype not in ["float32", "float16", "bfloat16"]:
               raise ValueError(F"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""							)
         __lowerCamelCase     :      Tuple       						=						router_dtype

         __lowerCamelCase     :      Dict       						=						router_ignore_padding_tokens
         __lowerCamelCase     :      str       						=						batch_prioritized_routing
         __lowerCamelCase     :      List[str]       						=						second_expert_policy
         __lowerCamelCase     :      Tuple       						=						normalize_router_prob_before_dropping
         __lowerCamelCase     :      Dict       						=						moe_eval_capacity_token_fraction
         __lowerCamelCase     :      Optional[int]       						=						moe_token_dropout
         __lowerCamelCase     :      Dict       						=						output_router_logits
         super().__init__(
             pad_token_id=_lowerCamelCase       ,  bos_token_id=_lowerCamelCase       ,  eos_token_id=_lowerCamelCase       ,  is_encoder_decoder=_lowerCamelCase       ,  decoder_start_token_id=_lowerCamelCase       ,  **_lowerCamelCase       ,  )



