



from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings

@maybe_allow_in_graph
class 	lowerCamelCase_    ( nn.Module						):






						'''simple docstring'''







						def __init__(	self      :			Any	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			int=0.0	,	__lowerCamelCase      :			Optional[int] = None	,	__lowerCamelCase      :			str = "geglu"	,	__lowerCamelCase      :			Optional[int] = None	,	__lowerCamelCase      :			bool = False	,	__lowerCamelCase      :			bool = False	,	__lowerCamelCase      :			bool = False	,	__lowerCamelCase      :			bool = False	,	__lowerCamelCase      :			bool = True	,	__lowerCamelCase      :			str = "layer_norm"	,	__lowerCamelCase      :			bool = False	,	)      ->				Union[str, Any]:
									super().__init__()
									A							:					Tuple              =						only_cross_attention

									A							:					List[Any]              =						(num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
									A							:					Any              =						(num_embeds_ada_norm is not None) and norm_type == "ada_norm"

									if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
												raise ValueError(
												    F"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
												    F""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.""" )

									# Define 3 blocks. Each block has its own normalization layer.
									# 1. Self-Attn
									if self.use_ada_layer_norm:
												A							:					str              =						AdaLayerNorm(__lowerCamelCase	,	__lowerCamelCase )
									elif self.use_ada_layer_norm_zero:
												A							:					Union[str, Any]              =						AdaLayerNormZero(__lowerCamelCase	,	__lowerCamelCase )
									else:
												A							:					Tuple              =						nn.LayerNorm(__lowerCamelCase	,	elementwise_affine=__lowerCamelCase )
									A							:					Optional[int]              =						Attention(
									    query_dim=__lowerCamelCase	,	heads=__lowerCamelCase	,	dim_head=__lowerCamelCase	,	dropout=__lowerCamelCase	,	bias=__lowerCamelCase	,	cross_attention_dim=cross_attention_dim if only_cross_attention else None	,	upcast_attention=__lowerCamelCase	,	)

									# 2. Cross-Attn
									if cross_attention_dim is not None or double_self_attention:
												# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
												# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
												# the second cross attention block.
												A							:					Any              =						(
												    AdaLayerNorm(__lowerCamelCase	,	__lowerCamelCase )
												    if self.use_ada_layer_norm
												    else nn.LayerNorm(__lowerCamelCase	,	elementwise_affine=__lowerCamelCase )
												)
												A							:					int              =						Attention(
												    query_dim=__lowerCamelCase	,	cross_attention_dim=cross_attention_dim if not double_self_attention else None	,	heads=__lowerCamelCase	,	dim_head=__lowerCamelCase	,	dropout=__lowerCamelCase	,	bias=__lowerCamelCase	,	upcast_attention=__lowerCamelCase	,	)  # is self-attn if encoder_hidden_states is none
									else:
												A							:					Union[str, Any]              =						None
												A							:					str              =						None

									# 3. Feed-forward
									A							:					Dict              =						nn.LayerNorm(__lowerCamelCase	,	elementwise_affine=__lowerCamelCase )
									A							:					int              =						FeedForward(__lowerCamelCase	,	dropout=__lowerCamelCase	,	activation_fn=__lowerCamelCase	,	final_dropout=__lowerCamelCase )

									# let chunk size default to None
									A							:					str              =						None
									A							:					Any              =						0







						def 			SCREAMING_SNAKE_CASE__		(	self      :			int	,	__lowerCamelCase      :			Optional[int]	,	__lowerCamelCase      :			int )      ->				Union[str, Any]:
									# Sets chunk feed-forward
									A							:					Tuple              =						chunk_size
									A							:					Union[str, Any]              =						dim







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Any	,	__lowerCamelCase      :			torch.FloatTensor	,	__lowerCamelCase      :			Optional[torch.FloatTensor] = None	,	__lowerCamelCase      :			Optional[torch.FloatTensor] = None	,	__lowerCamelCase      :			Optional[torch.FloatTensor] = None	,	__lowerCamelCase      :			Optional[torch.LongTensor] = None	,	__lowerCamelCase      :			Dict[str, Any] = None	,	__lowerCamelCase      :			Optional[torch.LongTensor] = None	,	)      ->				str:
									# Notice that normalization is always applied before the real computation in the following blocks.
									# 1. Self-Attention
									if self.use_ada_layer_norm:
												A							:					Tuple              =						self.norma(__lowerCamelCase	,	__lowerCamelCase )
									elif self.use_ada_layer_norm_zero:
												A       , A       , A       , A       , A							:					Dict              =						self.norma(
												    __lowerCamelCase	,	__lowerCamelCase	,	__lowerCamelCase	,	hidden_dtype=hidden_states.dtype )
									else:
												A							:					Union[str, Any]              =						self.norma(__lowerCamelCase )

									A							:					Optional[Any]              =						cross_attention_kwargs if cross_attention_kwargs is not None else {}

									A							:					Dict              =						self.attna(
									    __lowerCamelCase	,	encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None	,	attention_mask=__lowerCamelCase	,	**__lowerCamelCase	,	)
									if self.use_ada_layer_norm_zero:
												A							:					Dict              =						gate_msa.unsqueeze(1 ) * attn_output
									A							:					Optional[Any]              =						attn_output + hidden_states

									# 2. Cross-Attention
									if self.attna is not None:
												A							:					Optional[Any]              =						(
												    self.norma(__lowerCamelCase	,	__lowerCamelCase ) if self.use_ada_layer_norm else self.norma(__lowerCamelCase )
												)

												A							:					int              =						self.attna(
												    __lowerCamelCase	,	encoder_hidden_states=__lowerCamelCase	,	attention_mask=__lowerCamelCase	,	**__lowerCamelCase	,	)
												A							:					List[Any]              =						attn_output + hidden_states

									# 3. Feed-forward
									A							:					Union[str, Any]              =						self.norma(__lowerCamelCase )

									if self.use_ada_layer_norm_zero:
												A							:					Dict              =						norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

									if self._chunk_size is not None:
												# "feed_forward_chunk_size" can be used to save memory
												if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
															raise ValueError(
															    F"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.""" )

												A							:					str              =						norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
												A							:					Dict              =						torch.cat(
												    [self.ff(__lowerCamelCase ) for hid_slice in norm_hidden_states.chunk(__lowerCamelCase	,	dim=self._chunk_dim )]	,	dim=self._chunk_dim	,	)
									else:
												A							:					List[str]              =						self.ff(__lowerCamelCase )

									if self.use_ada_layer_norm_zero:
												A							:					Optional[Any]              =						gate_mlp.unsqueeze(1 ) * ff_output

									A							:					Union[str, Any]              =						ff_output + hidden_states

									return hidden_states







class 	lowerCamelCase_    ( nn.Module						):






						'''simple docstring'''







						def __init__(	self      :			int	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			Optional[int] = None	,	__lowerCamelCase      :			int = 4	,	__lowerCamelCase      :			float = 0.0	,	__lowerCamelCase      :			str = "geglu"	,	__lowerCamelCase      :			bool = False	,	)      ->				Dict:
									super().__init__()
									A							:					List[str]              =						int(dim * mult )
									A							:					Tuple              =						dim_out if dim_out is not None else dim

									if activation_fn == "gelu":
												A							:					List[str]              =						GELU(__lowerCamelCase	,	__lowerCamelCase )
									if activation_fn == "gelu-approximate":
												A							:					str              =						GELU(__lowerCamelCase	,	__lowerCamelCase	,	approximate="tanh" )
									elif activation_fn == "geglu":
												A							:					Any              =						GEGLU(__lowerCamelCase	,	__lowerCamelCase )
									elif activation_fn == "geglu-approximate":
												A							:					Optional[int]              =						ApproximateGELU(__lowerCamelCase	,	__lowerCamelCase )

									A							:					Any              =						nn.ModuleList([] )
									# project in
									self.net.append(__lowerCamelCase )
									# project dropout
									self.net.append(nn.Dropout(__lowerCamelCase ) )
									# project out
									self.net.append(nn.Linear(__lowerCamelCase	,	__lowerCamelCase ) )
									# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
									if final_dropout:
												self.net.append(nn.Dropout(__lowerCamelCase ) )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Union[str, Any]	,	__lowerCamelCase      :			Optional[int] )      ->				List[str]:
									for module in self.net:
												A							:					Optional[int]              =						module(__lowerCamelCase )
									return hidden_states







class 	lowerCamelCase_    ( nn.Module						):






						'''simple docstring'''







						def __init__(	self      :			Optional[int]	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			str = "none" )      ->				Optional[Any]:
									super().__init__()
									A							:					List[Any]              =						nn.Linear(__lowerCamelCase	,	__lowerCamelCase )
									A							:					int              =						approximate







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[Any]	,	__lowerCamelCase      :			str )      ->				List[str]:
									if gate.device.type != "mps":
												return F.gelu(__lowerCamelCase	,	approximate=self.approximate )
									# mps: gelu is not implemented for float16
									return F.gelu(gate.to(dtype=torch.floataa )	,	approximate=self.approximate ).to(dtype=gate.dtype )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			List[Any]	,	__lowerCamelCase      :			str )      ->				List[Any]:
									A							:					List[Any]              =						self.proj(__lowerCamelCase )
									A							:					Optional[int]              =						self.gelu(__lowerCamelCase )
									return hidden_states







class 	lowerCamelCase_    ( nn.Module						):






						'''simple docstring'''







						def __init__(	self      :			List[str]	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			int )      ->				Optional[Any]:
									super().__init__()
									A							:					str              =						nn.Linear(__lowerCamelCase	,	dim_out * 2 )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			List[str]	,	__lowerCamelCase      :			Union[str, Any] )      ->				Optional[int]:
									if gate.device.type != "mps":
												return F.gelu(__lowerCamelCase )
									# mps: gelu is not implemented for float16
									return F.gelu(gate.to(dtype=torch.floataa ) ).to(dtype=gate.dtype )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[int]	,	__lowerCamelCase      :			Tuple )      ->				Tuple:
									A       , A							:					Union[str, Any]              =						self.proj(__lowerCamelCase ).chunk(2	,	dim=-1 )
									return hidden_states * self.gelu(__lowerCamelCase )







class 	lowerCamelCase_    ( nn.Module						):






						'''simple docstring'''







						def __init__(	self      :			Tuple	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			int )      ->				Union[str, Any]:
									super().__init__()
									A							:					int              =						nn.Linear(__lowerCamelCase	,	__lowerCamelCase )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[Any]	,	__lowerCamelCase      :			Union[str, Any] )      ->				Optional[Any]:
									A							:					Tuple              =						self.proj(__lowerCamelCase )
									return x * torch.sigmoid(1.702 * x )







class 	lowerCamelCase_    ( nn.Module						):






						'''simple docstring'''







						def __init__(	self      :			List[str]	,	__lowerCamelCase      :			Union[str, Any]	,	__lowerCamelCase      :			Optional[int] )      ->				Union[str, Any]:
									super().__init__()
									A							:					List[str]              =						nn.Embedding(__lowerCamelCase	,	__lowerCamelCase )
									A							:					Dict              =						nn.SiLU()
									A							:					Any              =						nn.Linear(__lowerCamelCase	,	embedding_dim * 2 )
									A							:					Optional[Any]              =						nn.LayerNorm(__lowerCamelCase	,	elementwise_affine=__lowerCamelCase )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[int]	,	__lowerCamelCase      :			Any	,	__lowerCamelCase      :			Any )      ->				Optional[int]:
									A							:					Optional[int]              =						self.linear(self.silu(self.emb(__lowerCamelCase ) ) )
									A       , A							:					str              =						torch.chunk(__lowerCamelCase	,	2 )
									A							:					Optional[int]              =						self.norm(__lowerCamelCase ) * (1 + scale) + shift
									return x







class 	lowerCamelCase_    ( nn.Module						):






						'''simple docstring'''







						def __init__(	self      :			Union[str, Any]	,	__lowerCamelCase      :			Optional[Any]	,	__lowerCamelCase      :			Union[str, Any] )      ->				Optional[int]:
									super().__init__()

									A							:					Dict              =						CombinedTimestepLabelEmbeddings(__lowerCamelCase	,	__lowerCamelCase )

									A							:					Dict              =						nn.SiLU()
									A							:					Any              =						nn.Linear(__lowerCamelCase	,	6 * embedding_dim	,	bias=__lowerCamelCase )
									A							:					Tuple              =						nn.LayerNorm(__lowerCamelCase	,	elementwise_affine=__lowerCamelCase	,	eps=1e-6 )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[Any]	,	__lowerCamelCase      :			List[Any]	,	__lowerCamelCase      :			Tuple	,	__lowerCamelCase      :			Any	,	__lowerCamelCase      :			Dict=None )      ->				Dict:
									A							:					Union[str, Any]              =						self.linear(self.silu(self.emb(__lowerCamelCase	,	__lowerCamelCase	,	hidden_dtype=__lowerCamelCase ) ) )
									A       , A       , A       , A       , A       , A							:					Optional[Any]              =						emb.chunk(6	,	dim=1 )
									A							:					int              =						self.norm(__lowerCamelCase ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
									return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class 	lowerCamelCase_    ( nn.Module						):






						'''simple docstring'''







						def __init__(	self      :			str	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			int	,	__lowerCamelCase      :			Optional[str] = None	,	__lowerCamelCase      :			float = 1e-5 )      ->				Dict:
									super().__init__()
									A							:					Dict              =						num_groups
									A							:					List[str]              =						eps

									if act_fn is None:
												A							:					List[str]              =						None
									else:
												A							:					Optional[int]              =						get_activation(__lowerCamelCase )

									A							:					List[Any]              =						nn.Linear(__lowerCamelCase	,	out_dim * 2 )







						def 			SCREAMING_SNAKE_CASE__		(	self      :			str	,	__lowerCamelCase      :			Union[str, Any]	,	__lowerCamelCase      :			int )      ->				str:
									if self.act:
												A							:					int              =						self.act(__lowerCamelCase )
									A							:					Optional[int]              =						self.linear(__lowerCamelCase )
									A							:					Any              =						emb[:, :, None, None]
									A       , A							:					Union[str, Any]              =						emb.chunk(2	,	dim=1 )

									A							:					Optional[Any]              =						F.group_norm(__lowerCamelCase	,	self.num_groups	,	eps=self.eps )
									A							:					Any              =						x * (1 + scale) + shift
									return x




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow


if is_torch_available():
							import torch

							from transformers import XLMRobertaModel

@require_sentencepiece
@require_tokenizers
@require_torch
class 	lowerCamelCase_    ( unittest.TestCase						):






						'''simple docstring'''







						@slow
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Any )      ->				Union[str, Any]:
									A							:					Union[str, Any]              =						XLMRobertaModel.from_pretrained("xlm-roberta-base" )
									A							:					Tuple              =						torch.tensor([[0, 5_81, 1_02_69, 83, 9_99_42, 1_36, 6_07_42, 23, 70, 8_05_83, 1_82_76, 2]] )
									# The dog is cute and lives in the garden house

									A							:					Tuple              =						torch.Size((1, 12, 7_68) )  # batch_size, sequence_length, embedding_vector_dim
									A							:					List[str]              =						torch.tensor(
									    [[-0.0101, 0.1218, -0.0803, 0.0801, 0.1327, 0.0776, -0.1215, 0.2383, 0.3338, 0.3106, 0.0300, 0.0252]] )
									#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')
									#  xlmr.eval()
									#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
									with torch.no_grad():
												A							:					Tuple              =						model(__lowerCamelCase )["last_hidden_state"].detach()
									self.assertEqual(output.shape	,	__lowerCamelCase )
									# compare the actual values for a slice of last dim
									self.assertTrue(torch.allclose(output[:, :, -1]	,	__lowerCamelCase	,	atol=1e-3 ) )







						@slow
						def 			SCREAMING_SNAKE_CASE__		(	self      :			Optional[int] )      ->				int:
									A							:					str              =						XLMRobertaModel.from_pretrained("xlm-roberta-large" )
									A							:					List[Any]              =						torch.tensor([[0, 5_81, 1_02_69, 83, 9_99_42, 1_36, 6_07_42, 23, 70, 8_05_83, 1_82_76, 2]] )
									# The dog is cute and lives in the garden house

									A							:					Optional[Any]              =						torch.Size((1, 12, 10_24) )  # batch_size, sequence_length, embedding_vector_dim
									A							:					List[Any]              =						torch.tensor(
									    [[-0.0699, -0.0318, 0.0705, -0.1241, 0.0999, -0.0520, 0.1004, -0.1838, -0.4704, 0.1437, 0.0821, 0.0126]] )
									#  xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.large')
									#  xlmr.eval()
									#  expected_output_values_last_dim = xlmr.extract_features(input_ids[0])[:, :, -1]
									with torch.no_grad():
												A							:					Optional[int]              =						model(__lowerCamelCase )["last_hidden_state"].detach()
									self.assertEqual(output.shape	,	__lowerCamelCase )
									# compare the actual values for a slice of last dim
									self.assertTrue(torch.allclose(output[:, :, -1]	,	__lowerCamelCase	,	atol=1e-3 ) )