



from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings



@maybe_allow_in_graph
class 	lowerCamelCase      (						nn.Module  ):







					'''simple docstring'''







					def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase=0.0    ,	lowerCAmelCase = None    ,	lowerCAmelCase = "geglu"    ,	lowerCAmelCase = None    ,	lowerCAmelCase = False    ,	lowerCAmelCase = False    ,	lowerCAmelCase = False    ,	lowerCAmelCase = False    ,	lowerCAmelCase = True    ,	lowerCAmelCase = "layer_norm"    ,	lowerCAmelCase = False    ,	):
							super().__init__()
							UpperCAmelCase_						=				only_cross_attention

							UpperCAmelCase_						=				(num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
							UpperCAmelCase_						=				(num_embeds_ada_norm is not None) and norm_type == "ada_norm"

							if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
									raise ValueError(
									    f'''`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'''
									    f''' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'''     )

							# Define 3 blocks. Each block has its own normalization layer.
							# 1. Self-Attn
							if self.use_ada_layer_norm:
									UpperCAmelCase_						=				AdaLayerNorm(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )
							elif self.use_ada_layer_norm_zero:
									UpperCAmelCase_						=				AdaLayerNormZero(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )
							else:
									UpperCAmelCase_						=				nn.LayerNorm(_SCREAMING_SNAKE_CASE    ,	elementwise_affine=_SCREAMING_SNAKE_CASE     )
							UpperCAmelCase_						=				Attention(
							    query_dim=_SCREAMING_SNAKE_CASE    ,	heads=_SCREAMING_SNAKE_CASE    ,	dim_head=_SCREAMING_SNAKE_CASE    ,	dropout=_SCREAMING_SNAKE_CASE    ,	bias=_SCREAMING_SNAKE_CASE    ,	cross_attention_dim=cross_attention_dim if only_cross_attention else None    ,	upcast_attention=_SCREAMING_SNAKE_CASE    ,	)

							# 2. Cross-Attn
							if cross_attention_dim is not None or double_self_attention:
									# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
									# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
									# the second cross attention block.
									UpperCAmelCase_						=				(
									    AdaLayerNorm(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )
									    if self.use_ada_layer_norm
									    else nn.LayerNorm(_SCREAMING_SNAKE_CASE    ,	elementwise_affine=_SCREAMING_SNAKE_CASE     )
									)
									UpperCAmelCase_						=				Attention(
									    query_dim=_SCREAMING_SNAKE_CASE    ,	cross_attention_dim=cross_attention_dim if not double_self_attention else None    ,	heads=_SCREAMING_SNAKE_CASE    ,	dim_head=_SCREAMING_SNAKE_CASE    ,	dropout=_SCREAMING_SNAKE_CASE    ,	bias=_SCREAMING_SNAKE_CASE    ,	upcast_attention=_SCREAMING_SNAKE_CASE    ,	)  # is self-attn if encoder_hidden_states is none
							else:
									UpperCAmelCase_						=				None
									UpperCAmelCase_						=				None

							# 3. Feed-forward
							UpperCAmelCase_						=				nn.LayerNorm(_SCREAMING_SNAKE_CASE    ,	elementwise_affine=_SCREAMING_SNAKE_CASE     )
							UpperCAmelCase_						=				FeedForward(_SCREAMING_SNAKE_CASE    ,	dropout=_SCREAMING_SNAKE_CASE    ,	activation_fn=_SCREAMING_SNAKE_CASE    ,	final_dropout=_SCREAMING_SNAKE_CASE     )

							# let chunk size default to None
							UpperCAmelCase_						=				None
							UpperCAmelCase_						=				0



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase     ):
							UpperCAmelCase_						=				chunk_size
							UpperCAmelCase_						=				dim



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase = None    ,	lowerCAmelCase = None    ,	lowerCAmelCase = None    ,	lowerCAmelCase = None    ,	lowerCAmelCase = None    ,	lowerCAmelCase = None    ,	):
							# Notice that normalization is always applied before the real computation in the following blocks.
							# 1. Self-Attention
							if self.use_ada_layer_norm:
									UpperCAmelCase_						=				self.norma(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )
							elif self.use_ada_layer_norm_zero:
									UpperCAmelCase_					,						UpperCAmelCase_					,						UpperCAmelCase_					,						UpperCAmelCase_					,						UpperCAmelCase_						=				self.norma(
									    _SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	hidden_dtype=hidden_states.dtype     )
							else:
									UpperCAmelCase_						=				self.norma(_SCREAMING_SNAKE_CASE     )

							UpperCAmelCase_						=				cross_attention_kwargs if cross_attention_kwargs is not None else {}

							UpperCAmelCase_						=				self.attna(
							    _SCREAMING_SNAKE_CASE    ,	encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None    ,	attention_mask=_SCREAMING_SNAKE_CASE    ,	**_SCREAMING_SNAKE_CASE    ,	)
							if self.use_ada_layer_norm_zero:
									UpperCAmelCase_						=				gate_msa.unsqueeze(1     ) * attn_output
							UpperCAmelCase_						=				attn_output + hidden_states

							# 2. Cross-Attention
							if self.attna is not None:
									UpperCAmelCase_						=				(
									    self.norma(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     ) if self.use_ada_layer_norm else self.norma(_SCREAMING_SNAKE_CASE     )
									)

									UpperCAmelCase_						=				self.attna(
									    _SCREAMING_SNAKE_CASE    ,	encoder_hidden_states=_SCREAMING_SNAKE_CASE    ,	attention_mask=_SCREAMING_SNAKE_CASE    ,	**_SCREAMING_SNAKE_CASE    ,	)
									UpperCAmelCase_						=				attn_output + hidden_states

							# 3. Feed-forward
							UpperCAmelCase_						=				self.norma(_SCREAMING_SNAKE_CASE     )

							if self.use_ada_layer_norm_zero:
									UpperCAmelCase_						=				norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

							if self._chunk_size is not None:
									# "feed_forward_chunk_size" can be used to save memory
									if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
											raise ValueError(
											    f'''`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'''     )

									UpperCAmelCase_						=				norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
									UpperCAmelCase_						=				torch.cat(
									    [self.ff(_SCREAMING_SNAKE_CASE     ) for hid_slice in norm_hidden_states.chunk(_SCREAMING_SNAKE_CASE    ,	dim=self._chunk_dim     )]    ,	dim=self._chunk_dim    ,	)
							else:
									UpperCAmelCase_						=				self.ff(_SCREAMING_SNAKE_CASE     )

							if self.use_ada_layer_norm_zero:
									UpperCAmelCase_						=				gate_mlp.unsqueeze(1     ) * ff_output

							UpperCAmelCase_						=				ff_output + hidden_states

							return hidden_states





class 	lowerCamelCase      (						nn.Module  ):







					'''simple docstring'''







					def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase = None    ,	lowerCAmelCase = 4    ,	lowerCAmelCase = 0.0    ,	lowerCAmelCase = "geglu"    ,	lowerCAmelCase = False    ,	):
							super().__init__()
							UpperCAmelCase_						=				int(dim * mult     )
							UpperCAmelCase_						=				dim_out if dim_out is not None else dim

							if activation_fn == "gelu":
									UpperCAmelCase_						=				GELU(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )
							if activation_fn == "gelu-approximate":
									UpperCAmelCase_						=				GELU(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	approximate="tanh"     )
							elif activation_fn == "geglu":
									UpperCAmelCase_						=				GEGLU(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )
							elif activation_fn == "geglu-approximate":
									UpperCAmelCase_						=				ApproximateGELU(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )

							UpperCAmelCase_						=				nn.ModuleList([]     )
							# project in
							self.net.append(_SCREAMING_SNAKE_CASE     )
							# project dropout
							self.net.append(nn.Dropout(_SCREAMING_SNAKE_CASE     )     )
							# project out
							self.net.append(nn.Linear(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )     )
							# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
							if final_dropout:
									self.net.append(nn.Dropout(_SCREAMING_SNAKE_CASE     )     )



					def    A__		(			self    ,	lowerCAmelCase     ):
							for module in self.net:
									UpperCAmelCase_						=				module(_SCREAMING_SNAKE_CASE     )
							return hidden_states





class 	lowerCamelCase      (						nn.Module  ):







					'''simple docstring'''







					def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase = "none"     ):
							super().__init__()
							UpperCAmelCase_						=				nn.Linear(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )
							UpperCAmelCase_						=				approximate



					def    A__		(			self    ,	lowerCAmelCase     ):
							if gate.device.type != "mps":
									return F.gelu(_SCREAMING_SNAKE_CASE    ,	approximate=self.approximate     )
							# mps: gelu is not implemented for float16
							return F.gelu(gate.to(dtype=torch.floataa     )    ,	approximate=self.approximate     ).to(dtype=gate.dtype     )



					def    A__		(			self    ,	lowerCAmelCase     ):
							UpperCAmelCase_						=				self.proj(_SCREAMING_SNAKE_CASE     )
							UpperCAmelCase_						=				self.gelu(_SCREAMING_SNAKE_CASE     )
							return hidden_states





class 	lowerCamelCase      (						nn.Module  ):







					'''simple docstring'''







					def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase     ):
							super().__init__()
							UpperCAmelCase_						=				nn.Linear(_SCREAMING_SNAKE_CASE    ,	dim_out * 2     )



					def    A__		(			self    ,	lowerCAmelCase     ):
							if gate.device.type != "mps":
									return F.gelu(_SCREAMING_SNAKE_CASE     )
							# mps: gelu is not implemented for float16
							return F.gelu(gate.to(dtype=torch.floataa     )     ).to(dtype=gate.dtype     )



					def    A__		(			self    ,	lowerCAmelCase     ):
							UpperCAmelCase_					,						UpperCAmelCase_						=				self.proj(_SCREAMING_SNAKE_CASE     ).chunk(2    ,	dim=-1     )
							return hidden_states * self.gelu(_SCREAMING_SNAKE_CASE     )





class 	lowerCamelCase      (						nn.Module  ):







					'''simple docstring'''







					def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase     ):
							super().__init__()
							UpperCAmelCase_						=				nn.Linear(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )



					def    A__		(			self    ,	lowerCAmelCase     ):
							UpperCAmelCase_						=				self.proj(_SCREAMING_SNAKE_CASE     )
							return x * torch.sigmoid(1.702 * x     )





class 	lowerCamelCase      (						nn.Module  ):







					'''simple docstring'''







					def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase     ):
							super().__init__()
							UpperCAmelCase_						=				nn.Embedding(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )
							UpperCAmelCase_						=				nn.SiLU()
							UpperCAmelCase_						=				nn.Linear(_SCREAMING_SNAKE_CASE    ,	embedding_dim * 2     )
							UpperCAmelCase_						=				nn.LayerNorm(_SCREAMING_SNAKE_CASE    ,	elementwise_affine=_SCREAMING_SNAKE_CASE     )



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase     ):
							UpperCAmelCase_						=				self.linear(self.silu(self.emb(_SCREAMING_SNAKE_CASE     )     )     )
							UpperCAmelCase_					,						UpperCAmelCase_						=				torch.chunk(_SCREAMING_SNAKE_CASE    ,	2     )
							UpperCAmelCase_						=				self.norm(_SCREAMING_SNAKE_CASE     ) * (1 + scale) + shift
							return x





class 	lowerCamelCase      (						nn.Module  ):







					'''simple docstring'''







					def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase     ):
							super().__init__()

							UpperCAmelCase_						=				CombinedTimestepLabelEmbeddings(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE     )

							UpperCAmelCase_						=				nn.SiLU()
							UpperCAmelCase_						=				nn.Linear(_SCREAMING_SNAKE_CASE    ,	6 * embedding_dim    ,	bias=_SCREAMING_SNAKE_CASE     )
							UpperCAmelCase_						=				nn.LayerNorm(_SCREAMING_SNAKE_CASE    ,	elementwise_affine=_SCREAMING_SNAKE_CASE    ,	eps=1e-6     )



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase=None     ):
							UpperCAmelCase_						=				self.linear(self.silu(self.emb(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	hidden_dtype=_SCREAMING_SNAKE_CASE     )     )     )
							UpperCAmelCase_					,						UpperCAmelCase_					,						UpperCAmelCase_					,						UpperCAmelCase_					,						UpperCAmelCase_					,						UpperCAmelCase_						=				emb.chunk(6    ,	dim=1     )
							UpperCAmelCase_						=				self.norm(_SCREAMING_SNAKE_CASE     ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
							return x, gate_msa, shift_mlp, scale_mlp, gate_mlp





class 	lowerCamelCase      (						nn.Module  ):







					'''simple docstring'''







					def __init__(			self    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase    ,	lowerCAmelCase = None    ,	lowerCAmelCase = 1e-5     ):
							super().__init__()
							UpperCAmelCase_						=				num_groups
							UpperCAmelCase_						=				eps

							if act_fn is None:
									UpperCAmelCase_						=				None
							else:
									UpperCAmelCase_						=				get_activation(_SCREAMING_SNAKE_CASE     )

							UpperCAmelCase_						=				nn.Linear(_SCREAMING_SNAKE_CASE    ,	out_dim * 2     )



					def    A__		(			self    ,	lowerCAmelCase    ,	lowerCAmelCase     ):
							if self.act:
									UpperCAmelCase_						=				self.act(_SCREAMING_SNAKE_CASE     )
							UpperCAmelCase_						=				self.linear(_SCREAMING_SNAKE_CASE     )
							UpperCAmelCase_						=				emb[:, :, None, None]
							UpperCAmelCase_					,						UpperCAmelCase_						=				emb.chunk(2    ,	dim=1     )

							UpperCAmelCase_						=				F.group_norm(_SCREAMING_SNAKE_CASE    ,	self.num_groups    ,	eps=self.eps     )
							UpperCAmelCase_						=				x * (1 + scale) + shift
							return x






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


SCREAMING_SNAKE_CASE     				=						{"configuration_xlnet": ["XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP", "XLNetConfig"]}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			SCREAMING_SNAKE_CASE     				=						["XLNetTokenizer"]

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			SCREAMING_SNAKE_CASE     				=						["XLNetTokenizerFast"]

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			SCREAMING_SNAKE_CASE     				=						[
			    "XLNET_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "XLNetForMultipleChoice",
			    "XLNetForQuestionAnswering",
			    "XLNetForQuestionAnsweringSimple",
			    "XLNetForSequenceClassification",
			    "XLNetForTokenClassification",
			    "XLNetLMHeadModel",
			    "XLNetModel",
			    "XLNetPreTrainedModel",
			    "load_tf_weights_in_xlnet",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			SCREAMING_SNAKE_CASE     				=						[
			    "TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "TFXLNetForMultipleChoice",
			    "TFXLNetForQuestionAnsweringSimple",
			    "TFXLNetForSequenceClassification",
			    "TFXLNetForTokenClassification",
			    "TFXLNetLMHeadModel",
			    "TFXLNetMainLayer",
			    "TFXLNetModel",
			    "TFXLNetPreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_xlnet import XLNetTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_xlnet_fast import XLNetTokenizerFast

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_xlnet import (
						    XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
						    XLNetForMultipleChoice,
						    XLNetForQuestionAnswering,
						    XLNetForQuestionAnsweringSimple,
						    XLNetForSequenceClassification,
						    XLNetForTokenClassification,
						    XLNetLMHeadModel,
						    XLNetModel,
						    XLNetPreTrainedModel,
						    load_tf_weights_in_xlnet,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_xlnet import (
						    TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFXLNetForMultipleChoice,
						    TFXLNetForQuestionAnsweringSimple,
						    TFXLNetForSequenceClassification,
						    TFXLNetForTokenClassification,
						    TFXLNetLMHeadModel,
						    TFXLNetMainLayer,
						    TFXLNetModel,
						    TFXLNetPreTrainedModel,
						)

else:
			import sys

			SCREAMING_SNAKE_CASE     				=						_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

