



"""simple docstring"""





from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings



@maybe_allow_in_graph
class    _lowerCAmelCase    (				nn.Module		):





		"""simple docstring"""






		def __init__(					self							,    _lowercase							,    _lowercase							,    _lowercase							,    _lowercase=0.0							,    _lowercase = None							,    _lowercase = "geglu"							,    _lowercase = None							,    _lowercase = False							,    _lowercase = False							,    _lowercase = False							,    _lowercase = False							,    _lowercase = True							,    _lowercase = "layer_norm"							,    _lowercase = False							,    ) ->						List[str]:







								'''simple docstring'''

								super().__init__()
								snake_case_   : int						=			only_cross_attention

								snake_case_   : Tuple						=			(num_embeds_ada_norm is not None) and norm_type == '''ada_norm_zero'''
								snake_case_   : Optional[Any]						=			(num_embeds_ada_norm is not None) and norm_type == '''ada_norm'''

								if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
														raise ValueError(
														    f'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
														    f' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'   )

								# Define 3 blocks. Each block has its own normalization layer.
								# 1. Self-Attn
								if self.use_ada_layer_norm:
														snake_case_   : Union[str, Any]						=			AdaLayerNorm(UpperCAmelCase__							,    UpperCAmelCase__   )
								elif self.use_ada_layer_norm_zero:
														snake_case_   : List[Any]						=			AdaLayerNormZero(UpperCAmelCase__							,    UpperCAmelCase__   )
								else:
														snake_case_   : Tuple						=			nn.LayerNorm(UpperCAmelCase__							,    elementwise_affine=UpperCAmelCase__   )
								snake_case_   : List[Any]						=			Attention(
								    query_dim=UpperCAmelCase__							,    heads=UpperCAmelCase__							,    dim_head=UpperCAmelCase__							,    dropout=UpperCAmelCase__							,    bias=UpperCAmelCase__							,    cross_attention_dim=cross_attention_dim if only_cross_attention else None							,    upcast_attention=UpperCAmelCase__							,    )

								# 2. Cross-Attn
								if cross_attention_dim is not None or double_self_attention:
														# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
														# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
														# the second cross attention block.
														snake_case_   : List[Any]						=			(
														    AdaLayerNorm(UpperCAmelCase__							,    UpperCAmelCase__   )
														    if self.use_ada_layer_norm
														    else nn.LayerNorm(UpperCAmelCase__							,    elementwise_affine=UpperCAmelCase__   )
														)
														snake_case_   : Union[str, Any]						=			Attention(
														    query_dim=UpperCAmelCase__							,    cross_attention_dim=cross_attention_dim if not double_self_attention else None							,    heads=UpperCAmelCase__							,    dim_head=UpperCAmelCase__							,    dropout=UpperCAmelCase__							,    bias=UpperCAmelCase__							,    upcast_attention=UpperCAmelCase__							,    )  # is self-attn if encoder_hidden_states is none
								else:
														snake_case_   : Union[str, Any]						=			None
														snake_case_   : Optional[Any]						=			None

								# 3. Feed-forward
								snake_case_   : Dict						=			nn.LayerNorm(UpperCAmelCase__							,    elementwise_affine=UpperCAmelCase__   )
								snake_case_   : str						=			FeedForward(UpperCAmelCase__							,    dropout=UpperCAmelCase__							,    activation_fn=UpperCAmelCase__							,    final_dropout=UpperCAmelCase__   )

								# let chunk size default to None
								snake_case_   : List[str]						=			None
								snake_case_   : List[str]						=			0





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase   ) ->						List[str]:







								'''simple docstring'''

								snake_case_   : Optional[Any]						=			chunk_size
								snake_case_   : Optional[Any]						=			dim





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase = None							,    _lowercase = None							,    _lowercase = None							,    _lowercase = None							,    _lowercase = None							,    _lowercase = None							,    ) ->						int:







								'''simple docstring'''

								if self.use_ada_layer_norm:
														snake_case_   : Dict						=			self.norma(UpperCAmelCase__							,    UpperCAmelCase__   )
								elif self.use_ada_layer_norm_zero:
														snake_case_   : str						=			self.norma(
														    UpperCAmelCase__							,    UpperCAmelCase__							,    UpperCAmelCase__							,    hidden_dtype=hidden_states.dtype   )
								else:
														snake_case_   : Tuple						=			self.norma(UpperCAmelCase__   )

								snake_case_   : List[str]						=			cross_attention_kwargs if cross_attention_kwargs is not None else {}

								snake_case_   : Optional[Any]						=			self.attna(
								    UpperCAmelCase__							,    encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None							,    attention_mask=UpperCAmelCase__							,    **UpperCAmelCase__							,    )
								if self.use_ada_layer_norm_zero:
														snake_case_   : Optional[Any]						=			gate_msa.unsqueeze(1   ) * attn_output
								snake_case_   : Any						=			attn_output + hidden_states

								# 2. Cross-Attention
								if self.attna is not None:
														snake_case_   : str						=			(
														    self.norma(UpperCAmelCase__							,    UpperCAmelCase__   ) if self.use_ada_layer_norm else self.norma(UpperCAmelCase__   )
														)

														snake_case_   : int						=			self.attna(
														    UpperCAmelCase__							,    encoder_hidden_states=UpperCAmelCase__							,    attention_mask=UpperCAmelCase__							,    **UpperCAmelCase__							,    )
														snake_case_   : Tuple						=			attn_output + hidden_states

								# 3. Feed-forward
								snake_case_   : List[str]						=			self.norma(UpperCAmelCase__   )

								if self.use_ada_layer_norm_zero:
														snake_case_   : Union[str, Any]						=			norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

								if self._chunk_size is not None:
														# "feed_forward_chunk_size" can be used to save memory
														if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
																				raise ValueError(
																				    f'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'   )

														snake_case_   : Any						=			norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
														snake_case_   : List[str]						=			torch.cat(
														    [self.ff(UpperCAmelCase__   ) for hid_slice in norm_hidden_states.chunk(UpperCAmelCase__							,    dim=self._chunk_dim   )]							,    dim=self._chunk_dim							,    )
								else:
														snake_case_   : str						=			self.ff(UpperCAmelCase__   )

								if self.use_ada_layer_norm_zero:
														snake_case_   : int						=			gate_mlp.unsqueeze(1   ) * ff_output

								snake_case_   : Any						=			ff_output + hidden_states

								return hidden_states




class    _lowerCAmelCase    (				nn.Module		):





		"""simple docstring"""






		def __init__(					self							,    _lowercase							,    _lowercase = None							,    _lowercase = 4							,    _lowercase = 0.0							,    _lowercase = "geglu"							,    _lowercase = False							,    ) ->						Union[str, Any]:







								'''simple docstring'''

								super().__init__()
								snake_case_   : List[Any]						=			int(dim * mult   )
								snake_case_   : Optional[int]						=			dim_out if dim_out is not None else dim

								if activation_fn == "gelu":
														snake_case_   : List[Any]						=			GELU(UpperCAmelCase__							,    UpperCAmelCase__   )
								if activation_fn == "gelu-approximate":
														snake_case_   : int						=			GELU(UpperCAmelCase__							,    UpperCAmelCase__							,    approximate="""tanh"""   )
								elif activation_fn == "geglu":
														snake_case_   : Optional[int]						=			GEGLU(UpperCAmelCase__							,    UpperCAmelCase__   )
								elif activation_fn == "geglu-approximate":
														snake_case_   : Tuple						=			ApproximateGELU(UpperCAmelCase__							,    UpperCAmelCase__   )

								snake_case_   : Optional[Any]						=			nn.ModuleList([]   )
								# project in
								self.net.append(UpperCAmelCase__   )
								# project dropout
								self.net.append(nn.Dropout(UpperCAmelCase__   )   )
								# project out
								self.net.append(nn.Linear(UpperCAmelCase__							,    UpperCAmelCase__   )   )
								# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
								if final_dropout:
														self.net.append(nn.Dropout(UpperCAmelCase__   )   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						int:







								'''simple docstring'''

								for module in self.net:
														snake_case_   : List[Any]						=			module(UpperCAmelCase__   )
								return hidden_states




class    _lowerCAmelCase    (				nn.Module		):





		"""simple docstring"""






		def __init__(					self							,    _lowercase							,    _lowercase							,    _lowercase = "none"   ) ->						Optional[Any]:







								'''simple docstring'''

								super().__init__()
								snake_case_   : str						=			nn.Linear(UpperCAmelCase__							,    UpperCAmelCase__   )
								snake_case_   : Union[str, Any]						=			approximate





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						Optional[int]:







								'''simple docstring'''

								if gate.device.type != "mps":
														return F.gelu(UpperCAmelCase__							,    approximate=self.approximate   )
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa   )							,    approximate=self.approximate   ).to(dtype=gate.dtype   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						Any:







								'''simple docstring'''

								snake_case_   : Optional[int]						=			self.proj(UpperCAmelCase__   )
								snake_case_   : Optional[Any]						=			self.gelu(UpperCAmelCase__   )
								return hidden_states




class    _lowerCAmelCase    (				nn.Module		):





		"""simple docstring"""






		def __init__(					self							,    _lowercase							,    _lowercase   ) ->						int:







								'''simple docstring'''

								super().__init__()
								snake_case_   : List[Any]						=			nn.Linear(UpperCAmelCase__							,    dim_out * 2   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						Dict:







								'''simple docstring'''

								if gate.device.type != "mps":
														return F.gelu(UpperCAmelCase__   )
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa   )   ).to(dtype=gate.dtype   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						Union[str, Any]:







								'''simple docstring'''

								snake_case_   : Union[str, Any]						=			self.proj(UpperCAmelCase__   ).chunk(2							,    dim=-1   )
								return hidden_states * self.gelu(UpperCAmelCase__   )




class    _lowerCAmelCase    (				nn.Module		):





		"""simple docstring"""






		def __init__(					self							,    _lowercase							,    _lowercase   ) ->						Optional[Any]:







								'''simple docstring'''

								super().__init__()
								snake_case_   : Tuple						=			nn.Linear(UpperCAmelCase__							,    UpperCAmelCase__   )





		def    UpperCAmelCase__							(					self							,    _lowercase   ) ->						Tuple:







								'''simple docstring'''

								snake_case_   : Any						=			self.proj(UpperCAmelCase__   )
								return x * torch.sigmoid(1.702 * x   )




class    _lowerCAmelCase    (				nn.Module		):





		"""simple docstring"""






		def __init__(					self							,    _lowercase							,    _lowercase   ) ->						Dict:







								'''simple docstring'''

								super().__init__()
								snake_case_   : Tuple						=			nn.Embedding(UpperCAmelCase__							,    UpperCAmelCase__   )
								snake_case_   : Dict						=			nn.SiLU()
								snake_case_   : int						=			nn.Linear(UpperCAmelCase__							,    embedding_dim * 2   )
								snake_case_   : List[Any]						=			nn.LayerNorm(UpperCAmelCase__							,    elementwise_affine=UpperCAmelCase__   )





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase   ) ->						Optional[Any]:







								'''simple docstring'''

								snake_case_   : List[Any]						=			self.linear(self.silu(self.emb(UpperCAmelCase__   )   )   )
								snake_case_   : Any						=			torch.chunk(UpperCAmelCase__							,    2   )
								snake_case_   : Tuple						=			self.norm(UpperCAmelCase__   ) * (1 + scale) + shift
								return x




class    _lowerCAmelCase    (				nn.Module		):





		"""simple docstring"""






		def __init__(					self							,    _lowercase							,    _lowercase   ) ->						str:







								'''simple docstring'''

								super().__init__()

								snake_case_   : int						=			CombinedTimestepLabelEmbeddings(UpperCAmelCase__							,    UpperCAmelCase__   )

								snake_case_   : Any						=			nn.SiLU()
								snake_case_   : Optional[Any]						=			nn.Linear(UpperCAmelCase__							,    6 * embedding_dim							,    bias=UpperCAmelCase__   )
								snake_case_   : Optional[int]						=			nn.LayerNorm(UpperCAmelCase__							,    elementwise_affine=UpperCAmelCase__							,    eps=1E-6   )





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase							,    _lowercase							,    _lowercase=None   ) ->						Optional[Any]:







								'''simple docstring'''

								snake_case_   : Union[str, Any]						=			self.linear(self.silu(self.emb(UpperCAmelCase__							,    UpperCAmelCase__							,    hidden_dtype=UpperCAmelCase__   )   )   )
								snake_case_   : int						=			emb.chunk(6							,    dim=1   )
								snake_case_   : Dict						=			self.norm(UpperCAmelCase__   ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
								return x, gate_msa, shift_mlp, scale_mlp, gate_mlp




class    _lowerCAmelCase    (				nn.Module		):





		"""simple docstring"""






		def __init__(					self							,    _lowercase							,    _lowercase							,    _lowercase							,    _lowercase = None							,    _lowercase = 1E-5   ) ->						Optional[int]:







								'''simple docstring'''

								super().__init__()
								snake_case_   : Tuple						=			num_groups
								snake_case_   : List[Any]						=			eps

								if act_fn is None:
														snake_case_   : Tuple						=			None
								else:
														snake_case_   : Optional[int]						=			get_activation(UpperCAmelCase__   )

								snake_case_   : Optional[Any]						=			nn.Linear(UpperCAmelCase__							,    out_dim * 2   )





		def    UpperCAmelCase__							(					self							,    _lowercase							,    _lowercase   ) ->						List[Any]:







								'''simple docstring'''

								if self.act:
														snake_case_   : List[Any]						=			self.act(UpperCAmelCase__   )
								snake_case_   : Dict						=			self.linear(UpperCAmelCase__   )
								snake_case_   : int						=			emb[:, :, None, None]
								snake_case_   : List[Any]						=			emb.chunk(2							,    dim=1   )

								snake_case_   : str						=			F.group_norm(UpperCAmelCase__							,    self.num_groups							,    eps=self.eps   )
								snake_case_   : int						=			x * (1 + scale) + shift
								return x






"""simple docstring"""





def 	__lowerCAmelCase						(		__UpperCamelCase      :			int					, __UpperCamelCase      :			int    ):


						'''simple docstring'''






						if a < 0 or b < 0:
												raise ValueError("""the value of both inputs must be positive"""    )

						snake_case_   : List[str]						=			str(bin(__UpperCamelCase    )    )[2:]  # remove the leading "0b"
						snake_case_   : Union[str, Any]						=			str(bin(__UpperCamelCase    )    )[2:]  # remove the leading "0b"

						snake_case_   : int						=			max(len(__UpperCamelCase    )					, len(__UpperCamelCase    )    )

						return "0b" + "".join(
						    str(int(char_a != char_b    )    )
						    for char_a, char_b in zip(a_binary.zfill(__UpperCamelCase    )					, b_binary.zfill(__UpperCamelCase    )    )    )


if __name__ == "__main__":
					import doctest

					doctest.testmod()

