





from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings





@maybe_allow_in_graph
class      lowercase__    (							nn.Module  ):







  def __init__(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase=0.0    ,						_lowercase = None    ,						_lowercase = "geglu"    ,						_lowercase = None    ,						_lowercase = False    ,						_lowercase = False    ,						_lowercase = False    ,						_lowercase = False    ,						_lowercase = True    ,						_lowercase = "layer_norm"    ,						_lowercase = False    ,						):
       super().__init__()
       lowerCAmelCase_					:       int    =		only_cross_attention

       lowerCAmelCase_					:       Any    =		(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
       lowerCAmelCase_					:       List[str]    =		(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

       if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
            raise ValueError(
                F'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
                F' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'						)

       # Define 3 blocks. Each block has its own normalization layer.
       # 1. Self-Attn
       if self.use_ada_layer_norm:
            lowerCAmelCase_					:       List[Any]    =		AdaLayerNorm(UpperCAmelCase_    ,						UpperCAmelCase_						)
       elif self.use_ada_layer_norm_zero:
            lowerCAmelCase_					:       Union[str, Any]    =		AdaLayerNormZero(UpperCAmelCase_    ,						UpperCAmelCase_						)
       else:
            lowerCAmelCase_					:       Optional[int]    =		nn.LayerNorm(UpperCAmelCase_    ,						elementwise_affine=UpperCAmelCase_						)
       lowerCAmelCase_					:       Union[str, Any]    =		Attention(
           query_dim=UpperCAmelCase_    ,						heads=UpperCAmelCase_    ,						dim_head=UpperCAmelCase_    ,						dropout=UpperCAmelCase_    ,						bias=UpperCAmelCase_    ,						cross_attention_dim=cross_attention_dim if only_cross_attention else None    ,						upcast_attention=UpperCAmelCase_    ,						)

       # 2. Cross-Attn
       if cross_attention_dim is not None or double_self_attention:
            # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
            # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
            # the second cross attention block.
            lowerCAmelCase_					:       int    =		(
                AdaLayerNorm(UpperCAmelCase_    ,						UpperCAmelCase_						)
                if self.use_ada_layer_norm
                else nn.LayerNorm(UpperCAmelCase_    ,						elementwise_affine=UpperCAmelCase_						)
            )
            lowerCAmelCase_					:       str    =		Attention(
                query_dim=UpperCAmelCase_    ,						cross_attention_dim=cross_attention_dim if not double_self_attention else None    ,						heads=UpperCAmelCase_    ,						dim_head=UpperCAmelCase_    ,						dropout=UpperCAmelCase_    ,						bias=UpperCAmelCase_    ,						upcast_attention=UpperCAmelCase_    ,						)  # is self-attn if encoder_hidden_states is none
       else:
            lowerCAmelCase_					:       Optional[Any]    =		None
            lowerCAmelCase_					:       List[str]    =		None

       # 3. Feed-forward
       lowerCAmelCase_					:       str    =		nn.LayerNorm(UpperCAmelCase_    ,						elementwise_affine=UpperCAmelCase_						)
       lowerCAmelCase_					:       Dict    =		FeedForward(UpperCAmelCase_    ,						dropout=UpperCAmelCase_    ,						activation_fn=UpperCAmelCase_    ,						final_dropout=UpperCAmelCase_						)

       # let chunk size default to None
       lowerCAmelCase_					:       Union[str, Any]    =		None
       lowerCAmelCase_					:       int    =		0







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       Tuple    =		chunk_size
       lowerCAmelCase_					:       Tuple    =		dim







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase = None    ,						_lowercase = None    ,						_lowercase = None    ,						_lowercase = None    ,						_lowercase = None    ,						_lowercase = None    ,						):



       if self.use_ada_layer_norm:
            lowerCAmelCase_					:       str    =		self.norma(UpperCAmelCase_    ,						UpperCAmelCase_						)
       elif self.use_ada_layer_norm_zero:
            lowerCAmelCase_   ,  lowerCAmelCase_   ,  lowerCAmelCase_   ,  lowerCAmelCase_   ,  lowerCAmelCase_					:       List[Any]    =		self.norma(
                UpperCAmelCase_    ,						UpperCAmelCase_    ,						UpperCAmelCase_    ,						hidden_dtype=hidden_states.dtype						)
       else:
            lowerCAmelCase_					:       Dict    =		self.norma(UpperCAmelCase_						)

       lowerCAmelCase_					:       str    =		cross_attention_kwargs if cross_attention_kwargs is not None else {}

       lowerCAmelCase_					:       str    =		self.attna(
           UpperCAmelCase_    ,						encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None    ,						attention_mask=UpperCAmelCase_    ,						**UpperCAmelCase_    ,						)
       if self.use_ada_layer_norm_zero:
            lowerCAmelCase_					:       Union[str, Any]    =		gate_msa.unsqueeze(1						) * attn_output
       lowerCAmelCase_					:       Union[str, Any]    =		attn_output + hidden_states

       # 2. Cross-Attention
       if self.attna is not None:
            lowerCAmelCase_					:       Any    =		(
                self.norma(UpperCAmelCase_    ,						UpperCAmelCase_						) if self.use_ada_layer_norm else self.norma(UpperCAmelCase_						)
            )

            lowerCAmelCase_					:       Optional[Any]    =		self.attna(
                UpperCAmelCase_    ,						encoder_hidden_states=UpperCAmelCase_    ,						attention_mask=UpperCAmelCase_    ,						**UpperCAmelCase_    ,						)
            lowerCAmelCase_					:       Union[str, Any]    =		attn_output + hidden_states

       # 3. Feed-forward
       lowerCAmelCase_					:       Union[str, Any]    =		self.norma(UpperCAmelCase_						)

       if self.use_ada_layer_norm_zero:
            lowerCAmelCase_					:       Any    =		norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

       if self._chunk_size is not None:
            # "feed_forward_chunk_size" can be used to save memory
            if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                 raise ValueError(
                     F'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'						)

            lowerCAmelCase_					:       List[Any]    =		norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
            lowerCAmelCase_					:       Optional[Any]    =		torch.cat(
                [self.ff(UpperCAmelCase_						) for hid_slice in norm_hidden_states.chunk(UpperCAmelCase_    ,						dim=self._chunk_dim						)]    ,						dim=self._chunk_dim    ,						)
       else:
            lowerCAmelCase_					:       List[Any]    =		self.ff(UpperCAmelCase_						)

       if self.use_ada_layer_norm_zero:
            lowerCAmelCase_					:       Dict    =		gate_mlp.unsqueeze(1						) * ff_output

       lowerCAmelCase_					:       Tuple    =		ff_output + hidden_states

       return hidden_states





class      lowercase__    (							nn.Module  ):







  def __init__(	self    ,						_lowercase    ,						_lowercase = None    ,						_lowercase = 4    ,						_lowercase = 0.0    ,						_lowercase = "geglu"    ,						_lowercase = False    ,						):
       super().__init__()
       lowerCAmelCase_					:       Optional[Any]    =		int(dim * mult						)
       lowerCAmelCase_					:       int    =		dim_out if dim_out is not None else dim

       if activation_fn == "gelu":
            lowerCAmelCase_					:       Any    =		GELU(UpperCAmelCase_    ,						UpperCAmelCase_						)
       if activation_fn == "gelu-approximate":
            lowerCAmelCase_					:       Dict    =		GELU(UpperCAmelCase_    ,						UpperCAmelCase_    ,						approximate="""tanh"""						)
       elif activation_fn == "geglu":
            lowerCAmelCase_					:       Optional[int]    =		GEGLU(UpperCAmelCase_    ,						UpperCAmelCase_						)
       elif activation_fn == "geglu-approximate":
            lowerCAmelCase_					:       List[str]    =		ApproximateGELU(UpperCAmelCase_    ,						UpperCAmelCase_						)

       lowerCAmelCase_					:       Any    =		nn.ModuleList([]						)
       # project in
       self.net.append(UpperCAmelCase_						)
       # project dropout
       self.net.append(nn.Dropout(UpperCAmelCase_						)						)
       # project out
       self.net.append(nn.Linear(UpperCAmelCase_    ,						UpperCAmelCase_						)						)
       # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
       if final_dropout:
            self.net.append(nn.Dropout(UpperCAmelCase_						)						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):



       for module in self.net:
            lowerCAmelCase_					:       Dict    =		module(UpperCAmelCase_						)
       return hidden_states





class      lowercase__    (							nn.Module  ):







  def __init__(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase = "none"						):
       super().__init__()
       lowerCAmelCase_					:       Union[str, Any]    =		nn.Linear(UpperCAmelCase_    ,						UpperCAmelCase_						)
       lowerCAmelCase_					:       Optional[Any]    =		approximate







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):



       if gate.device.type != "mps":
            return F.gelu(UpperCAmelCase_    ,						approximate=self.approximate						)
       # mps: gelu is not implemented for float16
       return F.gelu(gate.to(dtype=torch.floataa						)    ,						approximate=self.approximate						).to(dtype=gate.dtype						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_					:       List[Any]    =		self.proj(UpperCAmelCase_						)
       lowerCAmelCase_					:       Any    =		self.gelu(UpperCAmelCase_						)
       return hidden_states





class      lowercase__    (							nn.Module  ):







  def __init__(	self    ,						_lowercase    ,						_lowercase						):
       super().__init__()
       lowerCAmelCase_					:       Optional[int]    =		nn.Linear(UpperCAmelCase_    ,						dim_out * 2						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):



       if gate.device.type != "mps":
            return F.gelu(UpperCAmelCase_						)
       # mps: gelu is not implemented for float16
       return F.gelu(gate.to(dtype=torch.floataa						)						).to(dtype=gate.dtype						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_   ,  lowerCAmelCase_					:       str    =		self.proj(UpperCAmelCase_						).chunk(2    ,						dim=-1						)
       return hidden_states * self.gelu(UpperCAmelCase_						)





class      lowercase__    (							nn.Module  ):







  def __init__(	self    ,						_lowercase    ,						_lowercase						):
       super().__init__()
       lowerCAmelCase_					:       Optional[Any]    =		nn.Linear(UpperCAmelCase_    ,						UpperCAmelCase_						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase						):
       lowerCAmelCase_					:       Optional[Any]    =		self.proj(UpperCAmelCase_						)
       return x * torch.sigmoid(1.702 * x						)





class      lowercase__    (							nn.Module  ):







  def __init__(	self    ,						_lowercase    ,						_lowercase						):
       super().__init__()
       lowerCAmelCase_					:       Any    =		nn.Embedding(UpperCAmelCase_    ,						UpperCAmelCase_						)
       lowerCAmelCase_					:       int    =		nn.SiLU()
       lowerCAmelCase_					:       int    =		nn.Linear(UpperCAmelCase_    ,						embedding_dim * 2						)
       lowerCAmelCase_					:       Union[str, Any]    =		nn.LayerNorm(UpperCAmelCase_    ,						elementwise_affine=UpperCAmelCase_						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase						):
       lowerCAmelCase_					:       Optional[int]    =		self.linear(self.silu(self.emb(UpperCAmelCase_						)						)						)
       lowerCAmelCase_   ,  lowerCAmelCase_					:       List[str]    =		torch.chunk(UpperCAmelCase_    ,						2						)
       lowerCAmelCase_					:       Optional[int]    =		self.norm(UpperCAmelCase_						) * (1 + scale) + shift
       return x





class      lowercase__    (							nn.Module  ):







  def __init__(	self    ,						_lowercase    ,						_lowercase						):
       super().__init__()

       lowerCAmelCase_					:       int    =		CombinedTimestepLabelEmbeddings(UpperCAmelCase_    ,						UpperCAmelCase_						)

       lowerCAmelCase_					:       int    =		nn.SiLU()
       lowerCAmelCase_					:       int    =		nn.Linear(UpperCAmelCase_    ,						6 * embedding_dim    ,						bias=UpperCAmelCase_						)
       lowerCAmelCase_					:       Optional[int]    =		nn.LayerNorm(UpperCAmelCase_    ,						elementwise_affine=UpperCAmelCase_    ,						eps=1e-6						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase=None						):
       lowerCAmelCase_					:       str    =		self.linear(self.silu(self.emb(UpperCAmelCase_    ,						UpperCAmelCase_    ,						hidden_dtype=UpperCAmelCase_						)						)						)
       lowerCAmelCase_   ,  lowerCAmelCase_   ,  lowerCAmelCase_   ,  lowerCAmelCase_   ,  lowerCAmelCase_   ,  lowerCAmelCase_					:       List[str]    =		emb.chunk(6    ,						dim=1						)
       lowerCAmelCase_					:       Optional[int]    =		self.norm(UpperCAmelCase_						) * (1 + scale_msa[:, None]) + shift_msa[:, None]
       return x, gate_msa, shift_mlp, scale_mlp, gate_mlp





class      lowercase__    (							nn.Module  ):







  def __init__(	self    ,						_lowercase    ,						_lowercase    ,						_lowercase    ,						_lowercase = None    ,						_lowercase = 1e-5						):
       super().__init__()
       lowerCAmelCase_					:       List[str]    =		num_groups
       lowerCAmelCase_					:       Union[str, Any]    =		eps

       if act_fn is None:
            lowerCAmelCase_					:       List[str]    =		None
       else:
            lowerCAmelCase_					:       Optional[Any]    =		get_activation(UpperCAmelCase_						)

       lowerCAmelCase_					:       Union[str, Any]    =		nn.Linear(UpperCAmelCase_    ,						out_dim * 2						)







  def 	UpperCAmelCase__		(	self    ,						_lowercase    ,						_lowercase						):



       if self.act:
            lowerCAmelCase_					:       int    =		self.act(UpperCAmelCase_						)
       lowerCAmelCase_					:       Optional[int]    =		self.linear(UpperCAmelCase_						)
       lowerCAmelCase_					:       List[str]    =		emb[:, :, None, None]
       lowerCAmelCase_   ,  lowerCAmelCase_					:       Union[str, Any]    =		emb.chunk(2    ,						dim=1						)

       lowerCAmelCase_					:       List[Any]    =		F.group_norm(UpperCAmelCase_    ,						self.num_groups    ,						eps=self.eps						)
       lowerCAmelCase_					:       List[Any]    =		x * (1 + scale) + shift
       return x







def 							_lowerCAmelCase (  _a   :					int					,   _a   :					bool = False			)	->			bool:
     if n == 2:
          return True
     if not n % 2 or n < 2:
          return False
     if n > 5 and n % 10 not in (1, 3, 7, 9):  # can quickly check last digit
          return False
     if n > 3_31_70_44_06_46_79_88_73_85_96_19_81 and not allow_probable:
          raise ValueError(
              """Warning: upper bound of deterministic test is exceeded. """
              """Pass allow_probable=True to allow probabilistic test. """
              """A return value of True indicates a probable prime."""			)
     # array bounds provided by analysis
     lowerCAmelCase_					:       Optional[int]    =		[
         20_47,
         1_37_36_53,
         25_32_60_01,
         32_15_03_17_51,
         2_15_23_02_89_87_47,
         3_47_47_49_66_03_83,
         3_41_55_00_71_72_83_21,
         1,
         3_82_51_23_05_65_46_41_30_51,
         1,
         1,
         31_86_65_85_78_34_03_11_51_16_74_61,
         3_31_70_44_06_46_79_88_73_85_96_19_81,
     ]

     lowerCAmelCase_					:       Dict    =		[2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41]
     for idx, _p in enumerate(_a					,   1			):
          if n < _p:
               # then we have our last prime to check
               lowerCAmelCase_					:       str    =		primes[:idx]
               break
     lowerCAmelCase_   ,  lowerCAmelCase_					:       Optional[Any]    =		n - 1, 0
     # break up n -1 into a power of 2 (s) and
     # remaining odd component
     # essentially, solve for d * 2 ** s == n - 1
     while d % 2 == 0:
          d //= 2
          s += 1
     for prime in plist:
          lowerCAmelCase_					:       Tuple    =		False
          for r in range(_a			):
               lowerCAmelCase_					:       Tuple    =		pow(_a					,   d * 2**r					,   _a			)
               # see article for analysis explanation for m
               if (r == 0 and m == 1) or ((m + 1) % n == 0):
                    lowerCAmelCase_					:       List[Any]    =		True
                    # this loop will not determine compositeness
                    break
          if pr:
               continue
          # if pr is False, then the above loop never evaluated to true,
          # and the n MUST be composite
          return False
     return True






def 							_lowerCAmelCase (  )	->			None:
     assert not miller_rabin(5_61			)
     assert miller_rabin(5_63			)
     # 2047

     assert not miller_rabin(83_82_01			)
     assert miller_rabin(83_82_07			)
     # 1_373_653

     assert not miller_rabin(17_31_60_01			)
     assert miller_rabin(17_31_60_17			)
     # 25_326_001

     assert not miller_rabin(30_78_38_66_41			)
     assert miller_rabin(30_78_38_66_53			)
     # 3_215_031_751

     assert not miller_rabin(1_71_30_45_57_48_01			)
     assert miller_rabin(1_71_30_45_57_48_19			)
     # 2_152_302_898_747

     assert not miller_rabin(2_77_97_99_72_83_07			)
     assert miller_rabin(2_77_97_99_72_83_27			)
     # 3_474_749_660_383

     assert not miller_rabin(1_13_85_00_23_90_94_41			)
     assert miller_rabin(1_13_85_00_23_90_95_27			)
     # 341_550_071_728_321

     assert not miller_rabin(1_27_50_41_01_88_48_80_43_51			)
     assert miller_rabin(1_27_50_41_01_88_48_80_43_91			)
     # 3_825_123_056_546_413_051

     assert not miller_rabin(7_96_66_46_44_58_50_77_87_79_18_67			)
     assert miller_rabin(7_96_66_46_44_58_50_77_87_79_19_51			)
     # 318_665_857_834_031_151_167_461

     assert not miller_rabin(55_28_40_67_74_46_64_78_97_66_03_33			)
     assert miller_rabin(55_28_40_67_74_46_64_78_97_66_03_59			)
     # 3_317_044_064_679_887_385_961_981
     # upper limit for probabilistic test


if __name__ == "__main__":
      test_miller_rabin()
