from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings




@maybe_allow_in_graph
class 	__snake_case	( nn.Module		):





      '''simple docstring'''






      def __init__(					self     :				str    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				int=0.0    ,  _UpperCamelCase     :				Optional[int] = None    ,  _UpperCamelCase     :				str = "geglu"    ,  _UpperCamelCase     :				Optional[int] = None    ,  _UpperCamelCase     :				bool = False    ,  _UpperCamelCase     :				bool = False    ,  _UpperCamelCase     :				bool = False    ,  _UpperCamelCase     :				bool = False    ,  _UpperCamelCase     :				bool = True    ,  _UpperCamelCase     :				str = "layer_norm"    ,  _UpperCamelCase     :				bool = False    ,  )				->Tuple:


          """simple docstring"""


          super().__init__()
          _lowerCamelCase     :		int 						=						only_cross_attention

          _lowerCamelCase     :		Tuple 						=						(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
          _lowerCamelCase     :		str 						=						(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

          if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
              raise ValueError(
                  F"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
                  F""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.""")

          # Define 3 blocks. Each block has its own normalization layer.
          # 1. Self-Attn
          if self.use_ada_layer_norm:
              _lowerCamelCase     :		Any 						=						AdaLayerNorm(_UpperCamelCase    ,  _UpperCamelCase)
          elif self.use_ada_layer_norm_zero:
              _lowerCamelCase     :		int 						=						AdaLayerNormZero(_UpperCamelCase    ,  _UpperCamelCase)
          else:
              _lowerCamelCase     :		Dict 						=						nn.LayerNorm(_UpperCamelCase    ,  elementwise_affine=_UpperCamelCase)
          _lowerCamelCase     :		Tuple 						=						Attention(
              query_dim=_UpperCamelCase    ,  heads=_UpperCamelCase    ,  dim_head=_UpperCamelCase    ,  dropout=_UpperCamelCase    ,  bias=_UpperCamelCase    ,  cross_attention_dim=cross_attention_dim if only_cross_attention else None    ,  upcast_attention=_UpperCamelCase    ,  )

          # 2. Cross-Attn
          if cross_attention_dim is not None or double_self_attention:
              # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
              # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
              # the second cross attention block.
              _lowerCamelCase     :		Tuple 						=						(
                  AdaLayerNorm(_UpperCamelCase    ,  _UpperCamelCase)
                  if self.use_ada_layer_norm
                  else nn.LayerNorm(_UpperCamelCase    ,  elementwise_affine=_UpperCamelCase)
              )
              _lowerCamelCase     :		Dict 						=						Attention(
                  query_dim=_UpperCamelCase    ,  cross_attention_dim=cross_attention_dim if not double_self_attention else None    ,  heads=_UpperCamelCase    ,  dim_head=_UpperCamelCase    ,  dropout=_UpperCamelCase    ,  bias=_UpperCamelCase    ,  upcast_attention=_UpperCamelCase    ,  )  # is self-attn if encoder_hidden_states is none
          else:
              _lowerCamelCase     :		Dict 						=						None
              _lowerCamelCase     :		Any 						=						None

          # 3. Feed-forward
          _lowerCamelCase     :		Dict 						=						nn.LayerNorm(_UpperCamelCase    ,  elementwise_affine=_UpperCamelCase)
          _lowerCamelCase     :		List[str] 						=						FeedForward(_UpperCamelCase    ,  dropout=_UpperCamelCase    ,  activation_fn=_UpperCamelCase    ,  final_dropout=_UpperCamelCase)

          # let chunk size default to None
          _lowerCamelCase     :		Union[str, Any] 						=						None
          _lowerCamelCase     :		Optional[Any] 						=						0



      def 	_SCREAMING_SNAKE_CASE    (					self     :				int    ,  _UpperCamelCase     :				Optional[int]    ,  _UpperCamelCase     :				int)				->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						chunk_size
          _lowerCamelCase     :		List[Any] 						=						dim




      def 	_SCREAMING_SNAKE_CASE    (					self     :				str    ,  _UpperCamelCase     :				torch.FloatTensor    ,  _UpperCamelCase     :				Optional[torch.FloatTensor] = None    ,  _UpperCamelCase     :				Optional[torch.FloatTensor] = None    ,  _UpperCamelCase     :				Optional[torch.FloatTensor] = None    ,  _UpperCamelCase     :				Optional[torch.LongTensor] = None    ,  _UpperCamelCase     :				Dict[str, Any] = None    ,  _UpperCamelCase     :				Optional[torch.LongTensor] = None    ,  )				->Optional[int]:


          """simple docstring"""


          if self.use_ada_layer_norm:
              _lowerCamelCase     :		List[Any] 						=						self.norma(_UpperCamelCase    ,  _UpperCamelCase)
          elif self.use_ada_layer_norm_zero:
              _lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase     :		str 						=						self.norma(
                  _UpperCamelCase    ,  _UpperCamelCase    ,  _UpperCamelCase    ,  hidden_dtype=hidden_states.dtype)
          else:
              _lowerCamelCase     :		List[str] 						=						self.norma(_UpperCamelCase)

          _lowerCamelCase     :		List[Any] 						=						cross_attention_kwargs if cross_attention_kwargs is not None else {}

          _lowerCamelCase     :		List[str] 						=						self.attna(
              _UpperCamelCase    ,  encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None    ,  attention_mask=_UpperCamelCase    ,  **_UpperCamelCase    ,  )
          if self.use_ada_layer_norm_zero:
              _lowerCamelCase     :		List[str] 						=						gate_msa.unsqueeze(1) * attn_output
          _lowerCamelCase     :		Optional[Any] 						=						attn_output + hidden_states

          # 2. Cross-Attention
          if self.attna is not None:
              _lowerCamelCase     :		List[Any] 						=						(
                  self.norma(_UpperCamelCase    ,  _UpperCamelCase) if self.use_ada_layer_norm else self.norma(_UpperCamelCase)
              )

              _lowerCamelCase     :		List[Any] 						=						self.attna(
                  _UpperCamelCase    ,  encoder_hidden_states=_UpperCamelCase    ,  attention_mask=_UpperCamelCase    ,  **_UpperCamelCase    ,  )
              _lowerCamelCase     :		Tuple 						=						attn_output + hidden_states

          # 3. Feed-forward
          _lowerCamelCase     :		List[str] 						=						self.norma(_UpperCamelCase)

          if self.use_ada_layer_norm_zero:
              _lowerCamelCase     :		Optional[int] 						=						norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

          if self._chunk_size is not None:
              # "feed_forward_chunk_size" can be used to save memory
              if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                  raise ValueError(
                      F"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.""")

              _lowerCamelCase     :		str 						=						norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
              _lowerCamelCase     :		List[str] 						=						torch.cat(
                  [self.ff(_UpperCamelCase) for hid_slice in norm_hidden_states.chunk(_UpperCamelCase    ,  dim=self._chunk_dim)]    ,  dim=self._chunk_dim    ,  )
          else:
              _lowerCamelCase     :		List[Any] 						=						self.ff(_UpperCamelCase)

          if self.use_ada_layer_norm_zero:
              _lowerCamelCase     :		str 						=						gate_mlp.unsqueeze(1) * ff_output

          _lowerCamelCase     :		int 						=						ff_output + hidden_states

          return hidden_states





class 	__snake_case	( nn.Module		):





      '''simple docstring'''






      def __init__(					self     :				str    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				Optional[int] = None    ,  _UpperCamelCase     :				int = 4    ,  _UpperCamelCase     :				float = 0.0    ,  _UpperCamelCase     :				str = "geglu"    ,  _UpperCamelCase     :				bool = False    ,  )				->int:


          """simple docstring"""


          super().__init__()
          _lowerCamelCase     :		List[str] 						=						int(dim * mult)
          _lowerCamelCase     :		Optional[int] 						=						dim_out if dim_out is not None else dim

          if activation_fn == "gelu":
              _lowerCamelCase     :		int 						=						GELU(_UpperCamelCase    ,  _UpperCamelCase)
          if activation_fn == "gelu-approximate":
              _lowerCamelCase     :		Any 						=						GELU(_UpperCamelCase    ,  _UpperCamelCase    ,  approximate="""tanh""")
          elif activation_fn == "geglu":
              _lowerCamelCase     :		Union[str, Any] 						=						GEGLU(_UpperCamelCase    ,  _UpperCamelCase)
          elif activation_fn == "geglu-approximate":
              _lowerCamelCase     :		Optional[Any] 						=						ApproximateGELU(_UpperCamelCase    ,  _UpperCamelCase)

          _lowerCamelCase     :		int 						=						nn.ModuleList([])
          # project in
          self.net.append(_UpperCamelCase)
          # project dropout
          self.net.append(nn.Dropout(_UpperCamelCase))
          # project out
          self.net.append(nn.Linear(_UpperCamelCase    ,  _UpperCamelCase))
          # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
          if final_dropout:
              self.net.append(nn.Dropout(_UpperCamelCase))




      def 	_SCREAMING_SNAKE_CASE    (					self     :				Optional[int]    ,  _UpperCamelCase     :				Tuple)				->Optional[Any]:


          """simple docstring"""


          for module in self.net:
              _lowerCamelCase     :		Optional[int] 						=						module(_UpperCamelCase)
          return hidden_states





class 	__snake_case	( nn.Module		):





      '''simple docstring'''






      def __init__(					self     :				Dict    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				str = "none")				->Any:


          """simple docstring"""


          super().__init__()
          _lowerCamelCase     :		int 						=						nn.Linear(_UpperCamelCase    ,  _UpperCamelCase)
          _lowerCamelCase     :		Optional[Any] 						=						approximate



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any]    ,  _UpperCamelCase     :				Optional[int])				->List[str]:


          """simple docstring"""


          if gate.device.type != "mps":
              return F.gelu(_UpperCamelCase    ,  approximate=self.approximate)
          # mps: gelu is not implemented for float16
          return F.gelu(gate.to(dtype=torch.floataa)    ,  approximate=self.approximate).to(dtype=gate.dtype)




      def 	_SCREAMING_SNAKE_CASE    (					self     :				List[str]    ,  _UpperCamelCase     :				Optional[int])				->Any:


          """simple docstring"""


          _lowerCamelCase     :		Optional[Any] 						=						self.proj(_UpperCamelCase)
          _lowerCamelCase     :		List[str] 						=						self.gelu(_UpperCamelCase)
          return hidden_states





class 	__snake_case	( nn.Module		):





      '''simple docstring'''






      def __init__(					self     :				str    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				int)				->Any:


          """simple docstring"""


          super().__init__()
          _lowerCamelCase     :		Tuple 						=						nn.Linear(_UpperCamelCase    ,  dim_out * 2)



      def 	_SCREAMING_SNAKE_CASE    (					self     :				Any    ,  _UpperCamelCase     :				int)				->List[str]:


          """simple docstring"""


          if gate.device.type != "mps":
              return F.gelu(_UpperCamelCase)
          # mps: gelu is not implemented for float16
          return F.gelu(gate.to(dtype=torch.floataa)).to(dtype=gate.dtype)




      def 	_SCREAMING_SNAKE_CASE    (					self     :				str    ,  _UpperCamelCase     :				str)				->int:


          """simple docstring"""


          _lowerCamelCase						,							_lowerCamelCase     :		Dict 						=						self.proj(_UpperCamelCase).chunk(2    ,  dim=-1)
          return hidden_states * self.gelu(_UpperCamelCase)





class 	__snake_case	( nn.Module		):





      '''simple docstring'''






      def __init__(					self     :				Dict    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				int)				->List[str]:


          """simple docstring"""


          super().__init__()
          _lowerCamelCase     :		Any 						=						nn.Linear(_UpperCamelCase    ,  _UpperCamelCase)




      def 	_SCREAMING_SNAKE_CASE    (					self     :				List[Any]    ,  _UpperCamelCase     :				List[Any])				->Tuple:


          """simple docstring"""


          _lowerCamelCase     :		Tuple 						=						self.proj(_UpperCamelCase)
          return x * torch.sigmoid(1.7_0_2 * x)





class 	__snake_case	( nn.Module		):





      '''simple docstring'''






      def __init__(					self     :				Optional[int]    ,  _UpperCamelCase     :				Union[str, Any]    ,  _UpperCamelCase     :				Dict)				->Any:


          """simple docstring"""


          super().__init__()
          _lowerCamelCase     :		Dict 						=						nn.Embedding(_UpperCamelCase    ,  _UpperCamelCase)
          _lowerCamelCase     :		List[Any] 						=						nn.SiLU()
          _lowerCamelCase     :		Union[str, Any] 						=						nn.Linear(_UpperCamelCase    ,  embedding_dim * 2)
          _lowerCamelCase     :		Optional[Any] 						=						nn.LayerNorm(_UpperCamelCase    ,  elementwise_affine=_UpperCamelCase)




      def 	_SCREAMING_SNAKE_CASE    (					self     :				int    ,  _UpperCamelCase     :				Any    ,  _UpperCamelCase     :				List[Any])				->Optional[int]:


          """simple docstring"""


          _lowerCamelCase     :		List[str] 						=						self.linear(self.silu(self.emb(_UpperCamelCase)))
          _lowerCamelCase						,							_lowerCamelCase     :		Optional[int] 						=						torch.chunk(_UpperCamelCase    ,  2)
          _lowerCamelCase     :		Dict 						=						self.norm(_UpperCamelCase) * (1 + scale) + shift
          return x





class 	__snake_case	( nn.Module		):





      '''simple docstring'''






      def __init__(					self     :				List[Any]    ,  _UpperCamelCase     :				Tuple    ,  _UpperCamelCase     :				List[str])				->Dict:


          """simple docstring"""


          super().__init__()

          _lowerCamelCase     :		str 						=						CombinedTimestepLabelEmbeddings(_UpperCamelCase    ,  _UpperCamelCase)

          _lowerCamelCase     :		List[Any] 						=						nn.SiLU()
          _lowerCamelCase     :		Optional[Any] 						=						nn.Linear(_UpperCamelCase    ,  6 * embedding_dim    ,  bias=_UpperCamelCase)
          _lowerCamelCase     :		Union[str, Any] 						=						nn.LayerNorm(_UpperCamelCase    ,  elementwise_affine=_UpperCamelCase    ,  eps=1E-6)




      def 	_SCREAMING_SNAKE_CASE    (					self     :				Union[str, Any]    ,  _UpperCamelCase     :				Optional[int]    ,  _UpperCamelCase     :				List[Any]    ,  _UpperCamelCase     :				List[str]    ,  _UpperCamelCase     :				Dict=None)				->int:


          """simple docstring"""


          _lowerCamelCase     :		Union[str, Any] 						=						self.linear(self.silu(self.emb(_UpperCamelCase    ,  _UpperCamelCase    ,  hidden_dtype=_UpperCamelCase)))
          _lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase     :		int 						=						emb.chunk(6    ,  dim=1)
          _lowerCamelCase     :		List[Any] 						=						self.norm(_UpperCamelCase) * (1 + scale_msa[:, None]) + shift_msa[:, None]
          return x, gate_msa, shift_mlp, scale_mlp, gate_mlp





class 	__snake_case	( nn.Module		):





      '''simple docstring'''






      def __init__(					self     :				List[Any]    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				int    ,  _UpperCamelCase     :				Optional[str] = None    ,  _UpperCamelCase     :				float = 1E-5)				->List[str]:


          """simple docstring"""


          super().__init__()
          _lowerCamelCase     :		int 						=						num_groups
          _lowerCamelCase     :		List[Any] 						=						eps

          if act_fn is None:
              _lowerCamelCase     :		str 						=						None
          else:
              _lowerCamelCase     :		List[Any] 						=						get_activation(_UpperCamelCase)

          _lowerCamelCase     :		Any 						=						nn.Linear(_UpperCamelCase    ,  out_dim * 2)




      def 	_SCREAMING_SNAKE_CASE    (					self     :				int    ,  _UpperCamelCase     :				Tuple    ,  _UpperCamelCase     :				Dict)				->Dict:


          """simple docstring"""


          if self.act:
              _lowerCamelCase     :		Optional[Any] 						=						self.act(_UpperCamelCase)
          _lowerCamelCase     :		Any 						=						self.linear(_UpperCamelCase)
          _lowerCamelCase     :		str 						=						emb[:, :, None, None]
          _lowerCamelCase						,							_lowerCamelCase     :		List[Any] 						=						emb.chunk(2    ,  dim=1)

          _lowerCamelCase     :		str 						=						F.group_norm(_UpperCamelCase    ,  self.num_groups    ,  eps=self.eps)
          _lowerCamelCase     :		List[Any] 						=						x * (1 + scale) + shift
          return x





import argparse
import os

import torch

from transformers import (
    XLNetConfig,
    XLNetForQuestionAnswering,
    XLNetForSequenceClassification,
    XLNetLMHeadModel,
    load_tf_weights_in_xlnet,
)
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


lowerCAmelCase			:							int     ={
    "cola": 2,
    "mnli": 3,
    "mrpc": 2,
    "sst-2": 2,
    "sts-b": 1,
    "qqp": 2,
    "qnli": 2,
    "rte": 2,
    "wnli": 2,
}


logging.set_verbosity_info()







def 						A__  (					__A   , __A   , __A   , __A=None	):







    '''simple docstring'''
    # Initialise PyTorch model
    _lowerCamelCase     :		Tuple 						=						XLNetConfig.from_json_file(__A	)

    _lowerCamelCase     :		List[Any] 						=						finetuning_task.lower() if finetuning_task is not None else """"""
    if finetuning_task in GLUE_TASKS_NUM_LABELS:
        print(F"""Building PyTorch XLNetForSequenceClassification model from configuration: {config}"""	)
        _lowerCamelCase     :		int 						=						finetuning_task
        _lowerCamelCase     :		Union[str, Any] 						=						GLUE_TASKS_NUM_LABELS[finetuning_task]
        _lowerCamelCase     :		int 						=						XLNetForSequenceClassification(__A	)
    elif "squad" in finetuning_task:
        _lowerCamelCase     :		Dict 						=						finetuning_task
        _lowerCamelCase     :		Optional[Any] 						=						XLNetForQuestionAnswering(__A	)
    else:
        _lowerCamelCase     :		Any 						=						XLNetLMHeadModel(__A	)

    # Load weights from tf checkpoint
    load_tf_weights_in_xlnet(__A   , __A   , __A	)

    # Save pytorch-model
    _lowerCamelCase     :		Optional[Any] 						=						os.path.join(__A   , __A	)
    _lowerCamelCase     :		Any 						=						os.path.join(__A   , __A	)
    print(F"""Save PyTorch model to {os.path.abspath(__A	)}"""	)
    torch.save(model.state_dict()   , __A	)
    print(F"""Save configuration file to {os.path.abspath(__A	)}"""	)
    with open(__A   , """w"""   , encoding="""utf-8"""	) as f:
        f.write(config.to_json_string()	)


if __name__ == "__main__":
    lowerCAmelCase			:							Dict     =argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        "--tf_checkpoint_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path."
    )
    parser.add_argument(
        "--xlnet_config_file",
        default=None,
        type=str,
        required=True,
        help=(
            "The config json file corresponding to the pre-trained XLNet model. \n"
            "This specifies the model architecture."
        ),
    )
    parser.add_argument(
        "--pytorch_dump_folder_path",
        default=None,
        type=str,
        required=True,
        help="Path to the folder to store the PyTorch model or dataset/vocab.",
    )
    parser.add_argument(
        "--finetuning_task",
        default=None,
        type=str,
        help="Name of a task on which the XLNet TensorFlow model was fine-tuned",
    )
    lowerCAmelCase			:							Union[str, Any]     =parser.parse_args()
    print(args)

    convert_xlnet_checkpoint_to_pytorch(
        args.tf_checkpoint_path, args.xlnet_config_file, args.pytorch_dump_folder_path, args.finetuning_task
    )




