

import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast







class 	__UpperCamelCase    (    unittest.TestCase						):

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   str  =    mock.Mock()
      _lowerCAmelCase       :   Optional[int]  =    500
      _lowerCAmelCase       :   Union[str, Any]  =    {}
      _lowerCAmelCase       :   Optional[Any]  =    HTTPError
      _lowerCAmelCase       :   List[Any]  =    {}

      # Download this model to make sure it's in the cache.
      _lowerCAmelCase       :   Union[str, Any]  =    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'      )

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('requests.Session.request'    ,return_value=_lowercase      ) as mock_head:
          _lowerCAmelCase       :   Optional[int]  =    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'      )
          # This check we did call the fake head request
          mock_head.assert_called()

  @require_tokenizers
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   int  =    mock.Mock()
      _lowerCAmelCase       :   Any  =    500
      _lowerCAmelCase       :   Optional[int]  =    {}
      _lowerCAmelCase       :   Any  =    HTTPError
      _lowerCAmelCase       :   Tuple  =    {}

      # Download this model to make sure it's in the cache.
      _lowerCAmelCase       :   Any  =    GPTaTokenizerFast.from_pretrained('gpt2'      )

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch('requests.Session.request'    ,return_value=_lowercase      ) as mock_head:
          _lowerCAmelCase       :   Tuple  =    GPTaTokenizerFast.from_pretrained('gpt2'      )
          # This check we did call the fake head request
          mock_head.assert_called()

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      try:
          _lowerCAmelCase       :   Dict  =    tempfile.mktemp()
          with open(_lowercase    ,'wb'      ) as f:
              http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'    ,_lowercase      )

          _lowerCAmelCase       :   Union[str, Any]  =    AlbertTokenizer.from_pretrained(_lowercase      )
      finally:
          os.remove(_lowercase      )

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile('tokenizer.json'      ):
          # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
          return
      try:
          with open('tokenizer.json'    ,'wb'      ) as f:
              http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'    ,_lowercase      )
          _lowerCAmelCase       :   Any  =    AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'      )
          # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
          self.assertEqual(tokenizer.vocab_size    ,1000      )
          # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
          os.remove('tokenizer.json'      )







  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Dict  =    AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'      )







@is_staging_test
class 	__UpperCamelCase    (    unittest.TestCase						):
  _UpperCAmelCase          =		["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]

  @classmethod
  def __lowerCamelCase		(		cls      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Any  =    TOKEN
      HfFolder.save_token(_lowercase      )

  @classmethod
  def __lowerCamelCase		(		cls      ):
      '''simple docstring'''

      try:
          delete_repo(token=cls._token    ,repo_id='test-tokenizer'      )
      except HTTPError:
          pass

      try:
          delete_repo(token=cls._token    ,repo_id='valid_org/test-tokenizer-org'      )
      except HTTPError:
          pass

      try:
          delete_repo(token=cls._token    ,repo_id='test-dynamic-tokenizer'      )
      except HTTPError:
          pass

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      with tempfile.TemporaryDirectory() as tmp_dir:
          _lowerCAmelCase       :   Dict  =    os.path.join(_lowercase    ,'vocab.txt'      )
          with open(_lowercase    ,'w'    ,encoding='utf-8'      ) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]      )      )
          _lowerCAmelCase       :   Any  =    BertTokenizer(_lowercase      )

      tokenizer.push_to_hub('test-tokenizer'    ,use_auth_token=self._token      )
      _lowerCAmelCase       :   List[str]  =    BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""      )
      self.assertDictEqual(new_tokenizer.vocab    ,tokenizer.vocab      )

      # Reset repo
      delete_repo(token=self._token    ,repo_id='test-tokenizer'      )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(_lowercase    ,repo_id='test-tokenizer'    ,push_to_hub=_lowercase    ,use_auth_token=self._token      )

      _lowerCAmelCase       :   Optional[Any]  =    BertTokenizer.from_pretrained(F"""{USER}/test-tokenizer"""      )
      self.assertDictEqual(new_tokenizer.vocab    ,tokenizer.vocab      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      with tempfile.TemporaryDirectory() as tmp_dir:
          _lowerCAmelCase       :   Optional[Any]  =    os.path.join(_lowercase    ,'vocab.txt'      )
          with open(_lowercase    ,'w'    ,encoding='utf-8'      ) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]      )      )
          _lowerCAmelCase       :   List[Any]  =    BertTokenizer(_lowercase      )

      tokenizer.push_to_hub('valid_org/test-tokenizer-org'    ,use_auth_token=self._token      )
      _lowerCAmelCase       :   Dict  =    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'      )
      self.assertDictEqual(new_tokenizer.vocab    ,tokenizer.vocab      )

      # Reset repo
      delete_repo(token=self._token    ,repo_id='valid_org/test-tokenizer-org'      )

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(
              _lowercase    ,repo_id='valid_org/test-tokenizer-org'    ,push_to_hub=_lowercase    ,use_auth_token=self._token      )

      _lowerCAmelCase       :   List[Any]  =    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'      )
      self.assertDictEqual(new_tokenizer.vocab    ,tokenizer.vocab      )







  @require_tokenizers
  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
          _lowerCAmelCase       :   Any  =    os.path.join(_lowercase    ,'vocab.txt'      )
          with open(_lowercase    ,'w'    ,encoding='utf-8'      ) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]      )      )
          _lowerCAmelCase       :   Tuple  =    CustomTokenizer(_lowercase      )

      # No fast custom tokenizer
      tokenizer.push_to_hub('test-dynamic-tokenizer'    ,use_auth_token=self._token      )

      _lowerCAmelCase       :   Optional[Any]  =    AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""    ,trust_remote_code=_lowercase      )
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__    ,'CustomTokenizer'      )

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
          _lowerCAmelCase       :   Dict  =    os.path.join(_lowercase    ,'vocab.txt'      )
          with open(_lowercase    ,'w'    ,encoding='utf-8'      ) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]      )      )

          _lowerCAmelCase       :   Any  =    BertTokenizerFast.from_pretrained(_lowercase      )
          bert_tokenizer.save_pretrained(_lowercase      )
          _lowerCAmelCase       :   Any  =    CustomTokenizerFast.from_pretrained(_lowercase      )

      tokenizer.push_to_hub('test-dynamic-tokenizer'    ,use_auth_token=self._token      )

      _lowerCAmelCase       :   int  =    AutoTokenizer.from_pretrained(F"""{USER}/test-dynamic-tokenizer"""    ,trust_remote_code=_lowercase      )
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__    ,'CustomTokenizerFast'      )
      _lowerCAmelCase       :   Any  =    AutoTokenizer.from_pretrained(
          F"""{USER}/test-dynamic-tokenizer"""    ,use_fast=_lowercase    ,trust_remote_code=_lowercase      )
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__    ,'CustomTokenizer'      )







class 	__UpperCamelCase    (    unittest.TestCase						):

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Tuple  =    Trie()
      trie.add('Hello 友達'      )
      self.assertEqual(trie.data    ,{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}      )
      trie.add('Hello'      )
      trie.data
      self.assertEqual(trie.data    ,{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[Any]  =    Trie()
      self.assertEqual(trie.split('[CLS] This is a extra_id_100'      )    ,['[CLS] This is a extra_id_100']      )
      trie.add('[CLS]'      )
      trie.add('extra_id_1'      )
      trie.add('extra_id_100'      )
      self.assertEqual(trie.split('[CLS] This is a extra_id_100'      )    ,['[CLS]', ' This is a ', 'extra_id_100']      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   int  =    Trie()
      trie.add('A'      )
      self.assertEqual(trie.split('ABC'      )    ,['A', 'BC']      )
      self.assertEqual(trie.split('BCA'      )    ,['BC', 'A']      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Tuple  =    Trie()
      trie.add('TOKEN]'      )
      trie.add('[SPECIAL_TOKEN]'      )
      self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'      )    ,['This is something ', '[SPECIAL_TOKEN]']      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[Any]  =    Trie()
      trie.add('A'      )
      trie.add('P'      )
      trie.add('[SPECIAL_TOKEN]'      )
      self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'      )    ,['This is something ', '[SPECIAL_TOKEN]']      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Union[str, Any]  =    Trie()
      trie.add('AB'      )
      trie.add('B'      )
      trie.add('C'      )
      self.assertEqual(trie.split('ABC'      )    ,['AB', 'C']      )

  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Optional[Any]  =    Trie()
      trie.add('ABC'      )
      trie.add('B'      )
      trie.add('CD'      )
      self.assertEqual(trie.split('ABCD'      )    ,['ABC', 'D']      )







  def __lowerCamelCase		(		self      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[str]  =    Trie()
      _lowerCAmelCase       :   List[Any]  =    trie.cut_text('ABC'    ,[0, 0, 2, 1, 2, 3]      )
      self.assertEqual(_lowercase    ,['AB', 'C']      )








"""simple docstring"""






import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin







class 	__UpperCamelCase    (    a__							,							a__						):

  @register_to_config
  def __init__(		self    ,_A = 128    ,_A = 256    ,_A = 2_0_0_0.0    ,_A = 768    ,_A = 12    ,_A = 12    ,_A = 64    ,_A = 2048    ,_A = 0.1    ,):
      '''simple docstring'''

      super().__init__()

      _lowerCAmelCase       :   int  =    nn.Sequential(
          nn.Linear(_A    ,d_model * 4    ,bias=_A      )    ,nn.SiLU()    ,nn.Linear(d_model * 4    ,d_model * 4    ,bias=_A      )    ,nn.SiLU()    ,)

      _lowerCAmelCase       :   Any  =    nn.Embedding(_A    ,_A      )
      _lowerCAmelCase       :   Tuple  =    False

      _lowerCAmelCase       :   Union[str, Any]  =    nn.Linear(_A    ,_A    ,bias=_A      )

      _lowerCAmelCase       :   int  =    nn.Dropout(p=_A      )

      _lowerCAmelCase       :   int  =    nn.ModuleList()
      for lyr_num in range(_A      ):
          # FiLM conditional T5 decoder
          _lowerCAmelCase       :   Any  =    DecoderLayer(d_model=_A    ,d_kv=_A    ,num_heads=_A    ,d_ff=_A    ,dropout_rate=_A      )
          self.decoders.append(_A      )

      _lowerCAmelCase       :   Optional[Any]  =    TaLayerNorm(_A      )

      _lowerCAmelCase       :   List[str]  =    nn.Dropout(p=_A      )
      _lowerCAmelCase       :   Optional[Any]  =    nn.Linear(_A    ,_A    ,bias=_A      )

  def __lowerCamelCase		(		self    ,_A    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Dict  =    torch.mul(query_input.unsqueeze(-1      )    ,key_input.unsqueeze(-2      )      )
      return mask.unsqueeze(-3      )







  def __lowerCamelCase		(		self    ,_A    ,_A    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase,					_lowerCAmelCase,					_lowerCAmelCase       :   Dict  =    decoder_input_tokens.shape
      assert decoder_noise_time.shape == (batch,)

      # decoder_noise_time is in [0, 1), so rescale to expected timing range.
      _lowerCAmelCase       :   Any  =    get_timestep_embedding(
          decoder_noise_time * self.config.max_decoder_noise_time    ,embedding_dim=self.config.d_model    ,max_period=self.config.max_decoder_noise_time    ,).to(dtype=self.dtype      )

      _lowerCAmelCase       :   Union[str, Any]  =    self.conditioning_emb(_A      ).unsqueeze(1      )

      assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

      _lowerCAmelCase       :   str  =    decoder_input_tokens.shape[1]

      # If we want to use relative positions for audio context, we can just offset
      # this sequence by the length of encodings_and_masks.
      _lowerCAmelCase       :   Union[str, Any]  =    torch.broadcast_to(
          torch.arange(_A    ,device=decoder_input_tokens.device      )    ,(batch, seq_length)    ,)

      _lowerCAmelCase       :   Any  =    self.position_encoding(_A      )

      _lowerCAmelCase       :   str  =    self.continuous_inputs_projection(_A      )
      inputs += position_encodings
      _lowerCAmelCase       :   int  =    self.dropout(_A      )

      # decoder: No padding present.
      _lowerCAmelCase       :   Union[str, Any]  =    torch.ones(
          decoder_input_tokens.shape[:2]    ,device=decoder_input_tokens.device    ,dtype=inputs.dtype      )

      # Translate encoding masks to encoder-decoder masks.
      _lowerCAmelCase       :   Optional[Any]  =    [(x, self.encoder_decoder_mask(_A    ,_A      )) for x, y in encodings_and_masks]

      # cross attend style: concat encodings
      _lowerCAmelCase       :   Dict  =    torch.cat([x[0] for x in encodings_and_encdec_masks]    ,dim=1      )
      _lowerCAmelCase       :   Tuple  =    torch.cat([x[1] for x in encodings_and_encdec_masks]    ,dim=-1      )

      for lyr in self.decoders:
          _lowerCAmelCase       :   Tuple  =    lyr(
              _A    ,conditioning_emb=_A    ,encoder_hidden_states=_A    ,encoder_attention_mask=_A    ,)[0]

      _lowerCAmelCase       :   Any  =    self.decoder_norm(_A      )
      _lowerCAmelCase       :   List[Any]  =    self.post_dropout(_A      )

      _lowerCAmelCase       :   int  =    self.spec_out(_A      )
      return spec_out







class 	__UpperCamelCase    (    nn.Module						):

  def __init__(		self    ,_A    ,_A    ,_A    ,_A    ,_A    ,_A=1E-6      ):
      '''simple docstring'''

      super().__init__()
      _lowerCAmelCase       :   Optional[Any]  =    nn.ModuleList()

      # cond self attention: layer 0
      self.layer.append(
          TaLayerSelfAttentionCond(d_model=_A    ,d_kv=_A    ,num_heads=_A    ,dropout_rate=_A      )      )

      # cross attention: layer 1
      self.layer.append(
          TaLayerCrossAttention(
              d_model=_A    ,d_kv=_A    ,num_heads=_A    ,dropout_rate=_A    ,layer_norm_epsilon=_A    ,)      )

      # Film Cond MLP + dropout: last layer
      self.layer.append(
          TaLayerFFCond(d_model=_A    ,d_ff=_A    ,dropout_rate=_A    ,layer_norm_epsilon=_A      )      )







  def __lowerCamelCase		(		self    ,_A    ,_A=None    ,_A=None    ,_A=None    ,_A=None    ,_A=None    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   Any  =    self.layer[0](
          _A    ,conditioning_emb=_A    ,attention_mask=_A    ,)

      if encoder_hidden_states is not None:
          _lowerCAmelCase       :   Any  =    torch.where(encoder_attention_mask > 0    ,0    ,-1E10      ).to(
              encoder_hidden_states.dtype      )

          _lowerCAmelCase       :   str  =    self.layer[1](
              _A    ,key_value_states=_A    ,attention_mask=_A    ,)

      # Apply Film Conditional Feed Forward layer
      _lowerCAmelCase       :   Optional[Any]  =    self.layer[-1](_A    ,_A      )

      return (hidden_states,)







class 	__UpperCamelCase    (    nn.Module						):

  def __init__(		self    ,_A    ,_A    ,_A    ,_A      ):
      '''simple docstring'''

      super().__init__()
      _lowerCAmelCase       :   Union[str, Any]  =    TaLayerNorm(_A      )
      _lowerCAmelCase       :   Any  =    TaFiLMLayer(in_features=d_model * 4    ,out_features=_A      )
      _lowerCAmelCase       :   Dict  =    Attention(query_dim=_A    ,heads=_A    ,dim_head=_A    ,out_bias=_A    ,scale_qk=_A      )
      _lowerCAmelCase       :   Tuple  =    nn.Dropout(_A      )







  def __lowerCamelCase		(		self    ,_A    ,_A=None    ,_A=None    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   int  =    self.layer_norm(_A      )

      if conditioning_emb is not None:
          _lowerCAmelCase       :   Union[str, Any]  =    self.FiLMLayer(_A    ,_A      )

      # Self-attention block
      _lowerCAmelCase       :   Union[str, Any]  =    self.attention(_A      )

      _lowerCAmelCase       :   Optional[Any]  =    hidden_states + self.dropout(_A      )

      return hidden_states







class 	__UpperCamelCase    (    nn.Module						):

  def __init__(		self    ,_A    ,_A    ,_A    ,_A    ,_A      ):
      '''simple docstring'''

      super().__init__()
      _lowerCAmelCase       :   List[str]  =    Attention(query_dim=_A    ,heads=_A    ,dim_head=_A    ,out_bias=_A    ,scale_qk=_A      )
      _lowerCAmelCase       :   Optional[int]  =    TaLayerNorm(_A    ,eps=_A      )
      _lowerCAmelCase       :   Tuple  =    nn.Dropout(_A      )







  def __lowerCamelCase		(		self    ,_A    ,_A=None    ,_A=None    ,):
      '''simple docstring'''

      _lowerCAmelCase       :   Union[str, Any]  =    self.layer_norm(_A      )
      _lowerCAmelCase       :   str  =    self.attention(
          _A    ,encoder_hidden_states=_A    ,attention_mask=attention_mask.squeeze(1      )    ,)
      _lowerCAmelCase       :   Any  =    hidden_states + self.dropout(_A      )
      return layer_output







class 	__UpperCamelCase    (    nn.Module						):

  def __init__(		self    ,_A    ,_A    ,_A    ,_A      ):
      '''simple docstring'''

      super().__init__()
      _lowerCAmelCase       :   Optional[int]  =    TaDenseGatedActDense(d_model=_A    ,d_ff=_A    ,dropout_rate=_A      )
      _lowerCAmelCase       :   Tuple  =    TaFiLMLayer(in_features=d_model * 4    ,out_features=_A      )
      _lowerCAmelCase       :   Any  =    TaLayerNorm(_A    ,eps=_A      )
      _lowerCAmelCase       :   Union[str, Any]  =    nn.Dropout(_A      )







  def __lowerCamelCase		(		self    ,_A    ,_A=None      ):
      '''simple docstring'''

      _lowerCAmelCase       :   int  =    self.layer_norm(_A      )
      if conditioning_emb is not None:
          _lowerCAmelCase       :   Union[str, Any]  =    self.film(_A    ,_A      )

      _lowerCAmelCase       :   str  =    self.DenseReluDense(_A      )
      _lowerCAmelCase       :   Tuple  =    hidden_states + self.dropout(_A      )
      return hidden_states







class 	__UpperCamelCase    (    nn.Module						):

  def __init__(		self    ,_A    ,_A    ,_A      ):
      '''simple docstring'''

      super().__init__()
      _lowerCAmelCase       :   Union[str, Any]  =    nn.Linear(_A    ,_A    ,bias=_A      )
      _lowerCAmelCase       :   Any  =    nn.Linear(_A    ,_A    ,bias=_A      )
      _lowerCAmelCase       :   Union[str, Any]  =    nn.Linear(_A    ,_A    ,bias=_A      )
      _lowerCAmelCase       :   Union[str, Any]  =    nn.Dropout(_A      )
      _lowerCAmelCase       :   int  =    NewGELUActivation()







  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Tuple  =    self.act(self.wi_a(_A      )      )
      _lowerCAmelCase       :   Optional[int]  =    self.wi_a(_A      )
      _lowerCAmelCase       :   Union[str, Any]  =    hidden_gelu * hidden_linear
      _lowerCAmelCase       :   Dict  =    self.dropout(_A      )

      _lowerCAmelCase       :   Dict  =    self.wo(_A      )
      return hidden_states







class 	__UpperCamelCase    (    nn.Module						):

  def __init__(		self    ,_A    ,_A=1E-6      ):
      '''simple docstring'''

      super().__init__()
      _lowerCAmelCase       :   Union[str, Any]  =    nn.Parameter(torch.ones(_A      )      )
      _lowerCAmelCase       :   Optional[int]  =    eps







  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   Optional[int]  =    hidden_states.to(torch.floataa      ).pow(2      ).mean(-1    ,keepdim=_A      )
      _lowerCAmelCase       :   List[Any]  =    hidden_states * torch.rsqrt(variance + self.variance_epsilon      )

      # convert into half-precision if necessary
      if self.weight.dtype in [torch.floataa, torch.bfloataa]:
          _lowerCAmelCase       :   Optional[int]  =    hidden_states.to(self.weight.dtype      )

      return self.weight * hidden_states







class 	__UpperCamelCase    (    nn.Module						):

  def __lowerCamelCase		(		self    ,_A      ):
      '''simple docstring'''

      return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi      ) * (input + 0.0_4_4_7_1_5 * torch.pow(_A    ,3.0      ))      ))







class 	__UpperCamelCase    (    nn.Module						):

  def __init__(		self    ,_A    ,_A      ):
      '''simple docstring'''

      super().__init__()
      _lowerCAmelCase       :   List[str]  =    nn.Linear(_A    ,out_features * 2    ,bias=_A      )







  def __lowerCamelCase		(		self    ,_A    ,_A      ):
      '''simple docstring'''

      _lowerCAmelCase       :   List[Any]  =    self.scale_bias(_A      )
      _lowerCAmelCase,					_lowerCAmelCase       :   List[Any]  =    torch.chunk(_A    ,2    ,-1      )
      _lowerCAmelCase       :   List[Any]  =    x * (1 + scale) + shift
      return x





