



'''simple docstring'''


from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     import os as original_os
     from os import path as original_path
     from os import rename as original_rename
     from os.path import dirname as original_dirname
     from os.path import join as original_join

     assert _test_patching.os is original_os
     assert _test_patching.path is original_path
     assert _test_patching.join is original_join

     assert _test_patching.renamed_os is original_os
     assert _test_patching.renamed_path is original_path
     assert _test_patching.renamed_join is original_join

     a_       		="__test_patch_submodule_mock__"
     with patch_submodule(_test_patching       ,	"os.path.join"       ,	lowercase__       ):
          # Every way to access os.path.join must be patched, and the rest must stay untouched

          # check os.path.join
          assert isinstance(_test_patching.os       ,	_PatchedModuleObj       )
          assert isinstance(_test_patching.os.path       ,	_PatchedModuleObj       )
          assert _test_patching.os.path.join is mock

          # check path.join
          assert isinstance(_test_patching.path       ,	_PatchedModuleObj       )
          assert _test_patching.path.join is mock

          # check join
          assert _test_patching.join is mock

          # check that the other attributes are untouched
          assert _test_patching.os.rename is original_rename
          assert _test_patching.path.dirname is original_dirname
          assert _test_patching.os.path.dirname is original_dirname

          # Even renamed modules or objects must be patched

          # check renamed_os.path.join
          assert isinstance(_test_patching.renamed_os       ,	_PatchedModuleObj       )
          assert isinstance(_test_patching.renamed_os.path       ,	_PatchedModuleObj       )
          assert _test_patching.renamed_os.path.join is mock

          # check renamed_path.join
          assert isinstance(_test_patching.renamed_path       ,	_PatchedModuleObj       )
          assert _test_patching.renamed_path.join is mock

          # check renamed_join
          assert _test_patching.renamed_join is mock

          # check that the other attributes are untouched
          assert _test_patching.renamed_os.rename is original_rename
          assert _test_patching.renamed_path.dirname is original_dirname
          assert _test_patching.renamed_os.path.dirname is original_dirname

     # check that everthing is back to normal when the patch is over

     assert _test_patching.os is original_os
     assert _test_patching.path is original_path
     assert _test_patching.join is original_join

     assert _test_patching.renamed_os is original_os
     assert _test_patching.renamed_path is original_path
     assert _test_patching.renamed_join is original_join






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     assert _test_patching.open is open

     a_       		="__test_patch_submodule_builtin_mock__"
     # _test_patching has "open" in its globals
     assert _test_patching.open is open
     with patch_submodule(_test_patching       ,	"open"       ,	lowercase__       ):
          assert _test_patching.open is mock

     # check that everthing is back to normal when the patch is over

     assert _test_patching.open is open






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     a_       		="__test_patch_submodule_missing_mock__"
     with patch_submodule(_test_patching       ,	"pandas.read_csv"       ,	lowercase__       ):
          pass






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     a_       		="__test_patch_submodule_missing_builtin_mock__"
     # _test_patching doesn't have "len" in its globals
     assert getattr(_test_patching       ,	"len"       ,	lowercase__       ) is None
     with patch_submodule(_test_patching       ,	"len"       ,	lowercase__       ):
          assert _test_patching.len is mock
     assert _test_patching.len is len






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     a_       		="__test_patch_submodule_start_and_stop_mock__"
     a_       		=patch_submodule(_test_patching       ,	"open"       ,	lowercase__       )
     assert _test_patching.open is open
     patch.start()
     assert _test_patching.open is mock
     patch.stop()
     assert _test_patching.open is open






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     from os import rename as original_rename
     from os.path import dirname as original_dirname
     from os.path import join as original_join

     a_       		="__test_patch_submodule_successive_join__"
     a_       		="__test_patch_submodule_successive_dirname__"
     a_       		="__test_patch_submodule_successive_rename__"
     assert _test_patching.os.path.join is original_join
     assert _test_patching.os.path.dirname is original_dirname
     assert _test_patching.os.rename is original_rename

     with patch_submodule(_test_patching       ,	"os.path.join"       ,	lowercase__       ):
          with patch_submodule(_test_patching       ,	"os.rename"       ,	lowercase__       ):
               with patch_submodule(_test_patching       ,	"os.path.dirname"       ,	lowercase__       ):
                    assert _test_patching.os.path.join is mock_join
                    assert _test_patching.os.path.dirname is mock_dirname
                    assert _test_patching.os.rename is mock_rename

    # try another order
     with patch_submodule(_test_patching       ,	"os.rename"       ,	lowercase__       ):
          with patch_submodule(_test_patching       ,	"os.path.join"       ,	lowercase__       ):
               with patch_submodule(_test_patching       ,	"os.path.dirname"       ,	lowercase__       ):
                    assert _test_patching.os.path.join is mock_join
                    assert _test_patching.os.path.dirname is mock_dirname
                    assert _test_patching.os.rename is mock_rename

     assert _test_patching.os.path.join is original_join
     assert _test_patching.os.path.dirname is original_dirname
     assert _test_patching.os.rename is original_rename






def  UpperCAmelCase_       (		):


     '''simple docstring'''







     a_       		="__test_patch_submodule_doesnt_exist_mock__"
     with patch_submodule(_test_patching       ,	"__module_that_doesn_exist__.__attribute_that_doesn_exist__"       ,	lowercase__       ):
          pass
     with patch_submodule(_test_patching       ,	"os.__attribute_that_doesn_exist__"       ,	lowercase__       ):
          pass







'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase							       =			logging.get_logger(__name__)

lowercase							       =			{
    '''google/switch-base-8''': '''https://huggingface.co/google/switch-base-8/blob/main/config.json''',
}







class 	UpperCAmelCase	(		__a):


 '''simple docstring'''




 __magic_name__					: int  =      "switch_transformers"
 __magic_name__					: List[Any]  =      ["past_key_values"]
 __magic_name__					: Union[str, Any]  =      {"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}




 def __init__(   self							,  lowerCAmelCase_=3_2_1_2_8							,  lowerCAmelCase_=7_6_8							,  lowerCAmelCase_=6_4							,  lowerCAmelCase_=2_0_4_8							,  lowerCAmelCase_=6_4							,  lowerCAmelCase_=1_2							,  lowerCAmelCase_=3							,  lowerCAmelCase_=1_2							,  lowerCAmelCase_=3							,  lowerCAmelCase_=1_2							,  lowerCAmelCase_=8							,  lowerCAmelCase_=False							,  lowerCAmelCase_=0.0_1							,  lowerCAmelCase_="float32"							,  lowerCAmelCase_=False							,  lowerCAmelCase_=3_2							,  lowerCAmelCase_=1_2_8							,  lowerCAmelCase_=0.1							,  lowerCAmelCase_=1e-6							,  lowerCAmelCase_=0.0_0_1							,  lowerCAmelCase_=0.0_0_1							,  lowerCAmelCase_=1.0							,  lowerCAmelCase_="relu"							,  lowerCAmelCase_=True							,  lowerCAmelCase_=False							,  lowerCAmelCase_=True							,  lowerCAmelCase_=0							,  lowerCAmelCase_=1							,  **lowerCAmelCase_							,  ) ->      Optional[int]:



      """simple docstring"""







      a_       		=vocab_size
      a_       		=d_model
      a_       		=d_kv
      a_       		=d_ff

      a_       		=num_sparse_encoder_layers

      a_       		=num_layers
      a_       		=(
          num_decoder_layers if num_decoder_layers is not None else self.num_layers
      )  # default = symmetry
      a_       		=num_sparse_decoder_layers

      # This tells us, each how many encoder layer we'll have to set a sparse layer.
      if self.num_sparse_encoder_layers > 0:
           a_       		=self.num_layers // self.num_sparse_encoder_layers
      else:
           a_       		=self.num_layers  # HACK: this will create 0 sparse layers

      # This tells us, each how many encoder layer we'll have to set a sparse layer.
      if self.num_sparse_decoder_layers > 0:
           a_       		=self.num_decoder_layers // self.num_sparse_decoder_layers
      else:
           a_       		=self.num_decoder_layers  # HACK: this will create 0 sparse layers

      a_       		=num_heads
      a_       		=num_experts
      a_       		=expert_capacity
      a_       		=router_bias
      a_       		=router_jitter_noise
      if router_dtype not in ["float32", "float16", "bfloat16"]:
           raise ValueError(f"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}""")
      a_       		=router_dtype

      a_       		=router_ignore_padding_tokens
      a_       		=relative_attention_num_buckets
      a_       		=relative_attention_max_distance

      a_       		=dropout_rate
      a_       		=layer_norm_epsilon
      a_       		=initializer_factor
      a_       		=feed_forward_proj
      a_       		=use_cache
      a_       		=add_router_probs

      a_       		=router_z_loss_coef
      a_       		=router_aux_loss_coef

      a_       		=self.feed_forward_proj.split("-")
      a_       		=act_info[-1]
      a_       		=act_info[0] == "gated"

      if len(lowerCAmelCase_) > 1 and act_info[0] != "gated" or len(lowerCAmelCase_) > 2:
           raise ValueError(
               f"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
               "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
               "'gated-gelu' or 'relu'")

      # for backwards compatibility
      if feed_forward_proj == "gated-gelu":
           a_       		="gelu_new"

      super().__init__(
          pad_token_id=lowerCAmelCase_							,  eos_token_id=lowerCAmelCase_							,  is_encoder_decoder=lowerCAmelCase_							,  **lowerCAmelCase_							,  )


