

'''simple docstring'''


import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    FalconForCausalLM,
						    FalconForQuestionAnswering,
						    FalconForSequenceClassification,
						    FalconForTokenClassification,
						    FalconModel,
						)



class 					lowercase						:


							def __init__(			self    ,			_snake_case    ,			_snake_case=3    ,			_snake_case=7    ,			_snake_case=True    ,			_snake_case=True    ,			_snake_case=False    ,			_snake_case=True    ,			_snake_case=99    ,			_snake_case=32    ,			_snake_case=5    ,			_snake_case=4    ,			_snake_case=37    ,			_snake_case="gelu"    ,			_snake_case=0.1    ,			_snake_case=0.1    ,			_snake_case=512    ,			_snake_case=16    ,			_snake_case=2    ,			_snake_case=0.02    ,			_snake_case=3    ,			_snake_case=4    ,			_snake_case=None    ,			)							->				List[Any]:
								UpperCAmelCase_		:      Tuple       	=					parent
								UpperCAmelCase_		:      List[str]       	=					batch_size
								UpperCAmelCase_		:      Any       	=					seq_length
								UpperCAmelCase_		:      Dict       	=					is_training
								UpperCAmelCase_		:      Optional[Any]       	=					use_input_mask
								UpperCAmelCase_		:      List[str]       	=					use_token_type_ids
								UpperCAmelCase_		:      Dict       	=					use_labels
								UpperCAmelCase_		:      Tuple       	=					vocab_size
								UpperCAmelCase_		:      int       	=					hidden_size
								UpperCAmelCase_		:      Union[str, Any]       	=					num_hidden_layers
								UpperCAmelCase_		:      Optional[int]       	=					num_attention_heads
								UpperCAmelCase_		:      Optional[int]       	=					intermediate_size
								UpperCAmelCase_		:      List[str]       	=					hidden_act
								UpperCAmelCase_		:      Optional[Any]       	=					hidden_dropout_prob
								UpperCAmelCase_		:      Any       	=					attention_probs_dropout_prob
								UpperCAmelCase_		:      str       	=					max_position_embeddings
								UpperCAmelCase_		:      str       	=					type_vocab_size
								UpperCAmelCase_		:      Dict       	=					type_sequence_label_size
								UpperCAmelCase_		:      Any       	=					initializer_range
								UpperCAmelCase_		:      str       	=					num_labels
								UpperCAmelCase_		:      Tuple       	=					num_choices
								UpperCAmelCase_		:      str       	=					scope


							def 						_snake_case							(			self)							->				List[str]:
								UpperCAmelCase_		:      Any       	=					ids_tensor([self.batch_size, self.seq_length]    ,			self.vocab_size)

								UpperCAmelCase_		:      Tuple       	=					None
								if self.use_input_mask:
									UpperCAmelCase_		:      Optional[Any]       	=					random_attention_mask([self.batch_size, self.seq_length])

								UpperCAmelCase_		:      Tuple       	=					None

								UpperCAmelCase_		:      int       	=					None
								UpperCAmelCase_		:      Tuple       	=					None
								UpperCAmelCase_		:      Dict       	=					None
								if self.use_labels:
									UpperCAmelCase_		:      Optional[int]       	=					ids_tensor([self.batch_size]    ,			self.type_sequence_label_size)
									UpperCAmelCase_		:      Optional[Any]       	=					ids_tensor([self.batch_size, self.seq_length]    ,			self.num_labels)
									UpperCAmelCase_		:      int       	=					ids_tensor([self.batch_size]    ,			self.num_choices)

								UpperCAmelCase_		:      Tuple       	=					self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


							def 						_snake_case							(			self)							->				Optional[int]:
								return FalconConfig(
								    vocab_size=self.vocab_size    ,			hidden_size=self.hidden_size    ,			num_hidden_layers=self.num_hidden_layers    ,			num_attention_heads=self.num_attention_heads    ,			intermediate_size=self.intermediate_size    ,			hidden_act=self.hidden_act    ,			hidden_dropout_prob=self.hidden_dropout_prob    ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,			max_position_embeddings=self.max_position_embeddings    ,			type_vocab_size=self.type_vocab_size    ,			is_decoder=_snake_case    ,			initializer_range=self.initializer_range    ,			pad_token_id=1    ,			new_decoder_architecture=_snake_case    ,			)


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case)							->				Dict:
								UpperCAmelCase_		:      Optional[Any]       	=					FalconModel(config=_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      str       	=					model(_snake_case    ,			attention_mask=_snake_case)
								UpperCAmelCase_		:      Tuple       	=					model(_snake_case)
								self.parent.assertEqual(result.last_hidden_state.shape    ,			(self.batch_size, self.seq_length, self.hidden_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			)							->				Optional[Any]:
								UpperCAmelCase_		:      Dict       	=					True
								UpperCAmelCase_		:      Any       	=					FalconModel(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      List[Any]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			)
								UpperCAmelCase_		:      Optional[Any]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			)
								UpperCAmelCase_		:      Union[str, Any]       	=					model(_snake_case    ,			attention_mask=_snake_case)
								self.parent.assertEqual(result.last_hidden_state.shape    ,			(self.batch_size, self.seq_length, self.hidden_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			)							->				Optional[int]:
								UpperCAmelCase_		:      List[str]       	=					FalconForCausalLM(config=_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Dict       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.parent.assertEqual(result.logits.shape    ,			(self.batch_size, self.seq_length, self.vocab_size))


							def 						_snake_case							(			self    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			_snake_case    ,			)							->				List[str]:
								UpperCAmelCase_		:      List[str]       	=					True
								UpperCAmelCase_		:      Optional[int]       	=					True
								UpperCAmelCase_		:      Tuple       	=					FalconForCausalLM(config=_snake_case)
								model.to(_snake_case)
								model.eval()

								# first forward pass
								UpperCAmelCase_		:      str       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			use_cache=_snake_case    ,			)
								UpperCAmelCase_		:      Tuple       	=					outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								UpperCAmelCase_		:      List[str]       	=					ids_tensor((self.batch_size, 3)    ,			config.vocab_size)
								UpperCAmelCase_		:      int       	=					ids_tensor((self.batch_size, 3)    ,			vocab_size=2)

								# append to next input_ids and
								UpperCAmelCase_		:      int       	=					torch.cat([input_ids, next_tokens]    ,			dim=-1)
								UpperCAmelCase_		:      List[str]       	=					torch.cat([input_mask, next_mask]    ,			dim=-1)

								UpperCAmelCase_		:      List[Any]       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			output_hidden_states=_snake_case    ,			)['hidden_states'][0]
								UpperCAmelCase_		:      str       	=					model(
								    _snake_case    ,			attention_mask=_snake_case    ,			encoder_hidden_states=_snake_case    ,			encoder_attention_mask=_snake_case    ,			past_key_values=_snake_case    ,			output_hidden_states=_snake_case    ,			)['hidden_states'][0]

								# select random slice
								UpperCAmelCase_		:      Optional[Any]       	=					ids_tensor((1,)    ,			output_from_past.shape[-1]).item()
								UpperCAmelCase_		:      Any       	=					output_from_no_past[:, -3:, random_slice_idx].detach()
								UpperCAmelCase_		:      Any       	=					output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(_snake_case    ,			_snake_case    ,			atol=1e-3))





							def 						_snake_case							(			self)							->				Tuple:
								UpperCAmelCase_		:      Tuple       	=					self.prepare_config_and_inputs()
								(
								    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    (
								    UpperCAmelCase_
								)	,    
								)		:      int       	=					config_and_inputs
								UpperCAmelCase_		:      str       	=					{'input_ids': input_ids, 'attention_mask': input_mask}
								return config, inputs_dict



@require_torch
class 					lowercase						(  a_,					a_,					a_,					unittest.TestCase ):
							_lowerCamelCase		:      Optional[int]=   (
							    (
							        FalconModel,
							        FalconForCausalLM,
							        FalconForSequenceClassification,
							        FalconForTokenClassification,
							        FalconForQuestionAnswering,
							    )
							    if is_torch_available()
							    else ()
							)
							_lowerCamelCase		:      List[Any]=   (FalconForCausalLM,) if is_torch_available() else ()
							_lowerCamelCase		:      List[str]=   (
							    {
							        "feature-extraction": FalconModel,
							        "text-classification": FalconForSequenceClassification,
							        "text-generation": FalconForCausalLM,
							        "question-answering": FalconForQuestionAnswering,
							        "token-classification": FalconForTokenClassification,
							        "zero-shot": FalconForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							_lowerCamelCase		:      Tuple=   False
							_lowerCamelCase		:      Optional[Any]=   False


							def 						_snake_case							(			self)							->				List[str]:
								UpperCAmelCase_		:      Any       	=					FalconModelTester(self)
								UpperCAmelCase_		:      List[str]       	=					ConfigTester(self    ,			config_class=_snake_case    ,			hidden_size=37)


							def 						_snake_case							(			self)							->				Union[str, Any]:
								self.config_tester.run_common_tests()


							def 						_snake_case							(			self)							->				Optional[int]:
								UpperCAmelCase_		:      Tuple       	=					self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_snake_case)


							def 						_snake_case							(			self)							->				Dict:
								UpperCAmelCase_	,    *UpperCAmelCase_		:      List[str]       	=					self.model_tester.prepare_config_and_inputs()
								for alibi in [True, False]:
									UpperCAmelCase_		:      Tuple       	=					alibi
									self.model_tester.create_and_check_model(_snake_case    ,			*_snake_case)


							def 						_snake_case							(			self)							->				Tuple:
								UpperCAmelCase_	,    UpperCAmelCase_		:      Dict       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      Union[str, Any]       	=					3
								UpperCAmelCase_		:      Optional[Any]       	=					input_dict['input_ids']
								UpperCAmelCase_		:      str       	=					input_ids.ne(1).to(_snake_case)
								UpperCAmelCase_		:      List[Any]       	=					ids_tensor([self.model_tester.batch_size]    ,			self.model_tester.type_sequence_label_size)
								UpperCAmelCase_		:      List[str]       	=					FalconForSequenceClassification(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Any       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.assertEqual(result.logits.shape    ,			(self.model_tester.batch_size, self.model_tester.num_labels))


							def 						_snake_case							(			self)							->				Tuple:
								UpperCAmelCase_	,    UpperCAmelCase_		:      List[Any]       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      Optional[int]       	=					3
								UpperCAmelCase_		:      Optional[int]       	=					'single_label_classification'
								UpperCAmelCase_		:      Any       	=					input_dict['input_ids']
								UpperCAmelCase_		:      List[Any]       	=					input_ids.ne(1).to(_snake_case)
								UpperCAmelCase_		:      Optional[Any]       	=					ids_tensor([self.model_tester.batch_size]    ,			self.model_tester.type_sequence_label_size)
								UpperCAmelCase_		:      int       	=					FalconForSequenceClassification(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Tuple       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.assertEqual(result.logits.shape    ,			(self.model_tester.batch_size, self.model_tester.num_labels))


							def 						_snake_case							(			self)							->				str:
								UpperCAmelCase_	,    UpperCAmelCase_		:      Optional[int]       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      Any       	=					input_dict['input_ids']
								UpperCAmelCase_		:      Tuple       	=					FalconForCausalLM(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      int       	=					model(_snake_case    ,			use_cache=_snake_case)
								UpperCAmelCase_		:      Optional[Any]       	=					input_ids.shape[0]
								UpperCAmelCase_		:      Any       	=					model._convert_to_rw_cache(result.past_key_values)
								UpperCAmelCase_		:      Tuple       	=					model._convert_cache_to_standard_format(_snake_case    ,			_snake_case)
								for layer in range(len(_snake_case)):
									for tensor_idx in range(2):
										self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3)
										self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4)
										self.assertTrue(
										    torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]))


							def 						_snake_case							(			self)							->				Union[str, Any]:
								UpperCAmelCase_	,    UpperCAmelCase_		:      str       	=					self.model_tester.prepare_config_and_inputs_for_common()
								UpperCAmelCase_		:      Tuple       	=					3
								UpperCAmelCase_		:      Any       	=					'multi_label_classification'
								UpperCAmelCase_		:      Optional[Any]       	=					input_dict['input_ids']
								UpperCAmelCase_		:      Tuple       	=					input_ids.ne(1).to(_snake_case)
								UpperCAmelCase_		:      Tuple       	=					ids_tensor(
								    [self.model_tester.batch_size, config.num_labels]    ,			self.model_tester.type_sequence_label_size).to(torch.float)
								UpperCAmelCase_		:      Optional[int]       	=					FalconForSequenceClassification(_snake_case)
								model.to(_snake_case)
								model.eval()
								UpperCAmelCase_		:      Dict       	=					model(_snake_case    ,			attention_mask=_snake_case    ,			labels=_snake_case)
								self.assertEqual(result.logits.shape    ,			(self.model_tester.batch_size, self.model_tester.num_labels))





							def 						_snake_case							(			self)							->				Dict:
								# Falcon can have different numbers of KV-heads than the number of query heads, so we need
								# to override this test to use the right head counts.
								for model_class in self.all_generative_model_classes:
									UpperCAmelCase_	,    UpperCAmelCase_		:      Union[str, Any]       	=					self.model_tester.prepare_config_and_inputs_for_common()

									# If it doesn't support cache, pass the test
									if not hasattr(_snake_case    ,			'use_cache'):
										return

									UpperCAmelCase_		:      List[Any]       	=					model_class(_snake_case).to(_snake_case)
									if "use_cache" not in inputs:
										UpperCAmelCase_		:      Dict       	=					True
									UpperCAmelCase_		:      Any       	=					model(**_snake_case)

									# If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
									if "past_key_values" not in outputs:
										return

									UpperCAmelCase_		:      Union[str, Any]       	=					(
									    getattr(_snake_case    ,			'decoder_layers'    ,			_snake_case)
									    or getattr(_snake_case    ,			'num_decoder_layers'    ,			_snake_case)
									    or config.num_hidden_layers
									)
									UpperCAmelCase_		:      Any       	=					getattr(_snake_case    ,			'num_kv_heads'    ,			config.num_attention_heads)
									UpperCAmelCase_		:      List[Any]       	=					getattr(_snake_case    ,			'd_model'    ,			config.hidden_size)
									UpperCAmelCase_		:      Any       	=					embed_dim // num_attention_heads

									UpperCAmelCase_		:      Optional[Any]       	=					outputs['past_key_values']
									self.assertEqual(len(_snake_case)    ,			_snake_case)

									UpperCAmelCase_	,    UpperCAmelCase_		:      Optional[int]       	=					inputs['input_ids'].shape
									for i in range(_snake_case):
										if config.new_decoder_architecture:
											UpperCAmelCase_		:      Union[str, Any]       	=					config.num_attention_heads
										elif config.multi_query:
											UpperCAmelCase_		:      int       	=					1
										self.assertEqual(len(past_kv[0])    ,			2)  # K V for the decoder = 2
										self.assertEqual(
										    past_kv[i][0].shape    ,			(batch_size, num_attention_heads, seq_length, per_head_embed_dim))
										self.assertEqual(
										    past_kv[i][1].shape    ,			(batch_size, num_attention_heads, seq_length, per_head_embed_dim))



@require_torch
class 					lowercase						(  unittest.TestCase ):


							@slow
							def 						_snake_case							(			self)							->				List[Any]:
								UpperCAmelCase_		:      int       	=					AutoTokenizer.from_pretrained('Rocketknight1/falcon-rw-1b')
								UpperCAmelCase_		:      List[str]       	=					FalconForCausalLM.from_pretrained('Rocketknight1/falcon-rw-1b')
								model.eval()
								model.to(_snake_case)
								UpperCAmelCase_		:      Any       	=					tokenizer('My favorite food is'    ,			return_tensors='pt').to(_snake_case)

								UpperCAmelCase_		:      Union[str, Any]       	=					(
								    'My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday.'
								)

								UpperCAmelCase_		:      Optional[Any]       	=					model.generate(**_snake_case    ,			do_sample=_snake_case    ,			max_new_tokens=19)
								UpperCAmelCase_		:      int       	=					tokenizer.batch_decode(_snake_case)[0]

								self.assertEqual(_snake_case    ,			_snake_case)


							@slow
							def 						_snake_case							(			self)							->				Any:
								# The big models are way too big for the CI, so we use tiny random models that resemble their
								# architectures but with much smaller and fewer layers
								for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
									UpperCAmelCase_		:      List[Any]       	=					AutoTokenizer.from_pretrained(_snake_case)
									UpperCAmelCase_		:      List[str]       	=					FalconForCausalLM.from_pretrained(_snake_case)
									model.eval()
									model.to(_snake_case)
									UpperCAmelCase_		:      Dict       	=					tokenizer('My favorite food is'    ,			return_tensors='pt').to(_snake_case)

									# We just test that these run without errors - the models are randomly initialized
									# and so the actual text outputs will be garbage
									model.generate(**_snake_case    ,			do_sample=_snake_case    ,			max_new_tokens=4)
									model.generate(**_snake_case    ,			do_sample=_snake_case    ,			max_new_tokens=4)
									model.generate(**_snake_case    ,			num_beams=2    ,			max_new_tokens=4)





							@slow
							def 						_snake_case							(			self)							->				Dict:
								# The big models are way too big for the CI, so we use tiny random models that resemble their
								# architectures but with much smaller and fewer layers
								with torch.no_grad():
									for repo in [
									    "Rocketknight1/falcon-rw-1b",
									    "Rocketknight1/tiny-random-falcon-7b",
									    "Rocketknight1/tiny-random-falcon-40b",
									]:
										UpperCAmelCase_		:      Optional[Any]       	=					AutoTokenizer.from_pretrained(_snake_case)
										UpperCAmelCase_		:      Tuple       	=					FalconForCausalLM.from_pretrained(_snake_case)
										model.eval()
										model.to(device=_snake_case)
										UpperCAmelCase_		:      int       	=					tokenizer('My favorite food is'    ,			return_tensors='pt').to(_snake_case)

										# Test results are the same with and without cache
										UpperCAmelCase_		:      str       	=					model.generate(**_snake_case    ,			do_sample=_snake_case    ,			max_new_tokens=20    ,			use_cache=_snake_case)
										UpperCAmelCase_		:      Dict       	=					model.generate(**_snake_case    ,			do_sample=_snake_case    ,			max_new_tokens=20    ,			use_cache=_snake_case)
										self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0)









'''simple docstring'''


from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__										= logging.get_logger(__name__)

lowerCAmelCase__										= {
    "tanreinama/GPTSAN-2.8B-spout_is_uniform": (
        "https://huggingface.co/tanreinama/GPTSAN-2.8B-spout_is_uniform/resolve/main/config.json"
    ),
}



class 					lowercase						(  a_ ):
							_lowerCamelCase		:      Any=   "gptsan-japanese"
							_lowerCamelCase		:      List[Any]=   [
							    "past_key_values",
							]
							_lowerCamelCase		:      Tuple=   {
							    "hidden_size": "d_model",
							    "num_attention_heads": "num_heads",
							    "num_hidden_layers": "num_layers",
							}


							def __init__(			self    ,			_snake_case=3_6000    ,			_snake_case=1280    ,			_snake_case=1024    ,			_snake_case=8192    ,			_snake_case=4096    ,			_snake_case=128    ,			_snake_case=10    ,			_snake_case=0    ,			_snake_case=16    ,			_snake_case=16    ,			_snake_case=128    ,			_snake_case=0.0    ,			_snake_case=1e-5    ,			_snake_case=False    ,			_snake_case=0.0    ,			_snake_case="float32"    ,			_snake_case=False    ,			_snake_case=False    ,			_snake_case=False    ,			_snake_case=0.002    ,			_snake_case=False    ,			_snake_case=True    ,			_snake_case=3_5998    ,			_snake_case=3_5995    ,			_snake_case=3_5999    ,			**_snake_case    ,			)							->				Optional[int]:
								UpperCAmelCase_		:      Dict       	=					vocab_size
								UpperCAmelCase_		:      Optional[int]       	=					max_position_embeddings
								UpperCAmelCase_		:      Any       	=					d_model
								UpperCAmelCase_		:      int       	=					d_ff
								UpperCAmelCase_		:      Tuple       	=					d_ext
								UpperCAmelCase_		:      Tuple       	=					d_spout
								UpperCAmelCase_		:      List[Any]       	=					num_switch_layers
								UpperCAmelCase_		:      Tuple       	=					num_ext_layers
								UpperCAmelCase_		:      Optional[int]       	=					num_switch_layers + num_ext_layers
								UpperCAmelCase_		:      Tuple       	=					num_heads
								UpperCAmelCase_		:      List[str]       	=					num_experts
								UpperCAmelCase_		:      List[str]       	=					expert_capacity
								UpperCAmelCase_		:      Optional[int]       	=					dropout_rate
								UpperCAmelCase_		:      List[Any]       	=					layer_norm_epsilon
								UpperCAmelCase_		:      Any       	=					router_bias
								UpperCAmelCase_		:      Optional[Any]       	=					router_jitter_noise
								UpperCAmelCase_		:      str       	=					router_dtype
								UpperCAmelCase_		:      List[str]       	=					router_ignore_padding_tokens
								UpperCAmelCase_		:      Optional[int]       	=					output_hidden_states
								UpperCAmelCase_		:      Union[str, Any]       	=					output_attentions
								UpperCAmelCase_		:      Optional[int]       	=					initializer_factor
								UpperCAmelCase_		:      Optional[int]       	=					output_router_logits
								UpperCAmelCase_		:      List[str]       	=					use_cache

								super().__init__(
								    separator_token_id=_snake_case    ,			pad_token_id=_snake_case    ,			eos_token_id=_snake_case    ,			**_snake_case    ,			)






