import copy
import os
from typing import TYPE_CHECKING, List, Union


if TYPE_CHECKING:
		pass

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_						=logging.get_logger(__name__)

UpperCAmelCase_						={
    """kakaobrain/align-base""": """https://huggingface.co/kakaobrain/align-base/resolve/main/config.json""",
}


class 							__UpperCamelCase			( UpperCAmelCase_  ):





				'''simple docstring'''



				__a							: List[str]										="align_text_model"



				def __init__( self       , UpperCAmelCase_=3_05_22       , UpperCAmelCase_=7_68       , UpperCAmelCase_=12       , UpperCAmelCase_=12       , UpperCAmelCase_=30_72       , UpperCAmelCase_="gelu"       , UpperCAmelCase_=0.1       , UpperCAmelCase_=0.1       , UpperCAmelCase_=5_12       , UpperCAmelCase_=2       , UpperCAmelCase_=0.02       , UpperCAmelCase_=1E-1_2       , UpperCAmelCase_=0       , UpperCAmelCase_="absolute"       , UpperCAmelCase_=True       , **UpperCAmelCase_       , ):
											super().__init__(**_snake_case						)

											lowerCAmelCase   =		vocab_size
											lowerCAmelCase   =		hidden_size
											lowerCAmelCase   =		num_hidden_layers
											lowerCAmelCase   =		num_attention_heads
											lowerCAmelCase   =		hidden_act
											lowerCAmelCase   =		intermediate_size
											lowerCAmelCase   =		hidden_dropout_prob
											lowerCAmelCase   =		attention_probs_dropout_prob
											lowerCAmelCase   =		max_position_embeddings
											lowerCAmelCase   =		type_vocab_size
											lowerCAmelCase   =		initializer_range
											lowerCAmelCase   =		layer_norm_eps
											lowerCAmelCase   =		position_embedding_type
											lowerCAmelCase   =		use_cache
											lowerCAmelCase   =		pad_token_id






				@classmethod
				def       __snake_case		( cls       , UpperCAmelCase_       , **UpperCAmelCase_						):
											cls._set_token_in_kwargs(_snake_case						)

											lowerCAmelCase   ,     lowerCAmelCase   =		cls.get_config_dict(_snake_case       , **_snake_case						)

											# get the text config dict if we are loading from AlignConfig
											if config_dict.get('''model_type'''						) == "align":
																		lowerCAmelCase   =		config_dict['''text_config''']

											if "model_type" in config_dict and hasattr(cls       , '''model_type'''						) and config_dict["model_type"] != cls.model_type:
																		logger.warning(
																		    F"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
																		    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""						)

											return cls.from_dict(_snake_case       , **_snake_case						)




class 							__UpperCamelCase			( UpperCAmelCase_  ):





				'''simple docstring'''



				__a							: str										="align_vision_model"



				def __init__( self       , UpperCAmelCase_ = 3       , UpperCAmelCase_ = 6_00       , UpperCAmelCase_ = 2.0       , UpperCAmelCase_ = 3.1       , UpperCAmelCase_ = 8       , UpperCAmelCase_ = [3, 3, 5, 3, 5, 5, 3]       , UpperCAmelCase_ = [32, 16, 24, 40, 80, 1_12, 1_92]       , UpperCAmelCase_ = [16, 24, 40, 80, 1_12, 1_92, 3_20]       , UpperCAmelCase_ = []       , UpperCAmelCase_ = [1, 2, 2, 2, 1, 2, 1]       , UpperCAmelCase_ = [1, 2, 2, 3, 3, 4, 1]       , UpperCAmelCase_ = [1, 6, 6, 6, 6, 6, 6]       , UpperCAmelCase_ = 0.25       , UpperCAmelCase_ = "swish"       , UpperCAmelCase_ = 25_60       , UpperCAmelCase_ = "mean"       , UpperCAmelCase_ = 0.02       , UpperCAmelCase_ = 0.001       , UpperCAmelCase_ = 0.99       , UpperCAmelCase_ = 0.2       , **UpperCAmelCase_       , ):
											super().__init__(**_snake_case						)

											lowerCAmelCase   =		num_channels
											lowerCAmelCase   =		image_size
											lowerCAmelCase   =		width_coefficient
											lowerCAmelCase   =		depth_coefficient
											lowerCAmelCase   =		depth_divisor
											lowerCAmelCase   =		kernel_sizes
											lowerCAmelCase   =		in_channels
											lowerCAmelCase   =		out_channels
											lowerCAmelCase   =		depthwise_padding
											lowerCAmelCase   =		strides
											lowerCAmelCase   =		num_block_repeats
											lowerCAmelCase   =		expand_ratios
											lowerCAmelCase   =		squeeze_expansion_ratio
											lowerCAmelCase   =		hidden_act
											lowerCAmelCase   =		hidden_dim
											lowerCAmelCase   =		pooling_type
											lowerCAmelCase   =		initializer_range
											lowerCAmelCase   =		batch_norm_eps
											lowerCAmelCase   =		batch_norm_momentum
											lowerCAmelCase   =		drop_connect_rate
											lowerCAmelCase   =		sum(_snake_case						) * 4






				@classmethod
				def       __snake_case		( cls       , UpperCAmelCase_       , **UpperCAmelCase_						):
											cls._set_token_in_kwargs(_snake_case						)

											lowerCAmelCase   ,     lowerCAmelCase   =		cls.get_config_dict(_snake_case       , **_snake_case						)

											# get the vision config dict if we are loading from AlignConfig
											if config_dict.get('''model_type'''						) == "align":
																		lowerCAmelCase   =		config_dict['''vision_config''']

											if "model_type" in config_dict and hasattr(cls       , '''model_type'''						) and config_dict["model_type"] != cls.model_type:
																		logger.warning(
																		    F"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
																		    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""						)

											return cls.from_dict(_snake_case       , **_snake_case						)




class 							__UpperCamelCase			( UpperCAmelCase_  ):





				'''simple docstring'''



				__a							: Optional[int]										="align"
				__a							: str										=True



				def __init__( self       , UpperCAmelCase_=None       , UpperCAmelCase_=None       , UpperCAmelCase_=6_40       , UpperCAmelCase_=1.0       , UpperCAmelCase_=0.02       , **UpperCAmelCase_       , ):
											super().__init__(**_snake_case						)

											if text_config is None:
																		lowerCAmelCase   =		{}
																		logger.info('''text_config is None. Initializing the AlignTextConfig with default values.'''						)

											if vision_config is None:
																		lowerCAmelCase   =		{}
																		logger.info('''vision_config is None. Initializing the AlignVisionConfig with default values.'''						)

											lowerCAmelCase   =		AlignTextConfig(**_snake_case						)
											lowerCAmelCase   =		AlignVisionConfig(**_snake_case						)

											lowerCAmelCase   =		projection_dim
											lowerCAmelCase   =		temperature_init_value
											lowerCAmelCase   =		initializer_range



				@classmethod
				def       __snake_case		( cls       , UpperCAmelCase_       , UpperCAmelCase_       , **UpperCAmelCase_						):
											return cls(text_config=text_config.to_dict()       , vision_config=vision_config.to_dict()       , **_snake_case						)






				def       __snake_case		( self						):
											lowerCAmelCase   =		copy.deepcopy(self.__dict__						)
											lowerCAmelCase   =		self.text_config.to_dict()
											lowerCAmelCase   =		self.vision_config.to_dict()
											lowerCAmelCase   =		self.__class__.model_type
											return output









from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_						=logging.get_logger(__name__)

UpperCAmelCase_						={
    """google/switch-base-8""": """https://huggingface.co/google/switch-base-8/blob/main/config.json""",
}


class 							__UpperCamelCase			( __UpperCAmelCase  ):





				'''simple docstring'''



				__a							: Any										="""switch_transformers"""
				__a							: Union[str, Any]										=["""past_key_values"""]
				__a							: Dict										={"""hidden_size""": """d_model""", """num_attention_heads""": """num_heads""", """num_hidden_layers""": """num_layers"""}



				def __init__( self       , UpperCAmelCase_=3_21_28       , UpperCAmelCase_=7_68       , UpperCAmelCase_=64       , UpperCAmelCase_=20_48       , UpperCAmelCase_=64       , UpperCAmelCase_=12       , UpperCAmelCase_=3       , UpperCAmelCase_=12       , UpperCAmelCase_=3       , UpperCAmelCase_=12       , UpperCAmelCase_=8       , UpperCAmelCase_=False       , UpperCAmelCase_=0.01       , UpperCAmelCase_="float32"       , UpperCAmelCase_=False       , UpperCAmelCase_=32       , UpperCAmelCase_=1_28       , UpperCAmelCase_=0.1       , UpperCAmelCase_=1E-6       , UpperCAmelCase_=0.001       , UpperCAmelCase_=0.001       , UpperCAmelCase_=1.0       , UpperCAmelCase_="relu"       , UpperCAmelCase_=True       , UpperCAmelCase_=False       , UpperCAmelCase_=True       , UpperCAmelCase_=0       , UpperCAmelCase_=1       , **UpperCAmelCase_       , ):
											lowerCAmelCase   =		vocab_size
											lowerCAmelCase   =		d_model
											lowerCAmelCase   =		d_kv
											lowerCAmelCase   =		d_ff

											lowerCAmelCase   =		num_sparse_encoder_layers

											lowerCAmelCase   =		num_layers
											lowerCAmelCase   =		(
											    num_decoder_layers if num_decoder_layers is not None else self.num_layers
											)  # default = symmetry
											lowerCAmelCase   =		num_sparse_decoder_layers

											# This tells us, each how many encoder layer we'll have to set a sparse layer.
											if self.num_sparse_encoder_layers > 0:
																		lowerCAmelCase   =		self.num_layers // self.num_sparse_encoder_layers
											else:
																		lowerCAmelCase   =		self.num_layers  # HACK: this will create 0 sparse layers

											# This tells us, each how many encoder layer we'll have to set a sparse layer.
											if self.num_sparse_decoder_layers > 0:
																		lowerCAmelCase   =		self.num_decoder_layers // self.num_sparse_decoder_layers
											else:
																		lowerCAmelCase   =		self.num_decoder_layers  # HACK: this will create 0 sparse layers

											lowerCAmelCase   =		num_heads
											lowerCAmelCase   =		num_experts
											lowerCAmelCase   =		expert_capacity
											lowerCAmelCase   =		router_bias
											lowerCAmelCase   =		router_jitter_noise
											if router_dtype not in ["float32", "float16", "bfloat16"]:
																		raise ValueError(F"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""						)
											lowerCAmelCase   =		router_dtype

											lowerCAmelCase   =		router_ignore_padding_tokens
											lowerCAmelCase   =		relative_attention_num_buckets
											lowerCAmelCase   =		relative_attention_max_distance

											lowerCAmelCase   =		dropout_rate
											lowerCAmelCase   =		layer_norm_epsilon
											lowerCAmelCase   =		initializer_factor
											lowerCAmelCase   =		feed_forward_proj
											lowerCAmelCase   =		use_cache
											lowerCAmelCase   =		add_router_probs

											lowerCAmelCase   =		router_z_loss_coef
											lowerCAmelCase   =		router_aux_loss_coef

											lowerCAmelCase   =		self.feed_forward_proj.split('''-'''						)
											lowerCAmelCase   =		act_info[-1]
											lowerCAmelCase   =		act_info[0] == '''gated'''

											if len(UpperCAmelCase_						) > 1 and act_info[0] != "gated" or len(UpperCAmelCase_						) > 2:
																		raise ValueError(
																		    F"""`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."""
																		    '''Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '''
																		    '''\'gated-gelu\' or \'relu\''''						)

											# for backwards compatibility
											if feed_forward_proj == "gated-gelu":
																		lowerCAmelCase   =		'''gelu_new'''

											super().__init__(
											    pad_token_id=UpperCAmelCase_       , eos_token_id=UpperCAmelCase_       , is_encoder_decoder=UpperCAmelCase_       , **UpperCAmelCase_       , )



