




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__lowerCamelCase   :						Union[str, Any]				=			{
    '''configuration_bert''': ['''BERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BertConfig''', '''BertOnnxConfig'''],
    '''tokenization_bert''': ['''BasicTokenizer''', '''BertTokenizer''', '''WordpieceTokenizer'''],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowerCamelCase   :						Any				=			['''BertTokenizerFast''']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowerCamelCase   :						List[str]				=			[
     '''BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''BertForMaskedLM''',
     '''BertForMultipleChoice''',
     '''BertForNextSentencePrediction''',
     '''BertForPreTraining''',
     '''BertForQuestionAnswering''',
     '''BertForSequenceClassification''',
     '''BertForTokenClassification''',
     '''BertLayer''',
     '''BertLMHeadModel''',
     '''BertModel''',
     '''BertPreTrainedModel''',
     '''load_tf_weights_in_bert''',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowerCamelCase   :						List[str]				=			[
     '''TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''TFBertEmbeddings''',
     '''TFBertForMaskedLM''',
     '''TFBertForMultipleChoice''',
     '''TFBertForNextSentencePrediction''',
     '''TFBertForPreTraining''',
     '''TFBertForQuestionAnswering''',
     '''TFBertForSequenceClassification''',
     '''TFBertForTokenClassification''',
     '''TFBertLMHeadModel''',
     '''TFBertMainLayer''',
     '''TFBertModel''',
     '''TFBertPreTrainedModel''',
 ]
try:
 if not is_tensorflow_text_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowerCamelCase   :						Optional[Any]				=			['''TFBertTokenizer''']

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __lowerCamelCase   :						int				=			[
     '''FlaxBertForCausalLM''',
     '''FlaxBertForMaskedLM''',
     '''FlaxBertForMultipleChoice''',
     '''FlaxBertForNextSentencePrediction''',
     '''FlaxBertForPreTraining''',
     '''FlaxBertForQuestionAnswering''',
     '''FlaxBertForSequenceClassification''',
     '''FlaxBertForTokenClassification''',
     '''FlaxBertModel''',
     '''FlaxBertPreTrainedModel''',
 ]

if TYPE_CHECKING:
 from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
 from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_bert_fast import BertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_bert import (
      BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      BertForMaskedLM,
      BertForMultipleChoice,
      BertForNextSentencePrediction,
      BertForPreTraining,
      BertForQuestionAnswering,
      BertForSequenceClassification,
      BertForTokenClassification,
      BertLayer,
      BertLMHeadModel,
      BertModel,
      BertPreTrainedModel,
      load_tf_weights_in_bert,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_bert import (
      TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFBertEmbeddings,
      TFBertForMaskedLM,
      TFBertForMultipleChoice,
      TFBertForNextSentencePrediction,
      TFBertForPreTraining,
      TFBertForQuestionAnswering,
      TFBertForSequenceClassification,
      TFBertForTokenClassification,
      TFBertLMHeadModel,
      TFBertMainLayer,
      TFBertModel,
      TFBertPreTrainedModel,
  )

 try:
  if not is_tensorflow_text_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_bert_tf import TFBertTokenizer

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_bert import (
      FlaxBertForCausalLM,
      FlaxBertForMaskedLM,
      FlaxBertForMultipleChoice,
      FlaxBertForNextSentencePrediction,
      FlaxBertForPreTraining,
      FlaxBertForQuestionAnswering,
      FlaxBertForSequenceClassification,
      FlaxBertForTokenClassification,
      FlaxBertModel,
      FlaxBertPreTrainedModel,
  )

else:
 import sys

 __lowerCamelCase   :						Optional[Any]				=			_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)









from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase   :						Optional[int]				=			logging.get_logger(__name__)

__lowerCamelCase   :						Dict				=			{}


class    lowerCamelCase    (		_lowerCamelCase						):






			'''simple docstring'''





			UpperCamelCase__								='''llama'''
			UpperCamelCase__								=['''past_key_values''']

			def __init__( self			:   Dict       ,						lowerCamelCase_			:   List[str]=32000       ,						lowerCamelCase_			:   List[Any]=4096       ,						lowerCamelCase_			:   Any=11008       ,						lowerCamelCase_			:   Optional[Any]=32       ,						lowerCamelCase_			:   Dict=32       ,						lowerCamelCase_			:   Optional[Any]=None       ,						lowerCamelCase_			:   Dict="silu"       ,						lowerCamelCase_			:   Optional[int]=2048       ,						lowerCamelCase_			:   int=0.0_2       ,						lowerCamelCase_			:   Tuple=1E-6       ,						lowerCamelCase_			:   Tuple=True       ,						lowerCamelCase_			:   List[str]=0       ,						lowerCamelCase_			:   List[str]=1       ,						lowerCamelCase_			:   Optional[int]=2       ,						lowerCamelCase_			:   List[str]=1       ,						lowerCamelCase_			:   Dict=False       ,						lowerCamelCase_			:   int=None       ,						**lowerCamelCase_			:   Tuple       ,						)						->   Tuple:
								__magic_name__							:	Any    							=							vocab_size
								__magic_name__							:	Optional[int]    							=							max_position_embeddings
								__magic_name__							:	Union[str, Any]    							=							hidden_size
								__magic_name__							:	Dict    							=							intermediate_size
								__magic_name__							:	Union[str, Any]    							=							num_hidden_layers
								__magic_name__							:	Tuple    							=							num_attention_heads

								# for backward compatibility
								if num_key_value_heads is None:
													__magic_name__							:	Optional[int]    							=							num_attention_heads

								__magic_name__							:	Optional[int]    							=							num_key_value_heads
								__magic_name__							:	List[str]    							=							hidden_act
								__magic_name__							:	List[str]    							=							initializer_range
								__magic_name__							:	Tuple    							=							rms_norm_eps
								__magic_name__							:	Optional[int]    							=							pretraining_tp
								__magic_name__							:	Any    							=							use_cache
								__magic_name__							:	int    							=							rope_scaling
								self._rope_scaling_validation()

								super().__init__(
								    pad_token_id=lowerCamelCase_       ,						bos_token_id=lowerCamelCase_       ,						eos_token_id=lowerCamelCase_       ,						tie_word_embeddings=lowerCamelCase_       ,						**lowerCamelCase_       ,						)


			def 							UpperCAmelCase__       ( self			:   str  )						->   Tuple:
								if self.rope_scaling is None:
													return

								if not isinstance(self.rope_scaling       ,						lowerCamelCase_  ) or len(self.rope_scaling  ) != 2:
													raise ValueError(
													    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
													    F'''got {self.rope_scaling}'''  )
								__magic_name__							:	List[str]    							=							self.rope_scaling.get('''type'''       ,						lowerCamelCase_  )
								__magic_name__							:	str    							=							self.rope_scaling.get('''factor'''       ,						lowerCamelCase_  )
								if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
													raise ValueError(
													    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''  )
								if rope_scaling_factor is None or not isinstance(lowerCamelCase_       ,						lowerCamelCase_  ) or rope_scaling_factor <= 1.0:
													raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''  )



