
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


lowerCamelCase__ :       Any						     =			logging.get_logger(__name__)

lowerCamelCase__ :       List[str]						     =			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCamelCase__ :       Tuple						     =			{
    "vocab_file": {"mobilebert-uncased": "https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"},
    "tokenizer_file": {
        "mobilebert-uncased": "https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"
    },
}

lowerCamelCase__ :       Tuple						     =			{"mobilebert-uncased": 512}


lowerCamelCase__ :       Union[str, Any]						     =			{}




class 				lowercase__(				_UpperCAmelCase   ):
	'''simple docstring'''

	UpperCamelCase				=  VOCAB_FILES_NAMES
	UpperCamelCase				=  PRETRAINED_VOCAB_FILES_MAP
	UpperCamelCase				=  PRETRAINED_INIT_CONFIGURATION
	UpperCamelCase				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCamelCase				=  MobileBertTokenizer





	def __init__(						self    :List[str]		, lowerCamelCase_    :str=None		, lowerCamelCase_    :int=None		, lowerCamelCase_    :Tuple=True		, lowerCamelCase_    :int="[UNK]"		, lowerCamelCase_    :Optional[int]="[SEP]"		, lowerCamelCase_    :int="[PAD]"		, lowerCamelCase_    :Tuple="[CLS]"		, lowerCamelCase_    :Dict="[MASK]"		, lowerCamelCase_    :str=True		, lowerCamelCase_    :Optional[Any]=None		, **lowerCamelCase_    :List[str]		, )							->			Tuple:







			'''simple docstring'''




			super().__init__(
			    lowercase__		, tokenizer_file=lowercase__		, do_lower_case=lowercase__		, unk_token=lowercase__		, sep_token=lowercase__		, pad_token=lowercase__		, cls_token=lowercase__		, mask_token=lowercase__		, tokenize_chinese_chars=lowercase__		, strip_accents=lowercase__		, **lowercase__		, )

			SCREAMING_SNAKE_CASE       :     Optional[Any]				= json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
			if (
			    normalizer_state.get('''lowercase'''		, lowercase__					) != do_lower_case
			    or normalizer_state.get('''strip_accents'''		, lowercase__					) != strip_accents
			    or normalizer_state.get('''handle_chinese_chars'''		, lowercase__					) != tokenize_chinese_chars
			):
					SCREAMING_SNAKE_CASE       :     Optional[Any]				= getattr(lowercase__		, normalizer_state.pop('''type'''					)					)
					SCREAMING_SNAKE_CASE       :     Any				= do_lower_case
					SCREAMING_SNAKE_CASE       :     Any				= strip_accents
					SCREAMING_SNAKE_CASE       :     Optional[int]				= tokenize_chinese_chars
					SCREAMING_SNAKE_CASE       :     Optional[int]				= normalizer_class(**lowercase__					)

			SCREAMING_SNAKE_CASE       :     Optional[int]				= do_lower_case





	def   __lowerCAmelCase (						self    :Any		, lowerCamelCase_    :Any		, lowerCamelCase_    :Union[str, Any]=None					)							->			List[str]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     List[str]				= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

			if token_ids_a:
					output += token_ids_a + [self.sep_token_id]

			return output





	def   __lowerCAmelCase (						self    :Optional[int]		, lowerCamelCase_    :Union[str, Any]		, lowerCamelCase_    :List[str] = None					)							->			Optional[Any]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Dict				= [self.sep_token_id]
			SCREAMING_SNAKE_CASE       :     Any				= [self.cls_token_id]
			if token_ids_a is None:
					return len(cls + token_ids_a + sep					) * [0]
			return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]







	def   __lowerCAmelCase (						self    :int		, lowerCamelCase_    :Union[str, Any]		, lowerCamelCase_    :Any = None					)							->			str:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= self._tokenizer.model.save(lowercase__		, name=lowercase__					)
			return tuple(lowercase__					)







"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase__ :       Tuple						     =			logging.get_logger(__name__)

lowerCamelCase__ :       str						     =			{
    "studio-ousia/luke-base": "https://huggingface.co/studio-ousia/luke-base/resolve/main/config.json",
    "studio-ousia/luke-large": "https://huggingface.co/studio-ousia/luke-large/resolve/main/config.json",
}




class 				lowercase__(				_UpperCAmelCase   ):
	'''simple docstring'''

	UpperCamelCase				=  """luke"""





	def __init__(						self    :Optional[int]		, lowerCamelCase_    :Union[str, Any]=5_02_67		, lowerCamelCase_    :int=50_00_00		, lowerCamelCase_    :Tuple=7_68		, lowerCamelCase_    :List[str]=2_56		, lowerCamelCase_    :Dict=12		, lowerCamelCase_    :Optional[int]=12		, lowerCamelCase_    :Optional[Any]=30_72		, lowerCamelCase_    :List[Any]="gelu"		, lowerCamelCase_    :Any=0.1		, lowerCamelCase_    :Union[str, Any]=0.1		, lowerCamelCase_    :List[str]=5_12		, lowerCamelCase_    :Optional[int]=2		, lowerCamelCase_    :Tuple=0.0_2		, lowerCamelCase_    :Optional[int]=1E-12		, lowerCamelCase_    :List[Any]=True		, lowerCamelCase_    :List[str]=None		, lowerCamelCase_    :str=1		, lowerCamelCase_    :Any=0		, lowerCamelCase_    :str=2		, **lowerCamelCase_    :List[Any]		, )							->			Optional[int]:







			'''simple docstring'''




			super().__init__(pad_token_id=lowerCamelCase_		, bos_token_id=lowerCamelCase_		, eos_token_id=lowerCamelCase_		, **lowerCamelCase_					)

			SCREAMING_SNAKE_CASE       :     Dict				= vocab_size
			SCREAMING_SNAKE_CASE       :     List[str]				= entity_vocab_size
			SCREAMING_SNAKE_CASE       :     str				= hidden_size
			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= entity_emb_size
			SCREAMING_SNAKE_CASE       :     List[Any]				= num_hidden_layers
			SCREAMING_SNAKE_CASE       :     Optional[int]				= num_attention_heads
			SCREAMING_SNAKE_CASE       :     Optional[Any]				= hidden_act
			SCREAMING_SNAKE_CASE       :     Optional[Any]				= intermediate_size
			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= hidden_dropout_prob
			SCREAMING_SNAKE_CASE       :     Dict				= attention_probs_dropout_prob
			SCREAMING_SNAKE_CASE       :     Dict				= max_position_embeddings
			SCREAMING_SNAKE_CASE       :     Tuple				= type_vocab_size
			SCREAMING_SNAKE_CASE       :     str				= initializer_range
			SCREAMING_SNAKE_CASE       :     Dict				= layer_norm_eps
			SCREAMING_SNAKE_CASE       :     Optional[int]				= use_entity_aware_attention
			SCREAMING_SNAKE_CASE       :     List[str]				= classifier_dropout





