



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowercase_	       =			logging.get_logger(__name__)

lowercase_	       =			{'''vocab_file''': '''vocab.txt'''}

lowercase_	       =			{
    '''vocab_file''': {
        '''YituTech/conv-bert-base''': '''https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt''',
        '''YituTech/conv-bert-medium-small''': (
            '''https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt'''
        ),
        '''YituTech/conv-bert-small''': '''https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt''',
    }
}

lowercase_	       =			{
    '''YituTech/conv-bert-base''': 5_12,
    '''YituTech/conv-bert-medium-small''': 5_12,
    '''YituTech/conv-bert-small''': 5_12,
}


lowercase_	       =			{
    '''YituTech/conv-bert-base''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-medium-small''': {'''do_lower_case''': True},
    '''YituTech/conv-bert-small''': {'''do_lower_case''': True},
}






class 						__a (     SCREAMING_SNAKE_CASE				):
	SCREAMING_SNAKE_CASE       =       VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE       =       PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE       =       PRETRAINED_INIT_CONFIGURATION
	SCREAMING_SNAKE_CASE       =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE       =       ConvBertTokenizer



	def __init__(						self							:    List[str]				,	snake_case_							:    List[Any]=None				,	snake_case_							:    Optional[Any]=None				,	snake_case_							:    str=True				,	snake_case_							:    Optional[Any]="[UNK]"				,	snake_case_							:    str="[SEP]"				,	snake_case_							:    Tuple="[PAD]"				,	snake_case_							:    Optional[int]="[CLS]"				,	snake_case_							:    Optional[int]="[MASK]"				,	snake_case_							:    int=True				,	snake_case_							:    List[Any]=None				,	**snake_case_							:    Union[str, Any]				,	)->							List[str]:
						super().__init__(
						    snake_case_				,	tokenizer_file=snake_case_				,	do_lower_case=snake_case_				,	unk_token=snake_case_				,	sep_token=snake_case_				,	pad_token=snake_case_				,	cls_token=snake_case_				,	mask_token=snake_case_				,	tokenize_chinese_chars=snake_case_				,	strip_accents=snake_case_				,	**snake_case_				,	)

						__lowerCAmelCase	   =json.loads(self.backend_tokenizer.normalizer.__getstate__())
						if (
						    normalizer_state.get("""lowercase"""				,	snake_case_) != do_lower_case
						    or normalizer_state.get("""strip_accents"""				,	snake_case_) != strip_accents
						    or normalizer_state.get("""handle_chinese_chars"""				,	snake_case_) != tokenize_chinese_chars
						):
											__lowerCAmelCase	   =getattr(snake_case_				,	normalizer_state.pop("""type"""))
											__lowerCAmelCase	   =do_lower_case
											__lowerCAmelCase	   =strip_accents
											__lowerCAmelCase	   =tokenize_chinese_chars
											__lowerCAmelCase	   =normalizer_class(**snake_case_)

						__lowerCAmelCase	   =do_lower_case



	def 		UpperCamelCase							(						self							:    Optional[Any]				,	snake_case_							:    Optional[int]				,	snake_case_							:    Union[str, Any]=None)->							List[Any]:
						__lowerCAmelCase	   =[self.cls_token_id] + token_ids_a + [self.sep_token_id]

						if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

						return output



	def 		UpperCamelCase							(						self							:    Union[str, Any]				,	snake_case_							:    List[int]				,	snake_case_							:    Optional[List[int]] = None)->							List[int]:
						__lowerCAmelCase	   =[self.sep_token_id]
						__lowerCAmelCase	   =[self.cls_token_id]
						if token_ids_a is None:
											return len(cls + token_ids_a + sep) * [0]
						return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



	def 		UpperCamelCase							(						self							:    int				,	snake_case_							:    str				,	snake_case_							:    Optional[str] = None)->							Tuple[str]:
						__lowerCAmelCase	   =self._tokenizer.model.save(snake_case_				,	name=snake_case_)
						return tuple(snake_case_)









from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_	       =			logging.get_logger(__name__)

lowercase_	       =			{
    '''google/realm-cc-news-pretrained-embedder''': (
        '''https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json'''
    ),
    '''google/realm-cc-news-pretrained-encoder''': (
        '''https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json'''
    ),
    '''google/realm-cc-news-pretrained-scorer''': (
        '''https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json'''
    ),
    '''google/realm-cc-news-pretrained-openqa''': (
        '''https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json'''
    ),
    '''google/realm-orqa-nq-openqa''': '''https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json''',
    '''google/realm-orqa-nq-reader''': '''https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json''',
    '''google/realm-orqa-wq-openqa''': '''https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json''',
    '''google/realm-orqa-wq-reader''': '''https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json''',
    # See all REALM models at https://huggingface.co/models?filter=realm
}






class 						__a (     SCREAMING_SNAKE_CASE				):
	SCREAMING_SNAKE_CASE       =       "realm"



	def __init__(						self							:    Tuple				,	snake_case_							:    Tuple=3_05_22				,	snake_case_							:    List[Any]=7_68				,	snake_case_							:    Any=1_28				,	snake_case_							:    Union[str, Any]=12				,	snake_case_							:    List[str]=12				,	snake_case_							:    Any=8				,	snake_case_							:    Any=30_72				,	snake_case_							:    int="gelu_new"				,	snake_case_							:    Optional[int]=0.1				,	snake_case_							:    Tuple=0.1				,	snake_case_							:    List[Any]=5_12				,	snake_case_							:    List[Any]=2				,	snake_case_							:    Any=0.0_2				,	snake_case_							:    List[str]=1e-12				,	snake_case_							:    Optional[Any]=2_56				,	snake_case_							:    Dict=10				,	snake_case_							:    str=1e-3				,	snake_case_							:    List[str]=5				,	snake_case_							:    Optional[Any]=3_20				,	snake_case_							:    Optional[Any]=13_35_37_18				,	snake_case_							:    Tuple=50_00				,	snake_case_							:    Union[str, Any]=1				,	snake_case_							:    Union[str, Any]=0				,	snake_case_							:    List[str]=2				,	**snake_case_							:    List[str]				,	)->							str:
						super().__init__(pad_token_id=snake_case_				,	bos_token_id=snake_case_				,	eos_token_id=snake_case_				,	**snake_case_)

						# Common config
						__lowerCAmelCase	   =vocab_size
						__lowerCAmelCase	   =max_position_embeddings
						__lowerCAmelCase	   =hidden_size
						__lowerCAmelCase	   =retriever_proj_size
						__lowerCAmelCase	   =num_hidden_layers
						__lowerCAmelCase	   =num_attention_heads
						__lowerCAmelCase	   =num_candidates
						__lowerCAmelCase	   =intermediate_size
						__lowerCAmelCase	   =hidden_act
						__lowerCAmelCase	   =hidden_dropout_prob
						__lowerCAmelCase	   =attention_probs_dropout_prob
						__lowerCAmelCase	   =initializer_range
						__lowerCAmelCase	   =type_vocab_size
						__lowerCAmelCase	   =layer_norm_eps

						# Reader config
						__lowerCAmelCase	   =span_hidden_size
						__lowerCAmelCase	   =max_span_width
						__lowerCAmelCase	   =reader_layer_norm_eps
						__lowerCAmelCase	   =reader_beam_size
						__lowerCAmelCase	   =reader_seq_len

						# Retrieval config
						__lowerCAmelCase	   =num_block_records
						__lowerCAmelCase	   =searcher_beam_size




