



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


__lowerCamelCase  =			logging.get_logger(__name__)

__lowerCamelCase  =			{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__lowerCamelCase  =			{
    """vocab_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

__lowerCamelCase  =			{
    """yjernite/retribert-base-uncased""": 512,
}


__lowerCamelCase  =			{
    """yjernite/retribert-base-uncased""": {"""do_lower_case""": True},
}


class   snake_case_ (__UpperCAmelCase  ):




       """simple docstring"""






       _lowerCamelCase			    =				VOCAB_FILES_NAMES
       _lowerCamelCase			    =				PRETRAINED_VOCAB_FILES_MAP
       _lowerCamelCase			    =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       _lowerCamelCase			    =				PRETRAINED_INIT_CONFIGURATION
       _lowerCamelCase			    =				RetriBertTokenizer
       _lowerCamelCase			    =				["""input_ids""", """attention_mask"""]



       def __init__(     self     ,lowercase=None     ,lowercase=None     ,lowercase=True     ,lowercase="[UNK]"     ,lowercase="[SEP]"     ,lowercase="[PAD]"     ,lowercase="[CLS]"     ,lowercase="[MASK]"     ,lowercase=True     ,lowercase=None     ,**lowercase     ,):

             """simple docstring"""





             super().__init__(
                 lowercase     ,tokenizer_file=lowercase     ,do_lower_case=lowercase     ,unk_token=lowercase     ,sep_token=lowercase     ,pad_token=lowercase     ,cls_token=lowercase     ,mask_token=lowercase     ,tokenize_chinese_chars=lowercase     ,strip_accents=lowercase     ,**lowercase     ,)

             UpperCAmelCase_  :	Optional[Any]				      =		json.loads(self.backend_tokenizer.normalizer.__getstate__())
             if (
                 normalizer_state.get("lowercase"     ,lowercase) != do_lower_case
                 or normalizer_state.get("strip_accents"     ,lowercase) != strip_accents
                 or normalizer_state.get("handle_chinese_chars"     ,lowercase) != tokenize_chinese_chars
             ):
                   UpperCAmelCase_  :	str				      =		getattr(lowercase     ,normalizer_state.pop("type"))
                   UpperCAmelCase_  :	Tuple				      =		do_lower_case
                   UpperCAmelCase_  :	int				      =		strip_accents
                   UpperCAmelCase_  :	Union[str, Any]				      =		tokenize_chinese_chars
                   UpperCAmelCase_  :	Dict				      =		normalizer_class(**lowercase)

             UpperCAmelCase_  :	Any				      =		do_lower_case



       def  A_   (     self     ,lowercase     ,lowercase=None):

             """simple docstring"""





             UpperCAmelCase_  :	Optional[int]				      =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                   output += token_ids_a + [self.sep_token_id]

             return output



       def  A_   (     self     ,lowercase     ,lowercase = None):

             """simple docstring"""





             UpperCAmelCase_  :	int				      =		[self.sep_token_id]
             UpperCAmelCase_  :	Tuple				      =		[self.cls_token_id]
             if token_ids_a is None:
                   return len(cls + token_ids_a + sep) * [0]
             return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



       def  A_   (     self     ,lowercase     ,lowercase = None):

             """simple docstring"""





             UpperCAmelCase_  :	Union[str, Any]				      =		self._tokenizer.model.save(lowercase     ,name=lowercase)
             return tuple(lowercase)





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase         =			logging.get_logger(__name__)

__lowerCamelCase         =			{
    '''microsoft/trocr-base-handwritten''': (
        '''https://huggingface.co/microsoft/trocr-base-handwritten/resolve/main/config.json'''
    ),
    # See all TrOCR models at https://huggingface.co/models?filter=trocr
}


class   snake_case_ (lowercase__  ):




       """simple docstring"""






       _lowerCamelCase			    =				"""trocr"""
       _lowerCamelCase			    =				["""past_key_values"""]
       _lowerCamelCase			    =				{
           """num_attention_heads""": """decoder_attention_heads""",
           """hidden_size""": """d_model""",
           """num_hidden_layers""": """decoder_layers""",
       }



       def __init__(     self     ,lowercase=50265     ,lowercase=1024     ,lowercase=12     ,lowercase=16     ,lowercase=4096     ,lowercase="gelu"     ,lowercase=512     ,lowercase=0.1     ,lowercase=0.0     ,lowercase=0.0     ,lowercase=2     ,lowercase=0.02     ,lowercase=0.0     ,lowercase=True     ,lowercase=False     ,lowercase=True     ,lowercase=True     ,lowercase=1     ,lowercase=0     ,lowercase=2     ,**lowercase     ,):

             """simple docstring"""





             UpperCAmelCase_  :	int				      =		vocab_size
             UpperCAmelCase_  :	Tuple				      =		d_model
             UpperCAmelCase_  :	List[str]				      =		decoder_layers
             UpperCAmelCase_  :	Union[str, Any]				      =		decoder_attention_heads
             UpperCAmelCase_  :	Optional[Any]				      =		decoder_ffn_dim
             UpperCAmelCase_  :	Tuple				      =		activation_function
             UpperCAmelCase_  :	List[Any]				      =		max_position_embeddings
             UpperCAmelCase_  :	List[Any]				      =		dropout
             UpperCAmelCase_  :	List[str]				      =		attention_dropout
             UpperCAmelCase_  :	Union[str, Any]				      =		activation_dropout
             UpperCAmelCase_  :	Optional[int]				      =		init_std
             UpperCAmelCase_  :	Dict				      =		decoder_layerdrop
             UpperCAmelCase_  :	List[str]				      =		use_cache
             UpperCAmelCase_  :	Dict				      =		scale_embedding
             UpperCAmelCase_  :	Dict				      =		use_learned_position_embeddings
             UpperCAmelCase_  :	Optional[Any]				      =		layernorm_embedding

             super().__init__(
                 pad_token_id=lowercase     ,bos_token_id=lowercase     ,eos_token_id=lowercase     ,decoder_start_token_id=lowercase     ,**lowercase     ,)
