





"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


lowerCAmelCase_			=							{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase_			=							{
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase_			=							{
    """google/electra-small-generator""": 512,
    """google/electra-base-generator""": 512,
    """google/electra-large-generator""": 512,
    """google/electra-small-discriminator""": 512,
    """google/electra-base-discriminator""": 512,
    """google/electra-large-discriminator""": 512,
}

lowerCAmelCase_			=							{
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}



class 	lowerCAmelCase					(      __UpperCAmelCase							):
							lowerCAmelCase__						 =  VOCAB_FILES_NAMES
							lowerCAmelCase__						 =  PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase__						 =  PRETRAINED_INIT_CONFIGURATION
							lowerCAmelCase__						 =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase__						 =  ElectraTokenizer






							def __init__(   self    ,  a__=None    ,  a__=None    ,  a__=True    ,  a__="[UNK]"    ,  a__="[SEP]"    ,  a__="[PAD]"    ,  a__="[CLS]"    ,  a__="[MASK]"    ,  a__=True    ,  a__=None    ,  **a__    ,  ):
														super().__init__(
														    a__    ,  tokenizer_file=a__    ,  do_lower_case=a__    ,  unk_token=a__    ,  sep_token=a__    ,  pad_token=a__    ,  cls_token=a__    ,  mask_token=a__    ,  tokenize_chinese_chars=a__    ,  strip_accents=a__    ,  **a__    ,  )

														_UpperCAmelCase     =     json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
														if (
														    normalizer_state.get('lowercase'    ,  a__     ) != do_lower_case
														    or normalizer_state.get('strip_accents'    ,  a__     ) != strip_accents
														    or normalizer_state.get('handle_chinese_chars'    ,  a__     ) != tokenize_chinese_chars
														):
																					_UpperCAmelCase     =     getattr(a__    ,  normalizer_state.pop('type'     )     )
																					_UpperCAmelCase     =     do_lower_case
																					_UpperCAmelCase     =     strip_accents
																					_UpperCAmelCase     =     tokenize_chinese_chars
																					_UpperCAmelCase     =     normalizer_class(**a__     )

														_UpperCAmelCase     =     do_lower_case






							def 							__A (   self    ,  a__    ,  a__=None     ):
														_UpperCAmelCase     =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output






							def 							__A (   self    ,  a__    ,  a__ = None     ):
														_UpperCAmelCase     =     [self.sep_token_id]
														_UpperCAmelCase     =     [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep     ) * [0]
														return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]






							def 							__A (   self    ,  a__    ,  a__ = None     ):
														_UpperCAmelCase     =     self._tokenizer.model.save(a__    ,  name=a__     )
														return tuple(a__     )













"""simple docstring"""






def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       int:






       """simple docstring"""




       return number | (1 << position)

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       int:






       """simple docstring"""




       return number & ~(1 << position)

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       int:






       """simple docstring"""




       return number ^ (1 << position)

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       bool:






       """simple docstring"""




       return ((number >> position) & 1) == 1

def       __lowerCamelCase    (       SCREAMING_SNAKE_CASE,SCREAMING_SNAKE_CASE							)		->       int:






       """simple docstring"""




       return int((number & (1 << position)) != 0							)


if __name__ == "__main__":
    import doctest

    doctest.testmod()






