


'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowercase    :    Any			     =  {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase    :    List[Any]			     =  {
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

_lowercase    :    Any			     =  {
    """google/electra-small-generator""": 512,
    """google/electra-base-generator""": 512,
    """google/electra-large-generator""": 512,
    """google/electra-small-discriminator""": 512,
    """google/electra-base-discriminator""": 512,
    """google/electra-large-discriminator""": 512,
}

_lowercase    :    Optional[int]			     =  {
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}






class  UpperCamelCase__(     lowerCAmelCase ):
				__magic_name__       :   List[Any]							= VOCAB_FILES_NAMES
				__magic_name__       :   List[Any]							= PRETRAINED_VOCAB_FILES_MAP
				__magic_name__       :   List[Any]							= PRETRAINED_INIT_CONFIGURATION
				__magic_name__       :   Dict							= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__magic_name__       :   Any							= ElectraTokenizer




				def __init__(	self    :				int   ,					lowerCAmelCase    :				Dict=None   ,					lowerCAmelCase    :				Any=None   ,					lowerCAmelCase    :				Union[str, Any]=True   ,					lowerCAmelCase    :				Union[str, Any]="[UNK]"   ,					lowerCAmelCase    :				Any="[SEP]"   ,					lowerCAmelCase    :				Dict="[PAD]"   ,					lowerCAmelCase    :				Tuple="[CLS]"   ,					lowerCAmelCase    :				str="[MASK]"   ,					lowerCAmelCase    :				Optional[int]=True   ,					lowerCAmelCase    :				Optional[int]=None   ,					**lowerCAmelCase    :				Union[str, Any]   ,					)->  int:

										"""simple docstring"""




										super().__init__(
										    lowerCAmelCase   ,					tokenizer_file=lowerCAmelCase   ,					do_lower_case=lowerCAmelCase   ,					unk_token=lowerCAmelCase   ,					sep_token=lowerCAmelCase   ,					pad_token=lowerCAmelCase   ,					cls_token=lowerCAmelCase   ,					mask_token=lowerCAmelCase   ,					tokenize_chinese_chars=lowerCAmelCase   ,					strip_accents=lowerCAmelCase   ,					**lowerCAmelCase   ,					)

										UpperCAmelCase         =			json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
										if (
										    normalizer_state.get('''lowercase'''   ,					lowerCAmelCase   ) != do_lower_case
										    or normalizer_state.get('''strip_accents'''   ,					lowerCAmelCase   ) != strip_accents
										    or normalizer_state.get('''handle_chinese_chars'''   ,					lowerCAmelCase   ) != tokenize_chinese_chars
										):
																UpperCAmelCase         =			getattr(lowerCAmelCase   ,					normalizer_state.pop('''type'''   )   )
																UpperCAmelCase         =			do_lower_case
																UpperCAmelCase         =			strip_accents
																UpperCAmelCase         =			tokenize_chinese_chars
																UpperCAmelCase         =			normalizer_class(**lowerCAmelCase   )

										UpperCAmelCase         =			do_lower_case




				def   a__(	self    :				List[Any]   ,					lowerCAmelCase    :				Optional[int]   ,					lowerCAmelCase    :				int=None   )->  List[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output




				def   a__(	self    :				Union[str, Any]   ,					lowerCAmelCase    :				List[int]   ,					lowerCAmelCase    :				Optional[List[int]] = None   )->  List[int]:

										"""simple docstring"""




										UpperCAmelCase         =			[self.sep_token_id]
										UpperCAmelCase         =			[self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]





				def   a__(	self    :				Union[str, Any]   ,					lowerCAmelCase    :				str   ,					lowerCAmelCase    :				Optional[str] = None   )->  Tuple[str]:

										"""simple docstring"""




										UpperCAmelCase         =			self._tokenizer.model.save(lowerCAmelCase   ,					name=lowerCAmelCase   )
										return tuple(lowerCAmelCase   )









'''simple docstring'''



from argparse import ArgumentParser

from . import BaseTransformersCLICommand


def    lowerCamelCase__   ( A     :    List[Any]  ):




						'''simple docstring'''






						return DownloadCommand(args.model   ,     args.cache_dir   ,     args.force   ,     args.trust_remote_code  )






class  UpperCamelCase__(     lowerCAmelCase ):




				@staticmethod
				def   a__(	lowerCAmelCase    :				ArgumentParser   )->  Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase         =			parser.add_parser('''download'''   )
										download_parser.add_argument(
										    '''--cache-dir'''   ,					type=lowerCAmelCase   ,					default=lowerCAmelCase   ,					help='''Path to location to store the models'''   )
										download_parser.add_argument(
										    '''--force'''   ,					action='''store_true'''   ,					help='''Force the model to be download even if already in cache-dir'''   )
										download_parser.add_argument(
										    '''--trust-remote-code'''   ,					action='''store_true'''   ,					help='''Whether or not to allow for custom models defined on the Hub in their own modeling files. Use only if you\'ve reviewed the code as it will execute on your local machine'''   ,					)
										download_parser.add_argument('''model'''   ,					type=lowerCAmelCase   ,					help='''Name of the model to download'''   )
										download_parser.set_defaults(func=lowerCAmelCase   )




				def __init__(	self    :				Dict   ,					lowerCAmelCase    :				str   ,					lowerCAmelCase    :				str   ,					lowerCAmelCase    :				bool   ,					lowerCAmelCase    :				bool   )->  Any:

										"""simple docstring"""




										UpperCAmelCase         =			model
										UpperCAmelCase         =			cache
										UpperCAmelCase         =			force
										UpperCAmelCase         =			trust_remote_code





				def   a__(	self    :				int   )->  Optional[Any]:

										"""simple docstring"""




										from ..models.auto import AutoModel, AutoTokenizer

										AutoModel.from_pretrained(
										    self._model   ,					cache_dir=self._cache   ,					force_download=self._force   ,					trust_remote_code=self._trust_remote_code   )
										AutoTokenizer.from_pretrained(
										    self._model   ,					cache_dir=self._cache   ,					force_download=self._force   ,					trust_remote_code=self._trust_remote_code   )





