




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__A						:							Optional[Any]							=  logging.get_logger(__name__)

__A						:							List[str]							=  {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__A						:							int							=  {
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

__A						:							Optional[int]							=  {
    """distilbert-base-uncased""": 5_1_2,
    """distilbert-base-uncased-distilled-squad""": 5_1_2,
    """distilbert-base-cased""": 5_1_2,
    """distilbert-base-cased-distilled-squad""": 5_1_2,
    """distilbert-base-german-cased""": 5_1_2,
    """distilbert-base-multilingual-cased""": 5_1_2,
}


__A						:							int							=  {
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     VOCAB_FILES_NAMES
							a__						  =     PRETRAINED_VOCAB_FILES_MAP
							a__						  =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							a__						  =     PRETRAINED_INIT_CONFIGURATION
							a__						  =     ['''input_ids''', '''attention_mask''']
							a__						  =     DistilBertTokenizer





							def __init__(					self       :	List[str]       ,     a       :	Tuple=None       ,     a       :	Tuple=None       ,     a       :	List[str]=True       ,     a       :	Union[str, Any]="[UNK]"       ,     a       :	Optional[Any]="[SEP]"       ,     a       :	Any="[PAD]"       ,     a       :	List[Any]="[CLS]"       ,     a       :	List[str]="[MASK]"       ,     a       :	Union[str, Any]=True       ,     a       :	Tuple=None       ,     **a       :	List[str]       ,     )			->					Optional[Any]:
										super().__init__(
										    a       ,     tokenizer_file=a       ,     do_lower_case=a       ,     unk_token=a       ,     sep_token=a       ,     pad_token=a       ,     cls_token=a       ,     mask_token=a       ,     tokenize_chinese_chars=a       ,     strip_accents=a       ,     **a       ,     )

										SCREAMING_SNAKE_CASE           =					json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
										if (
										    normalizer_state.get("""lowercase"""       ,     a			) != do_lower_case
										    or normalizer_state.get("""strip_accents"""       ,     a			) != strip_accents
										    or normalizer_state.get("""handle_chinese_chars"""       ,     a			) != tokenize_chinese_chars
										):
													SCREAMING_SNAKE_CASE           =					getattr(a       ,     normalizer_state.pop("""type"""			)			)
													SCREAMING_SNAKE_CASE           =					do_lower_case
													SCREAMING_SNAKE_CASE           =					strip_accents
													SCREAMING_SNAKE_CASE           =					tokenize_chinese_chars
													SCREAMING_SNAKE_CASE           =					normalizer_class(**a			)

										SCREAMING_SNAKE_CASE           =					do_lower_case





							def    _UpperCAmelCase     (					self       :	int       ,     a       :	int       ,     a       :	List[str]=None			)			->					List[Any]:
										SCREAMING_SNAKE_CASE           =					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

										return output





							def    _UpperCAmelCase     (					self       :	int       ,     a       :	List[int]       ,     a       :	Optional[List[int]] = None			)			->					List[int]:
										SCREAMING_SNAKE_CASE           =					[self.sep_token_id]
										SCREAMING_SNAKE_CASE           =					[self.cls_token_id]
										if token_ids_a is None:
													return len(cls + token_ids_a + sep			) * [0]
										return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







							def    _UpperCAmelCase     (					self       :	Any       ,     a       :	str       ,     a       :	Optional[str] = None			)			->					Tuple[str]:
										SCREAMING_SNAKE_CASE           =					self._tokenizer.model.save(a       ,     name=a			)
										return tuple(a			)












import importlib
import shutil
import threading
import warnings
from typing import List

import fsspec
import fsspec.asyn

from . import compression
from .hffilesystem import HfFileSystem


__A						:							Tuple							=  importlib.util.find_spec("""s3fs""") is not None

if _has_safs:
		from .safilesystem import SaFileSystem  # noqa: F401

__A						:							List[compression.BaseCompressedFileFileSystem]							=  [
    compression.BzaFileSystem,
    compression.GzipFileSystem,
    compression.LzaFileSystem,
    compression.XzFileSystem,
    compression.ZstdFileSystem,
]

# Register custom filesystems
for fs_class in COMPRESSION_FILESYSTEMS + [HfFileSystem]:
		if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
				warnings.warn(f'''A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.''')
		fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE   ):
			'''simple docstring'''
			if "://" in dataset_path:
						SCREAMING_SNAKE_CASE           =					dataset_path.split("""://"""   )[1]
			return dataset_path
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE   ):
			'''simple docstring'''
			if fs is not None and fs.protocol != "file":
						return True
			else:
						return False
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
			'''simple docstring'''
			SCREAMING_SNAKE_CASE           =					not is_remote_filesystem(SCREAMING_SNAKE_CASE   )
			if is_local:
						# LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
						shutil.move(fs._strip_protocol(SCREAMING_SNAKE_CASE   )					,	fs._strip_protocol(SCREAMING_SNAKE_CASE   )   )
			else:
						fs.mv(SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	recursive=SCREAMING_SNAKE_CASE   )




def 					lowerCamelCase_    (    ):
			'''simple docstring'''
			if hasattr(fsspec.asyn					,	"""reset_lock"""   ):
						# for future fsspec>2022.05.0
						fsspec.asyn.reset_lock()
			else:
						SCREAMING_SNAKE_CASE           =					None
						SCREAMING_SNAKE_CASE           =					None
						SCREAMING_SNAKE_CASE           =					threading.Lock()






