
"""simple docstring"""





import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE



class __lowerCAmelCase				(					_lowercase      ):




   """simple docstring"""







   snake_case	      =				""
   snake_case	      =				(
       None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
   )
   snake_case	      =				None  # compression type in fsspec. ex: "gzip"
   snake_case	      =				None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz




   def __init__(				self						:					List[str]       ,   _snake_case						:					str = ""       ,   _snake_case						:					Optional[str] = None       ,   _snake_case						:					Optional[dict] = None       ,   **_snake_case						:					Union[str, Any]	)      -> int:







        """simple docstring"""
        super().__init__(self       ,   **_snake_case	)
        # always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
        A_				 =						fsspec.open(
            _snake_case       ,   mode="rb"       ,   protocol=_snake_case       ,   compression=self.compression       ,   client_kwargs={
                "requote_redirect_url": False,  # see https://github.com/huggingface/datasets/pull/5459
                "trust_env": True,  # Enable reading proxy env variables.
                **(target_options or {}).pop("client_kwargs"       ,   {}	),  # To avoid issues if it was already passed.
            }       ,   **(target_options or {})       ,   )
        A_				 =						os.path.basename(self.file.path.split("::"	)[0]	)
        A_				 =						(
            self.compressed_name[: self.compressed_name.rindex("."	)]
            if "." in self.compressed_name
            else self.compressed_name
        )
        A_				 =						None




   @classmethod
   def      lowerCamelCase__     (				cls						:					str       ,   _snake_case						:					int	)      -> Tuple:







        """simple docstring"""
        return super()._strip_protocol(_snake_case	).lstrip("/"	)




   def      lowerCamelCase__     (				self						:					int	)      -> List[Any]:







        """simple docstring"""
        if self.dir_cache is None:
             A_				 =						{**self.file.fs.info(self.file.path	), "name": self.uncompressed_name}
             A_				 =						{f["name"]: f}




   def      lowerCamelCase__     (				self						:					int       ,   _snake_case						:					str	)      -> str:







        """simple docstring"""
        return self.file.open().read()






   def      lowerCamelCase__     (				self						:					int       ,   _snake_case						:					str       ,   _snake_case						:					str = "rb"       ,   _snake_case						:					Any=None       ,   _snake_case						:					Any=True       ,   _snake_case						:					Optional[int]=None       ,   **_snake_case						:					List[Any]       ,   )      -> Any:







        """simple docstring"""
        A_				 =						self._strip_protocol(_snake_case	)
        if mode != "rb":
             raise ValueError(F'Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\''	)
        return self.file.open()





class __lowerCAmelCase				(					_lowercase      ):




   """simple docstring"""







   snake_case	      =				"bz2"
   snake_case	      =				"bz2"
   snake_case	      =				".bz2"





class __lowerCAmelCase				(					_lowercase      ):




   """simple docstring"""







   snake_case	      =				"gzip"
   snake_case	      =				"gzip"
   snake_case	      =				".gz"





class __lowerCAmelCase				(					_lowercase      ):




   """simple docstring"""







   snake_case	      =				"lz4"
   snake_case	      =				"lz4"
   snake_case	      =				".lz4"





class __lowerCAmelCase				(					_lowercase      ):




   """simple docstring"""







   snake_case	      =				"xz"
   snake_case	      =				"xz"
   snake_case	      =				".xz"





class __lowerCAmelCase				(					_lowercase      ):




   """simple docstring"""







   snake_case	      =				"zstd"
   snake_case	      =				"zstd"
   snake_case	      =				".zst"




   def __init__(				self						:					Tuple       ,   _snake_case						:					str       ,   _snake_case						:					str = "rb"       ,   _snake_case						:					Optional[str] = None       ,   _snake_case						:					Optional[dict] = None       ,   _snake_case						:					int = DEFAULT_BLOCK_SIZE       ,   **_snake_case						:					Any       ,   )      -> Optional[int]:







        """simple docstring"""
        super().__init__(
            fo=_snake_case       ,   mode=_snake_case       ,   target_protocol=_snake_case       ,   target_options=_snake_case       ,   block_size=_snake_case       ,   **_snake_case       ,   )
        # We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
        #
        # File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
        #     out.close = close
        # AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
        #
        # see https://github.com/intake/filesystem_spec/issues/725
        A_				 =						self.file.__enter__



        class __lowerCAmelCase				:




           """simple docstring"""







           def __init__(				self						:					int       ,   _snake_case						:					Any	)      -> List[str]:







                """simple docstring"""
                A_				 =						file_




           def __enter__(				self						:					Tuple	)      -> str:







                """simple docstring"""
                self._file.__enter__()
                return self




           def __exit__(				self						:					List[Any]       ,   *_snake_case						:					Optional[Any]       ,   **_snake_case						:					int	)      -> Tuple:







                """simple docstring"""
                self._file.__exit__(*_snake_case       ,   **_snake_case	)




           def __iter__(				self						:					str	)      -> Union[str, Any]:







                """simple docstring"""
                return iter(self._file	)




           def      lowerCamelCase__     (				self						:					Union[str, Any]	)      -> Any:







                """simple docstring"""
                return next(self._file	)






           def __getattr__(				self						:					Any       ,   _snake_case						:					Any	)      -> List[str]:







                """simple docstring"""
                return getattr(self._file       ,   _snake_case	)

        def fixed_enter(*_snake_case						:					int       ,   **_snake_case						:					Dict	):
             return WrappedFile(_enter(*_snake_case       ,   **_snake_case	)	)

        A_				 =						fixed_enter






"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


UpperCamelCase_					:						Any				= logging.get_logger(__name__)

UpperCamelCase_					:						Any				= {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase_					:						List[str]				= {
    '''vocab_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase_					:						Union[str, Any]				= {
    '''yjernite/retribert-base-uncased''': 512,
}


UpperCamelCase_					:						str				= {
    '''yjernite/retribert-base-uncased''': {'''do_lower_case''': True},
}



class __lowerCAmelCase				(					_lowercase      ):




			"""simple docstring"""







			snake_case	      =				VOCAB_FILES_NAMES
			snake_case	      =				PRETRAINED_VOCAB_FILES_MAP
			snake_case	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			snake_case	      =				PRETRAINED_INIT_CONFIGURATION
			snake_case	      =				RetriBertTokenizer
			snake_case	      =				["input_ids", "attention_mask"]




			def __init__(				self						:					Tuple       ,   _snake_case						:					int=None       ,   _snake_case						:					Any=None       ,   _snake_case						:					str=True       ,   _snake_case						:					List[Any]="[UNK]"       ,   _snake_case						:					Optional[Any]="[SEP]"       ,   _snake_case						:					str="[PAD]"       ,   _snake_case						:					List[str]="[CLS]"       ,   _snake_case						:					List[str]="[MASK]"       ,   _snake_case						:					Dict=True       ,   _snake_case						:					Optional[int]=None       ,   **_snake_case						:					Tuple       ,   )      -> Union[str, Any]:







								"""simple docstring"""
								super().__init__(
								    _snake_case       ,   tokenizer_file=_snake_case       ,   do_lower_case=_snake_case       ,   unk_token=_snake_case       ,   sep_token=_snake_case       ,   pad_token=_snake_case       ,   cls_token=_snake_case       ,   mask_token=_snake_case       ,   tokenize_chinese_chars=_snake_case       ,   strip_accents=_snake_case       ,   **_snake_case       ,   )

								A_				 =						json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
								if (
								    normalizer_state.get("lowercase"       ,   _snake_case	) != do_lower_case
								    or normalizer_state.get("strip_accents"       ,   _snake_case	) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"       ,   _snake_case	) != tokenize_chinese_chars
								):
													A_				 =						getattr(_snake_case       ,   normalizer_state.pop("type"	)	)
													A_				 =						do_lower_case
													A_				 =						strip_accents
													A_				 =						tokenize_chinese_chars
													A_				 =						normalizer_class(**_snake_case	)

								A_				 =						do_lower_case




			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					List[Any]       ,   _snake_case						:					Union[str, Any]=None	)      -> List[Any]:







								"""simple docstring"""
								A_				 =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

								return output




			def      lowerCamelCase__     (				self						:					Any       ,   _snake_case						:					List[int]       ,   _snake_case						:					Optional[List[int]] = None	)      -> List[int]:







								"""simple docstring"""
								A_				 =						[self.sep_token_id]
								A_				 =						[self.cls_token_id]
								if token_ids_a is None:
													return len(cls + token_ids_a + sep	) * [0]
								return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]






			def      lowerCamelCase__     (				self						:					Union[str, Any]       ,   _snake_case						:					str       ,   _snake_case						:					Optional[str] = None	)      -> Tuple[str]:







								"""simple docstring"""
								A_				 =						self._tokenizer.model.save(_snake_case       ,   name=_snake_case	)
								return tuple(_snake_case	)




