






import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE


class 					_snake_case						(			_A ):
	_A          =   ''
	_A          =   (
	    None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
	)
	_A          =   None  # compression type in fsspec. ex: "gzip"
	_A          =   None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz






	def __init__(		self					,UpperCamelCase = ""					,UpperCamelCase = None					,UpperCamelCase = None					,**UpperCamelCase       )						->    Union[str, Any]:
					super().__init__(self					,**UpperCamelCase       )
					# always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
					snake_case__      :str         =   fsspec.open(
					    UpperCamelCase					,mode="rb"					,protocol=UpperCamelCase					,compression=self.compression					,client_kwargs={
					        "requote_redirect_url": False,  # see https://github.com/huggingface/datasets/pull/5459
					        "trust_env": True,  # Enable reading proxy env variables.
					        **(target_options or {}).pop("client_kwargs"					,{}       ),  # To avoid issues if it was already passed.
					    }					,**(target_options or {})					,)
					snake_case__      :Any         =   os.path.basename(self.file.path.split("::"       )[0]       )
					snake_case__      :List[Any]         =   (
					    self.compressed_name[: self.compressed_name.rindex("."       )]
					    if "." in self.compressed_name
					    else self.compressed_name
					)
					snake_case__      :Optional[int]         =   None






	@classmethod
	def    lowerCAmelCase_							(		cls					,UpperCamelCase       )						->    Union[str, Any]:
					# compressed file paths are always relative to the archive root
					return super()._strip_protocol(UpperCamelCase       ).lstrip("/"       )






	def    lowerCAmelCase_							(		self       )						->    List[Any]:
					if self.dir_cache is None:
									snake_case__      :Any         =   {**self.file.fs.info(self.file.path       ), "name": self.uncompressed_name}
									snake_case__      :List[str]         =   {f["name"]: f}






	def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    Optional[int]:
					return self.file.open().read()






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = "rb"					,UpperCamelCase=None					,UpperCamelCase=True					,UpperCamelCase=None					,**UpperCamelCase					,)						->    Union[str, Any]:
					snake_case__      :List[str]         =   self._strip_protocol(UpperCamelCase       )
					if mode != "rb":
									raise ValueError(f'Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\''       )
					return self.file.open()




class 					_snake_case						(			_A ):
	_A          =   'bz2'
	_A          =   'bz2'
	_A          =   '.bz2'




class 					_snake_case						(			_A ):
	_A          =   'gzip'
	_A          =   'gzip'
	_A          =   '.gz'




class 					_snake_case						(			_A ):
	_A          =   'lz4'
	_A          =   'lz4'
	_A          =   '.lz4'




class 					_snake_case						(			_A ):
	_A          =   'xz'
	_A          =   'xz'
	_A          =   '.xz'




class 					_snake_case						(			_A ):
	_A          =   'zstd'
	_A          =   'zstd'
	_A          =   '.zst'






	def __init__(		self					,UpperCamelCase					,UpperCamelCase = "rb"					,UpperCamelCase = None					,UpperCamelCase = None					,UpperCamelCase = DEFAULT_BLOCK_SIZE					,**UpperCamelCase					,)						->    Union[str, Any]:
					super().__init__(
					    fo=UpperCamelCase					,mode=UpperCamelCase					,target_protocol=UpperCamelCase					,target_options=UpperCamelCase					,block_size=UpperCamelCase					,**UpperCamelCase					,)
					# We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
					#
					# File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
					#     out.close = close
					# AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
					#
					# see https://github.com/intake/filesystem_spec/issues/725
					snake_case__      :Optional[Any]         =   self.file.__enter__


					class 					_snake_case						:






						def __init__(		self					,UpperCamelCase       )						->    Optional[int]:
										snake_case__      :Dict         =   file_






						def __enter__(		self       )						->    str:
										self._file.__enter__()
										return self






						def __exit__(		self					,*UpperCamelCase					,**UpperCamelCase       )						->    Dict:
										self._file.__exit__(*UpperCamelCase					,**UpperCamelCase       )






						def __iter__(		self       )						->    List[str]:
										return iter(self._file       )






						def    lowerCAmelCase_							(		self       )						->    Union[str, Any]:
										return next(self._file       )






						def __getattr__(		self					,UpperCamelCase       )						->    Union[str, Any]:
										return getattr(self._file					,UpperCamelCase       )

					def fixed_enter(*UpperCamelCase					,**UpperCamelCase       ):
									return WrappedFile(_enter(*UpperCamelCase					,**UpperCamelCase       )       )

					snake_case__      :List[str]         =   fixed_enter







from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


__UpperCAmelCase							:   str							=  logging.get_logger(__name__)

__UpperCAmelCase							:   List[Any]							=  {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase							:   List[Any]							=  {
    "vocab_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"
    },
    "merges_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"
    },
}

__UpperCAmelCase							:   str							=  {"allegro/herbert-base-cased": 5_1_4}
__UpperCAmelCase							:   List[str]							=  {}


class 					_snake_case						(			_A ):
	_A          =   VOCAB_FILES_NAMES
	_A          =   PRETRAINED_VOCAB_FILES_MAP
	_A          =   PRETRAINED_INIT_CONFIGURATION
	_A          =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A          =   HerbertTokenizer






	def __init__(		self					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase="<s>"					,UpperCamelCase="<unk>"					,UpperCamelCase="<pad>"					,UpperCamelCase="<mask>"					,UpperCamelCase="</s>"					,**UpperCamelCase					,)						->    Dict:
					super().__init__(
					    UpperCamelCase					,UpperCamelCase					,tokenizer_file=UpperCamelCase					,cls_token=UpperCamelCase					,unk_token=UpperCamelCase					,pad_token=UpperCamelCase					,mask_token=UpperCamelCase					,sep_token=UpperCamelCase					,**UpperCamelCase					,)






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Optional[int]         =   [self.cls_token_id]
					snake_case__      :Any         =   [self.sep_token_id]
					if token_ids_a is None:
									return cls + token_ids_a + sep

					return cls + token_ids_a + sep + token_ids_a + sep






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None					,UpperCamelCase = False       )						->    List[int]:
					if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=UpperCamelCase					,token_ids_a=UpperCamelCase					,already_has_special_tokens=UpperCamelCase       )

					if token_ids_a is None:
									return [1] + ([0] * len(UpperCamelCase       )) + [1]
					return [1] + ([0] * len(UpperCamelCase       )) + [1] + ([0] * len(UpperCamelCase       )) + [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Any         =   [self.sep_token_id]
					snake_case__      :Union[str, Any]         =   [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    Tuple[str]:
					snake_case__      :List[str]         =   self._tokenizer.model.save(UpperCamelCase					,name=UpperCamelCase       )
					return tuple(UpperCamelCase       )