






import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast


@dataclass
class 					_snake_case						(			datasets.BuilderConfig ):
 _A          =   None




class 					_snake_case						(			datasets.ArrowBasedBuilder ):
 _A          =   PandasConfig






 def    lowerCAmelCase_							(		self       )						->    str:
     return datasets.DatasetInfo(features=self.config.features       )






 def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    List[str]:
     if not self.config.data_files:
         raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'       )
     snake_case__      :Any         =   dl_manager.download_and_extract(self.config.data_files       )
     if isinstance(UpperCamelCase					,(str, list, tuple)       ):
         snake_case__      :List[Any]         =   data_files
         if isinstance(UpperCamelCase					,UpperCamelCase       ):
             snake_case__      :Any         =   [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         snake_case__      :int         =   [dl_manager.iter_files(UpperCamelCase       ) for file in files]
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,gen_kwargs={"files": files}       )]
     snake_case__      :Tuple         =   []
     for split_name, files in data_files.items():
         if isinstance(UpperCamelCase					,UpperCamelCase       ):
             snake_case__      :List[Any]         =   [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         snake_case__      :Optional[Any]         =   [dl_manager.iter_files(UpperCamelCase       ) for file in files]
         splits.append(datasets.SplitGenerator(name=UpperCamelCase					,gen_kwargs={"files": files}       )       )
     return splits






 def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    pa.Table:
     if self.config.features is not None:
         # more expensive cast to support nested features with keys in a different order
         # allows str <-> int/float or str to Audio for example
         snake_case__      :Any         =   table_cast(UpperCamelCase					,self.config.features.arrow_schema       )
     return pa_table






 def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    Optional[Any]:
     for i, file in enumerate(itertools.chain.from_iterable(UpperCamelCase       )       ):
         with open(UpperCamelCase					,"rb"       ) as f:
             snake_case__      :Optional[int]         =   pa.Table.from_pandas(pd.read_pickle(UpperCamelCase       )       )
             yield i, self._cast_table(UpperCamelCase       )







from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


__UpperCAmelCase							:   str							=  logging.get_logger(__name__)

__UpperCAmelCase							:   List[Any]							=  {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase							:   List[Any]							=  {
    "vocab_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"
    },
    "merges_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"
    },
}

__UpperCAmelCase							:   str							=  {"allegro/herbert-base-cased": 5_1_4}
__UpperCAmelCase							:   List[str]							=  {}


class 					_snake_case						(			_A ):
	_A          =   VOCAB_FILES_NAMES
	_A          =   PRETRAINED_VOCAB_FILES_MAP
	_A          =   PRETRAINED_INIT_CONFIGURATION
	_A          =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A          =   HerbertTokenizer






	def __init__(		self					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase="<s>"					,UpperCamelCase="<unk>"					,UpperCamelCase="<pad>"					,UpperCamelCase="<mask>"					,UpperCamelCase="</s>"					,**UpperCamelCase					,)						->    Dict:
					super().__init__(
					    UpperCamelCase					,UpperCamelCase					,tokenizer_file=UpperCamelCase					,cls_token=UpperCamelCase					,unk_token=UpperCamelCase					,pad_token=UpperCamelCase					,mask_token=UpperCamelCase					,sep_token=UpperCamelCase					,**UpperCamelCase					,)






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Optional[int]         =   [self.cls_token_id]
					snake_case__      :Any         =   [self.sep_token_id]
					if token_ids_a is None:
									return cls + token_ids_a + sep

					return cls + token_ids_a + sep + token_ids_a + sep






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None					,UpperCamelCase = False       )						->    List[int]:
					if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=UpperCamelCase					,token_ids_a=UpperCamelCase					,already_has_special_tokens=UpperCamelCase       )

					if token_ids_a is None:
									return [1] + ([0] * len(UpperCamelCase       )) + [1]
					return [1] + ([0] * len(UpperCamelCase       )) + [1] + ([0] * len(UpperCamelCase       )) + [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Any         =   [self.sep_token_id]
					snake_case__      :Union[str, Any]         =   [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    Tuple[str]:
					snake_case__      :List[str]         =   self._tokenizer.model.save(UpperCamelCase					,name=UpperCamelCase       )
					return tuple(UpperCamelCase       )