






import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





def 					lowercase_ (    __snake_case     :							List[Any]	,	__snake_case     :							Optional[int]      )     -> str:





    '''simple docstring'''


    assert isinstance(_lowerCamelCase	,	_lowerCamelCase      )
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize("keep_in_memory"	,	[False, True]      )
def 					lowercase_ (    __snake_case     :							Dict	,	__snake_case     :							Optional[int]	,	__snake_case     :							str      )     -> Tuple:





    '''simple docstring'''


    snake_case__      :Tuple         =   tmp_path / "cache"
    snake_case__      :Any         =   {"col_1": "string", "col_2": "int64", "col_3": "float64"}
    with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
        snake_case__      :List[Any]         =   JsonDatasetReader(_lowerCamelCase	,	cache_dir=_lowerCamelCase	,	keep_in_memory=_lowerCamelCase      ).read()
    _check_json_dataset(_lowerCamelCase	,	_lowerCamelCase      )





@pytest.mark.parametrize(
    "features"	,	[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]	,	)
def 					lowercase_ (    __snake_case     :							Tuple	,	__snake_case     :							Union[str, Any]	,	__snake_case     :							Union[str, Any]      )     -> Any:





    '''simple docstring'''


    snake_case__      :Dict         =   tmp_path / "cache"
    snake_case__      :List[str]         =   {"col_1": "string", "col_2": "int64", "col_3": "float64"}
    snake_case__      :Optional[Any]         =   features.copy() if features else default_expected_features
    snake_case__      :int         =   (
        Features({feature: Value(_lowerCamelCase      ) for feature, dtype in features.items()}      ) if features is not None else None
    )
    snake_case__      :Any         =   JsonDatasetReader(_lowerCamelCase	,	features=_lowerCamelCase	,	cache_dir=_lowerCamelCase      ).read()
    _check_json_dataset(_lowerCamelCase	,	_lowerCamelCase      )





@pytest.mark.parametrize(
    "features"	,	[
        None,
        {"col_3": "float64", "col_1": "string", "col_2": "int64"},
    ]	,	)
def 					lowercase_ (    __snake_case     :							int	,	__snake_case     :							str	,	__snake_case     :							List[Any]      )     -> List[str]:





    '''simple docstring'''


    snake_case__      :Union[str, Any]         =   tmp_path / "cache"
    snake_case__      :Dict         =   {"col_3": "float64", "col_1": "string", "col_2": "int64"}
    snake_case__      :Optional[Any]         =   features.copy() if features else default_expected_features
    snake_case__      :str         =   (
        Features({feature: Value(_lowerCamelCase      ) for feature, dtype in features.items()}      ) if features is not None else None
    )
    snake_case__      :Any         =   JsonDatasetReader(_lowerCamelCase	,	features=_lowerCamelCase	,	cache_dir=_lowerCamelCase      ).read()
    assert isinstance(_lowerCamelCase	,	_lowerCamelCase      )
    assert dataset.num_rows == 2
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_3", "col_1", "col_2"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype





def 					lowercase_ (    __snake_case     :							Optional[Any]	,	__snake_case     :							str      )     -> Any:





    '''simple docstring'''


    snake_case__      :List[str]         =   {"col_2": "int64", "col_3": "float64", "col_1": "string"}
    snake_case__      :int         =   features.copy()
    snake_case__      :Optional[Any]         =   (
        Features({feature: Value(_lowerCamelCase      ) for feature, dtype in features.items()}      ) if features is not None else None
    )
    snake_case__      :Union[str, Any]         =   tmp_path / "cache"
    snake_case__      :Dict         =   JsonDatasetReader(_lowerCamelCase	,	features=_lowerCamelCase	,	cache_dir=_lowerCamelCase      ).read()
    assert isinstance(_lowerCamelCase	,	_lowerCamelCase      )
    assert dataset.num_rows == 2
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_2", "col_3", "col_1"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize("split"	,	[None, NamedSplit("train"      ), "train", "test"]      )
def 					lowercase_ (    __snake_case     :							str	,	__snake_case     :							Optional[int]	,	__snake_case     :							Optional[int]      )     -> Tuple:





    '''simple docstring'''


    snake_case__      :Union[str, Any]         =   tmp_path / "cache"
    snake_case__      :Dict         =   {"col_1": "string", "col_2": "int64", "col_3": "float64"}
    snake_case__      :Optional[Any]         =   JsonDatasetReader(_lowerCamelCase	,	cache_dir=_lowerCamelCase	,	split=_lowerCamelCase      ).read()
    _check_json_dataset(_lowerCamelCase	,	_lowerCamelCase      )
    assert dataset.split == split if split else "train"





@pytest.mark.parametrize("path_type"	,	[str, list]      )
def 					lowercase_ (    __snake_case     :							Any	,	__snake_case     :							Tuple	,	__snake_case     :							List[Any]      )     -> Union[str, Any]:





    '''simple docstring'''


    if issubclass(_lowerCamelCase	,	_lowerCamelCase      ):
        snake_case__      :List[Any]         =   jsonl_path
    elif issubclass(_lowerCamelCase	,	_lowerCamelCase      ):
        snake_case__      :Optional[int]         =   [jsonl_path]
    snake_case__      :Tuple         =   tmp_path / "cache"
    snake_case__      :Tuple         =   {"col_1": "string", "col_2": "int64", "col_3": "float64"}
    snake_case__      :Union[str, Any]         =   JsonDatasetReader(_lowerCamelCase	,	cache_dir=_lowerCamelCase      ).read()
    _check_json_dataset(_lowerCamelCase	,	_lowerCamelCase      )





def 					lowercase_ (    __snake_case     :							Dict	,	__snake_case     :							Tuple	,	__snake_case     :							Union[str, Any]=("train",)      )     -> Any:





    '''simple docstring'''


    assert isinstance(_lowerCamelCase	,	_lowerCamelCase      )
    for split in splits:
        snake_case__      :List[str]         =   dataset_dict[split]
        assert dataset.num_rows == 4
        assert dataset.num_columns == 3
        assert dataset.column_names == ["col_1", "col_2", "col_3"]
        for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize("keep_in_memory"	,	[False, True]      )
def 					lowercase_ (    __snake_case     :							Optional[Any]	,	__snake_case     :							Dict	,	__snake_case     :							int      )     -> Optional[int]:





    '''simple docstring'''


    snake_case__      :List[Any]         =   tmp_path / "cache"
    snake_case__      :Optional[int]         =   {"col_1": "string", "col_2": "int64", "col_3": "float64"}
    with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
        snake_case__      :Optional[int]         =   JsonDatasetReader({"train": jsonl_path}	,	cache_dir=_lowerCamelCase	,	keep_in_memory=_lowerCamelCase      ).read()
    _check_json_datasetdict(_lowerCamelCase	,	_lowerCamelCase      )





@pytest.mark.parametrize(
    "features"	,	[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]	,	)
def 					lowercase_ (    __snake_case     :							str	,	__snake_case     :							Any	,	__snake_case     :							Tuple      )     -> Tuple:





    '''simple docstring'''


    snake_case__      :str         =   tmp_path / "cache"
    snake_case__      :List[Any]         =   {"col_1": "string", "col_2": "int64", "col_3": "float64"}
    snake_case__      :Any         =   features.copy() if features else default_expected_features
    snake_case__      :List[str]         =   (
        Features({feature: Value(_lowerCamelCase      ) for feature, dtype in features.items()}      ) if features is not None else None
    )
    snake_case__      :int         =   JsonDatasetReader({"train": jsonl_path}	,	features=_lowerCamelCase	,	cache_dir=_lowerCamelCase      ).read()
    _check_json_datasetdict(_lowerCamelCase	,	_lowerCamelCase      )





@pytest.mark.parametrize("split"	,	[None, NamedSplit("train"      ), "train", "test"]      )
def 					lowercase_ (    __snake_case     :							str	,	__snake_case     :							Union[str, Any]	,	__snake_case     :							List[Any]      )     -> Optional[int]:





    '''simple docstring'''


    if split:
        snake_case__      :int         =   {split: jsonl_path}
    else:
        snake_case__      :Optional[Any]         =   "train"
        snake_case__      :Dict         =   {"train": jsonl_path, "test": jsonl_path}
    snake_case__      :Union[str, Any]         =   tmp_path / "cache"
    snake_case__      :Optional[int]         =   {"col_1": "string", "col_2": "int64", "col_3": "float64"}
    snake_case__      :int         =   JsonDatasetReader(_lowerCamelCase	,	cache_dir=_lowerCamelCase      ).read()
    _check_json_datasetdict(_lowerCamelCase	,	_lowerCamelCase	,	splits=list(path.keys()      )      )
    assert all(dataset[split].split == split for split in path.keys()      )





def 					lowercase_ (    __snake_case     :							Any      )     -> Union[str, Any]:





    '''simple docstring'''


    return json.load(_lowerCamelCase      )





def 					lowercase_ (    __snake_case     :							Any      )     -> List[str]:





    '''simple docstring'''


    return [json.loads(_lowerCamelCase      ) for line in buffer]


class 					_snake_case						:






 @pytest.mark.parametrize("lines, load_json_function"					,[(True, load_json_lines), (False, load_json)]       )
 def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )						->    Optional[Any]:
     with io.BytesIO() as buffer:
         JsonDatasetWriter(UpperCamelCase					,UpperCamelCase					,lines=UpperCamelCase       ).write()
         buffer.seek(0       )
         snake_case__      :str         =   load_json_function(UpperCamelCase       )
     assert isinstance(UpperCamelCase					,UpperCamelCase       )
     assert isinstance(exported_content[0]					,UpperCamelCase       )
     assert len(UpperCamelCase       ) == 10






 @pytest.mark.parametrize(
     "orient, container, keys, len_at"					,[
         ("records", list, {"tokens", "labels", "answers", "id"}, None),
         ("split", dict, {"columns", "data"}, "data"),
         ("index", dict, set("0123456789"       ), None),
         ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
         ("values", list, None, None),
         ("table", dict, {"schema", "data"}, "data"),
     ]					,)
 def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )						->    str:
     with io.BytesIO() as buffer:
         JsonDatasetWriter(UpperCamelCase					,UpperCamelCase					,lines=UpperCamelCase					,orient=UpperCamelCase       ).write()
         buffer.seek(0       )
         snake_case__      :Union[str, Any]         =   load_json(UpperCamelCase       )
     assert isinstance(UpperCamelCase					,UpperCamelCase       )
     if keys:
         if container is dict:
             assert exported_content.keys() == keys
         else:
             assert exported_content[0].keys() == keys
     else:
         assert not hasattr(UpperCamelCase					,"keys"       ) and not hasattr(exported_content[0]					,"keys"       )
     if len_at:
         assert len(exported_content[len_at]       ) == 10
     else:
         assert len(UpperCamelCase       ) == 10






 @pytest.mark.parametrize("lines, load_json_function"					,[(True, load_json_lines), (False, load_json)]       )
 def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )						->    Optional[Any]:
     with io.BytesIO() as buffer:
         JsonDatasetWriter(UpperCamelCase					,UpperCamelCase					,lines=UpperCamelCase					,num_proc=2       ).write()
         buffer.seek(0       )
         snake_case__      :List[Any]         =   load_json_function(UpperCamelCase       )
     assert isinstance(UpperCamelCase					,UpperCamelCase       )
     assert isinstance(exported_content[0]					,UpperCamelCase       )
     assert len(UpperCamelCase       ) == 10






 @pytest.mark.parametrize(
     "orient, container, keys, len_at"					,[
         ("records", list, {"tokens", "labels", "answers", "id"}, None),
         ("split", dict, {"columns", "data"}, "data"),
         ("index", dict, set("0123456789"       ), None),
         ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
         ("values", list, None, None),
         ("table", dict, {"schema", "data"}, "data"),
     ]					,)
 def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )						->    Any:
     with io.BytesIO() as buffer:
         JsonDatasetWriter(UpperCamelCase					,UpperCamelCase					,lines=UpperCamelCase					,orient=UpperCamelCase					,num_proc=2       ).write()
         buffer.seek(0       )
         snake_case__      :Dict         =   load_json(UpperCamelCase       )
     assert isinstance(UpperCamelCase					,UpperCamelCase       )
     if keys:
         if container is dict:
             assert exported_content.keys() == keys
         else:
             assert exported_content[0].keys() == keys
     else:
         assert not hasattr(UpperCamelCase					,"keys"       ) and not hasattr(exported_content[0]					,"keys"       )
     if len_at:
         assert len(exported_content[len_at]       ) == 10
     else:
         assert len(UpperCamelCase       ) == 10






 def    lowerCAmelCase_							(		self					,UpperCamelCase       )						->    Optional[Any]:
     with pytest.raises(UpperCamelCase       ):
         with io.BytesIO() as buffer:
             JsonDatasetWriter(UpperCamelCase					,UpperCamelCase					,num_proc=0       )






 @pytest.mark.parametrize("compression, extension"					,[("gzip", "gz"), ("bz2", "bz2"), ("xz", "xz")]       )
 def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase					,UpperCamelCase       )						->    Tuple:
     snake_case__      :Dict         =   tmp_path_factory.mktemp("data"       ) / f'test.json.{extension}'
     snake_case__      :List[str]         =   str(shared_datadir / f'test_file.json.{extension}'       )
     JsonDatasetWriter(UpperCamelCase					,UpperCamelCase					,compression=UpperCamelCase       ).write()

     with fsspec.open(UpperCamelCase					,"rb"					,compression="infer"       ) as f:
         snake_case__      :Dict         =   f.read()
     with fsspec.open(UpperCamelCase					,"rb"					,compression="infer"       ) as f:
         snake_case__      :Any         =   f.read()
     assert exported_content == original_content







from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


__UpperCAmelCase							:   str							=  logging.get_logger(__name__)

__UpperCAmelCase							:   List[Any]							=  {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase							:   List[Any]							=  {
    "vocab_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"
    },
    "merges_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"
    },
}

__UpperCAmelCase							:   str							=  {"allegro/herbert-base-cased": 5_1_4}
__UpperCAmelCase							:   List[str]							=  {}


class 					_snake_case						(			_A ):
	_A          =   VOCAB_FILES_NAMES
	_A          =   PRETRAINED_VOCAB_FILES_MAP
	_A          =   PRETRAINED_INIT_CONFIGURATION
	_A          =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A          =   HerbertTokenizer






	def __init__(		self					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase=None					,UpperCamelCase="<s>"					,UpperCamelCase="<unk>"					,UpperCamelCase="<pad>"					,UpperCamelCase="<mask>"					,UpperCamelCase="</s>"					,**UpperCamelCase					,)						->    Dict:
					super().__init__(
					    UpperCamelCase					,UpperCamelCase					,tokenizer_file=UpperCamelCase					,cls_token=UpperCamelCase					,unk_token=UpperCamelCase					,pad_token=UpperCamelCase					,mask_token=UpperCamelCase					,sep_token=UpperCamelCase					,**UpperCamelCase					,)






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Optional[int]         =   [self.cls_token_id]
					snake_case__      :Any         =   [self.sep_token_id]
					if token_ids_a is None:
									return cls + token_ids_a + sep

					return cls + token_ids_a + sep + token_ids_a + sep






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None					,UpperCamelCase = False       )						->    List[int]:
					if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=UpperCamelCase					,token_ids_a=UpperCamelCase					,already_has_special_tokens=UpperCamelCase       )

					if token_ids_a is None:
									return [1] + ([0] * len(UpperCamelCase       )) + [1]
					return [1] + ([0] * len(UpperCamelCase       )) + [1] + ([0] * len(UpperCamelCase       )) + [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    List[int]:
					snake_case__      :Any         =   [self.sep_token_id]
					snake_case__      :Union[str, Any]         =   [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






	def    lowerCAmelCase_							(		self					,UpperCamelCase					,UpperCamelCase = None       )						->    Tuple[str]:
					snake_case__      :List[str]         =   self._tokenizer.model.save(UpperCamelCase					,name=UpperCamelCase       )
					return tuple(UpperCamelCase       )