





import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Tuple					,   UpperCamelCase__:							Optional[Any]    ):
		assert isinstance(UpperCamelCase__					,   UpperCamelCase__    )
		assert dataset.num_rows == 4
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_1", "col_2", "col_3"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype






@pytest.mark.parametrize("""keep_in_memory"""					,   [False, True]    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Union[str, Any]					,   UpperCamelCase__:							Optional[Any]					,   UpperCamelCase__:							Tuple    ):
		SCREAMING_SNAKE_CASE__			=      tmp_path / """cache"""
		SCREAMING_SNAKE_CASE__			=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				SCREAMING_SNAKE_CASE__			=      ParquetDatasetReader(UpperCamelCase__					,   cache_dir=UpperCamelCase__					,   keep_in_memory=UpperCamelCase__    ).read()
		_check_parquet_dataset(UpperCamelCase__					,   UpperCamelCase__    )






@pytest.mark.parametrize(
    """features"""					,   [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,   )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Tuple					,   UpperCamelCase__:							int					,   UpperCamelCase__:							Tuple    ):
		SCREAMING_SNAKE_CASE__			=      tmp_path / """cache"""
		SCREAMING_SNAKE_CASE__			=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		SCREAMING_SNAKE_CASE__			=      features.copy() if features else default_expected_features
		SCREAMING_SNAKE_CASE__			=      (
		    Features({feature: Value(UpperCamelCase__    ) for feature, dtype in features.items()}    ) if features is not None else None
		)
		SCREAMING_SNAKE_CASE__			=      ParquetDatasetReader(UpperCamelCase__					,   features=UpperCamelCase__					,   cache_dir=UpperCamelCase__    ).read()
		_check_parquet_dataset(UpperCamelCase__					,   UpperCamelCase__    )






@pytest.mark.parametrize("""split"""					,   [None, NamedSplit("""train"""    ), """train""", """test"""]    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[str]					,   UpperCamelCase__:							Optional[Any]					,   UpperCamelCase__:							Optional[Any]    ):
		SCREAMING_SNAKE_CASE__			=      tmp_path / """cache"""
		SCREAMING_SNAKE_CASE__			=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		SCREAMING_SNAKE_CASE__			=      ParquetDatasetReader(UpperCamelCase__					,   cache_dir=UpperCamelCase__					,   split=UpperCamelCase__    ).read()
		_check_parquet_dataset(UpperCamelCase__					,   UpperCamelCase__    )
		assert dataset.split == split if split else "train"






@pytest.mark.parametrize("""path_type"""					,   [str, list]    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Any					,   UpperCamelCase__:							Tuple					,   UpperCamelCase__:							Union[str, Any]    ):
		if issubclass(UpperCamelCase__					,   UpperCamelCase__    ):
				SCREAMING_SNAKE_CASE__			=      parquet_path
		elif issubclass(UpperCamelCase__					,   UpperCamelCase__    ):
				SCREAMING_SNAKE_CASE__			=      [parquet_path]
		SCREAMING_SNAKE_CASE__			=      tmp_path / """cache"""
		SCREAMING_SNAKE_CASE__			=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		SCREAMING_SNAKE_CASE__			=      ParquetDatasetReader(UpperCamelCase__					,   cache_dir=UpperCamelCase__    ).read()
		_check_parquet_dataset(UpperCamelCase__					,   UpperCamelCase__    )






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[Any]					,   UpperCamelCase__:							Dict					,   UpperCamelCase__:							Any=("train",)    ):
		assert isinstance(UpperCamelCase__					,   UpperCamelCase__    )
		for split in splits:
				SCREAMING_SNAKE_CASE__			=      dataset_dict[split]
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype






@pytest.mark.parametrize("""keep_in_memory"""					,   [False, True]    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[Any]					,   UpperCamelCase__:							Optional[int]					,   UpperCamelCase__:							Dict    ):
		SCREAMING_SNAKE_CASE__			=      tmp_path / """cache"""
		SCREAMING_SNAKE_CASE__			=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				SCREAMING_SNAKE_CASE__			=      ParquetDatasetReader(
				    {"""train""": parquet_path}					,   cache_dir=UpperCamelCase__					,   keep_in_memory=UpperCamelCase__    ).read()
		_check_parquet_datasetdict(UpperCamelCase__					,   UpperCamelCase__    )






@pytest.mark.parametrize(
    """features"""					,   [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,   )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							int					,   UpperCamelCase__:							List[str]					,   UpperCamelCase__:							Tuple    ):
		SCREAMING_SNAKE_CASE__			=      tmp_path / """cache"""
		SCREAMING_SNAKE_CASE__			=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		SCREAMING_SNAKE_CASE__			=      features.copy() if features else default_expected_features
		SCREAMING_SNAKE_CASE__			=      (
		    Features({feature: Value(UpperCamelCase__    ) for feature, dtype in features.items()}    ) if features is not None else None
		)
		SCREAMING_SNAKE_CASE__			=      ParquetDatasetReader({"""train""": parquet_path}					,   features=UpperCamelCase__					,   cache_dir=UpperCamelCase__    ).read()
		_check_parquet_datasetdict(UpperCamelCase__					,   UpperCamelCase__    )






@pytest.mark.parametrize("""split"""					,   [None, NamedSplit("""train"""    ), """train""", """test"""]    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Any					,   UpperCamelCase__:							Union[str, Any]					,   UpperCamelCase__:							str    ):
		if split:
				SCREAMING_SNAKE_CASE__			=      {split: parquet_path}
		else:
				SCREAMING_SNAKE_CASE__			=      """train"""
				SCREAMING_SNAKE_CASE__			=      {"""train""": parquet_path, """test""": parquet_path}
		SCREAMING_SNAKE_CASE__			=      tmp_path / """cache"""
		SCREAMING_SNAKE_CASE__			=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		SCREAMING_SNAKE_CASE__			=      ParquetDatasetReader(UpperCamelCase__					,   cache_dir=UpperCamelCase__    ).read()
		_check_parquet_datasetdict(UpperCamelCase__					,   UpperCamelCase__					,   splits=list(path.keys()    )    )
		assert all(dataset[split].split == split for split in path.keys()    )






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							str					,   UpperCamelCase__:							Optional[int]    ):
		SCREAMING_SNAKE_CASE__			=      ParquetDatasetWriter(UpperCamelCase__					,   tmp_path / """foo.parquet"""    )
		assert writer.write() > 0
		SCREAMING_SNAKE_CASE__			=      pq.ParquetFile(tmp_path / """foo.parquet"""    )
		SCREAMING_SNAKE_CASE__			=      pf.read()
		assert dataset.data.table == output_table






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							str					,   UpperCamelCase__:							List[Any]    ):
		SCREAMING_SNAKE_CASE__			=      str(shared_datadir / """test_image_rgb.jpg"""    )
		SCREAMING_SNAKE_CASE__			=      {"""image""": [image_path]}
		SCREAMING_SNAKE_CASE__			=      Features({"""image""": Image()}    )
		SCREAMING_SNAKE_CASE__			=      Dataset.from_dict(UpperCamelCase__					,   features=UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      ParquetDatasetWriter(UpperCamelCase__					,   tmp_path / """foo.parquet"""    )
		assert writer.write() > 0

		SCREAMING_SNAKE_CASE__			=      Dataset.from_parquet(str(tmp_path / """foo.parquet"""    )    )
		assert dataset.features == reloaded_dataset.features

		SCREAMING_SNAKE_CASE__			=      ParquetDatasetReader(str(tmp_path / """foo.parquet"""    )					,   streaming=UpperCamelCase__    ).read()
		assert dataset.features == reloaded_iterable_dataset.features






@pytest.mark.parametrize(
    """feature, expected"""					,   [
        (Features({"""foo""": Value("""int32"""    )}    ), None),
        (Features({"""image""": Image(), """foo""": Value("""int32"""    )}    ), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({"""nested""": Sequence(Audio()    )}    ), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]					,   )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[str]					,   UpperCamelCase__:							Any    ):
		assert get_writer_batch_size(UpperCamelCase__    ) == expected






import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					OpenAIGPTTokenizer
							lowerCamelCase_           =					OpenAIGPTTokenizerFast
							lowerCamelCase_           =					True
							lowerCamelCase_           =					False


							def      _snake_case  (							self    :Optional[Any]		)   ->   Dict:



									"""simple docstring"""



									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									SCREAMING_SNAKE_CASE__			=      [
									    """l""",
									    """o""",
									    """w""",
									    """e""",
									    """r""",
									    """s""",
									    """t""",
									    """i""",
									    """d""",
									    """n""",
									    """w</w>""",
									    """r</w>""",
									    """t</w>""",
									    """lo""",
									    """low""",
									    """er</w>""",
									    """low</w>""",
									    """lowest</w>""",
									    """newer</w>""",
									    """wider</w>""",
									    """<unk>""",
									]
									SCREAMING_SNAKE_CASE__			=      dict(zip(__A							,				range(len(__A		)		)		)		)
									SCREAMING_SNAKE_CASE__			=      ["""#version: 0.2""", """l o""", """lo w""", """e r</w>""", """"""]

									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""merges_file"""]		)
									with open(self.vocab_file							,				"""w"""		) as fp:
											fp.write(json.dumps(__A		)		)
									with open(self.merges_file							,				"""w"""		) as fp:
											fp.write("""\n""".join(__A		)		)


							def      _snake_case  (							self    :Union[str, Any]							,				__A    :str		)   ->   List[Any]:



									"""simple docstring"""



									return "lower newer", "lower newer"


							def      _snake_case  (							self    :Optional[Any]		)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      OpenAIGPTTokenizer(self.vocab_file							,				self.merges_file		)

									SCREAMING_SNAKE_CASE__			=      """lower"""
									SCREAMING_SNAKE_CASE__			=      ["""low""", """er</w>"""]
									SCREAMING_SNAKE_CASE__			=      tokenizer.tokenize(__A		)
									self.assertListEqual(__A							,				__A		)

									SCREAMING_SNAKE_CASE__			=      tokens + ["""<unk>"""]
									SCREAMING_SNAKE_CASE__			=      [14, 15, 20]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__A		)							,				__A		)


							def      _snake_case  (							self    :Optional[Any]							,				__A    :Optional[Any]=15		)   ->   Any:



									"""simple docstring"""



									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
													SCREAMING_SNAKE_CASE__			=      self.rust_tokenizer_class.from_pretrained(__A							,				**__A		)

													# Simple input
													SCREAMING_SNAKE_CASE__			=      """This is a simple input"""
													SCREAMING_SNAKE_CASE__			=      ["""This is a simple input 1""", """This is a simple input 2"""]
													SCREAMING_SNAKE_CASE__			=      ("""This is a simple input""", """This is a pair""")
													SCREAMING_SNAKE_CASE__			=      [
													    ("""This is a simple input 1""", """This is a simple input 2"""),
													    ("""This is a simple pair 1""", """This is a simple pair 2"""),
													]

													# Simple input tests
													self.assertRaises(__A							,				tokenizer_r.encode							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Simple input
													self.assertRaises(__A							,				tokenizer_r.encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Simple input
													self.assertRaises(
													    __A							,				tokenizer_r.batch_encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""							,				)

													# Pair input
													self.assertRaises(__A							,				tokenizer_r.encode							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Pair input
													self.assertRaises(__A							,				tokenizer_r.encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""		)

													# Pair input
													self.assertRaises(
													    __A							,				tokenizer_r.batch_encode_plus							,				__A							,				max_length=__A							,				padding="""max_length"""							,				)


							def      _snake_case  (							self    :Dict		)   ->   List[Any]:



									"""simple docstring"""



									pass




@require_ftfy
@require_spacy
@require_tokenizers
class   UpperCamelCase_       (     UpperCamelCase__    ):
							pass