


import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil






class     UpperCamelCase(  _a   ):


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Dict						)			->	Tuple:




											'''simple docstring'''

											__snake_case				=		pa.array(TypedSequence([1, 2, 3]						)						)
											self.assertEqual(arr.type     ,    pa.intaa()						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[int]						)			->	Tuple:




											'''simple docstring'''

											with self.assertRaises(SCREAMING_SNAKE_CASE						):
																	__snake_case				=		pa.array(TypedSequence([1, 2, 3]						)     ,    type=pa.intaa()						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		str						)			->	int:




											'''simple docstring'''

											with self.assertRaises(SCREAMING_SNAKE_CASE						):
																	__snake_case				=		pa.array(TypedSequence([1, 2, 3]     ,    try_type=Value("bool"						)     ,    type=Value("int64"						)						)						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]						)			->	Any:




											'''simple docstring'''

											__snake_case				=		pa.array(TypedSequence([1, 2, 3]     ,    type=Value("int32"						)						)						)
											self.assertEqual(arr.type     ,    pa.intaa()						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		str						)			->	Optional[int]:




											'''simple docstring'''

											with self.assertRaises((TypeError, pa.lib.ArrowInvalid)						):
																	__snake_case				=		pa.array(TypedSequence(["foo", "bar"]     ,    type=Value("int64"						)						)						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		int						)			->	Tuple:




											'''simple docstring'''

											__snake_case				=		pa.array(TypedSequence([1, 2, 3]     ,    try_type=Value("int32"						)						)						)
											self.assertEqual(arr.type     ,    pa.intaa()						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]						)			->	List[Any]:




											'''simple docstring'''

											__snake_case				=		pa.array(TypedSequence(["foo", "bar"]     ,    try_type=Value("int64"						)						)						)
											self.assertEqual(arr.type     ,    pa.string()						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		str						)			->	int:




											'''simple docstring'''

											__snake_case				=		pa.array(TypedSequence([[[1, 2, 3]]]     ,    type=ArrayaD((1, 3)     ,    "int64"						)						)						)
											self.assertEqual(arr.type     ,    ArrayaDExtensionType((1, 3)     ,    "int64"						)						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[Any]						)			->	Optional[Any]:




											'''simple docstring'''

											with self.assertRaises((TypeError, pa.lib.ArrowInvalid)						):
																	__snake_case				=		pa.array(TypedSequence(["foo", "bar"]     ,    type=ArrayaD((1, 3)     ,    "int64"						)						)						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Tuple						)			->	int:




											'''simple docstring'''

											__snake_case				=		pa.array(TypedSequence([[[1, 2, 3]]]     ,    try_type=ArrayaD((1, 3)     ,    "int64"						)						)						)
											self.assertEqual(arr.type     ,    ArrayaDExtensionType((1, 3)     ,    "int64"						)						)


					def 							SCREAMING_SNAKE_CASE_  (						self	:		Optional[int]						)			->	Any:




											'''simple docstring'''

											__snake_case				=		pa.array(TypedSequence(["foo", "bar"]     ,    try_type=ArrayaD((1, 3)     ,    "int64"						)						)						)
											self.assertEqual(arr.type     ,    pa.string()						)






					@require_pil
					def 							SCREAMING_SNAKE_CASE_  (						self	:		List[Any]						)			->	Optional[int]:




											'''simple docstring'''

											import PIL.Image

											__snake_case				=		PIL.Image.fromarray(np.arange(1_0     ,    dtype=np.uinta						).reshape(2     ,    5						)						)
											with patch(
											    "datasets.arrow_writer.cast_to_python_objects"     ,    side_effect=SCREAMING_SNAKE_CASE						) as mock_cast_to_python_objects:
																	__snake_case				=		pa.array(TypedSequence([{"path": None, "bytes": b"image_bytes"}, pil_image]     ,    type=Image()						)						)
																	__snake_case		,					__snake_case				=		mock_cast_to_python_objects.call_args_list[-1]
																	self.assertIn("optimize_list_casting"     ,    SCREAMING_SNAKE_CASE						)
																	self.assertFalse(kwargs["optimize_list_casting"]						)





def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase )						->	str:


						'''simple docstring'''






						__snake_case				=		pa.BufferReader(_lowerCAmelCase ) if isinstance(_lowerCAmelCase      ,	pa.Buffer ) else pa.memory_map(_lowerCAmelCase )
						__snake_case				=		pa.ipc.open_stream(_lowerCAmelCase )
						__snake_case				=		f.read_all()
						assert len(pa_table.to_batches() ) == expected_num_chunks
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
						del pa_table





@pytest.mark.parametrize("writer_batch_size"      ,	[None, 1, 10] )
@pytest.mark.parametrize(
    "fields"      ,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}] )
def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase )						->	List[Any]:


						'''simple docstring'''






						__snake_case				=		pa.BufferOutputStream()
						__snake_case				=		pa.schema(_lowerCAmelCase ) if fields else None
						with ArrowWriter(stream=_lowerCAmelCase      ,	schema=_lowerCAmelCase      ,	writer_batch_size=_lowerCAmelCase ) as writer:
												writer.write({"col_1": "foo", "col_2": 1} )
												writer.write({"col_1": "bar", "col_2": 2} )
												__snake_case		,					__snake_case				=		writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												__snake_case				=		{"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_lowerCAmelCase      ,	metadata=writer._schema.metadata )
						_check_output(output.getvalue()      ,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )





def 	_lowerCAmelCase       (    )						->	List[Any]:


						'''simple docstring'''






						__snake_case				=		pa.BufferOutputStream()
						__snake_case				=		Features({"labels": ClassLabel(names=["neg", "pos"] )} )
						with ArrowWriter(stream=_lowerCAmelCase      ,	features=_lowerCAmelCase ) as writer:
												writer.write({"labels": 0} )
												writer.write({"labels": 1} )
												__snake_case		,					__snake_case				=		writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert writer._schema == features.arrow_schema
						assert writer._schema.metadata == features.arrow_schema.metadata
						__snake_case				=		pa.BufferReader(output.getvalue() )
						__snake_case				=		pa.ipc.open_stream(_lowerCAmelCase )
						__snake_case				=		f.read_all()
						__snake_case				=		pa_table.schema
						assert pa_table.num_rows == 2
						assert schema == features.arrow_schema
						assert schema.metadata == features.arrow_schema.metadata
						assert features == Features.from_arrow_schema(_lowerCAmelCase )





@pytest.mark.parametrize("writer_batch_size"      ,	[None, 1, 10] )
def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	Dict:


						'''simple docstring'''






						__snake_case				=		pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_lowerCAmelCase      ,	writer_batch_size=_lowerCAmelCase      ,	hash_salt="split_name"      ,	check_duplicates=_lowerCAmelCase      ,	) as writer:
												with pytest.raises(_lowerCAmelCase ):
																		writer.write({"col_1": "foo", "col_2": 1}      ,	key=[1, 2] )
																		__snake_case		,					__snake_case				=		writer.finalize()





@pytest.mark.parametrize("writer_batch_size"      ,	[None, 2, 10] )
def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	Optional[int]:


						'''simple docstring'''






						__snake_case				=		pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_lowerCAmelCase      ,	writer_batch_size=_lowerCAmelCase      ,	hash_salt="split_name"      ,	check_duplicates=_lowerCAmelCase      ,	) as writer:
												with pytest.raises(_lowerCAmelCase ):
																		writer.write({"col_1": "foo", "col_2": 1}      ,	key=10 )
																		writer.write({"col_1": "bar", "col_2": 2}      ,	key=10 )
																		__snake_case		,					__snake_case				=		writer.finalize()





@pytest.mark.parametrize("writer_batch_size"      ,	[None, 2, 10] )
def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	List[str]:


						'''simple docstring'''






						__snake_case				=		pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_lowerCAmelCase      ,	writer_batch_size=_lowerCAmelCase      ,	hash_salt="split_name"      ,	check_duplicates=_lowerCAmelCase      ,	) as writer:
												writer.write({"col_1": "foo", "col_2": 1}      ,	key=1 )
												writer.write({"col_1": "bar", "col_2": 2}      ,	key=2 )
												__snake_case		,					__snake_case				=		writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						_check_output(output.getvalue()      ,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )





@pytest.mark.parametrize("writer_batch_size"      ,	[None, 1, 10] )
@pytest.mark.parametrize(
    "fields"      ,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}] )
def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase )						->	Tuple:


						'''simple docstring'''






						__snake_case				=		pa.BufferOutputStream()
						__snake_case				=		pa.schema(_lowerCAmelCase ) if fields else None
						with ArrowWriter(stream=_lowerCAmelCase      ,	schema=_lowerCAmelCase      ,	writer_batch_size=_lowerCAmelCase ) as writer:
												writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]} )
												writer.write_batch({"col_1": [], "col_2": []} )
												__snake_case		,					__snake_case				=		writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												__snake_case				=		{"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_lowerCAmelCase      ,	metadata=writer._schema.metadata )
						_check_output(output.getvalue()      ,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )





@pytest.mark.parametrize("writer_batch_size"      ,	[None, 1, 10] )
@pytest.mark.parametrize(
    "fields"      ,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}] )
def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase )						->	Tuple:


						'''simple docstring'''






						__snake_case				=		pa.BufferOutputStream()
						__snake_case				=		pa.schema(_lowerCAmelCase ) if fields else None
						with ArrowWriter(stream=_lowerCAmelCase      ,	schema=_lowerCAmelCase      ,	writer_batch_size=_lowerCAmelCase ) as writer:
												writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]} ) )
												__snake_case		,					__snake_case				=		writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												__snake_case				=		{"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_lowerCAmelCase      ,	metadata=writer._schema.metadata )
						_check_output(output.getvalue()      ,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )





@pytest.mark.parametrize("writer_batch_size"      ,	[None, 1, 10] )
@pytest.mark.parametrize(
    "fields"      ,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}] )
def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase )						->	Optional[int]:


						'''simple docstring'''






						__snake_case				=		pa.BufferOutputStream()
						__snake_case				=		pa.schema(_lowerCAmelCase ) if fields else None
						with ArrowWriter(stream=_lowerCAmelCase      ,	schema=_lowerCAmelCase      ,	writer_batch_size=_lowerCAmelCase ) as writer:
												writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]} ) )
												writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]} ) )
												__snake_case		,					__snake_case				=		writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												__snake_case				=		{"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(_lowerCAmelCase      ,	metadata=writer._schema.metadata )
						_check_output(output.getvalue()      ,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1 )





def 	_lowerCAmelCase       (    )						->	str:


						'''simple docstring'''






						with tempfile.TemporaryDirectory() as tmp_dir:
												__snake_case				=		{"col_1": pa.string(), "col_2": pa.intaa()}
												__snake_case				=		os.path.join(_lowerCAmelCase      ,	"test.arrow" )
												with ArrowWriter(path=_lowerCAmelCase      ,	schema=pa.schema(_lowerCAmelCase ) ) as writer:
																		writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]} )
																		__snake_case		,					__snake_case				=		writer.finalize()
												assert num_examples == 2
												assert num_bytes > 0
												assert writer._schema == pa.schema(_lowerCAmelCase      ,	metadata=writer._schema.metadata )
												_check_output(_lowerCAmelCase      ,	1 )





def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	Tuple:


						'''simple docstring'''






						if pa.types.is_list(_lowerCAmelCase ):
												return get_base_dtype(arr_type.value_type )
						else:
												return arr_type





def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase )						->	int:


						'''simple docstring'''






						if isinstance(lst[0]      ,	_lowerCAmelCase ):
												change_first_primitive_element_in_list(lst[0]      ,	_lowerCAmelCase )
						else:
												__snake_case				=		value





@pytest.mark.parametrize("optimized_int_type, expected_dtype"      ,	[(None, pa.intaa()), (Value("int32" ), pa.intaa())] )
@pytest.mark.parametrize("sequence"      ,	[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]] )
def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase      ,	_lowerCAmelCase )						->	Union[str, Any]:


						'''simple docstring'''






						__snake_case				=		pa.array(TypedSequence(_lowerCAmelCase      ,	optimized_int_type=_lowerCAmelCase ) )
						assert get_base_dtype(arr.type ) == expected_dtype





@pytest.mark.parametrize(
    "col, expected_dtype"      ,	[
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]      ,	)
@pytest.mark.parametrize("sequence"      ,	[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]] )
def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase      ,	_lowerCAmelCase )						->	int:


						'''simple docstring'''






						__snake_case				=		pa.array(OptimizedTypedSequence(_lowerCAmelCase      ,	col=_lowerCAmelCase ) )
						assert get_base_dtype(arr.type ) == expected_dtype

						# not in range
						if col != "other":
												# avoids errors due to in-place modifications
												__snake_case				=		copy.deepcopy(_lowerCAmelCase )
												__snake_case				=		np.iinfo(expected_dtype.to_pandas_dtype() ).max + 1
												change_first_primitive_element_in_list(_lowerCAmelCase      ,	_lowerCAmelCase )
												__snake_case				=		pa.array(OptimizedTypedSequence(_lowerCAmelCase      ,	col=_lowerCAmelCase ) )
												assert get_base_dtype(arr.type ) == pa.intaa()





@pytest.mark.parametrize("raise_exception"      ,	[False, True] )
def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase )						->	List[str]:


						'''simple docstring'''






						__snake_case				=		str(tmp_path / "dataset-train.arrow" )
						try:
												with ArrowWriter(path=_lowerCAmelCase ) as writer:
																		if raise_exception:
																								raise pa.lib.ArrowInvalid()
																		else:
																								writer.stream.close()
						except pa.lib.ArrowInvalid:
												pass
						finally:
												assert writer.stream.closed





def 	_lowerCAmelCase       (    _lowerCAmelCase )						->	Union[str, Any]:


						'''simple docstring'''






						__snake_case				=		"mock://dataset-train.arrow"
						with ArrowWriter(path=_lowerCAmelCase      ,	storage_options=mockfs.storage_options ) as writer:
												assert isinstance(writer._fs      ,	type(_lowerCAmelCase ) )
												assert writer._fs.storage_options == mockfs.storage_options
												writer.write({"col_1": "foo", "col_2": 1} )
												writer.write({"col_1": "bar", "col_2": 2} )
												__snake_case		,					__snake_case				=		writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert mockfs.exists(_lowerCAmelCase )





def 	_lowerCAmelCase       (    )						->	List[Any]:


						'''simple docstring'''






						__snake_case				=		pa.BufferOutputStream()
						with ParquetWriter(stream=_lowerCAmelCase ) as writer:
												writer.write({"col_1": "foo", "col_2": 1} )
												writer.write({"col_1": "bar", "col_2": 2} )
												__snake_case		,					__snake_case				=		writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						__snake_case				=		pa.BufferReader(output.getvalue() )
						__snake_case				=		pq.read_table(_lowerCAmelCase )
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}





@require_pil
@pytest.mark.parametrize("embed_local_files"      ,	[False, True] )
def 	_lowerCAmelCase       (    _lowerCAmelCase      ,	_lowerCAmelCase )						->	List[str]:


						'''simple docstring'''






						import PIL.Image

						__snake_case				=		str(tmp_path / "test_image_rgb.jpg" )
						PIL.Image.fromarray(np.zeros((5, 5)      ,	dtype=np.uinta ) ).save(_lowerCAmelCase      ,	format="png" )
						__snake_case				=		pa.BufferOutputStream()
						with ParquetWriter(
						    stream=_lowerCAmelCase      ,	features=Features({"image": Image()} )      ,	embed_local_files=_lowerCAmelCase ) as writer:
												writer.write({"image": image_path} )
												writer.finalize()
						__snake_case				=		pa.BufferReader(output.getvalue() )
						__snake_case				=		pq.read_table(_lowerCAmelCase )
						__snake_case				=		pa_table.to_pydict()
						if embed_local_files:
												assert isinstance(out["image"][0]["path"]      ,	_lowerCAmelCase )
												with open(_lowerCAmelCase      ,	"rb" ) as f:
																		assert out["image"][0]["bytes"] == f.read()
						else:
												assert out["image"][0]["path"] == image_path
												assert out["image"][0]["bytes"] is None





def 	_lowerCAmelCase       (    )						->	Optional[Any]:


						'''simple docstring'''






						__snake_case				=		pa.schema([pa.field("col_1"      ,	pa.string()      ,	nullable=_lowerCAmelCase )] )
						__snake_case				=		pa.BufferOutputStream()
						with ArrowWriter(stream=_lowerCAmelCase ) as writer:
												writer._build_writer(inferred_schema=_lowerCAmelCase )
												assert writer._schema == pa.schema([pa.field("col_1"      ,	pa.string() )] )









import warnings

from ...utils import logging
from .image_processing_beit import BeitImageProcessor


A							:					Any     						=     logging.get_logger(__name__)






class     UpperCamelCase(  _a   ):


					def __init__(						self	:		Optional[int]     ,    *SCREAMING_SNAKE_CASE	:		Tuple     ,    **SCREAMING_SNAKE_CASE	:		List[Any]						)			->	None:




											'''simple docstring'''

											warnings.warn(
											    "The class BeitFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"
											    " use BeitImageProcessor instead."     ,    SCREAMING_SNAKE_CASE     ,    )
											super().__init__(*SCREAMING_SNAKE_CASE     ,    **SCREAMING_SNAKE_CASE						)





