




import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   assert isinstance(UpperCamelCase__					,	UpperCamelCase__   )
   assert dataset.num_rows == 4
   assert dataset.num_columns == 3
   assert dataset.column_names == ["col_1", "col_2", "col_3"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("""keep_in_memory"""					,	[False, True]   )
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   SCREAMING_SNAKE_CASE           =					tmp_path / """cache"""
   SCREAMING_SNAKE_CASE           =					{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      SCREAMING_SNAKE_CASE           =					JsonDatasetReader(UpperCamelCase__					,	cache_dir=UpperCamelCase__					,	keep_in_memory=UpperCamelCase__   ).read()
   _check_json_dataset(UpperCamelCase__					,	UpperCamelCase__   )
@pytest.mark.parametrize(
    """features"""					,	[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,	)
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   SCREAMING_SNAKE_CASE           =					tmp_path / """cache"""
   SCREAMING_SNAKE_CASE           =					{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   SCREAMING_SNAKE_CASE           =					features.copy() if features else default_expected_features
   SCREAMING_SNAKE_CASE           =					(
       Features({feature: Value(UpperCamelCase__   ) for feature, dtype in features.items()}   ) if features is not None else None
   )
   SCREAMING_SNAKE_CASE           =					JsonDatasetReader(UpperCamelCase__					,	features=UpperCamelCase__					,	cache_dir=UpperCamelCase__   ).read()
   _check_json_dataset(UpperCamelCase__					,	UpperCamelCase__   )
@pytest.mark.parametrize(
    """features"""					,	[
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]					,	)
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   SCREAMING_SNAKE_CASE           =					tmp_path / """cache"""
   SCREAMING_SNAKE_CASE           =					{"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
   SCREAMING_SNAKE_CASE           =					features.copy() if features else default_expected_features
   SCREAMING_SNAKE_CASE           =					(
       Features({feature: Value(UpperCamelCase__   ) for feature, dtype in features.items()}   ) if features is not None else None
   )
   SCREAMING_SNAKE_CASE           =					JsonDatasetReader(UpperCamelCase__					,	features=UpperCamelCase__					,	cache_dir=UpperCamelCase__   ).read()
   assert isinstance(UpperCamelCase__					,	UpperCamelCase__   )
   assert dataset.num_rows == 2
   assert dataset.num_columns == 3
   assert dataset.column_names == ["col_3", "col_1", "col_2"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   # jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
   SCREAMING_SNAKE_CASE           =					{"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
   SCREAMING_SNAKE_CASE           =					features.copy()
   SCREAMING_SNAKE_CASE           =					(
       Features({feature: Value(UpperCamelCase__   ) for feature, dtype in features.items()}   ) if features is not None else None
   )
   SCREAMING_SNAKE_CASE           =					tmp_path / """cache"""
   SCREAMING_SNAKE_CASE           =					JsonDatasetReader(UpperCamelCase__					,	features=UpperCamelCase__					,	cache_dir=UpperCamelCase__   ).read()
   assert isinstance(UpperCamelCase__					,	UpperCamelCase__   )
   assert dataset.num_rows == 2
   assert dataset.num_columns == 3
   assert dataset.column_names == ["col_2", "col_3", "col_1"]
   for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("""split"""					,	[None, NamedSplit("""train"""   ), """train""", """test"""]   )
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   SCREAMING_SNAKE_CASE           =					tmp_path / """cache"""
   SCREAMING_SNAKE_CASE           =					{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   SCREAMING_SNAKE_CASE           =					JsonDatasetReader(UpperCamelCase__					,	cache_dir=UpperCamelCase__					,	split=UpperCamelCase__   ).read()
   _check_json_dataset(UpperCamelCase__					,	UpperCamelCase__   )
   assert dataset.split == split if split else "train"
@pytest.mark.parametrize("""path_type"""					,	[str, list]   )
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   if issubclass(UpperCamelCase__					,	UpperCamelCase__   ):
      SCREAMING_SNAKE_CASE           =					jsonl_path
   elif issubclass(UpperCamelCase__					,	UpperCamelCase__   ):
      SCREAMING_SNAKE_CASE           =					[jsonl_path]
   SCREAMING_SNAKE_CASE           =					tmp_path / """cache"""
   SCREAMING_SNAKE_CASE           =					{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   SCREAMING_SNAKE_CASE           =					JsonDatasetReader(UpperCamelCase__					,	cache_dir=UpperCamelCase__   ).read()
   _check_json_dataset(UpperCamelCase__					,	UpperCamelCase__   )
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE=("train",)   ):
   '''simple docstring'''
   assert isinstance(UpperCamelCase__					,	UpperCamelCase__   )
   for split in splits:
      SCREAMING_SNAKE_CASE           =					dataset_dict[split]
      assert dataset.num_rows == 4
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_1", "col_2", "col_3"]
      for feature, expected_dtype in expected_features.items():
         assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("""keep_in_memory"""					,	[False, True]   )
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   SCREAMING_SNAKE_CASE           =					tmp_path / """cache"""
   SCREAMING_SNAKE_CASE           =					{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
      SCREAMING_SNAKE_CASE           =					JsonDatasetReader({"""train""": jsonl_path}					,	cache_dir=UpperCamelCase__					,	keep_in_memory=UpperCamelCase__   ).read()
   _check_json_datasetdict(UpperCamelCase__					,	UpperCamelCase__   )
@pytest.mark.parametrize(
    """features"""					,	[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,	)
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   SCREAMING_SNAKE_CASE           =					tmp_path / """cache"""
   SCREAMING_SNAKE_CASE           =					{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   SCREAMING_SNAKE_CASE           =					features.copy() if features else default_expected_features
   SCREAMING_SNAKE_CASE           =					(
       Features({feature: Value(UpperCamelCase__   ) for feature, dtype in features.items()}   ) if features is not None else None
   )
   SCREAMING_SNAKE_CASE           =					JsonDatasetReader({"""train""": jsonl_path}					,	features=UpperCamelCase__					,	cache_dir=UpperCamelCase__   ).read()
   _check_json_datasetdict(UpperCamelCase__					,	UpperCamelCase__   )
@pytest.mark.parametrize("""split"""					,	[None, NamedSplit("""train"""   ), """train""", """test"""]   )
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE					,	SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   if split:
      SCREAMING_SNAKE_CASE           =					{split: jsonl_path}
   else:
      SCREAMING_SNAKE_CASE           =					"""train"""
      SCREAMING_SNAKE_CASE           =					{"""train""": jsonl_path, """test""": jsonl_path}
   SCREAMING_SNAKE_CASE           =					tmp_path / """cache"""
   SCREAMING_SNAKE_CASE           =					{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
   SCREAMING_SNAKE_CASE           =					JsonDatasetReader(UpperCamelCase__					,	cache_dir=UpperCamelCase__   ).read()
   _check_json_datasetdict(UpperCamelCase__					,	UpperCamelCase__					,	splits=list(path.keys()   )   )
   assert all(dataset[split].split == split for split in path.keys()   )
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   return json.load(UpperCamelCase__   )
def 					lowerCamelCase_    (    SCREAMING_SNAKE_CASE   ):
   '''simple docstring'''
   return [json.loads(UpperCamelCase__   ) for line in buffer]




class 			UpperCAmelCase_      :




       '''simple docstring'''



       @pytest.mark.parametrize("""lines, load_json_function"""       ,     [(True, load_json_lines), (False, load_json)]			)
       def    _UpperCAmelCase     (					self       :	List[str]       ,     a       :	str       ,     a       :	List[str]       ,     a       :	str			)			->					Dict:
          with io.BytesIO() as buffer:
             JsonDatasetWriter(__a       ,     __a       ,     lines=__a			).write()
             buffer.seek(0			)
             SCREAMING_SNAKE_CASE           =					load_json_function(__a			)
          assert isinstance(__a       ,     __a			)
          assert isinstance(exported_content[0]       ,     __a			)
          assert len(__a			) == 10





       @pytest.mark.parametrize(
           """orient, container, keys, len_at"""       ,     [
               ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
               ("""split""", dict, {"""columns""", """data"""}, """data"""),
               ("""index""", dict, set("""0123456789"""			), None),
               ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
               ("""values""", list, None, None),
               ("""table""", dict, {"""schema""", """data"""}, """data"""),
           ]       ,     )
       def    _UpperCAmelCase     (					self       :	Optional[int]       ,     a       :	Tuple       ,     a       :	Any       ,     a       :	List[str]       ,     a       :	Tuple       ,     a       :	Dict			)			->					int:
          with io.BytesIO() as buffer:
             JsonDatasetWriter(__a       ,     __a       ,     lines=__a       ,     orient=__a			).write()
             buffer.seek(0			)
             SCREAMING_SNAKE_CASE           =					load_json(__a			)
          assert isinstance(__a       ,     __a			)
          if keys:
             if container is dict:
                assert exported_content.keys() == keys
             else:
                assert exported_content[0].keys() == keys
          else:
             assert not hasattr(__a       ,     """keys"""			) and not hasattr(exported_content[0]       ,     """keys"""			)
          if len_at:
             assert len(exported_content[len_at]			) == 10
          else:
             assert len(__a			) == 10





       @pytest.mark.parametrize("""lines, load_json_function"""       ,     [(True, load_json_lines), (False, load_json)]			)
       def    _UpperCAmelCase     (					self       :	Union[str, Any]       ,     a       :	List[Any]       ,     a       :	Any       ,     a       :	Dict			)			->					int:
          with io.BytesIO() as buffer:
             JsonDatasetWriter(__a       ,     __a       ,     lines=__a       ,     num_proc=2			).write()
             buffer.seek(0			)
             SCREAMING_SNAKE_CASE           =					load_json_function(__a			)
          assert isinstance(__a       ,     __a			)
          assert isinstance(exported_content[0]       ,     __a			)
          assert len(__a			) == 10





       @pytest.mark.parametrize(
           """orient, container, keys, len_at"""       ,     [
               ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
               ("""split""", dict, {"""columns""", """data"""}, """data"""),
               ("""index""", dict, set("""0123456789"""			), None),
               ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
               ("""values""", list, None, None),
               ("""table""", dict, {"""schema""", """data"""}, """data"""),
           ]       ,     )
       def    _UpperCAmelCase     (					self       :	str       ,     a       :	Any       ,     a       :	int       ,     a       :	Tuple       ,     a       :	List[Any]       ,     a       :	Union[str, Any]			)			->					Dict:
          with io.BytesIO() as buffer:
             JsonDatasetWriter(__a       ,     __a       ,     lines=__a       ,     orient=__a       ,     num_proc=2			).write()
             buffer.seek(0			)
             SCREAMING_SNAKE_CASE           =					load_json(__a			)
          assert isinstance(__a       ,     __a			)
          if keys:
             if container is dict:
                assert exported_content.keys() == keys
             else:
                assert exported_content[0].keys() == keys
          else:
             assert not hasattr(__a       ,     """keys"""			) and not hasattr(exported_content[0]       ,     """keys"""			)
          if len_at:
             assert len(exported_content[len_at]			) == 10
          else:
             assert len(__a			) == 10





       def    _UpperCAmelCase     (					self       :	Optional[int]       ,     a       :	List[Any]			)			->					List[str]:
          with pytest.raises(__a			):
             with io.BytesIO() as buffer:
                JsonDatasetWriter(__a       ,     __a       ,     num_proc=0			)







       @pytest.mark.parametrize("""compression, extension"""       ,     [("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]			)
       def    _UpperCAmelCase     (					self       :	Tuple       ,     a       :	Tuple       ,     a       :	Union[str, Any]       ,     a       :	Union[str, Any]       ,     a       :	Any       ,     a       :	Optional[int]			)			->					Optional[int]:
          SCREAMING_SNAKE_CASE           =					tmp_path_factory.mktemp("""data"""			) / f"""test.json.{extension}"""
          SCREAMING_SNAKE_CASE           =					str(shared_datadir / f"""test_file.json.{extension}"""			)
          JsonDatasetWriter(__a       ,     __a       ,     compression=__a			).write()

          with fsspec.open(__a       ,     """rb"""       ,     compression="""infer"""			) as f:
             SCREAMING_SNAKE_CASE           =					f.read()
          with fsspec.open(__a       ,     """rb"""       ,     compression="""infer"""			) as f:
             SCREAMING_SNAKE_CASE           =					f.read()
          assert exported_content == original_content












import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class 			UpperCAmelCase_      (   A       ):




							'''simple docstring'''



							a__						  =     ['''image_processor''', '''tokenizer''']
							a__						  =     '''ViTImageProcessor'''
							a__						  =     ('''CLIPTokenizer''', '''CLIPTokenizerFast''')





							def __init__(					self       :	List[str]       ,     a       :	Dict=None       ,     a       :	List[str]=None       ,     **a       :	List[str]			)			->					Optional[Any]:
										SCREAMING_SNAKE_CASE           =					None
										if "feature_extractor" in kwargs:
													warnings.warn(
													    """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
													    """ instead."""       ,     a       ,     )
													SCREAMING_SNAKE_CASE           =					kwargs.pop("""feature_extractor"""			)

										SCREAMING_SNAKE_CASE           =					image_processor if image_processor is not None else feature_extractor
										if image_processor is None:
													raise ValueError("""You need to specify an `image_processor`."""			)
										if tokenizer is None:
													raise ValueError("""You need to specify a `tokenizer`."""			)

										super().__init__(a       ,     a			)





							def __call__(					self       :	List[Any]       ,     a       :	int=None       ,     a       :	Dict=None       ,     a       :	int=None       ,     a       :	Tuple=None       ,     **a       :	Any			)			->					Any:
										if text is None and visual_prompt is None and images is None:
													raise ValueError("""You have to specify either text, visual prompt or images."""			)

										if text is not None and visual_prompt is not None:
													raise ValueError("""You have to specify exactly one type of prompt. Either text or visual prompt."""			)

										if text is not None:
													SCREAMING_SNAKE_CASE           =					self.tokenizer(a       ,     return_tensors=a       ,     **a			)

										if visual_prompt is not None:
													SCREAMING_SNAKE_CASE           =					self.image_processor(a       ,     return_tensors=a       ,     **a			)

										if images is not None:
													SCREAMING_SNAKE_CASE           =					self.image_processor(a       ,     return_tensors=a       ,     **a			)

										if visual_prompt is not None and images is not None:
													SCREAMING_SNAKE_CASE           =					{
													    """pixel_values""": image_features.pixel_values,
													    """conditional_pixel_values""": prompt_features.pixel_values,
													}
													return encoding
										elif text is not None and images is not None:
													SCREAMING_SNAKE_CASE           =					image_features.pixel_values
													return encoding
										elif text is not None:
													return encoding
										elif visual_prompt is not None:
													SCREAMING_SNAKE_CASE           =					{
													    """conditional_pixel_values""": prompt_features.pixel_values,
													}
													return encoding
										else:
													return BatchEncoding(data=dict(**a			)       ,     tensor_type=a			)





							def    _UpperCAmelCase     (					self       :	Dict       ,     *a       :	Dict       ,     **a       :	Union[str, Any]			)			->					Any:
										return self.tokenizer.batch_decode(*a       ,     **a			)





							def    _UpperCAmelCase     (					self       :	Tuple       ,     *a       :	List[Any]       ,     **a       :	Union[str, Any]			)			->					Dict:
										return self.tokenizer.decode(*a       ,     **a			)





							@property
							def    _UpperCAmelCase     (					self       :	Any			)			->					Optional[Any]:
										warnings.warn(
										    """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""       ,     a       ,     )
										return self.image_processor_class







							@property
							def    _UpperCAmelCase     (					self       :	Optional[int]			)			->					str:
										warnings.warn(
										    """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""       ,     a       ,     )
										return self.image_processor






