





import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader







class   _UpperCamelCase  (       A_  ):







					'''simple docstring'''

					def __init__(						self    :  Optional[int] ,					__lowercase    :  NestedDataStructureLike[PathLike] ,					__lowercase    :  Optional[NamedSplit] = None ,					__lowercase    :  Optional[Features] = None ,					__lowercase    :  str = None ,					__lowercase    :  bool = False ,					__lowercase    :  bool = False ,					__lowercase    :  Optional[str] = None ,					__lowercase    :  Optional[int] = None ,					**__lowercase    :  Tuple ,					):






						'''simple docstring'''







						super().__init__(
						    __lowercase ,					split=__lowercase ,					features=__lowercase ,					cache_dir=__lowercase ,					keep_in_memory=__lowercase ,					streaming=__lowercase ,					num_proc=__lowercase ,					**__lowercase ,					)
						UpperCAmelCase_								=     field
						UpperCAmelCase_								=     path_or_paths if isinstance(__lowercase ,					__lowercase		) else {self.split: path_or_paths}
						UpperCAmelCase_								=     Json(
						    cache_dir=__lowercase ,					data_files=__lowercase ,					features=__lowercase ,					field=__lowercase ,					**__lowercase ,					)





					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[int]		):






						'''simple docstring'''







						if self.streaming:
							UpperCAmelCase_								=     self.builder.as_streaming_dataset(split=self.split		)
						# Build regular (map-style) dataset
						else:
							UpperCAmelCase_								=     None
							UpperCAmelCase_								=     None
							UpperCAmelCase_								=     None
							UpperCAmelCase_								=     None

							self.builder.download_and_prepare(
							    download_config=__lowercase ,					download_mode=__lowercase ,					verification_mode=__lowercase ,					base_path=__lowercase ,					num_proc=self.num_proc ,					)
							UpperCAmelCase_								=     self.builder.as_dataset(
							    split=self.split ,					verification_mode=__lowercase ,					in_memory=self.keep_in_memory		)
						return dataset







class   _UpperCamelCase  :







					'''simple docstring'''

					def __init__(						self    :  Tuple ,					__lowercase    :  Dataset ,					__lowercase    :  Union[PathLike, BinaryIO] ,					__lowercase    :  Optional[int] = None ,					__lowercase    :  Optional[int] = None ,					**__lowercase    :  Any ,					):






						'''simple docstring'''







						if num_proc is not None and num_proc <= 0:
							raise ValueError(F"""num_proc {num_proc} must be an integer > 0."""		)

						UpperCAmelCase_								=     dataset
						UpperCAmelCase_								=     path_or_buf
						UpperCAmelCase_								=     batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
						UpperCAmelCase_								=     num_proc
						UpperCAmelCase_								=     """utf-8"""
						UpperCAmelCase_								=     to_json_kwargs





					def        SCREAMING_SNAKE_CASE							(						self    :  Tuple		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.to_json_kwargs.pop("""path_or_buf""" ,					__lowercase		)
						UpperCAmelCase_								=     self.to_json_kwargs.pop("""orient""" ,					"""records"""		)
						UpperCAmelCase_								=     self.to_json_kwargs.pop("""lines""" ,					True if orient == """records""" else False		)
						UpperCAmelCase_								=     self.to_json_kwargs.pop("""index""" ,					False if orient in ["""split""", """table"""] else True		)
						UpperCAmelCase_								=     self.to_json_kwargs.pop("""compression""" ,					__lowercase		)

						if compression not in [None, "infer", "gzip", "bz2", "xz"]:
							raise NotImplementedError(F"""`datasets` currently does not support {compression} compression"""		)

						if isinstance(self.path_or_buf ,					(str, bytes, os.PathLike)		):
							with fsspec.open(self.path_or_buf ,					"""wb""" ,					compression=__lowercase		) as buffer:
								UpperCAmelCase_								=     self._write(file_obj=__lowercase ,					orient=__lowercase ,					lines=__lowercase ,					index=__lowercase ,					**self.to_json_kwargs		)
						else:
							if compression:
								raise NotImplementedError(
								    F"""The compression parameter is not supported when writing to a buffer, but compression={compression}"""
								    """ was passed. Please provide a local path instead."""		)
							UpperCAmelCase_								=     self._write(
							    file_obj=self.path_or_buf ,					orient=__lowercase ,					lines=__lowercase ,					index=__lowercase ,					**self.to_json_kwargs		)
						return written





					def        SCREAMING_SNAKE_CASE							(						self    :  List[str] ,					__lowercase    :  int		):






						'''simple docstring'''







						UpperCAmelCase_						,	UpperCAmelCase_						,	UpperCAmelCase_						,	UpperCAmelCase_						,	UpperCAmelCase_								=     args

						UpperCAmelCase_								=     query_table(
						    table=self.dataset.data ,					key=slice(__lowercase ,					offset + self.batch_size		) ,					indices=self.dataset._indices ,					)
						UpperCAmelCase_								=     batch.to_pandas().to_json(
						    path_or_buf=__lowercase ,					orient=__lowercase ,					lines=__lowercase ,					index=__lowercase ,					**__lowercase		)
						if not json_str.endswith("""\n"""		):
							json_str += "\n"
						return json_str.encode(self.encoding		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[Any] ,					__lowercase    :  BinaryIO ,					__lowercase    :  Tuple ,					__lowercase    :  Optional[Any] ,					__lowercase    :  Optional[Any] ,					**__lowercase    :  Union[str, Any] ,					):






						'''simple docstring'''







						UpperCAmelCase_								=     0

						if self.num_proc is None or self.num_proc == 1:
							for offset in logging.tqdm(
							    range(0 ,					len(self.dataset		) ,					self.batch_size		) ,					unit="""ba""" ,					disable=not logging.is_progress_bar_enabled() ,					desc="""Creating json from Arrow format""" ,					):
								UpperCAmelCase_								=     self._batch_json((offset, orient, lines, index, to_json_kwargs)		)
								written += file_obj.write(__lowercase		)
						else:
							UpperCAmelCase_						,	UpperCAmelCase_								=     len(self.dataset		), self.batch_size
							with multiprocessing.Pool(self.num_proc		) as pool:
								for json_str in logging.tqdm(
								    pool.imap(
								        self._batch_json ,					[(offset, orient, lines, index, to_json_kwargs) for offset in range(0 ,					__lowercase ,					__lowercase		)] ,					) ,					total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size ,					unit="""ba""" ,					disable=not logging.is_progress_bar_enabled() ,					desc="""Creating json from Arrow format""" ,					):
									written += file_obj.write(__lowercase		)

						return written









import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_vision_available


if is_vision_available():
				from PIL import Image

				from transformers import ChineseCLIPImageProcessor, ChineseCLIPProcessor







@require_vision
class   _UpperCamelCase  (       unittest.TestCase  ):







					'''simple docstring'''

					def        SCREAMING_SNAKE_CASE							(						self    :  List[Any]		):






						'''simple docstring'''







						UpperCAmelCase_								=     tempfile.mkdtemp()

						UpperCAmelCase_								=     [
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """[PAD]""",
						    """[MASK]""",
						    """的""",
						    """价""",
						    """格""",
						    """是""",
						    """15""",
						    """便""",
						    """alex""",
						    """##andra""",
						    """，""",
						    """。""",
						    """-""",
						    """t""",
						    """shirt""",
						]
						UpperCAmelCase_								=     os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES["""vocab_file"""]		)
						with open(self.vocab_file ,					"""w""" ,					encoding="""utf-8"""		) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)

						UpperCAmelCase_								=     {
						    """do_resize""": True,
						    """size""": {"""height""": 2_24, """width""": 2_24},
						    """do_center_crop""": True,
						    """crop_size""": {"""height""": 18, """width""": 18},
						    """do_normalize""": True,
						    """image_mean""": [0.48145466, 0.4578275, 0.40821073],
						    """image_std""": [0.26862954, 0.26130258, 0.27577711],
						    """do_convert_rgb""": True,
						}
						UpperCAmelCase_								=     os.path.join(self.tmpdirname ,					__lowercase		)
						with open(self.image_processor_file ,					"""w""" ,					encoding="""utf-8"""		) as fp:
							json.dump(__lowercase ,					__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  List[Any] ,					**__lowercase    :  Optional[Any]		):






						'''simple docstring'''







						return BertTokenizer.from_pretrained(self.tmpdirname ,					**__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[Any] ,					**__lowercase    :  Optional[Any]		):






						'''simple docstring'''







						return BertTokenizerFast.from_pretrained(self.tmpdirname ,					**__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  int ,					**__lowercase    :  int		):






						'''simple docstring'''







						return ChineseCLIPImageProcessor.from_pretrained(self.tmpdirname ,					**__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Any		):






						'''simple docstring'''







						shutil.rmtree(self.tmpdirname		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[int]		):






						'''simple docstring'''







						UpperCAmelCase_								=     [np.random.randint(2_55 ,					size=(3, 30, 4_00) ,					dtype=np.uinta		)]

						UpperCAmelCase_								=     [Image.fromarray(np.moveaxis(__lowercase ,					0 ,					-1		)		) for x in image_inputs]

						return image_inputs





					def        SCREAMING_SNAKE_CASE							(						self    :  List[Any]		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.get_tokenizer()
						UpperCAmelCase_								=     self.get_rust_tokenizer()
						UpperCAmelCase_								=     self.get_image_processor()

						UpperCAmelCase_								=     ChineseCLIPProcessor(tokenizer=__lowercase ,					image_processor=__lowercase		)
						processor_slow.save_pretrained(self.tmpdirname		)
						UpperCAmelCase_								=     ChineseCLIPProcessor.from_pretrained(self.tmpdirname ,					use_fast=__lowercase		)

						UpperCAmelCase_								=     ChineseCLIPProcessor(tokenizer=__lowercase ,					image_processor=__lowercase		)
						processor_fast.save_pretrained(self.tmpdirname		)
						UpperCAmelCase_								=     ChineseCLIPProcessor.from_pretrained(self.tmpdirname		)

						self.assertEqual(processor_slow.tokenizer.get_vocab() ,					tokenizer_slow.get_vocab()		)
						self.assertEqual(processor_fast.tokenizer.get_vocab() ,					tokenizer_fast.get_vocab()		)
						self.assertEqual(tokenizer_slow.get_vocab() ,					tokenizer_fast.get_vocab()		)
						self.assertIsInstance(processor_slow.tokenizer ,					__lowercase		)
						self.assertIsInstance(processor_fast.tokenizer ,					__lowercase		)

						self.assertEqual(processor_slow.image_processor.to_json_string() ,					image_processor.to_json_string()		)
						self.assertEqual(processor_fast.image_processor.to_json_string() ,					image_processor.to_json_string()		)
						self.assertIsInstance(processor_slow.image_processor ,					__lowercase		)
						self.assertIsInstance(processor_fast.image_processor ,					__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  int		):






						'''simple docstring'''







						UpperCAmelCase_								=     ChineseCLIPProcessor(tokenizer=self.get_tokenizer() ,					image_processor=self.get_image_processor()		)
						processor.save_pretrained(self.tmpdirname		)

						UpperCAmelCase_								=     self.get_tokenizer(cls_token="""(CLS)""" ,					sep_token="""(SEP)"""		)
						UpperCAmelCase_								=     self.get_image_processor(do_normalize=__lowercase		)

						UpperCAmelCase_								=     ChineseCLIPProcessor.from_pretrained(
						    self.tmpdirname ,					cls_token="""(CLS)""" ,					sep_token="""(SEP)""" ,					do_normalize=__lowercase		)

						self.assertEqual(processor.tokenizer.get_vocab() ,					tokenizer_add_kwargs.get_vocab()		)
						self.assertIsInstance(processor.tokenizer ,					__lowercase		)

						self.assertEqual(processor.image_processor.to_json_string() ,					image_processor_add_kwargs.to_json_string()		)
						self.assertIsInstance(processor.image_processor ,					__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Tuple		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.get_image_processor()
						UpperCAmelCase_								=     self.get_tokenizer()

						UpperCAmelCase_								=     ChineseCLIPProcessor(tokenizer=__lowercase ,					image_processor=__lowercase		)

						UpperCAmelCase_								=     self.prepare_image_inputs()

						UpperCAmelCase_								=     image_processor(__lowercase ,					return_tensors="""np"""		)
						UpperCAmelCase_								=     processor(images=__lowercase ,					return_tensors="""np"""		)

						for key in input_feat_extract.keys():
							self.assertAlmostEqual(input_feat_extract[key].sum() ,					input_processor[key].sum() ,					delta=1e-2		)





					def        SCREAMING_SNAKE_CASE							(						self    :  Optional[Any]		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.get_image_processor()
						UpperCAmelCase_								=     self.get_tokenizer()

						UpperCAmelCase_								=     ChineseCLIPProcessor(tokenizer=__lowercase ,					image_processor=__lowercase		)

						UpperCAmelCase_								=     """Alexandra，T-shirt的价格是15便士。"""

						UpperCAmelCase_								=     processor(text=__lowercase		)

						UpperCAmelCase_								=     tokenizer(__lowercase		)

						for key in encoded_tok.keys():
							self.assertListEqual(encoded_tok[key] ,					encoded_processor[key]		)





					def        SCREAMING_SNAKE_CASE							(						self    :  str		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.get_image_processor()
						UpperCAmelCase_								=     self.get_tokenizer()

						UpperCAmelCase_								=     ChineseCLIPProcessor(tokenizer=__lowercase ,					image_processor=__lowercase		)

						UpperCAmelCase_								=     """Alexandra，T-shirt的价格是15便士。"""
						UpperCAmelCase_								=     self.prepare_image_inputs()

						UpperCAmelCase_								=     processor(text=__lowercase ,					images=__lowercase		)

						self.assertListEqual(list(inputs.keys()		) ,					["""input_ids""", """token_type_ids""", """attention_mask""", """pixel_values"""]		)

						# test if it raises when no input is passed
						with pytest.raises(__lowercase		):
							processor()





					def        SCREAMING_SNAKE_CASE							(						self    :  List[Any]		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.get_image_processor()
						UpperCAmelCase_								=     self.get_tokenizer()

						UpperCAmelCase_								=     ChineseCLIPProcessor(tokenizer=__lowercase ,					image_processor=__lowercase		)

						UpperCAmelCase_								=     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

						UpperCAmelCase_								=     processor.batch_decode(__lowercase		)
						UpperCAmelCase_								=     tokenizer.batch_decode(__lowercase		)

						self.assertListEqual(__lowercase ,					__lowercase		)





					def        SCREAMING_SNAKE_CASE							(						self    :  List[Any]		):






						'''simple docstring'''







						UpperCAmelCase_								=     self.get_image_processor()
						UpperCAmelCase_								=     self.get_tokenizer()

						UpperCAmelCase_								=     ChineseCLIPProcessor(tokenizer=__lowercase ,					image_processor=__lowercase		)

						UpperCAmelCase_								=     """Alexandra，T-shirt的价格是15便士。"""
						UpperCAmelCase_								=     self.prepare_image_inputs()

						UpperCAmelCase_								=     processor(text=__lowercase ,					images=__lowercase		)

						self.assertListEqual(list(inputs.keys()		) ,					processor.model_input_names		)


