





'''simple docstring'''



import json
import os
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers.testing_utils import require_torch, require_vision, slow
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
	import torch

if is_vision_available():
	from PIL import Image

	from transformers import ImageGPTImageProcessor






class 							__lowercase       (      unittest.TestCase						):







	'''simple docstring'''

	def __init__(self     ,_lowerCamelCase     ,_lowerCamelCase=7     ,_lowerCamelCase=3     ,_lowerCamelCase=18     ,_lowerCamelCase=30     ,_lowerCamelCase=400     ,_lowerCamelCase=True     ,_lowerCamelCase=None     ,_lowerCamelCase=True     ,)							->							Dict:







				'''simple docstring'''





				__lowercase										=							size if size is not None else {'''height''': 18, '''width''': 18}
				__lowercase										=							parent
				__lowercase										=							batch_size
				__lowercase										=							num_channels
				__lowercase										=							image_size
				__lowercase										=							min_resolution
				__lowercase										=							max_resolution
				__lowercase										=							do_resize
				__lowercase										=							size
				__lowercase										=							do_normalize


	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				return {
				    # here we create 2 clusters for the sake of simplicity
				    "clusters": np.asarray(
				        [
				            [0.8_8_6_6_4_4_3_6_3_4_0_3_3_2_0_3, 0.6_6_1_8_8_2_9_3_6_9_5_4_4_9_8_3, 0.3_8_9_1_7_4_6_4_0_1_7_8_6_8_0_4],
				            [-0.6_0_4_2_5_5_9_1_4_6_8_8_1_1_0_4, -0.0_2_2_9_5_0_0_8_8_6_0_5_2_8_4_6_9, 0.5_4_2_3_7_9_7_3_6_9_0_0_3_2_9_6],
				        ] ),
				    "do_resize": self.do_resize,
				    "size": self.size,
				    "do_normalize": self.do_normalize,
				}






@require_torch
@require_vision
class 							__lowercase       (      lowerCAmelCase__ ,      unittest.TestCase						):







	'''simple docstring'''

	a    :				Any											=       ImageGPTImageProcessor if is_vision_available() else None


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				__lowercase										=							ImageGPTImageProcessingTester(self )


	@property
	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				return self.image_processor_tester.prepare_image_processor_dict()


	def 						_UpperCAmelCase     (self )							->							int:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class(**self.image_processor_dict )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''clusters''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''do_resize''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''size''' ) )
				self.assertTrue(hasattr(_lowerCamelCase     ,'''do_normalize''' ) )


	def 						_UpperCAmelCase     (self )							->							Optional[int]:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class.from_dict(self.image_processor_dict )
				self.assertEqual(image_processor.size     ,{'''height''': 18, '''width''': 18} )

				__lowercase										=							self.image_processing_class.from_dict(self.image_processor_dict     ,size=42 )
				self.assertEqual(image_processor.size     ,{'''height''': 42, '''width''': 42} )


	def 						_UpperCAmelCase     (self )							->							int:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class(**self.image_processor_dict )
				__lowercase										=							json.loads(image_processor.to_json_string() )
				for key, value in self.image_processor_dict.items():
							if key == "clusters":
										self.assertTrue(np.array_equal(_lowerCamelCase     ,obj[key] ) )
							else:
										self.assertEqual(obj[key]     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class(**self.image_processor_dict )

				with tempfile.TemporaryDirectory() as tmpdirname:
							__lowercase										=							os.path.join(_lowerCamelCase     ,'''image_processor.json''' )
							image_processor_first.to_json_file(_lowerCamelCase )
							__lowercase										=							self.image_processing_class.from_json_file(_lowerCamelCase ).to_dict()

				__lowercase										=							image_processor_first.to_dict()
				for key, value in image_processor_first.items():
							if key == "clusters":
										self.assertTrue(np.array_equal(_lowerCamelCase     ,image_processor_second[key] ) )
							else:
										self.assertEqual(image_processor_first[key]     ,_lowerCamelCase )


	def 						_UpperCAmelCase     (self )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase										=							self.image_processing_class(**self.image_processor_dict )

				with tempfile.TemporaryDirectory() as tmpdirname:
							image_processor_first.save_pretrained(_lowerCamelCase )
							__lowercase										=							self.image_processing_class.from_pretrained(_lowerCamelCase ).to_dict()

				__lowercase										=							image_processor_first.to_dict()
				for key, value in image_processor_first.items():
							if key == "clusters":
										self.assertTrue(np.array_equal(_lowerCamelCase     ,image_processor_second[key] ) )
							else:
										self.assertEqual(image_processor_first[key]     ,_lowerCamelCase )


	@unittest.skip('''ImageGPT requires clusters at initialization''' )
	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				pass





def    _lowerCAmelCase						(				):

			'''simple docstring'''





			__lowercase										=							load_dataset('''hf-internal-testing/fixtures_image_utils'''		,       split='''test'''				)

			__lowercase										=							Image.open(dataset[4]['''file''']				)
			__lowercase										=							Image.open(dataset[5]['''file''']				)

			__lowercase										=							[imagea, imagea]

			return images






@require_vision
@require_torch
class 							__lowercase       (      unittest.TestCase						):







	'''simple docstring'''

	@slow
	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							ImageGPTImageProcessor.from_pretrained('''openai/imagegpt-small''' )

				__lowercase										=							prepare_images()

				# test non-batched
				__lowercase										=							image_processing(images[0]     ,return_tensors='''pt''' )

				self.assertIsInstance(encoding.input_ids     ,torch.LongTensor )
				self.assertEqual(encoding.input_ids.shape     ,(1, 1024) )

				__lowercase										=							[306, 191, 191]
				self.assertEqual(encoding.input_ids[0, :3].tolist()     ,_lowerCamelCase )

				# test batched
				__lowercase										=							image_processing(_lowerCamelCase     ,return_tensors='''pt''' )

				self.assertIsInstance(encoding.input_ids     ,torch.LongTensor )
				self.assertEqual(encoding.input_ids.shape     ,(2, 1024) )

				__lowercase										=							[303, 13, 13]
				self.assertEqual(encoding.input_ids[1, -3:].tolist()     ,_lowerCamelCase )












'''simple docstring'''



import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


_SCREAMING_SNAKE_CASE					=    '''<<<<<<< This should probably be modified because it mentions: '''

_SCREAMING_SNAKE_CASE					=    '''=======
>>>>>>>
'''

_SCREAMING_SNAKE_CASE					=    [
    '''TextEncoderConfig''',
    '''ByteTextEncoder''',
    '''SubwordTextEncoder''',
    '''encoder_config''',
    '''maybe_build_from_corpus''',
    '''manual_dir''',
]

_SCREAMING_SNAKE_CASE					=    [
    # (pattern, replacement)
    # Order is important here for some replacements
    (R'''tfds\.core''', R'''datasets'''),
    (R'''tf\.io\.gfile\.GFile''', R'''open'''),
    (R'''tf\.([\w\d]+)''', R'''datasets.Value(\'\1\')'''),
    (R'''tfds\.features\.Text\(\)''', R'''datasets.Value(\'string\')'''),
    (R'''tfds\.features\.Text\(''', R'''datasets.Value(\'string\'),'''),
    (R'''features\s*=\s*tfds.features.FeaturesDict\(''', R'''features=datasets.Features('''),
    (R'''tfds\.features\.FeaturesDict\(''', R'''dict('''),
    (R'''The TensorFlow Datasets Authors''', R'''The TensorFlow Datasets Authors and the HuggingFace Datasets Authors'''),
    (R'''tfds\.''', R'''datasets.'''),
    (R'''dl_manager\.manual_dir''', R'''self.config.data_dir'''),
    (R'''self\.builder_config''', R'''self.config'''),
]




def    _lowerCAmelCase						(				lowerCamelCase_			:  Namespace				):
			return ConvertCommand(args.tfds_path		,       args.datasets_directory				)






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	@staticmethod
	def 						_UpperCAmelCase     (_lowerCamelCase )							->							Any:







				'''simple docstring'''





				__lowercase										=							parser.add_parser(
				    '''convert'''     ,help='''Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'''     ,)
				train_parser.add_argument(
				    '''--tfds_path'''     ,type=_lowerCamelCase     ,required=_lowerCamelCase     ,help='''Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'''     ,)
				train_parser.add_argument(
				    '''--datasets_directory'''     ,type=_lowerCamelCase     ,required=_lowerCamelCase     ,help='''Path to the HuggingFace Datasets folder.''' )
				train_parser.set_defaults(func=_lowerCamelCase )


	def __init__(self     ,_lowerCamelCase     ,_lowerCamelCase     ,*_lowerCamelCase )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							get_logger('''datasets-cli/converting''' )

				__lowercase										=							tfds_path
				__lowercase										=							datasets_directory


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				if os.path.isdir(self._tfds_path ):
							__lowercase										=							os.path.abspath(self._tfds_path )
				elif os.path.isfile(self._tfds_path ):
							__lowercase										=							os.path.dirname(self._tfds_path )
				else:
							raise ValueError('''--tfds_path is neither a directory nor a file. Please check path.''' )

				__lowercase										=							os.path.abspath(self._datasets_directory )

				self._logger.info(f"Converting datasets from {abs_tfds_path} to {abs_datasets_path}" )

				__lowercase										=							[]
				__lowercase										=							[]
				__lowercase										=							{}

				if os.path.isdir(self._tfds_path ):
							__lowercase										=							os.listdir(_lowerCamelCase )
				else:
							__lowercase										=							[os.path.basename(self._tfds_path )]

				for f_name in file_names:
							self._logger.info(f"Looking at file {f_name}" )
							__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )
							__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )

							if not os.path.isfile(_lowerCamelCase ) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
										self._logger.info('''Skipping file''' )
										continue

							with open(_lowerCamelCase     ,encoding='''utf-8''' ) as f:
										__lowercase										=							f.readlines()

							__lowercase										=							[]
							__lowercase										=							False
							__lowercase										=							False
							__lowercase										=							[]
							for line in lines:
										__lowercase										=							line

										# Convert imports
										if "import tensorflow.compat.v2 as tf" in out_line:
													continue
										elif "@tfds.core" in out_line:
													continue
										elif "builder=self" in out_line:
													continue
										elif "import tensorflow_datasets.public_api as tfds" in out_line:
													__lowercase										=							'''import datasets\n'''
										elif "import tensorflow" in out_line:
													# order is important here
													__lowercase										=							''''''
													continue
										elif "from absl import logging" in out_line:
													__lowercase										=							'''from datasets import logging\n'''
										elif "getLogger" in out_line:
													__lowercase										=							out_line.replace('''getLogger'''     ,'''get_logger''' )
										elif any(expression in out_line for expression in TO_HIGHLIGHT ):
													__lowercase										=							True
													__lowercase										=							list(filter(lambda _lowerCamelCase : e in out_line     ,_lowerCamelCase ) )
													out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(_lowerCamelCase ) + '''\n''' )
													out_lines.append(_lowerCamelCase )
													out_lines.append(_lowerCamelCase )
													continue
										else:
													for pattern, replacement in TO_CONVERT:
																__lowercase										=							re.sub(_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase )

                # Take care of saving utilities (to later move them together with main script)
										if "tensorflow_datasets" in out_line:
													__lowercase										=							re.match(R'''from\stensorflow_datasets.*import\s([^\.\r\n]+)'''     ,_lowerCamelCase )
													tfds_imports.extend(imp.strip() for imp in match.group(1 ).split(''',''' ) )
													__lowercase										=							'''from . import ''' + match.group(1 )

										# Check we have not forget anything
										if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
													raise ValueError(f"Error converting {out_line.strip()}" )

										if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
													__lowercase										=							True
										out_lines.append(_lowerCamelCase )

							if is_builder or "wmt" in f_name:
										# We create a new directory for each dataset
										__lowercase										=							f_name.replace('''.py'''     ,'''''' )
										__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )
										__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )
										os.makedirs(_lowerCamelCase     ,exist_ok=_lowerCamelCase )
										self._logger.info(f"Adding directory {output_dir}" )
										imports_to_builder_map.update({imp: output_dir for imp in tfds_imports} )
							else:
										# Utilities will be moved at the end
										utils_files.append(_lowerCamelCase )

							if needs_manual_update:
										with_manual_update.append(_lowerCamelCase )

							with open(_lowerCamelCase     ,'''w'''     ,encoding='''utf-8''' ) as f:
										f.writelines(_lowerCamelCase )
							self._logger.info(f"Converted in {output_file}" )

				for utils_file in utils_files:
							try:
										__lowercase										=							os.path.basename(_lowerCamelCase )
										__lowercase										=							imports_to_builder_map[f_name.replace('''.py'''     ,'''''' )]
										self._logger.info(f"Moving {dest_folder} to {utils_file}" )
										shutil.copy(_lowerCamelCase     ,_lowerCamelCase )
							except KeyError:
										self._logger.error(f"Cannot find destination folder for {utils_file}. Please copy manually." )

				if with_manual_update:
							for file_path in with_manual_update:
										self._logger.warning(
										    f"You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'." )





