





'''simple docstring'''



import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DPMSolverMultistepScheduler,
    TextToVideoSDPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import is_xformers_available, load_numpy, skip_mps, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()






@skip_mps
class 							__lowercase       (      lowerCAmelCase__ ,      unittest.TestCase						):







	'''simple docstring'''

	a    :				Optional[Any]											=       TextToVideoSDPipeline
	a    :				Optional[int]											=       TEXT_TO_IMAGE_PARAMS
	a    :				Optional[int]											=       TEXT_TO_IMAGE_BATCH_PARAMS
	# No `output_type`.
	a    :				str											=       frozenset(
	    [
	        "num_inference_steps",
	        "generator",
	        "latents",
	        "return_dict",
	        "callback",
	        "callback_steps",
	    ]						)


	def 						_UpperCAmelCase     (self )							->							Union[str, Any]:







				'''simple docstring'''





				torch.manual_seed(0 )
				__lowercase										=							UNetaDConditionModel(
				    block_out_channels=(32, 64, 64, 64)     ,layers_per_block=2     ,sample_size=32     ,in_channels=4     ,out_channels=4     ,down_block_types=('''CrossAttnDownBlock3D''', '''CrossAttnDownBlock3D''', '''CrossAttnDownBlock3D''', '''DownBlock3D''')     ,up_block_types=('''UpBlock3D''', '''CrossAttnUpBlock3D''', '''CrossAttnUpBlock3D''', '''CrossAttnUpBlock3D''')     ,cross_attention_dim=32     ,attention_head_dim=4     ,)
				__lowercase										=							DDIMScheduler(
				    beta_start=0.0_0_0_8_5     ,beta_end=0.0_1_2     ,beta_schedule='''scaled_linear'''     ,clip_sample=_lowerCamelCase     ,set_alpha_to_one=_lowerCamelCase     ,)
				torch.manual_seed(0 )
				__lowercase										=							AutoencoderKL(
				    block_out_channels=[32, 64]     ,in_channels=3     ,out_channels=3     ,down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']     ,up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']     ,latent_channels=4     ,sample_size=128     ,)
				torch.manual_seed(0 )
				__lowercase										=							CLIPTextConfig(
				    bos_token_id=0     ,eos_token_id=2     ,hidden_size=32     ,intermediate_size=37     ,layer_norm_eps=1E-0_5     ,num_attention_heads=4     ,num_hidden_layers=5     ,pad_token_id=1     ,vocab_size=1000     ,hidden_act='''gelu'''     ,projection_dim=512     ,)
				__lowercase										=							CLIPTextModel(_lowerCamelCase )
				__lowercase										=							CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip''' )

				__lowercase										=							{
				    '''unet''': unet,
				    '''scheduler''': scheduler,
				    '''vae''': vae,
				    '''text_encoder''': text_encoder,
				    '''tokenizer''': tokenizer,
				}
				return components


	def 						_UpperCAmelCase     (self     ,_lowerCamelCase     ,_lowerCamelCase=0 )							->							str:







				'''simple docstring'''





				if str(_lowerCamelCase ).startswith('''mps''' ):
							__lowercase										=							torch.manual_seed(_lowerCamelCase )
				else:
							__lowercase										=							torch.Generator(device=_lowerCamelCase ).manual_seed(_lowerCamelCase )
				__lowercase										=							{
				    '''prompt''': '''A painting of a squirrel eating a burger''',
				    '''generator''': generator,
				    '''num_inference_steps''': 2,
				    '''guidance_scale''': 6.0,
				    '''output_type''': '''pt''',
				}
				return inputs


	def 						_UpperCAmelCase     (self )							->							Optional[int]:







				'''simple docstring'''





				__lowercase										=							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
				__lowercase										=							self.get_dummy_components()
				__lowercase										=							TextToVideoSDPipeline(**_lowerCamelCase )
				__lowercase										=							sd_pipe.to(_lowerCamelCase )
				sd_pipe.set_progress_bar_config(disable=_lowerCamelCase )

				__lowercase										=							self.get_dummy_inputs(_lowerCamelCase )
				__lowercase										=							'''np'''
				__lowercase										=							sd_pipe(**_lowerCamelCase ).frames
				__lowercase										=							frames[0][-3:, -3:, -1]

				assert frames[0].shape == (64, 64, 3)
				__lowercase										=							np.array([1_5_8.0, 1_6_0.0, 1_5_3.0, 1_2_5.0, 1_0_0.0, 1_2_1.0, 1_1_1.0, 9_3.0, 1_1_3.0] )

				assert np.abs(image_slice.flatten() - expected_slice ).max() < 1E-2


	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				self._test_attention_slicing_forward_pass(test_mean_pixel_difference=_lowerCamelCase     ,expected_max_diff=3E-3 )


	@unittest.skipIf(
	    torch_device != '''cuda''' or not is_xformers_available()     ,reason='''XFormers attention is only available with CUDA and `xformers` installed'''     ,)
	def 						_UpperCAmelCase     (self )							->							Any:







				'''simple docstring'''





				self._test_xformers_attention_forwardGenerator_pass(test_mean_pixel_difference=_lowerCamelCase     ,expected_max_diff=1E-2 )


	@unittest.skip(reason='''Batching needs to be properly figured out first for this pipeline.''' )
	def 						_UpperCAmelCase     (self )							->							int:







				'''simple docstring'''





				pass


	@unittest.skip(reason='''Batching needs to be properly figured out first for this pipeline.''' )
	def 						_UpperCAmelCase     (self )							->							List[Any]:







				'''simple docstring'''





				pass


	@unittest.skip(reason='''`num_images_per_prompt` argument is not supported for this pipeline.''' )
	def 						_UpperCAmelCase     (self )							->							Union[str, Any]:







				'''simple docstring'''





				pass


	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				return super().test_progress_bar()






@slow
@skip_mps
class 							__lowercase       (      unittest.TestCase						):







	'''simple docstring'''

	def 						_UpperCAmelCase     (self )							->							Dict:







				'''simple docstring'''





				__lowercase										=							load_numpy(
				    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text_to_video/video.npy''' )

				__lowercase										=							TextToVideoSDPipeline.from_pretrained('''damo-vilab/text-to-video-ms-1.7b''' )
				__lowercase										=							DPMSolverMultistepScheduler.from_config(pipe.scheduler.config )
				__lowercase										=							pipe.to('''cuda''' )

				__lowercase										=							'''Spiderman is surfing'''
				__lowercase										=							torch.Generator(device='''cpu''' ).manual_seed(0 )

				__lowercase										=							pipe(_lowerCamelCase     ,generator=_lowerCamelCase     ,num_inference_steps=25     ,output_type='''pt''' ).frames
				__lowercase										=							video_frames.cpu().numpy()

				assert np.abs(expected_video - video ).mean() < 5E-2


	def 						_UpperCAmelCase     (self )							->							List[str]:







				'''simple docstring'''





				__lowercase										=							load_numpy(
				    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text_to_video/video_2step.npy''' )

				__lowercase										=							TextToVideoSDPipeline.from_pretrained('''damo-vilab/text-to-video-ms-1.7b''' )
				__lowercase										=							pipe.to('''cuda''' )

				__lowercase										=							'''Spiderman is surfing'''
				__lowercase										=							torch.Generator(device='''cpu''' ).manual_seed(0 )

				__lowercase										=							pipe(_lowerCamelCase     ,generator=_lowerCamelCase     ,num_inference_steps=2     ,output_type='''pt''' ).frames
				__lowercase										=							video_frames.cpu().numpy()

				assert np.abs(expected_video - video ).mean() < 5E-2












'''simple docstring'''



import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


_SCREAMING_SNAKE_CASE					=    '''<<<<<<< This should probably be modified because it mentions: '''

_SCREAMING_SNAKE_CASE					=    '''=======
>>>>>>>
'''

_SCREAMING_SNAKE_CASE					=    [
    '''TextEncoderConfig''',
    '''ByteTextEncoder''',
    '''SubwordTextEncoder''',
    '''encoder_config''',
    '''maybe_build_from_corpus''',
    '''manual_dir''',
]

_SCREAMING_SNAKE_CASE					=    [
    # (pattern, replacement)
    # Order is important here for some replacements
    (R'''tfds\.core''', R'''datasets'''),
    (R'''tf\.io\.gfile\.GFile''', R'''open'''),
    (R'''tf\.([\w\d]+)''', R'''datasets.Value(\'\1\')'''),
    (R'''tfds\.features\.Text\(\)''', R'''datasets.Value(\'string\')'''),
    (R'''tfds\.features\.Text\(''', R'''datasets.Value(\'string\'),'''),
    (R'''features\s*=\s*tfds.features.FeaturesDict\(''', R'''features=datasets.Features('''),
    (R'''tfds\.features\.FeaturesDict\(''', R'''dict('''),
    (R'''The TensorFlow Datasets Authors''', R'''The TensorFlow Datasets Authors and the HuggingFace Datasets Authors'''),
    (R'''tfds\.''', R'''datasets.'''),
    (R'''dl_manager\.manual_dir''', R'''self.config.data_dir'''),
    (R'''self\.builder_config''', R'''self.config'''),
]




def    _lowerCAmelCase						(				lowerCamelCase_			:  Namespace				):
			return ConvertCommand(args.tfds_path		,       args.datasets_directory				)






class 							__lowercase       (      lowerCAmelCase__						):







	'''simple docstring'''

	@staticmethod
	def 						_UpperCAmelCase     (_lowerCamelCase )							->							Any:







				'''simple docstring'''





				__lowercase										=							parser.add_parser(
				    '''convert'''     ,help='''Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'''     ,)
				train_parser.add_argument(
				    '''--tfds_path'''     ,type=_lowerCamelCase     ,required=_lowerCamelCase     ,help='''Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'''     ,)
				train_parser.add_argument(
				    '''--datasets_directory'''     ,type=_lowerCamelCase     ,required=_lowerCamelCase     ,help='''Path to the HuggingFace Datasets folder.''' )
				train_parser.set_defaults(func=_lowerCamelCase )


	def __init__(self     ,_lowerCamelCase     ,_lowerCamelCase     ,*_lowerCamelCase )							->							List[Any]:







				'''simple docstring'''





				__lowercase										=							get_logger('''datasets-cli/converting''' )

				__lowercase										=							tfds_path
				__lowercase										=							datasets_directory


	def 						_UpperCAmelCase     (self )							->							str:







				'''simple docstring'''





				if os.path.isdir(self._tfds_path ):
							__lowercase										=							os.path.abspath(self._tfds_path )
				elif os.path.isfile(self._tfds_path ):
							__lowercase										=							os.path.dirname(self._tfds_path )
				else:
							raise ValueError('''--tfds_path is neither a directory nor a file. Please check path.''' )

				__lowercase										=							os.path.abspath(self._datasets_directory )

				self._logger.info(f"Converting datasets from {abs_tfds_path} to {abs_datasets_path}" )

				__lowercase										=							[]
				__lowercase										=							[]
				__lowercase										=							{}

				if os.path.isdir(self._tfds_path ):
							__lowercase										=							os.listdir(_lowerCamelCase )
				else:
							__lowercase										=							[os.path.basename(self._tfds_path )]

				for f_name in file_names:
							self._logger.info(f"Looking at file {f_name}" )
							__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )
							__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )

							if not os.path.isfile(_lowerCamelCase ) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
										self._logger.info('''Skipping file''' )
										continue

							with open(_lowerCamelCase     ,encoding='''utf-8''' ) as f:
										__lowercase										=							f.readlines()

							__lowercase										=							[]
							__lowercase										=							False
							__lowercase										=							False
							__lowercase										=							[]
							for line in lines:
										__lowercase										=							line

										# Convert imports
										if "import tensorflow.compat.v2 as tf" in out_line:
													continue
										elif "@tfds.core" in out_line:
													continue
										elif "builder=self" in out_line:
													continue
										elif "import tensorflow_datasets.public_api as tfds" in out_line:
													__lowercase										=							'''import datasets\n'''
										elif "import tensorflow" in out_line:
													# order is important here
													__lowercase										=							''''''
													continue
										elif "from absl import logging" in out_line:
													__lowercase										=							'''from datasets import logging\n'''
										elif "getLogger" in out_line:
													__lowercase										=							out_line.replace('''getLogger'''     ,'''get_logger''' )
										elif any(expression in out_line for expression in TO_HIGHLIGHT ):
													__lowercase										=							True
													__lowercase										=							list(filter(lambda _lowerCamelCase : e in out_line     ,_lowerCamelCase ) )
													out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(_lowerCamelCase ) + '''\n''' )
													out_lines.append(_lowerCamelCase )
													out_lines.append(_lowerCamelCase )
													continue
										else:
													for pattern, replacement in TO_CONVERT:
																__lowercase										=							re.sub(_lowerCamelCase     ,_lowerCamelCase     ,_lowerCamelCase )

                # Take care of saving utilities (to later move them together with main script)
										if "tensorflow_datasets" in out_line:
													__lowercase										=							re.match(R'''from\stensorflow_datasets.*import\s([^\.\r\n]+)'''     ,_lowerCamelCase )
													tfds_imports.extend(imp.strip() for imp in match.group(1 ).split(''',''' ) )
													__lowercase										=							'''from . import ''' + match.group(1 )

										# Check we have not forget anything
										if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
													raise ValueError(f"Error converting {out_line.strip()}" )

										if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
													__lowercase										=							True
										out_lines.append(_lowerCamelCase )

							if is_builder or "wmt" in f_name:
										# We create a new directory for each dataset
										__lowercase										=							f_name.replace('''.py'''     ,'''''' )
										__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )
										__lowercase										=							os.path.join(_lowerCamelCase     ,_lowerCamelCase )
										os.makedirs(_lowerCamelCase     ,exist_ok=_lowerCamelCase )
										self._logger.info(f"Adding directory {output_dir}" )
										imports_to_builder_map.update({imp: output_dir for imp in tfds_imports} )
							else:
										# Utilities will be moved at the end
										utils_files.append(_lowerCamelCase )

							if needs_manual_update:
										with_manual_update.append(_lowerCamelCase )

							with open(_lowerCamelCase     ,'''w'''     ,encoding='''utf-8''' ) as f:
										f.writelines(_lowerCamelCase )
							self._logger.info(f"Converted in {output_file}" )

				for utils_file in utils_files:
							try:
										__lowercase										=							os.path.basename(_lowerCamelCase )
										__lowercase										=							imports_to_builder_map[f_name.replace('''.py'''     ,'''''' )]
										self._logger.info(f"Moving {dest_folder} to {utils_file}" )
										shutil.copy(_lowerCamelCase     ,_lowerCamelCase )
							except KeyError:
										self._logger.error(f"Cannot find destination folder for {utils_file}. Please copy manually." )

				if with_manual_update:
							for file_path in with_manual_update:
										self._logger.warning(
										    f"You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'." )





