




"""simple docstring"""






import importlib
import inspect
import json
import os
import re
import shutil
import sys
from pathlib import Path
from typing import Dict, Optional, Union
from urllib import request

from huggingface_hub import HfFolder, cached_download, hf_hub_download, model_info
from packaging import version

from .. import __version__
from . import DIFFUSERS_DYNAMIC_MODULE_NAME, HF_MODULES_CACHE, logging


lowercase__  		=		(
    """https://raw.githubusercontent.com/huggingface/diffusers/{revision}/examples/community/{pipeline}.py"""
)


lowercase__  		=		logging.get_logger(__name__)  # pylint: disable=invalid-name







def   _snake_case    (       ):
				_lowerCamelCase  :    Tuple					=					'https://pypi.org/pypi/diffusers/json'
				_lowerCamelCase  :    List[str]					=					json.loads(request.urlopen(lowercase__	).read()	)['releases'].keys()
				return sorted(lowercase__							,					key=lambda lowercase__	: version.Version(lowercase__	)	)







def   _snake_case    (       ):
				# This function has already been executed if HF_MODULES_CACHE already is in the Python path.
				if HF_MODULES_CACHE in sys.path:
								return

				sys.path.append(lowercase__	)
				os.makedirs(lowercase__							,					exist_ok=lowercase__	)
				_lowerCamelCase  :    str					=					Path(lowercase__	) / '__init__.py'
				if not init_path.exists():
								init_path.touch()







def   _snake_case    (       lowercase__	):
				init_hf_modules()
				_lowerCamelCase  :    Optional[int]					=					Path(lowercase__	) / name
				# If the parent module does not exist yet, recursively create it.
				if not dynamic_module_path.parent.exists():
								create_dynamic_module(dynamic_module_path.parent	)
				os.makedirs(lowercase__							,					exist_ok=lowercase__	)
				_lowerCamelCase  :    int					=					dynamic_module_path / '__init__.py'
				if not init_path.exists():
								init_path.touch()







def   _snake_case    (       lowercase__	):
				with open(lowercase__							,					'r'							,					encoding='utf-8'	) as f:
								_lowerCamelCase  :    Optional[int]					=					f.read()

				# Imports of the form `import .xxx`
				_lowerCamelCase  :    Dict					=					re.findall('^\s*import\s+\.(\S+)\s*$'							,					lowercase__							,					flags=re.MULTILINE	)
				# Imports of the form `from .xxx import yyy`
				relative_imports += re.findall('^\s*from\s+\.(\S+)\s+import'							,					lowercase__							,					flags=re.MULTILINE	)
				# Unique-ify
				return list(set(lowercase__	)	)







def   _snake_case    (       lowercase__	):
				_lowerCamelCase  :    Dict					=					False
				_lowerCamelCase  :    List[str]					=					[module_file]
				_lowerCamelCase  :    List[str]					=					[]

				# Let's recurse through all relative imports
				while not no_change:
								_lowerCamelCase  :    Optional[int]					=					[]
								for f in files_to_check:
												new_imports.extend(get_relative_imports(lowercase__	)	)

								_lowerCamelCase  :    Any					=					Path(lowercase__	).parent
								_lowerCamelCase  :    Dict					=					[str(module_path / m	) for m in new_imports]
								_lowerCamelCase  :    List[str]					=					[f for f in new_import_files if f not in all_relative_imports]
								_lowerCamelCase  :    Optional[Any]					=					[f'''{f}.py''' for f in new_import_files]

								_lowerCamelCase  :    Dict					=					len(lowercase__	) == 0
								all_relative_imports.extend(lowercase__	)

				return all_relative_imports







def   _snake_case    (       lowercase__	):
				with open(lowercase__							,					'r'							,					encoding='utf-8'	) as f:
								_lowerCamelCase  :    List[Any]					=					f.read()

				# Imports of the form `import xxx`
				_lowerCamelCase  :    List[Any]					=					re.findall('^\s*import\s+(\S+)\s*$'							,					lowercase__							,					flags=re.MULTILINE	)
				# Imports of the form `from xxx import yyy`
				imports += re.findall('^\s*from\s+(\S+)\s+import'							,					lowercase__							,					flags=re.MULTILINE	)
				# Only keep the top-level module
				_lowerCamelCase  :    Union[str, Any]					=					[imp.split('.'	)[0] for imp in imports if not imp.startswith('.'	)]

				# Unique-ify and test we got them all
				_lowerCamelCase  :    str					=					list(set(lowercase__	)	)
				_lowerCamelCase  :    Any					=					[]
				for imp in imports:
								try:
												importlib.import_module(lowercase__	)
								except ImportError:
												missing_packages.append(lowercase__	)

				if len(lowercase__	) > 0:
								raise ImportError(
								    'This modeling file requires the following packages that were not found in your environment: '
								    f'''{', '.join(lowercase__	)}. Run `pip install {' '.join(lowercase__	)}`'''	)

				return get_relative_imports(lowercase__	)







def   _snake_case    (       lowercase__							,					lowercase__	):
				_lowerCamelCase  :    int					=					module_path.replace(os.path.sep							,					'.'	)
				_lowerCamelCase  :    Optional[Any]					=					importlib.import_module(lowercase__	)

				if class_name is None:
								return find_pipeline_class(lowercase__	)
				return getattr(lowercase__							,					lowercase__	)







def   _snake_case    (       lowercase__	):
				from ..pipelines import DiffusionPipeline

				_lowerCamelCase  :    str					=					dict(inspect.getmembers(lowercase__							,					inspect.isclass	)	)

				_lowerCamelCase  :    Tuple					=					None
				for cls_name, cls in cls_members.items():
								if (
								    cls_name != DiffusionPipeline.__name__
								    and issubclass(cls							,					lowercase__	)
								    and cls.__module__.split('.'	)[0] != "diffusers"
								):
												if pipeline_class is not None:
																raise ValueError(
																    f'''Multiple classes that inherit from {DiffusionPipeline.__name__} have been found:'''
																    f''' {pipeline_class.__name__}, and {cls_name}. Please make sure to define only one in'''
																    f''' {loaded_module}.'''	)
												_lowerCamelCase  :    Optional[int]					=					cls

				return pipeline_class







def   _snake_case    (       lowercase__							,					lowercase__							,					lowercase__ = None							,					lowercase__ = False							,					lowercase__ = False							,					lowercase__ = None							,					lowercase__ = None							,					lowercase__ = None							,					lowercase__ = False							,					):
				_lowerCamelCase  :    int					=					str(lowercase__	)

				_lowerCamelCase  :    Union[str, Any]					=					os.path.join(lowercase__							,					lowercase__	)

				if os.path.isfile(lowercase__	):
								_lowerCamelCase  :    Union[str, Any]					=					module_file_or_url
								_lowerCamelCase  :    Tuple					=					'local'
				elif pretrained_model_name_or_path.count('/'	) == 0:
								_lowerCamelCase  :    List[str]					=					get_diffusers_versions()
								# cut ".dev0"
								_lowerCamelCase  :    Optional[Any]					=					'v' + '.'.join(__version__.split('.'	)[:3]	)

								# retrieve github version that matches
								if revision is None:
												_lowerCamelCase  :    Dict					=					latest_version if latest_version[1:] in available_versions else 'main'
												logger.info(f'''Defaulting to latest_version: {revision}.'''	)
								elif revision in available_versions:
												_lowerCamelCase  :    str					=					f'''v{revision}'''
								elif revision == "main":
												_lowerCamelCase  :    Tuple					=					revision
								else:
												raise ValueError(
												    f'''`custom_revision`: {revision} does not exist. Please make sure to choose one of'''
												    f''' {', '.join(available_versions + ['main']	)}.'''	)

								# community pipeline on GitHub
								_lowerCamelCase  :    Optional[int]					=					COMMUNITY_PIPELINES_URL.format(revision=lowercase__							,					pipeline=lowercase__	)
								try:
												_lowerCamelCase  :    Union[str, Any]					=					cached_download(
												    lowercase__							,					cache_dir=lowercase__							,					force_download=lowercase__							,					proxies=lowercase__							,					resume_download=lowercase__							,					local_files_only=lowercase__							,					use_auth_token=lowercase__							,					)
												_lowerCamelCase  :    Any					=					'git'
												_lowerCamelCase  :    Tuple					=					pretrained_model_name_or_path + '.py'
								except EnvironmentError:
												logger.error(f'''Could not locate the {module_file} inside {pretrained_model_name_or_path}.'''	)
												raise
				else:
								try:
												# Load from URL or cache if already cached
												_lowerCamelCase  :    Dict					=					hf_hub_download(
												    lowercase__							,					lowercase__							,					cache_dir=lowercase__							,					force_download=lowercase__							,					proxies=lowercase__							,					resume_download=lowercase__							,					local_files_only=lowercase__							,					use_auth_token=lowercase__							,					)
												_lowerCamelCase  :    str					=					os.path.join('local'							,					'--'.join(pretrained_model_name_or_path.split('/'	)	)	)
								except EnvironmentError:
												logger.error(f'''Could not locate the {module_file} inside {pretrained_model_name_or_path}.'''	)
												raise

    # Check we have all the requirements in our environment
				_lowerCamelCase  :    Tuple					=					check_imports(lowercase__	)

				# Now we move the module inside our cached dynamic modules.
				_lowerCamelCase  :    int					=					DIFFUSERS_DYNAMIC_MODULE_NAME + os.path.sep + submodule
				create_dynamic_module(lowercase__	)
				_lowerCamelCase  :    Optional[int]					=					Path(lowercase__	) / full_submodule
				if submodule == "local" or submodule == "git":
								# We always copy local files (we could hash the file to see if there was a change, and give them the name of
								# that hash, to only copy when there is a modification but it seems overkill for now).
								# The only reason we do the copy is to avoid putting too many folders in sys.path.
								shutil.copy(lowercase__							,					submodule_path / module_file	)
								for module_needed in modules_needed:
												_lowerCamelCase  :    Any					=					f'''{module_needed}.py'''
												shutil.copy(os.path.join(lowercase__							,					lowercase__	)							,					submodule_path / module_needed	)
				else:
								# Get the commit hash
								# TODO: we will get this info in the etag soon, so retrieve it from there and not here.
								if isinstance(lowercase__							,					lowercase__	):
												_lowerCamelCase  :    Tuple					=					use_auth_token
								elif use_auth_token is True:
												_lowerCamelCase  :    Optional[int]					=					HfFolder.get_token()
								else:
												_lowerCamelCase  :    int					=					None

								_lowerCamelCase  :    int					=					model_info(lowercase__							,					revision=lowercase__							,					token=lowercase__	).sha

								# The module file will end up being placed in a subfolder with the git hash of the repo. This way we get the
								# benefit of versioning.
								_lowerCamelCase  :    Dict					=					submodule_path / commit_hash
								_lowerCamelCase  :    List[str]					=					full_submodule + os.path.sep + commit_hash
								create_dynamic_module(lowercase__	)

								if not (submodule_path / module_file).exists():
												shutil.copy(lowercase__							,					submodule_path / module_file	)
								# Make sure we also have every file with relative
								for module_needed in modules_needed:
												if not (submodule_path / module_needed).exists():
																get_cached_module_file(
																    lowercase__							,					f'''{module_needed}.py'''							,					cache_dir=lowercase__							,					force_download=lowercase__							,					resume_download=lowercase__							,					proxies=lowercase__							,					use_auth_token=lowercase__							,					revision=lowercase__							,					local_files_only=lowercase__							,					)
				return os.path.join(lowercase__							,					lowercase__	)







def   _snake_case    (       lowercase__							,					lowercase__							,					lowercase__ = None							,					lowercase__ = None							,					lowercase__ = False							,					lowercase__ = False							,					lowercase__ = None							,					lowercase__ = None							,					lowercase__ = None							,					lowercase__ = False							,					**lowercase__							,					):
				_lowerCamelCase  :    int					=					get_cached_module_file(
				    lowercase__							,					lowercase__							,					cache_dir=lowercase__							,					force_download=lowercase__							,					resume_download=lowercase__							,					proxies=lowercase__							,					use_auth_token=lowercase__							,					revision=lowercase__							,					local_files_only=lowercase__							,					)
				return get_class_in_module(lowercase__							,					final_module.replace('.py'							,					''	)	)





"""simple docstring"""






from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_torch_dist_unique_port,
    require_torch_multi_gpu,
    require_torch_neuroncore,
)
from transformers.training_args import ParallelMode
from transformers.utils import logging


lowercase__  		=		logging.get_logger(__name__)


if is_torch_available():
			import torch
			from torch import nn
			from torch.utils.data import Dataset

			from transformers import Trainer






			class lowerCAmelCase__			(  lowercase		):






						'''simple docstring'''



						def __init__(						self						,   lowercase = 101						):
										_lowerCamelCase  :    Optional[int]					=					length
						def __len__(						self						):
										return self.length






						def __getitem__(						self						,   lowercase						):
										return i






			class lowerCAmelCase__			:






						'''simple docstring'''



						def __call__(						self						,   lowercase						):
										return {"input_ids": torch.tensor(lowercase						), "labels": torch.tensor(lowercase						)}






			class lowerCAmelCase__			(  nn.Module		):






						'''simple docstring'''



						def __init__(						self						):
										super().__init__()
										# Add some (unused) params otherwise DDP will complain.
										_lowerCamelCase  :    int					=					nn.Linear(120						,   80						)






						def       A_					(						self						,   lowercase						,   lowercase=None						):
										if labels is not None:
														return torch.tensor(0.0						,   device=input_ids.device						), input_ids
										else:
														return input_ids






class lowerCAmelCase__			(  lowercase		):






			'''simple docstring'''



			@require_torch_neuroncore
			def       A_					(						self						):
							_lowerCamelCase  :    Optional[Any]					=					F'''--nproc_per_node=2
            --master_port={get_torch_dist_unique_port()}
            {self.test_file_dir}/test_trainer_distributed.py
        '''.split()
							_lowerCamelCase  :    int					=					self.get_auto_remove_tmp_dir()
							_lowerCamelCase  :    str					=					F'''--output_dir {output_dir}'''.split()
							_lowerCamelCase  :    Tuple					=					['torchrun'] + distributed_args + args
							execute_subprocess_async(lowercase						,   env=self.get_env()						)
							# successful return here == success - any errors would have caused an error in the sub-call






class lowerCAmelCase__			(  lowercase		):






			'''simple docstring'''



			@require_torch_multi_gpu
			def       A_					(						self						):
							_lowerCamelCase  :    Union[str, Any]					=					F'''--nproc_per_node={torch.cuda.device_count()}
            --master_port={get_torch_dist_unique_port()}
            {self.test_file_dir}/test_trainer_distributed.py
        '''.split()
							_lowerCamelCase  :    List[str]					=					self.get_auto_remove_tmp_dir()
							_lowerCamelCase  :    Any					=					F'''--output_dir {output_dir}'''.split()
							_lowerCamelCase  :    Any					=					['torchrun'] + distributed_args + args
							execute_subprocess_async(lowercase						,   env=self.get_env()						)
							# successful return here == success - any errors would have caused an error in the sub-call


if __name__ == "__main__":
			# The script below is meant to be run under torch.distributed, on a machine with multiple GPUs:
			#
			# PYTHONPATH="src" python -m torch.distributed.run --nproc_per_node 2 --output_dir output_dir ./tests/test_trainer_distributed.py

			lowercase__  		=		HfArgumentParser((TrainingArguments,))
			lowercase__  		=		parser.parse_args_into_dataclasses()[0]

			logger.warning(
			    F"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, "
			    F"distributed training: {training_args.parallel_mode != ParallelMode.NOT_DISTRIBUTED}"
			)

			# Essentially, what we want to verify in the distributed case is that we get all samples back,
			# in the right order. (this is crucial for prediction for instance)
			for dataset_length in [101, 40, 7]:
						lowercase__  		=		DummyDataset(dataset_length)







						def   _snake_case    (       lowercase__	):
										_lowerCamelCase  :    int					=					list(range(len(lowercase__	)	)	)
										_lowerCamelCase  :    Optional[int]					=					p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
										if not success and training_args.local_rank == 0:
														logger.warning(
														    'Predictions and/or labels do not match expected results:\n  - predictions: '
														    f'''{p.predictions.tolist()}\n  - labels: {p.label_ids.tolist()}\n  - expected: {sequential}'''	)
										return {"success": success}

						lowercase__  		=		Trainer(
						    model=DummyModel(),
						    args=training_args,
						    data_collator=DummyDataCollator(),
						    eval_dataset=dataset,
						    compute_metrics=compute_metrics,
						)
						lowercase__  		=		trainer.evaluate()
						logger.info(metrics)
						if metrics["eval_success"] is not True:
									logger.error(metrics)
									exit(1)

						lowercase__  		=		trainer.predict(dataset)
						logger.info(p.metrics)
						if p.metrics["test_success"] is not True:
									logger.error(p.metrics)
									exit(1)

						lowercase__  		=		2

						lowercase__  		=		trainer.evaluate()
						logger.info(metrics)
						if metrics["eval_success"] is not True:
									logger.error(metrics)
									exit(1)

						lowercase__  		=		trainer.predict(dataset)
						logger.info(p.metrics)
						if p.metrics["test_success"] is not True:
									logger.error(p.metrics)
									exit(1)
						lowercase__  		=		None