



'''simple docstring'''



import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, XLMRobertaTokenizer

from diffusers import AltDiffusionPipeline, AutoencoderKL, DDIMScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.alt_diffusion.modeling_roberta_series import (
    RobertaSeriesConfig,
    RobertaSeriesModelWithTransformation,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()





class _UpperCamelCase		(     A      ,       A      ,       A      ,       unittest.TestCase		):





				'''simple docstring'''





				lowerCAmelCase__					    =    AltDiffusionPipeline
				lowerCAmelCase__					    =    TEXT_TO_IMAGE_PARAMS
				lowerCAmelCase__					    =    TEXT_TO_IMAGE_BATCH_PARAMS
				lowerCAmelCase__					    =    TEXT_TO_IMAGE_IMAGE_PARAMS
				lowerCAmelCase__					    =    TEXT_TO_IMAGE_IMAGE_PARAMS

				def 				__lowerCamelCase     (							self  :  Union[str, Any]):







											'''simple docstring'''

											torch.manual_seed(0)
											__lowercase       	=UNetaDConditionModel(
											    block_out_channels=(3_2, 6_4)  ,       layers_per_block=2  ,       sample_size=3_2  ,       in_channels=4  ,       out_channels=4  ,       down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D')  ,       up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D')  ,       cross_attention_dim=3_2  ,       )
											__lowercase       	=DDIMScheduler(
											    beta_start=0.0_0085  ,       beta_end=0.012  ,       beta_schedule='scaled_linear'  ,       clip_sample=_lowerCAmelCase  ,       set_alpha_to_one=_lowerCAmelCase  ,       )
											torch.manual_seed(0)
											__lowercase       	=AutoencoderKL(
											    block_out_channels=[3_2, 6_4]  ,       in_channels=3  ,       out_channels=3  ,       down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']  ,       up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']  ,       latent_channels=4  ,       )

											# TODO: address the non-deterministic text encoder (fails for save-load tests)
											# torch.manual_seed(0)
											# text_encoder_config = RobertaSeriesConfig(
											#     hidden_size=32,
											#     project_dim=32,
											#     intermediate_size=37,
											#     layer_norm_eps=1e-05,
											#     num_attention_heads=4,
											#     num_hidden_layers=5,
											#     vocab_size=5002,
											# )
											# text_encoder = RobertaSeriesModelWithTransformation(text_encoder_config)

											torch.manual_seed(0)
											__lowercase       	=CLIPTextConfig(
											    bos_token_id=0  ,       eos_token_id=2  ,       hidden_size=3_2  ,       projection_dim=3_2  ,       intermediate_size=3_7  ,       layer_norm_eps=1e-05  ,       num_attention_heads=4  ,       num_hidden_layers=5  ,       pad_token_id=1  ,       vocab_size=5_0_0_2  ,       )
											__lowercase       	=CLIPTextModel(_lowerCAmelCase)

											__lowercase       	=XLMRobertaTokenizer.from_pretrained('hf-internal-testing/tiny-xlm-roberta')
											__lowercase       	=7_7

											__lowercase       	={
											    'unet': unet,
											    'scheduler': scheduler,
											    'vae': vae,
											    'text_encoder': text_encoder,
											    'tokenizer': tokenizer,
											    'safety_checker': None,
											    'feature_extractor': None,
											}
											return components

				def 				__lowerCamelCase     (							self  :  List[Any]  ,       _lowerCAmelCase  :  Union[str, Any]  ,       _lowerCAmelCase  :  Dict=0):







											'''simple docstring'''

											if str(_lowerCAmelCase).startswith('mps'):
																		__lowercase       	=torch.manual_seed(_lowerCAmelCase)
											else:
																		__lowercase       	=torch.Generator(device=_lowerCAmelCase).manual_seed(_lowerCAmelCase)
											__lowercase       	={
											    'prompt': 'A painting of a squirrel eating a burger',
											    'generator': generator,
											    'num_inference_steps': 2,
											    'guidance_scale': 6.0,
											    'output_type': 'numpy',
											}
											return inputs

				def 				__lowerCamelCase     (							self  :  List[str]):







											'''simple docstring'''

											super().test_attention_slicing_forward_pass(expected_max_diff=3e-3)

				def 				__lowerCamelCase     (							self  :  Optional[int]):







											'''simple docstring'''

											super().test_inference_batch_single_identical(expected_max_diff=3e-3)

				def 				__lowerCamelCase     (							self  :  Optional[Any]):







											'''simple docstring'''

											__lowercase       	='cpu'  # ensure determinism for the device-dependent torch.Generator

											__lowercase       	=self.get_dummy_components()
											torch.manual_seed(0)
											__lowercase       	=RobertaSeriesConfig(
											    hidden_size=3_2  ,       project_dim=3_2  ,       intermediate_size=3_7  ,       layer_norm_eps=1e-05  ,       num_attention_heads=4  ,       num_hidden_layers=5  ,       vocab_size=5_0_0_2  ,       )
											# TODO: remove after fixing the non-deterministic text encoder
											__lowercase       	=RobertaSeriesModelWithTransformation(_lowerCAmelCase)
											__lowercase       	=text_encoder

											__lowercase       	=AltDiffusionPipeline(**_lowerCAmelCase)
											__lowercase       	=alt_pipe.to(_lowerCAmelCase)
											alt_pipe.set_progress_bar_config(disable=_lowerCAmelCase)

											__lowercase       	=self.get_dummy_inputs(_lowerCAmelCase)
											__lowercase       	='A photo of an astronaut'
											__lowercase       	=alt_pipe(**_lowerCAmelCase)
											__lowercase       	=output.images
											__lowercase       	=image[0, -3:, -3:, -1]

											assert image.shape == (1, 6_4, 6_4, 3)
											__lowercase       	=np.array(
											    [0.574_8162, 0.6044_7145, 0.4882_1217, 0.5010_0636, 0.543_1185, 0.4576_3683, 0.4965_7696, 0.4813_2733, 0.4757_3093])

											assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

				def 				__lowerCamelCase     (							self  :  str):







											'''simple docstring'''

											__lowercase       	='cpu'  # ensure determinism for the device-dependent torch.Generator

											__lowercase       	=self.get_dummy_components()
											__lowercase       	=PNDMScheduler(skip_prk_steps=_lowerCAmelCase)
											torch.manual_seed(0)
											__lowercase       	=RobertaSeriesConfig(
											    hidden_size=3_2  ,       project_dim=3_2  ,       intermediate_size=3_7  ,       layer_norm_eps=1e-05  ,       num_attention_heads=4  ,       num_hidden_layers=5  ,       vocab_size=5_0_0_2  ,       )
											# TODO: remove after fixing the non-deterministic text encoder
											__lowercase       	=RobertaSeriesModelWithTransformation(_lowerCAmelCase)
											__lowercase       	=text_encoder
											__lowercase       	=AltDiffusionPipeline(**_lowerCAmelCase)
											__lowercase       	=alt_pipe.to(_lowerCAmelCase)
											alt_pipe.set_progress_bar_config(disable=_lowerCAmelCase)

											__lowercase       	=self.get_dummy_inputs(_lowerCAmelCase)
											__lowercase       	=alt_pipe(**_lowerCAmelCase)
											__lowercase       	=output.images
											__lowercase       	=image[0, -3:, -3:, -1]

											assert image.shape == (1, 6_4, 6_4, 3)
											__lowercase       	=np.array(
											    [0.5160_5093, 0.570_7241, 0.4736_5507, 0.5057_8886, 0.563_3877, 0.464_2503, 0.518_2081, 0.4876_3484, 0.4908_4237])

											assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2





@slow
@require_torch_gpu
class _UpperCamelCase		(     unittest.TestCase		):





				'''simple docstring'''





				def 				__lowerCamelCase     (							self  :  str):







											'''simple docstring'''

											super().tearDown()
											gc.collect()
											torch.cuda.empty_cache()

				def 				__lowerCamelCase     (							self  :  Union[str, Any]):







											'''simple docstring'''

											__lowercase       	=AltDiffusionPipeline.from_pretrained('BAAI/AltDiffusion'  ,       safety_checker=_lowerCAmelCase)
											__lowercase       	=alt_pipe.to(_lowerCAmelCase)
											alt_pipe.set_progress_bar_config(disable=_lowerCAmelCase)

											__lowercase       	='A painting of a squirrel eating a burger'
											__lowercase       	=torch.manual_seed(0)
											__lowercase       	=alt_pipe([prompt]  ,       generator=_lowerCAmelCase  ,       guidance_scale=6.0  ,       num_inference_steps=2_0  ,       output_type='np')

											__lowercase       	=output.images

											__lowercase       	=image[0, -3:, -3:, -1]

											assert image.shape == (1, 5_1_2, 5_1_2, 3)
											__lowercase       	=np.array([0.1010, 0.0800, 0.0794, 0.0885, 0.0843, 0.0762, 0.0769, 0.0729, 0.0586])

											assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2

				def 				__lowerCamelCase     (							self  :  Dict):







											'''simple docstring'''

											__lowercase       	=DDIMScheduler.from_pretrained('BAAI/AltDiffusion'  ,       subfolder='scheduler')

											__lowercase       	=AltDiffusionPipeline.from_pretrained('BAAI/AltDiffusion'  ,       scheduler=_lowerCAmelCase  ,       safety_checker=_lowerCAmelCase)
											__lowercase       	=alt_pipe.to(_lowerCAmelCase)
											alt_pipe.set_progress_bar_config(disable=_lowerCAmelCase)

											__lowercase       	='A painting of a squirrel eating a burger'
											__lowercase       	=torch.manual_seed(0)

											__lowercase       	=alt_pipe([prompt]  ,       generator=_lowerCAmelCase  ,       num_inference_steps=2  ,       output_type='numpy')
											__lowercase       	=output.images

											__lowercase       	=image[0, -3:, -3:, -1]

											assert image.shape == (1, 5_1_2, 5_1_2, 3)
											__lowercase       	=np.array([0.4019, 0.4052, 0.3810, 0.4119, 0.3916, 0.3982, 0.4651, 0.4195, 0.5323])

											assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2








'''simple docstring'''



import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.local_sgd import LocalSGD


########################################################################
# This is a fully working simple example to use Accelerate
# with LocalSGD, which is a method to synchronize model
# parameters every K batches. It is different, but complementary
# to gradient accumulation.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowerCamelCase	 =       16
lowerCamelCase	 =       32



def   _A       (   _lowerCAmelCase  ,   _lowerCAmelCase = 16					):



							"""simple docstring"""



							__lowercase       	=AutoTokenizer.from_pretrained('bert-base-cased'					)
							__lowercase       	=load_dataset('glue'  ,   'mrpc'					)

							def tokenize_function(_lowerCAmelCase					):
														# max_length=None => use the model max length (it's actually the default)
														__lowercase       	=tokenizer(examples['sentence1']  ,   examples['sentence2']  ,   truncation=_lowerCAmelCase  ,   max_length=_lowerCAmelCase					)
														return outputs

							# Apply the method we just defined to all the examples in all the splits of the dataset
							# starting with the main process first:
							with accelerator.main_process_first():
														__lowercase       	=datasets.map(
														    _lowerCAmelCase  ,   batched=_lowerCAmelCase  ,   remove_columns=['idx', 'sentence1', 'sentence2']  ,   )

							# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
							# transformers library
							__lowercase       	=tokenized_datasets.rename_column('label'  ,   'labels'					)

							def collate_fn(_lowerCAmelCase					):
														# On TPU it's best to pad everything to the same length or training will be very slow.
														__lowercase       	=128 if accelerator.distributed_type == DistributedType.TPU else None
														# When using mixed precision we want round multiples of 8/16
														if accelerator.mixed_precision == "fp8":
																					__lowercase       	=16
														elif accelerator.mixed_precision != "no":
																					__lowercase       	=8
														else:
																					__lowercase       	=None

														return tokenizer.pad(
														    _lowerCAmelCase  ,   padding='longest'  ,   max_length=_lowerCAmelCase  ,   pad_to_multiple_of=_lowerCAmelCase  ,   return_tensors='pt'  ,   )

							# Instantiate dataloaders.
							__lowercase       	=DataLoader(
							    tokenized_datasets['train']  ,   shuffle=_lowerCAmelCase  ,   collate_fn=_lowerCAmelCase  ,   batch_size=_lowerCAmelCase					)
							__lowercase       	=DataLoader(
							    tokenized_datasets['validation']  ,   shuffle=_lowerCAmelCase  ,   collate_fn=_lowerCAmelCase  ,   batch_size=_lowerCAmelCase					)

							return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
		from accelerate.test_utils.training import mocked_dataloaders

		lowerCamelCase	 =       mocked_dataloaders  # noqa: F811



def   _A       (   _lowerCAmelCase  ,   _lowerCAmelCase					):



							"""simple docstring"""



							if os.environ.get('TESTING_MOCKED_DATALOADERS'  ,   _lowerCAmelCase					) == "1":
														__lowercase       	=2
							# New Code #
							__lowercase       	=int(args.gradient_accumulation_steps					)
							__lowercase       	=int(args.local_sgd_steps					)
							# Initialize accelerator
							__lowercase       	=Accelerator(
							    cpu=args.cpu  ,   mixed_precision=args.mixed_precision  ,   gradient_accumulation_steps=_lowerCAmelCase					)
							if accelerator.distributed_type not in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
														raise NotImplementedError('LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'					)
							# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
							__lowercase       	=config['lr']
							__lowercase       	=int(config['num_epochs']					)
							__lowercase       	=int(config['seed']					)
							__lowercase       	=int(config['batch_size']					)

							__lowercase       	=evaluate.load('glue'  ,   'mrpc'					)

							set_seed(_lowerCAmelCase					)
							__lowercase	,				__lowercase       	=get_dataloaders(_lowerCAmelCase  ,   _lowerCAmelCase					)
							# Instantiate the model (we build the model here so that the seed also control new weights initialization)
							__lowercase       	=AutoModelForSequenceClassification.from_pretrained('bert-base-cased'  ,   return_dict=_lowerCAmelCase					)

							# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
							# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
							# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
							__lowercase       	=model.to(accelerator.device					)

							# Instantiate optimizer
							__lowercase       	=AdamW(params=model.parameters()  ,   lr=_lowerCAmelCase					)

							# Instantiate scheduler
							__lowercase       	=get_linear_schedule_with_warmup(
							    optimizer=_lowerCAmelCase  ,   num_warmup_steps=100  ,   num_training_steps=(len(_lowerCAmelCase					) * num_epochs)  ,   )

							# Prepare everything
							# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
							# prepare method.
							__lowercase	,				__lowercase	,				__lowercase	,				__lowercase	,				__lowercase       	=accelerator.prepare(
							    _lowerCAmelCase  ,   _lowerCAmelCase  ,   _lowerCAmelCase  ,   _lowerCAmelCase  ,   _lowerCAmelCase					)

							# Now we train the model
							for epoch in range(_lowerCAmelCase					):
														model.train()
														with LocalSGD(
														    accelerator=_lowerCAmelCase  ,   model=_lowerCAmelCase  ,   local_sgd_steps=_lowerCAmelCase  ,   enabled=local_sgd_steps is not None					) as local_sgd:
																					for step, batch in enumerate(_lowerCAmelCase					):
																												# We could avoid this line since we set the accelerator with `device_placement=True`.
																												batch.to(accelerator.device					)
																												# New code #
																												# We use the new `accumulate` context manager to perform gradient accumulation
																												# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
																												with accelerator.accumulate(_lowerCAmelCase					):
																																			__lowercase       	=model(**_lowerCAmelCase					)
																																			__lowercase       	=output.loss
																																			accelerator.backward(_lowerCAmelCase					)
																																			optimizer.step()
																																			lr_scheduler.step()
																																			optimizer.zero_grad()
																																			# LocalSGD-specific line
																																			local_sgd.step()

														model.eval()
														for step, batch in enumerate(_lowerCAmelCase					):
																					# We could avoid this line since we set the accelerator with `device_placement=True`.
																					batch.to(accelerator.device					)
																					with torch.no_grad():
																												__lowercase       	=model(**_lowerCAmelCase					)
																					__lowercase       	=outputs.logits.argmax(dim=-1					)
																					__lowercase	,				__lowercase       	=accelerator.gather_for_metrics((predictions, batch['labels'])					)
																					metric.add_batch(
																					    predictions=_lowerCAmelCase  ,   references=_lowerCAmelCase  ,   )

														__lowercase       	=metric.compute()
														# Use accelerator.print to print only on the main process.
														accelerator.print(f"""epoch {epoch}:"""  ,   _lowerCAmelCase					)



def   _A       (   ):



							"""simple docstring"""



							__lowercase       	=argparse.ArgumentParser(description='Simple example of training script.'					)
							parser.add_argument(
							    '--mixed_precision'  ,   type=_lowerCAmelCase  ,   default=_lowerCAmelCase  ,   choices=['no', 'fp16', 'bf16', 'fp8']  ,   help='Whether to use mixed precision. Choose'
							    'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
							    'and an Nvidia Ampere GPU.'  ,   )
							# New Code #
							parser.add_argument(
							    '--gradient_accumulation_steps'  ,   type=_lowerCAmelCase  ,   default=1  ,   help='The number of minibatches to be ran before gradients are accumulated.'  ,   )
							parser.add_argument(
							    '--local_sgd_steps'  ,   type=_lowerCAmelCase  ,   default=8  ,   help='Number of local SGD steps or None to disable local SGD'					)
							parser.add_argument('--cpu'  ,   action='store_true'  ,   help='If passed, will train on the CPU.'					)
							__lowercase       	=parser.parse_args()
							__lowercase       	={'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
							training_function(_lowerCAmelCase  ,   _lowerCAmelCase					)


if __name__ == "__main__":
		main()



