





import argparse
import gc
import json
import os

import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


lowercase_					     =	16
lowercase_					     =	32




def     __UpperCamelCase						(_SCREAMING_SNAKE_CASE	)       ->     Optional[Any]:
  return int(x / 2**20	)
class        SCREAMING_SNAKE_CASE     :


   def __enter__(					self    :       Optional[int] )->     List[str]:





     """simple docstring"""







     gc.collect()
     torch.cuda.empty_cache()
     torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
     lowercase__			    =						torch.cuda.memory_allocated()
     return self


   def __exit__(					self    :       Any    ,  *a    :       int )->     Optional[int]:





     """simple docstring"""







     gc.collect()
     torch.cuda.empty_cache()
     lowercase__			    =						torch.cuda.memory_allocated()
     lowercase__			    =						torch.cuda.max_memory_allocated()
     lowercase__			    =						bamb(self.end - self.begin )
     lowercase__			    =						bamb(self.peak - self.begin )
     # print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")






def     __UpperCamelCase						(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE = 16	,		_SCREAMING_SNAKE_CASE = "bert-base-cased"	,		_SCREAMING_SNAKE_CASE = 320	,		_SCREAMING_SNAKE_CASE = 160	,		)       ->     Dict:
  lowercase__			    =						AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE	)
  lowercase__			    =						load_dataset(
      'glue'	,		'mrpc'	,		split={'train': F"""train[:{n_train}]""", 'validation': F"""validation[:{n_val}]"""}	)

  def tokenize_function(_SCREAMING_SNAKE_CASE	):
    # max_length=None => use the model max length (it's actually the default)
    lowercase__			    =						tokenizer(examples['sentence1']	,		examples['sentence2']	,		truncation=_SCREAMING_SNAKE_CASE	,		max_length=_SCREAMING_SNAKE_CASE	)
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  lowercase__			    =						datasets.map(
      _SCREAMING_SNAKE_CASE	,		batched=_SCREAMING_SNAKE_CASE	,		remove_columns=['idx', 'sentence1', 'sentence2']	,		load_from_cache_file=_SCREAMING_SNAKE_CASE	)

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  lowercase__			    =						tokenized_datasets.rename_column('label'	,		'labels'	)

  def collate_fn(_SCREAMING_SNAKE_CASE	):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    if accelerator.distributed_type == DistributedType.TPU:
      return tokenizer.pad(_SCREAMING_SNAKE_CASE	,		padding='max_length'	,		max_length=128	,		return_tensors='pt'	)
    return tokenizer.pad(_SCREAMING_SNAKE_CASE	,		padding='longest'	,		return_tensors='pt'	)

  # Instantiate dataloaders.
  lowercase__			    =						DataLoader(
      tokenized_datasets['train']	,		shuffle=_SCREAMING_SNAKE_CASE	,		collate_fn=_SCREAMING_SNAKE_CASE	,		batch_size=_SCREAMING_SNAKE_CASE	)
  lowercase__			    =						DataLoader(
      tokenized_datasets['validation']	,		shuffle=_SCREAMING_SNAKE_CASE	,		collate_fn=_SCREAMING_SNAKE_CASE	,		batch_size=_SCREAMING_SNAKE_CASE	)

  return train_dataloader, eval_dataloader




def     __UpperCamelCase						(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	)       ->     Optional[Any]:
  # Initialize accelerator
  lowercase__			    =						Accelerator()

  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  lowercase__			    =						config['lr']
  lowercase__			    =						int(config['num_epochs']	)
  lowercase__			    =						int(config['seed']	)
  lowercase__			    =						int(config['batch_size']	)
  lowercase__			    =						args.model_name_or_path

  set_seed(_SCREAMING_SNAKE_CASE	)
  lowercase__    ,					lowercase__			    =						get_dataloaders(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		args.n_train	,		args.n_val	)

  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  lowercase__			    =						AutoModelForSequenceClassification.from_pretrained(_SCREAMING_SNAKE_CASE	,		return_dict=_SCREAMING_SNAKE_CASE	)

  # Instantiate optimizer
  lowercase__			    =						(
      AdamW
      if accelerator.state.deepspeed_plugin is None
      or 'optimizer' not in accelerator.state.deepspeed_plugin.deepspeed_config
      else DummyOptim
  )
  lowercase__			    =						optimizer_cls(params=model.parameters()	,		lr=_SCREAMING_SNAKE_CASE	)

  if accelerator.state.deepspeed_plugin is not None:
    lowercase__			    =						accelerator.state.deepspeed_plugin.deepspeed_config[
        'gradient_accumulation_steps'
    ]
  else:
    lowercase__			    =						1
  lowercase__			    =						(len(_SCREAMING_SNAKE_CASE	) * num_epochs) // gradient_accumulation_steps

  # Instantiate scheduler
  if (
      accelerator.state.deepspeed_plugin is None
      or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
  ):
    lowercase__			    =						get_linear_schedule_with_warmup(
        optimizer=_SCREAMING_SNAKE_CASE	,		num_warmup_steps=0	,		num_training_steps=_SCREAMING_SNAKE_CASE	,		)
  else:
    lowercase__			    =						DummyScheduler(_SCREAMING_SNAKE_CASE	,		total_num_steps=_SCREAMING_SNAKE_CASE	,		warmup_num_steps=0	)

  # Prepare everything
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  lowercase__    ,					lowercase__    ,					lowercase__    ,					lowercase__    ,					lowercase__			    =						accelerator.prepare(
      _SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	)

  # We need to keep track of how many total steps we have iterated over
  lowercase__			    =						0
  # We also need to keep track of the stating epoch so files are named properly
  lowercase__			    =						0

  # Now we train the model
  lowercase__			    =						{}
  for epoch in range(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	):
    with TorchTracemalloc() as tracemalloc:
      model.train()
      for step, batch in enumerate(_SCREAMING_SNAKE_CASE	):
        lowercase__			    =						model(**_SCREAMING_SNAKE_CASE	)
        lowercase__			    =						outputs.loss
        lowercase__			    =						loss / gradient_accumulation_steps
        accelerator.backward(_SCREAMING_SNAKE_CASE	)
        if step % gradient_accumulation_steps == 0:
          optimizer.step()
          lr_scheduler.step()
          optimizer.zero_grad()

        overall_step += 1

        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
    accelerator.print('Memory before entering the train : {}'.format(bamb(tracemalloc.begin	)	)	)
    accelerator.print('Memory consumed at the end of the train (end-begin): {}'.format(tracemalloc.used	)	)
    accelerator.print('Peak Memory consumed during the train (max-begin): {}'.format(tracemalloc.peaked	)	)
    accelerator.print(
        'Total Peak Memory consumed during the train (max): {}'.format(
            tracemalloc.peaked + bamb(tracemalloc.begin	)	)	)
    lowercase__			    =						tracemalloc.peaked + bamb(tracemalloc.begin	)
    if args.peak_memory_upper_bound is not None:
      assert (
          train_total_peak_memory[F"""epoch-{epoch}"""] <= args.peak_memory_upper_bound
      ), "Peak memory usage exceeded the upper bound"

  accelerator.wait_for_everyone()
  if accelerator.is_main_process:
    with open(os.path.join(args.output_dir	,		'peak_memory_utilization.json'	)	,		'w'	) as f:
      json.dump(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	)




def     __UpperCamelCase						()       ->     int:
  lowercase__			    =						argparse.ArgumentParser(description='Simple example of training script tracking peak GPU memory usage.'	)
  parser.add_argument(
      '--model_name_or_path'	,		type=_SCREAMING_SNAKE_CASE	,		default='bert-base-cased'	,		help='Path to pretrained model or model identifier from huggingface.co/models.'	,		required=_SCREAMING_SNAKE_CASE	,		)
  parser.add_argument(
      '--output_dir'	,		type=_SCREAMING_SNAKE_CASE	,		default='.'	,		help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'	,		)
  parser.add_argument(
      '--peak_memory_upper_bound'	,		type=_SCREAMING_SNAKE_CASE	,		default=_SCREAMING_SNAKE_CASE	,		help='The upper bound of peak memory usage in MB. If set, the training will throw an error if the peak memory usage exceeds this value.'	,		)
  parser.add_argument(
      '--n_train'	,		type=_SCREAMING_SNAKE_CASE	,		default=320	,		help='Number of training examples to use.'	,		)
  parser.add_argument(
      '--n_val'	,		type=_SCREAMING_SNAKE_CASE	,		default=160	,		help='Number of validation examples to use.'	,		)
  parser.add_argument(
      '--num_epochs'	,		type=_SCREAMING_SNAKE_CASE	,		default=1	,		help='Number of train epochs.'	,		)
  lowercase__			    =						parser.parse_args()
  lowercase__			    =						{'lr': 2E-5, 'num_epochs': args.num_epochs, 'seed': 42, 'batch_size': 16}
  training_function(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	)


if __name__ == "__main__":
      main()









import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionSAGPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()
class        SCREAMING_SNAKE_CASE     (UpperCAmelCase    ,			UpperCAmelCase    ,			unittest.TestCase   ):
   _UpperCamelCase					:	List[Any]				      = StableDiffusionSAGPipeline
   _UpperCamelCase					:	str				      = TEXT_TO_IMAGE_PARAMS
   _UpperCamelCase					:	Dict				      = TEXT_TO_IMAGE_BATCH_PARAMS
   _UpperCamelCase					:	str				      = TEXT_TO_IMAGE_IMAGE_PARAMS
   _UpperCamelCase					:	Optional[int]				      = TEXT_TO_IMAGE_IMAGE_PARAMS
   _UpperCamelCase					:	Union[str, Any]				      = False


   def  SCREAMING_SNAKE_CASE_							(					self    :       List[str] )->     Dict:





     """simple docstring"""







     torch.manual_seed(0 )
     lowercase__			    =						UNetaDConditionModel(
         block_out_channels=(32, 64)    ,  layers_per_block=2    ,  sample_size=32    ,  in_channels=4    ,  out_channels=4    ,  down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D')    ,  up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D')    ,  cross_attention_dim=32    ,  )
     lowercase__			    =						DDIMScheduler(
         beta_start=0.00085    ,  beta_end=0.012    ,  beta_schedule='scaled_linear'    ,  clip_sample=a    ,  set_alpha_to_one=a    ,  )
     torch.manual_seed(0 )
     lowercase__			    =						AutoencoderKL(
         block_out_channels=[32, 64]    ,  in_channels=3    ,  out_channels=3    ,  down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']    ,  up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']    ,  latent_channels=4    ,  )
     torch.manual_seed(0 )
     lowercase__			    =						CLIPTextConfig(
         bos_token_id=0    ,  eos_token_id=2    ,  hidden_size=32    ,  intermediate_size=37    ,  layer_norm_eps=1E-0_5    ,  num_attention_heads=4    ,  num_hidden_layers=5    ,  pad_token_id=1    ,  vocab_size=1_000    ,  )
     lowercase__			    =						CLIPTextModel(a )
     lowercase__			    =						CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip' )

     lowercase__			    =						{
         'unet': unet,
         'scheduler': scheduler,
         'vae': vae,
         'text_encoder': text_encoder,
         'tokenizer': tokenizer,
         'safety_checker': None,
         'feature_extractor': None,
     }
     return components


   def  SCREAMING_SNAKE_CASE_							(					self    :       Tuple    ,  a    :       Optional[int]    ,  a    :       Any=0 )->     Union[str, Any]:





     """simple docstring"""







     if str(a ).startswith('mps' ):
       lowercase__			    =						torch.manual_seed(a )
     else:
       lowercase__			    =						torch.Generator(device=a ).manual_seed(a )
     lowercase__			    =						{
         'prompt': '.',
         'generator': generator,
         'num_inference_steps': 2,
         'guidance_scale': 1.0,
         'sag_scale': 1.0,
         'output_type': 'numpy',
     }
     return inputs


   def  SCREAMING_SNAKE_CASE_							(					self    :       Any )->     Any:





     """simple docstring"""







     super().test_inference_batch_single_identical(expected_max_diff=3E-3 )






@slow
@require_torch_gpu
class        SCREAMING_SNAKE_CASE     (unittest.TestCase   ):


   def  SCREAMING_SNAKE_CASE_							(					self    :       str )->     int:





     """simple docstring"""







     super().tearDown()
     gc.collect()
     torch.cuda.empty_cache()


   def  SCREAMING_SNAKE_CASE_							(					self    :       Tuple )->     Optional[Any]:





     """simple docstring"""







     lowercase__			    =						StableDiffusionSAGPipeline.from_pretrained('CompVis/stable-diffusion-v1-4' )
     lowercase__			    =						sag_pipe.to(a )
     sag_pipe.set_progress_bar_config(disable=a )

     lowercase__			    =						'.'
     lowercase__			    =						torch.manual_seed(0 )
     lowercase__			    =						sag_pipe(
         [prompt]    ,  generator=a    ,  guidance_scale=7.5    ,  sag_scale=1.0    ,  num_inference_steps=20    ,  output_type='np' )

     lowercase__			    =						output.images

     lowercase__			    =						image[0, -3:, -3:, -1]

     assert image.shape == (1, 512, 512, 3)
     lowercase__			    =						np.array([0.1568, 0.1738, 0.1695, 0.1693, 0.1507, 0.1705, 0.1547, 0.1751, 0.1949] )

     assert np.abs(image_slice.flatten() - expected_slice ).max() < 5E-2


   def  SCREAMING_SNAKE_CASE_							(					self    :       Tuple )->     Optional[Any]:





     """simple docstring"""







     lowercase__			    =						StableDiffusionSAGPipeline.from_pretrained('stabilityai/stable-diffusion-2-1-base' )
     lowercase__			    =						sag_pipe.to(a )
     sag_pipe.set_progress_bar_config(disable=a )

     lowercase__			    =						'.'
     lowercase__			    =						torch.manual_seed(0 )
     lowercase__			    =						sag_pipe(
         [prompt]    ,  generator=a    ,  guidance_scale=7.5    ,  sag_scale=1.0    ,  num_inference_steps=20    ,  output_type='np' )

     lowercase__			    =						output.images

     lowercase__			    =						image[0, -3:, -3:, -1]

     assert image.shape == (1, 512, 512, 3)
     lowercase__			    =						np.array([0.3459, 0.2876, 0.2537, 0.3002, 0.2671, 0.2160, 0.3026, 0.2262, 0.2371] )

     assert np.abs(image_slice.flatten() - expected_slice ).max() < 5E-2


   def  SCREAMING_SNAKE_CASE_							(					self    :       Optional[int] )->     List[Any]:





     """simple docstring"""







     lowercase__			    =						StableDiffusionSAGPipeline.from_pretrained('stabilityai/stable-diffusion-2-1-base' )
     lowercase__			    =						sag_pipe.to(a )
     sag_pipe.set_progress_bar_config(disable=a )

     lowercase__			    =						'.'
     lowercase__			    =						torch.manual_seed(0 )
     lowercase__			    =						sag_pipe(
         [prompt]    ,  width=768    ,  height=512    ,  generator=a    ,  guidance_scale=7.5    ,  sag_scale=1.0    ,  num_inference_steps=20    ,  output_type='np'    ,  )

     lowercase__			    =						output.images

     assert image.shape == (1, 512, 768, 3)


