






'''simple docstring'''

import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


SCREAMING_SNAKE_CASE__											=     1_6
SCREAMING_SNAKE_CASE__											=     3_2







def 					lowercase__				(   __UpperCamelCase	,		__UpperCamelCase	,		__UpperCamelCase	,		__UpperCamelCase	,		__UpperCamelCase = 16					)->				Optional[int]:
      UpperCamelCase       						=      AutoTokenizer.from_pretrained("""bert-base-cased"""					)
      UpperCamelCase       						=      DatasetDict(
          {
              """train""": dataset["""train"""].select(__UpperCamelCase					),
              """validation""": dataset["""train"""].select(__UpperCamelCase					),
              """test""": dataset["""validation"""],
          }					)

      def tokenize_function(__UpperCamelCase					):
            # max_length=None => use the model max length (it's actually the default)
            UpperCamelCase       						=      tokenizer(examples["""sentence1"""]	,		examples["""sentence2"""]	,		truncation=__UpperCamelCase	,		max_length=__UpperCamelCase					)
            return outputs

      # Apply the method we just defined to all the examples in all the splits of the dataset
      # starting with the main process first:
      with accelerator.main_process_first():
            UpperCamelCase       						=      datasets.map(
                __UpperCamelCase	,		batched=__UpperCamelCase	,		remove_columns=["""idx""", """sentence1""", """sentence2"""]	,		)

      # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
      # transformers library
      UpperCamelCase       						=      tokenized_datasets.rename_column("""label"""	,		"""labels"""					)

      def collate_fn(__UpperCamelCase					):
            # On TPU it's best to pad everything to the same length or training will be very slow.
            UpperCamelCase       						=      128 if accelerator.distributed_type == DistributedType.TPU else None
            # When using mixed precision we want round multiples of 8/16
            if accelerator.mixed_precision == "fp8":
                  UpperCamelCase       						=      16
            elif accelerator.mixed_precision != "no":
                  UpperCamelCase       						=      8
            else:
                  UpperCamelCase       						=      None

            return tokenizer.pad(
                __UpperCamelCase	,		padding="""longest"""	,		max_length=__UpperCamelCase	,		pad_to_multiple_of=__UpperCamelCase	,		return_tensors="""pt"""	,		)

      # Instantiate dataloaders.
      UpperCamelCase       						=      DataLoader(
          tokenized_datasets["""train"""]	,		shuffle=__UpperCamelCase	,		collate_fn=__UpperCamelCase	,		batch_size=__UpperCamelCase					)
      UpperCamelCase       						=      DataLoader(
          tokenized_datasets["""validation"""]	,		shuffle=__UpperCamelCase	,		collate_fn=__UpperCamelCase	,		batch_size=__UpperCamelCase					)

      UpperCamelCase       						=      DataLoader(
          tokenized_datasets["""test"""]	,		shuffle=__UpperCamelCase	,		collate_fn=__UpperCamelCase	,		batch_size=__UpperCamelCase					)

      return train_dataloader, eval_dataloader, test_dataloader







def 					lowercase__				(   __UpperCamelCase	,		__UpperCamelCase					)->				Any:
      # New Code #
      UpperCamelCase       						=      []
      # Download the dataset
      UpperCamelCase       						=      load_dataset("""glue"""	,		"""mrpc"""					)
      # Create our splits
      UpperCamelCase       						=      StratifiedKFold(n_splits=int(args.num_folds					)					)
      # Initialize accelerator
      UpperCamelCase       						=      Accelerator(cpu=args.cpu	,		mixed_precision=args.mixed_precision					)
      # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
      UpperCamelCase       						=      config["""lr"""]
      UpperCamelCase       						=      int(config["""num_epochs"""]					)
      UpperCamelCase       						=      int(config["""seed"""]					)
      UpperCamelCase       						=      int(config["""batch_size"""]					)

      UpperCamelCase       						=      evaluate.load("""glue"""	,		"""mrpc"""					)

      # If the batch size is too big we use gradient accumulation
      UpperCamelCase       						=      1
      if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
            UpperCamelCase       						=      batch_size // MAX_GPU_BATCH_SIZE
            UpperCamelCase       						=      MAX_GPU_BATCH_SIZE

      set_seed(__UpperCamelCase					)

      # New Code #
      # Create our folds:
      UpperCamelCase       						=      kfold.split(np.zeros(datasets["""train"""].num_rows					)	,		datasets["""train"""]["""label"""]					)
      UpperCamelCase       						=      []
      # Iterate over them
      for i, (train_idxs, valid_idxs) in enumerate(__UpperCamelCase					):
            UpperCamelCase						,UpperCamelCase						,UpperCamelCase       						=      get_fold_dataloaders(
                __UpperCamelCase	,		__UpperCamelCase	,		__UpperCamelCase	,		__UpperCamelCase	,		)
            # Instantiate the model (we build the model here so that the seed also control new weights initialization)
            UpperCamelCase       						=      AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""	,		return_dict=__UpperCamelCase					)

            # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
            # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
            # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
            UpperCamelCase       						=      model.to(accelerator.device					)

            # Instantiate optimizer
            UpperCamelCase       						=      AdamW(params=model.parameters()	,		lr=__UpperCamelCase					)

            # Instantiate scheduler
            UpperCamelCase       						=      get_linear_schedule_with_warmup(
                optimizer=__UpperCamelCase	,		num_warmup_steps=100	,		num_training_steps=(len(__UpperCamelCase					) * num_epochs) // gradient_accumulation_steps	,		)

            # Prepare everything
            # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
            # prepare method.
            UpperCamelCase						,UpperCamelCase						,UpperCamelCase						,UpperCamelCase						,UpperCamelCase       						=      accelerator.prepare(
                __UpperCamelCase	,		__UpperCamelCase	,		__UpperCamelCase	,		__UpperCamelCase	,		__UpperCamelCase					)

            # Now we train the model
            for epoch in range(__UpperCamelCase					):
                  model.train()
                  for step, batch in enumerate(__UpperCamelCase					):
                        # We could avoid this line since we set the accelerator with `device_placement=True`.
                        batch.to(accelerator.device					)
                        UpperCamelCase       						=      model(**__UpperCamelCase					)
                        UpperCamelCase       						=      outputs.loss
                        UpperCamelCase       						=      loss / gradient_accumulation_steps
                        accelerator.backward(__UpperCamelCase					)
                        if step % gradient_accumulation_steps == 0:
                              optimizer.step()
                              lr_scheduler.step()
                              optimizer.zero_grad()

                  model.eval()
                  for step, batch in enumerate(__UpperCamelCase					):
                        # We could avoid this line since we set the accelerator with `device_placement=True`.
                        batch.to(accelerator.device					)
                        with torch.no_grad():
                              UpperCamelCase       						=      model(**__UpperCamelCase					)
                        UpperCamelCase       						=      outputs.logits.argmax(dim=-1					)
                        UpperCamelCase						,UpperCamelCase       						=      accelerator.gather_for_metrics((predictions, batch["""labels"""])					)
                        metric.add_batch(
                            predictions=__UpperCamelCase	,		references=__UpperCamelCase	,		)

                  UpperCamelCase       						=      metric.compute()
                  # Use accelerator.print to print only on the main process.
                  accelerator.print(F"epoch {epoch}:"	,		__UpperCamelCase					)

            # New Code #
            # We also run predictions on the test set at the very end
            UpperCamelCase       						=      []
            for step, batch in enumerate(__UpperCamelCase					):
                  # We could avoid this line since we set the accelerator with `device_placement=True`.
                  batch.to(accelerator.device					)
                  with torch.no_grad():
                        UpperCamelCase       						=      model(**__UpperCamelCase					)
                  UpperCamelCase       						=      outputs.logits
                  UpperCamelCase						,UpperCamelCase       						=      accelerator.gather_for_metrics((predictions, batch["""labels"""])					)
                  fold_predictions.append(predictions.cpu()					)
                  if i == 0:
                        # We need all of the test predictions
                        test_references.append(references.cpu()					)
        # Use accelerator.print to print only on the main process.
            test_predictions.append(torch.cat(__UpperCamelCase	,		dim=0					)					)
            # We now need to release all our memory and get rid of the current model, optimizer, etc
            accelerator.free_memory()
      # New Code #
      # Finally we check the accuracy of our folded results:
      UpperCamelCase       						=      torch.cat(__UpperCamelCase	,		dim=0					)
      UpperCamelCase       						=      torch.stack(__UpperCamelCase	,		dim=0					).sum(dim=0					).div(int(args.num_folds					)					).argmax(dim=-1					)
      UpperCamelCase       						=      metric.compute(predictions=__UpperCamelCase	,		references=__UpperCamelCase					)
      accelerator.print("""Average test metrics from all folds:"""	,		__UpperCamelCase					)







def 					lowercase__				(   )->				List[str]:
      UpperCamelCase       						=      argparse.ArgumentParser(description="""Simple example of training script."""					)
      parser.add_argument(
          """--mixed_precision"""	,		type=__UpperCamelCase	,		default=__UpperCamelCase	,		choices=["""no""", """fp16""", """bf16""", """fp8"""]	,		help="""Whether to use mixed precision. Choose"""
          """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
          """and an Nvidia Ampere GPU."""	,		)
      parser.add_argument("""--cpu"""	,		action="""store_true"""	,		help="""If passed, will train on the CPU."""					)
      # New Code #
      parser.add_argument("""--num_folds"""	,		type=__UpperCamelCase	,		default=3	,		help="""The number of splits to perform across the dataset"""					)
      UpperCamelCase       						=      parser.parse_args()
      UpperCamelCase       						=      {"""lr""": 2E-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
      training_function(__UpperCamelCase	,		__UpperCamelCase					)


if __name__ == "__main__":
     main()








'''simple docstring'''

import inspect
import unittest

from transformers import MobileNetVaConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import MobileNetVaForImageClassification, MobileNetVaModel
     from transformers.models.mobilenet_va.modeling_mobilenet_va import MOBILENET_V1_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
     from PIL import Image

     from transformers import MobileNetVaImageProcessor






class        a_		(    lowerCamelCase       ):

  def 						A__		(   self   )						->   int:



        """simple docstring"""






        UpperCamelCase       						=      self.config_class(**self.inputs_dict   )
        self.parent.assertTrue(hasattr(_SCREAMING_SNAKE_CASE    ,	"""tf_padding"""   )   )
        self.parent.assertTrue(hasattr(_SCREAMING_SNAKE_CASE    ,	"""depth_multiplier"""   )   )






class        a_		:

  def __init__(   self    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE=13    ,	_SCREAMING_SNAKE_CASE=3    ,	_SCREAMING_SNAKE_CASE=32    ,	_SCREAMING_SNAKE_CASE=0.2_5    ,	_SCREAMING_SNAKE_CASE=8    ,	_SCREAMING_SNAKE_CASE=True    ,	_SCREAMING_SNAKE_CASE=1024    ,	_SCREAMING_SNAKE_CASE=32    ,	_SCREAMING_SNAKE_CASE="relu6"    ,	_SCREAMING_SNAKE_CASE=0.1    ,	_SCREAMING_SNAKE_CASE=0.0_2    ,	_SCREAMING_SNAKE_CASE=True    ,	_SCREAMING_SNAKE_CASE=True    ,	_SCREAMING_SNAKE_CASE=10    ,	_SCREAMING_SNAKE_CASE=None    ,	)						->   List[str]:



        """simple docstring"""






        UpperCamelCase       						=      parent
        UpperCamelCase       						=      batch_size
        UpperCamelCase       						=      num_channels
        UpperCamelCase       						=      image_size
        UpperCamelCase       						=      depth_multiplier
        UpperCamelCase       						=      min_depth
        UpperCamelCase       						=      tf_padding
        UpperCamelCase       						=      int(last_hidden_size * depth_multiplier   )
        UpperCamelCase       						=      output_stride
        UpperCamelCase       						=      hidden_act
        UpperCamelCase       						=      classifier_dropout_prob
        UpperCamelCase       						=      use_labels
        UpperCamelCase       						=      is_training
        UpperCamelCase       						=      num_labels
        UpperCamelCase       						=      initializer_range
        UpperCamelCase       						=      scope

  def 						A__		(   self   )						->   int:



        """simple docstring"""






        UpperCamelCase       						=      floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

        UpperCamelCase       						=      None
        UpperCamelCase       						=      None
        if self.use_labels:
              UpperCamelCase       						=      ids_tensor([self.batch_size]    ,	self.num_labels   )
              UpperCamelCase       						=      ids_tensor([self.batch_size, self.image_size, self.image_size]    ,	self.num_labels   )

        UpperCamelCase       						=      self.get_config()

        return config, pixel_values, labels, pixel_labels

  def 						A__		(   self   )						->   Optional[Any]:



        """simple docstring"""






        return MobileNetVaConfig(
            num_channels=self.num_channels    ,	image_size=self.image_size    ,	depth_multiplier=self.depth_multiplier    ,	min_depth=self.min_depth    ,	tf_padding=self.tf_padding    ,	hidden_act=self.hidden_act    ,	classifier_dropout_prob=self.classifier_dropout_prob    ,	initializer_range=self.initializer_range    ,	)

  def 						A__		(   self    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE   )						->   int:



        """simple docstring"""






        UpperCamelCase       						=      MobileNetVaModel(config=_SCREAMING_SNAKE_CASE   )
        model.to(_SCREAMING_SNAKE_CASE   )
        model.eval()
        UpperCamelCase       						=      model(_SCREAMING_SNAKE_CASE   )
        self.parent.assertEqual(
            result.last_hidden_state.shape    ,	(
                self.batch_size,
                self.last_hidden_size,
                self.image_size // self.output_stride,
                self.image_size // self.output_stride,
            )    ,	)

  def 						A__		(   self    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE   )						->   List[Any]:



        """simple docstring"""






        UpperCamelCase       						=      self.num_labels
        UpperCamelCase       						=      MobileNetVaForImageClassification(_SCREAMING_SNAKE_CASE   )
        model.to(_SCREAMING_SNAKE_CASE   )
        model.eval()
        UpperCamelCase       						=      model(_SCREAMING_SNAKE_CASE    ,	labels=_SCREAMING_SNAKE_CASE   )
        self.parent.assertEqual(result.logits.shape    ,	(self.batch_size, self.num_labels)   )


  def 						A__		(   self   )						->   Any:



        """simple docstring"""






        UpperCamelCase       						=      self.prepare_config_and_inputs()
        UpperCamelCase						,UpperCamelCase						,UpperCamelCase						,UpperCamelCase       						=      config_and_inputs
        UpperCamelCase       						=      {"""pixel_values""": pixel_values}
        return config, inputs_dict






@require_torch
class        a_		(    lowerCamelCase			, lowerCamelCase			, unittest.TestCase       ):
  lowercase  	=   (MobileNetVaModel, MobileNetVaForImageClassification) if is_torch_available() else ()
  lowercase  	=   (
      {"""feature-extraction""": MobileNetVaModel, """image-classification""": MobileNetVaForImageClassification}
      if is_torch_available()
      else {}
  )

  lowercase  	=   False
  lowercase  	=   False
  lowercase  	=   False
  lowercase  	=   False

  def 						A__		(   self   )						->   int:



        """simple docstring"""






        UpperCamelCase       						=      MobileNetVaModelTester(self   )
        UpperCamelCase       						=      MobileNetVaConfigTester(self    ,	config_class=_SCREAMING_SNAKE_CASE    ,	has_text_modality=_SCREAMING_SNAKE_CASE   )

  def 						A__		(   self   )						->   Optional[Any]:



        """simple docstring"""






        self.config_tester.run_common_tests()

  @unittest.skip(reason="""MobileNetV1 does not use inputs_embeds"""   )
  def 						A__		(   self   )						->   int:



        """simple docstring"""






        pass

  @unittest.skip(reason="""MobileNetV1 does not support input and output embeddings"""   )
  def 						A__		(   self   )						->   int:



        """simple docstring"""






        pass

  @unittest.skip(reason="""MobileNetV1 does not output attentions"""   )
  def 						A__		(   self   )						->   Dict:



        """simple docstring"""






        pass

  def 						A__		(   self   )						->   Union[str, Any]:



        """simple docstring"""






        UpperCamelCase						,UpperCamelCase       						=      self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
              UpperCamelCase       						=      model_class(_SCREAMING_SNAKE_CASE   )
              UpperCamelCase       						=      inspect.signature(model.forward   )
              # signature.parameters is an OrderedDict => so arg_names order is deterministic
              UpperCamelCase       						=      [*signature.parameters.keys()]

              UpperCamelCase       						=      ["""pixel_values"""]
              self.assertListEqual(arg_names[:1]    ,	_SCREAMING_SNAKE_CASE   )

  def 						A__		(   self   )						->   str:



        """simple docstring"""






        UpperCamelCase       						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE   )

  def 						A__		(   self   )						->   int:



        """simple docstring"""






        def check_hidden_states_output(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE   ):
              UpperCamelCase       						=      model_class(_SCREAMING_SNAKE_CASE   )
              model.to(_SCREAMING_SNAKE_CASE   )
              model.eval()

              with torch.no_grad():
                    UpperCamelCase       						=      model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE   )   )

              UpperCamelCase       						=      outputs.hidden_states

              UpperCamelCase       						=      26
              self.assertEqual(len(_SCREAMING_SNAKE_CASE   )    ,	_SCREAMING_SNAKE_CASE   )

        UpperCamelCase						,UpperCamelCase       						=      self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
              UpperCamelCase       						=      True
              check_hidden_states_output(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE   )

              # check that output_hidden_states also work using config
              del inputs_dict["output_hidden_states"]
              UpperCamelCase       						=      True

              check_hidden_states_output(_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE    ,	_SCREAMING_SNAKE_CASE   )

  def 						A__		(   self   )						->   str:



        """simple docstring"""






        UpperCamelCase       						=      self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*_SCREAMING_SNAKE_CASE   )


  @slow
  def 						A__		(   self   )						->   Dict:



        """simple docstring"""






        for model_name in MOBILENET_V1_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              UpperCamelCase       						=      MobileNetVaModel.from_pretrained(_SCREAMING_SNAKE_CASE   )
              self.assertIsNotNone(_SCREAMING_SNAKE_CASE   )







def 					lowercase__				(   )->				Optional[Any]:
      UpperCamelCase       						=      Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""					)
      return image






@require_torch
@require_vision
class        a_		(    unittest.TestCase       ):

  @cached_property
  def 						A__		(   self   )						->   Dict:



        """simple docstring"""






        return (
            MobileNetVaImageProcessor.from_pretrained("""google/mobilenet_v1_1.0_224"""   ) if is_vision_available() else None
        )


  @slow
  def 						A__		(   self   )						->   Union[str, Any]:



        """simple docstring"""






        UpperCamelCase       						=      MobileNetVaForImageClassification.from_pretrained("""google/mobilenet_v1_1.0_224"""   ).to(_SCREAMING_SNAKE_CASE   )

        UpperCamelCase       						=      self.default_image_processor
        UpperCamelCase       						=      prepare_img()
        UpperCamelCase       						=      image_processor(images=_SCREAMING_SNAKE_CASE    ,	return_tensors="""pt"""   ).to(_SCREAMING_SNAKE_CASE   )

        # forward pass
        with torch.no_grad():
              UpperCamelCase       						=      model(**_SCREAMING_SNAKE_CASE   )

        # verify the logits
        UpperCamelCase       						=      torch.Size((1, 1001)   )
        self.assertEqual(outputs.logits.shape    ,	_SCREAMING_SNAKE_CASE   )

        UpperCamelCase       						=      torch.tensor([-4.1_7_3_9, -1.1_2_3_3, 3.1_2_0_5]   ).to(_SCREAMING_SNAKE_CASE   )

        self.assertTrue(torch.allclose(outputs.logits[0, :3]    ,	_SCREAMING_SNAKE_CASE    ,	atol=1e-4   )   )
