
'''simple docstring'''

import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


__snake_case:  Dict								=			16
__snake_case:  Tuple								=			32






def 	_snake_case						(      A_       :    Accelerator     ,		A_       :    int = 16     ,		A_       :    str = "bert-base-cased"   ):
  """simple docstring"""







  a_      :	List[Any]      =							AutoTokenizer.from_pretrained(A_   )
  a_      :	int      =							load_dataset("""glue"""     ,		"""mrpc"""   )

  def tokenize_function(A_       :    Tuple   ):
    # max_length=None => use the model max length (it's actually the default)
    a_      :	Dict      =							tokenizer(examples["""sentence1"""]     ,		examples["""sentence2"""]     ,		truncation=A_     ,		max_length=A_   )
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  a_      :	List[str]      =							datasets.map(
      A_     ,		batched=A_     ,		remove_columns=["""idx""", """sentence1""", """sentence2"""]     ,		load_from_cache_file=A_   )

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  a_      :	List[Any]      =							tokenized_datasets.rename_column("""label"""     ,		"""labels"""   )

  def collate_fn(A_       :    int   ):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    if accelerator.distributed_type == DistributedType.TPU:
      return tokenizer.pad(A_     ,		padding="""max_length"""     ,		max_length=128     ,		return_tensors="""pt"""   )
    return tokenizer.pad(A_     ,		padding="""longest"""     ,		return_tensors="""pt"""   )

  # Instantiate dataloaders.
  a_      :	Tuple      =							DataLoader(
      tokenized_datasets["""train"""]     ,		shuffle=A_     ,		collate_fn=A_     ,		batch_size=A_   )
  a_      :	Optional[Any]      =							DataLoader(
      tokenized_datasets["""validation"""]     ,		shuffle=A_     ,		collate_fn=A_     ,		batch_size=A_   )

  return train_dataloader, eval_dataloader






def 	_snake_case						(      A_       :    List[Any]     ,		A_       :    Optional[Any]   ):
  """simple docstring"""







  a_      :	str      =							Accelerator()

  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  a_      :	List[str]      =							config["""lr"""]
  a_      :	List[Any]      =							int(config["""num_epochs"""]   )
  a_      :	int      =							int(config["""seed"""]   )
  a_      :	str      =							int(config["""batch_size"""]   )
  a_      :	Dict      =							args.model_name_or_path

  set_seed(A_   )
  a_      , a_      :	List[str]      =							get_dataloaders(A_     ,		A_     ,		A_   )

  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  a_      :	int      =							AutoModelForSequenceClassification.from_pretrained(A_     ,		return_dict=A_   )

  # Instantiate optimizer
  a_      :	Dict      =							(
      AdamW
      if accelerator.state.deepspeed_plugin is None
      or """optimizer""" not in accelerator.state.deepspeed_plugin.deepspeed_config
      else DummyOptim
  )
  a_      :	List[str]      =							optimizer_cls(params=model.parameters()     ,		lr=A_   )

  if accelerator.state.deepspeed_plugin is not None:
    a_      :	Optional[Any]      =							accelerator.state.deepspeed_plugin.deepspeed_config[
        """gradient_accumulation_steps"""
    ]
  else:
    a_      :	Union[str, Any]      =							1
  a_      :	List[str]      =							(len(A_   ) * num_epochs) // gradient_accumulation_steps

  # Instantiate scheduler
  if (
      accelerator.state.deepspeed_plugin is None
      or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
  ):
    a_      :	Dict      =							get_linear_schedule_with_warmup(
        optimizer=A_     ,		num_warmup_steps=0     ,		num_training_steps=A_     ,		)
  else:
    a_      :	Dict      =							DummyScheduler(A_     ,		total_num_steps=A_     ,		warmup_num_steps=0   )

  # Prepare everything
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  a_      , a_      , a_      , a_      , a_      :	Union[str, Any]      =							accelerator.prepare(
      A_     ,		A_     ,		A_     ,		A_     ,		A_   )

  # We need to keep track of how many total steps we have iterated over
  a_      :	int      =							0
  # We also need to keep track of the stating epoch so files are named properly
  a_      :	Tuple      =							0

  # Now we train the model
  a_      :	Optional[int]      =							evaluate.load("""glue"""     ,		"""mrpc"""   )
  a_      :	Union[str, Any]      =							0
  a_      :	Union[str, Any]      =							{}
  for epoch in range(A_     ,		A_   ):
    model.train()
    for step, batch in enumerate(A_   ):
      a_      :	Optional[int]      =							model(**A_   )
      a_      :	Union[str, Any]      =							outputs.loss
      a_      :	Dict      =							loss / gradient_accumulation_steps
      accelerator.backward(A_   )
      if step % gradient_accumulation_steps == 0:
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()

      overall_step += 1

    model.eval()
    a_      :	List[str]      =							0
    for step, batch in enumerate(A_   ):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device   )
      with torch.no_grad():
        a_      :	List[Any]      =							model(**A_   )
      a_      :	int      =							outputs.logits.argmax(dim=-1   )
      # It is slightly faster to call this once, than multiple times
      a_      , a_      :	Tuple      =							accelerator.gather(
          (predictions, batch["""labels"""])   )  # If we are in a multiprocess environment, the last batch has duplicates
      if accelerator.use_distributed:
        if step == len(A_   ) - 1:
          a_      :	Optional[int]      =							predictions[: len(eval_dataloader.dataset   ) - samples_seen]
          a_      :	Any      =							references[: len(eval_dataloader.dataset   ) - samples_seen]
        else:
          samples_seen += references.shape[0]
      metric.add_batch(
          predictions=A_     ,		references=A_     ,		)

    a_      :	Dict      =							metric.compute()
    # Use accelerator.print to print only on the main process.
    accelerator.print(f'''epoch {epoch}:'''     ,		A_   )
    a_      :	str      =							eval_metric["""accuracy"""]

    if best_performance < eval_metric["accuracy"]:
      a_      :	Union[str, Any]      =							eval_metric["""accuracy"""]

  if args.performance_lower_bound is not None:
    assert (
        args.performance_lower_bound <= best_performance
    ), f'''Best performance metric {best_performance} is lower than the lower bound {args.performance_lower_bound}'''

  accelerator.wait_for_everyone()
  if accelerator.is_main_process:
    with open(os.path.join(args.output_dir     ,		"""all_results.json"""   )     ,		"""w"""   ) as f:
      json.dump(A_     ,		A_   )






def 	_snake_case						(      ):
  """simple docstring"""







  a_      :	Tuple      =							argparse.ArgumentParser(description="""Simple example of training script tracking peak GPU memory usage."""   )
  parser.add_argument(
      """--model_name_or_path"""     ,		type=A_     ,		default="""bert-base-cased"""     ,		help="""Path to pretrained model or model identifier from huggingface.co/models."""     ,		required=A_     ,		)
  parser.add_argument(
      """--output_dir"""     ,		type=A_     ,		default="""."""     ,		help="""Optional save directory where all checkpoint folders will be stored. Default is the current working directory."""     ,		)
  parser.add_argument(
      """--performance_lower_bound"""     ,		type=A_     ,		default=A_     ,		help="""Optional lower bound for the performance metric. If set, the training will throw error when the performance metric drops below this value."""     ,		)
  parser.add_argument(
      """--num_epochs"""     ,		type=A_     ,		default=3     ,		help="""Number of train epochs."""     ,		)
  a_      :	Optional[int]      =							parser.parse_args()
  a_      :	Union[str, Any]      =							{"""lr""": 2E-5, """num_epochs""": args.num_epochs, """seed""": 42, """batch_size""": 16}
  training_function(A_     ,		A_   )


if __name__ == "__main__":
    main()




'''simple docstring'''

import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


__snake_case:  Dict								=			logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
    import torch_xla.core.xla_model as xm
    import torch_xla.debug.metrics as met


class  _UpperCAmelCase ( lowerCAmelCase__    ):

     """simple docstring"""

     def __init__(      self							,		*lowerCAmelCase_							,		lowerCAmelCase_=None							,		lowerCAmelCase_=None							,		lowerCAmelCase_=None							,		**lowerCAmelCase_ ):

       '''simple docstring'''

       super().__init__(*lowerCAmelCase_							,		**lowerCAmelCase_ )
       a_      :	Tuple      =							eval_examples
       a_      :	Optional[Any]      =							post_process_function
       a_      :	List[str]      =							quant_trainer_args
       a_      :	List[str]      =							1_28  # default number of calibration samples


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_=None ):

       '''simple docstring'''

       if calib_dataset is None and self.calib_dataset is None:
         raise ValueError("""Trainer: calibration requires an calib_dataset.""" )
       a_      :	str      =							calib_dataset if calib_dataset is not None else self.calib_dataset

       a_      :	Dict      =							self._remove_unused_columns(lowerCAmelCase_							,		description="""Calibration""" )

       return DataLoader(
           lowerCAmelCase_							,		batch_size=self.args.eval_batch_size							,		collate_fn=self.data_collator							,		drop_last=self.args.dataloader_drop_last							,		num_workers=self.args.dataloader_num_workers							,		pin_memory=self.args.dataloader_pin_memory							,		shuffle=lowerCAmelCase_							,		)


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_=None ):

       '''simple docstring'''

       a_      :	Union[str, Any]      =							self.train_dataset if calib_dataset is None else calib_dataset
       a_      :	int      =							self.get_calib_dataloader(lowerCAmelCase_ )

       a_      :	Dict      =							self.model
       quant_trainer.configure_model(lowerCAmelCase_							,		self.quant_trainer_args							,		calib=lowerCAmelCase_ )
       model.eval()
       quant_trainer.enable_calibration(lowerCAmelCase_ )

       logger.info("""***** Running calibration *****""" )
       logger.info(f'''  Num examples = {self.calib_num}''' )
       logger.info(f'''  Batch size = {calib_dataloader.batch_size}''' )

       for step, inputs in enumerate(lowerCAmelCase_ ):
         # Prediction step
         a_      , a_      , a_      :	Optional[int]      =							self.prediction_step(lowerCAmelCase_							,		lowerCAmelCase_							,		prediction_loss_only=lowerCAmelCase_ )
         if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
           break

       quant_trainer.finish_calibration(lowerCAmelCase_							,		self.quant_trainer_args )
       a_      :	Optional[int]      =							model


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_=None							,		lowerCAmelCase_=None							,		lowerCAmelCase_=None							,		lowerCAmelCase_ = "eval" ):

       '''simple docstring'''

       a_      :	str      =							self.eval_dataset if eval_dataset is None else eval_dataset
       a_      :	List[str]      =							self.get_eval_dataloader(lowerCAmelCase_ )
       a_      :	str      =							self.eval_examples if eval_examples is None else eval_examples

       # Temporarily disable metric computation, we will do it in the loop here.
       a_      :	List[str]      =							self.compute_metrics
       a_      :	Tuple      =							None
       a_      :	Optional[Any]      =							self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
       try:
         a_      :	List[str]      =							eval_loop(
             lowerCAmelCase_							,		description="""Evaluation"""							,		prediction_loss_only=True if compute_metrics is None else None							,		ignore_keys=lowerCAmelCase_							,		)
       finally:
         a_      :	int      =							compute_metrics

       if self.post_process_function is not None and self.compute_metrics is not None:
         a_      :	Optional[Any]      =							self.post_process_function(lowerCAmelCase_							,		lowerCAmelCase_							,		output.predictions )
         a_      :	Tuple      =							self.compute_metrics(lowerCAmelCase_ )

         # Prefix all keys with metric_key_prefix + '_'
         for key in list(metrics.keys() ):
           if not key.startswith(f'''{metric_key_prefix}_''' ):
             a_      :	Optional[int]      =							metrics.pop(lowerCAmelCase_ )

         self.log(lowerCAmelCase_ )
       else:
         a_      :	Any      =							{}

       if self.args.tpu_metrics_debug or self.args.debug:
         # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
         xm.master_print(met.metrics_report() )

       a_      :	Optional[Any]      =							self.callback_handler.on_evaluate(self.args							,		self.state							,		self.control							,		lowerCAmelCase_ )
       return metrics


     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_							,		lowerCAmelCase_							,		lowerCAmelCase_=None							,		lowerCAmelCase_ = "test" ):

       '''simple docstring'''

       a_      :	List[Any]      =							self.get_test_dataloader(lowerCAmelCase_ )

       # Temporarily disable metric computation, we will do it in the loop here.
       a_      :	Any      =							self.compute_metrics
       a_      :	Dict      =							None
       a_      :	Dict      =							self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
       try:
         a_      :	Dict      =							eval_loop(
             lowerCAmelCase_							,		description="""Prediction"""							,		prediction_loss_only=True if compute_metrics is None else None							,		ignore_keys=lowerCAmelCase_							,		)
       finally:
         a_      :	int      =							compute_metrics

       if self.post_process_function is None or self.compute_metrics is None:
         return output

       a_      :	Tuple      =							self.post_process_function(lowerCAmelCase_							,		lowerCAmelCase_							,		output.predictions							,		"""predict""" )
       a_      :	List[Any]      =							self.compute_metrics(lowerCAmelCase_ )

       # Prefix all keys with metric_key_prefix + '_'
       for key in list(metrics.keys() ):
         if not key.startswith(f'''{metric_key_prefix}_''' ):
           a_      :	Tuple      =							metrics.pop(lowerCAmelCase_ )

       return PredictionOutput(predictions=predictions.predictions							,		label_ids=predictions.label_ids							,		metrics=lowerCAmelCase_ )






     def 							_lowerCAmelCase  (      self							,		lowerCAmelCase_="./" ):

       '''simple docstring'''

       a_      :	Any      =							self.eval_dataset
       a_      :	Tuple      =							self.get_eval_dataloader(lowerCAmelCase_ )

       a_      :	List[str]      =							next(iter(lowerCAmelCase_ ) )

       # saving device - to make it consistent
       a_      :	Dict      =							torch.device("""cuda""" if torch.cuda.is_available() else """cpu""" )

       # convert to tuple
       a_      :	Optional[int]      =							tuple(v.to(lowerCAmelCase_ ) for k, v in batch.items() )

       logger.info("""Converting model to be onnx compatible""" )
       from pytorch_quantization.nn import TensorQuantizer

       a_      :	List[str]      =							True

       a_      :	str      =							self.model.to(lowerCAmelCase_ )

       model.eval()
       model.float()

       a_      :	Optional[int]      =							model.module if hasattr(lowerCAmelCase_							,		"""module""" ) else model
       quant_trainer.configure_model(lowerCAmelCase_							,		self.quant_trainer_args )

       a_      :	Union[str, Any]      =							os.path.join(lowerCAmelCase_							,		"""model.onnx""" )
       logger.info(f'''exporting model to {output_model_file}''' )

       a_      :	Dict      =							{0: """batch_size""", 1: """seq_len"""}

       torch.onnx.export(
           lowerCAmelCase_							,		lowerCAmelCase_							,		lowerCAmelCase_							,		export_params=lowerCAmelCase_							,		opset_version=13							,		do_constant_folding=lowerCAmelCase_							,		input_names=["""input_ids""", """attention_mask""", """token_type_ids"""]							,		output_names=["""output_start_logits""", """output_end_logits"""]							,		dynamic_axes={
               """input_ids""": axes,
               """attention_mask""": axes,
               """token_type_ids""": axes,
               """output_start_logits""": axes,
               """output_end_logits""": axes,
           }							,		verbose=lowerCAmelCase_							,		)
       logger.info("""onnx export finished""" )


