



"""simple docstring"""
import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


_UpperCAmelCase     :   str     =			1_6
_UpperCAmelCase     :   Optional[Any]     =			3_2





def      __magic_name__				(							lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase = 16 ):
   SCREAMING_SNAKE_CASE_: Union[str, Any]     			=AutoTokenizer.from_pretrained("""bert-base-cased""" )
   SCREAMING_SNAKE_CASE_: List[Any]     			=DatasetDict(
       {
           """train""": dataset["""train"""].select(lowerCamelCase_ ),
           """validation""": dataset["""train"""].select(lowerCamelCase_ ),
           """test""": dataset["""validation"""],
       } )

   def tokenize_function(lowercase ):
      # max_length=None => use the model max length (it's actually the default)
      SCREAMING_SNAKE_CASE_: Dict     			=tokenizer(examples["""sentence1"""]							,			examples["""sentence2"""]							,			truncation=lowerCamelCase_							,			max_length=lowerCamelCase_ )
      return outputs

   # Apply the method we just defined to all the examples in all the splits of the dataset
   # starting with the main process first:
   with accelerator.main_process_first():
      SCREAMING_SNAKE_CASE_: str     			=datasets.map(
          lowerCamelCase_							,			batched=lowerCamelCase_							,			remove_columns=["""idx""", """sentence1""", """sentence2"""]							,			)

   # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
   # transformers library
   SCREAMING_SNAKE_CASE_: str     			=tokenized_datasets.rename_column("""label"""							,			"""labels""" )

   def collate_fn(lowercase ):
      # On TPU it's best to pad everything to the same length or training will be very slow.
      SCREAMING_SNAKE_CASE_: str     			=128 if accelerator.distributed_type == DistributedType.TPU else None
      # When using mixed precision we want round multiples of 8/16
      if accelerator.mixed_precision == "fp8":
         SCREAMING_SNAKE_CASE_: str     			=16
      elif accelerator.mixed_precision != "no":
         SCREAMING_SNAKE_CASE_: Optional[int]     			=8
      else:
         SCREAMING_SNAKE_CASE_: Union[str, Any]     			=None

      return tokenizer.pad(
          lowerCamelCase_							,			padding="""longest"""							,			max_length=lowerCamelCase_							,			pad_to_multiple_of=lowerCamelCase_							,			return_tensors="""pt"""							,			)

   # Instantiate dataloaders.
   SCREAMING_SNAKE_CASE_: Optional[int]     			=DataLoader(
       tokenized_datasets["""train"""]							,			shuffle=lowerCamelCase_							,			collate_fn=lowerCamelCase_							,			batch_size=lowerCamelCase_ )
   SCREAMING_SNAKE_CASE_: str     			=DataLoader(
       tokenized_datasets["""validation"""]							,			shuffle=lowerCamelCase_							,			collate_fn=lowerCamelCase_							,			batch_size=lowerCamelCase_ )

   SCREAMING_SNAKE_CASE_: List[str]     			=DataLoader(
       tokenized_datasets["""test"""]							,			shuffle=lowerCamelCase_							,			collate_fn=lowerCamelCase_							,			batch_size=lowerCamelCase_ )

   return train_dataloader, eval_dataloader, test_dataloader





def      __magic_name__				(							lowercase							,			lowercase ):
   SCREAMING_SNAKE_CASE_: str     			=[]
   # Download the dataset
   SCREAMING_SNAKE_CASE_: int     			=load_dataset("""glue"""							,			"""mrpc""" )
   # Create our splits
   SCREAMING_SNAKE_CASE_: Union[str, Any]     			=StratifiedKFold(n_splits=int(args.num_folds ) )
   # Initialize accelerator
   SCREAMING_SNAKE_CASE_: Dict     			=Accelerator(cpu=args.cpu							,			mixed_precision=args.mixed_precision )
   # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
   SCREAMING_SNAKE_CASE_: Optional[Any]     			=config['''lr''']
   SCREAMING_SNAKE_CASE_: Tuple     			=int(config["""num_epochs"""] )
   SCREAMING_SNAKE_CASE_: List[str]     			=int(config["""seed"""] )
   SCREAMING_SNAKE_CASE_: Union[str, Any]     			=int(config["""batch_size"""] )

   SCREAMING_SNAKE_CASE_: Optional[int]     			=evaluate.load("""glue"""							,			"""mrpc""" )

   # If the batch size is too big we use gradient accumulation
   SCREAMING_SNAKE_CASE_: Optional[int]     			=1
   if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
      SCREAMING_SNAKE_CASE_: Dict     			=batch_size // MAX_GPU_BATCH_SIZE
      SCREAMING_SNAKE_CASE_: List[Any]     			=MAX_GPU_BATCH_SIZE

   set_seed(lowerCamelCase_ )

   # New Code #
   # Create our folds:
   SCREAMING_SNAKE_CASE_: Optional[Any]     			=kfold.split(np.zeros(datasets["""train"""].num_rows )							,			datasets["""train"""]["""label"""] )
   SCREAMING_SNAKE_CASE_: Optional[int]     			=[]
   # Iterate over them
   for i, (train_idxs, valid_idxs) in enumerate(lowerCamelCase_ ):
      SCREAMING_SNAKE_CASE_: List[Any]     			=get_fold_dataloaders(
          lowerCamelCase_							,			lowerCamelCase_							,			lowerCamelCase_							,			lowerCamelCase_							,			)
      # Instantiate the model (we build the model here so that the seed also control new weights initialization)
      SCREAMING_SNAKE_CASE_: Dict     			=AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""							,			return_dict=lowerCamelCase_ )

      # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
      # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
      # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
      SCREAMING_SNAKE_CASE_: List[str]     			=model.to(accelerator.device )

      # Instantiate optimizer
      SCREAMING_SNAKE_CASE_: int     			=AdamW(params=model.parameters()							,			lr=lowerCamelCase_ )

      # Instantiate scheduler
      SCREAMING_SNAKE_CASE_: List[Any]     			=get_linear_schedule_with_warmup(
          optimizer=lowerCamelCase_							,			num_warmup_steps=100							,			num_training_steps=(len(lowerCamelCase_ ) * num_epochs) // gradient_accumulation_steps							,			)

      # Prepare everything
      # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
      # prepare method.
      SCREAMING_SNAKE_CASE_: Optional[int]     			=accelerator.prepare(
          lowerCamelCase_							,			lowerCamelCase_							,			lowerCamelCase_							,			lowerCamelCase_							,			lowerCamelCase_ )

      # Now we train the model
      for epoch in range(lowerCamelCase_ ):
         model.train()
         for step, batch in enumerate(lowerCamelCase_ ):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device )
            SCREAMING_SNAKE_CASE_: List[str]     			=model(**lowerCamelCase_ )
            SCREAMING_SNAKE_CASE_: Dict     			=outputs.loss
            SCREAMING_SNAKE_CASE_: int     			=loss / gradient_accumulation_steps
            accelerator.backward(lowerCamelCase_ )
            if step % gradient_accumulation_steps == 0:
               optimizer.step()
               lr_scheduler.step()
               optimizer.zero_grad()

         model.eval()
         for step, batch in enumerate(lowerCamelCase_ ):
            # We could avoid this line since we set the accelerator with `device_placement=True`.
            batch.to(accelerator.device )
            with torch.no_grad():
               SCREAMING_SNAKE_CASE_: int     			=model(**lowerCamelCase_ )
            SCREAMING_SNAKE_CASE_: List[str]     			=outputs.logits.argmax(dim=-1 )
            SCREAMING_SNAKE_CASE_: Union[str, Any]     			=accelerator.gather_for_metrics((predictions, batch["""labels"""]) )
            metric.add_batch(
                predictions=lowerCamelCase_							,			references=lowerCamelCase_							,			)

         SCREAMING_SNAKE_CASE_: int     			=metric.compute()
         # Use accelerator.print to print only on the main process.
         accelerator.print(f'''epoch {epoch}:'''							,			lowerCamelCase_ )

      # New Code #
      # We also run predictions on the test set at the very end
      SCREAMING_SNAKE_CASE_: Any     			=[]
      for step, batch in enumerate(lowerCamelCase_ ):
         # We could avoid this line since we set the accelerator with `device_placement=True`.
         batch.to(accelerator.device )
         with torch.no_grad():
            SCREAMING_SNAKE_CASE_: str     			=model(**lowerCamelCase_ )
         SCREAMING_SNAKE_CASE_: Tuple     			=outputs.logits
         SCREAMING_SNAKE_CASE_: Any     			=accelerator.gather_for_metrics((predictions, batch["""labels"""]) )
         fold_predictions.append(predictions.cpu() )
         if i == 0:
            # We need all of the test predictions
            test_references.append(references.cpu() )
        # Use accelerator.print to print only on the main process.
      test_predictions.append(torch.cat(lowerCamelCase_							,			dim=0 ) )
      # We now need to release all our memory and get rid of the current model, optimizer, etc
      accelerator.free_memory()
   # New Code #
   # Finally we check the accuracy of our folded results:
   SCREAMING_SNAKE_CASE_: int     			=torch.cat(lowerCamelCase_							,			dim=0 )
   SCREAMING_SNAKE_CASE_: str     			=torch.stack(lowerCamelCase_							,			dim=0 ).sum(dim=0 ).div(int(args.num_folds ) ).argmax(dim=-1 )
   SCREAMING_SNAKE_CASE_: Any     			=metric.compute(predictions=lowerCamelCase_							,			references=lowerCamelCase_ )
   accelerator.print("""Average test metrics from all folds:"""							,			lowerCamelCase_ )





def      __magic_name__				(							):
   SCREAMING_SNAKE_CASE_: Tuple     			=argparse.ArgumentParser(description="""Simple example of training script.""" )
   parser.add_argument(
       """--mixed_precision"""							,			type=lowerCamelCase_							,			default=lowerCamelCase_							,			choices=["""no""", """fp16""", """bf16""", """fp8"""]							,			help="""Whether to use mixed precision. Choose"""
       """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
       """and an Nvidia Ampere GPU."""							,			)
   parser.add_argument("""--cpu"""							,			action="""store_true"""							,			help="""If passed, will train on the CPU.""" )
   # New Code #
   parser.add_argument("""--num_folds"""							,			type=lowerCamelCase_							,			default=3							,			help="""The number of splits to perform across the dataset""" )
   SCREAMING_SNAKE_CASE_: List[str]     			=parser.parse_args()
   SCREAMING_SNAKE_CASE_: Optional[Any]     			={'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
   training_function(lowerCamelCase_							,			lowerCamelCase_ )


if __name__ == "__main__":
   main()








"""simple docstring"""
import numpy as np
from matplotlib import pyplot as plt
from sklearn.datasets import load_iris
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier





def      __magic_name__				(							lowercase ):
   return (data["data"], data["target"])





def      __magic_name__				(							lowercase							,			lowercase ):
   SCREAMING_SNAKE_CASE_: Dict     			=XGBClassifier()
   classifier.fit(lowercase							,			lowercase )
   return classifier





def      __magic_name__				(							):
   SCREAMING_SNAKE_CASE_: Optional[Any]     			=load_iris()
   SCREAMING_SNAKE_CASE_			,		SCREAMING_SNAKE_CASE_: Tuple     			=data_handling(lowercase )
   SCREAMING_SNAKE_CASE_			,		SCREAMING_SNAKE_CASE_			,		SCREAMING_SNAKE_CASE_			,		SCREAMING_SNAKE_CASE_: List[Any]     			=train_test_split(
       lowercase							,			lowercase							,			test_size=0.25 )

   SCREAMING_SNAKE_CASE_: Tuple     			=iris["""target_names"""]

   # Create an XGBoost Classifier from the training data
   SCREAMING_SNAKE_CASE_: Optional[int]     			=xgboost(lowercase							,			lowercase )

   # Display the confusion matrix of the classifier with both training and test sets
   ConfusionMatrixDisplay.from_estimator(
       lowercase							,			lowercase							,			lowercase							,			display_labels=lowercase							,			cmap="""Blues"""							,			normalize="""true"""							,			)
   plt.title("""Normalized Confusion Matrix - IRIS Dataset""" )
   plt.show()


if __name__ == "__main__":
   import doctest

   doctest.testmod(verbose=True)
   main()



