






import argparse

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


a_		:						List[Any]             =    16
a_		:						Dict             =    32






def 				lowerCamelCase__				(_UpperCAmelCase							,       _UpperCAmelCase = 16):
			SCREAMING_SNAKE_CASE						= AutoTokenizer.from_pretrained('bert-base-cased')
			SCREAMING_SNAKE_CASE						= load_dataset('glue'							,       'mrpc')

			def tokenize_function(_UpperCAmelCase):
						# max_length=None => use the model max length (it's actually the default)
						SCREAMING_SNAKE_CASE						= tokenizer(examples['sentence1']							,       examples['sentence2']							,       truncation=UpperCamelCase__							,       max_length=UpperCamelCase__)
						return outputs

			# Apply the method we just defined to all the examples in all the splits of the dataset
			# starting with the main process first:
			with accelerator.main_process_first():
						SCREAMING_SNAKE_CASE						= datasets.map(
						    UpperCamelCase__							,       batched=UpperCamelCase__							,       remove_columns=['idx', 'sentence1', 'sentence2']							,       )

			# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
			# transformers library
			SCREAMING_SNAKE_CASE						= tokenized_datasets.rename_column('label'							,       'labels')

			def collate_fn(_UpperCAmelCase):
						# On TPU it's best to pad everything to the same length or training will be very slow.
						SCREAMING_SNAKE_CASE						= 128 if accelerator.distributed_type == DistributedType.TPU else None
						# When using mixed precision we want round multiples of 8/16
						if accelerator.mixed_precision == "fp8":
									SCREAMING_SNAKE_CASE						= 16
						elif accelerator.mixed_precision != "no":
									SCREAMING_SNAKE_CASE						= 8
						else:
									SCREAMING_SNAKE_CASE						= None

						return tokenizer.pad(
						    UpperCamelCase__							,       padding='longest'							,       max_length=UpperCamelCase__							,       pad_to_multiple_of=UpperCamelCase__							,       return_tensors='pt'							,       )

			# Instantiate dataloaders.
			SCREAMING_SNAKE_CASE						= DataLoader(
			    tokenized_datasets['train']							,       shuffle=UpperCamelCase__							,       collate_fn=UpperCamelCase__							,       batch_size=UpperCamelCase__							,       drop_last=UpperCamelCase__)
			SCREAMING_SNAKE_CASE						= DataLoader(
			    tokenized_datasets['validation']							,       shuffle=UpperCamelCase__							,       collate_fn=UpperCamelCase__							,       batch_size=UpperCamelCase__							,       drop_last=(accelerator.mixed_precision == 'fp8')							,       )

			return train_dataloader, eval_dataloader






def 				lowerCamelCase__				(_UpperCAmelCase							,       _UpperCAmelCase):
			SCREAMING_SNAKE_CASE						= Accelerator(cpu=args.cpu							,       mixed_precision=args.mixed_precision)
			# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
			SCREAMING_SNAKE_CASE						= config['lr']
			SCREAMING_SNAKE_CASE						= int(config['num_epochs'])
			SCREAMING_SNAKE_CASE						= int(config['seed'])
			SCREAMING_SNAKE_CASE						= int(config['batch_size'])

			SCREAMING_SNAKE_CASE						= evaluate.load('glue'							,       'mrpc')

			# If the batch size is too big we use gradient accumulation
			SCREAMING_SNAKE_CASE						= 1
			if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
						SCREAMING_SNAKE_CASE						= batch_size // MAX_GPU_BATCH_SIZE
						SCREAMING_SNAKE_CASE						= MAX_GPU_BATCH_SIZE

			set_seed(UpperCamelCase__)
			SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						= get_dataloaders(UpperCamelCase__							,       UpperCamelCase__)
			# Instantiate the model (we build the model here so that the seed also control new weights initialization)
			SCREAMING_SNAKE_CASE						= AutoModelForSequenceClassification.from_pretrained('bert-base-cased'							,       return_dict=UpperCamelCase__)

			# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
			# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
			# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
			SCREAMING_SNAKE_CASE						= model.to(accelerator.device)
			# Instantiate optimizer
			SCREAMING_SNAKE_CASE						= AdamW(params=model.parameters()							,       lr=UpperCamelCase__)

			# Instantiate scheduler
			SCREAMING_SNAKE_CASE						= get_linear_schedule_with_warmup(
			    optimizer=UpperCamelCase__							,       num_warmup_steps=100							,       num_training_steps=(len(UpperCamelCase__) * num_epochs) // gradient_accumulation_steps							,       )

			# Prepare everything
			# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
			# prepare method.

			SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						= accelerator.prepare(
			    UpperCamelCase__							,       UpperCamelCase__							,       UpperCamelCase__							,       UpperCamelCase__							,       UpperCamelCase__)

			# Now we train the model
			for epoch in range(UpperCamelCase__):
						model.train()
						for step, batch in enumerate(UpperCamelCase__):
									# We could avoid this line since we set the accelerator with `device_placement=True`.
									batch.to(accelerator.device)
									SCREAMING_SNAKE_CASE						= model(**UpperCamelCase__)
									SCREAMING_SNAKE_CASE						= outputs.loss
									SCREAMING_SNAKE_CASE						= loss / gradient_accumulation_steps
									accelerator.backward(UpperCamelCase__)
									if step % gradient_accumulation_steps == 0:
												optimizer.step()
												lr_scheduler.step()
												optimizer.zero_grad()

						model.eval()
						for step, batch in enumerate(UpperCamelCase__):
									# We could avoid this line since we set the accelerator with `device_placement=True`.
									batch.to(accelerator.device)
									with torch.no_grad():
												SCREAMING_SNAKE_CASE						= model(**UpperCamelCase__)
									SCREAMING_SNAKE_CASE						= outputs.logits.argmax(dim=-1)
									SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						= accelerator.gather_for_metrics((predictions, batch['labels']))
									metric.add_batch(
									    predictions=UpperCamelCase__							,       references=UpperCamelCase__							,       )

						SCREAMING_SNAKE_CASE						= metric.compute()
						# Use accelerator.print to print only on the main process.
						accelerator.print(F'''epoch {epoch}:'''							,       UpperCamelCase__)






def 				lowerCamelCase__				():
			SCREAMING_SNAKE_CASE						= argparse.ArgumentParser(description='Simple example of training script.')
			parser.add_argument(
			    '--mixed_precision'							,       type=UpperCamelCase__							,       default=UpperCamelCase__							,       choices=['no', 'fp16', 'bf16', 'fp8']							,       help='Whether to use mixed precision. Choose'
			    'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
			    'and an Nvidia Ampere GPU.'							,       )
			parser.add_argument('--cpu'							,       action='store_true'							,       help='If passed, will train on the CPU.')
			SCREAMING_SNAKE_CASE						= parser.parse_args()
			SCREAMING_SNAKE_CASE						= {'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
			training_function(UpperCamelCase__							,       UpperCamelCase__)


if __name__ == "__main__":
				main()












import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


a_		:						Optional[int]             =    '\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n'

a_		:						Tuple             =    '\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n'

a_		:						Optional[Any]             =    '\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for \'cvit-mkb-clsr\' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for \'cvit-mkb-clsr\' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "precision": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wnli\')  # \'wnli\' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wiki-ner\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'cvit-mkb-clsr\')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'precision@10\': 1.0}\n\n'






def 				lowerCamelCase__				(_UpperCAmelCase							,       _UpperCAmelCase):
			return float((preds == labels).mean())






def 				lowerCamelCase__				(_UpperCAmelCase							,       _UpperCAmelCase):
			SCREAMING_SNAKE_CASE						= simple_accuracy(_UpperCAmelCase							,       _UpperCAmelCase)
			SCREAMING_SNAKE_CASE						= float(fa_score(y_true=_UpperCAmelCase							,       y_pred=_UpperCAmelCase))
			return {
			    "accuracy": acc,
			    "f1": fa,
			}






def 				lowerCamelCase__				(_UpperCAmelCase							,       _UpperCAmelCase):
			SCREAMING_SNAKE_CASE						= np.array(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE						= np.array(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE						= en_sentvecs.shape[0]

			# mean centering
			SCREAMING_SNAKE_CASE						= en_sentvecs - np.mean(_UpperCAmelCase							,       axis=0)
			SCREAMING_SNAKE_CASE						= in_sentvecs - np.mean(_UpperCAmelCase							,       axis=0)

			SCREAMING_SNAKE_CASE						= cdist(_UpperCAmelCase							,       _UpperCAmelCase							,       'cosine')
			SCREAMING_SNAKE_CASE						= np.array(range(_UpperCAmelCase))
			SCREAMING_SNAKE_CASE						= sim.argsort(axis=1)[:, :10]
			SCREAMING_SNAKE_CASE						= np.any(preds == actual[:, None]							,       axis=1)
			return float(matches.mean())







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION   ,    _KWARGS_DESCRIPTION					)
class 		_snake_case  (  datasets.Metric					):






			def    SCREAMING_SNAKE_CASE__    (       self)      ->       Dict:
						if self.config_name not in [
						    "wnli",
						    "copa",
						    "sna",
						    "csqa",
						    "wstp",
						    "inltkh",
						    "bbca",
						    "cvit-mkb-clsr",
						    "iitp-mr",
						    "iitp-pr",
						    "actsa-sc",
						    "md",
						    "wiki-ner",
						]:
									raise KeyError(
									    'You should supply a configuration name selected in '
									    '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '
									    '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '
									    '"wiki-ner"]')
						return datasets.MetricInfo(
						    description=_DESCRIPTION		,					citation=_CITATION		,					inputs_description=_KWARGS_DESCRIPTION		,					features=datasets.Features(
						        {
						            'predictions': datasets.Value('int64')
						            if self.config_name != 'cvit-mkb-clsr'
						            else datasets.Sequence(datasets.Value('float32')),
						            'references': datasets.Value('int64')
						            if self.config_name != 'cvit-mkb-clsr'
						            else datasets.Sequence(datasets.Value('float32')),
						        })		,					codebase_urls=[]		,					reference_urls=[]		,					format='numpy' if self.config_name != 'cvit-mkb-clsr' else None		,					)







			def    SCREAMING_SNAKE_CASE__    (       self		,					a		,					a)      ->       List[Any]:
						if self.config_name == "cvit-mkb-clsr":
									return {"precision@10": precision_at_aa(a		,					a)}
						elif self.config_name in ["wiki-ner"]:
									return acc_and_fa(a		,					a)
						elif self.config_name in [
						    "wnli",
						    "copa",
						    "sna",
						    "csqa",
						    "wstp",
						    "inltkh",
						    "bbca",
						    "iitp-mr",
						    "iitp-pr",
						    "actsa-sc",
						    "md",
						]:
									return {"accuracy": simple_accuracy(a		,					a)}
						else:
									raise KeyError(
									    'You should supply a configuration name selected in '
									    '["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '
									    '"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '
									    '"wiki-ner"]')




