






'''simple docstring'''

import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A:      str  				=				1_6
A:      Optional[int]  				=				3_2






def 	_UpperCAmelCase    (  a		:	Accelerator						,			a		:	DatasetDict						,			a		:	List[int]						,			a		:	List[int]						,			a		:	int = 1_6							)			->			int:





						"""simple docstring"""
						lowercase_      :   Optional[Any]        =			AutoTokenizer.from_pretrained('bert-base-cased'							)
						lowercase_      :   str        =			DatasetDict(
						    {
						        'train': dataset['train'].select(snake_case__							),
						        'validation': dataset['train'].select(snake_case__							),
						        'test': dataset['validation'],
						    }							)

						def tokenize_function(a		:	Optional[Any]							):
												# max_length=None => use the model max length (it's actually the default)
												lowercase_      :   Union[str, Any]        =			tokenizer(examples['sentence1']						,			examples['sentence2']						,			truncation=snake_case__						,			max_length=snake_case__							)
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												lowercase_      :   List[Any]        =			datasets.map(
												    snake_case__						,			batched=snake_case__						,			remove_columns=['idx', 'sentence1', 'sentence2']						,			)

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						lowercase_      :   Optional[Any]        =			tokenized_datasets.rename_column('label'						,			'labels'							)

						def collate_fn(a		:	Optional[Any]							):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												lowercase_      :   Optional[int]        =			1_2_8 if accelerator.distributed_type == DistributedType.TPU else None
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		lowercase_      :   Union[str, Any]        =			1_6
												elif accelerator.mixed_precision != "no":
																		lowercase_      :   Any        =			8
												else:
																		lowercase_      :   Dict        =			None

												return tokenizer.pad(
												    snake_case__						,			padding='longest'						,			max_length=snake_case__						,			pad_to_multiple_of=snake_case__						,			return_tensors='pt'						,			)

						# Instantiate dataloaders.
						lowercase_      :   Dict        =			DataLoader(
						    tokenized_datasets['train']						,			shuffle=snake_case__						,			collate_fn=snake_case__						,			batch_size=snake_case__							)
						lowercase_      :   List[str]        =			DataLoader(
						    tokenized_datasets['validation']						,			shuffle=snake_case__						,			collate_fn=snake_case__						,			batch_size=snake_case__							)

						lowercase_      :   List[Any]        =			DataLoader(
						    tokenized_datasets['test']						,			shuffle=snake_case__						,			collate_fn=snake_case__						,			batch_size=snake_case__							)

						return train_dataloader, eval_dataloader, test_dataloader






def 	_UpperCAmelCase    (  a		:	str						,			a		:	List[str]							)			->			Union[str, Any]:





						"""simple docstring"""
						# New Code #
						lowercase_      :   Tuple        =			[]
						# Download the dataset
						lowercase_      :   Dict        =			load_dataset('glue'						,			'mrpc'							)
						# Create our splits
						lowercase_      :   Optional[Any]        =			StratifiedKFold(n_splits=int(args.num_folds							)							)
						# Initialize accelerator
						lowercase_      :   str        =			Accelerator(cpu=args.cpu						,			mixed_precision=args.mixed_precision							)
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						lowercase_      :   Dict        =			config['lr']
						lowercase_      :   str        =			int(config['num_epochs']							)
						lowercase_      :   Union[str, Any]        =			int(config['seed']							)
						lowercase_      :   Optional[int]        =			int(config['batch_size']							)

						lowercase_      :   int        =			evaluate.load('glue'						,			'mrpc'							)

						# If the batch size is too big we use gradient accumulation
						lowercase_      :   int        =			1
						if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
												lowercase_      :   Optional[int]        =			batch_size // MAX_GPU_BATCH_SIZE
												lowercase_      :   Dict        =			MAX_GPU_BATCH_SIZE

						set_seed(snake_case__							)

						# New Code #
						# Create our folds:
						lowercase_      :   List[Any]        =			kfold.split(np.zeros(datasets['train'].num_rows							)						,			datasets['train']['label']							)
						lowercase_      :   Optional[int]        =			[]
						# Iterate over them
						for i, (train_idxs, valid_idxs) in enumerate(snake_case__							):
												lowercase_  ,      lowercase_  ,      lowercase_      :   Tuple        =			get_fold_dataloaders(
												    snake_case__						,			snake_case__						,			snake_case__						,			snake_case__						,			)
												# Instantiate the model (we build the model here so that the seed also control new weights initialization)
												lowercase_      :   List[str]        =			AutoModelForSequenceClassification.from_pretrained('bert-base-cased'						,			return_dict=snake_case__							)

												# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
												# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
												# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
												lowercase_      :   Optional[int]        =			model.to(accelerator.device							)

												# Instantiate optimizer
												lowercase_      :   str        =			AdamW(params=model.parameters()						,			lr=snake_case__							)

												# Instantiate scheduler
												lowercase_      :   str        =			get_linear_schedule_with_warmup(
												    optimizer=snake_case__						,			num_warmup_steps=1_0_0						,			num_training_steps=(len(snake_case__							) * num_epochs) // gradient_accumulation_steps						,			)

												# Prepare everything
												# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
												# prepare method.
												lowercase_  ,      lowercase_  ,      lowercase_  ,      lowercase_  ,      lowercase_      :   List[str]        =			accelerator.prepare(
												    snake_case__						,			snake_case__						,			snake_case__						,			snake_case__						,			snake_case__							)

												# Now we train the model
												for epoch in range(snake_case__							):
																		model.train()
																		for step, batch in enumerate(snake_case__							):
																								# We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device							)
																								lowercase_      :   str        =			model(**snake_case__							)
																								lowercase_      :   Union[str, Any]        =			outputs.loss
																								lowercase_      :   List[Any]        =			loss / gradient_accumulation_steps
																								accelerator.backward(snake_case__							)
																								if step % gradient_accumulation_steps == 0:
																														optimizer.step()
																														lr_scheduler.step()
																														optimizer.zero_grad()

																		model.eval()
																		for step, batch in enumerate(snake_case__							):
																								# We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device							)
																								with torch.no_grad():
																														lowercase_      :   Optional[int]        =			model(**snake_case__							)
																								lowercase_      :   int        =			outputs.logits.argmax(dim=-1							)
																								lowercase_  ,      lowercase_      :   List[Any]        =			accelerator.gather_for_metrics((predictions, batch['labels'])							)
																								metric.add_batch(
																								    predictions=snake_case__						,			references=snake_case__						,			)

																		lowercase_      :   Tuple        =			metric.compute()
																		# Use accelerator.print to print only on the main process.
																		accelerator.print(f"epoch {epoch}:"						,			snake_case__							)

												# New Code #
												# We also run predictions on the test set at the very end
												lowercase_      :   str        =			[]
												for step, batch in enumerate(snake_case__							):
																		# We could avoid this line since we set the accelerator with `device_placement=True`.
																		batch.to(accelerator.device							)
																		with torch.no_grad():
																								lowercase_      :   Union[str, Any]        =			model(**snake_case__							)
																		lowercase_      :   str        =			outputs.logits
																		lowercase_  ,      lowercase_      :   int        =			accelerator.gather_for_metrics((predictions, batch['labels'])							)
																		fold_predictions.append(predictions.cpu()							)
																		if i == 0:
																								# We need all of the test predictions
																								test_references.append(references.cpu()							)
        # Use accelerator.print to print only on the main process.
												test_predictions.append(torch.cat(snake_case__						,			dim=0							)							)
												# We now need to release all our memory and get rid of the current model, optimizer, etc
												accelerator.free_memory()
						# New Code #
						# Finally we check the accuracy of our folded results:
						lowercase_      :   Union[str, Any]        =			torch.cat(snake_case__						,			dim=0							)
						lowercase_      :   Optional[int]        =			torch.stack(snake_case__						,			dim=0							).sum(dim=0							).div(int(args.num_folds							)							).argmax(dim=-1							)
						lowercase_      :   List[Any]        =			metric.compute(predictions=snake_case__						,			references=snake_case__							)
						accelerator.print('Average test metrics from all folds:'						,			snake_case__							)






def 	_UpperCAmelCase    (  )			->			Tuple:





						"""simple docstring"""
						lowercase_      :   Optional[int]        =			argparse.ArgumentParser(description='Simple example of training script.'							)
						parser.add_argument(
						    '--mixed_precision'						,			type=snake_case__						,			default=snake_case__						,			choices=['no', 'fp16', 'bf16', 'fp8']						,			help='Whether to use mixed precision. Choose'
						    'between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'
						    'and an Nvidia Ampere GPU.'						,			)
						parser.add_argument('--cpu'						,			action='store_true'						,			help='If passed, will train on the CPU.'							)
						# New Code #
						parser.add_argument('--num_folds'						,			type=snake_case__						,			default=3						,			help='The number of splits to perform across the dataset'							)
						lowercase_      :   Optional[Any]        =			parser.parse_args()
						lowercase_      :   str        =			{'lr': 2e-5, 'num_epochs': 3, 'seed': 4_2, 'batch_size': 1_6}
						training_function(snake_case__						,			snake_case__							)


if __name__ == "__main__":
			main()








'''simple docstring'''

import argparse


A:      List[Any]  				=				"docs/source/_static/js/custom.js"






def 	_UpperCAmelCase    (  a		:	Optional[Any]							)			->			Optional[Any]:





						"""simple docstring"""
						with open(a						,			encoding='utf-8'						,			newline='\n'							) as f:
												lowercase_      :   List[Any]        =			f.readlines()
						lowercase_      :   Dict        =			0

						# First let's put the right version
						while not lines[index].startswith('const stableVersion ='							):
												index += 1
						lowercase_      :   Dict        =			f"const stableVersion = \"v{version}\"\n"

						# Then update the dictionary
						while not lines[index].startswith('const versionMapping = {'							):
												index += 1

						# We go until the end
						while not lines[index].startswith('}'							):
												index += 1
						# We add the new version at the end
						lines[index - 1] += f"    \"v{version}\": \"v{version}\",\n"

						with open(a						,			'w'						,			encoding='utf-8'						,			newline='\n'							) as f:
												f.writelines(a							)


if __name__ == "__main__":
						A:      str  				=				argparse.ArgumentParser()
						parser.add_argument("--version", help="Release version.")
						A:      List[str]  				=				parser.parse_args()
						update_custom_js(args.version)
