
"""simple docstring"""






import argparse
from typing import List

import evaluate
import numpy as np
import torch
from datasets import DatasetDict, load_dataset

# New Code #
# We'll be using StratifiedKFold for this example
from sklearn.model_selection import StratifiedKFold
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to perform Cross Validation,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


lowerCamelCase__ :       List[Any]						     =			16
lowerCamelCase__ :       List[str]						     =			32





def __A						(   a_ :							Dict		,		a_ :							Optional[Any]		,		a_ :							str		,		a_ :							List[Any]		,		a_ :							str = 16     )->							List[Any]:




		'''simple docstring'''






		SCREAMING_SNAKE_CASE       :     List[str]				= AutoTokenizer.from_pretrained('''bert-base-cased'''     )
		SCREAMING_SNAKE_CASE       :     List[str]				= DatasetDict(
		    {
		        '''train''': dataset['''train'''].select(_lowercase     ),
		        '''validation''': dataset['''train'''].select(_lowercase     ),
		        '''test''': dataset['''validation'''],
		    }     )

		def tokenize_function(a_ :							Tuple     ):
				# max_length=None => use the model max length (it's actually the default)
				SCREAMING_SNAKE_CASE       :     str				= tokenizer(examples['''sentence1''']		,		examples['''sentence2''']		,		truncation=_lowercase		,		max_length=_lowercase     )
				return outputs

		# Apply the method we just defined to all the examples in all the splits of the dataset
		# starting with the main process first:
		with accelerator.main_process_first():
				SCREAMING_SNAKE_CASE       :     Dict				= datasets.map(
				    _lowercase		,		batched=_lowercase		,		remove_columns=['''idx''', '''sentence1''', '''sentence2''']		,		)

		# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
		# transformers library
		SCREAMING_SNAKE_CASE       :     Any				= tokenized_datasets.rename_column('''label'''		,		'''labels'''     )

		def collate_fn(a_ :							Optional[Any]     ):
				# On TPU it's best to pad everything to the same length or training will be very slow.
				SCREAMING_SNAKE_CASE       :     str				= 1_28 if accelerator.distributed_type == DistributedType.TPU else None
				# When using mixed precision we want round multiples of 8/16
				if accelerator.mixed_precision == "fp8":
						SCREAMING_SNAKE_CASE       :     Union[str, Any]				= 16
				elif accelerator.mixed_precision != "no":
						SCREAMING_SNAKE_CASE       :     Dict				= 8
				else:
						SCREAMING_SNAKE_CASE       :     Dict				= None

				return tokenizer.pad(
				    _lowercase		,		padding='''longest'''		,		max_length=_lowercase		,		pad_to_multiple_of=_lowercase		,		return_tensors='''pt'''		,		)

		# Instantiate dataloaders.
		SCREAMING_SNAKE_CASE       :     List[str]				= DataLoader(
		    tokenized_datasets['''train''']		,		shuffle=_lowercase		,		collate_fn=_lowercase		,		batch_size=_lowercase     )
		SCREAMING_SNAKE_CASE       :     List[Any]				= DataLoader(
		    tokenized_datasets['''validation''']		,		shuffle=_lowercase		,		collate_fn=_lowercase		,		batch_size=_lowercase     )

		SCREAMING_SNAKE_CASE       :     Dict				= DataLoader(
		    tokenized_datasets['''test''']		,		shuffle=_lowercase		,		collate_fn=_lowercase		,		batch_size=_lowercase     )

		return train_dataloader, eval_dataloader, test_dataloader





def __A						(   a_ :							Tuple		,		a_ :							Optional[int]     )->							List[Any]:




		'''simple docstring'''






		SCREAMING_SNAKE_CASE       :     str				= []
		# Download the dataset
		SCREAMING_SNAKE_CASE       :     Any				= load_dataset('''glue'''		,		'''mrpc'''     )
		# Create our splits
		SCREAMING_SNAKE_CASE       :     int				= StratifiedKFold(n_splits=int(args.num_folds     )     )
		# Initialize accelerator
		SCREAMING_SNAKE_CASE       :     List[str]				= Accelerator(cpu=args.cpu		,		mixed_precision=args.mixed_precision     )
		# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
		SCREAMING_SNAKE_CASE       :     Dict				= config['''lr''']
		SCREAMING_SNAKE_CASE       :     int				= int(config['''num_epochs''']     )
		SCREAMING_SNAKE_CASE       :     List[str]				= int(config['''seed''']     )
		SCREAMING_SNAKE_CASE       :     str				= int(config['''batch_size''']     )

		SCREAMING_SNAKE_CASE       :     Union[str, Any]				= evaluate.load('''glue'''		,		'''mrpc'''     )

		# If the batch size is too big we use gradient accumulation
		SCREAMING_SNAKE_CASE       :     int				= 1
		if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
				SCREAMING_SNAKE_CASE       :     Optional[Any]				= batch_size // MAX_GPU_BATCH_SIZE
				SCREAMING_SNAKE_CASE       :     Tuple				= MAX_GPU_BATCH_SIZE

		set_seed(_lowercase     )

		# New Code #
		# Create our folds:
		SCREAMING_SNAKE_CASE       :     List[str]				= kfold.split(np.zeros(datasets['''train'''].num_rows     )		,		datasets['''train''']['''label''']     )
		SCREAMING_SNAKE_CASE       :     Dict				= []
		# Iterate over them
		for i, (train_idxs, valid_idxs) in enumerate(_lowercase     ):
				SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     Union[str, Any]				= get_fold_dataloaders(
				    _lowercase		,		_lowercase		,		_lowercase		,		_lowercase		,		)
				# Instantiate the model (we build the model here so that the seed also control new weights initialization)
				SCREAMING_SNAKE_CASE       :     Optional[Any]				= AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''		,		return_dict=_lowercase     )

				# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
				# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
				# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
				SCREAMING_SNAKE_CASE       :     Tuple				= model.to(accelerator.device     )

				# Instantiate optimizer
				SCREAMING_SNAKE_CASE       :     List[str]				= AdamW(params=model.parameters()		,		lr=_lowercase     )

				# Instantiate scheduler
				SCREAMING_SNAKE_CASE       :     Optional[Any]				= get_linear_schedule_with_warmup(
				    optimizer=_lowercase		,		num_warmup_steps=1_00		,		num_training_steps=(len(_lowercase     ) * num_epochs) // gradient_accumulation_steps		,		)

				# Prepare everything
				# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
				# prepare method.
				SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     Union[str, Any]				= accelerator.prepare(
				    _lowercase		,		_lowercase		,		_lowercase		,		_lowercase		,		_lowercase     )

				# Now we train the model
				for epoch in range(_lowercase     ):
						model.train()
						for step, batch in enumerate(_lowercase     ):
								# We could avoid this line since we set the accelerator with `device_placement=True`.
								batch.to(accelerator.device     )
								SCREAMING_SNAKE_CASE       :     Optional[int]				= model(**_lowercase     )
								SCREAMING_SNAKE_CASE       :     int				= outputs.loss
								SCREAMING_SNAKE_CASE       :     str				= loss / gradient_accumulation_steps
								accelerator.backward(_lowercase     )
								if step % gradient_accumulation_steps == 0:
										optimizer.step()
										lr_scheduler.step()
										optimizer.zero_grad()

						model.eval()
						for step, batch in enumerate(_lowercase     ):
								# We could avoid this line since we set the accelerator with `device_placement=True`.
								batch.to(accelerator.device     )
								with torch.no_grad():
										SCREAMING_SNAKE_CASE       :     Tuple				= model(**_lowercase     )
								SCREAMING_SNAKE_CASE       :     Union[str, Any]				= outputs.logits.argmax(dim=-1     )
								SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     Union[str, Any]				= accelerator.gather_for_metrics((predictions, batch['''labels'''])     )
								metric.add_batch(
								    predictions=_lowercase		,		references=_lowercase		,		)

						SCREAMING_SNAKE_CASE       :     Optional[Any]				= metric.compute()
						# Use accelerator.print to print only on the main process.
						accelerator.print(F"epoch {epoch}:"		,		_lowercase     )

				# New Code #
				# We also run predictions on the test set at the very end
				SCREAMING_SNAKE_CASE       :     List[Any]				= []
				for step, batch in enumerate(_lowercase     ):
						# We could avoid this line since we set the accelerator with `device_placement=True`.
						batch.to(accelerator.device     )
						with torch.no_grad():
								SCREAMING_SNAKE_CASE       :     str				= model(**_lowercase     )
						SCREAMING_SNAKE_CASE       :     Dict				= outputs.logits
						SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     List[Any]				= accelerator.gather_for_metrics((predictions, batch['''labels'''])     )
						fold_predictions.append(predictions.cpu()     )
						if i == 0:
								# We need all of the test predictions
								test_references.append(references.cpu()     )
        # Use accelerator.print to print only on the main process.
				test_predictions.append(torch.cat(_lowercase		,		dim=0     )     )
				# We now need to release all our memory and get rid of the current model, optimizer, etc
				accelerator.free_memory()
		# New Code #
		# Finally we check the accuracy of our folded results:
		SCREAMING_SNAKE_CASE       :     str				= torch.cat(_lowercase		,		dim=0     )
		SCREAMING_SNAKE_CASE       :     List[str]				= torch.stack(_lowercase		,		dim=0     ).sum(dim=0     ).div(int(args.num_folds     )     ).argmax(dim=-1     )
		SCREAMING_SNAKE_CASE       :     Optional[int]				= metric.compute(predictions=_lowercase		,		references=_lowercase     )
		accelerator.print('''Average test metrics from all folds:'''		,		_lowercase     )





def __A						(   )->							Dict:




		'''simple docstring'''






		SCREAMING_SNAKE_CASE       :     List[str]				= argparse.ArgumentParser(description='''Simple example of training script.'''     )
		parser.add_argument(
		    '''--mixed_precision'''		,		type=_lowercase		,		default=_lowercase		,		choices=['''no''', '''fp16''', '''bf16''', '''fp8''']		,		help='''Whether to use mixed precision. Choose'''
		    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
		    '''and an Nvidia Ampere GPU.'''		,		)
		parser.add_argument('''--cpu'''		,		action='''store_true'''		,		help='''If passed, will train on the CPU.'''     )
		# New Code #
		parser.add_argument('''--num_folds'''		,		type=_lowercase		,		default=3		,		help='''The number of splits to perform across the dataset'''     )
		SCREAMING_SNAKE_CASE       :     Any				= parser.parse_args()
		SCREAMING_SNAKE_CASE       :     List[Any]				= {'''lr''': 2E-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
		training_function(_lowercase		,		_lowercase     )


if __name__ == "__main__":
					main()







"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


lowerCamelCase__ :       int						     =			logging.get_logger(__name__)




class 				lowercase__(				_UpperCAmelCase		,						_UpperCAmelCase   ):
	'''simple docstring'''

	UpperCamelCase				=  """maskformer-swin"""

	UpperCamelCase				=  {
	    """num_attention_heads""": """num_heads""",
	    """num_hidden_layers""": """num_layers""",
	}





	def __init__(						self    :Optional[int]		, lowerCamelCase_    :List[Any]=2_24		, lowerCamelCase_    :Tuple=4		, lowerCamelCase_    :Optional[Any]=3		, lowerCamelCase_    :List[str]=96		, lowerCamelCase_    :int=[2, 2, 6, 2]		, lowerCamelCase_    :Union[str, Any]=[3, 6, 12, 24]		, lowerCamelCase_    :Optional[int]=7		, lowerCamelCase_    :Tuple=4.0		, lowerCamelCase_    :Tuple=True		, lowerCamelCase_    :Dict=0.0		, lowerCamelCase_    :Any=0.0		, lowerCamelCase_    :List[Any]=0.1		, lowerCamelCase_    :Dict="gelu"		, lowerCamelCase_    :Optional[int]=False		, lowerCamelCase_    :List[str]=0.0_2		, lowerCamelCase_    :Any=1E-5		, lowerCamelCase_    :Union[str, Any]=None		, lowerCamelCase_    :List[str]=None		, **lowerCamelCase_    :Union[str, Any]		, )							->			Dict:







			'''simple docstring'''




			super().__init__(**lowerCamelCase_					)

			SCREAMING_SNAKE_CASE       :     int				= image_size
			SCREAMING_SNAKE_CASE       :     Optional[Any]				= patch_size
			SCREAMING_SNAKE_CASE       :     str				= num_channels
			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= embed_dim
			SCREAMING_SNAKE_CASE       :     List[Any]				= depths
			SCREAMING_SNAKE_CASE       :     List[str]				= len(lowerCamelCase_					)
			SCREAMING_SNAKE_CASE       :     str				= num_heads
			SCREAMING_SNAKE_CASE       :     Any				= window_size
			SCREAMING_SNAKE_CASE       :     List[str]				= mlp_ratio
			SCREAMING_SNAKE_CASE       :     str				= qkv_bias
			SCREAMING_SNAKE_CASE       :     Tuple				= hidden_dropout_prob
			SCREAMING_SNAKE_CASE       :     int				= attention_probs_dropout_prob
			SCREAMING_SNAKE_CASE       :     int				= drop_path_rate
			SCREAMING_SNAKE_CASE       :     Tuple				= hidden_act
			SCREAMING_SNAKE_CASE       :     Any				= use_absolute_embeddings
			SCREAMING_SNAKE_CASE       :     List[str]				= layer_norm_eps
			SCREAMING_SNAKE_CASE       :     List[str]				= initializer_range
			# we set the hidden_size attribute in order to make Swin work with VisionEncoderDecoderModel
			# this indicates the channel dimension after the last stage of the model
			SCREAMING_SNAKE_CASE       :     int				= int(embed_dim * 2 ** (len(lowerCamelCase_					) - 1)					)
			SCREAMING_SNAKE_CASE       :     Dict				= ['''stem'''] + [f"stage{idx}" for idx in range(1		, len(lowerCamelCase_					) + 1					)]
			SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     int				= get_aligned_output_features_output_indices(
			    out_features=lowerCamelCase_		, out_indices=lowerCamelCase_		, stage_names=self.stage_names					)





