


'''simple docstring'''



import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing how to properly calculate the metrics on the
# validation dataset when in a distributed system, and builds off the
# `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


_lowercase    :    Tuple			     =  16
_lowercase    :    Tuple			     =  32


def    lowerCamelCase__   ( A     :    Accelerator   ,     A     :    int = 16  ):




						'''simple docstring'''






						UpperCAmelCase         =			AutoTokenizer.from_pretrained('''bert-base-cased'''  )
						UpperCAmelCase         =			load_dataset('''glue'''   ,     '''mrpc'''  )

						def tokenize_function(A     :    Any  ):
												# max_length=None => use the model max length (it's actually the default)
												UpperCAmelCase         =			tokenizer(examples['''sentence1''']   ,     examples['''sentence2''']   ,     truncation=A   ,     max_length=A  )
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												UpperCAmelCase         =			datasets.map(
												    A   ,     batched=A   ,     remove_columns=['''idx''', '''sentence1''', '''sentence2''']   ,     )

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						UpperCAmelCase         =			tokenized_datasets.rename_column('''label'''   ,     '''labels'''  )

						def collate_fn(A     :    Optional[Any]  ):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												UpperCAmelCase         =			1_28 if accelerator.distributed_type == DistributedType.TPU else None
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		UpperCAmelCase         =			16
												elif accelerator.mixed_precision != "no":
																		UpperCAmelCase         =			8
												else:
																		UpperCAmelCase         =			None

												return tokenizer.pad(
												    A   ,     padding='''longest'''   ,     max_length=A   ,     pad_to_multiple_of=A   ,     return_tensors='''pt'''   ,     )

						# Instantiate dataloaders.
						UpperCAmelCase         =			DataLoader(
						    tokenized_datasets['''train''']   ,     shuffle=A   ,     collate_fn=A   ,     batch_size=A  )
						UpperCAmelCase         =			DataLoader(
						    tokenized_datasets['''validation''']   ,     shuffle=A   ,     collate_fn=A   ,     batch_size=A  )

						return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("""TESTING_MOCKED_DATALOADERS""", None) == "1":
	from accelerate.test_utils.training import mocked_dataloaders

	_lowercase    :    Union[str, Any]			     =  mocked_dataloaders  # noqa: F811


def    lowerCamelCase__   ( A     :    Dict   ,     A     :    int  ):




						'''simple docstring'''






						if os.environ.get('''TESTING_MOCKED_DATALOADERS'''   ,     A  ) == "1":
												UpperCAmelCase         =			2
						# Initialize accelerator
						UpperCAmelCase         =			Accelerator(cpu=args.cpu   ,     mixed_precision=args.mixed_precision  )
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						UpperCAmelCase         =			config['''lr''']
						UpperCAmelCase         =			int(config['''num_epochs''']  )
						UpperCAmelCase         =			int(config['''seed''']  )
						UpperCAmelCase         =			int(config['''batch_size''']  )

						UpperCAmelCase         =			evaluate.load('''glue'''   ,     '''mrpc'''  )

						# If the batch size is too big we use gradient accumulation
						UpperCAmelCase         =			1
						if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
												UpperCAmelCase         =			batch_size // MAX_GPU_BATCH_SIZE
												UpperCAmelCase         =			MAX_GPU_BATCH_SIZE

						set_seed(A  )
						UpperCAmelCase       ,       UpperCAmelCase         =			get_dataloaders(A   ,     A  )
						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						UpperCAmelCase         =			AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''   ,     return_dict=A  )

						# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
						# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
						# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
						UpperCAmelCase         =			model.to(accelerator.device  )

						# Instantiate optimizer
						UpperCAmelCase         =			AdamW(params=model.parameters()   ,     lr=A  )

						# Instantiate scheduler
						UpperCAmelCase         =			get_linear_schedule_with_warmup(
						    optimizer=A   ,     num_warmup_steps=1_00   ,     num_training_steps=(len(A  ) * num_epochs) // gradient_accumulation_steps   ,     )

						# Prepare everything
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase       ,       UpperCAmelCase         =			accelerator.prepare(
						    A   ,     A   ,     A   ,     A   ,     A  )

						# Now we train the model
						for epoch in range(A  ):
												model.train()
												for step, batch in enumerate(A  ):
																		# We could avoid this line since we set the accelerator with `device_placement=True`.
																		batch.to(accelerator.device  )
																		UpperCAmelCase         =			model(**A  )
																		UpperCAmelCase         =			outputs.loss
																		UpperCAmelCase         =			loss / gradient_accumulation_steps
																		accelerator.backward(A  )
																		if step % gradient_accumulation_steps == 0:
																								optimizer.step()
																								lr_scheduler.step()
																								optimizer.zero_grad()

												model.eval()
												UpperCAmelCase         =			0
												for step, batch in enumerate(A  ):
																		# We could avoid this line since we set the accelerator with `device_placement=True`.
																		batch.to(accelerator.device  )
																		with torch.no_grad():
																								UpperCAmelCase         =			model(**A  )
																		UpperCAmelCase         =			outputs.logits.argmax(dim=-1  )
																		UpperCAmelCase       ,       UpperCAmelCase         =			accelerator.gather((predictions, batch['''labels'''])  )
																		# New Code #
																		# First we check if it's a distributed system
																		if accelerator.use_distributed:
																								# Then see if we're on the last batch of our eval dataloader
																								if step == len(A  ) - 1:
																														# Last batch needs to be truncated on distributed systems as it contains additional samples
																														UpperCAmelCase         =			predictions[: len(eval_dataloader.dataset  ) - samples_seen]
																														UpperCAmelCase         =			references[: len(eval_dataloader.dataset  ) - samples_seen]
																								else:
																														# Otherwise we add the number of samples seen
																														samples_seen += references.shape[0]
            # All of this can be avoided if you use `Accelerator.gather_for_metrics` instead of `Accelerator.gather`:
            # accelerator.gather_for_metrics((predictions, batch["labels"]))
																		metric.add_batch(
																		    predictions=A   ,     references=A   ,     )

												UpperCAmelCase         =			metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(f"""epoch {epoch}:"""   ,     A  )


def    lowerCamelCase__   ( ):




						'''simple docstring'''






						UpperCAmelCase         =			argparse.ArgumentParser(description='''Simple example of training script.'''  )
						parser.add_argument(
						    '''--mixed_precision'''   ,     type=A   ,     default=A   ,     choices=['''no''', '''fp16''', '''bf16''', '''fp8''']   ,     help='''Whether to use mixed precision. Choose'''
						    '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
						    '''and an Nvidia Ampere GPU.'''   ,     )
						parser.add_argument('''--cpu'''   ,     action='''store_true'''   ,     help='''If passed, will train on the CPU.'''  )
						UpperCAmelCase         =			parser.parse_args()
						UpperCAmelCase         =			{'''lr''': 2E-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
						training_function(A   ,     A  )


if __name__ == "__main__":
	main()









'''simple docstring'''



import argparse
import os
import re

import packaging.version


_lowercase    :    Optional[int]			     =  """examples/"""
_lowercase    :    str			     =  {
    """examples""": (re.compile(r"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), """check_min_version(\"VERSION\")\n"""),
    """init""": (re.compile(r"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), """__version__ = \"VERSION\"\n"""),
    """setup""": (re.compile(r"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), r"""\1version=\"VERSION\","""),
    """doc""": (re.compile(r"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), """release = \"VERSION\"\n"""),
}
_lowercase    :    Dict			     =  {
    """init""": """src/transformers/__init__.py""",
    """setup""": """setup.py""",
}
_lowercase    :    List[Any]			     =  """README.md"""


def    lowerCamelCase__   ( A     :    int   ,     A     :    str   ,     A     :    Optional[Any]  ):




						'''simple docstring'''






						with open(A   ,     '''r'''   ,     encoding='''utf-8'''   ,     newline='''\n'''  ) as f:
												UpperCAmelCase         =			f.read()
						UpperCAmelCase       ,       UpperCAmelCase         =			REPLACE_PATTERNS[pattern]
						UpperCAmelCase         =			replace.replace('''VERSION'''   ,     A  )
						UpperCAmelCase         =			re_pattern.sub(A   ,     A  )
						with open(A   ,     '''w'''   ,     encoding='''utf-8'''   ,     newline='''\n'''  ) as f:
												f.write(A  )


def    lowerCamelCase__   ( A     :    Optional[int]  ):




						'''simple docstring'''






						for folder, directories, fnames in os.walk(A  ):
												# Removing some of the folders with non-actively maintained examples from the walk
												if "research_projects" in directories:
																		directories.remove('''research_projects'''  )
												if "legacy" in directories:
																		directories.remove('''legacy'''  )
												for fname in fnames:
																		if fname.endswith('''.py'''  ):
																								update_version_in_file(os.path.join(A   ,     A  )   ,     A   ,     pattern='''examples'''  )


def    lowerCamelCase__   ( A     :    str   ,     A     :    Dict=False  ):




						'''simple docstring'''






						for pattern, fname in REPLACE_FILES.items():
												update_version_in_file(A   ,     A   ,     A  )
						if not patch:
												update_version_in_examples(A  )


def    lowerCamelCase__   ( ):




						'''simple docstring'''






						UpperCAmelCase         =			'''🤗 Transformers currently provides the following architectures'''
						UpperCAmelCase         =			'''1. Want to contribute a new model?'''
						with open(A   ,     '''r'''   ,     encoding='''utf-8'''   ,     newline='''\n'''  ) as f:
												UpperCAmelCase         =			f.readlines()

						# Find the start of the list.
						UpperCAmelCase         =			0
						while not lines[start_index].startswith(_start_prompt  ):
												start_index += 1
						start_index += 1

						UpperCAmelCase         =			start_index
						# Update the lines in the model list.
						while not lines[index].startswith(_end_prompt  ):
												if lines[index].startswith('''1.'''  ):
																		UpperCAmelCase         =			lines[index].replace(
																		    '''https://huggingface.co/docs/transformers/main/model_doc'''   ,     '''https://huggingface.co/docs/transformers/model_doc'''   ,     )
												index += 1

						with open(A   ,     '''w'''   ,     encoding='''utf-8'''   ,     newline='''\n'''  ) as f:
												f.writelines(A  )


def    lowerCamelCase__   ( ):




						'''simple docstring'''






						with open(REPLACE_FILES['''init''']   ,     '''r'''  ) as f:
												UpperCAmelCase         =			f.read()
						UpperCAmelCase         =			REPLACE_PATTERNS['''init'''][0].search(A  ).groups()[0]
						return packaging.version.parse(A  )


def    lowerCamelCase__   ( A     :    Tuple=False  ):




						'''simple docstring'''






						UpperCAmelCase         =			get_version()
						if patch and default_version.is_devrelease:
												raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!'''  )
						if default_version.is_devrelease:
												UpperCAmelCase         =			default_version.base_version
						elif patch:
												UpperCAmelCase         =			f"""{default_version.major}.{default_version.minor}.{default_version.micro + 1}"""
						else:
												UpperCAmelCase         =			f"""{default_version.major}.{default_version.minor + 1}.0"""

						# Now let's ask nicely if that's the right one.
						UpperCAmelCase         =			input(f"""Which version are you releasing? [{default_version}]"""  )
						if len(A  ) == 0:
												UpperCAmelCase         =			default_version

						print(f"""Updating version to {version}."""  )
						global_version_update(A   ,     patch=A  )
						if not patch:
												print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''  )
												clean_main_ref_in_model_list()


def    lowerCamelCase__   ( ):




						'''simple docstring'''






						UpperCAmelCase         =			get_version()
						UpperCAmelCase         =			f"""{current_version.major}.{current_version.minor + 1}.0.dev0"""
						UpperCAmelCase         =			current_version.base_version

						# Check with the user we got that right.
						UpperCAmelCase         =			input(f"""Which version are we developing now? [{dev_version}]"""  )
						if len(A  ) == 0:
												UpperCAmelCase         =			dev_version

						print(f"""Updating version to {version}."""  )
						global_version_update(A  )
						print('''Cleaning main README, don\'t forget to run `make fix-copies`.'''  )
						clean_main_ref_in_model_list()


if __name__ == "__main__":
	_lowercase    :    Optional[Any]			     =  argparse.ArgumentParser()
	parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
	parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
	_lowercase    :    Union[str, Any]			     =  parser.parse_args()
	if not args.post_release:
		pre_release_work(patch=args.patch)
	elif args.patch:
		print("""Nothing to do after a patch :-)""")
	else:
		post_release_work()





