




"""simple docstring"""





import argparse

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


A   =       16
A   =       32

def 					_UpperCamelCase    (			UpperCamelCase				,       UpperCamelCase = 16							)			->			Union[str, Any]:


			"""simple docstring"""
			__UpperCAmelCase	: Tuple		=					AutoTokenizer.from_pretrained("bert-base-cased"							)
			__UpperCAmelCase	: str		=					load_dataset("glue"				,       "mrpc"							)

			def tokenize_function(UpperCamelCase							):
						# max_length=None => use the model max length (it's actually the default)
						__UpperCAmelCase	: Optional[Any]		=					tokenizer(examples["sentence1"]				,       examples["sentence2"]				,       truncation=UpperCamelCase				,       max_length=UpperCamelCase							)
						return outputs

			# Apply the method we just defined to all the examples in all the splits of the dataset
			# starting with the main process first:
			with accelerator.main_process_first():
						__UpperCAmelCase	: Dict		=					datasets.map(
						    UpperCamelCase				,       batched=UpperCamelCase				,       remove_columns=["idx", "sentence1", "sentence2"]				,       )

			# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
			# transformers library
			__UpperCAmelCase	: int		=					tokenized_datasets.rename_column("label"				,       "labels"							)

			def collate_fn(UpperCamelCase							):
						# On TPU it's best to pad everything to the same length or training will be very slow.
						__UpperCAmelCase	: Optional[int]		=					128 if accelerator.distributed_type == DistributedType.TPU else None
						# When using mixed precision we want round multiples of 8/16
						if accelerator.mixed_precision == "fp8":
									__UpperCAmelCase	: Union[str, Any]		=					16
						elif accelerator.mixed_precision != "no":
									__UpperCAmelCase	: Optional[Any]		=					8
						else:
									__UpperCAmelCase	: str		=					None

						return tokenizer.pad(
						    UpperCamelCase				,       padding="longest"				,       max_length=UpperCamelCase				,       pad_to_multiple_of=UpperCamelCase				,       return_tensors="pt"				,       )

			# Instantiate dataloaders.
			__UpperCAmelCase	: List[Any]		=					DataLoader(
			    tokenized_datasets["train"]				,       shuffle=UpperCamelCase				,       collate_fn=UpperCamelCase				,       batch_size=UpperCamelCase				,       drop_last=UpperCamelCase							)
			__UpperCAmelCase	: Optional[int]		=					DataLoader(
			    tokenized_datasets["validation"]				,       shuffle=UpperCamelCase				,       collate_fn=UpperCamelCase				,       batch_size=UpperCamelCase				,       drop_last=(accelerator.mixed_precision == "fp8")				,       )

			return train_dataloader, eval_dataloader

def 					_UpperCamelCase    (			UpperCamelCase				,       UpperCamelCase							)			->			Tuple:


			"""simple docstring"""
			# Initialize accelerator
			__UpperCAmelCase	: int		=					Accelerator(cpu=args.cpu				,       mixed_precision=args.mixed_precision							)
			# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
			__UpperCAmelCase	: int		=					config["lr"]
			__UpperCAmelCase	: Any		=					int(config["num_epochs"]							)
			__UpperCAmelCase	: Union[str, Any]		=					int(config["seed"]							)
			__UpperCAmelCase	: Dict		=					int(config["batch_size"]							)

			__UpperCAmelCase	: str		=					evaluate.load("glue"				,       "mrpc"							)

			# If the batch size is too big we use gradient accumulation
			__UpperCAmelCase	: List[str]		=					1
			if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
						__UpperCAmelCase	: Tuple		=					batch_size // MAX_GPU_BATCH_SIZE
						__UpperCAmelCase	: str		=					MAX_GPU_BATCH_SIZE

			set_seed(UpperCamelCase							)
			__UpperCAmelCase	,     __UpperCAmelCase	: Tuple		=					get_dataloaders(UpperCamelCase				,       UpperCamelCase							)
			# Instantiate the model (we build the model here so that the seed also control new weights initialization)
			__UpperCAmelCase	: str		=					AutoModelForSequenceClassification.from_pretrained("bert-base-cased"				,       return_dict=UpperCamelCase							)

			# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
			# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
			# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
			__UpperCAmelCase	: Tuple		=					model.to(accelerator.device							)
			# Instantiate optimizer
			__UpperCAmelCase	: Union[str, Any]		=					AdamW(params=model.parameters()				,       lr=UpperCamelCase							)

			# Instantiate scheduler
			__UpperCAmelCase	: List[str]		=					get_linear_schedule_with_warmup(
			    optimizer=UpperCamelCase				,       num_warmup_steps=100				,       num_training_steps=(len(UpperCamelCase							) * num_epochs) // gradient_accumulation_steps				,       )

			# Prepare everything
			# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
			# prepare method.

			__UpperCAmelCase	,     __UpperCAmelCase	,     __UpperCAmelCase	,     __UpperCAmelCase	,     __UpperCAmelCase	: str		=					accelerator.prepare(
			    UpperCamelCase				,       UpperCamelCase				,       UpperCamelCase				,       UpperCamelCase				,       UpperCamelCase							)

			# Now we train the model
			for epoch in range(UpperCamelCase							):
						model.train()
						for step, batch in enumerate(UpperCamelCase							):
									# We could avoid this line since we set the accelerator with `device_placement=True`.
									batch.to(accelerator.device							)
									__UpperCAmelCase	: Optional[int]		=					model(**UpperCamelCase							)
									__UpperCAmelCase	: Union[str, Any]		=					outputs.loss
									__UpperCAmelCase	: Union[str, Any]		=					loss / gradient_accumulation_steps
									accelerator.backward(UpperCamelCase							)
									if step % gradient_accumulation_steps == 0:
												optimizer.step()
												lr_scheduler.step()
												optimizer.zero_grad()

						model.eval()
						for step, batch in enumerate(UpperCamelCase							):
									# We could avoid this line since we set the accelerator with `device_placement=True`.
									batch.to(accelerator.device							)
									with torch.no_grad():
												__UpperCAmelCase	: Union[str, Any]		=					model(**UpperCamelCase							)
									__UpperCAmelCase	: Optional[int]		=					outputs.logits.argmax(dim=-1							)
									__UpperCAmelCase	,     __UpperCAmelCase	: int		=					accelerator.gather_for_metrics((predictions, batch["labels"])							)
									metric.add_batch(
									    predictions=UpperCamelCase				,       references=UpperCamelCase				,       )

						__UpperCAmelCase	: int		=					metric.compute()
						# Use accelerator.print to print only on the main process.
						accelerator.print(f"epoch {epoch}:"				,       UpperCamelCase							)

def 					_UpperCamelCase    (			)			->			Dict:


			"""simple docstring"""
			__UpperCAmelCase	: int		=					argparse.ArgumentParser(description="Simple example of training script."							)
			parser.add_argument(
			    "--mixed_precision"				,       type=UpperCamelCase				,       default=UpperCamelCase				,       choices=["no", "fp16", "bf16", "fp8"]				,       help="Whether to use mixed precision. Choose"
			    "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
			    "and an Nvidia Ampere GPU."				,       )
			parser.add_argument("--cpu"				,       action="store_true"				,       help="If passed, will train on the CPU."							)
			__UpperCAmelCase	: Union[str, Any]		=					parser.parse_args()
			__UpperCAmelCase	: Optional[Any]		=					{"lr": 2e-5, "num_epochs": 3, "seed": 42, "batch_size": 16}
			training_function(UpperCamelCase				,       UpperCamelCase							)


if __name__ == "__main__":
					main()








"""simple docstring"""





import os
import re
import shutil
import sys
import tempfile
import unittest

import black


A   =       os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, """utils"""))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
A   =       """    def __init__(self, config):
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = nn.Parameter(torch.zeros(config.vocab_size))

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states
"""



class      a__				(	unittest.TestCase   ):


			def       a_     (      self      :    Optional[Any]):


						"""simple docstring"""




						__UpperCAmelCase	: List[Any]		=					tempfile.mkdtemp()
						os.makedirs(os.path.join(self.transformer_dir      ,  "models/bert/"))
						__UpperCAmelCase	: Tuple		=					self.transformer_dir
						shutil.copy(
						    os.path.join(UpperCamelCase_      ,  "src/transformers/models/bert/modeling_bert.py")      ,  os.path.join(self.transformer_dir      ,  "models/bert/modeling_bert.py")      ,  )


			def       a_     (      self      :    int):


						"""simple docstring"""




						__UpperCAmelCase	: Optional[Any]		=					"src/transformers"
						shutil.rmtree(self.transformer_dir)


			def       a_     (      self      :    str      ,  UpperCamelCase_      :    str      ,  UpperCamelCase_      :    int      ,  UpperCamelCase_      :    Optional[int]      ,  UpperCamelCase_      :    List[str]=None):


						"""simple docstring"""




						__UpperCAmelCase	: str		=					comment + F"\nclass {class_name}(nn.Module):\n" + class_code
						if overwrite_result is not None:
									__UpperCAmelCase	: str		=					comment + F"\nclass {class_name}(nn.Module):\n" + overwrite_result
						__UpperCAmelCase	: Optional[int]		=					black.Mode(target_versions={black.TargetVersion.PYaa}      ,  line_length=119)
						__UpperCAmelCase	: Dict		=					black.format_str(UpperCamelCase_      ,  mode=UpperCamelCase_)
						__UpperCAmelCase	: Dict		=					os.path.join(self.transformer_dir      ,  "new_code.py")
						with open(UpperCamelCase_      ,  "w"      ,  newline="\n") as f:
									f.write(UpperCamelCase_)
						if overwrite_result is None:
									self.assertTrue(len(check_copies.is_copy_consistent(UpperCamelCase_)) == 0)
						else:
									check_copies.is_copy_consistent(f.name      ,  overwrite=UpperCamelCase_)
									with open(UpperCamelCase_      ,  "r") as f:
												self.assertTrue(f.read()      ,  UpperCamelCase_)


			def       a_     (      self      :    Tuple):


						"""simple docstring"""




						__UpperCAmelCase	: Any		=					check_copies.find_code_in_transformers("models.bert.modeling_bert.BertLMPredictionHead")
						self.assertEqual(UpperCamelCase_      ,  UpperCamelCase_)


			def       a_     (      self      :    Optional[int]):


						"""simple docstring"""




						self.check_copy_consistency(
						    "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"      ,  "BertLMPredictionHead"      ,  REFERENCE_CODE + "\n"      ,  )

						# With no empty line at the end
						self.check_copy_consistency(
						    "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"      ,  "BertLMPredictionHead"      ,  UpperCamelCase_      ,  )

						# Copy consistency with rename
						self.check_copy_consistency(
						    "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"      ,  "TestModelLMPredictionHead"      ,  re.sub("Bert"      ,  "TestModel"      ,  UpperCamelCase_)      ,  )

						# Copy consistency with a really long name
						__UpperCAmelCase	: Optional[Any]		=					"TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"
						self.check_copy_consistency(
						    F"# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}"      ,  F"{long_class_name}LMPredictionHead"      ,  re.sub("Bert"      ,  UpperCamelCase_      ,  UpperCamelCase_)      ,  )

						# Copy consistency with overwrite
						self.check_copy_consistency(
						    "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"      ,  "TestModelLMPredictionHead"      ,  UpperCamelCase_      ,  overwrite_result=re.sub("Bert"      ,  "TestModel"      ,  UpperCamelCase_)      ,  )


			def       a_     (      self      :    Dict):


						"""simple docstring"""




						__UpperCAmelCase	: List[str]		=					check_copies.LOCALIZED_READMES["README_zh-hans.md"]

						__UpperCAmelCase	: str		=					(
						    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"
						    " Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"
						    " Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"
						    " Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."
						    " **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"
						    " released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"
						    " lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"
						    " method has been applied to compress GPT2 into"
						    " [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"
						    " [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"
						    " Multilingual BERT into"
						    " [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"
						    " version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"
						    " (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"
						    " as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"
						    " Luong, Quoc V. Le, Christopher D. Manning."
						)
						__UpperCAmelCase	: List[str]		=					(
						    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
						    " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
						    " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
						    " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
						)
						__UpperCAmelCase	: str		=					(
						    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
						    " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
						    " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
						    " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."
						    " **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"
						    " [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"
						    " lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"
						    " method has been applied to compress GPT2 into"
						    " [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"
						    " [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"
						    " Multilingual BERT into"
						    " [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"
						    " version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"
						    " Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"
						    " than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"
						    " Christopher D. Manning 发布。\n"
						)

						__UpperCAmelCase	,     __UpperCAmelCase	: Any		=					check_copies.convert_to_localized_md(
						    UpperCamelCase_      ,  UpperCamelCase_      ,  localized_readme["format_model_list"])

						self.assertFalse(UpperCamelCase_)
						self.assertEqual(UpperCamelCase_      ,  UpperCamelCase_)

						__UpperCAmelCase	,     __UpperCAmelCase	: int		=					check_copies.convert_to_localized_md(
						    UpperCamelCase_      ,  UpperCamelCase_      ,  localized_readme["format_model_list"])

						# Check whether the number of models is equal to README.md after conversion.
						self.assertTrue(UpperCamelCase_)

						__UpperCAmelCase	: str		=					(
						    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"
						    " Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"
						    " Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"
						    " Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."
						)
						__UpperCAmelCase	: Optional[int]		=					(
						    "1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"
						    " the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
						    " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
						    " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
						)
						__UpperCAmelCase	: int		=					(
						    "1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"
						    " Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"
						    " Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"
						    " Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"
						)

						__UpperCAmelCase	,     __UpperCAmelCase	: List[str]		=					check_copies.convert_to_localized_md(
						    UpperCamelCase_      ,  UpperCamelCase_      ,  localized_readme["format_model_list"])

						# Check if the model link is synchronized.
						self.assertEqual(UpperCamelCase_      ,  UpperCamelCase_)


