



"""simple docstring"""





import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.local_sgd import LocalSGD


########################################################################
# This is a fully working simple example to use Accelerate
# with LocalSGD, which is a method to synchronize model
# parameters every K batches. It is different, but complementary
# to gradient accumulation.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


__lowerCAmelCase       :					Optional[int]      					=					16
__lowerCAmelCase       :					List[str]      					=					32
def 	__lowerCAmelCase						(		__UpperCamelCase      :			Accelerator					, __UpperCamelCase      :			int = 1_6    ):


						'''simple docstring'''






						snake_case_   : List[Any]						=			AutoTokenizer.from_pretrained("""bert-base-cased"""    )
						snake_case_   : Union[str, Any]						=			load_dataset("""glue"""					, """mrpc"""    )

						def tokenize_function(__UpperCamelCase      :			Tuple    ):
												# max_length=None => use the model max length (it's actually the default)
												snake_case_   : Union[str, Any]						=			tokenizer(examples["""sentence1"""]					, examples["""sentence2"""]					, truncation=__UpperCamelCase					, max_length=__UpperCamelCase    )
												return outputs

						# Apply the method we just defined to all the examples in all the splits of the dataset
						# starting with the main process first:
						with accelerator.main_process_first():
												snake_case_   : List[str]						=			datasets.map(
												    __UpperCamelCase					, batched=__UpperCamelCase					, remove_columns=["""idx""", """sentence1""", """sentence2"""]					, )

						# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
						# transformers library
						snake_case_   : Tuple						=			tokenized_datasets.rename_column("""label"""					, """labels"""    )

						def collate_fn(__UpperCamelCase      :			List[Any]    ):
												# On TPU it's best to pad everything to the same length or training will be very slow.
												snake_case_   : Tuple						=			1_2_8 if accelerator.distributed_type == DistributedType.TPU else None
												# When using mixed precision we want round multiples of 8/16
												if accelerator.mixed_precision == "fp8":
																		snake_case_   : Any						=			1_6
												elif accelerator.mixed_precision != "no":
																		snake_case_   : Any						=			8
												else:
																		snake_case_   : str						=			None

												return tokenizer.pad(
												    __UpperCamelCase					, padding="""longest"""					, max_length=__UpperCamelCase					, pad_to_multiple_of=__UpperCamelCase					, return_tensors="""pt"""					, )

						# Instantiate dataloaders.
						snake_case_   : List[str]						=			DataLoader(
						    tokenized_datasets["""train"""]					, shuffle=__UpperCamelCase					, collate_fn=__UpperCamelCase					, batch_size=__UpperCamelCase    )
						snake_case_   : List[Any]						=			DataLoader(
						    tokenized_datasets["""validation"""]					, shuffle=__UpperCamelCase					, collate_fn=__UpperCamelCase					, batch_size=__UpperCamelCase    )

						return train_dataloader, eval_dataloader


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
					from accelerate.test_utils.training import mocked_dataloaders

					__lowerCAmelCase       :					Optional[Any]      					=					mocked_dataloaders  # noqa: F811
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int					, __UpperCamelCase      :			Optional[Any]    ):


						'''simple docstring'''






						if os.environ.get("""TESTING_MOCKED_DATALOADERS"""					, __UpperCamelCase    ) == "1":
												snake_case_   : Union[str, Any]						=			2
						# New Code #
						snake_case_   : Any						=			int(args.gradient_accumulation_steps    )
						snake_case_   : List[Any]						=			int(args.local_sgd_steps    )
						# Initialize accelerator
						snake_case_   : Dict						=			Accelerator(
						    cpu=args.cpu					, mixed_precision=args.mixed_precision					, gradient_accumulation_steps=__UpperCamelCase    )
						if accelerator.distributed_type not in [DistributedType.NO, DistributedType.MULTI_CPU, DistributedType.MULTI_GPU]:
												raise NotImplementedError("""LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)"""    )
						# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
						snake_case_   : List[str]						=			config["""lr"""]
						snake_case_   : int						=			int(config["""num_epochs"""]    )
						snake_case_   : List[Any]						=			int(config["""seed"""]    )
						snake_case_   : int						=			int(config["""batch_size"""]    )

						snake_case_   : Union[str, Any]						=			evaluate.load("""glue"""					, """mrpc"""    )

						set_seed(__UpperCamelCase    )
						snake_case_			,		snake_case_   : str						=			get_dataloaders(__UpperCamelCase					, __UpperCamelCase    )
						# Instantiate the model (we build the model here so that the seed also control new weights initialization)
						snake_case_   : List[Any]						=			AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""					, return_dict=__UpperCamelCase    )

						# We could avoid this line since the accelerator is set with `device_placement=True` (default value).
						# Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
						# creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
						snake_case_   : Optional[Any]						=			model.to(accelerator.device    )

						# Instantiate optimizer
						snake_case_   : List[str]						=			AdamW(params=model.parameters()					, lr=__UpperCamelCase    )

						# Instantiate scheduler
						snake_case_   : int						=			get_linear_schedule_with_warmup(
						    optimizer=__UpperCamelCase					, num_warmup_steps=1_0_0					, num_training_steps=(len(__UpperCamelCase    ) * num_epochs)					, )

						# Prepare everything
						# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
						# prepare method.
						snake_case_			,		snake_case_			,		snake_case_			,		snake_case_			,		snake_case_   : Optional[Any]						=			accelerator.prepare(
						    __UpperCamelCase					, __UpperCamelCase					, __UpperCamelCase					, __UpperCamelCase					, __UpperCamelCase    )

						# Now we train the model
						for epoch in range(__UpperCamelCase    ):
												model.train()
												with LocalSGD(
												    accelerator=__UpperCamelCase					, model=__UpperCamelCase					, local_sgd_steps=__UpperCamelCase					, enabled=local_sgd_steps is not None    ) as local_sgd:
																		for step, batch in enumerate(__UpperCamelCase    ):
																								# We could avoid this line since we set the accelerator with `device_placement=True`.
																								batch.to(accelerator.device    )
																								# New code #
																								# We use the new `accumulate` context manager to perform gradient accumulation
																								# We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
																								with accelerator.accumulate(__UpperCamelCase    ):
																														snake_case_   : List[Any]						=			model(**__UpperCamelCase    )
																														snake_case_   : Dict						=			output.loss
																														accelerator.backward(__UpperCamelCase    )
																														optimizer.step()
																														lr_scheduler.step()
																														optimizer.zero_grad()
																														# LocalSGD-specific line
																														local_sgd.step()

												model.eval()
												for step, batch in enumerate(__UpperCamelCase    ):
																		# We could avoid this line since we set the accelerator with `device_placement=True`.
																		batch.to(accelerator.device    )
																		with torch.no_grad():
																								snake_case_   : Optional[int]						=			model(**__UpperCamelCase    )
																		snake_case_   : Optional[Any]						=			outputs.logits.argmax(dim=-1    )
																		snake_case_			,		snake_case_   : Optional[Any]						=			accelerator.gather_for_metrics((predictions, batch["""labels"""])    )
																		metric.add_batch(
																		    predictions=__UpperCamelCase					, references=__UpperCamelCase					, )

												snake_case_   : List[Any]						=			metric.compute()
												# Use accelerator.print to print only on the main process.
												accelerator.print(F'epoch {epoch}:'					, __UpperCamelCase    )
def 	__lowerCAmelCase						(		):


						'''simple docstring'''






						snake_case_   : Tuple						=			argparse.ArgumentParser(description="""Simple example of training script."""    )
						parser.add_argument(
						    """--mixed_precision"""					, type=__UpperCamelCase					, default=__UpperCamelCase					, choices=["""no""", """fp16""", """bf16""", """fp8"""]					, help="""Whether to use mixed precision. Choose"""
						    """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
						    """and an Nvidia Ampere GPU."""					, )
						# New Code #
						parser.add_argument(
						    """--gradient_accumulation_steps"""					, type=__UpperCamelCase					, default=1					, help="""The number of minibatches to be ran before gradients are accumulated."""					, )
						parser.add_argument(
						    """--local_sgd_steps"""					, type=__UpperCamelCase					, default=8					, help="""Number of local SGD steps or None to disable local SGD"""    )
						parser.add_argument("""--cpu"""					, action="""store_true"""					, help="""If passed, will train on the CPU."""    )
						snake_case_   : List[Any]						=			parser.parse_args()
						snake_case_   : Any						=			{"""lr""": 2E-5, """num_epochs""": 3, """seed""": 4_2, """batch_size""": 1_6}
						training_function(__UpperCamelCase					, __UpperCamelCase    )


if __name__ == "__main__":
					main()






"""simple docstring"""





from collections.abc import Callable
from math import pi, sqrt
from random import uniform
from statistics import mean
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int    ):


						'''simple docstring'''






						def is_in_circle(__UpperCamelCase      :			float					, __UpperCamelCase      :			float    ) -> bool:
												snake_case_   : Dict						=			sqrt((x**2) + (y**2)    )
												# Our circle has a radius of 1, so a distance
												# greater than 1 would land outside the circle.
												return distance_from_centre <= 1

						# The proportion of guesses that landed in the circle
						snake_case_   : Tuple						=			mean(
						    int(is_in_circle(uniform(-1.0					, 1.0    )					, uniform(-1.0					, 1.0    )    )    )
						    for _ in range(__UpperCamelCase    )    )
						# The ratio of the area for circle to square is pi/4.
						snake_case_   : Union[str, Any]						=			proportion * 4
						print(F'The estimated value of pi is {pi_estimate}'    )
						print(F'The numpy value of pi is {pi}'    )
						print(F'The total error is {abs(pi - pi_estimate    )}'    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int					, __UpperCamelCase      :			Callable[[float], float]					, __UpperCamelCase      :			float = 0.0					, __UpperCamelCase      :			float = 1.0					, ):


						'''simple docstring'''






						return mean(
						    function_to_integrate(uniform(__UpperCamelCase					, __UpperCamelCase    )    ) for _ in range(__UpperCamelCase    )    ) * (max_value - min_value)
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int					, __UpperCamelCase      :			float = 0.0					, __UpperCamelCase      :			float = 1.0    ):


						'''simple docstring'''






						def identity_function(__UpperCamelCase      :			float    ) -> float:
												return x

						snake_case_   : int						=			area_under_curve_estimator(
						    __UpperCamelCase					, __UpperCamelCase					, __UpperCamelCase					, __UpperCamelCase    )
						snake_case_   : str						=			(max_value * max_value - min_value * min_value) / 2

						print("""******************"""    )
						print(F'Estimating area under y=x where x varies from {min_value} to {max_value}'    )
						print(F'Estimated value is {estimated_value}'    )
						print(F'Expected value is {expected_value}'    )
						print(F'Total error is {abs(estimated_value - expected_value    )}'    )
						print("""******************"""    )
def 	__lowerCAmelCase						(		__UpperCamelCase      :			int    ):


						'''simple docstring'''






						def function_to_integrate(__UpperCamelCase      :			float    ) -> float:
												return sqrt(4.0 - x * x    )

						snake_case_   : List[Any]						=			area_under_curve_estimator(
						    __UpperCamelCase					, __UpperCamelCase					, 0.0					, 2.0    )

						print("""******************"""    )
						print("""Estimating pi using area_under_curve_estimator"""    )
						print(F'Estimated value is {estimated_value}'    )
						print(F'Expected value is {pi}'    )
						print(F'Total error is {abs(estimated_value - pi    )}'    )
						print("""******************"""    )


if __name__ == "__main__":
					import doctest

					doctest.testmod()

