




import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule


class    lowerCamelCase    (		_lowerCamelCase						):






			'''simple docstring'''





			UpperCamelCase__								=42
			UpperCamelCase__								=jnp.floataa
			UpperCamelCase__								=True

			def 							UpperCAmelCase__       ( self			:   Optional[int]  )						->   List[str]:
								super().setup()
								__magic_name__							:	str    							=							nn.Dense(5       ,						dtype=self.dtype  )


			def __call__( self			:   List[Any]       ,						*lowerCamelCase_			:   Dict       ,						**lowerCamelCase_			:   Any  )						->   Tuple:
								__magic_name__							:	Dict    							=							super().__call__(*lowerCamelCase_       ,						**lowerCamelCase_  )
								__magic_name__							:	Dict    							=							self.cls(outputs[2]  )
								return outputs[:2] + (cls_out,)






class    lowerCamelCase    (		_lowerCamelCase						):






			'''simple docstring'''





			UpperCamelCase__								=FlaxBigBirdForNaturalQuestionsModule







def 		lowercase__   (		__A:	Optional[Any]							,__A:	Union[str, Any]							,__A:	List[str]							,__A:	List[str]							,__A:	int							,__A:	Tuple				):





					'''simple docstring'''






					def cross_entropy(__A:	List[str]							,__A:	Optional[Any]							,__A:	int=None				):
										__magic_name__							:	Tuple    							=							logits.shape[-1]
										__magic_name__							:	int    							=							(labels[..., None] == jnp.arange(__A				)[None]).astype('''f4'''				)
										__magic_name__							:	Optional[int]    							=							jax.nn.log_softmax(__A							,axis=-1				)
										__magic_name__							:	Union[str, Any]    							=							-jnp.sum(labels * logits							,axis=-1				)
										if reduction is not None:
															__magic_name__							:	int    							=							reduction(__A				)
										return loss

					__magic_name__							:	str    							=							partial(__A							,reduction=jnp.mean				)
					__magic_name__							:	Union[str, Any]    							=							cross_entropy(__A							,__A				)
					__magic_name__							:	str    							=							cross_entropy(__A							,__A				)
					__magic_name__							:	List[str]    							=							cross_entropy(__A							,__A				)
					return (start_loss + end_loss + pooled_loss) / 3


@dataclass
class    lowerCamelCase    :






			'''simple docstring'''





			UpperCamelCase__								="google/bigbird-roberta-base"
			UpperCamelCase__								=3_0_0_0
			UpperCamelCase__								=1_0_5_0_0

			UpperCamelCase__								=1_2_8
			UpperCamelCase__								=3

			UpperCamelCase__								=1
			UpperCamelCase__								=5

			# tx_args
			UpperCamelCase__								=3e-5
			UpperCamelCase__								=0.0
			UpperCamelCase__								=2_0_0_0_0
			UpperCamelCase__								=0.00_95

			UpperCamelCase__								="bigbird-roberta-natural-questions"
			UpperCamelCase__								="training-expt"
			UpperCamelCase__								="data/nq-training.jsonl"
			UpperCamelCase__								="data/nq-validation.jsonl"

			def 							UpperCAmelCase__       ( self			:   Tuple  )						->   Optional[int]:
								os.makedirs(self.base_dir       ,						exist_ok=lowerCamelCase_  )
								__magic_name__							:	Union[str, Any]    							=							os.path.join(self.base_dir       ,						self.save_dir  )
								__magic_name__							:	Any    							=							self.batch_size_per_device * jax.device_count()






@dataclass
class    lowerCamelCase    :






			'''simple docstring'''





			UpperCamelCase__								=42
			UpperCamelCase__								=4_0_9_6  # no dynamic padding on TPUs

			def __call__( self			:   str       ,						lowerCamelCase_			:   Tuple  )						->   Dict:
								__magic_name__							:	Optional[Any]    							=							self.collate_fn(lowerCamelCase_  )
								__magic_name__							:	Union[str, Any]    							=							jax.tree_util.tree_map(lowerCamelCase_       ,						lowerCamelCase_  )
								return batch

			def 							UpperCAmelCase__       ( self			:   Union[str, Any]       ,						lowerCamelCase_			:   Dict  )						->   Any:
								__magic_name__							,    __magic_name__							:	Optional[Any]    							=							self.fetch_inputs(features['''input_ids''']  )
								__magic_name__							:	Optional[int]    							=							{
								    '''input_ids''': jnp.array(lowerCamelCase_       ,						dtype=jnp.intaa  ),
								    '''attention_mask''': jnp.array(lowerCamelCase_       ,						dtype=jnp.intaa  ),
								    '''start_labels''': jnp.array(features['''start_token''']       ,						dtype=jnp.intaa  ),
								    '''end_labels''': jnp.array(features['''end_token''']       ,						dtype=jnp.intaa  ),
								    '''pooled_labels''': jnp.array(features['''category''']       ,						dtype=jnp.intaa  ),
								}
								return batch

			def 							UpperCAmelCase__       ( self			:   Dict       ,						lowerCamelCase_			:   list  )						->   List[str]:
								__magic_name__							:	int    							=							[self._fetch_inputs(lowerCamelCase_  ) for ids in input_ids]
								return zip(*lowerCamelCase_  )


			def 							UpperCAmelCase__       ( self			:   Union[str, Any]       ,						lowerCamelCase_			:   list  )						->   Dict:
								__magic_name__							:	str    							=							[1 for _ in range(len(lowerCamelCase_  )  )]
								while len(lowerCamelCase_  ) < self.max_length:
													input_ids.append(self.pad_id  )
													attention_mask.append(0  )
								return input_ids, attention_mask







def 		lowercase__   (		__A:	List[str]							,__A:	Optional[Any]							,__A:	Optional[Any]=None				):





					'''simple docstring'''






					if seed is not None:
										__magic_name__							:	Union[str, Any]    							=							dataset.shuffle(seed=__A				)
					for i in range(len(__A				) // batch_size				):
										__magic_name__							:	Union[str, Any]    							=							dataset[i * batch_size : (i + 1) * batch_size]
										yield dict(__A				)







@partial(jax.pmap							,axis_name='''batch'''				)
def 		lowercase__   (		__A:	Union[str, Any]							,__A:	Optional[Any]							,**__A:	str				):





					'''simple docstring'''






					def loss_fn(__A:	List[str]				):
										__magic_name__							:	Optional[int]    							=							model_inputs.pop('''start_labels'''				)
										__magic_name__							:	str    							=							model_inputs.pop('''end_labels'''				)
										__magic_name__							:	Tuple    							=							model_inputs.pop('''pooled_labels'''				)

										__magic_name__							:	int    							=							state.apply_fn(**__A							,params=__A							,dropout_rng=__A							,train=__A				)
										__magic_name__							,    __magic_name__							,    __magic_name__							:	List[Any]    							=							outputs

										return state.loss_fn(
										    __A							,__A							,__A							,__A							,__A							,__A							,)

					__magic_name__							,    __magic_name__							:	int    							=							jax.random.split(__A				)
					__magic_name__							:	Dict    							=							jax.value_and_grad(__A				)
					__magic_name__							,    __magic_name__							:	Union[str, Any]    							=							grad_fn(state.params				)
					__magic_name__							:	Any    							=							jax.lax.pmean({'''loss''': loss}							,axis_name='''batch'''				)
					__magic_name__							:	int    							=							jax.lax.pmean(__A							,'''batch'''				)

					__magic_name__							:	Any    							=							state.apply_gradients(grads=__A				)
					return state, metrics, new_drp_rng







@partial(jax.pmap							,axis_name='''batch'''				)
def 		lowercase__   (		__A:	List[Any]							,**__A:	str				):





					'''simple docstring'''






					__magic_name__							:	Optional[int]    							=							model_inputs.pop('''start_labels'''				)
					__magic_name__							:	List[Any]    							=							model_inputs.pop('''end_labels'''				)
					__magic_name__							:	Any    							=							model_inputs.pop('''pooled_labels'''				)

					__magic_name__							:	Tuple    							=							state.apply_fn(**__A							,params=state.params							,train=__A				)
					__magic_name__							,    __magic_name__							,    __magic_name__							:	List[str]    							=							outputs

					__magic_name__							:	Dict    							=							state.loss_fn(__A							,__A							,__A							,__A							,__A							,__A				)
					__magic_name__							:	Union[str, Any]    							=							jax.lax.pmean({'''loss''': loss}							,axis_name='''batch'''				)
					return metrics


class    lowerCamelCase    (		train_state.TrainState						):






			'''simple docstring'''





			UpperCamelCase__								=struct.field(pytree_node=_lowerCamelCase						)






@dataclass
class    lowerCamelCase    :






			'''simple docstring'''





			UpperCamelCase__								=42
			UpperCamelCase__								=42
			UpperCamelCase__								=42
			UpperCamelCase__								=42
			UpperCamelCase__								=42
			UpperCamelCase__								=42
			UpperCamelCase__								=None

			def 							UpperCAmelCase__       ( self			:   Tuple       ,						lowerCamelCase_			:   str       ,						lowerCamelCase_			:   Tuple       ,						lowerCamelCase_			:   Union[str, Any]       ,						lowerCamelCase_			:   List[str]=None  )						->   str:
								__magic_name__							:	Any    							=							model.params
								__magic_name__							:	Any    							=							TrainState.create(
								    apply_fn=model.__call__       ,						params=lowerCamelCase_       ,						tx=lowerCamelCase_       ,						loss_fn=lowerCamelCase_       ,						)
								if ckpt_dir is not None:
													__magic_name__							,    __magic_name__							,    __magic_name__							,    __magic_name__							,    __magic_name__							:	Optional[Any]    							=							restore_checkpoint(lowerCamelCase_       ,						lowerCamelCase_  )
													__magic_name__							:	List[Any]    							=							{
													    '''lr''': args.lr,
													    '''init_lr''': args.init_lr,
													    '''warmup_steps''': args.warmup_steps,
													    '''num_train_steps''': num_train_steps,
													    '''weight_decay''': args.weight_decay,
													}
													__magic_name__							,    __magic_name__							:	str    							=							build_tx(**lowerCamelCase_  )
													__magic_name__							:	Union[str, Any]    							=							train_state.TrainState(
													    step=lowerCamelCase_       ,						apply_fn=model.__call__       ,						params=lowerCamelCase_       ,						tx=lowerCamelCase_       ,						opt_state=lowerCamelCase_       ,						)
													__magic_name__							:	Optional[int]    							=							args
													__magic_name__							:	Optional[int]    							=							data_collator
													__magic_name__							:	Optional[Any]    							=							lr
													__magic_name__							:	Any    							=							params
								__magic_name__							:	Dict    							=							jax_utils.replicate(lowerCamelCase_  )
								return state

			def 							UpperCAmelCase__       ( self			:   int       ,						lowerCamelCase_			:   Any       ,						lowerCamelCase_			:   Tuple       ,						lowerCamelCase_			:   str  )						->   Any:
								__magic_name__							:	List[Any]    							=							self.args
								__magic_name__							:	List[Any]    							=							len(lowerCamelCase_  ) // args.batch_size

								__magic_name__							:	str    							=							jax.random.PRNGKey(0  )
								__magic_name__							:	Any    							=							jax.random.split(lowerCamelCase_       ,						jax.device_count()  )
								for epoch in range(args.max_epochs  ):
													__magic_name__							:	Dict    							=							jnp.array(0       ,						dtype=jnp.floataa  )
													__magic_name__							:	str    							=							get_batched_dataset(lowerCamelCase_       ,						args.batch_size       ,						seed=lowerCamelCase_  )
													__magic_name__							:	Any    							=							0
													for batch in tqdm(lowerCamelCase_       ,						total=lowerCamelCase_       ,						desc=F'''Running EPOCH-{epoch}'''  ):
																		__magic_name__							:	List[str]    							=							self.data_collator(lowerCamelCase_  )
																		__magic_name__							,    __magic_name__							,    __magic_name__							:	str    							=							self.train_step_fn(lowerCamelCase_       ,						lowerCamelCase_       ,						**lowerCamelCase_  )
																		running_loss += jax_utils.unreplicate(metrics['''loss''']  )
																		i += 1
																		if i % args.logging_steps == 0:
																							__magic_name__							:	List[Any]    							=							jax_utils.unreplicate(state.step  )
																							__magic_name__							:	Optional[int]    							=							running_loss.item() / i
																							__magic_name__							:	str    							=							self.scheduler_fn(state_step - 1  )

																							__magic_name__							:	List[str]    							=							self.evaluate(lowerCamelCase_       ,						lowerCamelCase_  )
																							__magic_name__							:	Tuple    							=							{
																							    '''step''': state_step.item(),
																							    '''eval_loss''': eval_loss.item(),
																							    '''tr_loss''': tr_loss,
																							    '''lr''': lr.item(),
																							}
																							tqdm.write(str(lowerCamelCase_  )  )
																							self.logger.log(lowerCamelCase_       ,						commit=lowerCamelCase_  )

																		if i % args.save_steps == 0:
																							self.save_checkpoint(args.save_dir + F'''-e{epoch}-s{i}'''       ,						state=lowerCamelCase_  )

			def 							UpperCAmelCase__       ( self			:   str       ,						lowerCamelCase_			:   str       ,						lowerCamelCase_			:   List[str]  )						->   Optional[int]:
								__magic_name__							:	Any    							=							get_batched_dataset(lowerCamelCase_       ,						self.args.batch_size  )
								__magic_name__							:	List[str]    							=							len(lowerCamelCase_  ) // self.args.batch_size
								__magic_name__							:	List[Any]    							=							jnp.array(0       ,						dtype=jnp.floataa  )
								__magic_name__							:	Union[str, Any]    							=							0
								for batch in tqdm(lowerCamelCase_       ,						total=lowerCamelCase_       ,						desc='''Evaluating ... '''  ):
													__magic_name__							:	List[Any]    							=							self.data_collator(lowerCamelCase_  )
													__magic_name__							:	Tuple    							=							self.val_step_fn(lowerCamelCase_       ,						**lowerCamelCase_  )
													running_loss += jax_utils.unreplicate(metrics['''loss''']  )
													i += 1
								return running_loss / i


			def 							UpperCAmelCase__       ( self			:   Union[str, Any]       ,						lowerCamelCase_			:   Tuple       ,						lowerCamelCase_			:   str  )						->   Any:
								__magic_name__							:	Optional[Any]    							=							jax_utils.unreplicate(lowerCamelCase_  )
								print(F'''SAVING CHECKPOINT IN {save_dir}'''       ,						end=''' ... '''  )
								self.model_save_fn(lowerCamelCase_       ,						params=state.params  )
								with open(os.path.join(lowerCamelCase_       ,						'''opt_state.msgpack'''  )       ,						'''wb'''  ) as f:
													f.write(to_bytes(state.opt_state  )  )
								joblib.dump(self.args       ,						os.path.join(lowerCamelCase_       ,						'''args.joblib'''  )  )
								joblib.dump(self.data_collator       ,						os.path.join(lowerCamelCase_       ,						'''data_collator.joblib'''  )  )
								with open(os.path.join(lowerCamelCase_       ,						'''training_state.json'''  )       ,						'''w'''  ) as f:
													json.dump({'''step''': state.step.item()}       ,						lowerCamelCase_  )
								print('''DONE'''  )







def 		lowercase__   (		__A:	Any							,__A:	Union[str, Any]				):





					'''simple docstring'''






					print(F'''RESTORING CHECKPOINT FROM {save_dir}'''							,end=''' ... '''				)
					with open(os.path.join(__A							,'''flax_model.msgpack'''				)							,'''rb'''				) as f:
										__magic_name__							:	Union[str, Any]    							=							from_bytes(state.params							,f.read()				)

					with open(os.path.join(__A							,'''opt_state.msgpack'''				)							,'''rb'''				) as f:
										__magic_name__							:	Optional[Any]    							=							from_bytes(state.opt_state							,f.read()				)

					__magic_name__							:	Union[str, Any]    							=							joblib.load(os.path.join(__A							,'''args.joblib'''				)				)
					__magic_name__							:	Union[str, Any]    							=							joblib.load(os.path.join(__A							,'''data_collator.joblib'''				)				)

					with open(os.path.join(__A							,'''training_state.json'''				)							,'''r'''				) as f:
										__magic_name__							:	Any    							=							json.load(__A				)
					__magic_name__							:	Optional[int]    							=							training_state['''step''']

					print('''DONE'''				)
					return params, opt_state, step, args, data_collator







def 		lowercase__   (		__A:	Any							,__A:	Optional[int]							,__A:	str							,__A:	List[Any]				):





					'''simple docstring'''






					__magic_name__							:	str    							=							num_train_steps - warmup_steps
					__magic_name__							:	Any    							=							optax.linear_schedule(init_value=__A							,end_value=__A							,transition_steps=__A				)
					__magic_name__							:	Dict    							=							optax.linear_schedule(init_value=__A							,end_value=1e-7							,transition_steps=__A				)
					__magic_name__							:	Tuple    							=							optax.join_schedules(schedules=[warmup_fn, decay_fn]							,boundaries=[warmup_steps]				)
					return lr







def 		lowercase__   (		__A:	List[str]							,__A:	Optional[Any]							,__A:	Tuple							,__A:	Dict							,__A:	Optional[int]				):





					'''simple docstring'''






					def weight_decay_mask(__A:	str				):
										__magic_name__							:	Dict    							=							traverse_util.flatten_dict(__A				)
										__magic_name__							:	List[Any]    							=							{k: (v[-1] != '''bias''' and v[-2:] != ('''LayerNorm''', '''scale''')) for k, v in params.items()}
										return traverse_util.unflatten_dict(__A				)

					__magic_name__							:	int    							=							scheduler_fn(__A							,__A							,__A							,__A				)

					__magic_name__							:	List[Any]    							=							optax.adamw(learning_rate=__A							,weight_decay=__A							,mask=__A				)
					return tx, lr









from sklearn.metrics import mean_squared_error

import datasets


__lowerCamelCase   :						List[str]				=			'''\
@article{scikit-learn,
 title={Scikit-learn: Machine Learning in {P}ython},
 author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
 journal={Journal of Machine Learning Research},
 volume={12},
 pages={2825--2830},
 year={2011}
}
'''

__lowerCamelCase   :						Optional[int]				=			'''\
Mean Squared Error(MSE) is the average of the square of difference between the predicted
and actual values.
'''


__lowerCamelCase   :						str				=			'''
Args:
    predictions: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Estimated target values.
    references: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Ground truth (correct) target values.
    sample_weight: array-like of shape (n_samples,), default=None
        Sample weights.
    multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average"
        Defines aggregating of multiple output values. Array-like value defines weights used to average errors.

        "raw_values" : Returns a full set of errors in case of multioutput input.

        "uniform_average" : Errors of all outputs are averaged with uniform weight.

    squared : bool, default=True
        If True returns MSE value, if False returns RMSE (Root Mean Squared Error) value.

Returns:
    mse : mean squared error.
Examples:

    >>> mse_metric = datasets.load_metric("mse")
    >>> predictions = [2.5, 0.0, 2, 8]
    >>> references = [3, -0.5, 2, 7]
    >>> results = mse_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'mse\': 0.375}
    >>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False)
    >>> print(rmse_result)
    {\'mse\': 0.6123724356957945}

    If you\'re using multi-dimensional lists, then set the config as follows :

    >>> mse_metric = datasets.load_metric("mse", "multilist")
    >>> predictions = [[0.5, 1], [-1, 1], [7, -6]]
    >>> references = [[0, 2], [-1, 2], [8, -5]]
    >>> results = mse_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'mse\': 0.7083333333333334}
    >>> results = mse_metric.compute(predictions=predictions, references=references, multioutput=\'raw_values\')
    >>> print(results) # doctest: +NORMALIZE_WHITESPACE
    {\'mse\': array([0.41666667, 1. ])}
'''


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,_KWARGS_DESCRIPTION						)
class    lowerCamelCase    (		datasets.Metric						):






			'''simple docstring'''





			def 							UpperCAmelCase__       ( self			:   Dict  )						->   Tuple:
								return datasets.MetricInfo(
								    description=_DESCRIPTION       ,						citation=_CITATION       ,						inputs_description=_KWARGS_DESCRIPTION       ,						features=datasets.Features(self._get_feature_types()  )       ,						reference_urls=[
								        '''https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html'''
								    ]       ,						)

			def 							UpperCAmelCase__       ( self			:   List[Any]  )						->   Any:
								if self.config_name == "multilist":
													return {
													    "predictions": datasets.Sequence(datasets.Value('''float'''  )  ),
													    "references": datasets.Sequence(datasets.Value('''float'''  )  ),
													}
								else:
													return {
													    "predictions": datasets.Value('''float'''  ),
													    "references": datasets.Value('''float'''  ),
													}


			def 							UpperCAmelCase__       ( self			:   Union[str, Any]       ,						lowerCamelCase_			:   Optional[Any]       ,						lowerCamelCase_			:   int       ,						lowerCamelCase_			:   int=None       ,						lowerCamelCase_			:   List[Any]="uniform_average"       ,						lowerCamelCase_			:   List[str]=True  )						->   List[str]:
								__magic_name__							:	Tuple    							=							mean_squared_error(
								    lowerCamelCase_       ,						lowerCamelCase_       ,						sample_weight=lowerCamelCase_       ,						multioutput=lowerCamelCase_       ,						squared=lowerCamelCase_  )

								return {"mse": mse}



