





import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule




class   UpperCamelCase_       (     UpperCamelCase__    ):
							lowerCamelCase_           =					42
							lowerCamelCase_           =					jnp.floataa
							lowerCamelCase_           =					True


							def      _snake_case  (							self    :Optional[int]		)   ->   List[Any]:



									"""simple docstring"""



									super().setup()
									SCREAMING_SNAKE_CASE__			=      nn.Dense(5							,				dtype=self.dtype		)


							def __call__(							self    :Union[str, Any]							,				*__A    :Optional[Any]							,				**__A    :Optional[Any]		)   ->   Any:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      super().__call__(*__A							,				**__A		)
									SCREAMING_SNAKE_CASE__			=      self.cls(outputs[2]		)
									return outputs[:2] + (cls_out,)




class   UpperCamelCase_       (     UpperCamelCase__    ):
							lowerCamelCase_           =					FlaxBigBirdForNaturalQuestionsModule






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[str]					,   UpperCamelCase__:							List[str]					,   UpperCamelCase__:							Tuple					,   UpperCamelCase__:							List[str]					,   UpperCamelCase__:							Optional[int]					,   UpperCamelCase__:							Optional[Any]    ):
		def cross_entropy(UpperCamelCase__:							List[Any]					,   UpperCamelCase__:							Tuple					,   UpperCamelCase__:							Union[str, Any]=None    ):
				SCREAMING_SNAKE_CASE__			=      logits.shape[-1]
				SCREAMING_SNAKE_CASE__			=      (labels[..., None] == jnp.arange(UpperCamelCase__    )[None]).astype("""f4"""    )
				SCREAMING_SNAKE_CASE__			=      jax.nn.log_softmax(UpperCamelCase__					,   axis=-1    )
				SCREAMING_SNAKE_CASE__			=      -jnp.sum(labels * logits					,   axis=-1    )
				if reduction is not None:
						SCREAMING_SNAKE_CASE__			=      reduction(UpperCamelCase__    )
				return loss

		SCREAMING_SNAKE_CASE__			=      partial(UpperCamelCase__					,   reduction=jnp.mean    )
		SCREAMING_SNAKE_CASE__			=      cross_entropy(UpperCamelCase__					,   UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      cross_entropy(UpperCamelCase__					,   UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      cross_entropy(UpperCamelCase__					,   UpperCamelCase__    )
		return (start_loss + end_loss + pooled_loss) / 3




@dataclass
class   UpperCamelCase_       :
							lowerCamelCase_           =					"google/bigbird-roberta-base"
							lowerCamelCase_           =					30_00
							lowerCamelCase_           =					1_05_00

							lowerCamelCase_           =					1_28
							lowerCamelCase_           =					3

							lowerCamelCase_           =					1
							lowerCamelCase_           =					5

							# tx_args
							lowerCamelCase_           =					3e-5
							lowerCamelCase_           =					0.0
							lowerCamelCase_           =					2_00_00
							lowerCamelCase_           =					0.0095

							lowerCamelCase_           =					"bigbird-roberta-natural-questions"
							lowerCamelCase_           =					"training-expt"
							lowerCamelCase_           =					"data/nq-training.jsonl"
							lowerCamelCase_           =					"data/nq-validation.jsonl"


							def      _snake_case  (							self    :int		)   ->   List[str]:



									"""simple docstring"""



									os.makedirs(self.base_dir							,				exist_ok=__A		)
									SCREAMING_SNAKE_CASE__			=      os.path.join(self.base_dir							,				self.save_dir		)
									SCREAMING_SNAKE_CASE__			=      self.batch_size_per_device * jax.device_count()




@dataclass
class   UpperCamelCase_       :
							lowerCamelCase_           =					42
							lowerCamelCase_           =					40_96  # no dynamic padding on TPUs


							def __call__(							self    :str							,				__A    :Union[str, Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.collate_fn(__A		)
									SCREAMING_SNAKE_CASE__			=      jax.tree_util.tree_map(__A							,				__A		)
									return batch


							def      _snake_case  (							self    :List[str]							,				__A    :Union[str, Any]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.fetch_inputs(features["""input_ids"""]		)
									SCREAMING_SNAKE_CASE__			=      {
									    """input_ids""": jnp.array(__A							,				dtype=jnp.intaa		),
									    """attention_mask""": jnp.array(__A							,				dtype=jnp.intaa		),
									    """start_labels""": jnp.array(features["""start_token"""]							,				dtype=jnp.intaa		),
									    """end_labels""": jnp.array(features["""end_token"""]							,				dtype=jnp.intaa		),
									    """pooled_labels""": jnp.array(features["""category"""]							,				dtype=jnp.intaa		),
									}
									return batch


							def      _snake_case  (							self    :List[Any]							,				__A    :list		)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      [self._fetch_inputs(__A		) for ids in input_ids]
									return zip(*__A		)


							def      _snake_case  (							self    :Union[str, Any]							,				__A    :list		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      [1 for _ in range(len(__A		)		)]
									while len(__A		) < self.max_length:
											input_ids.append(self.pad_id		)
											attention_mask.append(0		)
									return input_ids, attention_mask






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[Any]					,   UpperCamelCase__:							Any					,   UpperCamelCase__:							int=None    ):
		if seed is not None:
				SCREAMING_SNAKE_CASE__			=      dataset.shuffle(seed=UpperCamelCase__    )
		for i in range(len(UpperCamelCase__    ) // batch_size    ):
				SCREAMING_SNAKE_CASE__			=      dataset[i * batch_size : (i + 1) * batch_size]
				yield dict(UpperCamelCase__    )






@partial(jax.pmap					,   axis_name="""batch"""    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[Any]					,   UpperCamelCase__:							int					,   **UpperCamelCase__:							Dict    ):
		def loss_fn(UpperCamelCase__:							Any    ):
				SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""start_labels"""    )
				SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""end_labels"""    )
				SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""pooled_labels"""    )

				SCREAMING_SNAKE_CASE__			=      state.apply_fn(**UpperCamelCase__					,   params=UpperCamelCase__					,   dropout_rng=UpperCamelCase__					,   train=UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      outputs

				return state.loss_fn(
				    UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__					,   )

		SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      jax.random.split(UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      jax.value_and_grad(UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      grad_fn(state.params    )
		SCREAMING_SNAKE_CASE__			=      jax.lax.pmean({"""loss""": loss}					,   axis_name="""batch"""    )
		SCREAMING_SNAKE_CASE__			=      jax.lax.pmean(UpperCamelCase__					,   """batch"""    )

		SCREAMING_SNAKE_CASE__			=      state.apply_gradients(grads=UpperCamelCase__    )
		return state, metrics, new_drp_rng






@partial(jax.pmap					,   axis_name="""batch"""    )
def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[Any]					,   **UpperCamelCase__:							Optional[Any]    ):
		SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""start_labels"""    )
		SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""end_labels"""    )
		SCREAMING_SNAKE_CASE__			=      model_inputs.pop("""pooled_labels"""    )

		SCREAMING_SNAKE_CASE__			=      state.apply_fn(**UpperCamelCase__					,   params=state.params					,   train=UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      outputs

		SCREAMING_SNAKE_CASE__			=      state.loss_fn(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      jax.lax.pmean({"""loss""": loss}					,   axis_name="""batch"""    )
		return metrics




class   UpperCamelCase_       (     train_state.TrainState    ):
							lowerCamelCase_           =					struct.field(pytree_node=UpperCamelCase__    )




@dataclass
class   UpperCamelCase_       :
							lowerCamelCase_           =					42
							lowerCamelCase_           =					42
							lowerCamelCase_           =					42
							lowerCamelCase_           =					42
							lowerCamelCase_           =					42
							lowerCamelCase_           =					42
							lowerCamelCase_           =					None


							def      _snake_case  (							self    :Dict							,				__A    :int							,				__A    :str							,				__A    :Optional[Any]							,				__A    :Union[str, Any]=None		)   ->   str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      model.params
									SCREAMING_SNAKE_CASE__			=      TrainState.create(
									    apply_fn=model.__call__							,				params=__A							,				tx=__A							,				loss_fn=__A							,				)
									if ckpt_dir is not None:
											SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      restore_checkpoint(__A							,				__A		)
											SCREAMING_SNAKE_CASE__			=      {
											    """lr""": args.lr,
											    """init_lr""": args.init_lr,
											    """warmup_steps""": args.warmup_steps,
											    """num_train_steps""": num_train_steps,
											    """weight_decay""": args.weight_decay,
											}
											SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      build_tx(**__A		)
											SCREAMING_SNAKE_CASE__			=      train_state.TrainState(
											    step=__A							,				apply_fn=model.__call__							,				params=__A							,				tx=__A							,				opt_state=__A							,				)
											SCREAMING_SNAKE_CASE__			=      args
											SCREAMING_SNAKE_CASE__			=      data_collator
											SCREAMING_SNAKE_CASE__			=      lr
											SCREAMING_SNAKE_CASE__			=      params
									SCREAMING_SNAKE_CASE__			=      jax_utils.replicate(__A		)
									return state


							def      _snake_case  (							self    :Union[str, Any]							,				__A    :Any							,				__A    :Optional[Any]							,				__A    :Optional[Any]		)   ->   List[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.args
									SCREAMING_SNAKE_CASE__			=      len(__A		) // args.batch_size

									SCREAMING_SNAKE_CASE__			=      jax.random.PRNGKey(0		)
									SCREAMING_SNAKE_CASE__			=      jax.random.split(__A							,				jax.device_count()		)
									for epoch in range(args.max_epochs		):
											SCREAMING_SNAKE_CASE__			=      jnp.array(0							,				dtype=jnp.floataa		)
											SCREAMING_SNAKE_CASE__			=      get_batched_dataset(__A							,				args.batch_size							,				seed=__A		)
											SCREAMING_SNAKE_CASE__			=      0
											for batch in tqdm(__A							,				total=__A							,				desc=f'''Running EPOCH-{epoch}'''		):
													SCREAMING_SNAKE_CASE__			=      self.data_collator(__A		)
													SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.train_step_fn(__A							,				__A							,				**__A		)
													running_loss += jax_utils.unreplicate(metrics["""loss"""]		)
													i += 1
													if i % args.logging_steps == 0:
															SCREAMING_SNAKE_CASE__			=      jax_utils.unreplicate(state.step		)
															SCREAMING_SNAKE_CASE__			=      running_loss.item() / i
															SCREAMING_SNAKE_CASE__			=      self.scheduler_fn(state_step - 1		)

															SCREAMING_SNAKE_CASE__			=      self.evaluate(__A							,				__A		)
															SCREAMING_SNAKE_CASE__			=      {
															    """step""": state_step.item(),
															    """eval_loss""": eval_loss.item(),
															    """tr_loss""": tr_loss,
															    """lr""": lr.item(),
															}
															tqdm.write(str(__A		)		)
															self.logger.log(__A							,				commit=__A		)

													if i % args.save_steps == 0:
															self.save_checkpoint(args.save_dir + f'''-e{epoch}-s{i}'''							,				state=__A		)


							def      _snake_case  (							self    :str							,				__A    :Union[str, Any]							,				__A    :Optional[int]		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      get_batched_dataset(__A							,				self.args.batch_size		)
									SCREAMING_SNAKE_CASE__			=      len(__A		) // self.args.batch_size
									SCREAMING_SNAKE_CASE__			=      jnp.array(0							,				dtype=jnp.floataa		)
									SCREAMING_SNAKE_CASE__			=      0
									for batch in tqdm(__A							,				total=__A							,				desc="""Evaluating ... """		):
											SCREAMING_SNAKE_CASE__			=      self.data_collator(__A		)
											SCREAMING_SNAKE_CASE__			=      self.val_step_fn(__A							,				**__A		)
											running_loss += jax_utils.unreplicate(metrics["""loss"""]		)
											i += 1
									return running_loss / i


							def      _snake_case  (							self    :int							,				__A    :int							,				__A    :Tuple		)   ->   Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      jax_utils.unreplicate(__A		)
									print(f'''SAVING CHECKPOINT IN {save_dir}'''							,				end=""" ... """		)
									self.model_save_fn(__A							,				params=state.params		)
									with open(os.path.join(__A							,				"""opt_state.msgpack"""		)							,				"""wb"""		) as f:
											f.write(to_bytes(state.opt_state		)		)
									joblib.dump(self.args							,				os.path.join(__A							,				"""args.joblib"""		)		)
									joblib.dump(self.data_collator							,				os.path.join(__A							,				"""data_collator.joblib"""		)		)
									with open(os.path.join(__A							,				"""training_state.json"""		)							,				"""w"""		) as f:
											json.dump({"""step""": state.step.item()}							,				__A		)
									print("""DONE"""		)






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							List[str]					,   UpperCamelCase__:							str    ):
		print(f'''RESTORING CHECKPOINT FROM {save_dir}'''					,   end=""" ... """    )
		with open(os.path.join(UpperCamelCase__					,   """flax_model.msgpack"""    )					,   """rb"""    ) as f:
				SCREAMING_SNAKE_CASE__			=      from_bytes(state.params					,   f.read()    )

		with open(os.path.join(UpperCamelCase__					,   """opt_state.msgpack"""    )					,   """rb"""    ) as f:
				SCREAMING_SNAKE_CASE__			=      from_bytes(state.opt_state					,   f.read()    )

		SCREAMING_SNAKE_CASE__			=      joblib.load(os.path.join(UpperCamelCase__					,   """args.joblib"""    )    )
		SCREAMING_SNAKE_CASE__			=      joblib.load(os.path.join(UpperCamelCase__					,   """data_collator.joblib"""    )    )

		with open(os.path.join(UpperCamelCase__					,   """training_state.json"""    )					,   """r"""    ) as f:
				SCREAMING_SNAKE_CASE__			=      json.load(UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      training_state["""step"""]

		print("""DONE"""    )
		return params, opt_state, step, args, data_collator






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							int					,   UpperCamelCase__:							Optional[Any]					,   UpperCamelCase__:							Optional[int]					,   UpperCamelCase__:							Optional[int]    ):
		SCREAMING_SNAKE_CASE__			=      num_train_steps - warmup_steps
		SCREAMING_SNAKE_CASE__			=      optax.linear_schedule(init_value=UpperCamelCase__					,   end_value=UpperCamelCase__					,   transition_steps=UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      optax.linear_schedule(init_value=UpperCamelCase__					,   end_value=1e-7					,   transition_steps=UpperCamelCase__    )
		SCREAMING_SNAKE_CASE__			=      optax.join_schedules(schedules=[warmup_fn, decay_fn]					,   boundaries=[warmup_steps]    )
		return lr






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							Optional[Any]					,   UpperCamelCase__:							Optional[int]					,   UpperCamelCase__:							List[str]					,   UpperCamelCase__:							Optional[int]					,   UpperCamelCase__:							Dict    ):
		def weight_decay_mask(UpperCamelCase__:							Optional[Any]    ):
				SCREAMING_SNAKE_CASE__			=      traverse_util.flatten_dict(UpperCamelCase__    )
				SCREAMING_SNAKE_CASE__			=      {k: (v[-1] != """bias""" and v[-2:] != ("""LayerNorm""", """scale""")) for k, v in params.items()}
				return traverse_util.unflatten_dict(UpperCamelCase__    )

		SCREAMING_SNAKE_CASE__			=      scheduler_fn(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )

		SCREAMING_SNAKE_CASE__			=      optax.adamw(learning_rate=UpperCamelCase__					,   weight_decay=UpperCamelCase__					,   mask=UpperCamelCase__    )
		return tx, lr






from math import pow, sqrt






def 				SCREAMING_SNAKE_CASE__							(       *UpperCamelCase__:							float    ):
		SCREAMING_SNAKE_CASE__			=      len(UpperCamelCase__    ) > 0 and all(value > 0.0 for value in values    )
		return result






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							float					,   UpperCamelCase__:							float    ):
		return (
		    round(sqrt(molar_mass_a / molar_mass_a    )					,   6    )
		    if validate(UpperCamelCase__					,   UpperCamelCase__    )
		    else ValueError("""Input Error: Molar mass values must greater than 0."""    )
		)






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							float					,   UpperCamelCase__:							float					,   UpperCamelCase__:							float    ):
		return (
		    round(effusion_rate * sqrt(molar_mass_a / molar_mass_a    )					,   6    )
		    if validate(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )
		    else ValueError(
		        """Input Error: Molar mass and effusion rate values must greater than 0."""    )
		)






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							float					,   UpperCamelCase__:							float					,   UpperCamelCase__:							float    ):
		return (
		    round(effusion_rate / sqrt(molar_mass_a / molar_mass_a    )					,   6    )
		    if validate(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )
		    else ValueError(
		        """Input Error: Molar mass and effusion rate values must greater than 0."""    )
		)






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							float					,   UpperCamelCase__:							float					,   UpperCamelCase__:							float    ):
		return (
		    round(molar_mass / pow(effusion_rate_a / effusion_rate_a					,   2    )					,   6    )
		    if validate(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )
		    else ValueError(
		        """Input Error: Molar mass and effusion rate values must greater than 0."""    )
		)






def 				SCREAMING_SNAKE_CASE__							(       UpperCamelCase__:							float					,   UpperCamelCase__:							float					,   UpperCamelCase__:							float    ):
		return (
		    round(pow(effusion_rate_a / effusion_rate_a					,   2    ) / molar_mass					,   6    )
		    if validate(UpperCamelCase__					,   UpperCamelCase__					,   UpperCamelCase__    )
		    else ValueError(
		        """Input Error: Molar mass and effusion rate values must greater than 0."""    )
		)