
"""simple docstring"""






import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule




class 				lowercase__(				UpperCamelCase__   ):
	'''simple docstring'''

	UpperCamelCase				=  42
	UpperCamelCase				=  jnp.floataa
	UpperCamelCase				=  True





	def   __lowerCAmelCase (						self    :Optional[int]					)							->			List[Any]:







			'''simple docstring'''




			super().setup()
			SCREAMING_SNAKE_CASE       :     int				= nn.Dense(5		, dtype=self.dtype					)







	def __call__(						self    :Union[str, Any]		, *lowerCamelCase_    :Optional[Any]		, **lowerCamelCase_    :Optional[Any]					)							->			Any:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Tuple				= super().__call__(*__A		, **__A					)
			SCREAMING_SNAKE_CASE       :     Tuple				= self.cls(outputs[2]					)
			return outputs[:2] + (cls_out,)





class 				lowercase__(				UpperCamelCase__   ):
	'''simple docstring'''

	UpperCamelCase				=  FlaxBigBirdForNaturalQuestionsModule





def __A						(   a_ :							List[str]		,		a_ :							List[str]		,		a_ :							Tuple		,		a_ :							List[str]		,		a_ :							Optional[int]		,		a_ :							Optional[Any]     )->							Tuple:




		'''simple docstring'''






		def cross_entropy(a_ :							List[Any]		,		a_ :							Tuple		,		a_ :							Union[str, Any]=None     ):
				SCREAMING_SNAKE_CASE       :     str				= logits.shape[-1]
				SCREAMING_SNAKE_CASE       :     Dict				= (labels[..., None] == jnp.arange(UpperCamelCase__     )[None]).astype('''f4'''     )
				SCREAMING_SNAKE_CASE       :     Any				= jax.nn.log_softmax(UpperCamelCase__		,		axis=-1     )
				SCREAMING_SNAKE_CASE       :     Optional[int]				= -jnp.sum(labels * logits		,		axis=-1     )
				if reduction is not None:
						SCREAMING_SNAKE_CASE       :     List[str]				= reduction(UpperCamelCase__     )
				return loss

		SCREAMING_SNAKE_CASE       :     List[Any]				= partial(UpperCamelCase__		,		reduction=jnp.mean     )
		SCREAMING_SNAKE_CASE       :     int				= cross_entropy(UpperCamelCase__		,		UpperCamelCase__     )
		SCREAMING_SNAKE_CASE       :     List[Any]				= cross_entropy(UpperCamelCase__		,		UpperCamelCase__     )
		SCREAMING_SNAKE_CASE       :     Optional[int]				= cross_entropy(UpperCamelCase__		,		UpperCamelCase__     )
		return (start_loss + end_loss + pooled_loss) / 3




@dataclass
class 				lowercase__:
	'''simple docstring'''

	UpperCamelCase				=  """google/bigbird-roberta-base"""
	UpperCamelCase				=  30_00
	UpperCamelCase				=  1_05_00

	UpperCamelCase				=  1_28
	UpperCamelCase				=  3

	UpperCamelCase				=  1
	UpperCamelCase				=  5

	# tx_args
	UpperCamelCase				=  3E-5
	UpperCamelCase				=  0.0
	UpperCamelCase				=  2_00_00
	UpperCamelCase				=  0.0095

	UpperCamelCase				=  """bigbird-roberta-natural-questions"""
	UpperCamelCase				=  """training-expt"""
	UpperCamelCase				=  """data/nq-training.jsonl"""
	UpperCamelCase				=  """data/nq-validation.jsonl"""





	def   __lowerCAmelCase (						self    :int					)							->			List[str]:







			'''simple docstring'''




			os.makedirs(self.base_dir		, exist_ok=__A					)
			SCREAMING_SNAKE_CASE       :     str				= os.path.join(self.base_dir		, self.save_dir					)
			SCREAMING_SNAKE_CASE       :     List[Any]				= self.batch_size_per_device * jax.device_count()





@dataclass
class 				lowercase__:
	'''simple docstring'''

	UpperCamelCase				=  42
	UpperCamelCase				=  40_96  # no dynamic padding on TPUs





	def __call__(						self    :str		, lowerCamelCase_    :Union[str, Any]					)							->			Union[str, Any]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= self.collate_fn(__A					)
			SCREAMING_SNAKE_CASE       :     Union[str, Any]				= jax.tree_util.tree_map(__A		, __A					)
			return batch





	def   __lowerCAmelCase (						self    :List[str]		, lowerCamelCase_    :Union[str, Any]					)							->			List[Any]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     Optional[Any]				= self.fetch_inputs(features['''input_ids''']					)
			SCREAMING_SNAKE_CASE       :     List[str]				= {
			    '''input_ids''': jnp.array(__A		, dtype=jnp.intaa					),
			    '''attention_mask''': jnp.array(__A		, dtype=jnp.intaa					),
			    '''start_labels''': jnp.array(features['''start_token''']		, dtype=jnp.intaa					),
			    '''end_labels''': jnp.array(features['''end_token''']		, dtype=jnp.intaa					),
			    '''pooled_labels''': jnp.array(features['''category''']		, dtype=jnp.intaa					),
			}
			return batch





	def   __lowerCAmelCase (						self    :List[Any]		, lowerCamelCase_    :list					)							->			Tuple:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Optional[Any]				= [self._fetch_inputs(__A					) for ids in input_ids]
			return zip(*__A					)







	def   __lowerCAmelCase (						self    :Union[str, Any]		, lowerCamelCase_    :list					)							->			str:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Dict				= [1 for _ in range(len(__A					)					)]
			while len(__A					) < self.max_length:
					input_ids.append(self.pad_id					)
					attention_mask.append(0					)
			return input_ids, attention_mask





def __A						(   a_ :							List[Any]		,		a_ :							Any		,		a_ :							int=None     )->							Dict:




		'''simple docstring'''






		if seed is not None:
				SCREAMING_SNAKE_CASE       :     List[str]				= dataset.shuffle(seed=UpperCamelCase__     )
		for i in range(len(UpperCamelCase__     ) // batch_size     ):
				SCREAMING_SNAKE_CASE       :     int				= dataset[i * batch_size : (i + 1) * batch_size]
				yield dict(UpperCamelCase__     )





@partial(jax.pmap		,		axis_name='''batch'''     )
def __A						(   a_ :							Optional[Any]		,		a_ :							int		,		**a_ :							Dict     )->							int:




		'''simple docstring'''






		def loss_fn(a_ :							Any     ):
				SCREAMING_SNAKE_CASE       :     Optional[Any]				= model_inputs.pop('''start_labels'''     )
				SCREAMING_SNAKE_CASE       :     int				= model_inputs.pop('''end_labels'''     )
				SCREAMING_SNAKE_CASE       :     str				= model_inputs.pop('''pooled_labels'''     )

				SCREAMING_SNAKE_CASE       :     int				= state.apply_fn(**UpperCamelCase__		,		params=UpperCamelCase__		,		dropout_rng=UpperCamelCase__		,		train=UpperCamelCase__     )
				SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     int				= outputs

				return state.loss_fn(
				    UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__		,		)

		SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     Optional[Any]				= jax.random.split(UpperCamelCase__     )
		SCREAMING_SNAKE_CASE       :     List[Any]				= jax.value_and_grad(UpperCamelCase__     )
		SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     Union[str, Any]				= grad_fn(state.params     )
		SCREAMING_SNAKE_CASE       :     Union[str, Any]				= jax.lax.pmean({'''loss''': loss}		,		axis_name='''batch'''     )
		SCREAMING_SNAKE_CASE       :     Optional[int]				= jax.lax.pmean(UpperCamelCase__		,		'''batch'''     )

		SCREAMING_SNAKE_CASE       :     int				= state.apply_gradients(grads=UpperCamelCase__     )
		return state, metrics, new_drp_rng





@partial(jax.pmap		,		axis_name='''batch'''     )
def __A						(   a_ :							Optional[Any]		,		**a_ :							Optional[Any]     )->							Optional[Any]:




		'''simple docstring'''






		SCREAMING_SNAKE_CASE       :     Tuple				= model_inputs.pop('''start_labels'''     )
		SCREAMING_SNAKE_CASE       :     Tuple				= model_inputs.pop('''end_labels'''     )
		SCREAMING_SNAKE_CASE       :     Union[str, Any]				= model_inputs.pop('''pooled_labels'''     )

		SCREAMING_SNAKE_CASE       :     Dict				= state.apply_fn(**UpperCamelCase__		,		params=state.params		,		train=UpperCamelCase__     )
		SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     int				= outputs

		SCREAMING_SNAKE_CASE       :     int				= state.loss_fn(UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__     )
		SCREAMING_SNAKE_CASE       :     List[str]				= jax.lax.pmean({'''loss''': loss}		,		axis_name='''batch'''     )
		return metrics




class 				lowercase__(				train_state.TrainState   ):
	'''simple docstring'''

	UpperCamelCase				=  struct.field(pytree_node=UpperCamelCase__   )





@dataclass
class 				lowercase__:
	'''simple docstring'''

	UpperCamelCase				=  42
	UpperCamelCase				=  42
	UpperCamelCase				=  42
	UpperCamelCase				=  42
	UpperCamelCase				=  42
	UpperCamelCase				=  42
	UpperCamelCase				=  None





	def   __lowerCAmelCase (						self    :Dict		, lowerCamelCase_    :int		, lowerCamelCase_    :str		, lowerCamelCase_    :Optional[Any]		, lowerCamelCase_    :Union[str, Any]=None					)							->			str:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Optional[Any]				= model.params
			SCREAMING_SNAKE_CASE       :     List[Any]				= TrainState.create(
			    apply_fn=model.__call__		, params=__A		, tx=__A		, loss_fn=__A		, )
			if ckpt_dir is not None:
					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     Optional[Any]				= restore_checkpoint(__A		, __A					)
					SCREAMING_SNAKE_CASE       :     Optional[int]				= {
					    '''lr''': args.lr,
					    '''init_lr''': args.init_lr,
					    '''warmup_steps''': args.warmup_steps,
					    '''num_train_steps''': num_train_steps,
					    '''weight_decay''': args.weight_decay,
					}
					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     List[Any]				= build_tx(**__A					)
					SCREAMING_SNAKE_CASE       :     List[Any]				= train_state.TrainState(
					    step=__A		, apply_fn=model.__call__		, params=__A		, tx=__A		, opt_state=__A		, )
					SCREAMING_SNAKE_CASE       :     str				= args
					SCREAMING_SNAKE_CASE       :     Tuple				= data_collator
					SCREAMING_SNAKE_CASE       :     Dict				= lr
					SCREAMING_SNAKE_CASE       :     Dict				= params
			SCREAMING_SNAKE_CASE       :     str				= jax_utils.replicate(__A					)
			return state





	def   __lowerCAmelCase (						self    :Union[str, Any]		, lowerCamelCase_    :Any		, lowerCamelCase_    :Optional[Any]		, lowerCamelCase_    :Optional[Any]					)							->			List[Any]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     str				= self.args
			SCREAMING_SNAKE_CASE       :     Tuple				= len(__A					) // args.batch_size

			SCREAMING_SNAKE_CASE       :     Optional[int]				= jax.random.PRNGKey(0					)
			SCREAMING_SNAKE_CASE       :     Dict				= jax.random.split(__A		, jax.device_count()					)
			for epoch in range(args.max_epochs					):
					SCREAMING_SNAKE_CASE       :     List[Any]				= jnp.array(0		, dtype=jnp.floataa					)
					SCREAMING_SNAKE_CASE       :     str				= get_batched_dataset(__A		, args.batch_size		, seed=__A					)
					SCREAMING_SNAKE_CASE       :     Union[str, Any]				= 0
					for batch in tqdm(__A		, total=__A		, desc=f"Running EPOCH-{epoch}"					):
							SCREAMING_SNAKE_CASE       :     List[Any]				= self.data_collator(__A					)
							SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE,					SCREAMING_SNAKE_CASE       :     List[str]				= self.train_step_fn(__A		, __A		, **__A					)
							running_loss += jax_utils.unreplicate(metrics['''loss''']					)
							i += 1
							if i % args.logging_steps == 0:
									SCREAMING_SNAKE_CASE       :     Dict				= jax_utils.unreplicate(state.step					)
									SCREAMING_SNAKE_CASE       :     List[str]				= running_loss.item() / i
									SCREAMING_SNAKE_CASE       :     str				= self.scheduler_fn(state_step - 1					)

									SCREAMING_SNAKE_CASE       :     List[Any]				= self.evaluate(__A		, __A					)
									SCREAMING_SNAKE_CASE       :     Dict				= {
									    '''step''': state_step.item(),
									    '''eval_loss''': eval_loss.item(),
									    '''tr_loss''': tr_loss,
									    '''lr''': lr.item(),
									}
									tqdm.write(str(__A					)					)
									self.logger.log(__A		, commit=__A					)

							if i % args.save_steps == 0:
									self.save_checkpoint(args.save_dir + f"-e{epoch}-s{i}"		, state=__A					)





	def   __lowerCAmelCase (						self    :str		, lowerCamelCase_    :Union[str, Any]		, lowerCamelCase_    :Optional[int]					)							->			Optional[Any]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Any				= get_batched_dataset(__A		, self.args.batch_size					)
			SCREAMING_SNAKE_CASE       :     List[str]				= len(__A					) // self.args.batch_size
			SCREAMING_SNAKE_CASE       :     Dict				= jnp.array(0		, dtype=jnp.floataa					)
			SCREAMING_SNAKE_CASE       :     int				= 0
			for batch in tqdm(__A		, total=__A		, desc='''Evaluating ... '''					):
					SCREAMING_SNAKE_CASE       :     Tuple				= self.data_collator(__A					)
					SCREAMING_SNAKE_CASE       :     Optional[Any]				= self.val_step_fn(__A		, **__A					)
					running_loss += jax_utils.unreplicate(metrics['''loss''']					)
					i += 1
			return running_loss / i







	def   __lowerCAmelCase (						self    :int		, lowerCamelCase_    :int		, lowerCamelCase_    :Tuple					)							->			Optional[int]:







			'''simple docstring'''




			SCREAMING_SNAKE_CASE       :     Optional[Any]				= jax_utils.unreplicate(__A					)
			print(f"SAVING CHECKPOINT IN {save_dir}"		, end=''' ... '''					)
			self.model_save_fn(__A		, params=state.params					)
			with open(os.path.join(__A		, '''opt_state.msgpack'''					)		, '''wb'''					) as f:
					f.write(to_bytes(state.opt_state					)					)
			joblib.dump(self.args		, os.path.join(__A		, '''args.joblib'''					)					)
			joblib.dump(self.data_collator		, os.path.join(__A		, '''data_collator.joblib'''					)					)
			with open(os.path.join(__A		, '''training_state.json'''					)		, '''w'''					) as f:
					json.dump({'''step''': state.step.item()}		, __A					)
			print('''DONE'''					)





def __A						(   a_ :							List[str]		,		a_ :							str     )->							Optional[Any]:




		'''simple docstring'''






		print(F"RESTORING CHECKPOINT FROM {save_dir}"		,		end=''' ... '''     )
		with open(os.path.join(UpperCamelCase__		,		'''flax_model.msgpack'''     )		,		'''rb'''     ) as f:
				SCREAMING_SNAKE_CASE       :     Dict				= from_bytes(state.params		,		f.read()     )

		with open(os.path.join(UpperCamelCase__		,		'''opt_state.msgpack'''     )		,		'''rb'''     ) as f:
				SCREAMING_SNAKE_CASE       :     Tuple				= from_bytes(state.opt_state		,		f.read()     )

		SCREAMING_SNAKE_CASE       :     str				= joblib.load(os.path.join(UpperCamelCase__		,		'''args.joblib'''     )     )
		SCREAMING_SNAKE_CASE       :     Optional[Any]				= joblib.load(os.path.join(UpperCamelCase__		,		'''data_collator.joblib'''     )     )

		with open(os.path.join(UpperCamelCase__		,		'''training_state.json'''     )		,		'''r'''     ) as f:
				SCREAMING_SNAKE_CASE       :     Tuple				= json.load(UpperCamelCase__     )
		SCREAMING_SNAKE_CASE       :     Union[str, Any]				= training_state['''step''']

		print('''DONE'''     )
		return params, opt_state, step, args, data_collator





def __A						(   a_ :							int		,		a_ :							Optional[Any]		,		a_ :							Optional[int]		,		a_ :							Optional[int]     )->							List[str]:




		'''simple docstring'''






		SCREAMING_SNAKE_CASE       :     Any				= num_train_steps - warmup_steps
		SCREAMING_SNAKE_CASE       :     Dict				= optax.linear_schedule(init_value=UpperCamelCase__		,		end_value=UpperCamelCase__		,		transition_steps=UpperCamelCase__     )
		SCREAMING_SNAKE_CASE       :     Union[str, Any]				= optax.linear_schedule(init_value=UpperCamelCase__		,		end_value=1E-7		,		transition_steps=UpperCamelCase__     )
		SCREAMING_SNAKE_CASE       :     str				= optax.join_schedules(schedules=[warmup_fn, decay_fn]		,		boundaries=[warmup_steps]     )
		return lr





def __A						(   a_ :							Optional[Any]		,		a_ :							Optional[int]		,		a_ :							List[str]		,		a_ :							Optional[int]		,		a_ :							Dict     )->							List[Any]:




		'''simple docstring'''






		def weight_decay_mask(a_ :							Optional[Any]     ):
				SCREAMING_SNAKE_CASE       :     Any				= traverse_util.flatten_dict(UpperCamelCase__     )
				SCREAMING_SNAKE_CASE       :     str				= {k: (v[-1] != '''bias''' and v[-2:] != ('''LayerNorm''', '''scale''')) for k, v in params.items()}
				return traverse_util.unflatten_dict(UpperCamelCase__     )

		SCREAMING_SNAKE_CASE       :     List[Any]				= scheduler_fn(UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__		,		UpperCamelCase__     )

		SCREAMING_SNAKE_CASE       :     str				= optax.adamw(learning_rate=UpperCamelCase__		,		weight_decay=UpperCamelCase__		,		mask=UpperCamelCase__     )
		return tx, lr




"""simple docstring"""






import math





def __A						(   a_ :							list		,		a_ :							int     )->							int:




		'''simple docstring'''






		SCREAMING_SNAKE_CASE       :     Optional[int]				= len(a_     )
		SCREAMING_SNAKE_CASE       :     Optional[Any]				= int(math.floor(math.sqrt(a_     )     )     )
		SCREAMING_SNAKE_CASE       :     List[str]				= 0
		while arr[min(a_		,		a_     ) - 1] < x:
				SCREAMING_SNAKE_CASE       :     Optional[Any]				= step
				step += int(math.floor(math.sqrt(a_     )     )     )
				if prev >= n:
						return -1

		while arr[prev] < x:
				SCREAMING_SNAKE_CASE       :     Any				= prev + 1
				if prev == min(a_		,		a_     ):
						return -1
		if arr[prev] == x:
				return prev
		return -1


if __name__ == "__main__":
					lowerCamelCase__ :       Union[str, Any]						     =			input("Enter numbers separated by a comma:\n").strip()
					lowerCamelCase__ :       List[str]						     =			[int(item) for item in user_input.split(",")]
					lowerCamelCase__ :       Dict						     =			int(input("Enter the number to be searched:\n"))
					lowerCamelCase__ :       Tuple						     =			jump_search(arr, x)
					if res == -1:
										print("Number not found!")
					else:
										print(f'''Number {x} is at index {res}''')





