





"""simple docstring"""


import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaTokenizer, GPTJConfig, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, tooslow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
    import jax
    import jax.numpy as jnp

    from transformers.modeling_flax_pytorch_utils import (
        convert_pytorch_state_dict_to_flax,
        load_flax_weights_in_pytorch_model,
    )
    from transformers.models.gptj.modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel

if is_torch_available():
    import torch


class 			A__	:


     """simple docstring"""

     def __init__(	self:						str     ,		__a:						List[Any]     ,		__a:						Any=14     ,		__a:						Optional[Any]=7     ,		__a:						Optional[int]=True     ,		__a:						List[Any]=True     ,		__a:						int=False     ,		__a:						str=True     ,		__a:						Optional[int]=99     ,		__a:						Union[str, Any]=32     ,		__a:						Union[str, Any]=4     ,		__a:						List[Any]=4     ,		__a:						int=4     ,		__a:						List[str]=37     ,		__a:						int="gelu"     ,		__a:						List[Any]=0.1     ,		__a:						List[str]=0.1     ,		__a:						Optional[Any]=512     ,		__a:						int=0.02     ,		)->    List[Any]:
           lowerCamelCase					:					List[Any]        =	parent
           lowerCamelCase					:					Optional[int]        =	batch_size
           lowerCamelCase					:					int        =	seq_length
           lowerCamelCase					:					int        =	is_training
           lowerCamelCase					:					int        =	use_input_mask
           lowerCamelCase					:					Union[str, Any]        =	use_token_type_ids
           lowerCamelCase					:					Dict        =	use_labels
           lowerCamelCase					:					Any        =	vocab_size
           lowerCamelCase					:					int        =	hidden_size
           lowerCamelCase					:					Dict        =	rotary_dim
           lowerCamelCase					:					Optional[Any]        =	num_hidden_layers
           lowerCamelCase					:					List[str]        =	num_attention_heads
           lowerCamelCase					:					List[Any]        =	intermediate_size
           lowerCamelCase					:					int        =	hidden_act
           lowerCamelCase					:					List[Any]        =	hidden_dropout_prob
           lowerCamelCase					:					Tuple        =	attention_probs_dropout_prob
           lowerCamelCase					:					str        =	max_position_embeddings
           lowerCamelCase					:					int        =	initializer_range
           lowerCamelCase					:					Tuple        =	None
           lowerCamelCase					:					List[Any]        =	vocab_size - 1
           lowerCamelCase					:					Dict        =	vocab_size - 1
           lowerCamelCase					:					Tuple        =	vocab_size - 1

     def 		a__						(	self:						Optional[Any]	)->    Optional[int]:
           lowerCamelCase					:					Optional[int]        =	ids_tensor([self.batch_size, self.seq_length]     ,		self.vocab_size	)

           lowerCamelCase					:					str        =	None
           if self.use_input_mask:
                 lowerCamelCase					:					Any        =	random_attention_mask([self.batch_size, self.seq_length]	)

           lowerCamelCase					:					List[Any]        =	GPTJConfig(
               vocab_size=self.vocab_size     ,		n_embd=self.hidden_size     ,		n_layer=self.num_hidden_layers     ,		n_head=self.num_attention_heads     ,		n_positions=self.max_position_embeddings     ,		use_cache=__A     ,		bos_token_id=self.bos_token_id     ,		eos_token_id=self.eos_token_id     ,		pad_token_id=self.pad_token_id     ,		rotary_dim=self.rotary_dim     ,		)

           return (config, input_ids, input_mask)

     def 		a__						(	self:						Optional[int]	)->    List[Any]:
           lowerCamelCase					:					str        =	self.prepare_config_and_inputs()
           lowerCamelCase     ,			lowerCamelCase     ,			lowerCamelCase					:					Union[str, Any]        =	config_and_inputs
           lowerCamelCase					:					Tuple        =	{"""input_ids""": input_ids, """attention_mask""": attention_mask}
           return config, inputs_dict

     def 		a__						(	self:						int     ,		__a:						int     ,		__a:						Any     ,		__a:						Any     ,		__a:						Optional[Any]	)->    Dict:
           lowerCamelCase					:					str        =	20
           lowerCamelCase					:					str        =	model_class_name(__A	)

           lowerCamelCase					:					Optional[int]        =	model.init_cache(input_ids.shape[0]     ,		__A	)
           lowerCamelCase					:					Dict        =	jnp.ones((input_ids.shape[0], max_decoder_length)     ,		dtype="""i4"""	)

           lowerCamelCase					:					Any        =	jnp.broadcast_to(
               jnp.arange(input_ids.shape[-1] - 1	)[None, :]     ,		(input_ids.shape[0], input_ids.shape[-1] - 1)	)
           lowerCamelCase					:					Tuple        =	model(
               input_ids[:, :-1]     ,		attention_mask=__A     ,		past_key_values=__A     ,		position_ids=__A     ,		)

           lowerCamelCase					:					str        =	jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]     ,		dtype="""i4"""	)
           lowerCamelCase					:					Any        =	model(
               input_ids[:, -1:]     ,		attention_mask=__A     ,		past_key_values=outputs_cache.past_key_values     ,		position_ids=__A     ,		)

           lowerCamelCase					:					Any        =	model(__A	)

           lowerCamelCase					:					Tuple        =	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
           self.parent.assertTrue(diff < 1e-3     ,		msg=f'Max diff is {diff}'	)







     def 		a__						(	self:						Dict     ,		__a:						List[Any]     ,		__a:						List[str]     ,		__a:						Optional[Any]     ,		__a:						Union[str, Any]	)->    Optional[int]:
           lowerCamelCase					:					str        =	20
           lowerCamelCase					:					int        =	model_class_name(__A	)

           lowerCamelCase					:					str        =	jnp.concatenate(
               [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])	)]     ,		axis=-1     ,		)

           lowerCamelCase					:					Tuple        =	model.init_cache(input_ids.shape[0]     ,		__A	)
           lowerCamelCase					:					Any        =	jnp.broadcast_to(
               jnp.arange(input_ids.shape[-1] - 1	)[None, :]     ,		(input_ids.shape[0], input_ids.shape[-1] - 1)	)

           lowerCamelCase					:					List[str]        =	model(
               input_ids[:, :-1]     ,		attention_mask=__A     ,		past_key_values=__A     ,		position_ids=__A     ,		)
           lowerCamelCase					:					List[str]        =	jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]     ,		dtype="""i4"""	)
           lowerCamelCase					:					Tuple        =	model(
               input_ids[:, -1:]     ,		past_key_values=outputs_cache.past_key_values     ,		attention_mask=__A     ,		position_ids=__A     ,		)

           lowerCamelCase					:					Tuple        =	model(__A     ,		attention_mask=__A	)

           lowerCamelCase					:					Union[str, Any]        =	np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
           self.parent.assertTrue(diff < 1e-3     ,		msg=f'Max diff is {diff}'	)


@require_flax
class 			A__	(						UpperCamelCase__ ,     UpperCamelCase__ ,     unittest.TestCase):


     """simple docstring"""
     snake_case__	:					Union[str, Any]													=(FlaxGPTJModel, FlaxGPTJForCausalLM) if is_flax_available() else ()
     snake_case__	:					Optional[Any]													=(FlaxGPTJForCausalLM,) if is_flax_available() else ()

     def 		a__						(	self:						Tuple	)->    Union[str, Any]:
           lowerCamelCase					:					Optional[int]        =	FlaxGPTJModelTester(self	)

     def 		a__						(	self:						Union[str, Any]	)->    int:



           for model_class_name in self.all_model_classes:
                 lowerCamelCase     ,			lowerCamelCase     ,			lowerCamelCase					:					Optional[int]        =	self.model_tester.prepare_config_and_inputs()
                 self.model_tester.check_use_cache_forward(__A     ,		__A     ,		__A     ,		__A	)

     def 		a__						(	self:						Dict	)->    Dict:



           for model_class_name in self.all_model_classes:
                 lowerCamelCase     ,			lowerCamelCase     ,			lowerCamelCase					:					List[str]        =	self.model_tester.prepare_config_and_inputs()
                 self.model_tester.check_use_cache_forward_with_attn_mask(
                     __A     ,		__A     ,		__A     ,		__A	)

     @tooslow
     def 		a__						(	self:						List[str]	)->    int:
           lowerCamelCase					:					Any        =	GPTaTokenizer.from_pretrained("""gpt2"""     ,		pad_token="""<|endoftext|>"""     ,		padding_side="""left"""	)
           lowerCamelCase					:					Union[str, Any]        =	tokenizer(["""Hello this is a long string""", """Hey"""]     ,		return_tensors="""np"""     ,		padding=__A     ,		truncation=__A	)

           lowerCamelCase					:					int        =	FlaxGPTJForCausalLM.from_pretrained("""EleutherAI/gpt-j-6B"""	)
           lowerCamelCase					:					int        =	False
           lowerCamelCase					:					Optional[Any]        =	model.config.eos_token_id

           lowerCamelCase					:					Union[str, Any]        =	jax.jit(model.generate	)

           lowerCamelCase					:					str        =	jit_generate(
               inputs["""input_ids"""]     ,		attention_mask=inputs["""attention_mask"""]     ,		pad_token_id=tokenizer.pad_token_id	).sequences

           lowerCamelCase					:					Union[str, Any]        =	tokenizer.batch_decode(__A     ,		skip_special_tokens=__A	)

           lowerCamelCase					:					List[str]        =	[
               """Hello this is a long string of text.\n\nI'm trying to get the text of the""",
               """Hey, I'm a little late to the party. I'm going to""",
           ]

           self.assertListEqual(__A     ,		__A	)

     @is_pt_flax_cross_test
     def 		a__						(	self:						Any	)->    str:
           lowerCamelCase     ,			lowerCamelCase					:					Any        =	self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                 with self.subTest(model_class.__name__	):
                       # prepare inputs
                       lowerCamelCase					:					Any        =	self._prepare_for_class(__A     ,		__A	)
                       lowerCamelCase					:					List[Any]        =	{k: torch.tensor(v.tolist()	) for k, v in prepared_inputs_dict.items()}

                       # load corresponding PyTorch class
                       lowerCamelCase					:					List[Any]        =	model_class.__name__[4:]  # Skip the "Flax" at the beginning
                       lowerCamelCase					:					int        =	getattr(__A     ,		__A	)

                       lowerCamelCase     ,			lowerCamelCase					:					Optional[int]        =	pt_inputs["""input_ids"""].shape
                       lowerCamelCase					:					List[str]        =	np.random.randint(0     ,		seq_length - 1     ,		size=(batch_size,)	)
                       for batch_idx, start_index in enumerate(__A	):
                             lowerCamelCase					:					Dict        =	0
                             lowerCamelCase					:					List[str]        =	1
                             lowerCamelCase					:					Any        =	0
                             lowerCamelCase					:					List[str]        =	1
                       lowerCamelCase					:					Optional[Any]        =	pt_model_class(__A	).eval()
                       lowerCamelCase					:					Optional[Any]        =	model_class(__A     ,		dtype=jnp.floataa	)

                       lowerCamelCase					:					int        =	convert_pytorch_state_dict_to_flax(pt_model.state_dict()     ,		__A	)
                       lowerCamelCase					:					Any        =	fx_state

                       with torch.no_grad():
                             lowerCamelCase					:					int        =	pt_model(**__A	).to_tuple()

                       lowerCamelCase					:					Any        =	fx_model(**__A	).to_tuple()
                       self.assertEqual(len(__A	)     ,		len(__A	)     ,		"""Output lengths differ between Flax and PyTorch"""	)
                       for fx_output, pt_output in zip(__A     ,		__A	):
                             self.assert_almost_equals(fx_output[:, -1]     ,		pt_output[:, -1].numpy()     ,		4e-2	)

                       with tempfile.TemporaryDirectory() as tmpdirname:
                             pt_model.save_pretrained(__A	)
                             lowerCamelCase					:					Any        =	model_class.from_pretrained(__A     ,		from_pt=__A	)

                       lowerCamelCase					:					int        =	fx_model_loaded(**__A	).to_tuple()
                       self.assertEqual(
                           len(__A	)     ,		len(__A	)     ,		"""Output lengths differ between Flax and PyTorch"""	)
                       for fx_output_loaded, pt_output in zip(__A     ,		__A	):
                             self.assert_almost_equals(fx_output_loaded[:, -1]     ,		pt_output[:, -1].numpy()     ,		4e-2	)

     @is_pt_flax_cross_test
     def 		a__						(	self:						Any	)->    Dict:
           lowerCamelCase     ,			lowerCamelCase					:					Tuple        =	self.model_tester.prepare_config_and_inputs_for_common()
           for model_class in self.all_model_classes:
                 with self.subTest(model_class.__name__	):
                       # prepare inputs
                       lowerCamelCase					:					List[Any]        =	self._prepare_for_class(__A     ,		__A	)
                       lowerCamelCase					:					List[Any]        =	{k: torch.tensor(v.tolist()	) for k, v in prepared_inputs_dict.items()}

                       # load corresponding PyTorch class
                       lowerCamelCase					:					List[Any]        =	model_class.__name__[4:]  # Skip the "Flax" at the beginning
                       lowerCamelCase					:					Union[str, Any]        =	getattr(__A     ,		__A	)

                       lowerCamelCase					:					List[str]        =	pt_model_class(__A	).eval()
                       lowerCamelCase					:					Tuple        =	model_class(__A     ,		dtype=jnp.floataa	)

                       lowerCamelCase					:					Optional[Any]        =	load_flax_weights_in_pytorch_model(__A     ,		fx_model.params	)
                       lowerCamelCase     ,			lowerCamelCase					:					str        =	pt_inputs["""input_ids"""].shape
                       lowerCamelCase					:					int        =	np.random.randint(0     ,		seq_length - 1     ,		size=(batch_size,)	)
                       for batch_idx, start_index in enumerate(__A	):
                             lowerCamelCase					:					Union[str, Any]        =	0
                             lowerCamelCase					:					int        =	1
                             lowerCamelCase					:					Dict        =	0
                             lowerCamelCase					:					str        =	1

                       # make sure weights are tied in PyTorch
                       pt_model.tie_weights()

                       with torch.no_grad():
                             lowerCamelCase					:					Union[str, Any]        =	pt_model(**__A	).to_tuple()

                       lowerCamelCase					:					Optional[int]        =	fx_model(**__A	).to_tuple()
                       self.assertEqual(len(__A	)     ,		len(__A	)     ,		"""Output lengths differ between Flax and PyTorch"""	)
                       for fx_output, pt_output in zip(__A     ,		__A	):
                             self.assert_almost_equals(fx_output[:, -1]     ,		pt_output[:, -1].numpy()     ,		4e-2	)

                       with tempfile.TemporaryDirectory() as tmpdirname:
                             fx_model.save_pretrained(__A	)
                             lowerCamelCase					:					Dict        =	pt_model_class.from_pretrained(__A     ,		from_flax=__A	)

                       with torch.no_grad():
                             lowerCamelCase					:					int        =	pt_model_loaded(**__A	).to_tuple()

                       self.assertEqual(
                           len(__A	)     ,		len(__A	)     ,		"""Output lengths differ between Flax and PyTorch"""	)
                       for fx_output, pt_output in zip(__A     ,		__A	):
                             self.assert_almost_equals(fx_output[:, -1]     ,		pt_output[:, -1].numpy()     ,		4e-2	)







     @tooslow
     def 		a__						(	self:						Union[str, Any]	)->    Tuple:



           for model_class_name in self.all_model_classes:
                 lowerCamelCase					:					Union[str, Any]        =	model_class_name.from_pretrained("""EleutherAI/gpt-j-6B"""	)
                 lowerCamelCase					:					Optional[int]        =	model(np.ones((1, 1)	)	)
                 self.assertIsNotNone(__A	)








"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase      :Optional[int]	    =		logging.get_logger(__name__)

__lowerCamelCase      :List[str]	    =		{
    'google/realm-cc-news-pretrained-embedder': (
        'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json'
    ),
    'google/realm-cc-news-pretrained-encoder': (
        'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json'
    ),
    'google/realm-cc-news-pretrained-scorer': (
        'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json'
    ),
    'google/realm-cc-news-pretrained-openqa': (
        'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json'
    ),
    'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json',
    'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json',
    'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json',
    'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json',
    # See all REALM models at https://huggingface.co/models?filter=realm
}


class 			A__	(						__lowercase):


     """simple docstring"""
     snake_case__	:					Optional[Any]													='''realm'''

     def __init__(	self:						Union[str, Any]     ,		__a:						List[Any]=30_522     ,		__a:						List[Any]=768     ,		__a:						List[Any]=128     ,		__a:						Union[str, Any]=12     ,		__a:						Union[str, Any]=12     ,		__a:						Optional[Any]=8     ,		__a:						Dict=3_072     ,		__a:						List[Any]="gelu_new"     ,		__a:						List[Any]=0.1     ,		__a:						Tuple=0.1     ,		__a:						Optional[Any]=512     ,		__a:						Optional[int]=2     ,		__a:						str=0.02     ,		__a:						int=1e-1_2     ,		__a:						Optional[Any]=256     ,		__a:						Any=10     ,		__a:						Dict=1e-3     ,		__a:						Optional[Any]=5     ,		__a:						Dict=320     ,		__a:						Tuple=13_353_718     ,		__a:						List[Any]=5_000     ,		__a:						Dict=1     ,		__a:						int=0     ,		__a:						Dict=2     ,		**__a:						List[str]     ,		)->    Any:
           super().__init__(pad_token_id=__a     ,		bos_token_id=__a     ,		eos_token_id=__a     ,		**__a	)

           # Common config
           lowerCamelCase					:					Optional[Any]        =	vocab_size
           lowerCamelCase					:					str        =	max_position_embeddings
           lowerCamelCase					:					Dict        =	hidden_size
           lowerCamelCase					:					Dict        =	retriever_proj_size
           lowerCamelCase					:					Optional[Any]        =	num_hidden_layers
           lowerCamelCase					:					List[str]        =	num_attention_heads
           lowerCamelCase					:					Tuple        =	num_candidates
           lowerCamelCase					:					int        =	intermediate_size
           lowerCamelCase					:					Dict        =	hidden_act
           lowerCamelCase					:					List[str]        =	hidden_dropout_prob
           lowerCamelCase					:					Dict        =	attention_probs_dropout_prob
           lowerCamelCase					:					Optional[int]        =	initializer_range
           lowerCamelCase					:					Dict        =	type_vocab_size
           lowerCamelCase					:					Optional[Any]        =	layer_norm_eps

           # Reader config
           lowerCamelCase					:					List[str]        =	span_hidden_size
           lowerCamelCase					:					Dict        =	max_span_width
           lowerCamelCase					:					Optional[Any]        =	reader_layer_norm_eps
           lowerCamelCase					:					Optional[int]        =	reader_beam_size
           lowerCamelCase					:					List[Any]        =	reader_seq_len

           # Retrieval config
           lowerCamelCase					:					int        =	num_block_records
           lowerCamelCase					:					Dict        =	searcher_beam_size




