






'''simple docstring'''







import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel




class 		UpperCamelCase__    :




		'''simple docstring'''





		def __init__(      self   ,_lowerCAmelCase   ,_lowerCAmelCase=13   ,_lowerCAmelCase=7   ,_lowerCAmelCase=True   ,_lowerCAmelCase=True   ,_lowerCAmelCase=False   ,_lowerCAmelCase=True   ,_lowerCAmelCase=99   ,_lowerCAmelCase=32   ,_lowerCAmelCase=5   ,_lowerCAmelCase=4   ,_lowerCAmelCase=37   ,_lowerCAmelCase="gelu"   ,_lowerCAmelCase=0.1   ,_lowerCAmelCase=0.1   ,_lowerCAmelCase=5_12   ,_lowerCAmelCase=16   ,_lowerCAmelCase=2   ,_lowerCAmelCase=0.02   ,_lowerCAmelCase=3   ,_lowerCAmelCase=4   ,_lowerCAmelCase=None   ,):
								lowerCamelCase__         =  parent
								lowerCamelCase__         =  batch_size
								lowerCamelCase__         =  seq_length
								lowerCamelCase__         =  is_training
								lowerCamelCase__         =  use_input_mask
								lowerCamelCase__         =  use_token_type_ids
								lowerCamelCase__         =  use_labels
								lowerCamelCase__         =  vocab_size
								lowerCamelCase__         =  hidden_size
								lowerCamelCase__         =  num_hidden_layers
								lowerCamelCase__         =  num_attention_heads
								lowerCamelCase__         =  intermediate_size
								lowerCamelCase__         =  hidden_act
								lowerCamelCase__         =  hidden_dropout_prob
								lowerCamelCase__         =  attention_probs_dropout_prob
								lowerCamelCase__         =  max_position_embeddings
								lowerCamelCase__         =  type_vocab_size
								lowerCamelCase__         =  type_sequence_label_size
								lowerCamelCase__         =  initializer_range
								lowerCamelCase__         =  num_labels
								lowerCamelCase__         =  num_choices
								lowerCamelCase__         =  scope






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  ids_tensor([self.batch_size, self.seq_length]   ,self.vocab_size       )

								lowerCamelCase__         =  None
								if self.use_input_mask:
														lowerCamelCase__         =  random_attention_mask([self.batch_size, self.seq_length]       )

								lowerCamelCase__         =  None
								if self.use_token_type_ids:
														lowerCamelCase__         =  ids_tensor([self.batch_size, self.seq_length]   ,self.type_vocab_size       )

								lowerCamelCase__         =  None
								lowerCamelCase__         =  None
								lowerCamelCase__         =  None
								if self.use_labels:
														lowerCamelCase__         =  ids_tensor([self.batch_size]   ,self.type_sequence_label_size       )
														lowerCamelCase__         =  ids_tensor([self.batch_size, self.seq_length]   ,self.num_labels       )
														lowerCamelCase__         =  ids_tensor([self.batch_size]   ,self.num_choices       )

								lowerCamelCase__         =  self.get_config()

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






		def 			UpperCamelCase_					(      self       ):
								return OpenLlamaConfig(
								    vocab_size=self.vocab_size   ,hidden_size=self.hidden_size   ,num_hidden_layers=self.num_hidden_layers   ,num_attention_heads=self.num_attention_heads   ,intermediate_size=self.intermediate_size   ,hidden_act=self.hidden_act   ,hidden_dropout_prob=self.hidden_dropout_prob   ,attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,max_position_embeddings=self.max_position_embeddings   ,type_vocab_size=self.type_vocab_size   ,is_decoder=UpperCamelCase_   ,initializer_range=self.initializer_range   ,use_stable_embedding=UpperCamelCase_   ,)






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  OpenLlamaModel(config=UpperCamelCase_       )
								model.to(UpperCamelCase_       )
								model.eval()
								lowerCamelCase__         =  model(UpperCamelCase_   ,attention_mask=UpperCamelCase_       )
								lowerCamelCase__         =  model(UpperCamelCase_       )
								self.parent.assertEqual(result.last_hidden_state.shape   ,(self.batch_size, self.seq_length, self.hidden_size)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,):
								lowerCamelCase__         =  True
								lowerCamelCase__         =  OpenLlamaModel(UpperCamelCase_       )
								model.to(UpperCamelCase_       )
								model.eval()
								lowerCamelCase__         =  model(
								    UpperCamelCase_   ,attention_mask=UpperCamelCase_   ,encoder_hidden_states=UpperCamelCase_   ,encoder_attention_mask=UpperCamelCase_   ,)
								lowerCamelCase__         =  model(
								    UpperCamelCase_   ,attention_mask=UpperCamelCase_   ,encoder_hidden_states=UpperCamelCase_   ,)
								lowerCamelCase__         =  model(UpperCamelCase_   ,attention_mask=UpperCamelCase_       )
								self.parent.assertEqual(result.last_hidden_state.shape   ,(self.batch_size, self.seq_length, self.hidden_size)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,):
								lowerCamelCase__         =  OpenLlamaForCausalLM(config=UpperCamelCase_       )
								model.to(UpperCamelCase_       )
								model.eval()
								lowerCamelCase__         =  model(UpperCamelCase_   ,attention_mask=UpperCamelCase_   ,labels=UpperCamelCase_       )
								self.parent.assertEqual(result.logits.shape   ,(self.batch_size, self.seq_length, self.vocab_size)       )






		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,_lowerCAmelCase   ,):
								lowerCamelCase__         =  True
								lowerCamelCase__         =  True
								lowerCamelCase__         =  OpenLlamaForCausalLM(config=UpperCamelCase_       )
								model.to(UpperCamelCase_       )
								model.eval()

								# first forward pass
								lowerCamelCase__         =  model(
								    UpperCamelCase_   ,attention_mask=UpperCamelCase_   ,encoder_hidden_states=UpperCamelCase_   ,encoder_attention_mask=UpperCamelCase_   ,use_cache=UpperCamelCase_   ,)
								lowerCamelCase__         =  outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								lowerCamelCase__         =  ids_tensor((self.batch_size, 3)   ,config.vocab_size       )
								lowerCamelCase__         =  ids_tensor((self.batch_size, 3)   ,vocab_size=2       )

								# append to next input_ids and
								lowerCamelCase__         =  torch.cat([input_ids, next_tokens]   ,dim=-1       )
								lowerCamelCase__         =  torch.cat([input_mask, next_mask]   ,dim=-1       )

								lowerCamelCase__         =  model(
								    UpperCamelCase_   ,attention_mask=UpperCamelCase_   ,encoder_hidden_states=UpperCamelCase_   ,encoder_attention_mask=UpperCamelCase_   ,output_hidden_states=UpperCamelCase_   ,)['hidden_states'][0]
								lowerCamelCase__         =  model(
								    UpperCamelCase_   ,attention_mask=UpperCamelCase_   ,encoder_hidden_states=UpperCamelCase_   ,encoder_attention_mask=UpperCamelCase_   ,past_key_values=UpperCamelCase_   ,output_hidden_states=UpperCamelCase_   ,)['hidden_states'][0]

								# select random slice
								lowerCamelCase__         =  ids_tensor((1,)   ,output_from_past.shape[-1]       ).item()
								lowerCamelCase__         =  output_from_no_past[:, -3:, random_slice_idx].detach()
								lowerCamelCase__         =  output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(UpperCamelCase_   ,UpperCamelCase_   ,atol=1E-3       )       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.prepare_config_and_inputs()
								(
								    lowerCamelCase__    
								)         =  config_and_inputs
								lowerCamelCase__         =  {'input_ids': input_ids, 'attention_mask': input_mask}
								return config, inputs_dict




@require_torch
class 		UpperCamelCase__    (a   ,a   ,a   ,unittest.TestCase			):




		'''simple docstring'''





		_UpperCamelCase				  =					(
		    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
		)
		_UpperCamelCase				  =					(OpenLlamaForCausalLM,) if is_torch_available() else ()
		_UpperCamelCase				  =					(
		    {
		        'feature-extraction': OpenLlamaModel,
		        'text-classification': OpenLlamaForSequenceClassification,
		        'text-generation': OpenLlamaForCausalLM,
		        'zero-shot': OpenLlamaForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		_UpperCamelCase				  =					False
		_UpperCamelCase				  =					False






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  OpenLlamaModelTester(self       )
								lowerCamelCase__         =  ConfigTester(self   ,config_class=UpperCamelCase_   ,hidden_size=37       )






		def 			UpperCamelCase_					(      self       ):
								self.config_tester.run_common_tests()






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*UpperCamelCase_       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
														lowerCamelCase__         =  type
														self.model_tester.create_and_check_model(*UpperCamelCase_       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs_for_common()
								lowerCamelCase__         =  3
								lowerCamelCase__         =  input_dict['input_ids']
								lowerCamelCase__         =  input_ids.ne(1       ).to(UpperCamelCase_       )
								lowerCamelCase__         =  ids_tensor([self.model_tester.batch_size]   ,self.model_tester.type_sequence_label_size       )
								lowerCamelCase__         =  OpenLlamaForSequenceClassification(UpperCamelCase_       )
								model.to(UpperCamelCase_       )
								model.eval()
								lowerCamelCase__         =  model(UpperCamelCase_   ,attention_mask=UpperCamelCase_   ,labels=UpperCamelCase_       )
								self.assertEqual(result.logits.shape   ,(self.model_tester.batch_size, self.model_tester.num_labels)       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs_for_common()
								lowerCamelCase__         =  3
								lowerCamelCase__         =  'single_label_classification'
								lowerCamelCase__         =  input_dict['input_ids']
								lowerCamelCase__         =  input_ids.ne(1       ).to(UpperCamelCase_       )
								lowerCamelCase__         =  ids_tensor([self.model_tester.batch_size]   ,self.model_tester.type_sequence_label_size       )
								lowerCamelCase__         =  OpenLlamaForSequenceClassification(UpperCamelCase_       )
								model.to(UpperCamelCase_       )
								model.eval()
								lowerCamelCase__         =  model(UpperCamelCase_   ,attention_mask=UpperCamelCase_   ,labels=UpperCamelCase_       )
								self.assertEqual(result.logits.shape   ,(self.model_tester.batch_size, self.model_tester.num_labels)       )






		def 			UpperCamelCase_					(      self       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs_for_common()
								lowerCamelCase__         =  3
								lowerCamelCase__         =  'multi_label_classification'
								lowerCamelCase__         =  input_dict['input_ids']
								lowerCamelCase__         =  input_ids.ne(1       ).to(UpperCamelCase_       )
								lowerCamelCase__         =  ids_tensor(
								    [self.model_tester.batch_size, config.num_labels]   ,self.model_tester.type_sequence_label_size       ).to(torch.float       )
								lowerCamelCase__         =  OpenLlamaForSequenceClassification(UpperCamelCase_       )
								model.to(UpperCamelCase_       )
								model.eval()
								lowerCamelCase__         =  model(UpperCamelCase_   ,attention_mask=UpperCamelCase_   ,labels=UpperCamelCase_       )
								self.assertEqual(result.logits.shape   ,(self.model_tester.batch_size, self.model_tester.num_labels)       )






		@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test"""       )
		def 			UpperCamelCase_					(      self       ):
								pass






		@parameterized.expand([("""linear""",), ("""dynamic""",)]       )
		def 			UpperCamelCase_					(      self   ,_lowerCAmelCase       ):
								lowerCamelCase__         =  self.model_tester.prepare_config_and_inputs_for_common()
								lowerCamelCase__         =  ids_tensor([1, 10]   ,config.vocab_size       )
								lowerCamelCase__         =  ids_tensor([1, int(config.max_position_embeddings * 1.5       )]   ,config.vocab_size       )

								set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
								lowerCamelCase__         =  OpenLlamaModel(UpperCamelCase_       )
								original_model.to(UpperCamelCase_       )
								original_model.eval()
								lowerCamelCase__         =  original_model(UpperCamelCase_       ).last_hidden_state
								lowerCamelCase__         =  original_model(UpperCamelCase_       ).last_hidden_state

								set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
								lowerCamelCase__         =  {'type': scaling_type, 'factor': 10.0}
								lowerCamelCase__         =  OpenLlamaModel(UpperCamelCase_       )
								scaled_model.to(UpperCamelCase_       )
								scaled_model.eval()
								lowerCamelCase__         =  scaled_model(UpperCamelCase_       ).last_hidden_state
								lowerCamelCase__         =  scaled_model(UpperCamelCase_       ).last_hidden_state

								# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
								# maximum sequence length, so the outputs for the short input should match.
								if scaling_type == "dynamic":
														self.assertTrue(torch.allclose(UpperCamelCase_   ,UpperCamelCase_   ,atol=1E-5       )       )
								else:
														self.assertFalse(torch.allclose(UpperCamelCase_   ,UpperCamelCase_   ,atol=1E-5       )       )

								# The output should be different for long inputs
								self.assertFalse(torch.allclose(UpperCamelCase_   ,UpperCamelCase_   ,atol=1E-5       )       )









'''simple docstring'''







import re

import jax.numpy as jnp
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.random import PRNGKey

from ..utils import logging


UpperCamelCase    :      Union[str, Any]  			=		logging.get_logger(__name__)
def 						A__			(				__lowerCAmelCase		:			int						):
						lowerCamelCase__         =  R"""\w+[.]\d+"""
						lowerCamelCase__         =  re.findall(__lowerCAmelCase   ,    __lowerCAmelCase						)
						for pat in pats:
												lowerCamelCase__         =  key.replace(__lowerCAmelCase   ,    """_""".join(pat.split("""."""						)						)						)
						return key
def 						A__			(				__lowerCAmelCase		:			Union[str, Any]   ,    __lowerCAmelCase		:			Union[str, Any]   ,    __lowerCAmelCase		:			Tuple						):
						lowerCamelCase__         =  pt_tuple_key[:-1] + ("""scale""",)
						if (
						    any("""norm""" in str_ for str_ in pt_tuple_key						)
						    and (pt_tuple_key[-1] == "bias")
						    and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict)
						    and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict)
						):
												lowerCamelCase__         =  pt_tuple_key[:-1] + ("""scale""",)
												return renamed_pt_tuple_key, pt_tensor
						elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict:
												lowerCamelCase__         =  pt_tuple_key[:-1] + ("""scale""",)
												return renamed_pt_tuple_key, pt_tensor

						# embedding
						if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict:
												lowerCamelCase__         =  pt_tuple_key[:-1] + ("""embedding""",)
												return renamed_pt_tuple_key, pt_tensor

						# conv layer
						lowerCamelCase__         =  pt_tuple_key[:-1] + ("""kernel""",)
						if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4:
												lowerCamelCase__         =  pt_tensor.transpose(2   ,    3   ,    1   ,    0						)
												return renamed_pt_tuple_key, pt_tensor

						# linear layer
						lowerCamelCase__         =  pt_tuple_key[:-1] + ("""kernel""",)
						if pt_tuple_key[-1] == "weight":
												lowerCamelCase__         =  pt_tensor.T
												return renamed_pt_tuple_key, pt_tensor

						# old PyTorch layer norm weight
						lowerCamelCase__         =  pt_tuple_key[:-1] + ("""weight""",)
						if pt_tuple_key[-1] == "gamma":
												return renamed_pt_tuple_key, pt_tensor

						# old PyTorch layer norm bias
						lowerCamelCase__         =  pt_tuple_key[:-1] + ("""bias""",)
						if pt_tuple_key[-1] == "beta":
												return renamed_pt_tuple_key, pt_tensor

						return pt_tuple_key, pt_tensor





def 						A__			(				__lowerCAmelCase		:			int   ,    __lowerCAmelCase		:			Union[str, Any]   ,    __lowerCAmelCase		:			Tuple=42						):
						# Step 1: Convert pytorch tensor to numpy
						lowerCamelCase__         =  {k: v.numpy() for k, v in pt_state_dict.items()}

						# Step 2: Since the model is stateless, get random Flax params
						lowerCamelCase__         =  flax_model.init_weights(PRNGKey(__lowerCAmelCase						)						)

						lowerCamelCase__         =  flatten_dict(__lowerCAmelCase						)
						lowerCamelCase__         =  {}

						# Need to change some parameters name to match Flax names
						for pt_key, pt_tensor in pt_state_dict.items():
												lowerCamelCase__         =  rename_key(__lowerCAmelCase						)
												lowerCamelCase__         =  tuple(renamed_pt_key.split("""."""						)						)

												# Correctly rename weight parameters
												lowerCamelCase__       ,			lowerCamelCase__         =  rename_key_and_reshape_tensor(__lowerCAmelCase   ,    __lowerCAmelCase   ,    __lowerCAmelCase						)

												if flax_key in random_flax_state_dict:
																		if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
																								raise ValueError(
																								    F'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
																								    F'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''						)

        # also add unexpected weight so that warning is thrown
												lowerCamelCase__         =  jnp.asarray(__lowerCAmelCase						)

						return unflatten_dict(__lowerCAmelCase						)

