



"""simple docstring"""

import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel







class    __lowercase     :







						"""simple docstring"""







						def __init__(self   ,		lowercase__   ,		lowercase__=13   ,		lowercase__=7   ,		lowercase__=True   ,		lowercase__=True   ,		lowercase__=False   ,		lowercase__=True   ,		lowercase__=99   ,		lowercase__=32   ,		lowercase__=5   ,		lowercase__=4   ,		lowercase__=37   ,		lowercase__="gelu"   ,		lowercase__=0.1   ,		lowercase__=0.1   ,		lowercase__=5_12   ,		lowercase__=16   ,		lowercase__=2   ,		lowercase__=0.02   ,		lowercase__=3   ,		lowercase__=4   ,		lowercase__=None   ,		):
											snake_case_					:		List[str] 					=					parent
											snake_case_					:		int 					=					batch_size
											snake_case_					:		int 					=					seq_length
											snake_case_					:		Dict 					=					is_training
											snake_case_					:		List[Any] 					=					use_input_mask
											snake_case_					:		List[Any] 					=					use_token_type_ids
											snake_case_					:		Union[str, Any] 					=					use_labels
											snake_case_					:		Optional[int] 					=					vocab_size
											snake_case_					:		Dict 					=					hidden_size
											snake_case_					:		Optional[Any] 					=					num_hidden_layers
											snake_case_					:		Any 					=					num_attention_heads
											snake_case_					:		Dict 					=					intermediate_size
											snake_case_					:		Union[str, Any] 					=					hidden_act
											snake_case_					:		Dict 					=					hidden_dropout_prob
											snake_case_					:		Dict 					=					attention_probs_dropout_prob
											snake_case_					:		Union[str, Any] 					=					max_position_embeddings
											snake_case_					:		Any 					=					type_vocab_size
											snake_case_					:		List[Any] 					=					type_sequence_label_size
											snake_case_					:		Optional[int] 					=					initializer_range
											snake_case_					:		Union[str, Any] 					=					num_labels
											snake_case_					:		List[str] 					=					num_choices
											snake_case_					:		Optional[int] 					=					scope







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		List[str] 					=					ids_tensor([self.batch_size, self.seq_length]   ,		self.vocab_size       )

											snake_case_					:		Any 					=					None
											if self.use_input_mask:
																snake_case_					:		str 					=					random_attention_mask([self.batch_size, self.seq_length]       )

											snake_case_					:		str 					=					None
											if self.use_token_type_ids:
																snake_case_					:		Dict 					=					ids_tensor([self.batch_size, self.seq_length]   ,		self.type_vocab_size       )

											snake_case_					:		Optional[int] 					=					None
											snake_case_					:		List[str] 					=					None
											snake_case_					:		str 					=					None
											if self.use_labels:
																snake_case_					:		int 					=					ids_tensor([self.batch_size]   ,		self.type_sequence_label_size       )
																snake_case_					:		List[str] 					=					ids_tensor([self.batch_size, self.seq_length]   ,		self.num_labels       )
																snake_case_					:		List[Any] 					=					ids_tensor([self.batch_size]   ,		self.num_choices       )

											snake_case_					:		List[str] 					=					self.get_config()

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







						def 						__UpperCamelCase							(self       ):
											return OpenLlamaConfig(
											    vocab_size=self.vocab_size   ,		hidden_size=self.hidden_size   ,		num_hidden_layers=self.num_hidden_layers   ,		num_attention_heads=self.num_attention_heads   ,		intermediate_size=self.intermediate_size   ,		hidden_act=self.hidden_act   ,		hidden_dropout_prob=self.hidden_dropout_prob   ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,		max_position_embeddings=self.max_position_embeddings   ,		type_vocab_size=self.type_vocab_size   ,		is_decoder=lowercase__   ,		initializer_range=self.initializer_range   ,		use_stable_embedding=lowercase__   ,		)







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__       ):
											snake_case_					:		Optional[int] 					=					OpenLlamaModel(config=lowercase__       )
											model.to(lowercase__       )
											model.eval()
											snake_case_					:		Dict 					=					model(lowercase__   ,		attention_mask=lowercase__       )
											snake_case_					:		Optional[int] 					=					model(lowercase__       )
											self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.hidden_size)       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		List[Any] 					=					True
											snake_case_					:		Tuple 					=					OpenLlamaModel(lowercase__       )
											model.to(lowercase__       )
											model.eval()
											snake_case_					:		List[Any] 					=					model(
											    lowercase__   ,		attention_mask=lowercase__   ,		encoder_hidden_states=lowercase__   ,		encoder_attention_mask=lowercase__   ,		)
											snake_case_					:		Dict 					=					model(
											    lowercase__   ,		attention_mask=lowercase__   ,		encoder_hidden_states=lowercase__   ,		)
											snake_case_					:		Any 					=					model(lowercase__   ,		attention_mask=lowercase__       )
											self.parent.assertEqual(result.last_hidden_state.shape   ,		(self.batch_size, self.seq_length, self.hidden_size)       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		str 					=					OpenLlamaForCausalLM(config=lowercase__       )
											model.to(lowercase__       )
											model.eval()
											snake_case_					:		Dict 					=					model(lowercase__   ,		attention_mask=lowercase__   ,		labels=lowercase__       )
											self.parent.assertEqual(result.logits.shape   ,		(self.batch_size, self.seq_length, self.vocab_size)       )







						def 						__UpperCamelCase							(self   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		lowercase__   ,		):
											snake_case_					:		List[str] 					=					True
											snake_case_					:		Dict 					=					True
											snake_case_					:		int 					=					OpenLlamaForCausalLM(config=lowercase__       )
											model.to(lowercase__       )
											model.eval()

											# first forward pass
											snake_case_					:		Optional[Any] 					=					model(
											    lowercase__   ,		attention_mask=lowercase__   ,		encoder_hidden_states=lowercase__   ,		encoder_attention_mask=lowercase__   ,		use_cache=lowercase__   ,		)
											snake_case_					:		Optional[int] 					=					outputs.past_key_values

											# create hypothetical multiple next token and extent to next_input_ids
											snake_case_					:		List[str] 					=					ids_tensor((self.batch_size, 3)   ,		config.vocab_size       )
											snake_case_					:		List[Any] 					=					ids_tensor((self.batch_size, 3)   ,		vocab_size=2       )

											# append to next input_ids and
											snake_case_					:		Optional[Any] 					=					torch.cat([input_ids, next_tokens]   ,		dim=-1       )
											snake_case_					:		Optional[Any] 					=					torch.cat([input_mask, next_mask]   ,		dim=-1       )

											snake_case_					:		List[str] 					=					model(
											    lowercase__   ,		attention_mask=lowercase__   ,		encoder_hidden_states=lowercase__   ,		encoder_attention_mask=lowercase__   ,		output_hidden_states=lowercase__   ,		)["""hidden_states"""][0]
											snake_case_					:		List[Any] 					=					model(
											    lowercase__   ,		attention_mask=lowercase__   ,		encoder_hidden_states=lowercase__   ,		encoder_attention_mask=lowercase__   ,		past_key_values=lowercase__   ,		output_hidden_states=lowercase__   ,		)["""hidden_states"""][0]

											# select random slice
											snake_case_					:		Union[str, Any] 					=					ids_tensor((1,)   ,		output_from_past.shape[-1]       ).item()
											snake_case_					:		str 					=					output_from_no_past[:, -3:, random_slice_idx].detach()
											snake_case_					:		List[Any] 					=					output_from_past[:, :, random_slice_idx].detach()

											self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

											# test that outputs are equal for slice
											self.parent.assertTrue(torch.allclose(lowercase__   ,		lowercase__   ,		atol=1e-3       )       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		str 					=					self.prepare_config_and_inputs()
											(
											    (
											    snake_case_
											)					,			(
											    snake_case_
											)					,			(
											    snake_case_
											)					,			(
											    snake_case_
											)					,			(
											    snake_case_
											)					,			(
											    snake_case_
											)					,			(
											    snake_case_
											)					,			
											)					:		Union[str, Any] 					=					config_and_inputs
											snake_case_					:		Dict 					=					{"""input_ids""": input_ids, """attention_mask""": input_mask}
											return config, inputs_dict







@require_torch
class    __lowercase     (					_UpperCAmelCase					,    _UpperCAmelCase					,    _UpperCAmelCase					,    unittest.TestCase):







						"""simple docstring"""







						_A		:							Any							       =		(
						    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
						)
						_A		:							Union[str, Any]							       =		(OpenLlamaForCausalLM,) if is_torch_available() else ()
						_A		:							List[str]							       =		(
						    {
						        """feature-extraction""": OpenLlamaModel,
						        """text-classification""": OpenLlamaForSequenceClassification,
						        """text-generation""": OpenLlamaForCausalLM,
						        """zero-shot""": OpenLlamaForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)
						_A		:							Optional[int]							       =		False
						_A		:							Tuple							       =		False







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Tuple 					=					OpenLlamaModelTester(self       )
											snake_case_					:		Optional[int] 					=					ConfigTester(self   ,		config_class=lowercase__   ,		hidden_size=37       )







						def 						__UpperCamelCase							(self       ):
											self.config_tester.run_common_tests()







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Dict 					=					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*lowercase__       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Any 					=					self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
																snake_case_					:		List[str] 					=					type
																self.model_tester.create_and_check_model(*lowercase__       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					,			snake_case_					:		Dict 					=					self.model_tester.prepare_config_and_inputs_for_common()
											snake_case_					:		Optional[Any] 					=					3
											snake_case_					:		Dict 					=					input_dict["""input_ids"""]
											snake_case_					:		List[str] 					=					input_ids.ne(1       ).to(lowercase__       )
											snake_case_					:		Optional[int] 					=					ids_tensor([self.model_tester.batch_size]   ,		self.model_tester.type_sequence_label_size       )
											snake_case_					:		List[str] 					=					OpenLlamaForSequenceClassification(lowercase__       )
											model.to(lowercase__       )
											model.eval()
											snake_case_					:		List[str] 					=					model(lowercase__   ,		attention_mask=lowercase__   ,		labels=lowercase__       )
											self.assertEqual(result.logits.shape   ,		(self.model_tester.batch_size, self.model_tester.num_labels)       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					,			snake_case_					:		str 					=					self.model_tester.prepare_config_and_inputs_for_common()
											snake_case_					:		int 					=					3
											snake_case_					:		str 					=					"""single_label_classification"""
											snake_case_					:		Optional[Any] 					=					input_dict["""input_ids"""]
											snake_case_					:		Optional[int] 					=					input_ids.ne(1       ).to(lowercase__       )
											snake_case_					:		Optional[int] 					=					ids_tensor([self.model_tester.batch_size]   ,		self.model_tester.type_sequence_label_size       )
											snake_case_					:		Optional[Any] 					=					OpenLlamaForSequenceClassification(lowercase__       )
											model.to(lowercase__       )
											model.eval()
											snake_case_					:		Dict 					=					model(lowercase__   ,		attention_mask=lowercase__   ,		labels=lowercase__       )
											self.assertEqual(result.logits.shape   ,		(self.model_tester.batch_size, self.model_tester.num_labels)       )







						def 						__UpperCamelCase							(self       ):
											snake_case_					,			snake_case_					:		List[Any] 					=					self.model_tester.prepare_config_and_inputs_for_common()
											snake_case_					:		int 					=					3
											snake_case_					:		Any 					=					"""multi_label_classification"""
											snake_case_					:		Optional[int] 					=					input_dict["""input_ids"""]
											snake_case_					:		Tuple 					=					input_ids.ne(1       ).to(lowercase__       )
											snake_case_					:		Union[str, Any] 					=					ids_tensor(
											    [self.model_tester.batch_size, config.num_labels]   ,		self.model_tester.type_sequence_label_size       ).to(torch.float       )
											snake_case_					:		Any 					=					OpenLlamaForSequenceClassification(lowercase__       )
											model.to(lowercase__       )
											model.eval()
											snake_case_					:		str 					=					model(lowercase__   ,		attention_mask=lowercase__   ,		labels=lowercase__       )
											self.assertEqual(result.logits.shape   ,		(self.model_tester.batch_size, self.model_tester.num_labels)       )







						@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test"""       )
						def 						__UpperCamelCase							(self       ):
											pass







						@parameterized.expand([("""linear""",), ("""dynamic""",)]       )
						def 						__UpperCamelCase							(self   ,		lowercase__       ):
											snake_case_					,			snake_case_					:		int 					=					self.model_tester.prepare_config_and_inputs_for_common()
											snake_case_					:		List[Any] 					=					ids_tensor([1, 10]   ,		config.vocab_size       )
											snake_case_					:		str 					=					ids_tensor([1, int(config.max_position_embeddings * 1.5       )]   ,		config.vocab_size       )

											set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
											snake_case_					:		Tuple 					=					OpenLlamaModel(lowercase__       )
											original_model.to(lowercase__       )
											original_model.eval()
											snake_case_					:		str 					=					original_model(lowercase__       ).last_hidden_state
											snake_case_					:		Optional[Any] 					=					original_model(lowercase__       ).last_hidden_state

											set_seed(42       )  # Fixed seed at init time so the two models get the same random weights
											snake_case_					:		Union[str, Any] 					=					{"""type""": scaling_type, """factor""": 10.0}
											snake_case_					:		int 					=					OpenLlamaModel(lowercase__       )
											scaled_model.to(lowercase__       )
											scaled_model.eval()
											snake_case_					:		str 					=					scaled_model(lowercase__       ).last_hidden_state
											snake_case_					:		Optional[Any] 					=					scaled_model(lowercase__       ).last_hidden_state

											# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
											# maximum sequence length, so the outputs for the short input should match.
											if scaling_type == "dynamic":
																self.assertTrue(torch.allclose(lowercase__   ,		lowercase__   ,		atol=1e-5       )       )
											else:
																self.assertFalse(torch.allclose(lowercase__   ,		lowercase__   ,		atol=1e-5       )       )

											# The output should be different for long inputs
											self.assertFalse(torch.allclose(lowercase__   ,		lowercase__   ,		atol=1e-5       )       )









"""simple docstring"""

import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto.configuration_auto import CONFIG_MAPPING


a_			   =					logging.get_logger(__name__)







class    __lowercase     (					_UpperCAmelCase):







						"""simple docstring"""







						_A		:							Optional[int]							       =		"""upernet"""







						def __init__(self   ,		lowercase__=None   ,		lowercase__=5_12   ,		lowercase__=0.02   ,		lowercase__=[1, 2, 3, 6]   ,		lowercase__=True   ,		lowercase__=0.4   ,		lowercase__=3_84   ,		lowercase__=2_56   ,		lowercase__=1   ,		lowercase__=False   ,		lowercase__=2_55   ,		**lowercase__   ,		):
											super().__init__(**lowercase__       )

											if backbone_config is None:
																logger.info("""`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone."""       )
																snake_case_					:		List[Any] 					=					CONFIG_MAPPING["""resnet"""](out_features=["""stage1""", """stage2""", """stage3""", """stage4"""]       )
											elif isinstance(lowercase__   ,		lowercase__       ):
																snake_case_					:		Tuple 					=					backbone_config.get("""model_type"""       )
																snake_case_					:		List[str] 					=					CONFIG_MAPPING[backbone_model_type]
																snake_case_					:		List[Any] 					=					config_class.from_dict(lowercase__       )

											snake_case_					:		List[Any] 					=					backbone_config
											snake_case_					:		Optional[Any] 					=					hidden_size
											snake_case_					:		Any 					=					initializer_range
											snake_case_					:		str 					=					pool_scales
											snake_case_					:		Dict 					=					use_auxiliary_head
											snake_case_					:		str 					=					auxiliary_loss_weight
											snake_case_					:		List[str] 					=					auxiliary_in_channels
											snake_case_					:		Optional[Any] 					=					auxiliary_channels
											snake_case_					:		Any 					=					auxiliary_num_convs
											snake_case_					:		List[Any] 					=					auxiliary_concat_input
											snake_case_					:		List[str] 					=					loss_ignore_index







						def 						__UpperCamelCase							(self       ):
											snake_case_					:		Dict 					=					copy.deepcopy(self.__dict__       )
											snake_case_					:		Union[str, Any] 					=					self.backbone_config.to_dict()
											snake_case_					:		Any 					=					self.__class__.model_type
											return output




