



'''simple docstring'''

import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel



class 		lowerCamelCase__      :




					'''simple docstring'''



					def __init__(  self				:	Union[str, Any]     ,  UpperCamelCase_				:	Dict     ,  UpperCamelCase_				:	Optional[int]=13     ,  UpperCamelCase_				:	Optional[Any]=7     ,  UpperCamelCase_				:	int=True     ,  UpperCamelCase_				:	Tuple=True     ,  UpperCamelCase_				:	Tuple=False     ,  UpperCamelCase_				:	Optional[Any]=True     ,  UpperCamelCase_				:	Optional[int]=99     ,  UpperCamelCase_				:	int=32     ,  UpperCamelCase_				:	int=5     ,  UpperCamelCase_				:	List[str]=4     ,  UpperCamelCase_				:	Dict=37     ,  UpperCamelCase_				:	Union[str, Any]="gelu"     ,  UpperCamelCase_				:	Union[str, Any]=0.1     ,  UpperCamelCase_				:	str=0.1     ,  UpperCamelCase_				:	Optional[Any]=512     ,  UpperCamelCase_				:	Dict=16     ,  UpperCamelCase_				:	Dict=2     ,  UpperCamelCase_				:	Optional[int]=0.02     ,  UpperCamelCase_				:	Any=3     ,  UpperCamelCase_				:	Union[str, Any]=4     ,  UpperCamelCase_				:	List[Any]=None     ,  )  ->       int:



						'''simple docstring'''





						_lowercase					:       Any       =		parent
						_lowercase					:       Dict       =		batch_size
						_lowercase					:       Any       =		seq_length
						_lowercase					:       Optional[int]       =		is_training
						_lowercase					:       Optional[Any]       =		use_input_mask
						_lowercase					:       Union[str, Any]       =		use_token_type_ids
						_lowercase					:       Optional[Any]       =		use_labels
						_lowercase					:       List[Any]       =		vocab_size
						_lowercase					:       List[Any]       =		hidden_size
						_lowercase					:       Optional[Any]       =		num_hidden_layers
						_lowercase					:       Optional[Any]       =		num_attention_heads
						_lowercase					:       Optional[Any]       =		intermediate_size
						_lowercase					:       Union[str, Any]       =		hidden_act
						_lowercase					:       List[Any]       =		hidden_dropout_prob
						_lowercase					:       List[Any]       =		attention_probs_dropout_prob
						_lowercase					:       Dict       =		max_position_embeddings
						_lowercase					:       List[str]       =		type_vocab_size
						_lowercase					:       Optional[int]       =		type_sequence_label_size
						_lowercase					:       str       =		initializer_range
						_lowercase					:       Union[str, Any]       =		num_labels
						_lowercase					:       Union[str, Any]       =		num_choices
						_lowercase					:       Dict       =		scope




					def 						__UpperCAmelCase     (  self				:	Any					)  ->       int:



						'''simple docstring'''





						_lowercase					:       Dict       =		ids_tensor([self.batch_size, self.seq_length]     ,  self.vocab_size					)

						_lowercase					:       List[Any]       =		None
						if self.use_input_mask:
							_lowercase					:       Union[str, Any]       =		random_attention_mask([self.batch_size, self.seq_length]					)

						_lowercase					:       int       =		None
						if self.use_token_type_ids:
							_lowercase					:       Optional[Any]       =		ids_tensor([self.batch_size, self.seq_length]     ,  self.type_vocab_size					)

						_lowercase					:       Optional[Any]       =		None
						_lowercase					:       Optional[int]       =		None
						_lowercase					:       Optional[Any]       =		None
						if self.use_labels:
							_lowercase					:       int       =		ids_tensor([self.batch_size]     ,  self.type_sequence_label_size					)
							_lowercase					:       Optional[Any]       =		ids_tensor([self.batch_size, self.seq_length]     ,  self.num_labels					)
							_lowercase					:       Any       =		ids_tensor([self.batch_size]     ,  self.num_choices					)

						_lowercase					:       Dict       =		self.get_config()

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




					def 						__UpperCAmelCase     (  self				:	Optional[Any]					)  ->       Optional[Any]:



						'''simple docstring'''





						return OpenLlamaConfig(
						    vocab_size=self.vocab_size     ,  hidden_size=self.hidden_size     ,  num_hidden_layers=self.num_hidden_layers     ,  num_attention_heads=self.num_attention_heads     ,  intermediate_size=self.intermediate_size     ,  hidden_act=self.hidden_act     ,  hidden_dropout_prob=self.hidden_dropout_prob     ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,  max_position_embeddings=self.max_position_embeddings     ,  type_vocab_size=self.type_vocab_size     ,  is_decoder=UpperCamelCase_     ,  initializer_range=self.initializer_range     ,  use_stable_embedding=UpperCamelCase_     ,  )




					def 						__UpperCAmelCase     (  self				:	Optional[int]     ,  UpperCamelCase_				:	Union[str, Any]     ,  UpperCamelCase_				:	Union[str, Any]     ,  UpperCamelCase_				:	Union[str, Any]     ,  UpperCamelCase_				:	Union[str, Any]     ,  UpperCamelCase_				:	Union[str, Any]     ,  UpperCamelCase_				:	List[Any]     ,  UpperCamelCase_				:	Union[str, Any]					)  ->       List[Any]:



						'''simple docstring'''





						_lowercase					:       List[str]       =		OpenLlamaModel(config=UpperCamelCase_					)
						model.to(UpperCamelCase_					)
						model.eval()
						_lowercase					:       Optional[Any]       =		model(UpperCamelCase_     ,  attention_mask=UpperCamelCase_					)
						_lowercase					:       List[Any]       =		model(UpperCamelCase_					)
						self.parent.assertEqual(result.last_hidden_state.shape     ,  (self.batch_size, self.seq_length, self.hidden_size)					)




					def 						__UpperCAmelCase     (  self				:	Tuple     ,  UpperCamelCase_				:	List[Any]     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Union[str, Any]     ,  UpperCamelCase_				:	Optional[int]     ,  UpperCamelCase_				:	Union[str, Any]     ,  UpperCamelCase_				:	Dict     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Tuple     ,  )  ->       Optional[int]:



						'''simple docstring'''





						_lowercase					:       Optional[int]       =		True
						_lowercase					:       int       =		OpenLlamaModel(UpperCamelCase_					)
						model.to(UpperCamelCase_					)
						model.eval()
						_lowercase					:       Union[str, Any]       =		model(
						    UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  encoder_hidden_states=UpperCamelCase_     ,  encoder_attention_mask=UpperCamelCase_     ,  )
						_lowercase					:       Union[str, Any]       =		model(
						    UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  encoder_hidden_states=UpperCamelCase_     ,  )
						_lowercase					:       Union[str, Any]       =		model(UpperCamelCase_     ,  attention_mask=UpperCamelCase_					)
						self.parent.assertEqual(result.last_hidden_state.shape     ,  (self.batch_size, self.seq_length, self.hidden_size)					)




					def 						__UpperCAmelCase     (  self				:	Optional[Any]     ,  UpperCamelCase_				:	int     ,  UpperCamelCase_				:	List[Any]     ,  UpperCamelCase_				:	Optional[int]     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	Any     ,  UpperCamelCase_				:	List[Any]     ,  UpperCamelCase_				:	Any     ,  UpperCamelCase_				:	List[str]     ,  )  ->       Any:



						'''simple docstring'''





						_lowercase					:       Tuple       =		OpenLlamaForCausalLM(config=UpperCamelCase_					)
						model.to(UpperCamelCase_					)
						model.eval()
						_lowercase					:       int       =		model(UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  labels=UpperCamelCase_					)
						self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.seq_length, self.vocab_size)					)




					def 						__UpperCAmelCase     (  self				:	str     ,  UpperCamelCase_				:	List[Any]     ,  UpperCamelCase_				:	Union[str, Any]     ,  UpperCamelCase_				:	List[str]     ,  UpperCamelCase_				:	str     ,  UpperCamelCase_				:	List[Any]     ,  UpperCamelCase_				:	Optional[Any]     ,  UpperCamelCase_				:	Optional[Any]     ,  UpperCamelCase_				:	Dict     ,  UpperCamelCase_				:	Optional[Any]     ,  )  ->       Optional[Any]:



						'''simple docstring'''





						_lowercase					:       List[Any]       =		True
						_lowercase					:       Optional[Any]       =		True
						_lowercase					:       Optional[Any]       =		OpenLlamaForCausalLM(config=UpperCamelCase_					)
						model.to(UpperCamelCase_					)
						model.eval()

						# first forward pass
						_lowercase					:       Any       =		model(
						    UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  encoder_hidden_states=UpperCamelCase_     ,  encoder_attention_mask=UpperCamelCase_     ,  use_cache=UpperCamelCase_     ,  )
						_lowercase					:       Any       =		outputs.past_key_values

						# create hypothetical multiple next token and extent to next_input_ids
						_lowercase					:       Any       =		ids_tensor((self.batch_size, 3)     ,  config.vocab_size					)
						_lowercase					:       Tuple       =		ids_tensor((self.batch_size, 3)     ,  vocab_size=2					)

						# append to next input_ids and
						_lowercase					:       Union[str, Any]       =		torch.cat([input_ids, next_tokens]     ,  dim=-1					)
						_lowercase					:       List[Any]       =		torch.cat([input_mask, next_mask]     ,  dim=-1					)

						_lowercase					:       int       =		model(
						    UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  encoder_hidden_states=UpperCamelCase_     ,  encoder_attention_mask=UpperCamelCase_     ,  output_hidden_states=UpperCamelCase_     ,  )['hidden_states'][0]
						_lowercase					:       List[Any]       =		model(
						    UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  encoder_hidden_states=UpperCamelCase_     ,  encoder_attention_mask=UpperCamelCase_     ,  past_key_values=UpperCamelCase_     ,  output_hidden_states=UpperCamelCase_     ,  )['hidden_states'][0]

						# select random slice
						_lowercase					:       Any       =		ids_tensor((1,)     ,  output_from_past.shape[-1]					).item()
						_lowercase					:       Union[str, Any]       =		output_from_no_past[:, -3:, random_slice_idx].detach()
						_lowercase					:       Dict       =		output_from_past[:, :, random_slice_idx].detach()

						self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]					)

						# test that outputs are equal for slice
						self.parent.assertTrue(torch.allclose(UpperCamelCase_     ,  UpperCamelCase_     ,  atol=1E-3					)					)




					def 						__UpperCAmelCase     (  self				:	Optional[int]					)  ->       int:



						'''simple docstring'''





						_lowercase					:       List[Any]       =		self.prepare_config_and_inputs()
						(
						    (
						    _lowercase
						)  ,    (
						    _lowercase
						)  ,    (
						    _lowercase
						)  ,    (
						    _lowercase
						)  ,    (
						    _lowercase
						)  ,    (
						    _lowercase
						)  ,    (
						    _lowercase
						)  ,    
						)					:       Optional[Any]       =		config_and_inputs
						_lowercase					:       str       =		{'input_ids': input_ids, 'attention_mask': input_mask}
						return config, inputs_dict






@require_torch
class 		lowerCamelCase__      (						A	,   A	,   A	,   unittest.TestCase     ):




					'''simple docstring'''



					A_						    =						(
					    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
					)
					A_						    =						(OpenLlamaForCausalLM,) if is_torch_available() else ()
					A_						    =						(
					    {
					        """feature-extraction""": OpenLlamaModel,
					        """text-classification""": OpenLlamaForSequenceClassification,
					        """text-generation""": OpenLlamaForCausalLM,
					        """zero-shot""": OpenLlamaForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					A_						    =						False
					A_						    =						False




					def 						__UpperCAmelCase     (  self				:	Any					)  ->       Union[str, Any]:



						'''simple docstring'''





						_lowercase					:       Dict       =		OpenLlamaModelTester(self					)
						_lowercase					:       Dict       =		ConfigTester(self     ,  config_class=UpperCamelCase_     ,  hidden_size=37					)




					def 						__UpperCAmelCase     (  self				:	Dict					)  ->       Optional[Any]:



						'''simple docstring'''





						self.config_tester.run_common_tests()




					def 						__UpperCAmelCase     (  self				:	Dict					)  ->       Optional[Any]:



						'''simple docstring'''





						_lowercase					:       Union[str, Any]       =		self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	Any					)  ->       Tuple:



						'''simple docstring'''





						_lowercase					:       Union[str, Any]       =		self.model_tester.prepare_config_and_inputs()
						for type in ["absolute", "relative_key", "relative_key_query"]:
							_lowercase					:       int       =		type
							self.model_tester.create_and_check_model(*UpperCamelCase_					)




					def 						__UpperCAmelCase     (  self				:	Any					)  ->       Optional[int]:



						'''simple docstring'''





						_lowercase  ,    _lowercase					:       int       =		self.model_tester.prepare_config_and_inputs_for_common()
						_lowercase					:       Optional[int]       =		3
						_lowercase					:       Optional[Any]       =		input_dict['input_ids']
						_lowercase					:       Dict       =		input_ids.ne(1					).to(UpperCamelCase_					)
						_lowercase					:       Dict       =		ids_tensor([self.model_tester.batch_size]     ,  self.model_tester.type_sequence_label_size					)
						_lowercase					:       str       =		OpenLlamaForSequenceClassification(UpperCamelCase_					)
						model.to(UpperCamelCase_					)
						model.eval()
						_lowercase					:       int       =		model(UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  labels=UpperCamelCase_					)
						self.assertEqual(result.logits.shape     ,  (self.model_tester.batch_size, self.model_tester.num_labels)					)




					def 						__UpperCAmelCase     (  self				:	int					)  ->       str:



						'''simple docstring'''





						_lowercase  ,    _lowercase					:       List[Any]       =		self.model_tester.prepare_config_and_inputs_for_common()
						_lowercase					:       Optional[int]       =		3
						_lowercase					:       int       =		'single_label_classification'
						_lowercase					:       Dict       =		input_dict['input_ids']
						_lowercase					:       int       =		input_ids.ne(1					).to(UpperCamelCase_					)
						_lowercase					:       List[Any]       =		ids_tensor([self.model_tester.batch_size]     ,  self.model_tester.type_sequence_label_size					)
						_lowercase					:       Any       =		OpenLlamaForSequenceClassification(UpperCamelCase_					)
						model.to(UpperCamelCase_					)
						model.eval()
						_lowercase					:       Tuple       =		model(UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  labels=UpperCamelCase_					)
						self.assertEqual(result.logits.shape     ,  (self.model_tester.batch_size, self.model_tester.num_labels)					)




					def 						__UpperCAmelCase     (  self				:	str					)  ->       List[str]:



						'''simple docstring'''





						_lowercase  ,    _lowercase					:       Dict       =		self.model_tester.prepare_config_and_inputs_for_common()
						_lowercase					:       Any       =		3
						_lowercase					:       List[str]       =		'multi_label_classification'
						_lowercase					:       List[Any]       =		input_dict['input_ids']
						_lowercase					:       List[str]       =		input_ids.ne(1					).to(UpperCamelCase_					)
						_lowercase					:       Optional[int]       =		ids_tensor(
						    [self.model_tester.batch_size, config.num_labels]     ,  self.model_tester.type_sequence_label_size					).to(torch.float					)
						_lowercase					:       Union[str, Any]       =		OpenLlamaForSequenceClassification(UpperCamelCase_					)
						model.to(UpperCamelCase_					)
						model.eval()
						_lowercase					:       str       =		model(UpperCamelCase_     ,  attention_mask=UpperCamelCase_     ,  labels=UpperCamelCase_					)
						self.assertEqual(result.logits.shape     ,  (self.model_tester.batch_size, self.model_tester.num_labels)					)




					@unittest.skip('Open-Llama buffers include complex numbers, which breaks this test'					)
					def 						__UpperCAmelCase     (  self				:	List[Any]					)  ->       List[Any]:



						'''simple docstring'''





						pass




					@parameterized.expand([('linear',), ('dynamic',)]					)
					def 						__UpperCAmelCase     (  self				:	str     ,  UpperCamelCase_				:	List[str]					)  ->       Tuple:



						'''simple docstring'''





						_lowercase  ,    _lowercase					:       List[Any]       =		self.model_tester.prepare_config_and_inputs_for_common()
						_lowercase					:       Optional[Any]       =		ids_tensor([1, 10]     ,  config.vocab_size					)
						_lowercase					:       Union[str, Any]       =		ids_tensor([1, int(config.max_position_embeddings * 1.5					)]     ,  config.vocab_size					)

						set_seed(42					)  # Fixed seed at init time so the two models get the same random weights
						_lowercase					:       List[str]       =		OpenLlamaModel(UpperCamelCase_					)
						original_model.to(UpperCamelCase_					)
						original_model.eval()
						_lowercase					:       Optional[int]       =		original_model(UpperCamelCase_					).last_hidden_state
						_lowercase					:       Dict       =		original_model(UpperCamelCase_					).last_hidden_state

						set_seed(42					)  # Fixed seed at init time so the two models get the same random weights
						_lowercase					:       List[str]       =		{'type': scaling_type, 'factor': 10.0}
						_lowercase					:       Any       =		OpenLlamaModel(UpperCamelCase_					)
						scaled_model.to(UpperCamelCase_					)
						scaled_model.eval()
						_lowercase					:       Tuple       =		scaled_model(UpperCamelCase_					).last_hidden_state
						_lowercase					:       int       =		scaled_model(UpperCamelCase_					).last_hidden_state

						# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
						# maximum sequence length, so the outputs for the short input should match.
						if scaling_type == "dynamic":
							self.assertTrue(torch.allclose(UpperCamelCase_     ,  UpperCamelCase_     ,  atol=1E-5					)					)
						else:
							self.assertFalse(torch.allclose(UpperCamelCase_     ,  UpperCamelCase_     ,  atol=1E-5					)					)

						# The output should be different for long inputs
						self.assertFalse(torch.allclose(UpperCamelCase_     ,  UpperCamelCase_     ,  atol=1E-5					)					)





'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_A				:    Optional[Any]     						={'''configuration_xlnet''': ['''XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XLNetConfig''']}

try:
		if not is_sentencepiece_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    Tuple     						=['''XLNetTokenizer''']

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    str     						=['''XLNetTokenizerFast''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    Any     						=[
		    '''XLNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''XLNetForMultipleChoice''',
		    '''XLNetForQuestionAnswering''',
		    '''XLNetForQuestionAnsweringSimple''',
		    '''XLNetForSequenceClassification''',
		    '''XLNetForTokenClassification''',
		    '''XLNetLMHeadModel''',
		    '''XLNetModel''',
		    '''XLNetPreTrainedModel''',
		    '''load_tf_weights_in_xlnet''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_A				:    str     						=[
		    '''TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFXLNetForMultipleChoice''',
		    '''TFXLNetForQuestionAnsweringSimple''',
		    '''TFXLNetForSequenceClassification''',
		    '''TFXLNetForTokenClassification''',
		    '''TFXLNetLMHeadModel''',
		    '''TFXLNetMainLayer''',
		    '''TFXLNetModel''',
		    '''TFXLNetPreTrainedModel''',
		]


if TYPE_CHECKING:
		from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

		try:
				if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_xlnet import XLNetTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_xlnet_fast import XLNetTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_xlnet import (
				    XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
				    XLNetForMultipleChoice,
				    XLNetForQuestionAnswering,
				    XLNetForQuestionAnsweringSimple,
				    XLNetForSequenceClassification,
				    XLNetForTokenClassification,
				    XLNetLMHeadModel,
				    XLNetModel,
				    XLNetPreTrainedModel,
				    load_tf_weights_in_xlnet,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_xlnet import (
				    TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFXLNetForMultipleChoice,
				    TFXLNetForQuestionAnsweringSimple,
				    TFXLNetForSequenceClassification,
				    TFXLNetForTokenClassification,
				    TFXLNetLMHeadModel,
				    TFXLNetMainLayer,
				    TFXLNetModel,
				    TFXLNetPreTrainedModel,
				)

else:
		import sys

		_A				:    str     						=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
