





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel




class   UpperCamelCase_       :


							def __init__(							self    :Tuple							,				__A    :str							,				__A    :Optional[int]=13							,				__A    :Union[str, Any]=7							,				__A    :Optional[Any]=True							,				__A    :Any=True							,				__A    :List[Any]=False							,				__A    :List[Any]=True							,				__A    :Any=99							,				__A    :List[str]=32							,				__A    :int=5							,				__A    :Any=4							,				__A    :List[Any]=37							,				__A    :int="gelu"							,				__A    :Tuple=0.1							,				__A    :Optional[int]=0.1							,				__A    :Dict=512							,				__A    :Optional[Any]=16							,				__A    :List[Any]=2							,				__A    :Dict=0.0_2							,				__A    :Dict=3							,				__A    :Dict=4							,				__A    :int=None							,				)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      parent
									SCREAMING_SNAKE_CASE__			=      batch_size
									SCREAMING_SNAKE_CASE__			=      seq_length
									SCREAMING_SNAKE_CASE__			=      is_training
									SCREAMING_SNAKE_CASE__			=      use_input_mask
									SCREAMING_SNAKE_CASE__			=      use_token_type_ids
									SCREAMING_SNAKE_CASE__			=      use_labels
									SCREAMING_SNAKE_CASE__			=      vocab_size
									SCREAMING_SNAKE_CASE__			=      hidden_size
									SCREAMING_SNAKE_CASE__			=      num_hidden_layers
									SCREAMING_SNAKE_CASE__			=      num_attention_heads
									SCREAMING_SNAKE_CASE__			=      intermediate_size
									SCREAMING_SNAKE_CASE__			=      hidden_act
									SCREAMING_SNAKE_CASE__			=      hidden_dropout_prob
									SCREAMING_SNAKE_CASE__			=      attention_probs_dropout_prob
									SCREAMING_SNAKE_CASE__			=      max_position_embeddings
									SCREAMING_SNAKE_CASE__			=      type_vocab_size
									SCREAMING_SNAKE_CASE__			=      type_sequence_label_size
									SCREAMING_SNAKE_CASE__			=      initializer_range
									SCREAMING_SNAKE_CASE__			=      num_labels
									SCREAMING_SNAKE_CASE__			=      num_choices
									SCREAMING_SNAKE_CASE__			=      scope


							def      _snake_case  (							self    :Optional[int]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size, self.seq_length]							,				self.vocab_size		)

									SCREAMING_SNAKE_CASE__			=      None
									if self.use_input_mask:
											SCREAMING_SNAKE_CASE__			=      random_attention_mask([self.batch_size, self.seq_length]		)

									SCREAMING_SNAKE_CASE__			=      None
									if self.use_token_type_ids:
											SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size, self.seq_length]							,				self.type_vocab_size		)

									SCREAMING_SNAKE_CASE__			=      None
									SCREAMING_SNAKE_CASE__			=      None
									SCREAMING_SNAKE_CASE__			=      None
									if self.use_labels:
											SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size]							,				self.type_sequence_label_size		)
											SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size, self.seq_length]							,				self.num_labels		)
											SCREAMING_SNAKE_CASE__			=      ids_tensor([self.batch_size]							,				self.num_choices		)

									SCREAMING_SNAKE_CASE__			=      self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


							def      _snake_case  (							self    :Optional[int]		)   ->   int:



									"""simple docstring"""



									return OpenLlamaConfig(
									    vocab_size=self.vocab_size							,				hidden_size=self.hidden_size							,				num_hidden_layers=self.num_hidden_layers							,				num_attention_heads=self.num_attention_heads							,				intermediate_size=self.intermediate_size							,				hidden_act=self.hidden_act							,				hidden_dropout_prob=self.hidden_dropout_prob							,				attention_probs_dropout_prob=self.attention_probs_dropout_prob							,				max_position_embeddings=self.max_position_embeddings							,				type_vocab_size=self.type_vocab_size							,				is_decoder=__A							,				initializer_range=self.initializer_range							,				use_stable_embedding=__A							,				)


							def      _snake_case  (							self    :Optional[Any]							,				__A    :Optional[int]							,				__A    :Optional[Any]							,				__A    :Union[str, Any]							,				__A    :List[str]							,				__A    :Dict							,				__A    :Dict							,				__A    :Union[str, Any]		)   ->   Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      OpenLlamaModel(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A		)
									SCREAMING_SNAKE_CASE__			=      model(__A		)
									self.parent.assertEqual(result.last_hidden_state.shape							,				(self.batch_size, self.seq_length, self.hidden_size)		)


							def      _snake_case  (							self    :Optional[int]							,				__A    :Union[str, Any]							,				__A    :List[Any]							,				__A    :str							,				__A    :Dict							,				__A    :Dict							,				__A    :Optional[Any]							,				__A    :Tuple							,				__A    :Dict							,				__A    :Union[str, Any]							,				)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      True
									SCREAMING_SNAKE_CASE__			=      OpenLlamaModel(__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(
									    __A							,				attention_mask=__A							,				encoder_hidden_states=__A							,				encoder_attention_mask=__A							,				)
									SCREAMING_SNAKE_CASE__			=      model(
									    __A							,				attention_mask=__A							,				encoder_hidden_states=__A							,				)
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A		)
									self.parent.assertEqual(result.last_hidden_state.shape							,				(self.batch_size, self.seq_length, self.hidden_size)		)


							def      _snake_case  (							self    :Any							,				__A    :Tuple							,				__A    :Any							,				__A    :Optional[int]							,				__A    :str							,				__A    :Optional[Any]							,				__A    :List[Any]							,				__A    :str							,				__A    :Tuple							,				__A    :List[Any]							,				)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      OpenLlamaForCausalLM(config=__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A							,				labels=__A		)
									self.parent.assertEqual(result.logits.shape							,				(self.batch_size, self.seq_length, self.vocab_size)		)


							def      _snake_case  (							self    :Union[str, Any]							,				__A    :List[str]							,				__A    :Optional[int]							,				__A    :Dict							,				__A    :Tuple							,				__A    :Tuple							,				__A    :Optional[Any]							,				__A    :str							,				__A    :Optional[Any]							,				__A    :Tuple							,				)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      True
									SCREAMING_SNAKE_CASE__			=      True
									SCREAMING_SNAKE_CASE__			=      OpenLlamaForCausalLM(config=__A		)
									model.to(__A		)
									model.eval()

									# first forward pass
									SCREAMING_SNAKE_CASE__			=      model(
									    __A							,				attention_mask=__A							,				encoder_hidden_states=__A							,				encoder_attention_mask=__A							,				use_cache=__A							,				)
									SCREAMING_SNAKE_CASE__			=      outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									SCREAMING_SNAKE_CASE__			=      ids_tensor((self.batch_size, 3)							,				config.vocab_size		)
									SCREAMING_SNAKE_CASE__			=      ids_tensor((self.batch_size, 3)							,				vocab_size=2		)

									# append to next input_ids and
									SCREAMING_SNAKE_CASE__			=      torch.cat([input_ids, next_tokens]							,				dim=-1		)
									SCREAMING_SNAKE_CASE__			=      torch.cat([input_mask, next_mask]							,				dim=-1		)

									SCREAMING_SNAKE_CASE__			=      model(
									    __A							,				attention_mask=__A							,				encoder_hidden_states=__A							,				encoder_attention_mask=__A							,				output_hidden_states=__A							,				)["""hidden_states"""][0]
									SCREAMING_SNAKE_CASE__			=      model(
									    __A							,				attention_mask=__A							,				encoder_hidden_states=__A							,				encoder_attention_mask=__A							,				past_key_values=__A							,				output_hidden_states=__A							,				)["""hidden_states"""][0]

									# select random slice
									SCREAMING_SNAKE_CASE__			=      ids_tensor((1,)							,				output_from_past.shape[-1]		).item()
									SCREAMING_SNAKE_CASE__			=      output_from_no_past[:, -3:, random_slice_idx].detach()
									SCREAMING_SNAKE_CASE__			=      output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(__A							,				__A							,				atol=1E-3		)		)


							def      _snake_case  (							self    :List[Any]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.prepare_config_and_inputs()
									(
									    (
									    SCREAMING_SNAKE_CASE__						
									)	,						(
									    SCREAMING_SNAKE_CASE__						
									)	,						(
									    SCREAMING_SNAKE_CASE__						
									)	,						(
									    SCREAMING_SNAKE_CASE__						
									)	,						(
									    SCREAMING_SNAKE_CASE__						
									)	,						(
									    SCREAMING_SNAKE_CASE__						
									)	,						(
									    SCREAMING_SNAKE_CASE__						
									)	,												
									)			=      config_and_inputs
									SCREAMING_SNAKE_CASE__			=      {"""input_ids""": input_ids, """attention_mask""": input_mask}
									return config, inputs_dict




@require_torch
class   UpperCamelCase_       (     UpperCamelCase__				,  UpperCamelCase__				,  UpperCamelCase__				,  unittest.TestCase    ):
							lowerCamelCase_           =					(
							    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
							)
							lowerCamelCase_           =					(OpenLlamaForCausalLM,) if is_torch_available() else ()
							lowerCamelCase_           =					(
							    {
							        "feature-extraction": OpenLlamaModel,
							        "text-classification": OpenLlamaForSequenceClassification,
							        "text-generation": OpenLlamaForCausalLM,
							        "zero-shot": OpenLlamaForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowerCamelCase_           =					False
							lowerCamelCase_           =					False


							def      _snake_case  (							self    :Union[str, Any]		)   ->   Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      OpenLlamaModelTester(self		)
									SCREAMING_SNAKE_CASE__			=      ConfigTester(self							,				config_class=__A							,				hidden_size=37		)


							def      _snake_case  (							self    :List[Any]		)   ->   str:



									"""simple docstring"""



									self.config_tester.run_common_tests()


							def      _snake_case  (							self    :List[Any]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*__A		)


							def      _snake_case  (							self    :Optional[Any]		)   ->   List[str]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
											SCREAMING_SNAKE_CASE__			=      type
											self.model_tester.create_and_check_model(*__A		)


							def      _snake_case  (							self    :List[Any]		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs_for_common()
									SCREAMING_SNAKE_CASE__			=      3
									SCREAMING_SNAKE_CASE__			=      input_dict["""input_ids"""]
									SCREAMING_SNAKE_CASE__			=      input_ids.ne(1		).to(__A		)
									SCREAMING_SNAKE_CASE__			=      ids_tensor([self.model_tester.batch_size]							,				self.model_tester.type_sequence_label_size		)
									SCREAMING_SNAKE_CASE__			=      OpenLlamaForSequenceClassification(__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A							,				labels=__A		)
									self.assertEqual(result.logits.shape							,				(self.model_tester.batch_size, self.model_tester.num_labels)		)


							def      _snake_case  (							self    :Dict		)   ->   Dict:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs_for_common()
									SCREAMING_SNAKE_CASE__			=      3
									SCREAMING_SNAKE_CASE__			=      """single_label_classification"""
									SCREAMING_SNAKE_CASE__			=      input_dict["""input_ids"""]
									SCREAMING_SNAKE_CASE__			=      input_ids.ne(1		).to(__A		)
									SCREAMING_SNAKE_CASE__			=      ids_tensor([self.model_tester.batch_size]							,				self.model_tester.type_sequence_label_size		)
									SCREAMING_SNAKE_CASE__			=      OpenLlamaForSequenceClassification(__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A							,				labels=__A		)
									self.assertEqual(result.logits.shape							,				(self.model_tester.batch_size, self.model_tester.num_labels)		)


							def      _snake_case  (							self    :Any		)   ->   int:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs_for_common()
									SCREAMING_SNAKE_CASE__			=      3
									SCREAMING_SNAKE_CASE__			=      """multi_label_classification"""
									SCREAMING_SNAKE_CASE__			=      input_dict["""input_ids"""]
									SCREAMING_SNAKE_CASE__			=      input_ids.ne(1		).to(__A		)
									SCREAMING_SNAKE_CASE__			=      ids_tensor(
									    [self.model_tester.batch_size, config.num_labels]							,				self.model_tester.type_sequence_label_size		).to(torch.float		)
									SCREAMING_SNAKE_CASE__			=      OpenLlamaForSequenceClassification(__A		)
									model.to(__A		)
									model.eval()
									SCREAMING_SNAKE_CASE__			=      model(__A							,				attention_mask=__A							,				labels=__A		)
									self.assertEqual(result.logits.shape							,				(self.model_tester.batch_size, self.model_tester.num_labels)		)


							@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test"""		)
							def      _snake_case  (							self    :Any		)   ->   List[str]:



									"""simple docstring"""



									pass


							@parameterized.expand([("""linear""",), ("""dynamic""",)]		)
							def      _snake_case  (							self    :Tuple							,				__A    :str		)   ->   Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__	,						SCREAMING_SNAKE_CASE__			=      self.model_tester.prepare_config_and_inputs_for_common()
									SCREAMING_SNAKE_CASE__			=      ids_tensor([1, 10]							,				config.vocab_size		)
									SCREAMING_SNAKE_CASE__			=      ids_tensor([1, int(config.max_position_embeddings * 1.5		)]							,				config.vocab_size		)

									set_seed(42		)  # Fixed seed at init time so the two models get the same random weights
									SCREAMING_SNAKE_CASE__			=      OpenLlamaModel(__A		)
									original_model.to(__A		)
									original_model.eval()
									SCREAMING_SNAKE_CASE__			=      original_model(__A		).last_hidden_state
									SCREAMING_SNAKE_CASE__			=      original_model(__A		).last_hidden_state

									set_seed(42		)  # Fixed seed at init time so the two models get the same random weights
									SCREAMING_SNAKE_CASE__			=      {"""type""": scaling_type, """factor""": 10.0}
									SCREAMING_SNAKE_CASE__			=      OpenLlamaModel(__A		)
									scaled_model.to(__A		)
									scaled_model.eval()
									SCREAMING_SNAKE_CASE__			=      scaled_model(__A		).last_hidden_state
									SCREAMING_SNAKE_CASE__			=      scaled_model(__A		).last_hidden_state

									# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
									# maximum sequence length, so the outputs for the short input should match.
									if scaling_type == "dynamic":
											self.assertTrue(torch.allclose(__A							,				__A							,				atol=1E-5		)		)
									else:
											self.assertFalse(torch.allclose(__A							,				__A							,				atol=1E-5		)		)

									# The output should be different for long inputs
									self.assertFalse(torch.allclose(__A							,				__A							,				atol=1E-5		)		)






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


_lowerCamelCase   				=			{
    'configuration_layoutlmv3': [
        'LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'LayoutLMv3Config',
        'LayoutLMv3OnnxConfig',
    ],
    'processing_layoutlmv3': ['LayoutLMv3Processor'],
    'tokenization_layoutlmv3': ['LayoutLMv3Tokenizer'],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCamelCase   				=			['LayoutLMv3TokenizerFast']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCamelCase   				=			[
	    'LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'LayoutLMv3ForQuestionAnswering',
	    'LayoutLMv3ForSequenceClassification',
	    'LayoutLMv3ForTokenClassification',
	    'LayoutLMv3Model',
	    'LayoutLMv3PreTrainedModel',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCamelCase   				=			[
	    'TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'TFLayoutLMv3ForQuestionAnswering',
	    'TFLayoutLMv3ForSequenceClassification',
	    'TFLayoutLMv3ForTokenClassification',
	    'TFLayoutLMv3Model',
	    'TFLayoutLMv3PreTrainedModel',
	]

try:
	if not is_vision_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCamelCase   				=			['LayoutLMv3FeatureExtractor']
	_lowerCamelCase   				=			['LayoutLMv3ImageProcessor']


if TYPE_CHECKING:
	from .configuration_layoutlmva import (
	    LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    LayoutLMvaConfig,
	    LayoutLMvaOnnxConfig,
	)
	from .processing_layoutlmva import LayoutLMvaProcessor
	from .tokenization_layoutlmva import LayoutLMvaTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_layoutlmva import (
		    LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
		    LayoutLMvaForQuestionAnswering,
		    LayoutLMvaForSequenceClassification,
		    LayoutLMvaForTokenClassification,
		    LayoutLMvaModel,
		    LayoutLMvaPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_layoutlmva import (
		    TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFLayoutLMvaForQuestionAnswering,
		    TFLayoutLMvaForSequenceClassification,
		    TFLayoutLMvaForTokenClassification,
		    TFLayoutLMvaModel,
		    TFLayoutLMvaPreTrainedModel,
		)

	try:
		if not is_vision_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor
		from .image_processing_layoutlmva import LayoutLMvaImageProcessor

else:
	import sys

	_lowerCamelCase   				=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)